diff --git a/.bazelignore b/.bazelignore index e0df23f9..1ef6fd4c 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,4 +1,3 @@ .* +third_party/modules .zig-cache -.devenv -venv/ diff --git a/.bazelrc b/.bazelrc index 09c50737..e386fc58 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,15 +1,21 @@ # Faster hash function startup --digest_function=blake3 +# Disable WORKSPACE +common --enable_workspace=false + +# Don't need for now +common --lockfile_mode=update + # Needed by LLVM and/or XLA common --experimental_repo_remote_exec +common --experimental_remote_cache_eviction_retries=5 # Self explanatory common --enable_platform_specific_config # Ensure host tools are built optimized common --host_compilation_mode=opt -common --@rules_zig//zig/settings:host_mode=release_safe # Suppress all warnings, LLVM and friends are too noisy common --copt='-w' @@ -20,33 +26,35 @@ common --cxxopt='-std=c++17' common --host_cxxopt='-std=c++17' common --enable_runfiles -# Ensure rules_python doesn't rely on the system's Python for stage-1 bootstrap -common --@rules_python//python/config_settings:bootstrap_impl=script +# Ensure the local env doesn't leak into the build +common --incompatible_strict_action_env # We don't to check the built files in the sandbox or repositories common --experimental_check_output_files=false -# Expose Zig's shared cache directories to sandboxed actions when the -# selected sandbox implementation supports host path mounts. -# -# These flags are consumed by linux-sandbox, and by Bazel's Docker sandbox. -# They are ignored by processwrapper-sandbox and macOS sandbox-exec, which use -# a symlinked execroot rather than bind mounts. +# Performance +common --legacy_external_runfiles=false + +common --experimental_output_directory_naming_scheme=diff_against_dynamic_baseline + +# Mount the zig cache into the sandboxes +build:macos --sandbox_add_mount_pair=/var/tmp build:linux --sandbox_add_mount_pair=/tmp -build:linux --sandbox_writable_path=/tmp -build:linux --sandbox_add_mount_pair=/var/tmp -build:linux --sandbox_writable_path=/var/tmp # Ensure workers are sandboxed build --worker_sandboxing +# Allows symlinks with spaces +build --experimental_inprocess_symlink_creation + # Build in debug for now, by default common --compilation_mode=dbg # Various cc options +common --experimental_cc_static_library common --experimental_cc_shared_library common --incompatible_use_cc_configure_from_rules_cc -common --incompatible_autoload_externally=+@rules_cc,+@rules_python,+@rules_shell,+@rules_java +common --experimental_starlark_cc_import common --experimental_platform_in_output_dir @@ -56,33 +64,28 @@ build --experimental_inmemory_sandbox_stashes # clang by default common --compiler=clang +# Enable resolving proto compiler from toolchains +common --incompatible_enable_proto_toolchain_resolution + # Allow empty globs since some legacy rules common --incompatible_disallow_empty_glob=false # Use the C linker to link Zig code -common --@rules_zig//zig/settings:use_cc_common_link=true +common --@rules_zig//zig/settings:use_cc_common_link=True +# Zig self hosted backend is too fragile for now +common --@rules_zig//zig/settings:zigopt=-fllvm -common --@rules_zig//zig/settings:use_standalone_translate_c=True - -common --@llvm//config:experimental_stub_libgcc_s=true +common --@toolchains_llvm_bootstrapped//config:experimental_stub_libgcc_s=True common --macos_minimum_os=13.0 -# Enable stamping -common --stamp=true -common --workspace_status_command=./tools/workspace_status.sh - common --@rules_cc//cc/toolchains/args/archiver_flags:use_libtool_on_macos=false common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1 -common --xcode_version_config=//:disable_xcode # Required for protobuf upb on Linux common:linux --copt=-fPIC -# Prefer prebuilt protoc (finally!) -common --@com_google_protobuf//bazel/toolchains:prefer_prebuilt_protoc=true - # Print test errors in the console test --test_output=errors @@ -105,43 +108,9 @@ common:alldebug --strategy=local common:release --compilation_mode=opt common:release --copt=-Ofast -common:release --strip=always common:release --@rules_zig//zig/settings:mode=release_safe -common:release --@rules_zig//zig/settings:zigopt=-fllvm # --config=native builds everything optimized for the local CPU common:native --copt=-march=native common:native --@rules_zig//zig/settings:zigopt=-mcpu=native - -# Remote execution and caching with BuildBuddy -common:remote --bes_results_url=https://app.buildbuddy.io/invocation/ -common:remote --bes_backend=grpcs://remote.buildbuddy.io -common:remote --remote_cache=grpcs://remote.buildbuddy.io -common:remote --experimental_remote_downloader=grpcs://remote.buildbuddy.io -common:remote --remote_executor=grpcs://remote.buildbuddy.io -common:remote --remote_download_toplevel -common:remote --remote_download_outputs=minimal -common:remote --nobuild_runfile_links -common:remote --remote_timeout=3600 -common:remote --experimental_remote_cache_chunking -common:remote --noexperimental_throttle_remote_action_building -common:remote --remote_cache_compression -# Re-runs actions instead of failing on transient lost remote/disk cache inputs. -common:remote --rewind_lost_inputs -# Falls back to local downloads if the remote asset downloader is unavailable. -common:remote --experimental_remote_downloader_local_fallback -common:remote --grpc_keepalive_time=30s -common:remote --jobs=2000 -common:remote --extra_execution_platforms=@zml//:rbe_platform -common:remote --experimental_repo_remote_exec=true - -# Runtime profiling helpers for `bazel run`. -run:nsys --@zml//platforms:cuda=true -run:nsys --run_under=//tools:nsys - -run:rocprofv3 --@zml//platforms:rocm=true -run:rocprofv3 --run_under=//tools:rocprofv3 - -run:neuron-profile --@zml//platforms:neuron=true -run:neuron-profile --@zml//platforms:cpu=false -run:neuron-profile --run_under='//tools/neuron:profile /tmp/zml-neuron-profile' +common:native --@rules_rust//rust/settings:extra_rustc_flag=-Ctarget-cpu=native diff --git a/.bazelversion b/.bazelversion index b13d146a..f9c71a52 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -9.3.0 +8.5.1 diff --git a/.github/ISSUE_TEMPLATE/llmd--bug-report.md b/.github/ISSUE_TEMPLATE/llmd--bug-report.md deleted file mode 100644 index b9629993..00000000 --- a/.github/ISSUE_TEMPLATE/llmd--bug-report.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: 'LLMD: Bug report' -about: If you hit a bug in LLMD, let us know -title: "[LLMD]" -labels: LLMD -assignees: '' -type: Bug - ---- - -**Describe the bug** -A clear and concise description of what the bug is. - -**To Reproduce** -Steps to reproduce the behavior: -1. Go to '...' -2. Click on '....' -3. Scroll down to '....' -4. See error - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Configuration:** - - Model: [e.g. `google/gemma-4-31B-it`] - - OS: [e.g. `Ubuntu 24.04`] - - Platform: [`cuda` / `rocm` / `oneapi` / `metal` / `tpu`] - - Accelerator: [e.g. `AMD MI300X`] - - Driver version: [e.g. `595.71.05`] - - Version: [e.g. 0.1.2 or docker image `sha256@`] - - Command line: [e.g. `docker run ...`] - -**Additional context** -Add any other context about the problem here. diff --git a/.github/runs-on.yml b/.github/runs-on.yml index aefeccc9..09d88bf4 100644 --- a/.github/runs-on.yml +++ b/.github/runs-on.yml @@ -3,30 +3,10 @@ images: platform: "linux" arch: "amd64" ami: "ami-04a92520784b93e73" - - ubuntu24-arm64: - platform: "linux" - arch: "arm64" - ami: "ami-035c36ba717c3d79f" - -runners: - large-runner-amd64: - cpu: [16] - family: ["c8", "c7"] - image: ubuntu24-amd64 - volume: 80gb - extras: ["s3-cache"] - - large-runner-arm64: - cpu: [16] - family: ["c8g", "c7g"] - image: ubuntu24-arm64 - volume: 80gb - extras: ["s3-cache"] - - small-runner-amd64: - cpu: [4] - family: ["c8", "c7"] - image: ubuntu24-amd64 - volume: 80gb - extras: ["s3-cache"] + disk: large + preinstall: | + #!/bin/bash + wget https://github.com/jbdalido/bazel-remote/releases/download/test/bazel-remote -O /usr/local/bin/bzlcache + chmod +x /usr/local/bin/bzlcache + echo "Run bazel cache" + systemd-run --property=Restart=always /usr/local/bin/bzlcache --max_size=1000 --grpc_address 127.0.0.1:15501 --dir=/tmp/cache --s3.region eu-west-3 --s3.bucket runs-on-s3bucketcache-jycgicewbssh --s3.auth_method iam_role --s3.prefix cache --s3.endpoint=s3.eu-west-3.amazonaws.com --storage_mode uncompressed diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 31bc46e8..48bc9320 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,174 +12,98 @@ concurrency: cancel-in-progress: ${{ !contains(github.ref, 'master')}} jobs: - ci: - name: ci-${{ matrix.target }}-${{ matrix.arch }} + platforms: strategy: - fail-fast: false matrix: - include: - - target: cpu - arch: amd64 - bazel_common_opts: "" - run_tests: true - runner_labels: - - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-cpu-amd64 - - runner=large-runner-amd64 - - - target: neuron - arch: amd64 - bazel_common_opts: "--@zml//platforms:neuron=true --@zml//platforms:cpu=false" - run_tests: false - runner_labels: - - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-neuron-amd64 - - runner=large-runner-amd64 - - - target: tpu - arch: amd64 - bazel_common_opts: "--@zml//platforms:tpu=true --@zml//platforms:cpu=false" - run_tests: false - runner_labels: - - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-tpu-amd64 - - runner=large-runner-amd64 - + runtimes: - target: cuda - arch: amd64 - bazel_common_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" - run_tests: true - runner_labels: ["self-hosted", "linux", "x64", "platform=cuda"] - - - target: cuda - arch: arm64 - bazel_common_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" - run_tests: false - runner_labels: - - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-cuda-arm64 - - runner=large-runner-arm64 - + bazel_opts: "--@zml//platforms:cuda=true --@zml//platforms:cpu=false" + enable_cache: false + user: actions + runs-on: ["self-hosted", "platform=cuda"] - target: rocm - arch: amd64 - bazel_common_opts: "--@zml//platforms:rocm=true --@zml//platforms:cpu=false" - run_tests: true - runner_labels: ["self-hosted", "linux", "x64", "platform=rocm"] - - - target: oneapi - arch: amd64 - bazel_common_opts: "--@zml//platforms:oneapi=true --@zml//platforms:cpu=false" - run_tests: false - runner_labels: - - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-oneapi-amd64 - - runner=large-runner-amd64 - - - target: metal - arch: arm64 - bazel_common_opts: "--@zml//platforms:metal=true --@zml//platforms:cpu=false" - bazel_build_opts: "--platforms=@zml//platforms:macos_arm64" - run_tests: false - runner_labels: - - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-metal-arm64 - - runner=large-runner-amd64 - - - target: furiosa - arch: amd64 - bazel_common_opts: "--@zml//platforms:furiosa=true --@zml//platforms:cpu=false" - run_tests: false - runner_labels: - - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-ci-furiosa-amd64 - - runner=large-runner-amd64 - - runs-on: ${{ matrix.runner_labels }} + bazel_opts: "--@zml//platforms:rocm=true --@zml//platforms:cpu=false" + enable_cache: false + user: actions + runs-on: ["self-hosted", "platform=rocm"] + - target: cpu + bazel_opts: "--@zml//platforms:cpu=true" + enable_cache: true + user: runner + runs-on: + [ + "runs-on", + "runner=4cpu-linux-x64", + "image=ubuntu24-amd64", + "run-id=${{ github.run_id }}", + "extras=s3-cache", + ] + runs-on: ${{ matrix.runtimes.runs-on }} steps: - - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 + - uses: runs-on/action@v1 - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@v4 + + - uses: runs-on/snapshot@v1 + if: ${{ matrix.runtimes.enable_cache }} + with: + path: /home/${{ matrix.runtimes.user }}/.cache/bazel-disk + volume_size: 350 + + - name: TMP fix for the mount user + if: ${{ matrix.runtimes.enable_cache }} + run: sudo chown -R ${{ matrix.runtimes.user }}:${{ matrix.runtimes.user }} /home/${{ matrix.runtimes.user }}/.cache/ - name: Ensure self hosted cache permissions - if: ${{ contains(matrix.runner_labels, 'self-hosted') }} + if: ${{ !matrix.runtimes.enable_cache }} run: mkdir -p /tmp/zig-cache && mkdir -p /tmp/zml && sudo chmod -R 777 /tmp/zig-cache && sudo chmod -R 777 /tmp/zml - - name: Setup Bazel - uses: bazel-contrib/setup-bazel@104e3eea9447cdaa4b00d0fb46191a3985d26ab3 # 0.19.1 + - uses: bazel-contrib/setup-bazel@0.15.0 with: - bazelisk-version: 1.29.0 - bazelisk-cache: true - cache-save: false + bazelisk-version: 1.26.0 + bazelisk-cache: false + # todo: use `--config=debug` instead of `-c opt` when static archive bug is fixed bazelrc: | - startup --experimental_remote_repo_contents_cache + common -c opt common --verbose_failures common --keep_going common --test_output=streamed common --color=yes common --show_timestamps - common --config=remote - common --strategy=TestRunner=local - ${{ format('common:remote --remote_header=x-buildbuddy-api-key={0}', secrets.BUILDBUDDY_CI_API_KEY) || '' }} + common --disk_cache=/home/${{ matrix.runtimes.user }}/.cache/bazel-disk + common --strip=always - - name: Capture self hosted storage usage before - id: storage_before - if: ${{ contains(matrix.runner_labels, 'self-hosted') }} - continue-on-error: true - shell: bash - run: &measure-self-hosted-storage | - output_base=$(bazel info output_base) || output_base="" - bazel_size="unavailable" - if [[ -n "$output_base" ]]; then - bazel_size=$(du -sh --si -- "$output_base" | cut -f1) || bazel_size="unavailable" - fi - disks=$(df -H --local --output=source,size,used,avail,pcent,target | - awk '$1 ~ /^\/dev\// && $1 !~ /^\/dev\/loop/ && !seen[$1]++ { - printf "%s%s: %s / %s used (%s), %s free", separator, $6, $3, $2, $5, $4 - separator = "; " - } END {if (separator == "") printf "none"}') || disks="unavailable" - { - printf 'output_base=%s\n' "$output_base" - printf 'bazel_size=%s\n' "$bazel_size" - printf 'disks=%s\n' "$disks" - } >> "$GITHUB_OUTPUT" + - name: ZML / bazel build //... + run: bazel build ${{ matrix.runtimes.bazel_opts }} //... - - run: bazel mod graph - - run: bazel query ${{ matrix.bazel_common_opts }} --output=graph //... - - run: bazel cquery ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... - - run: bazel build ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... - - if: ${{ matrix.run_tests }} - run: bazel test ${{ matrix.bazel_common_opts }} ${{ matrix.bazel_build_opts }} //... - - - name: Capture self hosted storage usage after - id: storage_after - if: ${{ always() && contains(matrix.runner_labels, 'self-hosted') }} - continue-on-error: true - shell: bash - run: *measure-self-hosted-storage - - - name: Report self hosted storage usage - if: ${{ always() && contains(matrix.runner_labels, 'self-hosted') }} - env: - STORAGE_JOB: ${{ github.job }}-${{ matrix.target }}-${{ matrix.arch }} - OUTPUT_BASE: ${{ steps.storage_after.outputs.output_base || steps.storage_before.outputs.output_base || 'unavailable' }} - SIZE_BEFORE: ${{ steps.storage_before.outputs.bazel_size || 'unavailable' }} - SIZE_AFTER: ${{ steps.storage_after.outputs.bazel_size || 'unavailable' }} - DISKS_BEFORE: ${{ steps.storage_before.outputs.disks || 'unavailable' }} - DISKS_AFTER: ${{ steps.storage_after.outputs.disks || 'unavailable' }} - run: | - message=$(printf 'Runner: %s\nBazel output base: %s\nBazel directory size: %s -> %s\nDisks before: %s\nDisks after: %s' \ - "$RUNNER_NAME" "$OUTPUT_BASE" "$SIZE_BEFORE" "$SIZE_AFTER" "$DISKS_BEFORE" "$DISKS_AFTER") - message=${message//'%'/'%25'} - message=${message//$'\r'/'%0D'} - message=${message//$'\n'/'%0A'} - printf '::notice title=Disk usage - %s::%s\n' "$STORAGE_JOB" "$message" - - lint: + - name: ZML / bazel test //... + run: bazel test ${{ matrix.runtimes.bazel_opts }} //... + ci: runs-on: - - runs-on=${{ github.run_id }}-${{ github.run_attempt }}-lint - - runner=small-runner-amd64 + [ + "runs-on", + "runner=4cpu-linux-x64", + "image=ubuntu24-amd64", + "run-id=${{ github.run_id }}", + "extras=s3-cache", + ] steps: - - uses: runs-on/action@17b59092eb37ebfdbb034460892887c09100eaa3 # v2.5.0 + - uses: runs-on/action@v1 - name: Checkout - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + uses: actions/checkout@v4 + + - uses: runs-on/snapshot@v1 + with: + path: /home/runner/.cache/bazel-disk + volume_size: 350 + + - name: TMP fix for the mount user + run: sudo chown -R runner:runner /home/runner/.cache/ - name: Setup Zig - uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 + uses: mlugg/setup-zig@v2 with: - version: 0.17.0 + version: 0.15.2 mirror: https://mirror.zml.ai/zig - name: Zig Format @@ -187,3 +111,27 @@ jobs: - name: No autofix in Zig code run: bash -c "! git grep -l '// autofix' -- '*.zig'" + + - uses: bazel-contrib/setup-bazel@0.15.0 + with: + bazelisk-version: 1.26.0 + bazelisk-cache: false + # todo: use `--config=debug` instead of `-c opt` when static archive bug is fixed + bazelrc: | + common -c opt + common --verbose_failures + common --keep_going + common --test_output=streamed + common --color=yes + common --show_timestamps + common --disk_cache=/home/runner/.cache/bazel-disk + common --strip=always + + - name: ZML / bazel mod explain + run: bazel mod explain + + - name: ZML / bazel mod graph + run: bazel mod graph + + - name: ZML / bazel query //... + run: bazel query //... diff --git a/.gitignore b/.gitignore index 0ec34365..a4b52b95 100644 --- a/.gitignore +++ b/.gitignore @@ -31,14 +31,3 @@ venv # Editor specific ## Neovim .nvim.local.lua - -# Devenv -.devenv* -devenv.local.nix -devenv.local.yaml - -# direnv -.direnv - -# pre-commit -.pre-commit-config.yaml diff --git a/.helix/languages.toml b/.helix/languages.toml deleted file mode 100644 index 529cdcf3..00000000 --- a/.helix/languages.toml +++ /dev/null @@ -1,19 +0,0 @@ -[[language]] -auto-format = true -file-types = [ "zig" ] -language-servers = [ "zls" ] -name = "zig" -roots = [ "build.zig", "build.zig.zon", "zls.build.json" ] -scope = "source.zig" - -[language.indent] -tab-width = 4 -unit = " " - -[language-server.zls] -command = "sh" -args = [ - "-c", - "exec \"$(git rev-parse --show-toplevel)/tools/zls.sh\" \"$@\"", - "zls" -] diff --git a/.nvim.lua b/.nvim.lua index 9982de80..a1f9ad95 100644 --- a/.nvim.lua +++ b/.nvim.lua @@ -4,63 +4,33 @@ local cwd = vim.fn.getcwd() -- Load the local config file if it exists local f, err = loadfile(".nvim.local.lua") if not err then - local ok, err_f = pcall(f) - if not ok then - vim.print(err_f) - end + f() else - vim.lsp.config("zls", { - cmd = { cwd .. "/tools/zls.sh" }, - }) - vim.lsp.enable("zls") + local capabilities = vim.lsp.protocol.make_client_capabilities() - function runFixAll() - local params = vim.lsp.util.make_range_params() - params.context = { - only = { "source.fixAll" }, - diagnostics = {}, - } + -- Prepend CWD to relative paths + local zls_cmd = cwd .. "/../zml/tools/zls.sh" - -- Send synchronous request (timeout 2 seconds) - local responses = vim.lsp.buf_request_sync(0, "textDocument/codeAction", params, 1000) - if not responses then - return - end + vim.lsp.config["zls"] = { + capabilities = capabilities, + cmd = { zls_cmd }, + root_marker = { "build.zig" }, + filetypes = {"zig"}, + } + vim.lsp.enable('zls') - for _, resp in pairs(responses) do - for _, action in ipairs(resp.result or {}) do - -- If action has edits, apply them - if action.edit then - vim.lsp.util.apply_workspace_edit(action.edit, "utf-16") - end - -- If the action has a command, execute it - if action.command then - vim.lsp.buf.execute_command(action.command) - end - end - end - end - - vim.api.nvim_create_autocmd("BufWritePre", { - pattern = { "*.zig", "*.zon" }, - callback = runFixAll, - }) - - local ok, conform = pcall(require, "conform") - - if ok then - conform.formatters_by_ft.bzl = { "buildifier" } - conform.formatters.buildifier = { - command = cwd .. "/tools/buildifier.sh", - } - - vim.api.nvim_create_autocmd("FileType", { - pattern = "bzl", - callback = function() - vim.keymap.set("n", "f", function() - conform.format() - end, { remap = false, silent = true }) - end, - }) - end + vim.api.nvim_create_autocmd('BufWritePre',{ + pattern = {"*.zig", "*.zon"}, + callback = function(ev) + vim.lsp.buf.code_action({ + context = { only = { "source.organizeImports" } }, + apply = true, + }) + vim.lsp.buf.code_action({ + context = { only = { "source.fixAll" } }, + apply = true, + }) + vim.loop.sleep(100) + end + }) end diff --git a/.vscode/settings.json b/.vscode/settings.json index 772705fb..82050af6 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,13 +9,15 @@ "**/bazel-*": true }, "files.watcherExclude": { - "**/bazel-*": true + "**/bazel-*": true, }, // "files.trimTrailingWhitespace": true, "files.insertFinalNewline": true, "bazel.buildifierFixOnFormat": true, "bazel.queriesShareServer": true, - "bazel.pathsToIgnore": ["."], + "bazel.pathsToIgnore": [ + "." + ], "bazel.buildifierExecutable": "@buildifier_prebuilt//:buildifier", "C_Cpp.formatting": "clangFormat", "C_Cpp.clang_format_style": "Microsoft", @@ -26,7 +28,10 @@ "editor.suggest.insertMode": "replace", "editor.stickyScroll.defaultModel": "foldingProviderModel", "editor.codeActionsOnSave": { - "source.fixAll": "explicit" + "source.fixAll": "explicit", } - } + }, + "rust-analyzer.linkedProjects": [ + "${workspaceFolder}/zml/tokenizer/hftokenizers/Cargo.toml" + ] } diff --git a/.zed/settings.json b/.zed/settings.json index 55eff7c8..141b19f4 100644 --- a/.zed/settings.json +++ b/.zed/settings.json @@ -8,12 +8,12 @@ "formatter": "language_server", "code_actions_on_format": { "source.fixAll": true, - "source.organizeImports": true - } + "source.organizeImports": true, + }, }, "YAML": { - "tab_size": 2 - } + "tab_size": 2, + }, }, "file_scan_exclusions": [ "**/.git", @@ -25,17 +25,17 @@ "**/Thumbs.db", "**/.classpath", "**/.settings", - "**/bazel-*" + "**/bazel-*", ], "tab_size": 4, "inlay_hints": { - "enabled": true + "enabled": true, }, "lsp": { "zls": { "binary": { - "path": "tools/zls.sh" - } - } - } + "path": "tools/zls.sh", + }, + }, + }, } diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index fe0f5d9c..00000000 --- a/AGENTS.md +++ /dev/null @@ -1,35 +0,0 @@ -# Repository Guidelines - -## Project Structure & Module Organization - -ZML is a Zig project built with Bazel. Core library code lives in `zml/`, with IO under `zml/io/`, tokenizers under `zml/tokenizer/`, attention implementations under `zml/attention/`, and MoE support under `zml/moe/`. Shared Zig utilities are in `stdx/`; FFI and C helpers are in `ffi/` and `upb/`. Bazel rules and workspace helpers live in `bazel/` and `third_party/`. Runnable examples are in `examples/`, while user-facing documentation is in `docs/`. Keep tests close to implementation files unless a package already has a separate test file. - -## Build, Test, and Development Commands - -- `bazel build //zml`: builds the main ZML library. -- `bazel test //zml:test`: runs the core Zig test target. -- `bazel test //stdx:test`: runs shared utility tests. -- `bazel test //zml/tokenizer:test`: runs tokenizer tests. -- `bazel run //examples/mnist`: runs the quick MNIST smoke test. -- `bazel run //examples/llm -- --model=hf://... --prompt="..."`: runs the LLM example. -- `bazel build //...` and `bazel test //...`: use before broad or shared changes. -- `./tools/buildifier.sh`: formats Bazel/Starlark files. -- `echo "$(bazel info output_base)/$(bazel cquery --output=files "$(bazel cquery "filter('zig_toolchain', deps(//zml))" 2>/dev/null | cut -d' ' -f1 | head -n1)" 2>/dev/null | rg "/lib\$")/std"`: find the current Zig standard library used by Bazel. - -Append platform flags when relevant, for example `--@zml//platforms:cuda=true`, `--@zml//platforms:rocm=true`, `--@zml//platforms:tpu=true`, or `--@zml//platforms:cpu=false`. - -## Coding Style & Naming Conventions - -Follow the Zig style guide and format Zig code with `zig fmt`; CI checks `zig fmt --check` outside `third_party/`. Use four-space indentation. Prefer `const x: Foo = .{ .bar = 1 };`, `pub fn method(self: Foo)`, and module imports such as `const foo = @import("foo.zig"); foo.bar()`. Use `PascalCase` for types and `lowerCamelCase` for functions, fields, and locals. Add comments only for non-obvious invariants, ownership, tensor shape semantics, or platform behavior. - -## Testing Guidelines - -Tests are generally inline Zig `test "..." {}` blocks near the code they cover. Use descriptive test names and narrow Bazel targets first, then broaden when touching shared APIs. For tensor or graph changes, cover shape/tag inference and runtime behavior when practical. CI builds all targets and runs tests on CPU, CUDA, and ROCm where available. - -## Commit & Pull Request Guidelines - -Recent history uses scoped, imperative commit subjects such as `zml/tensor: add Tensor.onMemory()` or `workspace: use latest version of upstreamable rules_zig`. Keep subjects concise and mention the affected area first. Pull requests should include a short description, relevant linked issue, platform impact if any, and the exact `bazel build` or `bazel test` commands run. Include screenshots only for docs or UI-visible changes. - -## Quirks - -If during build, you encounter an `AccessDenied` error, the solution is not to use the `--repo_env` bazel argument to override the zig cache. The solution is to run `sudo chmod -R 777`on `/tmp/zig-cache` and `/var/tmp/zig-cache`. This is due to the fact that all users share the same zig cache directory, and bazel runs as a different user than the one who created the cache. diff --git a/BUILD.bazel b/BUILD.bazel index 734a3792..5dbfe326 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,48 +1,13 @@ -load("@@apple_support+//xcode:xcode_config.bzl", "xcode_config") -load("@rules_zig//zig/translate-c:toolchain.bzl", "translate_c_toolchain") -load("@rules_zig//zig/zls:zls_completion.bzl", "zls_completion") - -# Dummy xcode_config so we don't need to build xcode_locator in the repository rule. -xcode_config( - name = "disable_xcode", - visibility = ["//visibility:public"], -) - -alias( - name = "translate-c", - actual = "@translate-c//:translate-c", - visibility = ["//visibility:public"], -) - -translate_c_toolchain( - name = "translate_c_toolchain_impl", - runtime_modules = [ - "@translate-c//:c_builtins", - "@translate-c//:helpers", - ], - translate_c = "@translate-c//:translate-c", -) - -toolchain( - name = "translate_c_toolchain", - target_settings = ["@rules_zig//zig/config/use_standalone_translate_c:enabled"], - toolchain = ":translate_c_toolchain_impl", - toolchain_type = "@rules_zig//zig/translate-c:toolchain_type", - visibility = ["//visibility:public"], -) +load("//third_party/zls:zls_completion.bzl", "zls_completion") zls_completion( name = "completion", visibility = ["//visibility:public"], deps = [ - "//bin/zml-smi", - "//examples/llm", + "//examples/llama", "//examples/mnist", - "//examples/sharding", - "//examples/io:playground", "//stdx", "//zml", - "//zml/tokenizer/iree", ], ) @@ -53,11 +18,11 @@ platform( "@platforms//os:linux", # ubuntu:22.04 has GNU C Library (Ubuntu GLIBC 2.35-0ubuntu3.10) # But apparently not in this docker://ubuntu:22.04 image? - "@llvm//constraints/libc:gnu.2.28", + "@toolchains_llvm_bootstrapped//constraints/libc:gnu.2.31", "@bazel_tools//tools/cpp:clang", ], exec_properties = { - "container-image": "docker://mirror.gcr.io/library/ubuntu:22.04", + "container-image": "docker://ubuntu:22.04", "OSFamily": "Linux", }, visibility = ["//visibility:public"], diff --git a/MODULE.bazel b/MODULE.bazel index e9c4c9d3..b0d54b40 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -2,107 +2,33 @@ module( name = "zml", ) -bazel_dep(name = "abseil-cpp", version = "20260526.0", repo_name = "com_google_absl") - -# Work around abseil-cpp#2071 until the BCR release includes the Clang BMI2 fix. -single_version_override( - module_name = "abseil-cpp", - patch_strip = 1, - patches = [ - "//:third_party/abseil-cpp/bmi2intrin.patch", - ], - version = "20260526.0", -) - -bazel_dep(name = "bazel_lib", version = "3.7.2") +bazel_dep(name = "abseil-cpp", version = "20260107.1") +bazel_dep(name = "aspect_bazel_lib", version = "2.22.5") bazel_dep(name = "aspect_rules_py", version = "1.8.4") bazel_dep(name = "bazel_skylib", version = "1.9.0") - -## Needed by iree -bazel_dep(name = "libbacktrace", version = "1.0.0-20250926-7939218") bazel_dep(name = "patchelf", version = "0.18.0.bcr.1") bazel_dep(name = "pcre2", version = "10.45") -bazel_dep(name = "platforms", version = "1.1.0") -bazel_dep(name = "protobuf", version = "35.0", repo_name = "com_google_protobuf") -bazel_dep(name = "pybind11_bazel", version = "3.0.1") -bazel_dep(name = "re2", version = "2025-11-05", repo_name = "com_googlesource_code_re2") -bazel_dep(name = "rules_cc", version = "0.2.19") -bazel_dep(name = "rules_distroless", version = "0.8.0") - -# Once the rules_distroless can parse unspaced apt version constraints -# Update the rules_distroless version and remove the patch -# https://github.com/bazel-contrib/rules_distroless/pull/229 -single_version_override( - module_name = "rules_distroless", - patch_strip = 1, - patches = [ - "//:third_party/rules_distroless/accept-nospace-version-constraints.patch", - "//:third_party/rules_distroless/flat-repositories.patch", - ], - version = "0.8.0", -) - +bazel_dep(name = "platforms", version = "1.0.0") +bazel_dep(name = "protobuf", version = "33.1", repo_name = "com_google_protobuf") +bazel_dep(name = "rules_cc", version = "0.2.16") +bazel_dep(name = "rules_distroless", version = "0.6.2") bazel_dep(name = "rules_license", version = "1.0.0") bazel_dep(name = "rules_oci", version = "2.2.7") bazel_dep(name = "rules_proto", version = "7.1.0") -bazel_dep(name = "rules_python", version = "1.9.0") +bazel_dep(name = "rules_python", version = "1.8.4") +bazel_dep(name = "rules_rust", version = "0.68.1") +bazel_dep(name = "rules_rs", version = "0.0.28") bazel_dep(name = "rules_shell", version = "0.6.1") +bazel_dep(name = "rules_zig", version = "0.12.2") +bazel_dep(name = "toolchains_llvm_bootstrapped", version = "0.5.5") +bazel_dep(name = "with_cfg.bzl", version = "0.14.1") -# repo_name = None prevents rules_swift from eagerly auto-registering toolchains. -bazel_dep(name = "rules_swift", version = "3.6.1", repo_name = None) -bazel_dep(name = "rules_zig", version = "0.17.0") - -bazel_dep(name = "tar.bzl", version = "0.10.4") -bazel_dep(name = "llvm", version = "0.8.24") -bazel_dep(name = "libxml2", version = "2.15.3") -bazel_dep(name = "xz", version = "5.4.5.bcr.9", repo_name = "llvm_xz") -bazel_dep(name = "zlib-ng", version = "2.3.3", repo_name = "llvm_zlib") -bazel_dep(name = "zstd", version = "1.5.7.bcr.1", repo_name = "llvm_zstd") -bazel_dep(name = "with_cfg.bzl", version = "0.14.4") -bazel_dep(name = "highway", version = "1.4.0") -bazel_dep(name = "grpc", version = "1.80.0") -single_version_override( - module_name = "grpc", - patch_strip = 1, - patches = [ - "//:third_party/grpc/protobuf-35-upb-load.patch", - "//:third_party/grpc/grpc++_unsecure-visibility.patch", - "//:third_party/grpc/plugin-flags.patch", - ], - version = "1.80.0", -) - -single_version_override( - module_name = "envoy_api", - version = "0.0.0-20250128-4de3c74", -) - -# Added to support bazel 9+ since an old version is pulled by GRPC. -bazel_dep(name = "rules_foreign_cc", version = "0.15.1") - -bazel_dep(name = "buildifier_prebuilt", version = "8.5.1", dev_dependency = True) - -# BSDTar toolchains -tar_toolchains = use_extension("@tar.bzl//tar:extensions.bzl", "toolchains") -use_repo( - tar_toolchains, - "bsd_tar_toolchains_darwin_amd64", - "bsd_tar_toolchains_darwin_arm64", - "bsd_tar_toolchains_linux_amd64", - "bsd_tar_toolchains_linux_arm64", - "bsd_tar_toolchains_windows_amd64", - "bsd_tar_toolchains_windows_arm64", -) -### +bazel_dep(name = "buildifier_prebuilt", version = "8.2.0.2", dev_dependency = True) ### Zig toolchain zig = use_extension("@rules_zig//zig:extensions.bzl", "zig") zig.index(file = "//bazel:zig_index.json") -zig.toolchain( - # Ensure Zig actions are run locally only - extra_exec_compatible_with = ["//platforms:local_execution"], - zig_version = "0.17.0", -) +zig.toolchain(zig_version = "0.16.0-dev.2611+f996d2866") zig.mirrors(urls = [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -124,37 +50,23 @@ use_repo(zig, "zig_toolchains") register_toolchains("@rules_zig//zig/target:all") register_toolchains("@zig_toolchains//:all") - -register_toolchains("//:translate_c_toolchain") - -# Add the local execution platform to the set of eligible execution platforms -# This is used so that Zig toolchains that are only declared as compatible with -# the constraints described by this platform be selected and matched with only -# this platform only. -# This ensures Zig actions only run locally because they're slower remotely -# due to the Zig compilation model which concentrates one action for the whole -# build graph. -register_execution_platforms("//platforms:local_execution_platform") - -register_toolchains("//third_party/zls:toolchain") - ### ### LLVM Bootstrapped toolchains -register_toolchains("@llvm//toolchain:all") +register_toolchains( + "@toolchains_llvm_bootstrapped//toolchain:all", +) ### uv = use_extension("@rules_python//python/uv:uv.bzl", "uv", dev_dependency = True) -uv.configure(version = "0.9.30") +uv.configure(version = "0.6.5") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") -use_repo(python, "python_3_12", "python_versions") +use_repo(python, "python_versions") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") - -pip_neuron = use_extension("@rules_python//python/extensions:pip.bzl", "pip") -pip_neuron.parse( +pip.parse( download_only = True, extra_pip_args = [ "--abi=cp312", @@ -169,9 +81,8 @@ pip_neuron.parse( hub_name = "neuron_py_deps", python_version = "3.12", requirements_lock = "//platforms/neuron:requirements.lock.txt", - target_platforms = ["linux_x86_64"], ) -use_repo(pip_neuron, "neuron_py_deps") +use_repo(pip, "neuron_py_deps") # For downloading models pip.parse( @@ -182,170 +93,41 @@ pip.parse( ) use_repo(pip, "huggingface_hub") -pip.parse( - download_only = True, - # fsspec[gcs] -> gcsfs -> fsspec is a circular dependency, install them together. - experimental_requirement_cycles = { - "fsspec": [ - "fsspec", - "gcsfs", - ], - }, - hub_name = "xprof", - python_version = "3.12", - requirements_lock = "//tools/xprof:requirements.lock.txt", -) -use_repo(pip, "xprof") - -cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_plugin") -use_repo(cpu, "libzml_cpu_darwin_amd64", "libzml_cpu_darwin_arm64", "libzml_cpu_linux_amd64") +cpu = use_extension("//platforms/cpu:cpu.bzl", "cpu_pjrt_plugin") +use_repo(cpu, "libpjrt_cpu_darwin_amd64", "libpjrt_cpu_darwin_arm64", "libpjrt_cpu_linux_amd64") cuda = use_extension("//platforms/cuda:cuda.bzl", "cuda_packages") -use_repo(cuda, "cuda_nvml_dev_linux_sbsa", "cuda_nvml_dev_linux_x86_64", "cuda_nvtx_linux_sbsa", "cuda_nvtx_linux_x86_64", "libzml_cuda_linux_amd64", "libzml_cuda_linux_arm64", "zlib1g_linux_amd64", "zlib1g_linux_arm64") +use_repo(cuda, "libpjrt_cuda") rocm = use_extension("//platforms/rocm:rocm.bzl", "rocm_packages") -use_repo( - rocm, - "amdrocm-amdsmi", - "amdrocm-base", - "amdrocm-blas", - "amdrocm-dnn", - "amdrocm-fft", - "amdrocm-llvm", - "amdrocm-math-common", - "amdrocm-profiler-base", - "amdrocm-rccl", - "amdrocm-runtime", - "amdrocm-solver", - "amdrocm-sparse", - "amdrocm-sysdeps", - "libatomic1", - "libdrm-common", - "libzml_rocm", -) - -oneapi = use_extension("//platforms/oneapi:oneapi.bzl", "oneapi_packages") -use_repo(oneapi, "libzml_oneapi") - -furiosa = use_extension("//platforms/furiosa:furiosa.bzl", "furiosa_packages") -use_repo(furiosa, "libzml_furiosa") +use_repo(rocm, "hipblaslt", "libpjrt_rocm", "rocblas") tpu = use_extension("//platforms/tpu:tpu.bzl", "tpu_packages") -use_repo(tpu, "libzml_tpu") +use_repo(tpu, "libpjrt_tpu") neuron = use_extension("//platforms/neuron:neuron.bzl", "neuron_packages") -use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "aws-neuronx-tools", "libgcc_s1", "libgomp1", "libstdcpp6", "libzml_neuron") +use_repo(neuron, "aws-neuronx-collectives", "aws-neuronx-runtime-lib", "libgomp1", "libpjrt_neuron", "zlib1g") -http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +zls = use_extension("//third_party/zls:zls_toolchain.bzl", "zls_toolchains") +use_repo(zls, "zls_toolchains") -http_archive( - name = "sevenzip_darwin_amd64", - build_file_content = """exports_files(["7zz"])""", - sha256 = "26aa75bc262bb10bf0805617b95569c3035c2c590a99f7db55c7e9607b2685e0", - urls = ["https://www.7-zip.org/a/7z2501-mac.tar.xz"], -) - -# Host tools for the portable AEA extraction path. -http_archive( - name = "sevenzip_darwin_arm64", - build_file_content = """exports_files(["7zz"])""", - sha256 = "26aa75bc262bb10bf0805617b95569c3035c2c590a99f7db55c7e9607b2685e0", - urls = ["https://www.7-zip.org/a/7z2501-mac.tar.xz"], -) - -http_archive( - name = "sevenzip_linux_amd64", - build_file_content = """exports_files(["7zz"])""", - sha256 = "4ca3b7c6f2f67866b92622818b58233dc70367be2f36b498eb0bdeaaa44b53f4", - urls = ["https://www.7-zip.org/a/7z2501-linux-x64.tar.xz"], -) - -http_archive( - name = "ipsw_linux_amd64", - build_file_content = """exports_files(["ipsw"])""", - sha256 = "923c1caa44d3adaf2b792725c6209e879a13ab68af462ae5adfe1d611e2f3653", - urls = ["https://github.com/blacktop/ipsw/releases/download/v3.1.729/ipsw_3.1.729_linux_x86_64.tar.gz"], -) - -http_archive( - name = "sevenzip_linux_arm64", - build_file_content = """exports_files(["7zz"])""", - sha256 = "39c5140f02ce4436599303c59a149f654cb1bbc47cdc105a942120d747ae040d", - urls = ["https://www.7-zip.org/a/7z2501-linux-arm64.tar.xz"], -) - -http_archive( - name = "ipsw_linux_arm64", - build_file_content = """exports_files(["ipsw"])""", - sha256 = "fdef33729a0cb1d6be0cbffe677a97fb167e6af07fda3eafd03137ce607e92b4", - urls = ["https://github.com/blacktop/ipsw/releases/download/v3.1.729/ipsw_3.1.729_linux_arm64.tar.gz"], -) - -http_aea_archive = use_repo_rule("//bazel:http_aea_archive.bzl", "http_aea_archive") - -http_aea_archive( - name = "metal_toolchain", - archive_decryption_key = "v7VbaT39JmQPp5p4N3HoVPAKcZ5vyhY+nnjMRD79z5E=", - build_file = "//platforms/metal:metal_toolchain.BUILD.bazel", - canonical_id = "com.apple.MobileAsset.MetalToolchain-17B54", - includes = ["Metal.xctoolchain/*"], - sha256 = "84f1cec3984878294777349e794f7bff0407e501a728bd600dc8605fa36abf6f", - urls = [ - "https://updates.cdn-apple.com/2025/mobileassets/022-20354/192ADD28-31EE-43C8-972D-E5BED5FB1845/com_apple_MobileAsset_MetalToolchain/637B0E26-E00A-488B-BA3B-CB50A8ECFD74.aar", - ], -) - -metal = use_extension("//platforms/metal:metal.bzl", "metal_packages") -use_repo(metal, "libzml_metal") +register_toolchains("@zls_toolchains//:all") non_module_deps = use_extension("//:third_party/non_module_deps.bzl", "non_module_deps") -use_repo( - non_module_deps, - "arocc", - "cloud_accelerator_diagnostics", - "com_google_sentencepiece", - "cuda_tile", - "cute_ir", - "flashinfer_cutlass_moe_linux_amd64", - "flashinfer_cutlass_moe_linux_arm64", - "flydsl", - "iree", - "libvaxis", - "linenoise", - "mnist", - "org_swig_swig", - "stb", - "translate-c", - "uucode", - "xla", - "zigimg", - "zio", - "zls", - "zls_diffz", - "zls_known_folders", - "zls_lsp", -) +use_repo(non_module_deps, "arocc", "com_google_sentencepiece", "flashattn", "linenoise", "mnist", "org_swig_swig", "translate-c", "xla") xla = use_extension("//third_party/xla:xla.bzl", "xla") use_repo( xla, "com_github_grpc_grpc", - "eigen_archive", - "farmhash_archive", - "farmhash_gpu_archive", - "highwayhash", - "hwloc", "llvm-raw", "local_config_cuda", "local_config_remote_execution", "local_config_rocm", "local_config_sycl", "local_config_tensorrt", - "ml_dtypes_py", "python_version_repo", "rules_ml_toolchain", - "shardy", - "snappy", "stablehlo", "sycl_configure", "sycl_configure_ext", @@ -354,22 +136,39 @@ use_repo( ) llvm = use_extension("//third_party/xla:llvm.bzl", "llvm") -llvm.configure(targets = [ - "X86", - "NVPTX", - "AMDGPU", - "AArch64", -]) +llvm.configure() use_repo(llvm, "llvm-project") -apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") -apt.install( - name = "apt_cpu", - lock = "//platforms/cpu:packages.lock.json", - manifest = "//platforms/cpu:packages.yaml", +rust = use_extension("@rules_rust//rust:extensions.bzl", "rust") +rust.toolchain( + edition = "2024", + extra_target_triples = [ + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + ], + versions = ["1.92.0"], ) -use_repo(apt, "apt_cpu") +use_repo(rust, "rust_toolchains") +register_toolchains("@rust_toolchains//:all") + +crate = use_extension("@rules_rs//rs:extensions.bzl", "crate") +crate.from_cargo( + name = "crates", + cargo_lock = "//zml/tokenizer/hftokenizers:Cargo.lock", + cargo_toml = "//zml/tokenizer/hftokenizers:Cargo.toml", + platform_triples = [ + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-unknown-linux-gnu", + "x86_64-unknown-linux-gnu", + ], +) +use_repo(crate, "crates") + +apt = use_extension("@rules_distroless//apt:extensions.bzl", "apt") apt.install( name = "apt_cuda", lock = "//platforms/cuda:packages.lock.json", @@ -384,14 +183,6 @@ apt.install( ) use_repo(apt, "apt_rocm") -apt.install( - name = "apt_oneapi", - lock = "//platforms/oneapi:packages.lock.json", - manifest = "//platforms/oneapi:packages.yaml", - resolve_transitive = False, -) -use_repo(apt, "apt_oneapi") - apt.install( name = "apt_neuron", lock = "//platforms/neuron:packages.lock.json", @@ -406,16 +197,10 @@ oci.pull( image = "gcr.io/distroless/cc-debian12", platforms = ["linux/amd64"], ) -oci.pull( - name = "debian_12", - digest = "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", - image = "mirror.gcr.io/library/debian:12", - platforms = ["linux/amd64"], -) -use_repo( - oci, - "debian_12", - "debian_12_linux_amd64", - "distroless_cc_debian12", - "distroless_cc_debian12_linux_amd64", +use_repo(oci, "distroless_cc_debian12", "distroless_cc_debian12_linux_amd64") + +git_override( + module_name = "rules_zig", + commit = "6c5d6a07b4ef48be894e7418bcae37bac54e1c2e", + remote = "https://github.com/zml/rules_zig.git", ) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 1e746605..cc67df7a 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -1,55 +1,47 @@ { - "lockFileVersion": 28, + "lockFileVersion": 24, "registryFileHashes": { "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", - "https://bcr.bazel.build/modules/abseil-cpp/20260526.0/MODULE.bazel": "b4a76a8d70ba08aa0253fd5e374ca5a5fa6b15aba985a7bbe4467c138954830f", - "https://bcr.bazel.build/modules/abseil-cpp/20260526.0/source.json": "a39a26fec1b7037bdd7ea6c25cc098b99ae80af10ff0b6ec8c4bfe0df694f451", - "https://bcr.bazel.build/modules/abseil-py/2.1.0/MODULE.bazel": "5ebe5bf853769c65707e5c28f216798f7a4b1042015e6a36e6d03094d94bec8a", - "https://bcr.bazel.build/modules/abseil-py/2.1.0/source.json": "0e8fc4f088ce07099c1cd6594c20c7ddbb48b4b3c0849b7d94ba94be88ff042b", + "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", + "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", + "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", + "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", + "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", + "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", + "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", + "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/MODULE.bazel": "e33b3801443f5fd64465262084534115db76363df13d2168a42bbfacc747be81", + "https://bcr.bazel.build/modules/abseil-cpp/20260107.1/source.json": "7a9a88969b1e79268cf613728ca8ff8fa4bc4b1a9abee9ec1fb5f113ca751971", "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", - "https://bcr.bazel.build/modules/apple_support/1.13.0/MODULE.bazel": "7c8cdea7e031b7f9f67f0b497adf6d2c6a2675e9304ca93a9af6ed84eef5a524", "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", - "https://bcr.bazel.build/modules/apple_support/1.17.1/MODULE.bazel": "655c922ab1209978a94ef6ca7d9d43e940cd97d9c172fb55f94d91ac53f8610b", - "https://bcr.bazel.build/modules/apple_support/1.21.0/MODULE.bazel": "ac1824ed5edf17dee2fdd4927ada30c9f8c3b520be1b5fd02a5da15bc10bff3e", - "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", - "https://bcr.bazel.build/modules/apple_support/1.22.1/MODULE.bazel": "90bd1a660590f3ceffbdf524e37483094b29352d85317060b2327fff8f3f4458", + "https://bcr.bazel.build/modules/apple_support/1.23.0/MODULE.bazel": "317d47e3f65b580e7fb4221c160797fda48e32f07d2dfff63d754ef2316dcd25", "https://bcr.bazel.build/modules/apple_support/1.23.1/MODULE.bazel": "53763fed456a968cf919b3240427cf3a9d5481ec5466abc9d5dc51bc70087442", "https://bcr.bazel.build/modules/apple_support/1.24.1/MODULE.bazel": "f46e8ddad60aef170ee92b2f3d00ef66c147ceafea68b6877cb45bd91737f5f8", - "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", - "https://bcr.bazel.build/modules/apple_support/2.3.0/MODULE.bazel": "d48f824ae8eeea5f837eb3038cef3615075d996d3e82eb9187192c2820605a81", - "https://bcr.bazel.build/modules/apple_support/2.5.2/MODULE.bazel": "bb1b7081e390a8d3885f642cb0073dfcf8e7a36525e079fa8f5835af7079c0fe", - "https://bcr.bazel.build/modules/apple_support/2.5.4/MODULE.bazel": "964e9268b37edb64e9a0fa525e5b5e535ba56bf63cadc68375f92e69fcb734f6", - "https://bcr.bazel.build/modules/apple_support/2.8.0/MODULE.bazel": "c45f5176057092afba0be6c48a9ea02101666802f8be536947e099e94757f8ed", - "https://bcr.bazel.build/modules/apple_support/2.8.0/source.json": "b7f6d87669c206adaa7d82cc44733349c63e341b80a0e049496ebd91d78c3472", - "https://bcr.bazel.build/modules/aspect_bazel_lib/1.31.2/MODULE.bazel": "7bee702b4862612f29333590f4b658a5832d433d6f8e4395f090e8f4e85d442f", - "https://bcr.bazel.build/modules/aspect_bazel_lib/1.38.0/MODULE.bazel": "6307fec451ba9962c1c969eb516ebfe1e46528f7fa92e1c9ac8646bef4cdaa3f", - "https://bcr.bazel.build/modules/aspect_bazel_lib/1.40.3/MODULE.bazel": "668e6bcb4d957fc0e284316dba546b705c8d43c857f87119619ee83c4555b859", + "https://bcr.bazel.build/modules/apple_support/2.2.0/MODULE.bazel": "5e32bc42e413a4544df7afee7fa4c7aff73e670a44e56e8e45ce1954332034ad", + "https://bcr.bazel.build/modules/apple_support/2.2.0/source.json": "368bf1de5dace2a411e792e6df2ea5384eb8db3899680ed51b37eb48b88fd8ac", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.21.2/MODULE.bazel": "276347663a25b0d5bd6cad869252bea3e160c4d980e764b15f3bae7f80b30624", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/MODULE.bazel": "004ba890363d05372a97248c37205ae64b6fa31047629cd2c0895a9d0c7779e8", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.22.5/source.json": "ac2c3213df8f985785f1d0aeb7f0f73d5324e6e67d593d9b9470fb74a25d4a9b", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.7.2/MODULE.bazel": "780d1a6522b28f5edb7ea09630748720721dfe27690d65a2d33aa7509de77e07", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", - "https://bcr.bazel.build/modules/aspect_rules_js/1.33.1/MODULE.bazel": "db3e7f16e471cf6827059d03af7c21859e7a0d2bc65429a3a11f005d46fc501b", - "https://bcr.bazel.build/modules/aspect_rules_js/1.39.0/MODULE.bazel": "aece421d479e3c31dc3e5f6d49a12acc2700457c03c556650ec7a0ff23fc0d95", - "https://bcr.bazel.build/modules/aspect_rules_lint/0.12.0/MODULE.bazel": "e767c5dbfeb254ec03275a7701b5cfde2c4d2873676804bc7cb27ddff3728fed", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/MODULE.bazel": "4ab4a78e9d65cf15e755807f7ae504655e1771cefd7c3343ad34dbbb625b9564", "https://bcr.bazel.build/modules/aspect_rules_py/1.8.4/source.json": "7cd80ad811f0c2b3aad3c878506ae602fce2fe53b78487ecfbcc8bcde07d220e", + "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.2/MODULE.bazel": "598e7fe3b54f5fa64fdbeead1027653963a359cc23561d43680006f3b463d5a4", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/MODULE.bazel": "37c764292861c2f70314efa9846bb6dbb44fc0308903b3285da6528305450183", "https://bcr.bazel.build/modules/aspect_tools_telemetry/0.3.3/source.json": "605086bbc197743a0d360f7ddc550a1d4dfa0441bc807236e17170f636153348", - "https://bcr.bazel.build/modules/bazel_features/0.1.0/MODULE.bazel": "47011d645b0f949f42ee67f2e8775188a9cf4a0a1528aa2fa4952f2fd00906fd", - "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", - "https://bcr.bazel.build/modules/bazel_features/1.13.0/MODULE.bazel": "c14c33c7c3c730612bdbe14ebbb5e61936b6f11322ea95a6e91cd1ba962f94df", "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d", "https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a", "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", "https://bcr.bazel.build/modules/bazel_features/1.20.0/MODULE.bazel": "8b85300b9c8594752e0721a37210e34879d23adc219ed9dc8f4104a4a1750920", "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", - "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", "https://bcr.bazel.build/modules/bazel_features/1.24.0/MODULE.bazel": "4796b4c25b47053e9bbffa792b3792d07e228ff66cd0405faef56a978708acd4", "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", @@ -58,155 +50,56 @@ "https://bcr.bazel.build/modules/bazel_features/1.32.0/MODULE.bazel": "095d67022a58cb20f7e20e1aefecfa65257a222c18a938e2914fd257b5f1ccdc", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", - "https://bcr.bazel.build/modules/bazel_features/1.36.0/MODULE.bazel": "596cb62090b039caf1cad1d52a8bc35cf188ca9a4e279a828005e7ee49a1bec3", "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", - "https://bcr.bazel.build/modules/bazel_features/1.39.0/MODULE.bazel": "28739425c1fc283c91931619749c832b555e60bcd1010b40d8441ce0a5cf726d", + "https://bcr.bazel.build/modules/bazel_features/1.38.0/source.json": "31ba776c122b54a2885e23651642e32f087a87bf025465f8040751894b571277", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", - "https://bcr.bazel.build/modules/bazel_features/1.42.0/MODULE.bazel": "e8ca15cb2639c5f12183db6dcb678735555d0cdd739b32a0418b6532b5e565f8", - "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", - "https://bcr.bazel.build/modules/bazel_features/1.43.0/MODULE.bazel": "defa2226f06ba20550d6548c3a2ea2a7929634437a52973869c20c225450eb91", - "https://bcr.bazel.build/modules/bazel_features/1.47.0/MODULE.bazel": "e34df3cb35b1684cfa69923a61ae3803595babd3942cd306a488d51400886b30", - "https://bcr.bazel.build/modules/bazel_features/1.50.0/MODULE.bazel": "2083ef9c7a469f520890483ccf8e0189d6e71e2117e7752e15e6554433d5ae3e", - "https://bcr.bazel.build/modules/bazel_features/1.50.0/source.json": "e0ee3debde2789ff56e4452e612d126925ba9ab64d4bde79c67f099d2902df9b", "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-beta.1/MODULE.bazel": "407729e232f611c3270005b016b437005daa7b1505826798ea584169a476e878", "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", - "https://bcr.bazel.build/modules/bazel_lib/3.1.0/MODULE.bazel": "6809765c14e3c766a9b9286c7b0ec56ed87a73326e48fe01749f0c0fdcfe3287", - "https://bcr.bazel.build/modules/bazel_lib/3.2.2/MODULE.bazel": "e2c890c8a515d6bca9c66d47718aa9e44b458fde64ec7204b8030bf2d349058c", - "https://bcr.bazel.build/modules/bazel_lib/3.7.2/MODULE.bazel": "cbc920fff00a55b2fd663d40850560a6fb17fd5ef3a9f3d6cbf0d32908dd8a40", - "https://bcr.bazel.build/modules/bazel_lib/3.7.2/source.json": "7a9c2b06a113c4962d329bb68befdb90d0447788a19aabaa32b476a0458e7d28", + "https://bcr.bazel.build/modules/bazel_lib/3.2.0/MODULE.bazel": "39b50d94b9be6bda507862254e20c263f9b950e3160112348d10a938be9ce2c2", + "https://bcr.bazel.build/modules/bazel_lib/3.2.0/source.json": "a6f45a903134bebbf33a6166dd42b4c7ab45169de094b37a85f348ca41170a84", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", - "https://bcr.bazel.build/modules/bazel_skylib/1.4.0/MODULE.bazel": "2ab127ef8d56a739a99bb2ce00ec4c7d1ecc7977d4370c0ca6efd0d8f03d6d99", "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d", "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", - "https://bcr.bazel.build/modules/bazel_skylib/1.8.0/MODULE.bazel": "2fb3fb53675f6adfc1ca5bfbd5cfb655ae350fba4706d924a8ec7e3ba945671c", "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67", "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/MODULE.bazel": "72997b29dfd95c3fa0d0c48322d05590418edef451f8db8db5509c57875fb4b7", "https://bcr.bazel.build/modules/bazel_skylib/1.9.0/source.json": "7ad77c1e8c1b84222d9b3f3cae016a76639435744c19330b0b37c0a3c9da7dc0", - "https://bcr.bazel.build/modules/bazel_worker_api/0.0.8/MODULE.bazel": "396c1ef53835aafe3d42ce6619080531ee770648303731f16cfaa33fa056bf0c", - "https://bcr.bazel.build/modules/bazel_worker_api/0.0.8/source.json": "abaf8ac9d2ab2f47bda9af4c0c080ff7907378888e1f4bc62a0539dd13ba61e8", - "https://bcr.bazel.build/modules/bazel_worker_java/0.0.8/MODULE.bazel": "e76479eae70bd4e8f5f4c2dfc5d03ab971cfb18750246c7b3f3454c5c2ee6629", - "https://bcr.bazel.build/modules/bazel_worker_java/0.0.8/source.json": "9395c4679444bc47bf7e51a710366a4480aa371c6f6bed01868e2fabcf11acec", - "https://bcr.bazel.build/modules/boringssl/0.0.0-20230215-5c22014/MODULE.bazel": "4b03dc0d04375fa0271174badcd202ed249870c8e895b26664fd7298abea7282", - "https://bcr.bazel.build/modules/boringssl/0.0.0-20240530-2db0eb3/MODULE.bazel": "d0405b762c5e87cd445b7015f2b8da5400ef9a8dbca0bfefa6c1cea79d528a97", - "https://bcr.bazel.build/modules/boringssl/0.20240913.0/MODULE.bazel": "fcaa7503a5213290831a91ed1eb538551cf11ac0bc3a6ad92d0fef92c5bd25fb", - "https://bcr.bazel.build/modules/boringssl/0.20241024.0/MODULE.bazel": "b540cff73d948cb79cb0bc108d7cef391d2098a25adabfda5043e4ef548dbc87", - "https://bcr.bazel.build/modules/boringssl/0.20241024.0/source.json": "d843092e682b84188c043ac742965d7f96e04c846c7e338187e03238674909a9", - "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/MODULE.bazel": "77f2a1958d1d07376dd3ce3ae16540f2c1b01921c1fd21930827271260e75a66", - "https://bcr.bazel.build/modules/buildifier_prebuilt/8.5.1/source.json": "ae9f3d9dc7bec033976cf47165a78788bebad2b5c272241063ae31bad8664fd4", - "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", - "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", - "https://bcr.bazel.build/modules/c-ares/1.19.1/MODULE.bazel": "73bca21720772370ff91cc8e88bbbaf14897720c6473e87c1ddc0f848284c313", - "https://bcr.bazel.build/modules/c-ares/1.19.1/source.json": "56bfa95b01e4e0012e90eaa9f1bab823c48c3af4454286b889b9cc74f5098da1", - "https://bcr.bazel.build/modules/cel-spec/0.15.0/MODULE.bazel": "e1eed53d233acbdcf024b4b0bc1528116d92c29713251b5154078ab1348cb600", - "https://bcr.bazel.build/modules/cel-spec/0.24.0/MODULE.bazel": "e310c7aff8490ed689ccafd32729b77a660b9547f5a5ba9b20e967011c324b36", - "https://bcr.bazel.build/modules/cel-spec/0.24.0/source.json": "522d08bc22524e07863276dd0f038f446a83166e91281dcfc07d5b8433c8d89e", - "https://bcr.bazel.build/modules/civetweb/1.16/MODULE.bazel": "46a38f9daeb57392e3827fce7d40926be0c802bd23cdd6bfd3a96c804de42fae", - "https://bcr.bazel.build/modules/civetweb/1.16/source.json": "ba8b9585adb8355cb51b999d57172fd05e7a762c56b8d4bac6db42c99de3beb7", - "https://bcr.bazel.build/modules/curl/8.7.1/MODULE.bazel": "088221c35a2939c555e6e47cb31a81c15f8b59f4daa8009b1e9271a502d33485", - "https://bcr.bazel.build/modules/curl/8.8.0/MODULE.bazel": "7da3b3e79b0b4ee8f8c95d640bc6ad7b430ce66ef6e9c9d2bc29b3b5ef85f6fe", - "https://bcr.bazel.build/modules/curl/8.8.0/source.json": "d7d138b6878cf38891692fee0649ace35357fd549b425614d571786f054374d4", - "https://bcr.bazel.build/modules/cython/3.0.11-1/MODULE.bazel": "868b3f5c956c3657420d2302004c6bb92606bfa47e314bab7f2ba0630c7c966c", - "https://bcr.bazel.build/modules/cython/3.0.11-1/source.json": "da318be900b8ca9c3d1018839d3bebc5a8e1645620d0848fa2c696d4ecf7c296", - "https://bcr.bazel.build/modules/envoy_api/0.0.0-20250128-4de3c74/MODULE.bazel": "1fe72489212c530086e3ffb0e018b2bfef4663200ca03571570f9f006bef1d75", - "https://bcr.bazel.build/modules/envoy_api/0.0.0-20250128-4de3c74/source.json": "028519164a2e24563f4b43d810fdedc702daed90e71e7042d45ba82ad807b46f", + "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/MODULE.bazel": "a9b689711d5b69f9db741649b218c119b9fdf82924ba390415037e09798edd03", + "https://bcr.bazel.build/modules/buildifier_prebuilt/8.2.0.2/source.json": "51eb0a4b38aaaeab7fa64361576d616c4d8bfd0f17a0a10184aeab7084d79f8e", + "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", + "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/MODULE.bazel": "f1b7bb2dd53e8f2ef984b39485ec8a44e9076dda5c4b8efd2fb4c6a6e856a31d", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.3/source.json": "ebe931bfe362e4b41e59ee00a528db6074157ff2ced92eb9e970acab2e1089c9", - "https://bcr.bazel.build/modules/gazelle/0.27.0/MODULE.bazel": "3446abd608295de6d90b4a8a118ed64a9ce11dcb3dda2dc3290a22056bd20996", - "https://bcr.bazel.build/modules/gazelle/0.30.0/MODULE.bazel": "f888a1effe338491f35f0e0e85003b47bb9d8295ccba73c37e07702d8d31c65b", - "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", - "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", - "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", - "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0", - "https://bcr.bazel.build/modules/gazelle/0.37.0/MODULE.bazel": "d1327ba0907d0275ed5103bfbbb13518f6c04955b402213319d0d6c0ce9839d4", - "https://bcr.bazel.build/modules/gazelle/0.39.1/MODULE.bazel": "1fa3fefad240e535066fd0e6950dfccd627d36dc699ee0034645e51dbde3980f", - "https://bcr.bazel.build/modules/gazelle/0.45.0/MODULE.bazel": "ecd19ebe9f8e024e1ccffb6d997cc893a974bcc581f1ae08f386bdd448b10687", - "https://bcr.bazel.build/modules/gazelle/0.47.0/MODULE.bazel": "b61bb007c4efad134aa30ee7f4a8e2a39b22aa5685f005edaa022fbd1de43ebc", - "https://bcr.bazel.build/modules/gazelle/0.51.3/MODULE.bazel": "618a729142f66de1e2cb776d026413763be5b80d5e3d29ffb9d3d90c5defde90", - "https://bcr.bazel.build/modules/gazelle/0.51.3/source.json": "fbe5312a01fb4a2a58caff4a0d40dcf384b6f0bda75e85546663360da1d7538a", - "https://bcr.bazel.build/modules/google_benchmark/1.8.4/MODULE.bazel": "c6d54a11dcf64ee63545f42561eda3fd94c1b5f5ebe1357011de63ae33739d5e", - "https://bcr.bazel.build/modules/google_benchmark/1.8.4/source.json": "84590f7bc5a1fd99e1ef274ee16bb41c214f705e62847b42e705010dfa81fe53", - "https://bcr.bazel.build/modules/googleapis-cc/1.0.0/MODULE.bazel": "cf01757e7590c56140a4b81638ff2b3e7074769e6271720bbf738fcda25b6fc2", - "https://bcr.bazel.build/modules/googleapis-cc/1.0.0/source.json": "ab0e3a2ee9968a8848f59872fbbfa3e1f768597d71d2229e6caa319d357967c7", - "https://bcr.bazel.build/modules/googleapis-go/1.0.0/MODULE.bazel": "0a207a4c49da28c5cc1f7b3aeb23c2f7828c85c14aa8d9db0e30357a8d2250ed", - "https://bcr.bazel.build/modules/googleapis-go/1.0.0/source.json": "ef189be4e7853e1ebc6123fe20b71822bf9896bd1f8eed8f68505c4585f72a48", - "https://bcr.bazel.build/modules/googleapis-grpc-cc/1.0.0/MODULE.bazel": "3553358a9d8d96026c9e28d9fb6c268574950d0be7fa9b4c0aeaf3c37c73f2d3", - "https://bcr.bazel.build/modules/googleapis-grpc-cc/1.0.0/source.json": "fa7b79043b3c82bf74f1f2fa45af289e19b247375868d0752db2c114a1c7366c", - "https://bcr.bazel.build/modules/googleapis-java/1.0.0/MODULE.bazel": "d633989337d069b5a95e6101777319681d7a4af4677e36801f11839d6512095c", - "https://bcr.bazel.build/modules/googleapis-java/1.0.0/source.json": "ee59e2de37e4b531172870ac0296afa38f1ea004105ee21b2793c31a9d0ddccd", - "https://bcr.bazel.build/modules/googleapis-python/1.0.0/MODULE.bazel": "0ccd1614a914fb524b3ac267f9c97f9a5cd5412b027f0176b81a725882ec42ff", - "https://bcr.bazel.build/modules/googleapis-python/1.0.0/source.json": "24364f075ec5e6d5e0cfc5d651bd833b119e0cea1a45d51330c93461e3586e42", - "https://bcr.bazel.build/modules/googleapis-rules-registry/1.0.0/MODULE.bazel": "97c6a4d413b373d4cc97065da3de1b2166e22cbbb5f4cc9f05760bfa83619e24", - "https://bcr.bazel.build/modules/googleapis-rules-registry/1.0.0/source.json": "cf611c836a60e98e2e2ab2de8004f119e9f06878dcf4ea2d95a437b1b7a89fe9", - "https://bcr.bazel.build/modules/googleapis/0.0.0-20240326-1c8d509c5/MODULE.bazel": "a4b7e46393c1cdcc5a00e6f85524467c48c565256b22b5fae20f84ab4a999a68", - "https://bcr.bazel.build/modules/googleapis/0.0.0-20240819-fe8ba054a/MODULE.bazel": "117b7c7be7327ed5d6c482274533f2dbd78631313f607094d4625c28203cacdf", - "https://bcr.bazel.build/modules/googleapis/0.0.0-20241220-5e258e33.bcr.1/MODULE.bazel": "ee6c30f82ecd476e61f019fb1151aaab380ea419958ff274ef2f0efca7969f5c", - "https://bcr.bazel.build/modules/googleapis/0.0.0-20251003-2193a2bf/MODULE.bazel": "cc9e5ed294ed9ebf42cdbbdddd2df29048519e3797004df1e3f369f31ff4f2d4", - "https://bcr.bazel.build/modules/googleapis/0.0.0-20251003-2193a2bf/source.json": "21558a194c519e27262cca9cf031bb166a666a8b7fb89993b700c828f8dc0857", + "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", + "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", - "https://bcr.bazel.build/modules/googletest/1.17.0.bcr.2/MODULE.bazel": "827f54f492a3ce549c940106d73de332c2b30cebd0c20c0bc5d786aba7f116cb", - "https://bcr.bazel.build/modules/googletest/1.17.0.bcr.2/source.json": "3664514073a819992320ffbce5825e4238459df344d8b01748af2208f8d2e1eb", "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", - "https://bcr.bazel.build/modules/grpc-java/1.62.2/MODULE.bazel": "99b8771e8c7cacb130170fed2a10c9e8fed26334a93e73b42d2953250885a158", - "https://bcr.bazel.build/modules/grpc-java/1.66.0/MODULE.bazel": "86ff26209fac846adb89db11f3714b3dc0090fb2fb81575673cc74880cda4e7e", - "https://bcr.bazel.build/modules/grpc-proto/0.0.0-20240627-ec30f58/MODULE.bazel": "88de79051e668a04726e9ea94a481ec6f1692086735fd6f488ab908b3b909238", - "https://bcr.bazel.build/modules/grpc/1.80.0/MODULE.bazel": "35ef253a0a31055c8a2495640516224647545b51673b0ccb956b116ee92ef554", - "https://bcr.bazel.build/modules/grpc/1.80.0/source.json": "55bf4cdcf278a450185ff93bef843a0d7db0a83d0b9761d9c1dee91faa9cdf46", - "https://bcr.bazel.build/modules/highway/1.4.0/MODULE.bazel": "22d48d947bda0c685d6174e62f9b6690089fefb87dfff4943cd420410ae82f68", - "https://bcr.bazel.build/modules/highway/1.4.0/source.json": "ae87bf2499d4902a15d760e14b7a1e94dc5dd0564bec256b9b0ed46c0e835f89", + "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5", "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", - "https://bcr.bazel.build/modules/jsoncpp/1.9.6.bcr.2/MODULE.bazel": "64f508885f907ac2518039b3d0c5c1703a8f6137d9f5d635067ba93d33c2670a", - "https://bcr.bazel.build/modules/jsoncpp/1.9.6.bcr.2/source.json": "13199fa0a267ca46814e9e6ab313a71890c8f1822e57376fdc23a2d2cd384051", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", - "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/MODULE.bazel": "4cd5b9355f1c2533c30f01904b71ac1b1a9bf61c43c81ba9972a5bf8f216476b", - "https://bcr.bazel.build/modules/libbacktrace/1.0.0-20250926-7939218/source.json": "83239989d09748f63dee4bdd430d261e217a012670e066e2238925a303a65d89", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", - "https://bcr.bazel.build/modules/libpfm/4.11.0/source.json": "caaffb3ac2b59b8aac456917a4ecf3167d40478ee79f15ab7a877ec9273937c9", - "https://bcr.bazel.build/modules/libxml2/2.15.3/MODULE.bazel": "a6bffafd8aa8b0c13eaa4bf25d55be188ec8548549395f03e98c39bab264adaa", - "https://bcr.bazel.build/modules/libxml2/2.15.3/source.json": "fd7c420cb14981416c9fadd0db555bd1e249355e0e5b63b2ce15b92996b5fbfb", - "https://bcr.bazel.build/modules/llvm/0.8.24/MODULE.bazel": "e03e030b1d0b44bb05c52665c2f43de9260af9152f666262e11e25f6c072aae3", - "https://bcr.bazel.build/modules/llvm/0.8.24/source.json": "924ed48858c1c081698f1992c7b489434375de901c47b7046e203459840cb544", - "https://bcr.bazel.build/modules/mbedtls/3.6.0/MODULE.bazel": "8e380e4698107c5f8766264d4df92e36766248447858db28187151d884995a09", - "https://bcr.bazel.build/modules/mbedtls/3.6.0/source.json": "1dbe7eb5258050afcc3806b9d43050f71c6f539ce0175535c670df606790b30c", - "https://bcr.bazel.build/modules/nlohmann_json/3.11.3/MODULE.bazel": "87023db2f55fc3a9949c7b08dc711fae4d4be339a80a99d04453c4bb3998eefc", - "https://bcr.bazel.build/modules/nlohmann_json/3.12.0.bcr.1/MODULE.bazel": "a1c8bb07b5b91d971727c635f449d05623ac9608f6fe4f5f04254ea12f08e349", - "https://bcr.bazel.build/modules/nlohmann_json/3.12.0.bcr.1/source.json": "93f82a5ae985eb935c539bfee95e04767187818189241ac956f3ccadbdb8fb02", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", - "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/MODULE.bazel": "cc18734138dd18c912c6ce2a59186db28f85d8058c99c9f21b46ca3e0aba0ebe", - "https://bcr.bazel.build/modules/opencensus-cpp/0.0.0-20230502-50eb5de.bcr.2/source.json": "7c135f9d42bb3b045669c3c6ab3bb3c208e00b46aca4422eea64c29811a5b240", - "https://bcr.bazel.build/modules/opencensus-proto/0.4.1.bcr.2/MODULE.bazel": "789706a714855f92c5c8cfcf1ef32bbb64dcd3b7c9066756ad7986ec59709d29", - "https://bcr.bazel.build/modules/opencensus-proto/0.4.1.bcr.2/source.json": "aadf3f53e08b72376506b7c4ea3d167010c9efb160d7d6e1e304ed646bac1b36", - "https://bcr.bazel.build/modules/opencensus-proto/0.4.1/MODULE.bazel": "4a2e8b4d0b544002502474d611a5a183aa282251e14f6a01afe841c0c1b10372", - "https://bcr.bazel.build/modules/openssl/3.3.1.bcr.1/MODULE.bazel": "49c0c07e8fb87b480bccb842cfee1b32617f11dac590f732573c69058699a3d1", - "https://bcr.bazel.build/modules/openssl/3.3.1.bcr.1/source.json": "0c0872e048bbea052a9c541fb47019481a19201ba5555a71d762ad591bf94e1f", - "https://bcr.bazel.build/modules/opentelemetry-cpp/1.19.0/MODULE.bazel": "3455326c08b28415648a3d60d8e3c811847ebdbe64474f75b25878f25585aea1", - "https://bcr.bazel.build/modules/opentelemetry-cpp/1.19.0/source.json": "4e48137e4c3ecb99401ff99876df8fa330598d7da051869bec643446e8a8ff95", - "https://bcr.bazel.build/modules/opentelemetry-proto/1.4.0.bcr.1/MODULE.bazel": "5ceaf25e11170d22eded4c8032728b4a3f273765fccda32f9e94f463755c4167", - "https://bcr.bazel.build/modules/opentelemetry-proto/1.5.0/MODULE.bazel": "7543d91a53b98e7b5b37c5a0865b93bff12c1ee022b1e322cd236b968894b030", - "https://bcr.bazel.build/modules/opentelemetry-proto/1.5.0/source.json": "046b721ce203e88cdaad44d7dd17a86b7200eab9388b663b234e72e13ff7b143", - "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/MODULE.bazel": "b3925269f63561b8b880ae7cf62ccf81f6ece55b62cd791eda9925147ae116ec", - "https://bcr.bazel.build/modules/opentracing-cpp/1.6.0/source.json": "da1cb1add160f5e5074b7272e9db6fd8f1b3336c15032cd0a653af9d2f484aed", + "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", - "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", - "https://bcr.bazel.build/modules/package_metadata/0.0.7/MODULE.bazel": "7adb03933fc8401f495800cf4eafcff0edc6da0ff55c7db223ef69d19f689486", - "https://bcr.bazel.build/modules/package_metadata/0.0.7/source.json": "50639625e937b56115012674c797cca7a05a96b4878c87d803c13dc2b31de8a0", + "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/MODULE.bazel": "35320668481400503dea7c75313473b9ffd6dea4e7ba583b96a896199b0088d5", "https://bcr.bazel.build/modules/patchelf/0.18.0.bcr.1/source.json": "d877d271ef5d531407885322054c99c7f0617a9a85ff99c0a70f4ced7f13e13a", "https://bcr.bazel.build/modules/pcre2/10.45/MODULE.bazel": "9bdc838aa2e21be030d7181385e45a5c87b14575bb7ff35d10b5be3d56913bed", @@ -220,63 +113,28 @@ "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580", - "https://bcr.bazel.build/modules/platforms/1.1.0/MODULE.bazel": "1c0c09f5bdcf4b3f924720d2478a3711cb39f4977019ca5988685e5b7e18b3d2", - "https://bcr.bazel.build/modules/platforms/1.1.0/source.json": "fcf351c47596c939140ab0d333dfdd08ed1ea6ce33c2fe70c12493a301cf1344", - "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/MODULE.bazel": "116ad46e97c1d2aeb020fe2899a342a7e703574ce7c0faf7e4810f938c974a9a", - "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0.bcr.1/source.json": "e813cce2d450708cfcb26e309c5172583a7440776edf354e83e6788c768e5cca", - "https://bcr.bazel.build/modules/prometheus-cpp/1.3.0/MODULE.bazel": "ce82e086bbc0b60267e970f6a54b2ca6d0f22d3eb6633e00e2cc2899c700f3d8", + "https://bcr.bazel.build/modules/platforms/1.0.0/source.json": "f4ff1fd412e0246fd38c82328eb209130ead81d62dcd5a9e40910f867f733d96", "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", - "https://bcr.bazel.build/modules/protobuf/23.1/MODULE.bazel": "88b393b3eb4101d18129e5db51847cd40a5517a53e81216144a8c32dfeeca52a", - "https://bcr.bazel.build/modules/protobuf/24.4/MODULE.bazel": "7bc7ce5f2abf36b3b7b7c8218d3acdebb9426aeb35c2257c96445756f970eb12", - "https://bcr.bazel.build/modules/protobuf/25.6/MODULE.bazel": "fc0ae073b47c7ede88b825ff79e64f1c058967c7a87a86cdf4abecd9e0516625", - "https://bcr.bazel.build/modules/protobuf/26.0/MODULE.bazel": "8402da964092af40097f4a205eec2a33fd4a7748dc43632b7d1629bfd9a2b856", - "https://bcr.bazel.build/modules/protobuf/27.0-rc2/MODULE.bazel": "b2b0dbafd57b6bec0ca9b251da02e628c357dab53a097570aa7d79d020f107cf", "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", - "https://bcr.bazel.build/modules/protobuf/27.2/MODULE.bazel": "32450b50673882e4c8c3d10a83f3bc82161b213ed2f80d17e38bece8f165c295", - "https://bcr.bazel.build/modules/protobuf/29.0-rc2.bcr.1/MODULE.bazel": "52f4126f63a2f0bbf36b99c2a87648f08467a4eaf92ba726bc7d6a500bbf770c", "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92", "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", - "https://bcr.bazel.build/modules/protobuf/29.3/MODULE.bazel": "77480eea5fb5541903e49683f24dc3e09f4a79e0eea247414887bb9fc0066e94", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", - "https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573", - "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", - "https://bcr.bazel.build/modules/protobuf/30.0/MODULE.bazel": "0e736de5d52ad7824113f47e65256a26ee74b689ba859c5447a0663e5a075409", - "https://bcr.bazel.build/modules/protobuf/31.1/MODULE.bazel": "379a389bb330b7b8c1cdf331cc90bf3e13de5614799b3b52cdb7c6f389f6b38e", - "https://bcr.bazel.build/modules/protobuf/32.1/MODULE.bazel": "89cd2866a9cb07fee9ff74c41ceace11554f32e0d849de4e23ac55515cfada4d", - "https://bcr.bazel.build/modules/protobuf/33.4/MODULE.bazel": "114775b816b38b6d0ca620450d6b02550c60ceedfdc8d9a229833b34a223dc42", - "https://bcr.bazel.build/modules/protobuf/34.0.bcr.1/MODULE.bazel": "74e541b0ba877813da786a11707d4e394433c157841d5111a36be0d44b907931", - "https://bcr.bazel.build/modules/protobuf/35.0/MODULE.bazel": "ee97170c013d94c366bbb5fe8a97b0650477cbc00e5e0f6a2c297484bc34d81b", - "https://bcr.bazel.build/modules/protobuf/35.0/source.json": "2162e80adef862606ba85ae42e0df85eb519376fd6ab49caddee6fb776a1281e", - "https://bcr.bazel.build/modules/protoc-gen-validate/1.0.4.bcr.2/MODULE.bazel": "c4bd2c850211ff5b7dadf9d2d0496c1c922fdedc303c775b01dfd3b3efc907ed", - "https://bcr.bazel.build/modules/protoc-gen-validate/1.0.4/MODULE.bazel": "b8913c154b16177990f6126d2d2477d187f9ddc568e95ee3e2d50fc65d2c494a", - "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1.bcr.1/MODULE.bazel": "4bf09676b62fa587ae07e073420a76ec8766dcce7545e5f8c68cfa8e484b5120", - "https://bcr.bazel.build/modules/protoc-gen-validate/1.2.1.bcr.1/source.json": "c19071ebc4b53b5f1cfab9c66eefaf6e4179eb8a998970d07b1077687e777f29", + "https://bcr.bazel.build/modules/protobuf/33.1/MODULE.bazel": "982c8a0cceab4d790076f72b7677faf836b0dfadc2b66a34aab7232116c4ae39", + "https://bcr.bazel.build/modules/protobuf/33.1/source.json": "992c237a40899425648213bf79b05f08c6e8dcd619f96cd944b4511b0276fbd8", "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", - "https://bcr.bazel.build/modules/pybind11_bazel/2.13.6/MODULE.bazel": "2d746fda559464b253b2b2e6073cb51643a2ac79009ca02100ebbc44b4548656", - "https://bcr.bazel.build/modules/pybind11_bazel/3.0.1/MODULE.bazel": "67899804c3dbdf169d107ce7c00e60a38a46d097c7ff3460bdebe192cbc77f56", - "https://bcr.bazel.build/modules/pybind11_bazel/3.0.1/source.json": "5bcb3ae55c0d5a159bd910154d7ee8873bdf61dd7b13cd0a2bb456011ec706ae", - "https://bcr.bazel.build/modules/rapidjson/1.1.0.bcr.20241007/MODULE.bazel": "82fbcb2e42f9e0040e76ccc74c06c3e46dfd33c64ca359293f8b84df0e6dff4c", - "https://bcr.bazel.build/modules/rapidjson/1.1.0.bcr.20241007/source.json": "5c42389ad0e21fc06b95ad7c0b730008271624a2fa3292e0eab5f30e15adeee3", - "https://bcr.bazel.build/modules/re2/2021-09-01/MODULE.bazel": "bcb6b96f3b071e6fe2d8bed9cc8ada137a105f9d2c5912e91d27528b3d123833", + "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/source.json": "6900fdc8a9e95866b8c0d4ad4aba4d4236317b5c1cd04c502df3f0d33afed680", "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", - "https://bcr.bazel.build/modules/re2/2024-05-01/MODULE.bazel": "55a3f059538f381107824e7d00df5df6d061ba1fb80e874e4909c0f0549e8f3e", "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "b4963dda9b31080be1905ef085ecd7dd6cd47c05c79b9cdf83ade83ab2ab271a", + "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/source.json": "2ff292be6ef3340325ce8a045ecc326e92cbfab47c7cbab4bd85d28971b97ac4", "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", - "https://bcr.bazel.build/modules/re2/2025-08-12.bcr.1/MODULE.bazel": "e09b434b122bfb786a69179f9b325e35cb1856c3f56a7a81dd61609260ed46e1", - "https://bcr.bazel.build/modules/re2/2025-11-05/MODULE.bazel": "d97e240364f132017500a3af816493af7efd4be40aab2bb403ed449cc96f70a3", - "https://bcr.bazel.build/modules/re2/2025-11-05/source.json": "7df1394aabda1c9bc188a302f5d54b1c657924edd04ebc57d2be29dbd7efd141", "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", - "https://bcr.bazel.build/modules/rules_android/0.7.1/MODULE.bazel": "a806fc382a774252f228a40e3b11b9fcc6276f8778c7fb33e9f72937c6258363", - "https://bcr.bazel.build/modules/rules_android/0.7.1/source.json": "151440aed3f0f73a00d4ed5cec5d31f63a6fef9b95d8fab1eb1810150fa525f2", - "https://bcr.bazel.build/modules/rules_apple/3.13.0/MODULE.bazel": "b4559a2c6281ca3165275bb36c1f0ac74666632adc5bdb680e366de7ce845f43", + "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", - "https://bcr.bazel.build/modules/rules_apple/4.1.0/MODULE.bazel": "76e10fd4a48038d3fc7c5dc6e63b7063bbf5304a2e3bd42edda6ec660eebea68", - "https://bcr.bazel.build/modules/rules_apple/4.1.0/source.json": "8ee81e1708756f81b343a5eb2b2f0b953f1d25c4ab3d4a68dc02754872e80715", - "https://bcr.bazel.build/modules/rules_buf/0.1.1/MODULE.bazel": "6189aec18a4f7caff599ad41b851ab7645d4f1e114aa6431acf9b0666eb92162", + "https://bcr.bazel.build/modules/rules_apple/3.16.0/source.json": "d8b5fe461272018cc07cfafce11fe369c7525330804c37eec5a82f84cd475366", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", @@ -285,156 +143,92 @@ "https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87", "https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a", "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", - "https://bcr.bazel.build/modules/rules_cc/0.0.5/MODULE.bazel": "be41f87587998fe8890cd82ea4e848ed8eb799e053c224f78f3ff7fe1a1d9b74", + "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", - "https://bcr.bazel.build/modules/rules_cc/0.1.2/MODULE.bazel": "557ddc3a96858ec0d465a87c0a931054d7dcfd6583af2c7ed3baf494407fd8d0", - "https://bcr.bazel.build/modules/rules_cc/0.1.4/MODULE.bazel": "bb03a452a7527ac25a7518fb86a946ef63df860b9657d8323a0c50f8504fb0b9", "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", - "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", - "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.16/MODULE.bazel": "9242fa89f950c6ef7702801ab53922e99c69b02310c39fb6e62b2bd30df2a1d4", - "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", - "https://bcr.bazel.build/modules/rules_cc/0.2.18/MODULE.bazel": "4460ec36adc8f722a6a2a4ac9374cb91f2acebadaa93fc37966129afb3dece87", - "https://bcr.bazel.build/modules/rules_cc/0.2.19/MODULE.bazel": "d5e0f05b63273281a16654eb6b1a8742a75ec153ac8b4f0419949d6e401e46f0", - "https://bcr.bazel.build/modules/rules_cc/0.2.20/MODULE.bazel": "f5c07bce5ddcb99be21a0812ff5aadb439e688b7449c6542152363b2fd859c1a", - "https://bcr.bazel.build/modules/rules_cc/0.2.25/MODULE.bazel": "4a3d4f3606d3b3190be495dbc497acca552807d1d5540661ef0d1658472882e3", - "https://bcr.bazel.build/modules/rules_cc/0.2.25/source.json": "8df28a9a97b878fe45f0a4f4adf019869dda51a60ee2f6e15a0c79656a25058d", + "https://bcr.bazel.build/modules/rules_cc/0.2.16/source.json": "d03d5cde49376d87e14ec14b666c56075e5e3926930327fd5d0484a1ff2ac1cc", "https://bcr.bazel.build/modules/rules_cc/0.2.4/MODULE.bazel": "1ff1223dfd24f3ecf8f028446d4a27608aa43c3f41e346d22838a4223980b8cc", "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_cc/0.2.9/MODULE.bazel": "34263f1dca62ea664265438cef714d7db124c03e1ed55ebb4f1dc860164308d1", - "https://bcr.bazel.build/modules/rules_distroless/0.8.0/MODULE.bazel": "a3e473943b685190e20668f5149f62a116fdcbeba66c7b9778f71d2aed4533c1", - "https://bcr.bazel.build/modules/rules_distroless/0.8.0/source.json": "a6931e20d97a13675693adb8b6cd3a0e6eb5218db43dfc83b75566ddc4158efc", - "https://bcr.bazel.build/modules/rules_foreign_cc/0.10.1/MODULE.bazel": "b9527010e5fef060af92b6724edb3691970a5b1f76f74b21d39f7d433641be60", - "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/MODULE.bazel": "c2c60d26c79fda484acb95cdbec46e89d6b28b4845cb277160ce1e0c8622bb88", - "https://bcr.bazel.build/modules/rules_foreign_cc/0.15.1/source.json": "a161811a63ba8a859086da3b7ff3ad04f2e9c255d7727b41087103fc0eb22f55", + "https://bcr.bazel.build/modules/rules_distroless/0.6.2/MODULE.bazel": "eafe8b473c0c78038d9a6e1dbd9278b7bf83457ddc78207297483bfbe1012d56", + "https://bcr.bazel.build/modules/rules_distroless/0.6.2/source.json": "830d7a25ce382e571e58791fbed3b47e6da3237805624bab8997ae59d763bdc2", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", - "https://bcr.bazel.build/modules/rules_go/0.33.0/MODULE.bazel": "a2b11b64cd24bf94f57454f53288a5dacfe6cb86453eee7761b7637728c1910c", - "https://bcr.bazel.build/modules/rules_go/0.38.1/MODULE.bazel": "fb8e73dd3b6fc4ff9d260ceacd830114891d49904f5bda1c16bc147bcc254f71", - "https://bcr.bazel.build/modules/rules_go/0.39.1/MODULE.bazel": "d34fb2a249403a5f4339c754f1e63dc9e5ad70b47c5e97faee1441fc6636cd61", - "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", - "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", - "https://bcr.bazel.build/modules/rules_go/0.45.1/MODULE.bazel": "6d7884f0edf890024eba8ab31a621faa98714df0ec9d512389519f0edff0281a", - "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", - "https://bcr.bazel.build/modules/rules_go/0.48.0/MODULE.bazel": "d00ebcae0908ee3f5e6d53f68677a303d6d59a77beef879598700049c3980a03", - "https://bcr.bazel.build/modules/rules_go/0.49.0/MODULE.bazel": "61cfc1ba17123356d1b12b6c50f6e0162b2cc7fd6f51753c12471e973a0f72a5", - "https://bcr.bazel.build/modules/rules_go/0.50.1/MODULE.bazel": "b91a308dc5782bb0a8021ad4330c81fea5bda77f96b9e4c117b9b9c8f6665ee0", - "https://bcr.bazel.build/modules/rules_go/0.53.0/MODULE.bazel": "a4ed760d3ac0dbc0d7b967631a9a3fd9100d28f7d9fcf214b4df87d4bfff5f9a", - "https://bcr.bazel.build/modules/rules_go/0.59.0/MODULE.bazel": "b7e43e7414a3139a7547d1b4909b29085fbe5182b6c58cbe1ed4c6272815aeae", - "https://bcr.bazel.build/modules/rules_go/0.62.0/MODULE.bazel": "8ee616065c3d2b2f7ac0880108316ce8d0c332b3a30aad24e95c0bc124ec853e", - "https://bcr.bazel.build/modules/rules_go/0.62.0/source.json": "36d781c558eb7d3bd49dc5e190455714f174232372f15207ab200b4348bda3e6", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", - "https://bcr.bazel.build/modules/rules_java/5.5.0/MODULE.bazel": "486ad1aa15cdc881af632b4b1448b0136c76025a1fe1ad1b65c5899376b83a50", "https://bcr.bazel.build/modules/rules_java/6.0.0/MODULE.bazel": "8a43b7df601a7ec1af61d79345c17b31ea1fedc6711fd4abfd013ea612978e39", "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", "https://bcr.bazel.build/modules/rules_java/6.4.0/MODULE.bazel": "e986a9fe25aeaa84ac17ca093ef13a4637f6107375f64667a15999f77db6c8f6", "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", - "https://bcr.bazel.build/modules/rules_java/7.1.0/MODULE.bazel": "30d9135a2b6561c761bd67bd4990da591e6bdc128790ce3e7afd6a3558b2fb64", "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", "https://bcr.bazel.build/modules/rules_java/7.3.2/MODULE.bazel": "50dece891cfdf1741ea230d001aa9c14398062f2b7c066470accace78e412bc2", - "https://bcr.bazel.build/modules/rules_java/7.4.0/MODULE.bazel": "a592852f8a3dd539e82ee6542013bf2cadfc4c6946be8941e189d224500a8934", "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", - "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1", + "https://bcr.bazel.build/modules/rules_java/8.14.0/MODULE.bazel": "717717ed40cc69994596a45aec6ea78135ea434b8402fb91b009b9151dd65615", + "https://bcr.bazel.build/modules/rules_java/8.14.0/source.json": "8a88c4ca9e8759da53cddc88123880565c520503321e2566b4e33d0287a3d4bc", "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", "https://bcr.bazel.build/modules/rules_java/8.6.0/MODULE.bazel": "9c064c434606d75a086f15ade5edb514308cccd1544c2b2a89bbac4310e41c71", "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", "https://bcr.bazel.build/modules/rules_java/8.8.0/MODULE.bazel": "de589d0880911ac007abd521b9f0ddcd8b0dbd05c8553e6f8124a050b83acf7d", - "https://bcr.bazel.build/modules/rules_java/8.9.0/MODULE.bazel": "e17c876cb53dcd817b7b7f0d2985b710610169729e8c371b2221cacdcd3dce4a", - "https://bcr.bazel.build/modules/rules_java/9.1.0/MODULE.bazel": "ee63f27e36a3fada80342869361182f120a9819c74320e8e65b1e04ba0cd7a9d", - "https://bcr.bazel.build/modules/rules_java/9.3.0/MODULE.bazel": "f657c72d65ac449caae9abf2e68e66c0d36f9416848c4c4903d0b3234229e7f2", - "https://bcr.bazel.build/modules/rules_java/9.3.0/source.json": "59ae7e662c3c7042b88bbb42ad12483523e234c65ebe4c51611baa43e85cb248", "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", "https://bcr.bazel.build/modules/rules_jvm_external/5.3/MODULE.bazel": "bf93870767689637164657731849fb887ad086739bd5d360d90007a581d5527d", - "https://bcr.bazel.build/modules/rules_jvm_external/6.0/MODULE.bazel": "37c93a5a78d32e895d52f86a8d0416176e915daabd029ccb5594db422e87c495", "https://bcr.bazel.build/modules/rules_jvm_external/6.1/MODULE.bazel": "75b5fec090dbd46cf9b7d8ea08cf84a0472d92ba3585b476f44c326eda8059c4", - "https://bcr.bazel.build/modules/rules_jvm_external/6.2/MODULE.bazel": "36a6e52487a855f33cb960724eb56547fa87e2c98a0474c3acad94339d7f8e99", "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0", "https://bcr.bazel.build/modules/rules_jvm_external/6.7/MODULE.bazel": "e717beabc4d091ecb2c803c2d341b88590e9116b8bf7947915eeb33aab4f96dd", - "https://bcr.bazel.build/modules/rules_jvm_external/6.9/MODULE.bazel": "07c5db05527db7744a54fcffd653e1550d40e0540207a7f7e6d0a4de5bef8274", - "https://bcr.bazel.build/modules/rules_jvm_external/6.9/source.json": "b12970214f3cc144b26610caeb101fa622d910f1ab3d98f0bae1058edbd00bd4", + "https://bcr.bazel.build/modules/rules_jvm_external/6.7/source.json": "5426f412d0a7fc6b611643376c7e4a82dec991491b9ce5cb1cfdd25fe2e92be4", "https://bcr.bazel.build/modules/rules_kotlin/1.9.0/MODULE.bazel": "ef85697305025e5a61f395d4eaede272a5393cee479ace6686dba707de804d59", - "https://bcr.bazel.build/modules/rules_kotlin/1.9.5/MODULE.bazel": "043a16a572f610558ec2030db3ff0c9938574e7dd9f58bded1bb07c0192ef025", "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3", - "https://bcr.bazel.build/modules/rules_kotlin/2.3.20/MODULE.bazel": "3443d53d275e14fecfebd0b491f01d06ea3883c04a1b3336e7ae9d5ec9066bef", - "https://bcr.bazel.build/modules/rules_kotlin/2.3.20/source.json": "5a5553cffea43f2c5156c8ad0de4a14ad95413ceb39cd4d08f50e2aea86927e8", + "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5", "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", - "https://bcr.bazel.build/modules/rules_license/0.0.8/MODULE.bazel": "5669c6fe49b5134dbf534db681ad3d67a2d49cfc197e4a95f1ca2fd7f3aebe96", "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", - "https://bcr.bazel.build/modules/rules_nodejs/5.8.2/MODULE.bazel": "6bc03c8f37f69401b888023bf511cb6ee4781433b0cb56236b2e55a21e3a026a", "https://bcr.bazel.build/modules/rules_oci/2.2.7/MODULE.bazel": "f6150e4b224d459f7f6523ef65967464ca4efdd266c7fbf2f5a2a51011957e0c", "https://bcr.bazel.build/modules/rules_oci/2.2.7/source.json": "b099f02af330f47f19dc67fc9300ef6e1937a8c86882690db0e7a2fcea8c7f6b", - "https://bcr.bazel.build/modules/rules_perl/0.2.4/MODULE.bazel": "5f5af7be4bf5fb88d91af7469518f0fd2161718aefc606188f7cd51f436ca938", - "https://bcr.bazel.build/modules/rules_perl/0.2.4/source.json": "574317d6b3c7e4843fe611b76f15e62a1889949f5570702e1ee4ad335ea3c339", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "1e5b502e2e1a9e825eef74476a5a1ee524a92297085015a052510b09a1a09483", - "https://bcr.bazel.build/modules/rules_proto/6.0.0/MODULE.bazel": "b531d7f09f58dce456cd61b4579ce8c86b38544da75184eadaf0a7cb7966453f", "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73", "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2", "https://bcr.bazel.build/modules/rules_proto/7.1.0/MODULE.bazel": "002d62d9108f75bb807cd56245d45648f38275cb3a99dcd45dfb864c5d74cb96", "https://bcr.bazel.build/modules/rules_proto/7.1.0/source.json": "39f89066c12c24097854e8f57ab8558929f9c8d474d34b2c00ac04630ad8940e", "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", - "https://bcr.bazel.build/modules/rules_python/0.20.0/MODULE.bazel": "bfe14d17f20e3fe900b9588f526f52c967a6f281e47a1d6b988679bd15082286", - "https://bcr.bazel.build/modules/rules_python/0.22.0/MODULE.bazel": "b8057bafa11a9e0f4b08fc3b7cd7bee0dcbccea209ac6fc9a3ff051cd03e19e9", - "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", - "https://bcr.bazel.build/modules/rules_python/0.34.0/MODULE.bazel": "1d623d026e075b78c9fde483a889cda7996f5da4f36dffb24c246ab30f06513a", - "https://bcr.bazel.build/modules/rules_python/0.35.0/MODULE.bazel": "c3657951764cdcdb5a7370d5e885fad5e8c1583320aad18d46f9f110d2c22755", - "https://bcr.bazel.build/modules/rules_python/0.37.1/MODULE.bazel": "3faeb2d9fa0a81f8980643ee33f212308f4d93eea4b9ce6f36d0b742e71e9500", - "https://bcr.bazel.build/modules/rules_python/0.37.2/MODULE.bazel": "b5ffde91410745750b6c13be1c5dc4555ef5bc50562af4a89fd77807fdde626a", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/rules_python/0.40.0/MODULE.bazel": "9d1a3cd88ed7d8e39583d9ffe56ae8a244f67783ae89b60caafc9f5cf318ada7", "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", - "https://bcr.bazel.build/modules/rules_python/1.2.0/MODULE.bazel": "5aeeb48b2a6c19d668b48adf2b8a2b209a6310c230db0ce77450f148a89846e4", - "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", - "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", - "https://bcr.bazel.build/modules/rules_python/1.5.1/MODULE.bazel": "acfe65880942d44a69129d4c5c3122d57baaf3edf58ae5a6bd4edea114906bf5", "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", - "https://bcr.bazel.build/modules/rules_python/1.6.3/MODULE.bazel": "a7b80c42cb3de5ee2a5fa1abc119684593704fcd2fec83165ebe615dec76574f", - "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", - "https://bcr.bazel.build/modules/rules_python/1.8.0/MODULE.bazel": "c151c025dbcc93d8f62ab68ecc313c9176a868a0e6386981bf2a12aec77cbe7b", - "https://bcr.bazel.build/modules/rules_python/1.9.0/MODULE.bazel": "afc3a05f29f09f2d3ee95ad99a145250dab41a2b2d8d6f82cc91936b3213282c", - "https://bcr.bazel.build/modules/rules_python/1.9.0/source.json": "3921ea0b65298d51aead5b9e4a82203d7be9b5918619b58b53f1c259f4e63169", - "https://bcr.bazel.build/modules/rules_robolectric/4.14.1.2/MODULE.bazel": "d44fec647d0aeb67b9f3b980cf68ba634976f3ae7ccd6c07d790b59b87a4f251", - "https://bcr.bazel.build/modules/rules_robolectric/4.14.1.2/source.json": "37c10335f2361c337c5c1f34ed36d2da70534c23088062b33a8bdaab68aa9dea", - "https://bcr.bazel.build/modules/rules_rust/0.51.0/MODULE.bazel": "2b6d1617ac8503bfdcc0e4520c20539d4bba3a691100bee01afe193ceb0310f9", - "https://bcr.bazel.build/modules/rules_rust/0.69.0/MODULE.bazel": "4326fec48f2fef0d514de46346f7f77e200c82936dd08b91c9ef039fbdad5c10", - "https://bcr.bazel.build/modules/rules_rust/0.69.0/source.json": "0d094307d690cc18b3ab003998697be8070a206f65592c5c8476999796f11c4b", + "https://bcr.bazel.build/modules/rules_python/1.8.4/MODULE.bazel": "33e3971e66161a3e955f7a0d411a8d1f291c4ce4c561851512466f3c77ff8ece", + "https://bcr.bazel.build/modules/rules_python/1.8.4/source.json": "9fbc0e57bae52cddcc3831d668bce87a47e0c655104a85098d4459dd9a3b0a10", + "https://bcr.bazel.build/modules/rules_rs/0.0.28/MODULE.bazel": "3c3830eda824d0476818443dd4a3a2fa0765484589db874d18e94fb6ee50a60f", + "https://bcr.bazel.build/modules/rules_rs/0.0.28/source.json": "3480135f6ef8111ec7abe2b7cc164af40354b4eaf8f885170567922a70a2dc99", + "https://bcr.bazel.build/modules/rules_rust/0.66.0/MODULE.bazel": "86ef763a582f4739a27029bdcc6c562258ed0ea6f8d58294b049e215ceb251b3", + "https://bcr.bazel.build/modules/rules_rust/0.68.1/MODULE.bazel": "8d3332ef4079673385eb81f8bd68b012decc04ac00c9d5a01a40eff90301732c", + "https://bcr.bazel.build/modules/rules_rust/0.68.1/source.json": "3378e746f81b62457fdfd37391244fa8ff075ba85c05931ee4f3a20ac1efe963", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", "https://bcr.bazel.build/modules/rules_shell/0.4.0/MODULE.bazel": "0f8f11bb3cd11755f0b48c1de0bbcf62b4b34421023aa41a2fc74ef68d9584f0", "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", - "https://bcr.bazel.build/modules/rules_shell/0.8.0/MODULE.bazel": "f6a89f1d6a669a26f28fe814503857055d76306b79cfc11d12399af08d0b80ae", - "https://bcr.bazel.build/modules/rules_shell/0.8.0/source.json": "eb53cc815bc503c6683c5fe12d943f98883f81fc22f51403ec8a95610cba4195", + "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", - "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", - "https://bcr.bazel.build/modules/rules_swift/2.5.0/MODULE.bazel": "6d2fec72be0ad2e1fc2eab536cf9d3fb149037c6d2aa678078f3512c06a819e1", - "https://bcr.bazel.build/modules/rules_swift/3.1.2/MODULE.bazel": "72c8f5cf9d26427cee6c76c8e3853eb46ce6b0412a081b2b6db6e8ad56267400", - "https://bcr.bazel.build/modules/rules_swift/3.6.1/MODULE.bazel": "5a68bb05b6b8044f8a7bd7184d56150189e7eca05522ce525517c234fed50ed8", - "https://bcr.bazel.build/modules/rules_swift/3.6.1/source.json": "9af2e521ffdb41961b6b9ac05293bd6519ee2fb52849a2be658a20b0221cb8db", - "https://bcr.bazel.build/modules/rules_zig/0.17.0/MODULE.bazel": "2c1c08adc986cfdb5fee97e21982cac846f8a20a6a8054ae9cab8b02721c2011", - "https://bcr.bazel.build/modules/rules_zig/0.17.0/source.json": "488f71f57c596ed9b7635ebeb28066b23c6fc1802c30749d06bdad261d7b04a2", - "https://bcr.bazel.build/modules/stardoc/0.5.0/MODULE.bazel": "f9f1f46ba8d9c3362648eea571c6f9100680efc44913618811b58cc9c02cd678", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/source.json": "40fc69dfaac64deddbb75bd99cdac55f4427d9ca0afbe408576a65428427a186", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", "https://bcr.bazel.build/modules/stardoc/0.5.4/MODULE.bazel": "6569966df04610b8520957cb8e97cf2e9faac2c0309657c537ab51c16c18a2a4", @@ -445,53 +239,35 @@ "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", - "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", - "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/source.json": "5fba48bbe0ba48761f9e9f75f92876cafb5d07c0ce059cc7a8027416de94a05b", - "https://bcr.bazel.build/modules/tar.bzl/0.10.4/MODULE.bazel": "e8f9ff79199e8d9eaad7f1b0a77ad74b30bb82d794b87d8ca942bead5de83ae9", - "https://bcr.bazel.build/modules/tar.bzl/0.10.4/source.json": "20143442376c03426f6135292ba02d825cb75308aa47e6bf42dd4cc5a435c2ff", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/source.json": "32bd87e5f4d7acc57c5b2ff7c325ae3061d5e242c0c4c214ae87e0f1c13e54cb", "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", "https://bcr.bazel.build/modules/tar.bzl/0.5.1/MODULE.bazel": "7c2eb3dcfc53b0f3d6f9acdfd911ca803eaf92aadf54f8ca6e4c1f3aee288351", "https://bcr.bazel.build/modules/tar.bzl/0.5.5/MODULE.bazel": "4bfab9bbc7a1966c2c5f7371f5848f5e2d27c465951b4435adc9aaf00ed681da", "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", + "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", + "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/MODULE.bazel": "a2f1469b2920fcc9c043c87301b7a207d320231615abecd7771d89e516fc3d0b", + "https://bcr.bazel.build/modules/toolchains_llvm_bootstrapped/0.5.5/source.json": "8498c094703da13a859d19dd2a200152f75870fb7f22a9770df99d230a4f50d1", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", - "https://bcr.bazel.build/modules/upb/0.0.0-20230516-61a97ef/MODULE.bazel": "c0df5e35ad55e264160417fd0875932ee3c9dda63d9fccace35ac62f45e1b6f9", - "https://bcr.bazel.build/modules/windows_support/0.4.1/MODULE.bazel": "2287f95ac65c48ea05197f626a9b5711ea3c88cea61f2a5782882bdd55fdeb51", - "https://bcr.bazel.build/modules/windows_support/0.4.1/source.json": "bdb72e6717b00135e0a4a8512b3f6b626df0baea0bde74bf7312c7484f791287", "https://bcr.bazel.build/modules/with_cfg.bzl/0.12.0/MODULE.bazel": "b573395fe63aef4299ba095173e2f62ccfee5ad9bbf7acaa95dba73af9fc2b38", "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/MODULE.bazel": "aa0ef3f6c67dd35db7ac38c76785dc02f48b53f34e16b20487e70f51b5324d0a", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/MODULE.bazel": "be2276f83878525d8f228b9ec777daee5eb7baba6100dad45081510d0cec35e3", - "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.4/source.json": "63d661a0c144ca74fd8b0aa61e458c7627554c7d4a8e0a4d1ce00975437c776a", - "https://bcr.bazel.build/modules/xds/0.0.0-20240423-555b57e/MODULE.bazel": "cea509976a77e34131411684ef05a1d6ad194dd71a8d5816643bc5b0af16dc0f", - "https://bcr.bazel.build/modules/xds/0.0.0-20251210-ee656c7/MODULE.bazel": "03e1c50b16fc2403de5dd98ff8e2dc0b404865740434b8705c7c0853241f943d", - "https://bcr.bazel.build/modules/xds/0.0.0-20251210-ee656c7/source.json": "69b87f3bff38ba29399c012f3e3fc0f239ac4e5c1ae66ebd9481be6d8461da55", - "https://bcr.bazel.build/modules/xz/5.4.5.bcr.9/MODULE.bazel": "ff7cf8a5334a96b7b9ffcbe73418d45929ce125efebac39c1ee7d14bd2ae7a9b", - "https://bcr.bazel.build/modules/xz/5.4.5.bcr.9/source.json": "abf61f3c1630eab7b81722f710060b6f31e7c160511bcbfaad74bb4385883051", + "https://bcr.bazel.build/modules/with_cfg.bzl/0.14.1/source.json": "4666d3035f69063ecd136f3f0c68e424f1b1dd97ffc9cceb679522c0be38761b", "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/MODULE.bazel": "9bcb7151b3cd4681b89d350530eaf7b45e32a44dda94843b8932b0cb1cd4594a", "https://bcr.bazel.build/modules/yq.bzl/0.3.1/source.json": "f0b0f204a2a6b0e34b4c9541efe8c04f2ef1af65948daa784eccea738b21dbd2", - "https://bcr.bazel.build/modules/zlib-ng/2.3.3.bcr.2/MODULE.bazel": "86130924115bf8dfc4aba286fb49a2ec8d32430a21771db9ccf5381acf0d10cc", - "https://bcr.bazel.build/modules/zlib-ng/2.3.3.bcr.2/source.json": "7aa4112eb2fbb99757e97cf0b7a4e2bdb3464b2b3cfb351bc807ffcb34bd349f", - "https://bcr.bazel.build/modules/zlib-ng/2.3.3/MODULE.bazel": "0afa781a5f354b4fd811a5c9a086e111daf50e09d8e98ab1cf7c46eeacc33aac", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", - "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", - "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.1/MODULE.bazel": "6a9fe6e3fc865715a7be9823ce694ceb01e364c35f7a846bf0d2b34762bc066b", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", - "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", - "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.1/MODULE.bazel": "c5977176dd8555be7a9d598512ae0cae11831259c06f00a5e5e0037d5db4e3f5", - "https://bcr.bazel.build/modules/zstd/1.5.7.bcr.1/source.json": "aa95e0b5aac9d80195b9047d223beaf26b1948be55d49f0e803e180e9ccc6e75" + "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198" }, "selectedYankedVersions": {}, "moduleExtensions": { "@@aspect_rules_py+//py:extensions.bzl%py_tools": { "general": { - "bzlTransitiveDigest": "6o8lyMtImPc3Si+zcGss6WH8IY80mTWhC0MC3r0ra8Y=", + "bzlTransitiveDigest": "GLVIk55uU2jb460gpL1FFbE/tBFqzMxRfta5QXYfjQo=", "usagesDigest": "9BiP3ASHlGl4doGhzXJwK17taNzv/q6x/IZ4Hb6LxNY=", - "recordedInputs": [ - "REPO_MAPPING:aspect_rules_py+,aspect_tools_telemetry_report aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report", - "REPO_MAPPING:aspect_rules_py+,bazel_skylib bazel_skylib+", - "REPO_MAPPING:aspect_rules_py+,bazel_tools bazel_tools" - ], + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, "generatedRepoSpecs": { "rules_py_tools.darwin_amd64": { "repoRuleId": "@@aspect_rules_py+//py/private/toolchain:repo.bzl%prebuilt_tool_repo", @@ -533,54 +309,130 @@ "downloaded_file_path": "pex-2.3.1-py2.py3-none-any.whl" } } - } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_rules_py+", + "aspect_tools_telemetry_report", + "aspect_tools_telemetry++telemetry+aspect_tools_telemetry_report" + ], + [ + "aspect_rules_py+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "aspect_rules_py+", + "bazel_tools", + "bazel_tools" + ] + ] } }, "@@aspect_tools_telemetry+//:extension.bzl%telemetry": { "general": { "bzlTransitiveDigest": "MePriaXmQNSqUfE+YQvEtzBc2bU1mjITINIffFiZYgo=", - "usagesDigest": "pyph4lSDn5CHIn4s2mtKJTkRaLV/o2zIFgnrpufr/8o=", - "recordedInputs": [ - "REPO_MAPPING:aspect_tools_telemetry+,bazel_lib bazel_lib+", - "REPO_MAPPING:aspect_tools_telemetry+,bazel_skylib bazel_skylib+" - ], + "usagesDigest": "73l8h4f8amAmbKge5Ixr/4s4BE7L7fqDBTTAMrNSHHg=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, "generatedRepoSpecs": { "aspect_tools_telemetry_report": { "repoRuleId": "@@aspect_tools_telemetry+//:extension.bzl%tel_repository", "attributes": { "deps": { "aspect_rules_py": "1.8.4", + "rules_rs": "0.0.28", "aspect_tools_telemetry": "0.3.3" } } } - } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_tools_telemetry+", + "bazel_lib", + "bazel_lib+" + ], + [ + "aspect_tools_telemetry+", + "bazel_skylib", + "bazel_skylib+" + ] + ] } }, - "@@rules_android+//rules/android_sdk_repository:rule.bzl%android_sdk_repository_extension": { + "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { - "bzlTransitiveDigest": "aLDGLKolYFwLsBssOQfwOs+bo3Aof0yKkW0HQRPX3ms=", - "usagesDigest": "iEGI2aNDMkHt9LXCdViLNUUOslpiVj2DrevWWXZEFnU=", - "recordedInputs": [], + "bzlTransitiveDigest": "AEsQvKwdstptzt98AO1bJYfoWpPUn2upx53ra6o/Rc8=", + "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, "generatedRepoSpecs": { - "androidsdk": { - "repoRuleId": "@@rules_android+//rules/android_sdk_repository:rule.bzl%_android_sdk_repository", - "attributes": {} + "com_github_jetbrains_kotlin_git": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository", + "attributes": { + "urls": [ + "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip" + ], + "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88" + } + }, + "com_github_jetbrains_kotlin": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_capabilities_repository", + "attributes": { + "git_repository_name": "com_github_jetbrains_kotlin_git", + "compiler_version": "1.9.23" + } + }, + "com_github_google_ksp": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:ksp.bzl%ksp_compiler_plugin_repository", + "attributes": { + "urls": [ + "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip" + ], + "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d", + "strip_version": "1.9.23-1.0.20" + } + }, + "com_github_pinterest_ktlint": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985", + "urls": [ + "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint" + ], + "executable": true + } + }, + "rules_android": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806", + "strip_prefix": "rules_android-0.1.1", + "urls": [ + "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip" + ] + } } - } + }, + "recordedRepoMappingEntries": [ + [ + "rules_kotlin+", + "bazel_tools", + "bazel_tools" + ] + ] } }, "@@rules_oci+//oci:extensions.bzl%oci": { "general": { - "bzlTransitiveDigest": "cbfE3+b9HnO8wGMMbdSzaHKjcON7SL8gb4fKTNiVHug=", - "usagesDigest": "JLbamplho20rviuebatMGqnQnaoj32ewCzLuRVusGoU=", - "recordedInputs": [ - "REPO_MAPPING:aspect_bazel_lib+,bazel_tools bazel_tools", - "REPO_MAPPING:bazel_features+,bazel_tools bazel_tools", - "REPO_MAPPING:rules_oci+,aspect_bazel_lib aspect_bazel_lib+", - "REPO_MAPPING:rules_oci+,bazel_features bazel_features+", - "REPO_MAPPING:rules_oci+,bazel_skylib bazel_skylib+" - ], + "bzlTransitiveDigest": "nCIRDeZJX85AXhvovlzr0Sx1f3ZKx8IS8xvTHWlUlqM=", + "usagesDigest": "IIGPr69MT/mSUo6fNUffrIiU31EqGjlEYm5qFmNjmUc=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, "generatedRepoSpecs": { "distroless_cc_debian12_linux_amd64": { "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", @@ -611,35 +463,6 @@ "reproducible": true } }, - "debian_12_linux_amd64": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_pull", - "attributes": { - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "mirror.gcr.io", - "repository": "library/debian", - "identifier": "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", - "platform": "linux/amd64", - "target_name": "debian_12_linux_amd64", - "bazel_tags": [] - } - }, - "debian_12": { - "repoRuleId": "@@rules_oci+//oci/private:pull.bzl%oci_alias", - "attributes": { - "target_name": "debian_12", - "www_authenticate_challenges": {}, - "scheme": "https", - "registry": "mirror.gcr.io", - "repository": "library/debian", - "identifier": "sha256:bc960ef50e6feed90686c593361df158517556ed1d2d98e5d1df3724024e0f49", - "platforms": { - "@@platforms//cpu:x86_64": "@debian_12_linux_amd64" - }, - "bzlmod_repository": "debian_12", - "reproducible": true - } - }, "oci_crane_darwin_amd64": { "repoRuleId": "@@rules_oci+//oci:repositories.bzl%crane_repositories", "attributes": { @@ -757,111 +580,370 @@ "moduleExtensionMetadata": { "explicitRootModuleDirectDeps": [ "distroless_cc_debian12", - "distroless_cc_debian12_linux_amd64", - "debian_12", - "debian_12_linux_amd64" + "distroless_cc_debian12_linux_amd64" ], "explicitRootModuleDirectDevDeps": [], "useAllRepos": "NO", "reproducible": false - } + }, + "recordedRepoMappingEntries": [ + [ + "aspect_bazel_lib+", + "bazel_tools", + "bazel_tools" + ], + [ + "bazel_features+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_oci+", + "aspect_bazel_lib", + "aspect_bazel_lib+" + ], + [ + "rules_oci+", + "bazel_features", + "bazel_features+" + ], + [ + "rules_oci+", + "bazel_skylib", + "bazel_skylib+" + ] + ] + } + }, + "@@rules_python+//python/extensions:config.bzl%config": { + "general": { + "bzlTransitiveDigest": "X49Lbeli5+XwNYRk+isOvNKFMIF8g/rnxL3Fdlwrffc=", + "usagesDigest": "p2al+dDKI5UlCyNvheMVynbWSGbdiji/jMz53fMNfJA=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "rules_python_internal": { + "repoRuleId": "@@rules_python+//python/private:internal_config_repo.bzl%internal_config_repo", + "attributes": { + "transition_setting_generators": {}, + "transition_settings": [] + } + }, + "pypi__build": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", + "sha256": "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__click": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", + "sha256": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__colorama": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__importlib_metadata": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", + "sha256": "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__installer": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", + "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__more_itertools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", + "sha256": "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__packaging": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", + "sha256": "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pep517": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", + "sha256": "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pip": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", + "sha256": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pip_tools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", + "sha256": "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pyproject_hooks": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", + "sha256": "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__setuptools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/90/99/158ad0609729111163fc1f674a5a42f2605371a4cf036d0441070e2f7455/setuptools-78.1.1-py3-none-any.whl", + "sha256": "c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__tomli": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", + "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__wheel": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", + "sha256": "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__zipp": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", + "sha256": "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + } + }, + "recordedRepoMappingEntries": [ + [ + "rules_python+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_python+", + "pypi__build", + "rules_python++config+pypi__build" + ], + [ + "rules_python+", + "pypi__click", + "rules_python++config+pypi__click" + ], + [ + "rules_python+", + "pypi__colorama", + "rules_python++config+pypi__colorama" + ], + [ + "rules_python+", + "pypi__importlib_metadata", + "rules_python++config+pypi__importlib_metadata" + ], + [ + "rules_python+", + "pypi__installer", + "rules_python++config+pypi__installer" + ], + [ + "rules_python+", + "pypi__more_itertools", + "rules_python++config+pypi__more_itertools" + ], + [ + "rules_python+", + "pypi__packaging", + "rules_python++config+pypi__packaging" + ], + [ + "rules_python+", + "pypi__pep517", + "rules_python++config+pypi__pep517" + ], + [ + "rules_python+", + "pypi__pip", + "rules_python++config+pypi__pip" + ], + [ + "rules_python+", + "pypi__pip_tools", + "rules_python++config+pypi__pip_tools" + ], + [ + "rules_python+", + "pypi__pyproject_hooks", + "rules_python++config+pypi__pyproject_hooks" + ], + [ + "rules_python+", + "pypi__setuptools", + "rules_python++config+pypi__setuptools" + ], + [ + "rules_python+", + "pypi__tomli", + "rules_python++config+pypi__tomli" + ], + [ + "rules_python+", + "pypi__wheel", + "rules_python++config+pypi__wheel" + ], + [ + "rules_python+", + "pypi__zipp", + "rules_python++config+pypi__zipp" + ] + ] } }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { - "bzlTransitiveDigest": "ZtcPw94gqLP8+ocBKipnnyLIivxEC/OUYen7Kx7hnEA=", - "usagesDigest": "Ucfum/vsVnlkhtglTERDLTj+bBeG7dcpbV9h0LP9HOQ=", - "recordedInputs": [ - "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", - "REPO_MAPPING:rules_python+,platforms platforms" - ], + "bzlTransitiveDigest": "CJBWuaTerWIqLpLWl77vjWFD1RRpXfzC2UGMwEk6eVg=", + "usagesDigest": "lPQI4kxB6q1JLGHa7OFxCNrWxnLjeMKg5ogOIB5OF6U=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, "generatedRepoSpecs": { - "uv_0_9_30_aarch64_apple_darwin": { + "uv_0_6_5_aarch64_apple_darwin": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.9.30", + "version": "0.6.5", "platform": "aarch64-apple-darwin", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-apple-darwin.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-apple-darwin.tar.gz" ], - "sha256": "03a5d9ec7f7d588446b2ec226d13ff6300055e55365eca8f3fab39f342b0e805" + "sha256": "4952c6275778fbc7e5cf6b5739ffd944c47063134d6ea43a7037b0a028ef771e" } }, - "uv_0_9_30_aarch64_unknown_linux_gnu": { + "uv_0_6_5_aarch64_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.9.30", + "version": "0.6.5", "platform": "aarch64-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-aarch64-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-aarch64-unknown-linux-gnu.tar.gz" ], - "sha256": "6aadf3c71600d594e16dabf382cc15282ead4c5ca768599b6bcb43c5004d9aa8" + "sha256": "ebf0707ab949b948ad14250ed93f968fec954e20d12ce6bc9fce74b0ce7e4900" } }, - "uv_0_9_30_powerpc64_unknown_linux_gnu": { + "uv_0_6_5_powerpc64_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.9.30", + "version": "0.6.5", "platform": "powerpc64-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64-unknown-linux-gnu.tar.gz" ], - "sha256": "ac4cd1a021462885932f6023b005a4835cca4c72bb60dec186ee2be4b60dca6f" + "sha256": "1f8fd8dde91258380dbcd4b9fc65f15a7dc3af6cfffefb9a4f99d32c4c67436d" } }, - "uv_0_9_30_powerpc64le_unknown_linux_gnu": { + "uv_0_6_5_powerpc64le_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.9.30", + "version": "0.6.5", "platform": "powerpc64le-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-powerpc64le-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-powerpc64le-unknown-linux-gnu.tar.gz" ], - "sha256": "73b8cbc560c6b2fa205358365d4e174abdf50cfcf57dc36a447572c56eba5ae4" + "sha256": "d2b6e19ca7ecd8d7e0d1ddced965aa91a3c822d07a7f7657743945245c1f627a" } }, - "uv_0_9_30_s390x_unknown_linux_gnu": { + "uv_0_6_5_s390x_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.9.30", + "version": "0.6.5", "platform": "s390x-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-s390x-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-s390x-unknown-linux-gnu.tar.gz" ], - "sha256": "b35975bb9e5c2c418b428d0316cc6e3c7a6eff710c69212be14005c192f54516" + "sha256": "43a3e39f19fd5d098bb8c896b6c0f99a146253c271d78409cf31c2f55f2763d0" } }, - "uv_0_9_30_x86_64_apple_darwin": { + "uv_0_6_5_x86_64_apple_darwin": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.9.30", + "version": "0.6.5", "platform": "x86_64-apple-darwin", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-apple-darwin.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-apple-darwin.tar.gz" ], - "sha256": "ce069bf750567e9a4a31d6e285d1eae75d444d8a281409b641235903943b7681" + "sha256": "d336d862694e1c36afea55eb15f5df02aa77c04a168b781df515d641e23285f0" } }, - "uv_0_9_30_x86_64_pc_windows_msvc": { + "uv_0_6_5_x86_64_pc_windows_msvc": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.9.30", + "version": "0.6.5", "platform": "x86_64-pc-windows-msvc", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-pc-windows-msvc.zip" + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-pc-windows-msvc.zip" ], - "sha256": "875981be7908295937dee09532bb66d576986d4f223259e171b0c767c885897a" + "sha256": "a3e9f0913189c38b804bf6bb3d348af305a8ceabc690be92b6e111245a821640" } }, - "uv_0_9_30_x86_64_unknown_linux_gnu": { + "uv_0_6_5_x86_64_unknown_linux_gnu": { "repoRuleId": "@@rules_python+//python/uv/private:uv_repository.bzl%uv_repository", "attributes": { - "version": "0.9.30", + "version": "0.6.5", "platform": "x86_64-unknown-linux-gnu", "urls": [ - "https://github.com/astral-sh/uv/releases/download/0.9.30/uv-x86_64-unknown-linux-gnu.tar.gz" + "https://github.com/astral-sh/uv/releases/download/0.6.5/uv-x86_64-unknown-linux-gnu.tar.gz" ], - "sha256": "8b3762374972daa7a74bbc6896cc73229ca69a07403dd9f9ea3805a51ffd7582" + "sha256": "8fc9895719a1291ecd193cb86f9282ff3649cef797d29eacc74c4f573aab1e2f" } }, "uv": { @@ -869,55 +951,55 @@ "attributes": { "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", "toolchain_names": [ - "0_9_30_aarch64_apple_darwin", - "0_9_30_aarch64_unknown_linux_gnu", - "0_9_30_powerpc64_unknown_linux_gnu", - "0_9_30_powerpc64le_unknown_linux_gnu", - "0_9_30_s390x_unknown_linux_gnu", - "0_9_30_x86_64_apple_darwin", - "0_9_30_x86_64_pc_windows_msvc", - "0_9_30_x86_64_unknown_linux_gnu" + "0_6_5_aarch64_apple_darwin", + "0_6_5_aarch64_unknown_linux_gnu", + "0_6_5_powerpc64_unknown_linux_gnu", + "0_6_5_powerpc64le_unknown_linux_gnu", + "0_6_5_s390x_unknown_linux_gnu", + "0_6_5_x86_64_apple_darwin", + "0_6_5_x86_64_pc_windows_msvc", + "0_6_5_x86_64_unknown_linux_gnu" ], "toolchain_implementations": { - "0_9_30_aarch64_apple_darwin": "@uv_0_9_30_aarch64_apple_darwin//:uv_toolchain", - "0_9_30_aarch64_unknown_linux_gnu": "@uv_0_9_30_aarch64_unknown_linux_gnu//:uv_toolchain", - "0_9_30_powerpc64_unknown_linux_gnu": "@uv_0_9_30_powerpc64_unknown_linux_gnu//:uv_toolchain", - "0_9_30_powerpc64le_unknown_linux_gnu": "@uv_0_9_30_powerpc64le_unknown_linux_gnu//:uv_toolchain", - "0_9_30_s390x_unknown_linux_gnu": "@uv_0_9_30_s390x_unknown_linux_gnu//:uv_toolchain", - "0_9_30_x86_64_apple_darwin": "@uv_0_9_30_x86_64_apple_darwin//:uv_toolchain", - "0_9_30_x86_64_pc_windows_msvc": "@uv_0_9_30_x86_64_pc_windows_msvc//:uv_toolchain", - "0_9_30_x86_64_unknown_linux_gnu": "@uv_0_9_30_x86_64_unknown_linux_gnu//:uv_toolchain" + "0_6_5_aarch64_apple_darwin": "@uv_0_6_5_aarch64_apple_darwin//:uv_toolchain", + "0_6_5_aarch64_unknown_linux_gnu": "@uv_0_6_5_aarch64_unknown_linux_gnu//:uv_toolchain", + "0_6_5_powerpc64_unknown_linux_gnu": "@uv_0_6_5_powerpc64_unknown_linux_gnu//:uv_toolchain", + "0_6_5_powerpc64le_unknown_linux_gnu": "@uv_0_6_5_powerpc64le_unknown_linux_gnu//:uv_toolchain", + "0_6_5_s390x_unknown_linux_gnu": "@uv_0_6_5_s390x_unknown_linux_gnu//:uv_toolchain", + "0_6_5_x86_64_apple_darwin": "@uv_0_6_5_x86_64_apple_darwin//:uv_toolchain", + "0_6_5_x86_64_pc_windows_msvc": "@uv_0_6_5_x86_64_pc_windows_msvc//:uv_toolchain", + "0_6_5_x86_64_unknown_linux_gnu": "@uv_0_6_5_x86_64_unknown_linux_gnu//:uv_toolchain" }, "toolchain_compatible_with": { - "0_9_30_aarch64_apple_darwin": [ + "0_6_5_aarch64_apple_darwin": [ "'@@platforms//os:macos'", "'@@platforms//cpu:aarch64'" ], - "0_9_30_aarch64_unknown_linux_gnu": [ + "0_6_5_aarch64_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:aarch64'" ], - "0_9_30_powerpc64_unknown_linux_gnu": [ + "0_6_5_powerpc64_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:ppc'" ], - "0_9_30_powerpc64le_unknown_linux_gnu": [ + "0_6_5_powerpc64le_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:ppc64le'" ], - "0_9_30_s390x_unknown_linux_gnu": [ + "0_6_5_s390x_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:s390x'" ], - "0_9_30_x86_64_apple_darwin": [ + "0_6_5_x86_64_apple_darwin": [ "'@@platforms//os:macos'", "'@@platforms//cpu:x86_64'" ], - "0_9_30_x86_64_pc_windows_msvc": [ + "0_6_5_x86_64_pc_windows_msvc": [ "'@@platforms//os:windows'", "'@@platforms//cpu:x86_64'" ], - "0_9_30_x86_64_unknown_linux_gnu": [ + "0_6_5_x86_64_unknown_linux_gnu": [ "'@@platforms//os:linux'", "'@@platforms//cpu:x86_64'" ] @@ -925,24 +1007,36 @@ "toolchain_target_settings": {} } } - } + }, + "recordedRepoMappingEntries": [ + [ + "rules_python+", + "bazel_tools", + "bazel_tools" + ], + [ + "rules_python+", + "platforms", + "platforms" + ] + ] } }, "@@rules_zig+//zig:extensions.bzl%zig": { "general": { - "bzlTransitiveDigest": "hPjDj28qKZsSFAE45ZtQPcgDkeRXAcwxiuAQyeEF7ng=", - "usagesDigest": "FDLiNUluzC0WBatQ/5WFpxzyQP4ubA/DUgTwOzYLGw4=", - "recordedInputs": [ - "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", - "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", - "FILE:@@//bazel/zig_index.json 387d1ab130896456fb092fb08bfaf0c57ccba854121955c752052600d1643d37", - "FILE:@@rules_zig+//zig/private/versions.json a33370258a702099b35362d4220fe8f02c869c2b845abcbf0aa78200abab097a" - ], + "bzlTransitiveDigest": "p1+tPOh3UlhcIaGGf1ycZCCC6I4NIHhiiHS0nAhhgHA=", + "usagesDigest": "8TQ1JYfn2xQJIQya4EwCvLiuZAdGIocYzOMc1au4GNs=", + "recordedFileInputs": { + "@@//bazel/zig_index.json": "0b2647eea9809f05c33dac461c6572e85e0401221bca32ebb6353100132f29c5", + "@@rules_zig+//zig/private/versions.json": "6eb85ebaee72c4e6fbc82d8ad73f69e0899d839b70a7e19caa792ab6a5fa0c36" + }, + "recordedDirentsInputs": {}, + "envVariables": {}, "generatedRepoSpecs": { - "zig_0.17.0_aarch64-linux": { + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -957,30 +1051,17 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/", - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds" + "https://ziglang.org/builds/" ], - "sha256": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", - "zig_version": "0.17.0", + "sha256": "d4219908cded6efb97a7326e4159f82607e3956d6d56efb1580187bf0872603e", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "aarch64-linux" } }, - "zig_0.17.0_aarch64-macos": { + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", + "url": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2611+f996d2866.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -995,30 +1076,17 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/", - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds" + "https://ziglang.org/builds/" ], - "sha256": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", - "zig_version": "0.17.0", + "sha256": "1eb526bb948c1c1bffdb5be318c044fc0216457be28b0d92aa2d92294b9d32c9", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "aarch64-macos" } }, - "zig_0.17.0_aarch64-windows": { + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", + "url": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2611+f996d2866.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1033,30 +1101,17 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/", - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds" + "https://ziglang.org/builds/" ], - "sha256": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", - "zig_version": "0.17.0", + "sha256": "42a612662453e47ad4001eb19c1d219dec6f254913988e2ebe6c95e4b5da3e82", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "aarch64-windows" } }, - "zig_0.17.0_x86_64-linux": { + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2611+f996d2866.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1071,30 +1126,17 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/", - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds" + "https://ziglang.org/builds/" ], - "sha256": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", - "zig_version": "0.17.0", + "sha256": "0188a70a3f84fe1918ac391a8b64bd241eb4c028c44ea023e853743cbecd3009", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "x86_64-linux" } }, - "zig_0.17.0_x86_64-macos": { + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", + "url": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2611+f996d2866.tar.xz", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1109,30 +1151,17 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/", - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds" + "https://ziglang.org/builds/" ], - "sha256": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", - "zig_version": "0.17.0", + "sha256": "5ba0c08bd2e9c74d18c6ea25483226a8f8922f4e50d09d9219be842894b41334", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "x86_64-macos" } }, - "zig_0.17.0_x86_64-windows": { + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows": { "repoRuleId": "@@rules_zig+//zig/private/repo:zig_repository.bzl%zig_repository", "attributes": { - "url": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", + "url": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2611+f996d2866.zip", "mirrors": [ "https://pkg.machengine.org/zig", "https://zigmirror.hryx.net/zig", @@ -1147,23 +1176,10 @@ "https://zig.karearl.com/zig", "https://pkg.earth/zig", "https://fs.liujiacai.net/zigbuilds", - "https://ziglang.org/builds/", - "https://pkg.machengine.org/zig", - "https://zigmirror.hryx.net/zig", - "https://zig.linus.dev/zig", - "https://zig.squirl.dev", - "https://zig.florent.dev", - "https://zig.mirror.mschae23.de/zig", - "https://zigmirror.meox.dev", - "https://ziglang.freetls.fastly.net", - "https://zig.tilok.dev", - "https://zig-mirror.tsimnet.eu/zig", - "https://zig.karearl.com/zig", - "https://pkg.earth/zig", - "https://fs.liujiacai.net/zigbuilds" + "https://ziglang.org/builds/" ], - "sha256": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", - "zig_version": "0.17.0", + "sha256": "3ec5cac4b74b5fe6e04f092ffb6e025993538ecf07970a428919d638efafa542", + "zig_version": "0.16.0-dev.2611+f996d2866", "platform": "x86_64-windows" } }, @@ -1171,108 +1187,75 @@ "repoRuleId": "@@rules_zig+//zig/private/repo:toolchains_repo.bzl%toolchains_repo", "attributes": { "names": [ - "zig_0.17.0_aarch64-linux", - "zig_0.17.0_aarch64-macos", - "zig_0.17.0_aarch64-windows", - "zig_0.17.0_x86_64-linux", - "zig_0.17.0_x86_64-macos", - "zig_0.17.0_x86_64-windows" + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux", + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos", + "zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows", + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux", + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos", + "zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows" ], "labels": [ - "@zig_0.17.0_aarch64-linux//:zig_toolchain", - "@zig_0.17.0_aarch64-macos//:zig_toolchain", - "@zig_0.17.0_aarch64-windows//:zig_toolchain", - "@zig_0.17.0_x86_64-linux//:zig_toolchain", - "@zig_0.17.0_x86_64-macos//:zig_toolchain", - "@zig_0.17.0_x86_64-windows//:zig_toolchain" + "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-linux//:zig_toolchain", + "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-macos//:zig_toolchain", + "@zig_0.16.0-dev.2611_Pf996d2866_aarch64-windows//:zig_toolchain", + "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-linux//:zig_toolchain", + "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-macos//:zig_toolchain", + "@zig_0.16.0-dev.2611_Pf996d2866_x86_64-windows//:zig_toolchain" ], "zig_versions": [ - "0.17.0", - "0.17.0", - "0.17.0", - "0.17.0", - "0.17.0", - "0.17.0" + "0.16.0-dev.2611+f996d2866", + "0.16.0-dev.2611+f996d2866", + "0.16.0-dev.2611+f996d2866", + "0.16.0-dev.2611+f996d2866", + "0.16.0-dev.2611+f996d2866", + "0.16.0-dev.2611+f996d2866" ], "exec_lengths": [ - 3, - 3, - 3, - 3, - 3, - 3 + 2, + 2, + 2, + 2, + 2, + 2 ], "exec_constraints": [ "@platforms//os:linux", "@platforms//cpu:aarch64", - "'@@//platforms:local_execution'", "@platforms//os:macos", "@platforms//cpu:aarch64", - "'@@//platforms:local_execution'", "@platforms//os:windows", "@platforms//cpu:aarch64", - "'@@//platforms:local_execution'", "@platforms//os:linux", "@platforms//cpu:x86_64", - "'@@//platforms:local_execution'", "@platforms//os:macos", "@platforms//cpu:x86_64", - "'@@//platforms:local_execution'", "@platforms//os:windows", - "@platforms//cpu:x86_64", - "'@@//platforms:local_execution'" - ], - "target_compatible_lengths": [ - 0, - 0, - 0, - 0, - 0, - 0 - ], - "target_compatible_constraints": [], - "target_settings_lengths": [ - 0, - 0, - 0, - 0, - 0, - 0 - ], - "target_settings": [] + "@platforms//cpu:x86_64" + ] } } - } - } - }, - "@@rules_zig+//zig/zls:extensions.bzl%zls": { - "general": { - "bzlTransitiveDigest": "8UokkNRK4sShFNMJ3aDabiCS4Q/ldgChV/NYNszfmCg=", - "usagesDigest": "kOrP52crJGijpHNzo4S7msXvbGpfmzvkE4tuhpRWTsI=", - "recordedInputs": [ - "REPO_MAPPING:rules_zig+,bazel_skylib bazel_skylib+", - "REPO_MAPPING:rules_zig+,bazel_tools bazel_tools", - "FILE:@@rules_zig+//zig/zls/private/versions.json 127803289dfd79b5255db5f74be1d9a2f790f03292d6147009af702d888aba69" - ], - "generatedRepoSpecs": { - "zls_toolchains": { - "repoRuleId": "@@rules_zig+//zig/zls/private/repo:toolchains_repo.bzl%toolchains_repo", - "attributes": { - "names": [], - "labels": [], - "zig_versions": [], - "exec_lengths": [], - "exec_constraints": [] - } - } - } + }, + "recordedRepoMappingEntries": [ + [ + "rules_zig+", + "bazel_skylib", + "bazel_skylib+" + ], + [ + "rules_zig+", + "bazel_tools", + "bazel_tools" + ] + ] } }, "@@yq.bzl+//yq:extensions.bzl%yq": { "general": { "bzlTransitiveDigest": "iWlctm1cJYZLESuQEBygCQc6adq+oIRMb4Dmk0rYxkA=", - "usagesDigest": "1CP5kLHwnlFZ3obmnV0eEOp+80JltkcSimeCHXe8/+E=", - "recordedInputs": [], + "usagesDigest": "J3TR36exv3ePI9O2y+uqrzTDmT6s6ADkTsROB++nm6I=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, "generatedRepoSpecs": { "yq_darwin_amd64": { "repoRuleId": "@@yq.bzl+//yq/toolchain:platforms.bzl%yq_platform_repo", @@ -1336,815 +1319,91 @@ "user_repository_name": "yq" } } - } + }, + "recordedRepoMappingEntries": [] } } }, "facts": { - "@@rules_go+//go:extensions.bzl%go_sdk": { - "1.21.1": { - "aix_ppc64": [ - "go1.21.1.aix-ppc64.tar.gz", - "ad5c160e2ff7989f818572f2391ca000d587569ff68d8591a695382a863719d0" - ], - "darwin_amd64": [ - "go1.21.1.darwin-amd64.tar.gz", - "809f5b0ef4f7dcdd5f51e9630a5b2e5a1006f22a047126d61560cdc365678a19" - ], - "darwin_arm64": [ - "go1.21.1.darwin-arm64.tar.gz", - "ffd40391a1e995855488b008ad9326ff8c2e81803a6e80894401003bae47fcf1" - ], - "dragonfly_amd64": [ - "go1.21.1.dragonfly-amd64.tar.gz", - "646f5db8ba479032aea07c597ccd187bc75c161f18e7a1785c62ab8cb103cb3e" - ], - "freebsd_386": [ - "go1.21.1.freebsd-386.tar.gz", - "9919a9a4dc82371aba3da5b7c830bcb6249fc1502cd26d959eb340a60e41ee01" - ], - "freebsd_amd64": [ - "go1.21.1.freebsd-amd64.tar.gz", - "2571f10f6047e04d87c1f5986a05e5e8f7b511faf98803ef12b66d563845d2a1" - ], - "freebsd_arm64": [ - "go1.21.1.freebsd-arm64.tar.gz", - "4946118e72deccf194a9bfcfedbafe52e45b5fe9441fd3acfced760b27a20f9a" - ], - "freebsd_armv6l": [ - "go1.21.1.freebsd-arm.tar.gz", - "6d4dab314ed6adff68a74e3bce516e36fd89c7bf4523c2c105f96e04933fc982" - ], - "freebsd_riscv64": [ - "go1.21.1.freebsd-riscv64.tar.gz", - "591bd28e2dac581facb20c1529f8c29590d7eaaccbcdc93d12a234461b7bfe89" - ], - "illumos_amd64": [ - "go1.21.1.illumos-amd64.tar.gz", - "829548b591675df5f113317ac011bfc3381da7a96bd6a1c748694370818b4497" - ], - "linux_386": [ - "go1.21.1.linux-386.tar.gz", - "b93850666cdadbd696a986cf7b03111fe99db8c34a9aaa113d7c96d0081e1901" - ], - "linux_amd64": [ - "go1.21.1.linux-amd64.tar.gz", - "b3075ae1ce5dab85f89bc7905d1632de23ca196bd8336afd93fa97434cfa55ae" - ], - "linux_arm64": [ - "go1.21.1.linux-arm64.tar.gz", - "7da1a3936a928fd0b2602ed4f3ef535b8cd1990f1503b8d3e1acc0fa0759c967" - ], - "linux_armv6l": [ - "go1.21.1.linux-armv6l.tar.gz", - "f3716a43f59ae69999841d6007b42c9e286e8d8ce470656fb3e70d7be2d7ca85" - ], - "linux_loong64": [ - "go1.21.1.linux-loong64.tar.gz", - "2273cc4ed398d8b341aee01fd62f3845045b2f439b84b28720a7f3d3552f5d41" - ], - "linux_mips": [ - "go1.21.1.linux-mips.tar.gz", - "2ba529210bae7ba7b8c48b0084f713b41e8c109ae75637820fd5b72be9f3faff" - ], - "linux_mips64": [ - "go1.21.1.linux-mips64.tar.gz", - "f2cef18856b17cc5c81cc8b0a35b1ca08d2b04d4b88adc07d190560cb464e2f6" - ], - "linux_mips64le": [ - "go1.21.1.linux-mips64le.tar.gz", - "3aa007a41f533b50eae2491bbd29926ada09357367a8aa05e7e50ec50c78acf9" - ], - "linux_mipsle": [ - "go1.21.1.linux-mipsle.tar.gz", - "c0059f49c3ea31bebe972f47d460d49addb78641081a7000e79e730464bcce95" - ], - "linux_ppc64": [ - "go1.21.1.linux-ppc64.tar.gz", - "6871fd3e6dae3897112dc4a98988406461ef1018f4a17c87689d977eba8dbf20" - ], - "linux_ppc64le": [ - "go1.21.1.linux-ppc64le.tar.gz", - "eddf018206f8a5589bda75252b72716d26611efebabdca5d0083ec15e9e41ab7" - ], - "linux_riscv64": [ - "go1.21.1.linux-riscv64.tar.gz", - "fac64ed26e003f49f1d77f6d2c4cf951422aecbce12232d9ec1bf4585fc44ee1" - ], - "linux_s390x": [ - "go1.21.1.linux-s390x.tar.gz", - "a83b3e8eb4dbf76294e773055eb51397510ff4d612a247bad9903560267bba6d" - ], - "netbsd_386": [ - "go1.21.1.netbsd-386.tar.gz", - "c919f2ba4755e6aebeea6e7f19abf1c5ed13d37c2930867d10a86adfd0686eba" - ], - "netbsd_amd64": [ - "go1.21.1.netbsd-amd64.tar.gz", - "b2dd9913063caa2b5f960e8ebc6c225293b4155e9ebd88d43afd55bb81ca8931" - ], - "netbsd_arm64": [ - "go1.21.1.netbsd-arm64.tar.gz", - "5d245ac932d421389c1bfcedc38733367ae407dbdc162a6e7bfca5677fb15269" - ], - "netbsd_armv6l": [ - "go1.21.1.netbsd-arm.tar.gz", - "4d1756a9845917b07c8ec4999b8e019a2bd2a8b3d5d16da5da0ef3ce84d2139c" - ], - "openbsd_386": [ - "go1.21.1.openbsd-386.tar.gz", - "ce094e551227522aaf3111a94a7f115c5a2b4f862c42d349026503544a40fb96" - ], - "openbsd_amd64": [ - "go1.21.1.openbsd-amd64.tar.gz", - "321c31c42131db2bd0e9d67dd98450f55b281e8e6d8eb681594c39e6688ccbb2" - ], - "openbsd_arm64": [ - "go1.21.1.openbsd-arm64.tar.gz", - "4d65bcb383895be8d6c05507f2b410e15448b0610e1f5fb23ece9ff4e045f7a8" - ], - "openbsd_armv6l": [ - "go1.21.1.openbsd-arm.tar.gz", - "481254b99b9861b65c9eaab137af15311fca618f9f17f2bd872795368c726666" - ], - "plan9_386": [ - "go1.21.1.plan9-386.tar.gz", - "93ea25cbb8063cd4d5917b8ca58a72c26b6cb9e6c1d785e30867ae8ad8290484" - ], - "plan9_amd64": [ - "go1.21.1.plan9-amd64.tar.gz", - "14d61633059651d12cc02b2f53739016f1b07063a4140147e44a196413950553" - ], - "plan9_armv6l": [ - "go1.21.1.plan9-arm.tar.gz", - "adfacbcfb5e7415802187fb6aa8b5397d589fdedcd5d0eac3b86352da7390603" - ], - "solaris_amd64": [ - "go1.21.1.solaris-amd64.tar.gz", - "b86a223e52e41de3ff180f90f1d7e705d30f96271f9a81005e20f25be2658990" - ], - "windows_386": [ - "go1.21.1.windows-386.zip", - "170256c820f466f29d64876f25f4dfa4029ed9902a0a9095d8bd603aecf4d83b" - ], - "windows_amd64": [ - "go1.21.1.windows-amd64.zip", - "10a4f5b63215d11d1770453733dbcbf024f3f74872f84e28d7ea59f0250316c6" - ], - "windows_arm64": [ - "go1.21.1.windows-arm64.zip", - "41135ce6e0ced4bc1e459cb96bd4090c9dc2062e24179c3f337d855af9b560ef" - ], - "windows_armv6l": [ - "go1.21.1.windows-arm.zip", - "d7a2472999a80dbf6bbcb6783a2b4299916055473263117295022f18e7fad487" - ] - }, - "1.22.0": { - "aix_ppc64": [ - "go1.22.0.aix-ppc64.tar.gz", - "190e105fc4133a8b5bb1492f368fa89aa4b729270441120714be7ee82e871ebc" - ], - "darwin_amd64": [ - "go1.22.0.darwin-amd64.tar.gz", - "ebca81df938d2d1047cc992be6c6c759543cf309d401b86af38a6aed3d4090f4" - ], - "darwin_arm64": [ - "go1.22.0.darwin-arm64.tar.gz", - "bf8e388b09134164717cd52d3285a4ab3b68691b80515212da0e9f56f518fb1e" - ], - "dragonfly_amd64": [ - "go1.22.0.dragonfly-amd64.tar.gz", - "357ab446200effa26c73ccaf3e8551426428950bf2490859fb296a09e53228b1" - ], - "freebsd_386": [ - "go1.22.0.freebsd-386.tar.gz", - "b8065da37783e8b9e7086365a54d74537e832c92311b61101a66989ab2458d8e" - ], - "freebsd_amd64": [ - "go1.22.0.freebsd-amd64.tar.gz", - "50f421c7f217083ac94aab1e09400cb9c2fea7d337679ec11f1638a11460da30" - ], - "freebsd_arm64": [ - "go1.22.0.freebsd-arm64.tar.gz", - "e23385e5c640787fa02cd58f2301ea09e162c4d99f8ca9fa6d52766f428a933d" - ], - "freebsd_armv6l": [ - "go1.22.0.freebsd-arm.tar.gz", - "c9c8b305f90903536f4981bad9f029828c2483b3216ca1783777344fbe603f2d" - ], - "freebsd_riscv64": [ - "go1.22.0.freebsd-riscv64.tar.gz", - "c8f94d1de6024546194d58e7b9370dc7ea06176aad94a675b0062c25c40cb645" - ], - "illumos_amd64": [ - "go1.22.0.illumos-amd64.tar.gz", - "d6792f11ad6ee5fc42d2fe51e1f1683471aa2ee4f20e3ad1be22a4afdbd38e7f" - ], - "linux_386": [ - "go1.22.0.linux-386.tar.gz", - "1e209c4abde069067ac9afb341c8003db6a210f8173c77777f02d3a524313da3" - ], - "linux_amd64": [ - "go1.22.0.linux-amd64.tar.gz", - "f6c8a87aa03b92c4b0bf3d558e28ea03006eb29db78917daec5cfb6ec1046265" - ], - "linux_arm64": [ - "go1.22.0.linux-arm64.tar.gz", - "6a63fef0e050146f275bf02a0896badfe77c11b6f05499bb647e7bd613a45a10" - ], - "linux_armv6l": [ - "go1.22.0.linux-armv6l.tar.gz", - "0525f92f79df7ed5877147bce7b955f159f3962711b69faac66bc7121d36dcc4" - ], - "linux_loong64": [ - "go1.22.0.linux-loong64.tar.gz", - "b4b1d425cc113608452a32768469b6e34e538fd072bde9f508a75c8dbbdb843f" - ], - "linux_mips": [ - "go1.22.0.linux-mips.tar.gz", - "ec0f9194df750c78492c02e4c70ffe6f3864f2511e47c894415320db752719f8" - ], - "linux_mips64": [ - "go1.22.0.linux-mips64.tar.gz", - "47e938d215e4968ab42afb7307792e5e64184a717e8f176d0de7c411af96d63d" - ], - "linux_mips64le": [ - "go1.22.0.linux-mips64le.tar.gz", - "c162a1a516b8bc8839fc0c0283ad90f6b511e5aca3da4939ed4800f124b9e72c" - ], - "linux_mipsle": [ - "go1.22.0.linux-mipsle.tar.gz", - "6ce3e9a06e3a8ca0312dc1f85046b6914c19852eba5393c5cbf26acf698f8977" - ], - "linux_ppc64": [ - "go1.22.0.linux-ppc64.tar.gz", - "5ae23bf460731eb078f5700b01a50a303308b9f7174a2994892e28bf061f7c85" - ], - "linux_ppc64le": [ - "go1.22.0.linux-ppc64le.tar.gz", - "0e57f421df9449066f00155ce98a5be93744b3d81b00ee4c2c9b511be2a31d93" - ], - "linux_riscv64": [ - "go1.22.0.linux-riscv64.tar.gz", - "afe9cedcdbd6fdff27c57efd30aa5ce0f666f471fed5fa96cd4fb38d6b577086" - ], - "linux_s390x": [ - "go1.22.0.linux-s390x.tar.gz", - "2e546a3583ba7bd3988f8f476245698f6a93dfa9fe206a8ca8f85c1ceecb2446" - ], - "netbsd_386": [ - "go1.22.0.netbsd-386.tar.gz", - "9b7e2dbd43a169bff18bf533a6c2f462eebe04126ab01c138d0d669c046e1658" - ], - "netbsd_amd64": [ - "go1.22.0.netbsd-amd64.tar.gz", - "b11995c271d2256dfe85cf54882ca3655e18c49c4d7db0502bff9977767894e7" - ], - "netbsd_arm64": [ - "go1.22.0.netbsd-arm64.tar.gz", - "499918ccfedde78264d194244d251bf41d95cf606cc0adad13b36b8103cb974f" - ], - "netbsd_armv6l": [ - "go1.22.0.netbsd-arm.tar.gz", - "b57a3aa9c862300ec0ee8609ce5e0f430f132044f351677fd34711a504081872" - ], - "openbsd_386": [ - "go1.22.0.openbsd-386.tar.gz", - "379e0829104c18a139d78b33378e6bd3ab2e0792f6c43b4c38e7f20d8d77b99d" - ], - "openbsd_amd64": [ - "go1.22.0.openbsd-amd64.tar.gz", - "ceb0c97ffc3bfaf74e1df843cb8571d7fc3173a08432f0f42112495df6a31520" - ], - "openbsd_arm64": [ - "go1.22.0.openbsd-arm64.tar.gz", - "358801cab7122ab50d7d92727644f26d818e9e973403f09e85c2e935a625db75" - ], - "openbsd_armv6l": [ - "go1.22.0.openbsd-arm.tar.gz", - "8af5aea3df539bc95ed412c0a176fe84baced70ea1dd29f4aa82d0e9ce27fd9f" - ], - "plan9_386": [ - "go1.22.0.plan9-386.tar.gz", - "fa42c545c9025c45ca9af176dc13a0f4af0cc26bacff6fcb35bb4a170ac538e8" - ], - "plan9_amd64": [ - "go1.22.0.plan9-amd64.tar.gz", - "d8cf64f37a1dfd8e190c5a303c43ab2d49324868f098d88a3106072d137a5a0b" - ], - "plan9_armv6l": [ - "go1.22.0.plan9-arm.tar.gz", - "86fd6165f0cbb47ad551094f74b3e5a6c5e09de858d8b99de72d978d41be6e2a" - ], - "solaris_amd64": [ - "go1.22.0.solaris-amd64.tar.gz", - "a6c12651768d3a74f16104502b4b7bef513ea6b646d99990a28d934c261d1689" - ], - "windows_386": [ - "go1.22.0.windows-386.zip", - "553d44928509965cbda02a45b35ab01cf8b925534bc526a34e2d9dc7794b57e8" - ], - "windows_amd64": [ - "go1.22.0.windows-amd64.zip", - "78b3158fe3aa358e0b6c9f26ecd338f9a11441e88bc434ae2e9f0ca2b0cc4dd3" - ], - "windows_arm64": [ - "go1.22.0.windows-arm64.zip", - "31a61e41d06a3bb2189a303f5f3e777ca4b454eff439f0a67bc2b166330021f4" - ], - "windows_armv6l": [ - "go1.22.0.windows-arm.zip", - "495c7dfaea4e2bf48643662bb622e4ce6378d6d9840015238ad4b8792b99ddbf" - ] - }, - "1.22.4": { - "aix_ppc64": [ - "go1.22.4.aix-ppc64.tar.gz", - "b9647fa9fc83a0cc5d4f092a19eaeaecf45f063a5aa7d4962fde65aeb7ae6ce1" - ], - "darwin_amd64": [ - "go1.22.4.darwin-amd64.tar.gz", - "c95967f50aa4ace34af0c236cbdb49a9a3e80ee2ad09d85775cb4462a5c19ed3" - ], - "darwin_arm64": [ - "go1.22.4.darwin-arm64.tar.gz", - "242b78dc4c8f3d5435d28a0d2cec9b4c1aa999b601fb8aa59fb4e5a1364bf827" - ], - "dragonfly_amd64": [ - "go1.22.4.dragonfly-amd64.tar.gz", - "f2fbb51af4719d3616efb482d6ed2b96579b474156f85a7ddc6f126764feec4b" - ], - "freebsd_386": [ - "go1.22.4.freebsd-386.tar.gz", - "7c54884bb9f274884651d41e61d1bc12738863ad1497e97ea19ad0e9aa6bf7b5" - ], - "freebsd_amd64": [ - "go1.22.4.freebsd-amd64.tar.gz", - "88d44500e1701dd35797619774d6dd51bf60f45a8338b0a82ddc018e4e63fb78" - ], - "freebsd_arm64": [ - "go1.22.4.freebsd-arm64.tar.gz", - "726dc093cf020277be45debf03c3b02b43c2efb3e2a5d4fba8f52579d65327dc" - ], - "freebsd_armv6l": [ - "go1.22.4.freebsd-arm.tar.gz", - "3d9efe47db142a22679aba46b1772e3900b0d87ae13bd2b3bc80dbf2ac0b2cd6" - ], - "freebsd_riscv64": [ - "go1.22.4.freebsd-riscv64.tar.gz", - "5f6b67e5e32f1d6ccb2d4dcb44934a5e2e870a877ba7443d86ec43cfc28afa71" - ], - "illumos_amd64": [ - "go1.22.4.illumos-amd64.tar.gz", - "d56ecc2f85b6418a21ef83879594d0c42ab4f65391a676bb12254870e6690d63" - ], - "linux_386": [ - "go1.22.4.linux-386.tar.gz", - "47a2a8d249a91eb8605c33bceec63aedda0441a43eac47b4721e3975ff916cec" - ], - "linux_amd64": [ - "go1.22.4.linux-amd64.tar.gz", - "ba79d4526102575196273416239cca418a651e049c2b099f3159db85e7bade7d" - ], - "linux_arm64": [ - "go1.22.4.linux-arm64.tar.gz", - "a8e177c354d2e4a1b61020aca3562e27ea3e8f8247eca3170e3fa1e0c2f9e771" - ], - "linux_armv6l": [ - "go1.22.4.linux-armv6l.tar.gz", - "e2b143fbacbc9cbd448e9ef41ac3981f0488ce849af1cf37e2341d09670661de" - ], - "linux_loong64": [ - "go1.22.4.linux-loong64.tar.gz", - "e2ff9436e4b34bf6926b06d97916e26d67a909a2effec17967245900f0816f1d" - ], - "linux_mips": [ - "go1.22.4.linux-mips.tar.gz", - "73f0dcc60458c4770593b05a7bc01cc0d31fc98f948c0c2334812c7a1f2fc3f1" - ], - "linux_mips64": [ - "go1.22.4.linux-mips64.tar.gz", - "417af97fc2630a647052375768be4c38adcc5af946352ea5b28613ea81ca5d45" - ], - "linux_mips64le": [ - "go1.22.4.linux-mips64le.tar.gz", - "7486e2d7dd8c98eb44df815ace35a7fe7f30b7c02326e3741bd934077508139b" - ], - "linux_mipsle": [ - "go1.22.4.linux-mipsle.tar.gz", - "69479c8aad301e459a8365b40cad1074a0dbba5defb9291669f94809c4c4be6e" - ], - "linux_ppc64": [ - "go1.22.4.linux-ppc64.tar.gz", - "dd238847e65bc3e2745caca475a5db6522a2fcf85cf6c38fc36a06642b19efd7" - ], - "linux_ppc64le": [ - "go1.22.4.linux-ppc64le.tar.gz", - "a3e5834657ef92523f570f798fed42f1f87bc18222a16815ec76b84169649ec4" - ], - "linux_riscv64": [ - "go1.22.4.linux-riscv64.tar.gz", - "56a827ff7dc6245bcd7a1e9288dffaa1d8b0fd7468562264c1523daf3b4f1b4a" - ], - "linux_s390x": [ - "go1.22.4.linux-s390x.tar.gz", - "7590c3e278e2dc6040aae0a39da3ca1eb2e3921673a7304cc34d588c45889eec" - ], - "netbsd_386": [ - "go1.22.4.netbsd-386.tar.gz", - "ddd2eebe34471a2502de6c5dad04ab27c9fc80cbde7a9ad5b3c66ecec4504e1d" - ], - "netbsd_amd64": [ - "go1.22.4.netbsd-amd64.tar.gz", - "33af79f6f935f6fbacc5d23876450b3567b79348fc065beef8e64081127dd234" - ], - "netbsd_arm64": [ - "go1.22.4.netbsd-arm64.tar.gz", - "c9a2971dec9f6d320c6f2b049b2353c6d0a2d35e87b8a4b2d78a2f0d62545f8e" - ], - "netbsd_armv6l": [ - "go1.22.4.netbsd-arm.tar.gz", - "fa3550ebd5375a70b3bcd342b5a71f4bd271dcbbfaf4eabefa2144ab5d8924b6" - ], - "openbsd_386": [ - "go1.22.4.openbsd-386.tar.gz", - "d21af022331bfdc2b5b161d616c3a1a4573d33cf7a30416ee509a8f3641deb47" - ], - "openbsd_amd64": [ - "go1.22.4.openbsd-amd64.tar.gz", - "72c0094c43f7e5722ec49c2a3e9dfa7a1123ac43a5f3a63eecf3e3795d3ff0ae" - ], - "openbsd_arm64": [ - "go1.22.4.openbsd-arm64.tar.gz", - "a7ab8d4e0b02bf06ed144ba42c61c0e93ee00f2b433415dfd4ad4b6e79f31650" - ], - "openbsd_armv6l": [ - "go1.22.4.openbsd-arm.tar.gz", - "1096831ea3c5ea3ca57d14251d9eda3786889531eb40d7d6775dcaa324d4b065" - ], - "openbsd_ppc64": [ - "go1.22.4.openbsd-ppc64.tar.gz", - "9716327c8a628358798898dc5148c49dbbeb5196bf2cbf088e550721a6e4f60b" - ], - "plan9_386": [ - "go1.22.4.plan9-386.tar.gz", - "a8dd4503c95c32a502a616ab78870a19889c9325fe9bd31eb16dd69346e4bfa8" - ], - "plan9_amd64": [ - "go1.22.4.plan9-amd64.tar.gz", - "5423a25808d76fe5aca8607a2e5ac5673abf45446b168cb5e9d8519ee9fe39a1" - ], - "plan9_armv6l": [ - "go1.22.4.plan9-arm.tar.gz", - "6af939ad583f5c85c09c53728ab7d38c3cc2b39167562d6c18a07c5c6608b370" - ], - "solaris_amd64": [ - "go1.22.4.solaris-amd64.tar.gz", - "e8cabe69c03085725afdb32a6f9998191a3e55a747b270d835fd05000d56abba" - ], - "windows_386": [ - "go1.22.4.windows-386.zip", - "aca4e2c37278a10f1c70dd0df142f7d66b50334fcee48978d409202d308d6d25" - ], - "windows_amd64": [ - "go1.22.4.windows-amd64.zip", - "26321c4d945a0035d8a5bc4a1965b0df401ff8ceac66ce2daadabf9030419a98" - ], - "windows_arm64": [ - "go1.22.4.windows-arm64.zip", - "8a2daa9ea28cbdafddc6171aefed384f4e5b6e714fb52116fe9ed25a132f37ed" - ], - "windows_armv6l": [ - "go1.22.4.windows-arm.zip", - "5fcd0671a49cecf39b41021621ee1b6e7aa1370f37122b72e80d4fd4185833b6" - ] - }, - "1.24.6": { - "aix_ppc64": [ - "go1.24.6.aix-ppc64.tar.gz", - "5cafb2927af95969fc3d0884be1ad4df54f19103d86dda928d9f76494a5ae6a6" - ], - "darwin_amd64": [ - "go1.24.6.darwin-amd64.tar.gz", - "4a8d7a32052f223e71faab424a69430455b27b3fff5f4e651f9d97c3e51a8746" - ], - "darwin_arm64": [ - "go1.24.6.darwin-arm64.tar.gz", - "4e29202c49573b953be7cc3500e1f8d9e66ddd12faa8cf0939a4951411e09a2a" - ], - "dragonfly_amd64": [ - "go1.24.6.dragonfly-amd64.tar.gz", - "c6e21bf8347f7a1c653dd4136c8c27a858515e98501d2843023a4bb3f1f7fb63" - ], - "freebsd_386": [ - "go1.24.6.freebsd-386.tar.gz", - "9cd74ad74f3ad833e92529f2fd9b0d7d9ffaab46307eccadb0afcf9a1ba09553" - ], - "freebsd_amd64": [ - "go1.24.6.freebsd-amd64.tar.gz", - "4983e2b10ae1f754e4eb07e1e589691c7e1d0dc428a92c16bd0e2ba03cc23ed9" - ], - "freebsd_arm": [ - "go1.24.6.freebsd-arm.tar.gz", - "a8da621d8282a91ee17b257a46f2606391c019cc1a7d7be628638792ca8033ad" - ], - "freebsd_arm64": [ - "go1.24.6.freebsd-arm64.tar.gz", - "76a75ad5125217c268029c0ad9c7295cc7f6042fe9cba4bebf9a89f7f42ad8af" - ], - "freebsd_riscv64": [ - "go1.24.6.freebsd-riscv64.tar.gz", - "ac206417d8460662f26d46dc2ad0488b2f9e22039946069ba4b48a0cb646e8b0" - ], - "illumos_amd64": [ - "go1.24.6.illumos-amd64.tar.gz", - "c54199c46ec823c857c52335f859b493593433d71fe479b9e7f95d868a144ae2" - ], - "linux_386": [ - "go1.24.6.linux-386.tar.gz", - "bb5bf69d75e7edbc93339824753a1a4655a928451a2c5e13ff90959ad69e065b" - ], - "linux_amd64": [ - "go1.24.6.linux-amd64.tar.gz", - "bbca37cc395c974ffa4893ee35819ad23ebb27426df87af92e93a9ec66ef8712" - ], - "linux_arm64": [ - "go1.24.6.linux-arm64.tar.gz", - "124ea6033a8bf98aa9fbab53e58d134905262d45a022af3a90b73320f3c3afd5" - ], - "linux_armv6l": [ - "go1.24.6.linux-armv6l.tar.gz", - "7feb4d25f5e72f94fda81c99d4adb6630dfa2c35211e0819417d53af6e71809e" - ], - "linux_loong64": [ - "go1.24.6.linux-loong64.tar.gz", - "8424d9c52b254255d2720769366af73802747e4eaf339c6e86bb52e5a1809fcb" - ], - "linux_mips": [ - "go1.24.6.linux-mips.tar.gz", - "791964a6ce65f604a2099b7e61f71128cc7a3a4fcee858218d2ce99033fc2b09" - ], - "linux_mips64": [ - "go1.24.6.linux-mips64.tar.gz", - "2f5282fdf2eaca44515728d518caec888b0dc4948effecb8e7f2ddff1ff9aa9e" - ], - "linux_mips64le": [ - "go1.24.6.linux-mips64le.tar.gz", - "64ddc6b28907b4ab997f2d81c226be403511a59a6100536560af26ccfb10b4a4" - ], - "linux_mipsle": [ - "go1.24.6.linux-mipsle.tar.gz", - "5ca12d2d4939dea977100e76885f4e08cd3ef3b73823853527b58ecac4598fd3" - ], - "linux_ppc64": [ - "go1.24.6.linux-ppc64.tar.gz", - "9a4a61ffbba7a58b3ef97a6d3b2b394168f613f9ca6092016981a88c78a57f86" - ], - "linux_ppc64le": [ - "go1.24.6.linux-ppc64le.tar.gz", - "63fc9559a3d6dfd63aa902f714375b879bbc848466181c035c122489b9646e27" - ], - "linux_riscv64": [ - "go1.24.6.linux-riscv64.tar.gz", - "e92c19ff15a9004fe43e1f62d433555250ef9fe3fabee3dc29ebd4802c2b8021" - ], - "linux_s390x": [ - "go1.24.6.linux-s390x.tar.gz", - "4cde28d9ffb6eef86bf8dac6852a45db335009f67e60bee3e477dd0ba0ff9704" - ], - "netbsd_386": [ - "go1.24.6.netbsd-386.tar.gz", - "d238045676104d2aaad9da53c4ba3f489cf81b26d3315d2caad2b6e240664423" - ], - "netbsd_amd64": [ - "go1.24.6.netbsd-amd64.tar.gz", - "9f8a9245cf146f1ebdac6785f71646c3dd0edf07dcdb8e5a8df85ec6447e5308" - ], - "netbsd_arm": [ - "go1.24.6.netbsd-arm.tar.gz", - "2359c8af6b2d6979910060478a57fd216824682e0908ea1a4412ea8dc4a55db7" - ], - "netbsd_arm64": [ - "go1.24.6.netbsd-arm64.tar.gz", - "99101158c796997ec85023d395c133fa90c782bd5faa56f8c3824817d28b0ee8" - ], - "openbsd_386": [ - "go1.24.6.openbsd-386.tar.gz", - "b6193420489f24d5560b87ccb38caf52cecb308d892d99bf21d1ef0d2c99de04" - ], - "openbsd_amd64": [ - "go1.24.6.openbsd-amd64.tar.gz", - "8dc0d94afbc65be9cafebe543b88a841dcf3429e50f668ea714200ef9f4489bd" - ], - "openbsd_arm": [ - "go1.24.6.openbsd-arm.tar.gz", - "c29563d14d8e4b5fa89f130afac40abba7a1ee1e87fcc759a95f1e9b5831bf68" - ], - "openbsd_arm64": [ - "go1.24.6.openbsd-arm64.tar.gz", - "3d97bdc5e1997cccfaa398242c52e8a97d3b626d988a65248bbdbf2f1ab188bf" - ], - "openbsd_ppc64": [ - "go1.24.6.openbsd-ppc64.tar.gz", - "54cf517933db0bff58f3e527e7fa194f505d562c71b1fb0050d3365fcdda4914" - ], - "openbsd_riscv64": [ - "go1.24.6.openbsd-riscv64.tar.gz", - "f7bdd8f882ecbc01ff6c1782ce3e29298445dd99b3dd4299423257df307c7b0d" - ], - "plan9_386": [ - "go1.24.6.plan9-386.tar.gz", - "9bc0a801dc0d8d0705ac9493219b926b2e3c0105b8921c6c2fce69d14c4b9007" - ], - "plan9_amd64": [ - "go1.24.6.plan9-amd64.tar.gz", - "7133c28acc277d3c5e8317fd9454b3a0902138c384466f818644b504b8b89ffe" - ], - "plan9_arm": [ - "go1.24.6.plan9-arm.tar.gz", - "c4e2aa8aa4353f75fc083ca728b73917467cef88c6f5f917ca420578672c6d58" - ], - "solaris_amd64": [ - "go1.24.6.solaris-amd64.tar.gz", - "cfcb4f1f7e987c6dee893c4401546516cc271c1ac7d7fb37a32b25343dc7df32" - ], - "windows_386": [ - "go1.24.6.windows-386.zip", - "39b4d31b933d2f7c8913e70fb0fffef27252e441c386eac5c13632cbb28dfb6e" - ], - "windows_amd64": [ - "go1.24.6.windows-amd64.zip", - "4fbc8af2cfca9e5059019b5150a426eb78e1e57718bf08f0e52b1c942a2782bf" - ], - "windows_arm64": [ - "go1.24.6.windows-arm64.zip", - "45c41b237d00e92e4cf8adce11b4c5258048b47a92bfbb1f4ef3b928d6fcb0b2" - ] - }, - "1.25.0": { - "aix_ppc64": [ - "go1.25.0.aix-ppc64.tar.gz", - "e5234a7dac67bc86c528fe9752fc9d63557918627707a733ab4cac1a6faed2d4" - ], - "darwin_amd64": [ - "go1.25.0.darwin-amd64.tar.gz", - "5bd60e823037062c2307c71e8111809865116714d6f6b410597cf5075dfd80ef" - ], - "darwin_arm64": [ - "go1.25.0.darwin-arm64.tar.gz", - "544932844156d8172f7a28f77f2ac9c15a23046698b6243f633b0a0b00c0749c" - ], - "dragonfly_amd64": [ - "go1.25.0.dragonfly-amd64.tar.gz", - "5ed3cf9a810a1483822538674f1336c06b51aa1b94d6d545a1a0319a48177120" - ], - "freebsd_386": [ - "go1.25.0.freebsd-386.tar.gz", - "abea5d5c6697e6b5c224731f2158fe87c602996a2a233ac0c4730cd57bf8374e" - ], - "freebsd_amd64": [ - "go1.25.0.freebsd-amd64.tar.gz", - "86e6fe0a29698d7601c4442052dac48bd58d532c51cccb8f1917df648138730b" - ], - "freebsd_arm": [ - "go1.25.0.freebsd-arm.tar.gz", - "d90b78e41921f72f30e8bbc81d9dec2cff7ff384a33d8d8debb24053e4336bfe" - ], - "freebsd_arm64": [ - "go1.25.0.freebsd-arm64.tar.gz", - "451d0da1affd886bfb291b7c63a6018527b269505db21ce6e14724f22ab0662e" - ], - "freebsd_riscv64": [ - "go1.25.0.freebsd-riscv64.tar.gz", - "7b565f76bd8bda46549eeaaefe0e53b251e644c230577290c0f66b1ecdb3cdbe" - ], - "illumos_amd64": [ - "go1.25.0.illumos-amd64.tar.gz", - "b1e1fdaab1ad25aa1c08d7a36c97d45d74b98b89c3f78c6d2145f77face54a2c" - ], - "linux_386": [ - "go1.25.0.linux-386.tar.gz", - "8c602dd9d99bc9453b3995d20ce4baf382cc50855900a0ece5de9929df4a993a" - ], - "linux_amd64": [ - "go1.25.0.linux-amd64.tar.gz", - "2852af0cb20a13139b3448992e69b868e50ed0f8a1e5940ee1de9e19a123b613" - ], - "linux_arm64": [ - "go1.25.0.linux-arm64.tar.gz", - "05de75d6994a2783699815ee553bd5a9327d8b79991de36e38b66862782f54ae" - ], - "linux_armv6l": [ - "go1.25.0.linux-armv6l.tar.gz", - "a5a8f8198fcf00e1e485b8ecef9ee020778bf32a408a4e8873371bfce458cd09" - ], - "linux_loong64": [ - "go1.25.0.linux-loong64.tar.gz", - "cab86b1cf761b1cb3bac86a8877cfc92e7b036fc0d3084123d77013d61432afc" - ], - "linux_mips": [ - "go1.25.0.linux-mips.tar.gz", - "d66b6fb74c3d91b9829dc95ec10ca1f047ef5e89332152f92e136cf0e2da5be1" - ], - "linux_mips64": [ - "go1.25.0.linux-mips64.tar.gz", - "4082e4381a8661bc2a839ff94ba3daf4f6cde20f8fb771b5b3d4762dc84198a2" - ], - "linux_mips64le": [ - "go1.25.0.linux-mips64le.tar.gz", - "70002c299ec7f7175ac2ef673b1b347eecfa54ae11f34416a6053c17f855afcc" - ], - "linux_mipsle": [ - "go1.25.0.linux-mipsle.tar.gz", - "b00a3a39eff099f6df9f1c7355bf28e4589d0586f42d7d4a394efb763d145a73" - ], - "linux_ppc64": [ - "go1.25.0.linux-ppc64.tar.gz", - "df166f33bd98160662560a72ff0b4ba731f969a80f088922bddcf566a88c1ec1" - ], - "linux_ppc64le": [ - "go1.25.0.linux-ppc64le.tar.gz", - "0f18a89e7576cf2c5fa0b487a1635d9bcbf843df5f110e9982c64df52a983ad0" - ], - "linux_riscv64": [ - "go1.25.0.linux-riscv64.tar.gz", - "c018ff74a2c48d55c8ca9b07c8e24163558ffec8bea08b326d6336905d956b67" - ], - "linux_s390x": [ - "go1.25.0.linux-s390x.tar.gz", - "34e5a2e19f2292fbaf8783e3a241e6e49689276aef6510a8060ea5ef54eee408" - ], - "netbsd_386": [ - "go1.25.0.netbsd-386.tar.gz", - "f8586cdb7aa855657609a5c5f6dbf523efa00c2bbd7c76d3936bec80aa6c0aba" - ], - "netbsd_amd64": [ - "go1.25.0.netbsd-amd64.tar.gz", - "ae8dc1469385b86a157a423bb56304ba45730de8a897615874f57dd096db2c2a" - ], - "netbsd_arm": [ - "go1.25.0.netbsd-arm.tar.gz", - "1ff7e4cc764425fc9dd6825eaee79d02b3c7cafffbb3691687c8d672ade76cb7" - ], - "netbsd_arm64": [ - "go1.25.0.netbsd-arm64.tar.gz", - "e1b310739f26724216aa6d7d7208c4031f9ff54c9b5b9a796ddc8bebcb4a5f16" - ], - "openbsd_386": [ - "go1.25.0.openbsd-386.tar.gz", - "4802a9b20e533da91adb84aab42e94aa56cfe3e5475d0550bed3385b182e69d8" - ], - "openbsd_amd64": [ - "go1.25.0.openbsd-amd64.tar.gz", - "c016cd984bebe317b19a4f297c4f50def120dc9788490540c89f28e42f1dabe1" - ], - "openbsd_arm": [ - "go1.25.0.openbsd-arm.tar.gz", - "a1e31d0bf22172ddde42edf5ec811ef81be43433df0948ece52fecb247ccfd8d" - ], - "openbsd_arm64": [ - "go1.25.0.openbsd-arm64.tar.gz", - "343ea8edd8c218196e15a859c6072d0dd3246fbbb168481ab665eb4c4140458d" - ], - "openbsd_ppc64": [ - "go1.25.0.openbsd-ppc64.tar.gz", - "694c14da1bcaeb5e3332d49bdc2b6d155067648f8fe1540c5de8f3cf8e157154" - ], - "openbsd_riscv64": [ - "go1.25.0.openbsd-riscv64.tar.gz", - "aa510ad25cf54c06cd9c70b6d80ded69cb20188ac6e1735655eef29ff7e7885f" - ], - "plan9_386": [ - "go1.25.0.plan9-386.tar.gz", - "46f8cef02086cf04bf186c5912776b56535178d4cb319cd19c9fdbdd29231986" - ], - "plan9_amd64": [ - "go1.25.0.plan9-amd64.tar.gz", - "29b34391d84095e44608a228f63f2f88113a37b74a79781353ec043dfbcb427b" - ], - "plan9_arm": [ - "go1.25.0.plan9-arm.tar.gz", - "0a047107d13ebe7943aaa6d54b1d7bbd2e45e68ce449b52915a818da715799c2" - ], - "solaris_amd64": [ - "go1.25.0.solaris-amd64.tar.gz", - "9977f9e4351984364a3b2b78f8b88bfd1d339812356d5237678514594b7d3611" - ], - "windows_386": [ - "go1.25.0.windows-386.zip", - "df9f39db82a803af0db639e3613a36681ab7a42866b1384b3f3a1045663961a7" - ], - "windows_amd64": [ - "go1.25.0.windows-amd64.zip", - "89efb4f9b30812eee083cc1770fdd2913c14d301064f6454851428f9707d190b" - ], - "windows_arm64": [ - "go1.25.0.windows-arm64.zip", - "27bab004c72b3d7bd05a69b6ec0fc54a309b4b78cc569dd963d8b3ec28bfdb8c" - ] - } + "@@rules_rs+//rs:extensions.bzl%crate": { + "ahash_0.8.12": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1.0\"},{\"name\":\"const-random\",\"optional\":true,\"req\":\"^0.1.17\"},{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.3.2\"},{\"kind\":\"dev\",\"name\":\"fnv\",\"req\":\"^1.0.5\"},{\"kind\":\"dev\",\"name\":\"fxhash\",\"req\":\"^0.2.1\"},{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.1\"},{\"kind\":\"dev\",\"name\":\"hashbrown\",\"req\":\"^0.14.3\"},{\"kind\":\"dev\",\"name\":\"hex\",\"req\":\"^0.4.2\"},{\"kind\":\"dev\",\"name\":\"no-panic\",\"req\":\"^0.1.10\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"once_cell\",\"req\":\"^1.18.0\",\"target\":\"cfg(not(all(target_arch = \\\"arm\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"pcg-mwc\",\"req\":\"^0.2.1\"},{\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"seahash\",\"req\":\"^4.0\"},{\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.117\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.59\"},{\"kind\":\"dev\",\"name\":\"smallvec\",\"req\":\"^1.13.1\"},{\"kind\":\"build\",\"name\":\"version_check\",\"req\":\"^0.9.4\"},{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"zerocopy\",\"req\":\"^0.8.24\"}],\"features\":{\"atomic-polyfill\":[\"dep:portable-atomic\",\"once_cell/critical-section\"],\"compile-time-rng\":[\"const-random\"],\"default\":[\"std\",\"runtime-rng\"],\"nightly-arm-aes\":[],\"no-rng\":[],\"runtime-rng\":[\"getrandom\"],\"std\":[]}}", + "aho-corasick_1.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.17\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.4.0\"}],\"features\":{\"default\":[\"std\",\"perf-literal\"],\"logging\":[\"dep:log\"],\"perf-literal\":[\"dep:memchr\"],\"std\":[\"memchr?/std\"]}}", + "base64_0.13.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"=0.3.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.6.1\"},{\"kind\":\"dev\",\"name\":\"structopt\",\"req\":\"^0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[]}}", + "bitflags_2.9.1": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"arbitrary\",\"req\":\"^1.0\"},{\"name\":\"bytemuck\",\"optional\":true,\"req\":\"^1.12\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"bytemuck\",\"req\":\"^1.12.2\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.103\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0.19\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.18\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"zerocopy\",\"req\":\"^0.8\"}],\"features\":{\"example_generated\":[],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[]}}", + "byteorder_1.5.0": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9.2\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"i128\":[],\"std\":[]}}", + "castaway_0.2.4": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1\"},{\"name\":\"rustversion\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"std\":[\"alloc\"]}}", + "cc_1.2.33": "{\"dependencies\":[{\"default_features\":false,\"name\":\"jobserver\",\"optional\":true,\"req\":\"^0.1.30\"},{\"default_features\":false,\"name\":\"libc\",\"optional\":true,\"req\":\"^0.2.62\",\"target\":\"cfg(unix)\"},{\"name\":\"shlex\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3\"}],\"features\":{\"jobserver\":[],\"parallel\":[\"dep:libc\",\"dep:jobserver\"]}}", + "cfg-if_1.0.0": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"]}}", + "compact_str_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"borsh\",\"optional\":true,\"req\":\"^1\"},{\"name\":\"bytes\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"castaway\",\"req\":\"^0.2.3\"},{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"cfg-if\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"diesel\",\"optional\":true,\"req\":\"^2\"},{\"name\":\"itoa\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"markup\",\"optional\":true,\"req\":\"^0.15\"},{\"default_features\":false,\"features\":[\"std\"],\"name\":\"proptest\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"quickcheck\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"quickcheck_macros\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"rkyv\",\"optional\":true,\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rkyv\",\"req\":\"^0.8.8\"},{\"name\":\"rustversion\",\"req\":\"^1\"},{\"name\":\"ryu\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"derive\",\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"features\":[\"union\"],\"name\":\"smallvec\",\"optional\":true,\"req\":\"^1\"},{\"default_features\":false,\"name\":\"sqlx\",\"optional\":true,\"req\":\"^0.8\"},{\"name\":\"static_assertions\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"test-case\",\"req\":\"^3\"},{\"kind\":\"dev\",\"name\":\"test-strategy\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"zeroize\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"borsh\":[\"dep:borsh\"],\"bytes\":[\"dep:bytes\"],\"default\":[\"std\"],\"diesel\":[\"dep:diesel\"],\"markup\":[\"dep:markup\"],\"proptest\":[\"dep:proptest\"],\"quickcheck\":[\"dep:quickcheck\"],\"rkyv\":[\"dep:rkyv\"],\"serde\":[\"dep:serde\"],\"smallvec\":[\"dep:smallvec\"],\"sqlx\":[\"dep:sqlx\",\"std\"],\"sqlx-mysql\":[\"sqlx\",\"sqlx/mysql\"],\"sqlx-postgres\":[\"sqlx\",\"sqlx/postgres\"],\"sqlx-sqlite\":[\"sqlx\",\"sqlx/sqlite\"],\"std\":[],\"zeroize\":[\"dep:zeroize\"]}}", + "crossbeam-deque_0.8.6": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-epoch\",\"req\":\"^0.9.17\"},{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"std\":[\"crossbeam-epoch/std\",\"crossbeam-utils/std\"]}}", + "crossbeam-epoch_0.9.18": "{\"dependencies\":[{\"default_features\":false,\"name\":\"crossbeam-utils\",\"req\":\"^0.8.18\"},{\"name\":\"loom-crate\",\"optional\":true,\"package\":\"loom\",\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"loom\":[\"loom-crate\",\"crossbeam-utils/loom\"],\"nightly\":[\"crossbeam-utils/nightly\"],\"std\":[\"alloc\",\"crossbeam-utils/std\"]}}", + "crossbeam-utils_0.8.21": "{\"dependencies\":[{\"name\":\"loom\",\"optional\":true,\"req\":\"^0.7.1\",\"target\":\"cfg(crossbeam_loom)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"}],\"features\":{\"default\":[\"std\"],\"nightly\":[],\"std\":[]}}", + "darling_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"darling_macro\",\"req\":\"=0.20.10\"},{\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.9\",\"target\":\"cfg(compiletests)\"},{\"kind\":\"dev\",\"name\":\"syn\",\"req\":\"^2.0.15\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"default\":[\"suggestions\"],\"diagnostics\":[\"darling_core/diagnostics\"],\"suggestions\":[\"darling_core/suggestions\"]}}", + "darling_core_0.20.10": "{\"dependencies\":[{\"name\":\"fnv\",\"req\":\"^1.0.7\"},{\"name\":\"ident_case\",\"req\":\"^1.0.1\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"strsim\",\"optional\":true,\"req\":\"^0.11.1\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"diagnostics\":[],\"suggestions\":[\"strsim\"]}}", + "darling_macro_0.20.10": "{\"dependencies\":[{\"name\":\"darling_core\",\"req\":\"=0.20.10\"},{\"name\":\"quote\",\"req\":\"^1.0.18\"},{\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{}}", + "dary_heap_0.3.7": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"alloc\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1\"}],\"features\":{\"extra\":[],\"unstable\":[],\"unstable_nightly\":[]}}", + "derive_builder_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_macro\",\"req\":\"=0.20.2\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.4\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.38\",\"target\":\"cfg(compiletests)\"}],\"features\":{\"alloc\":[\"derive_builder_macro/alloc\"],\"clippy\":[\"derive_builder_macro/clippy\"],\"default\":[\"std\"],\"std\":[\"derive_builder_macro/lib_has_std\"]}}", + "derive_builder_core_0.20.2": "{\"dependencies\":[{\"name\":\"darling\",\"req\":\"^0.20.10\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.37\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[],\"clippy\":[],\"lib_has_std\":[]}}", + "derive_builder_macro_0.20.2": "{\"dependencies\":[{\"name\":\"derive_builder_core\",\"req\":\"=0.20.2\"},{\"features\":[\"full\",\"extra-traits\"],\"name\":\"syn\",\"req\":\"^2.0.15\"}],\"features\":{\"alloc\":[\"derive_builder_core/alloc\"],\"clippy\":[\"derive_builder_core/clippy\"],\"lib_has_std\":[\"derive_builder_core/lib_has_std\"]}}", + "either_1.13.0": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.0\"}],\"features\":{\"default\":[\"use_std\"],\"use_std\":[]}}", + "esaxx-rs_0.1.10": "{\"dependencies\":[{\"kind\":\"build\",\"name\":\"cc\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"}],\"features\":{\"cpp\":[\"cc\"],\"default\":[\"cpp\"]}}", + "fnv_1.0.7": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "getrandom_0.3.3": "{\"dependencies\":[{\"name\":\"cfg-if\",\"req\":\"^1\"},{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"js-sys\",\"optional\":true,\"req\":\"^0.3.77\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\"), target_feature = \\\"atomics\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(all(any(target_os = \\\"linux\\\", target_os = \\\"android\\\"), not(any(all(target_os = \\\"linux\\\", target_env = \\\"\\\"), getrandom_backend = \\\"custom\\\", getrandom_backend = \\\"linux_raw\\\", getrandom_backend = \\\"rdrand\\\", getrandom_backend = \\\"rndr\\\"))))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"dragonfly\\\", target_os = \\\"freebsd\\\", target_os = \\\"hurd\\\", target_os = \\\"illumos\\\", target_os = \\\"cygwin\\\", all(target_os = \\\"horizon\\\", target_arch = \\\"arm\\\")))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"haiku\\\", target_os = \\\"redox\\\", target_os = \\\"nto\\\", target_os = \\\"aix\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"ios\\\", target_os = \\\"visionos\\\", target_os = \\\"watchos\\\", target_os = \\\"tvos\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(any(target_os = \\\"macos\\\", target_os = \\\"openbsd\\\", target_os = \\\"vita\\\", target_os = \\\"emscripten\\\"))\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"netbsd\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"solaris\\\")\"},{\"default_features\":false,\"name\":\"libc\",\"req\":\"^0.2.154\",\"target\":\"cfg(target_os = \\\"vxworks\\\")\"},{\"default_features\":false,\"name\":\"r-efi\",\"req\":\"^5.1\",\"target\":\"cfg(all(target_os = \\\"uefi\\\", getrandom_backend = \\\"efi_rng\\\"))\"},{\"default_features\":false,\"name\":\"wasi\",\"req\":\"^0.14\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", target_os = \\\"wasi\\\", target_env = \\\"p2\\\"))\"},{\"default_features\":false,\"name\":\"wasm-bindgen\",\"optional\":true,\"req\":\"^0.2.98\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"},{\"kind\":\"dev\",\"name\":\"wasm-bindgen-test\",\"req\":\"^0.3\",\"target\":\"cfg(all(target_arch = \\\"wasm32\\\", any(target_os = \\\"unknown\\\", target_os = \\\"none\\\")))\"}],\"features\":{\"rustc-dep-of-std\":[\"dep:compiler_builtins\",\"dep:core\"],\"std\":[],\"wasm_js\":[\"dep:wasm-bindgen\",\"dep:js-sys\"]}}", + "ident_case_1.0.1": "{\"dependencies\":[],\"features\":{}}", + "itertools_0.14.0": "{\"dependencies\":[{\"features\":[\"html_reports\"],\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.4.0\"},{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"paste\",\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"permutohedron\",\"req\":\"^0.2\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"use_std\"],\"use_alloc\":[],\"use_std\":[\"use_alloc\",\"either/use_std\"]}}", + "itoa_1.0.14": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"}],\"features\":{}}", + "libc_0.2.169": "{\"dependencies\":[{\"name\":\"rustc-std-workspace-core\",\"optional\":true,\"req\":\"^1.0.0\"}],\"features\":{\"align\":[],\"const-extern-fn\":[],\"default\":[\"std\"],\"extra_traits\":[],\"rustc-dep-of-std\":[\"align\",\"rustc-std-workspace-core\"],\"std\":[],\"use_std\":[\"std\"]}}", + "log_0.4.22": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"proc-macro2\",\"req\":\"^1.0.63\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_test\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"sval\",\"optional\":true,\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval\",\"req\":\"^2.1\"},{\"kind\":\"dev\",\"name\":\"sval_derive\",\"req\":\"^2.1\"},{\"default_features\":false,\"name\":\"sval_ref\",\"optional\":true,\"req\":\"^2.1\"},{\"default_features\":false,\"features\":[\"inline-i128\"],\"name\":\"value-bag\",\"optional\":true,\"req\":\"^1.7\"},{\"features\":[\"test\"],\"kind\":\"dev\",\"name\":\"value-bag\",\"req\":\"^1.7\"}],\"features\":{\"kv\":[],\"kv_serde\":[\"kv_std\",\"value-bag/serde\",\"serde\"],\"kv_std\":[\"std\",\"kv\",\"value-bag/error\"],\"kv_sval\":[\"kv\",\"value-bag/sval\",\"sval\",\"sval_ref\"],\"kv_unstable\":[\"kv\",\"value-bag\"],\"kv_unstable_serde\":[\"kv_serde\",\"kv_unstable_std\"],\"kv_unstable_std\":[\"kv_std\",\"kv_unstable\"],\"kv_unstable_sval\":[\"kv_sval\",\"kv_unstable\"],\"max_level_debug\":[],\"max_level_error\":[],\"max_level_info\":[],\"max_level_off\":[],\"max_level_trace\":[],\"max_level_warn\":[],\"release_max_level_debug\":[],\"release_max_level_error\":[],\"release_max_level_info\":[],\"release_max_level_off\":[],\"release_max_level_trace\":[],\"release_max_level_warn\":[],\"std\":[]}}", + "macro_rules_attribute-proc_macro_0.2.0": "{\"dependencies\":[],\"features\":{\"verbose-expansions\":[]}}", + "macro_rules_attribute_0.2.0": "{\"dependencies\":[{\"name\":\"macro_rules_attribute-proc_macro\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.10.0\"},{\"name\":\"paste\",\"req\":\"^1.0.7\"},{\"kind\":\"dev\",\"name\":\"pin-project-lite\",\"req\":\"^0.2.8\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.136\"}],\"features\":{\"better-docs\":[],\"default\":[],\"verbose-expansions\":[\"macro_rules_attribute-proc_macro/verbose-expansions\"]}}", + "memchr_2.7.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1.2\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.20\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"libc\":[],\"logging\":[\"dep:log\"],\"rustc-dep-of-std\":[\"core\",\"compiler_builtins\"],\"std\":[\"alloc\"],\"use_std\":[\"std\"]}}", + "minimal-lexical_0.2.1": "{\"dependencies\":[],\"features\":{\"alloc\":[],\"compact\":[],\"default\":[\"std\"],\"lint\":[],\"nightly\":[],\"std\":[]}}", + "monostate-impl_0.1.13": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"default_features\":false,\"features\":[\"parsing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.59\"}],\"features\":{}}", + "monostate_0.1.13": "{\"dependencies\":[{\"name\":\"monostate-impl\",\"req\":\"=0.1.13\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0.99\"}],\"features\":{}}", + "nom_7.1.3": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2.3\"},{\"default_features\":false,\"name\":\"minimal-lexical\",\"req\":\"^0.2.0\"},{\"kind\":\"dev\",\"name\":\"proptest\",\"req\":\"^1.0.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"docsrs\":[],\"std\":[\"alloc\",\"memchr/std\",\"minimal-lexical/std\"]}}", + "once_cell_1.20.2": "{\"dependencies\":[{\"name\":\"critical-section\",\"optional\":true,\"req\":\"^1.1.3\"},{\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"critical-section\",\"req\":\"^1.1.3\"},{\"default_features\":false,\"name\":\"parking_lot_core\",\"optional\":true,\"req\":\"^0.9.10\"},{\"default_features\":false,\"name\":\"portable-atomic\",\"optional\":true,\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"regex\",\"req\":\"^1.10.6\"}],\"features\":{\"alloc\":[\"race\"],\"atomic-polyfill\":[\"critical-section\"],\"critical-section\":[\"dep:critical-section\",\"portable-atomic\"],\"default\":[\"std\"],\"parking_lot\":[\"dep:parking_lot_core\"],\"portable-atomic\":[\"dep:portable-atomic\"],\"race\":[],\"std\":[\"alloc\"],\"unstable\":[]}}", + "onig_6.5.1": "{\"dependencies\":[{\"name\":\"bitflags\",\"req\":\"^2.4.0\"},{\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(windows)\"},{\"name\":\"once_cell\",\"req\":\"^1.12\"},{\"default_features\":false,\"name\":\"onig_sys\",\"req\":\"^69.9.1\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"onig_sys/generate\"],\"posix-api\":[\"onig_sys/posix-api\"],\"print-debug\":[\"onig_sys/print-debug\"],\"std-pattern\":[]}}", + "onig_sys_69.9.1": "{\"dependencies\":[{\"features\":[\"runtime\"],\"kind\":\"build\",\"name\":\"bindgen\",\"optional\":true,\"req\":\"^0.71\"},{\"kind\":\"build\",\"name\":\"cc\",\"req\":\"^1.0\"},{\"kind\":\"build\",\"name\":\"pkg-config\",\"req\":\"^0.3.16\"}],\"features\":{\"default\":[\"generate\"],\"generate\":[\"bindgen\"],\"posix-api\":[],\"print-debug\":[]}}", + "paste_1.0.15": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"paste-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "pkg-config_0.3.32": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"lazy_static\",\"req\":\"^1\"}],\"features\":{}}", + "ppv-lite86_0.2.20": "{\"dependencies\":[{\"features\":[\"simd\",\"derive\"],\"name\":\"zerocopy\",\"req\":\"^0.7\"}],\"features\":{\"default\":[\"std\"],\"no_simd\":[],\"simd\":[],\"std\":[]}}", + "proc-macro2_1.0.92": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quote\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4\"},{\"name\":\"unicode-ident\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"proc-macro\"],\"nightly\":[],\"proc-macro\":[],\"span-locations\":[]}}", + "quote_1.0.37": "{\"dependencies\":[{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.80\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.66\"}],\"features\":{\"default\":[\"proc-macro\"],\"proc-macro\":[\"proc-macro2/proc-macro\"]}}", + "r-efi_5.3.0": "{\"dependencies\":[{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0.0\"}],\"features\":{\"efiapi\":[],\"examples\":[\"native\"],\"native\":[],\"rustc-dep-of-std\":[\"core\"]}}", + "rand_0.9.1": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.2.1\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.4\"},{\"default_features\":false,\"name\":\"rand_chacha\",\"optional\":true,\"req\":\"^0.9.0\"},{\"default_features\":false,\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rand_pcg\",\"req\":\"^0.9.0\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1.7\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0.103\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"std_rng\",\"os_rng\",\"small_rng\",\"thread_rng\"],\"log\":[\"dep:log\"],\"nightly\":[],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\",\"rand_core/serde\"],\"simd_support\":[],\"small_rng\":[],\"std\":[\"rand_core/std\",\"rand_chacha?/std\",\"alloc\"],\"std_rng\":[\"dep:rand_chacha\"],\"thread_rng\":[\"std\",\"std_rng\",\"os_rng\"],\"unbiased\":[]}}", + "rand_chacha_0.9.0": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"simd\"],\"name\":\"ppv-lite86\",\"req\":\"^0.2.14\"},{\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"os_rng\"],\"kind\":\"dev\",\"name\":\"rand_core\",\"req\":\"^0.9.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"}],\"features\":{\"default\":[\"std\"],\"os_rng\":[\"rand_core/os_rng\"],\"serde\":[\"dep:serde\"],\"std\":[\"ppv-lite86/std\",\"rand_core/std\"]}}", + "rand_core_0.9.3": "{\"dependencies\":[{\"name\":\"getrandom\",\"optional\":true,\"req\":\"^0.3.0\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"os_rng\":[\"dep:getrandom\"],\"serde\":[\"dep:serde\"],\"std\":[\"getrandom?/std\"]}}", + "rayon-cond_0.4.0": "{\"dependencies\":[{\"name\":\"either\",\"req\":\"^1.6\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"rayon\",\"req\":\"^1.7\"}],\"features\":{}}", + "rayon-core_1.12.1": "{\"dependencies\":[{\"name\":\"crossbeam-deque\",\"req\":\"^0.8.1\"},{\"name\":\"crossbeam-utils\",\"req\":\"^0.8.0\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"^0.2\",\"target\":\"cfg(unix)\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"kind\":\"dev\",\"name\":\"scoped-tls\",\"req\":\"^1.0\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\"]}}", + "rayon_1.10.0": "{\"dependencies\":[{\"default_features\":false,\"name\":\"either\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"},{\"name\":\"rayon-core\",\"req\":\"^1.12.1\"},{\"name\":\"wasm_sync\",\"optional\":true,\"req\":\"^0.1.0\"}],\"features\":{\"web_spin_lock\":[\"dep:wasm_sync\",\"rayon-core/web_spin_lock\"]}}", + "regex-automata_0.4.9": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"default_features\":false,\"features\":[\"std\"],\"kind\":\"dev\",\"name\":\"bstr\",\"req\":\"^1.3.0\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"name\":\"log\",\"optional\":true,\"req\":\"^0.4.14\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"optional\":true,\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"alloc\":[],\"default\":[\"std\",\"syntax\",\"perf\",\"unicode\",\"meta\",\"nfa\",\"dfa\",\"hybrid\"],\"dfa\":[\"dfa-build\",\"dfa-search\",\"dfa-onepass\"],\"dfa-build\":[\"nfa-thompson\",\"dfa-search\"],\"dfa-onepass\":[\"nfa-thompson\"],\"dfa-search\":[],\"hybrid\":[\"alloc\",\"nfa-thompson\"],\"internal-instrument\":[\"internal-instrument-pikevm\"],\"internal-instrument-pikevm\":[\"logging\",\"std\"],\"logging\":[\"dep:log\",\"aho-corasick?/logging\",\"memchr?/logging\"],\"meta\":[\"syntax\",\"nfa-pikevm\"],\"nfa\":[\"nfa-thompson\",\"nfa-pikevm\",\"nfa-backtrack\"],\"nfa-backtrack\":[\"nfa-thompson\"],\"nfa-pikevm\":[\"nfa-thompson\"],\"nfa-thompson\":[\"alloc\"],\"perf\":[\"perf-inline\",\"perf-literal\"],\"perf-inline\":[],\"perf-literal\":[\"perf-literal-substring\",\"perf-literal-multisubstring\"],\"perf-literal-multisubstring\":[\"std\",\"dep:aho-corasick\"],\"perf-literal-substring\":[\"aho-corasick?/perf-literal\",\"dep:memchr\"],\"std\":[\"regex-syntax?/std\",\"memchr?/std\",\"aho-corasick?/std\",\"alloc\"],\"syntax\":[\"dep:regex-syntax\",\"alloc\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"unicode-word-boundary\",\"regex-syntax?/unicode\"],\"unicode-age\":[\"regex-syntax?/unicode-age\"],\"unicode-bool\":[\"regex-syntax?/unicode-bool\"],\"unicode-case\":[\"regex-syntax?/unicode-case\"],\"unicode-gencat\":[\"regex-syntax?/unicode-gencat\"],\"unicode-perl\":[\"regex-syntax?/unicode-perl\"],\"unicode-script\":[\"regex-syntax?/unicode-script\"],\"unicode-segment\":[\"regex-syntax?/unicode-segment\"],\"unicode-word-boundary\":[]}}", + "regex-syntax_0.8.5": "{\"dependencies\":[{\"features\":[\"derive\"],\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1.3.0\"}],\"features\":{\"arbitrary\":[\"dep:arbitrary\"],\"default\":[\"std\",\"unicode\"],\"std\":[],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\"],\"unicode-age\":[],\"unicode-bool\":[],\"unicode-case\":[],\"unicode-gencat\":[],\"unicode-perl\":[],\"unicode-script\":[],\"unicode-segment\":[]}}", + "regex_1.11.1": "{\"dependencies\":[{\"default_features\":false,\"name\":\"aho-corasick\",\"optional\":true,\"req\":\"^1.0.0\"},{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.69\"},{\"kind\":\"dev\",\"name\":\"doc-comment\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"atty\",\"humantime\",\"termcolor\"],\"kind\":\"dev\",\"name\":\"env_logger\",\"req\":\"^0.9.3\"},{\"default_features\":false,\"name\":\"memchr\",\"optional\":true,\"req\":\"^2.6.0\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"^1.17.1\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^1.0.3\"},{\"default_features\":false,\"features\":[\"alloc\",\"syntax\",\"meta\",\"nfa-pikevm\"],\"name\":\"regex-automata\",\"req\":\"^0.4.8\"},{\"default_features\":false,\"name\":\"regex-syntax\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"regex-test\",\"req\":\"^0.1.0\"}],\"features\":{\"default\":[\"std\",\"perf\",\"unicode\",\"regex-syntax/default\"],\"logging\":[\"aho-corasick?/logging\",\"memchr?/logging\",\"regex-automata/logging\"],\"pattern\":[],\"perf\":[\"perf-cache\",\"perf-dfa\",\"perf-onepass\",\"perf-backtrack\",\"perf-inline\",\"perf-literal\"],\"perf-backtrack\":[\"regex-automata/nfa-backtrack\"],\"perf-cache\":[],\"perf-dfa\":[\"regex-automata/hybrid\"],\"perf-dfa-full\":[\"regex-automata/dfa-build\",\"regex-automata/dfa-search\"],\"perf-inline\":[\"regex-automata/perf-inline\"],\"perf-literal\":[\"dep:aho-corasick\",\"dep:memchr\",\"regex-automata/perf-literal\"],\"perf-onepass\":[\"regex-automata/dfa-onepass\"],\"std\":[\"aho-corasick?/std\",\"memchr?/std\",\"regex-automata/std\",\"regex-syntax/std\"],\"unicode\":[\"unicode-age\",\"unicode-bool\",\"unicode-case\",\"unicode-gencat\",\"unicode-perl\",\"unicode-script\",\"unicode-segment\",\"regex-automata/unicode\",\"regex-syntax/unicode\"],\"unicode-age\":[\"regex-automata/unicode-age\",\"regex-syntax/unicode-age\"],\"unicode-bool\":[\"regex-automata/unicode-bool\",\"regex-syntax/unicode-bool\"],\"unicode-case\":[\"regex-automata/unicode-case\",\"regex-syntax/unicode-case\"],\"unicode-gencat\":[\"regex-automata/unicode-gencat\",\"regex-syntax/unicode-gencat\"],\"unicode-perl\":[\"regex-automata/unicode-perl\",\"regex-automata/unicode-word-boundary\",\"regex-syntax/unicode-perl\"],\"unicode-script\":[\"regex-automata/unicode-script\",\"regex-syntax/unicode-script\"],\"unicode-segment\":[\"regex-automata/unicode-segment\",\"regex-syntax/unicode-segment\"],\"unstable\":[\"pattern\"],\"use_std\":[\"std\"]}}", + "rustversion_1.0.21": "{\"dependencies\":[{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.49\"}],\"features\":{}}", + "ryu_1.0.18": "{\"dependencies\":[{\"name\":\"no-panic\",\"optional\":true,\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"num_cpus\",\"req\":\"^1.8\"},{\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"rand_xorshift\",\"req\":\"^0.3\"}],\"features\":{\"small\":[]}}", + "serde_1.0.216": "{\"dependencies\":[{\"name\":\"serde_derive\",\"req\":\"=1.0.216\",\"target\":\"cfg(any())\"},{\"name\":\"serde_derive\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1\"}],\"features\":{\"alloc\":[],\"default\":[\"std\"],\"derive\":[\"serde_derive\"],\"rc\":[],\"std\":[],\"unstable\":[]}}", + "serde_derive_1.0.216": "{\"dependencies\":[{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"default_features\":false,\"features\":[\"proc-macro\"],\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1\"},{\"default_features\":false,\"features\":[\"clone-impls\",\"derive\",\"parsing\",\"printing\",\"proc-macro\"],\"name\":\"syn\",\"req\":\"^2.0.81\"}],\"features\":{\"default\":[],\"deserialize_in_place\":[]}}", + "serde_json_1.0.134": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1.0.11\"},{\"name\":\"indexmap\",\"optional\":true,\"req\":\"^2.2.3\"},{\"kind\":\"dev\",\"name\":\"indoc\",\"req\":\"^2.0.2\"},{\"name\":\"itoa\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"memchr\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"ryu\",\"req\":\"^1.0\"},{\"default_features\":false,\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"features\":[\"derive\"],\"kind\":\"dev\",\"name\":\"serde\",\"req\":\"^1.0.194\"},{\"kind\":\"dev\",\"name\":\"serde_bytes\",\"req\":\"^0.11.10\"},{\"kind\":\"dev\",\"name\":\"serde_derive\",\"req\":\"^1.0.166\"},{\"kind\":\"dev\",\"name\":\"serde_stacker\",\"req\":\"^0.1.8\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"alloc\":[\"serde/alloc\"],\"arbitrary_precision\":[],\"default\":[\"std\"],\"float_roundtrip\":[],\"preserve_order\":[\"indexmap\",\"std\"],\"raw_value\":[],\"std\":[\"memchr/std\",\"serde/std\"],\"unbounded_depth\":[]}}", + "shlex_1.3.0": "{\"dependencies\":[],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "smallvec_1.13.2": "{\"dependencies\":[{\"name\":\"arbitrary\",\"optional\":true,\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"bincode\",\"req\":\"^1.0.1\"},{\"kind\":\"dev\",\"name\":\"debugger_test\",\"req\":\"^0.1.0\"},{\"kind\":\"dev\",\"name\":\"debugger_test_parser\",\"req\":\"^0.1.0\"},{\"default_features\":false,\"name\":\"serde\",\"optional\":true,\"req\":\"^1\"}],\"features\":{\"const_generics\":[],\"const_new\":[\"const_generics\"],\"debugger_visualizer\":[],\"drain_filter\":[],\"drain_keep_rest\":[\"drain_filter\"],\"may_dangle\":[],\"specialization\":[],\"union\":[],\"write\":[]}}", + "spm_precompiled_0.1.4": "{\"dependencies\":[{\"name\":\"base64\",\"req\":\"^0.13\"},{\"name\":\"nom\",\"req\":\"^7.1.1\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.9\"}],\"features\":{}}", + "static_assertions_1.1.0": "{\"dependencies\":[],\"features\":{\"nightly\":[]}}", + "strsim_0.11.1": "{\"dependencies\":[],\"features\":{}}", + "syn_2.0.91": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"automod\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"flate2\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"insta\",\"req\":\"^1\"},{\"default_features\":false,\"name\":\"proc-macro2\",\"req\":\"^1.0.91\"},{\"default_features\":false,\"name\":\"quote\",\"optional\":true,\"req\":\"^1.0.35\"},{\"kind\":\"dev\",\"name\":\"rayon\",\"req\":\"^1\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1\"},{\"features\":[\"blocking\"],\"kind\":\"dev\",\"name\":\"reqwest\",\"req\":\"^0.12\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"syn-test-suite\",\"req\":\"^0\"},{\"kind\":\"dev\",\"name\":\"tar\",\"req\":\"^0.4.16\",\"target\":\"cfg(not(miri))\"},{\"kind\":\"dev\",\"name\":\"termcolor\",\"req\":\"^1\"},{\"name\":\"unicode-ident\",\"req\":\"^1\"},{\"kind\":\"dev\",\"name\":\"walkdir\",\"req\":\"^2.3.2\",\"target\":\"cfg(not(miri))\"}],\"features\":{\"clone-impls\":[],\"default\":[\"derive\",\"parsing\",\"printing\",\"clone-impls\",\"proc-macro\"],\"derive\":[],\"extra-traits\":[],\"fold\":[],\"full\":[],\"parsing\":[],\"printing\":[\"dep:quote\"],\"proc-macro\":[\"proc-macro2/proc-macro\",\"quote?/proc-macro\"],\"test\":[\"syn-test-suite/all-features\"],\"visit\":[],\"visit-mut\":[]}}", + "thiserror-impl_2.0.12": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.74\"},{\"name\":\"quote\",\"req\":\"^1.0.35\"},{\"name\":\"syn\",\"req\":\"^2.0.87\"}],\"features\":{}}", + "thiserror_2.0.12": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"anyhow\",\"req\":\"^1.0.73\"},{\"kind\":\"dev\",\"name\":\"ref-cast\",\"req\":\"^1.0.18\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0.13\"},{\"name\":\"thiserror-impl\",\"req\":\"=2.0.12\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"^1.0.81\"}],\"features\":{\"default\":[\"std\"],\"std\":[]}}", + "tokenizers_0.22.2": "{\"dependencies\":[{\"features\":[\"serde\"],\"name\":\"ahash\",\"req\":\"^0.8.11\"},{\"name\":\"aho-corasick\",\"req\":\"^1.1\"},{\"kind\":\"dev\",\"name\":\"assert_approx_eq\",\"req\":\"^1.1\"},{\"features\":[\"serde\"],\"name\":\"compact_str\",\"req\":\"^0.9\"},{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.6\"},{\"features\":[\"serde\"],\"name\":\"dary_heap\",\"req\":\"^0.3.6\"},{\"name\":\"derive_builder\",\"req\":\"^0.20\"},{\"default_features\":false,\"name\":\"esaxx-rs\",\"req\":\"^0.1.10\"},{\"name\":\"fancy-regex\",\"optional\":true,\"req\":\"^0.14\"},{\"name\":\"getrandom\",\"req\":\"^0.3\"},{\"default_features\":false,\"features\":[\"ureq\"],\"name\":\"hf-hub\",\"optional\":true,\"req\":\"^0.4.1\"},{\"name\":\"indicatif\",\"optional\":true,\"req\":\"^0.18\"},{\"name\":\"itertools\",\"req\":\"^0.14\"},{\"name\":\"log\",\"req\":\"^0.4\"},{\"name\":\"macro_rules_attribute\",\"req\":\"^0.2.0\"},{\"name\":\"monostate\",\"req\":\"^0.1.12\"},{\"default_features\":false,\"name\":\"onig\",\"optional\":true,\"req\":\"^6.5.1\"},{\"name\":\"paste\",\"req\":\"^1.0.14\"},{\"name\":\"rand\",\"req\":\"^0.9\"},{\"name\":\"rayon\",\"req\":\"^1.10\"},{\"name\":\"rayon-cond\",\"req\":\"^0.4\"},{\"name\":\"regex\",\"req\":\"^1.10\"},{\"name\":\"regex-syntax\",\"req\":\"^0.8\"},{\"features\":[\"derive\"],\"name\":\"serde\",\"req\":\"^1.0\"},{\"name\":\"serde_json\",\"req\":\"^1.0\"},{\"name\":\"spm_precompiled\",\"req\":\"^0.1.3\"},{\"kind\":\"dev\",\"name\":\"tempfile\",\"req\":\"^3.10\"},{\"name\":\"thiserror\",\"req\":\"^2\"},{\"kind\":\"dev\",\"name\":\"tracing\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"tracing-subscriber\",\"req\":\"^0.3.18\"},{\"name\":\"unicode-normalization-alignments\",\"req\":\"^0.1\"},{\"name\":\"unicode-segmentation\",\"req\":\"^1.11\"},{\"name\":\"unicode_categories\",\"req\":\"^0.1\"}],\"features\":{\"default\":[\"progressbar\",\"onig\",\"esaxx_fast\"],\"esaxx_fast\":[\"esaxx-rs/cpp\"],\"http\":[\"hf-hub\"],\"progressbar\":[\"indicatif\"],\"rustls-tls\":[\"hf-hub?/rustls-tls\"],\"unstable_wasm\":[\"fancy-regex\",\"getrandom/wasm_js\"]}}", + "unicode-ident_1.0.14": "{\"dependencies\":[{\"default_features\":false,\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"fst\",\"req\":\"^0.4\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8\"},{\"kind\":\"dev\",\"name\":\"roaring\",\"req\":\"^0.10\"},{\"default_features\":false,\"kind\":\"dev\",\"name\":\"ucd-trie\",\"req\":\"^0.1\"},{\"kind\":\"dev\",\"name\":\"unicode-xid\",\"req\":\"^0.2.6\"}],\"features\":{}}", + "unicode-normalization-alignments_0.1.12": "{\"dependencies\":[{\"name\":\"smallvec\",\"req\":\"^1.0\"}],\"features\":{}}", + "unicode-segmentation_1.12.0": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"criterion\",\"req\":\"^0.5\"},{\"kind\":\"dev\",\"name\":\"quickcheck\",\"req\":\"^0.7\"}],\"features\":{\"no_std\":[]}}", + "unicode_categories_0.1.1": "{\"dependencies\":[],\"features\":{}}", + "version_check_0.9.5": "{\"dependencies\":[],\"features\":{}}", + "wasi_0.14.2+wasi-0.2.4": "{\"dependencies\":[{\"name\":\"compiler_builtins\",\"optional\":true,\"req\":\"^0.1\"},{\"name\":\"core\",\"optional\":true,\"package\":\"rustc-std-workspace-core\",\"req\":\"^1.0\"},{\"name\":\"rustc-std-workspace-alloc\",\"optional\":true,\"req\":\"^1.0\"},{\"features\":[\"bitflags\"],\"name\":\"wit-bindgen-rt\",\"req\":\"^0.39.0\"}],\"features\":{\"default\":[\"std\"],\"rustc-dep-of-std\":[\"compiler_builtins\",\"core\",\"rustc-std-workspace-alloc\"],\"std\":[]}}", + "wit-bindgen-rt_0.39.0": "{\"dependencies\":[{\"name\":\"bitflags\",\"optional\":true,\"req\":\"^2.3.3\"},{\"name\":\"futures\",\"optional\":true,\"req\":\"^0.3.30\"},{\"name\":\"once_cell\",\"optional\":true,\"req\":\"^1.19.0\"}],\"features\":{\"async\":[\"dep:futures\",\"dep:once_cell\"]}}", + "zerocopy-derive_0.7.35": "{\"dependencies\":[{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"name\":\"syn\",\"req\":\"^2.0.31\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"}],\"features\":{}}", + "zerocopy-derive_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"dissimilar\",\"req\":\"^1.0.9\"},{\"kind\":\"dev\",\"name\":\"libc\",\"req\":\"=0.2.163\"},{\"kind\":\"dev\",\"name\":\"once_cell\",\"req\":\"=1.9\"},{\"kind\":\"dev\",\"name\":\"prettyplease\",\"req\":\"=0.2.17\"},{\"name\":\"proc-macro2\",\"req\":\"^1.0.1\"},{\"name\":\"quote\",\"req\":\"^1.0.10\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"full\"],\"name\":\"syn\",\"req\":\"^2.0.46\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"}],\"features\":{}}", + "zerocopy_0.7.35": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"assert_matches\",\"req\":\"^1.5\"},{\"default_features\":false,\"name\":\"byteorder\",\"optional\":true,\"req\":\"^1.3\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.85\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.7.35\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.7.35\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\"],\"alloc\":[],\"default\":[\"byteorder\"],\"derive\":[\"zerocopy-derive\"],\"simd\":[],\"simd-nightly\":[\"simd\"]}}", + "zerocopy_0.8.26": "{\"dependencies\":[{\"kind\":\"dev\",\"name\":\"either\",\"req\":\"=1.13.0\"},{\"kind\":\"dev\",\"name\":\"elain\",\"req\":\"^0.3.0\"},{\"kind\":\"dev\",\"name\":\"itertools\",\"req\":\"^0.11\"},{\"default_features\":false,\"features\":[\"small_rng\"],\"kind\":\"dev\",\"name\":\"rand\",\"req\":\"^0.8.5\"},{\"kind\":\"dev\",\"name\":\"rustversion\",\"req\":\"^1.0\"},{\"kind\":\"dev\",\"name\":\"static_assertions\",\"req\":\"^1.1\"},{\"features\":[\"diff\"],\"kind\":\"dev\",\"name\":\"trybuild\",\"req\":\"=1.0.89\"},{\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\",\"target\":\"cfg(any())\"},{\"name\":\"zerocopy-derive\",\"optional\":true,\"req\":\"=0.8.26\"},{\"kind\":\"dev\",\"name\":\"zerocopy-derive\",\"req\":\"=0.8.26\"}],\"features\":{\"__internal_use_only_features_that_work_on_stable\":[\"alloc\",\"derive\",\"simd\",\"std\"],\"alloc\":[],\"derive\":[\"zerocopy-derive\"],\"float-nightly\":[],\"simd\":[],\"simd-nightly\":[\"simd\"],\"std\":[\"alloc\"]}}" } - }, - "factsVersions": {} + } } diff --git a/README.md b/README.md index d9e68b81..b511fc1d 100644 --- a/README.md +++ b/README.md @@ -2,139 +2,197 @@ Website | Getting Started - | Documentation + | Documentation | Discord | Contributing [ZML]: https://zml.ai/ [Getting Started]: #getting-started -[Documentation]: ./docs/README.md +[Documentation]: https://docs.zml.ai [Contributing]: ./CONTRIBUTING.md [Discord]: https://discord.gg/6y72SN2E7H -# About +# Bonjour 👋 -ZML is a production inference stack, purpose-built to decouple AI workloads from proprietary hardware. +At ZML, we are creating exciting AI products on top of our high-performance +AI inference stack. Our stack is built for production, using the amazing +[Zig](https://ziglang.org) language, [MLIR](https://mlir.llvm.org), and the +power of [Bazel](https://bazel.build). -Any model, many hardwares, one codebase, peak performance. +
+
Take me straight to getting started or give me a taste 🥐!
+
-Compiled directly to NVIDIA, AMD, INTEL, TPU, Trainium for peak hardware performance on any accelerator. No rewriting. +--- + +  + +# We're happy to share! +We're very happy to share our inference stack with the World and hope it allows +you, too, to build cool and exciting AI projects. + +To give you a glimpse of what you can do with ZML, here is an early demo: + +
+ +It shows a prototype running a LLama3 model sharded on 1 NVIDIA RTX 4090, 1 AMD +6800XT, and 1 Google Cloud TPU v2. All accelerators were hosted in different +locations, with activations being passed over a VPN. + +All processes used the same model code, cross-compiled on a Mac, and copied onto +the servers. + +For more inspiration, see also the examples below or check out the +[examples](./examples) folder. + + + +# Getting started -It is built using the -[Zig](https://ziglang.org) language, [MLIR](https://mlir.llvm.org), and [Bazel](https://bazel.build). -# Getting Started ## Prerequisites We use `bazel` to build ZML and its dependencies. The only prerequisite is -`bazel`, which we recommend installing through `bazelisk`. +`bazel`, which we recommend to download through `bazelisk`, a version manager +for `bazel`. + +**Please note: If you do not wish to install `bazel`** system-wide, we provide +[bazel.sh](bazel.sh) which downloads it to your home folder +and runs it. + +**Install Bazel** (recommended): + +
### macOS + -```bash +``` brew install bazelisk ``` +
+ +
### Linux + -```bash -curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.28.0/bazelisk-linux-amd64' +``` +curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-amd64' chmod +x /usr/local/bin/bazel ``` +
-## 30-Second Smoke Test -Run the MNIST example: +## Run a pre-packaged model -```bash -bazel run //examples/mnist +We have implemented a variety of example models in ZML. See our reference +implementations in the +[examples](https://github.com/zml/zml/tree/master/examples/) folder. + +### MNIST + +The [classic](https://en.wikipedia.org/wiki/MNIST_database) handwritten digits +recognition task. The model is tasked to recognize a handwritten digit, which +has been converted to a 28x28 pixel monochrome image. `Bazel` will download a +pre-trained model, and the test dataset. The program will load the model, +compile it, and classify a randomly picked example from the test dataset. + +On the command line: + +``` +bazel run --config=release //examples/mnist + +# or +./bazel.sh run --config=release //examples/mnist ``` -This downloads a small pretrained MNIST model, compiles it, loads the weights, and -classifies a random handwritten digit. -## LLM Quickstart -The main LLM example is [`//examples/llm`](./examples/llm). It currently supports: +### Meta Llama 3.1 8B -- Llama 3.1 / 3.2 -- Qwen 3.5 -- LFM 2.5 +This model has restrictions, see +[here](https://huggingface.co/meta-llama/Llama-3.1-8B-Instruct). It **requires +approval from Meta on Huggingface**, which can take a few hours to get granted. -Authenticate with Hugging Face if you want to load gated repos such as Meta -Llama: +Once you've been granted access, you're ready to download a gated model like +`Llama-3.1-8B-Instruct`! -```bash -bazel run //tools/hf -- auth login +First, you need to download the model using the [huggingface-cli](https://huggingface.co/docs/huggingface_hub/en/guides/cli). +Note you don't need to install it yourself, +you can just use the packaged version `bazel run //tools/hf --`. + +``` +bazel run //tools/hf -- download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct --exclude='*.pth' ``` -Alternatively, set the `HF_TOKEN` environment variable. +Then, you can run the model. -Then run a prompt directly: - -```bash -bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" +``` +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" ``` -Open the interactive chat loop by omitting `--prompt`: +You can also try `Llama-3.1-70B-Instruct` if you have enough memory. -```bash -bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct +### Meta Llama 3.2 1B + +Like the 8B model above, this model also requires approval. See +[here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for access requirements. + +``` +bazel run //tools/hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct -- --prompt="What is the capital of France?" ``` -You can also load from: - -- a local directory: `--model=/var/models/meta-llama/Llama-3.2-1B-Instruct` -- S3: `--model=s3://bucket/path/to/model` +For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. ## Running Models on GPU / TPU -Append one or more platform flags when compiling or running: +You can compile models for accelerator runtimes by appending one or more of the +following arguments to the command line when compiling / running a model: - NVIDIA CUDA: `--@zml//platforms:cuda=true` - AMD RoCM: `--@zml//platforms:rocm=true` -- Intel OneAPI: `--@zml//platforms:oneapi=true` - Google TPU: `--@zml//platforms:tpu=true` -- AWS Trainium / Inferentia 2: `--@zml//platforms:neuron=true` -- Disable CPU compilation: `--@zml//platforms:cpu=false` +- AWS Trainium/Inferentia 2: `--@zml//platforms:neuron=true` +- **AVOID CPU:** `--@zml//platforms:cpu=false` -Example on CUDA: +The latter, avoiding compilation for CPU, cuts down compilation time. -```bash -bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" +So, to run the Llama 3.1 8B model from above on your host sporting an NVIDIA GPU, +run the following: + +``` +bazel run --config=release //examples/llama \ + --@zml//platforms:cuda=true \ + -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct \ + --prompt="What is the capital of France?" ``` -Example on ROCm: - -```bash -bazel run //examples/llm --@zml//platforms:rocm=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" -``` - -Example on Intel OneAPI: - -```bash -bazel run //examples/llm --@zml//platforms:cpu=false --@zml//platforms:oneapi=true -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="Write a haiku about Zig" -``` ## Run Tests -```bash +``` bazel test //zml:test ``` -# Examples -- [`examples/llm`](./examples/llm): unified LLM CLI for Llama, Qwen, and LFM -- [`examples/mnist`](./examples/mnist): smallest end-to-end model run -- [`examples/sharding`](./examples/sharding): logical mesh, partitioners, shard-local execution, profiler output -- [`examples/io`](./examples/io): inspect and load local, `hf://`, `https://`, and `s3://` repositories through the VFS layer -- [`examples/benchmark`](./examples/benchmark): measure loading and execution performance +# A taste of ZML + + + +## MNIST -# A Taste Of ZML ```zig +const std = @import("std"); +const zml = @import("zml"); + +/// Model definition const Mnist = struct { fc1: Layer, fc2: Layer, @@ -143,84 +201,65 @@ const Mnist = struct { weight: zml.Tensor, bias: zml.Tensor, - pub fn init(store: zml.io.TensorStore.View) Layer { - return .{ - .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), - .bias = store.createReplicatedTensor("bias", .{.d_out}), - }; - } - pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - return self.weight.dot(input, .d).add(self.bias).relu().withTags(.{.d}); + return self.weight.matmul(input).add(self.bias).relu(); } }; - pub fn init(store: zml.io.TensorStore.View) Mnist { - return .{ - .fc1 = .init(store.withPrefix("fc1")), - .fc2 = .init(store.withPrefix("fc2")), - }; - } - - pub fn load( - self: *const Mnist, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *const zml.io.TensorStore, - meshes: []const *const zml.Mesh, - ) !zml.Bufferized(Mnist) { - return zml.io.load(Mnist, self, allocator, io, platform, store, .{ - .meshes = meshes, - .parallelism = 1, - .dma_chunks = 1, - .dma_chunk_size = 16 * 1024 * 1024, - }); - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { - self.fc1.weight.deinit(); - self.fc1.bias.deinit(); - self.fc2.weight.deinit(); - self.fc2.bias.deinit(); - } - /// just two linear layers + relu activation pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { - var x = input.flatten().convert(.f32).withTags(.{.d}); + std.log.info("Compiling for target: {s}", .{@tagName(input.getContext().target())}); + var x = input.flattenAll().convert(.f32); const layers: []const Layer = &.{ self.fc1, self.fc2 }; for (layers) |layer| { - x = layer.forward(x); + x = zml.call(layer, .forward, .{x}); } - return x.argMax(0).indices.convert(.u8); + return x.argMax(0, .u8).indices; } }; ``` -For a full walkthrough, see: -- [Getting Started](./docs/tutorials/getting_started.md) -- [Writing your first model](./docs/tutorials/write_first_model.md) -- [ZML Concepts](./docs/learn/concepts.md) -- [Deploying on a server](./docs/howtos/deploy_on_server.md) -# Where To Go Next +## Tagged Tensors + +```zig +const Sdpa = struct { + pub fn forward(_: Sdpa, ctx: *zml.Context, q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor) zml.Tensor { + const q = q_.withTags(.{ .b, .h, .q, .hd }); + const k = k_.withTags(.{ .b, .h, .k, .hd }); + const v = v_.withTags(.{ .b, .h, .k, .hd }); + const attn_mask = zml.nn.causalAttnMask(ctx, .{ .q = q.dim(.q), .k = k.dim(.k) }, q.dtype(), null); + return zml.nn.sdpa(ctx, q, k, v, .{ .attn_mask = attn_mask }); + } +}; +``` + + + + +# Where to go next: + +You might want to check out more [examples](./examples), read through the +[documentation directly on GitHub](./docs/README.md), or, for the full rendering +experience, browse the +[online documentation with included API reference](https://docs.zml.ai). + -- Run more examples in [`./examples`](./examples) -- Read the example-specific notes in [`examples/llm/README.md`](./examples/llm/README.md) -- Learn tagged dimensions in [`working_with_tensors.md`](./docs/tutorials/working_with_tensors.md) -- Start building a model with [`write_first_model.md`](./docs/tutorials/write_first_model.md) -- Explore deployment in [`deploy_on_server.md`](./docs/howtos/deploy_on_server.md) # Contributing See [here][Contributing]. + + # License ZML is licensed under the [Apache 2.0 license](./LICENSE). -# Thanks To Our Contributors + + +# Thanks to our contributors diff --git a/bazel/BUILD.bazel b/bazel/BUILD.bazel index eb2b2ab2..bb4a1781 100644 --- a/bazel/BUILD.bazel +++ b/bazel/BUILD.bazel @@ -1,24 +1,10 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") -cc_library( - name = "retain_runfiles_init", - srcs = ["runfiles_init.c"], - # Retain the constructor even when its object is in a static archive. - # Mach-O adds an underscore to the exported C symbol. - linkopts = select({ - "@platforms//os:macos": ["-Wl,-u,__bazel_runfiles_init"], - "//conditions:default": ["-Wl,-u,_bazel_runfiles_init"], - }), -) - zig_library( name = "bazel", main = "bazel.zig", - tags = ["manual"], visibility = ["//visibility:public"], deps = [ - ":retain_runfiles_init", "//stdx", "@rules_zig//zig/runfiles", ], diff --git a/bazel/bazel.zig b/bazel/bazel.zig index 67513227..6d5540b0 100644 --- a/bazel/bazel.zig +++ b/bazel/bazel.zig @@ -1,27 +1,52 @@ const std = @import("std"); +const builtin = @import("builtin"); const bazel_runfiles = @import("runfiles"); const stdx = @import("stdx"); var runfiles_global: bazel_runfiles.Runfiles = undefined; -pub export fn _bazel_runfiles_init_impl(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { - _ = argc; - _ = envp; - var threaded: std.Io.Threaded = .init_single_threaded; - runfiles_global = bazel_runfiles.Runfiles.create(.{ - .allocator = std.heap.c_allocator, - .io = threaded.io(), - .argv0 = std.mem.span(argv[0]), - .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, - .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, - }) catch { +const init_array_section = switch (builtin.object_format) { + .macho => "__DATA,__mod_init_func", + .elf => ".init_array", + else => "", +}; + +export const _ linksection(init_array_section) = &struct { + fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { + _ = argc; // autofix + _ = envp; // autofix + var threaded: std.Io.Threaded = .init_single_threaded; + runfiles_global = bazel_runfiles.Runfiles.create(.{ + .allocator = std.heap.c_allocator, + .io = threaded.io(), + .argv0 = std.mem.span(argv[0]), + .directory = if (std.c.getenv("RUNFILES_DIRECTORY")) |directory| std.mem.span(directory) else null, + .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, + }) catch { + std.debug.panic("Unable to find runfiles", .{}); + } orelse { + std.debug.panic("Unable to initialize runfiles", .{}); + }; + } +}.call; + +pub fn init(allocator: std.mem.Allocator, io: std.Io, argv: std.process.Args, environ_map: *std.process.Environ.Map) !void { + runfiles_global = try allocator.create(bazel_runfiles.Runfiles); + runfiles_global.* = try bazel_runfiles.Runfiles.create(.{ + .allocator = allocator, + .io = io, + .argv = argv, + .environ_map = environ_map, + }) orelse { std.debug.panic("Unable to find runfiles", .{}); - } orelse { - std.debug.panic("Unable to initialize runfiles", .{}); }; } +pub fn initSimple(init_: std.process.Init) !void { + try init(init_.arena.allocator(), init_.io, init_.minimal.args, init_.environ_map); +} + pub fn runfiles(source_repository: []const u8) !bazel_runfiles.Runfiles.WithSourceRepo { return runfiles_global.withSourceRepo(source_repository); } diff --git a/bazel/cc_import.bzl b/bazel/cc_import.bzl index fc6ef727..7c0e8de5 100644 --- a/bazel/cc_import.bzl +++ b/bazel/cc_import.bzl @@ -1,5 +1,4 @@ -load("@rules_cc//cc:defs.bzl", "cc_library", _cc_import = "cc_import") -load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load("@bazel_tools//tools/build_defs/cc:cc_import.bzl", _cc_import = "cc_import") load(":patchelf.bzl", "patchelf") def _cc_import_runfiles_impl(ctx): @@ -71,7 +70,7 @@ def cc_import( interface_library = interface_library, visibility = visibility, ) - cc_library( + native.cc_library( name = name, data = data, deps = deps + [name + ".cc_import"], diff --git a/bazel/http_aea_archive.bzl b/bazel/http_aea_archive.bzl deleted file mode 100644 index 439a6a41..00000000 --- a/bazel/http_aea_archive.bzl +++ /dev/null @@ -1,164 +0,0 @@ -"""Repository rule for downloading and extracting Apple Encrypted Archive assets on macOS and Linux AMD64/ARM64.""" - -load("@bazel_lib//lib:repo_utils.bzl", "repo_utils") - -def _fail_result(step, result): - if result.return_code != 0: - fail("%s failed with exit code %d\nstdout:\n%s\nstderr:\n%s" % ( - step, - result.return_code, - result.stdout, - result.stderr, - )) - -def _execute(rctx, step, argv, quiet = True): - result = rctx.execute([str(arg) for arg in argv], quiet = quiet) - _fail_result(step, result) - return result - -def _host_tools(rctx): - platform = repo_utils.platform(rctx) - if platform not in ["darwin_amd64", "darwin_arm64", "linux_amd64", "linux_arm64"]: - fail("http_aea_archive supports macOS and Linux AMD64/ARM64 repository hosts; got %s/%s" % (rctx.os.name, rctx.os.arch)) - ipsw = Label("@ipsw_{}//:ipsw".format(platform)) if platform.startswith("linux_") else None - sevenzip = Label("@sevenzip_{}//:7zz".format(platform)) - return (ipsw, sevenzip) - -def _single_restore_dmg(restore_dirs): - dmgs = [] - entries = [] - for restore_dir in restore_dirs: - if not restore_dir.exists: - continue - for entry in restore_dir.readdir(): - entries.append(str(entry)) - if not entry.is_dir and entry.basename.endswith(".dmg"): - dmgs.append(entry) - - if len(dmgs) != 1: - fail("expected exactly one DMG under AssetData/Restore, got %d; entries: %s" % ( - len(dmgs), - ", ".join(sorted(entries)), - )) - - return dmgs[0] - -def _linux_restore_dirs(rctx, ipsw, aar): - # Create an explicit empty config. It avoids loading the user's ipsw configuration. - rctx.file("ipsw-config.yaml", "{}\n") - _execute(rctx, "ipsw ota extract", [ - rctx.path(ipsw), - "--config", - rctx.path("ipsw-config.yaml"), - "ota", - "extract", - aar, - "--key-val", - rctx.attr.archive_decryption_key, - "--pattern", - "AssetData/Restore/.*\\.dmg$", # pattern to match DMG files under AssetData/Restore - "--output", - rctx.path("ipsw-output"), - "--confirm", - "--no-color", - ]) - output = rctx.path("ipsw-output") - restore_dirs = [] - if output.exists: - # ipsw prefixes extracted paths with an asset-specific directory. - for entry in output.readdir(): - if entry.is_dir: - restore_dirs.append(entry.get_child("AssetData", "Restore")) - return restore_dirs - -def _http_aea_archive_impl(rctx): - ipsw, sevenzip = _host_tools(rctx) - - urls = [] - if rctx.attr.url: - urls.append(rctx.attr.url) - urls.extend(rctx.attr.urls) - if not urls: - fail("http_aea_archive requires url or urls") - - if not rctx.attr.sha256 and not rctx.attr.integrity: - fail("http_aea_archive requires sha256 or integrity for reproducible downloads") - if not rctx.attr.archive_decryption_key: - fail("archive_decryption_key is required") - - aar = rctx.path("asset.aar") # path to the downloaded Apple Encrypted Archive (AAR) file - download_kwargs = { - "canonical_id": rctx.attr.canonical_id, - "integrity": rctx.attr.integrity, - "output": aar, - "sha256": rctx.attr.sha256, - "url": urls, - } - if rctx.attr.netrc: - download_kwargs["auth"] = rctx.use_netrc( - rctx.read_netrc(rctx.attr.netrc), - urls, - rctx.attr.auth_patterns, - ) - - rctx.download(**download_kwargs) - - if ipsw: - restore_dirs = _linux_restore_dirs(rctx, ipsw, aar) - else: - _execute(rctx, "aa patch", [ - "/usr/bin/aa", - "patch", - "-i", - aar, - "-key-value", - "base64:%s" % rctx.attr.archive_decryption_key, - "-src", - "/var/empty", - "-dst", - rctx.path("."), - ]) - restore_dirs = [rctx.path("AssetData/Restore")] - dmg = _single_restore_dmg(restore_dirs) - - extract_args = [ - rctx.path(sevenzip), - "x", - "-y", - dmg, - ] - extract_args.extend(rctx.attr.includes) - _execute(rctx, "7z extract dmg", extract_args) - - # Central cleanup of temporary files and directories after extraction. - # If absent, rctx.delete will handle it gracefully. - rctx.delete("asset.aar") - rctx.delete("AssetData") - rctx.delete("ipsw-output") - rctx.delete("ipsw-config.yaml") - - rctx.template("BUILD.bazel", rctx.attr.build_file) - - return None - -http_aea_archive = repository_rule( - implementation = _http_aea_archive_impl, - attrs = { - "url": attr.string(), - "urls": attr.string_list(), - "archive_decryption_key": attr.string(mandatory = True), - "sha256": attr.string(), - "integrity": attr.string(), - "canonical_id": attr.string(), - "netrc": attr.string(), - "auth_patterns": attr.string_dict(), - "build_file": attr.label( - allow_single_file = True, - ), - "includes": attr.string_list( - default = [], - doc = "Patterns to pass to 7z for selective extraction from the DMG. If empty, extracts everything.", - ), - }, - doc = "Downloads and extracts an AEA-wrapped AppleArchive asset on macOS and Linux AMD64/ARM64.", -) diff --git a/bazel/http_deb_archive.bzl b/bazel/http_deb_archive.bzl index 968001d6..91b52f5c 100644 --- a/bazel/http_deb_archive.bzl +++ b/bazel/http_deb_archive.bzl @@ -1,253 +1,46 @@ -load("@bazel_lib//lib:repo_utils.bzl", "repo_utils") -load( - "@bazel_tools//tools/build_defs/repo:cache.bzl", - "DEFAULT_CANONICAL_ID_ENV", - "get_default_canonical_id", -) load( "@bazel_tools//tools/build_defs/repo:utils.bzl", "get_auth", "patch", - "update_attrs", "workspace_and_buildfile", ) -def _get_source_urls(rctx): - if not rctx.attr.url and not rctx.attr.urls: - fail("At least one of url and urls must be provided") - - source_urls = [] - if rctx.attr.urls: - source_urls = rctx.attr.urls - if rctx.attr.url: - source_urls = [rctx.attr.url] + source_urls - return source_urls - -def _prefixed_pattern(strip_prefix, pattern): - if not strip_prefix: - return pattern - if not pattern: - return strip_prefix - if pattern == strip_prefix or pattern.startswith(strip_prefix + "/"): - return pattern - return strip_prefix + "/" + pattern - -def _downloaded_archive_name(rctx, source_urls): - if rctx.attr.type: - return ".downloaded.archive." + rctx.attr.type - - basename = source_urls[0].split("?", 1)[0].split("/")[-1] - if basename: - return ".downloaded." + basename - return ".downloaded.archive" - -def _placeholder_dir_from_exclude(pattern): - if pattern.endswith("/*"): - return pattern[:-2].strip("/") - return None - -def _ensure_placeholder_dirs(rctx, add_prefix): - for exclude in rctx.attr.excludes: - directory = _placeholder_dir_from_exclude(exclude) - if not directory: - continue - if add_prefix: - directory = add_prefix + "/" + directory - keep_file = directory + "/.http_deb_archive.keep" - rctx.file(keep_file, "") - rctx.delete(keep_file) - -def _host_bsdtar_label(rctx): - platform = repo_utils.platform(rctx) - binary = "tar.exe" if platform.startswith("windows_") else "tar" - return Label("@bsd_tar_toolchains_{}//:{}".format(platform, binary)) - -def _update_integrity_attrs(rctx, attrs, archive_info, remote_files_info, remote_patches_info): - integrity_override = {} - if not rctx.attr.sha256 and not rctx.attr.integrity: - integrity_override["integrity"] = archive_info.integrity - - remote_file_integrity = {path: info.integrity for path, info in remote_files_info.items()} - if rctx.attr.remote_file_integrity != remote_file_integrity: - integrity_override["remote_file_integrity"] = remote_file_integrity - - # TODO(cerisier): Remove this "if" when we no longer support bazel < 8.5.0 - if remote_patches_info: - remote_patch_integrity = {url: info.integrity for url, info in remote_patches_info.items()} - if rctx.attr.remote_patches != remote_patch_integrity: - integrity_override["remote_patches"] = remote_patch_integrity - - if not integrity_override: - return rctx.repo_metadata(reproducible = True) - - return rctx.repo_metadata( - attrs_for_reproducibility = update_attrs(rctx.attr, attrs.keys(), integrity_override), - ) - -# TODO(cerisier): Remove when we no longer support bazel < 8.5.0 -def symlink_files(ctx): - # type: (repository_ctx) -> None - """Utility function for symlinking local files. - - This is intended to be used in the implementation function of a repository rule. It assumes the - parameter `files` is present in `ctx.attr`. - - Existing files will be overwritten. - - Args: - ctx: The repository context of the repository rule calling this utility - function. - """ - for path, label in ctx.attr.files.items(): - src_path = ctx.path(label) - - # On Windows `ctx.symlink` may be implemented as a copy, so the file MUST be watched - ctx.watch(src_path) - if not src_path.exists: - fail("Input %s does not exist" % label) - if ctx.path(path).exists: - ctx.delete(path) - ctx.symlink(src_path, path) - -# TODO(cerisier): Remove when we no longer support bazel < 8.5.0 -def download_remote_files(ctx, auth = None): - """Utility function for downloading remote files. - - This rule is intended to be used in the implementation function of - a repository rule. It assumes the parameters `remote_file_urls` and - `remote_file_integrity` to be present in `ctx.attr`. - - Existing files will be overwritten. - - Args: - ctx: The repository context of the repository rule calling this utility - function. - auth: An optional dict specifying authentication information for some of the URLs. - - Returns: - dict mapping file paths to a download info. - """ - pending = { - path: ctx.download( - remote_file_urls, - path, - canonical_id = ctx.attr.canonical_id, - auth = get_auth(ctx, remote_file_urls) if auth == None else auth, - integrity = ctx.attr.remote_file_integrity.get(path, ""), - block = False, - # Overlaid files may be shell scripts. - executable = True, - ) - for path, remote_file_urls in ctx.attr.remote_file_urls.items() - } - - # Wait until the requests are done - return {path: token.wait() for path, token in pending.items()} - def _http_deb_archive_impl(rctx): - source_urls = _get_source_urls(rctx) - if rctx.attr.build_file and rctx.attr.build_file_content: fail("Only one of build_file and build_file_content can be provided.") - - archive = _downloaded_archive_name(rctx, source_urls) - archive_info = rctx.download( - source_urls, - archive, - rctx.attr.sha256, - canonical_id = rctx.attr.canonical_id or get_default_canonical_id(rctx, source_urls), - auth = get_auth(rctx, source_urls), - integrity = rctx.attr.integrity, + download_info = rctx.download_and_extract( + url = rctx.attr.urls, + output = "tmp", + sha256 = rctx.attr.sha256, + type = "deb", + stripPrefix = "", + canonical_id = " ".join(rctx.attr.urls), + auth = get_auth(rctx, rctx.attr.urls), ) - host_bsdtar = _host_bsdtar_label(rctx) - res = rctx.execute([ - rctx.path(host_bsdtar), - "-xf", - archive, - "--include=data.tar.*", - ]) for ext in ["gz", "xz", "zst"]: - data = "data.tar.{}".format(ext) + data = "tmp/data.tar.{}".format(ext) if rctx.path(data).exists: - archive = data + rctx.extract( + archive = data, + output = "", + stripPrefix = rctx.attr.strip_prefix, + ) + rctx.delete("tmp") break - - strip_prefix = rctx.attr.strip_prefix.strip("/") - includes = [_prefixed_pattern(strip_prefix, include) for include in rctx.attr.includes] - excludes = [_prefixed_pattern(strip_prefix, exclude) for exclude in rctx.attr.excludes] - - if strip_prefix and not includes: - includes = [strip_prefix, strip_prefix + "/*"] - - args = [] - for include in includes: - args.extend(["--include", include]) - for exclude in excludes: - args.extend(["--exclude", exclude]) - if strip_prefix: - args.extend(["--strip-components", str(len(strip_prefix.split("/")))]) - args.extend(rctx.attr.bsdtar_extra_args) - - add_prefix = rctx.attr.add_prefix.strip("/") - keep_file = None - if add_prefix: - keep_file = add_prefix + "/.http_deb_archive.keep" - rctx.file(keep_file, "") - args.extend(["-C", add_prefix]) - - args.extend(["-xf", archive]) - res = rctx.execute([rctx.path(host_bsdtar)] + args) - if res.return_code != 0: - fail("Failed to extract archive: {}\n{}".format(res.stderr, res.stdout)) - - if keep_file: - rctx.delete(keep_file) - _ensure_placeholder_dirs(rctx, add_prefix) - rctx.delete(archive) - workspace_and_buildfile(rctx) - remote_files_info = download_remote_files(rctx) - remote_patches_info = patch(rctx) - symlink_files(rctx) - - return _update_integrity_attrs(rctx, _http_deb_archive_attrs, archive_info, remote_files_info, remote_patches_info) - -_http_deb_archive_attrs = { - "url": attr.string(), - "urls": attr.string_list(), - "sha256": attr.string(), - "integrity": attr.string(), - "netrc": attr.string(), - "auth_patterns": attr.string_dict(), - "canonical_id": attr.string(), - "strip_prefix": attr.string(), - "add_prefix": attr.string(default = ""), - "files": attr.string_keyed_label_dict(default = {}), - "type": attr.string(), - "patches": attr.label_list(default = []), - "remote_file_urls": attr.string_list_dict(default = {}), - "remote_file_integrity": attr.string_dict(default = {}), - "remote_module_file_urls": attr.string_list(default = []), - "remote_module_file_integrity": attr.string(default = ""), - "remote_patches": attr.string_dict(default = {}), - "remote_patch_strip": attr.int(default = 0), - "patch_tool": attr.string(default = ""), - "patch_args": attr.string_list(default = []), - "patch_strip": attr.int(default = 0), - "patch_cmds": attr.string_list(default = []), - "patch_cmds_win": attr.string_list(default = []), - "build_file": attr.label(allow_single_file = True), - "build_file_content": attr.string(), - "workspace_file": attr.label(), - "workspace_file_content": attr.string(), - "includes": attr.string_list(default = []), - "excludes": attr.string_list(default = []), - "bsdtar_extra_args": attr.string_list(default = []), -} + patch(rctx) http_deb_archive = repository_rule( - implementation = _http_deb_archive_impl, - attrs = _http_deb_archive_attrs, - environ = [DEFAULT_CANONICAL_ID_ENV], + _http_deb_archive_impl, + attrs = { + "urls": attr.string_list(mandatory = True), + "sha256": attr.string(mandatory = True), + "strip_prefix": attr.string(), + "build_file": attr.label(allow_single_file = True), + "build_file_content": attr.string(), + "patches": attr.label_list(allow_files = True), + "workspace_file": attr.label(allow_single_file = True), + "workspace_file_content": attr.string(), + }, ) diff --git a/bazel/patchelf.bzl b/bazel/patchelf.bzl index bafaa6ec..ac79033c 100644 --- a/bazel/patchelf.bzl +++ b/bazel/patchelf.bzl @@ -1,78 +1,76 @@ def _patchelf_impl(ctx): - output_name = ctx.attr.soname or ctx.file.src.basename - output = ctx.actions.declare_file("{}/{}".format(ctx.label.name, output_name)) - - inputs = [ctx.file.src] - args = ctx.actions.args() - - # --- soname ----------------------------------------------------------- + output_name = ctx.file.src.basename if ctx.attr.soname: - args.add("--set-soname", ctx.attr.soname) + output_name = ctx.attr.soname + output = ctx.actions.declare_file("{}/{}".format(ctx.attr.name, output_name)) - # --- DT_NEEDED -------------------------------------------------------- - args.add_all(ctx.attr.remove_needed, before_each = "--remove-needed") - args.add_all(ctx.attr.add_needed, before_each = "--add-needed") - for old, new in ctx.attr.replace_needed.items(): - args.add("--replace-needed") - args.add(old) - args.add(new) - - # --- RPATH / RUNPATH -------------------------------------------------- - # patchelf only accepts one rpath operation per invocation. - rpath_ops = [ - bool(ctx.attr.set_rpath), - bool(ctx.attr.add_rpath), - ctx.attr.remove_rpath, + commands = [ + "set -e", + 'cp -f "$2" "$3"', + 'chmod +w "$3"', ] - if len([op for op in rpath_ops if op]) > 1: - fail("patchelf: set_rpath, add_rpath and remove_rpath are mutually exclusive") + + if ctx.attr.soname: + commands.append(""" "$1" --set-soname '{}' "$3" """.format(ctx.attr.soname)) + if ctx.attr.remove_needed: + for v in ctx.attr.remove_needed: + commands.append(""" "$1" --remove-needed '{}' "$3" """.format(v)) + if ctx.attr.add_needed: + for v in ctx.attr.add_needed: + commands.append(""" "$1" --add-needed '{}' "$3" """.format(v)) + if ctx.attr.replace_needed: + for k, v in ctx.attr.replace_needed.items(): + commands.append(""" "$1" --replace-needed '{}' '{}' "$3" """.format(k, v)) if ctx.attr.set_rpath: - args.add("--force-rpath") - args.add("--set-rpath", ctx.attr.set_rpath) - elif ctx.attr.add_rpath: - args.add("--force-rpath") - args.add("--add-rpath", ":".join(ctx.attr.add_rpath)) - elif ctx.attr.remove_rpath: - args.add("--remove-rpath") + commands.append(""" "$1" --set-rpath '{}' "$3" """.format(ctx.attr.set_rpath)) + if ctx.attr.add_rpath: + for path in ctx.attr.add_rpath: + commands.append(""" "$1" --add-rpath '{}' "$3" """.format(path)) + if ctx.attr.remove_rpath: + for path in ctx.attr.remove_rpath: + commands.append(""" "$1" --remove-rpath '{}' "$3" """.format(path)) - # --- dynamic symbol renames ------------------------------------------ + renamed_syms = ctx.actions.declare_file("{}.rename.txt".format(ctx.label.name)) if ctx.attr.rename_dynamic_symbols: - symbols = ctx.actions.declare_file(ctx.label.name + ".symbols") - ctx.actions.write(symbols, "".join([ - "{} {}\n".format(old, new) - for old, new in ctx.attr.rename_dynamic_symbols.items() - ])) - inputs.append(symbols) - args.add("--rename-dynamic-symbols", symbols) + content = "\n".join([ + "{} {}".format(k, v) + for k, v in ctx.attr.rename_dynamic_symbols.items() + ]) + ctx.actions.write(renamed_syms, content) + commands.append(""" "$1" --rename-dynamic-symbols '{}' "$3" """.format(renamed_syms.path)) + else: + ctx.actions.write(renamed_syms, "") - args.add("--output", output) - args.add(ctx.file.src) - - ctx.actions.run( - executable = ctx.executable._patchelf, - inputs = inputs, + ctx.actions.run_shell( + inputs = [ctx.file.src, renamed_syms], outputs = [output], - arguments = [args], - mnemonic = "Patchelf", - progress_message = "Patching ELF %{output}", + arguments = [ctx.executable._patchelf.path, ctx.file.src.path, output.path], + command = "\n".join(commands), + tools = [ctx.executable._patchelf], ) - return [DefaultInfo(files = depset([output]))] + + return [ + DefaultInfo( + files = depset([output]), + ), + ] patchelf = rule( implementation = _patchelf_impl, attrs = { "src": attr.label(allow_single_file = True, mandatory = True), - "soname": attr.string(doc = "Set DT_SONAME; also used as the output file name."), + "soname": attr.string(), "add_needed": attr.string_list(), "remove_needed": attr.string_list(), - "replace_needed": attr.string_dict(doc = "old -> new DT_NEEDED replacements."), - "rename_dynamic_symbols": attr.string_dict(doc = "old -> new symbol renames."), + "replace_needed": attr.string_dict(), + "rename_dynamic_symbols": attr.string_dict(), "set_rpath": attr.string(), - "add_rpath": attr.string_list(doc = "Entries appended to the existing RPATH (joined with ':')."), - "remove_rpath": attr.bool(default = False, doc = "Strip the DT_RPATH/DT_RUNPATH entry entirely."), + "add_rpath": attr.string_list(), + "remove_rpath": attr.string_list(), "_patchelf": attr.label( default = "@patchelf", + allow_single_file = True, executable = True, cfg = "exec", ), diff --git a/bazel/runfiles_init.c b/bazel/runfiles_init.c deleted file mode 100644 index 770a4b91..00000000 --- a/bazel/runfiles_init.c +++ /dev/null @@ -1,15 +0,0 @@ -extern void _bazel_runfiles_init_impl(int argc, char **argv, char **envp); - -// Preserve the existing startup ABI. Registration happens in C; -// runfiles discovery and initialization remain in Zig. -// glibc and Darwin's dyld pass argc/argv/envp to constructors; musl does not. -// The original Zig initializer already made this assumption to read argv[0]. -// -// Let C emit the platform's constructor registration. Zig's incremental ELF -// linker does not yet handle .init_array's section type, flags, and size correctly. -// TODO: Revert this workaround once Zig issue #37453 is fixed: -// https://codeberg.org/ziglang/zig/issues/37453 -__attribute__((constructor)) -void _bazel_runfiles_init(int argc, char **argv, char **envp) { - _bazel_runfiles_init_impl(argc, argv, envp); -} diff --git a/bazel/stamp/BUILD.bazel b/bazel/stamp/BUILD.bazel deleted file mode 100644 index 387f189c..00000000 --- a/bazel/stamp/BUILD.bazel +++ /dev/null @@ -1,37 +0,0 @@ -load("@bazel_lib//lib:run_binary.bzl", "run_binary") -load("@rules_cc//cc:cc_binary.bzl", "cc_binary") -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") - -run_binary( - name = "stamp_cc_src", - outs = ["stamp.c"], - args = [ - "$(location stamp.c)", - "bazel-out/stable-status.txt", - "bazel-out/volatile-status.txt", - ], - stamp = 1, - tool = ":genstamp", - visibility = ["//visibility:private"], -) - -cc_binary( - name = "genstamp", - srcs = ["genstamp.c"], - visibility = ["//visibility:private"], -) - -cc_library( - name = "cc", - srcs = [":stamp_cc_src"], - visibility = ["//visibility:public"], -) - -zig_library( - name = "zig", - import_name = "bazel/stamp", - main = "stamp.zig", - visibility = ["//visibility:public"], - deps = [":cc"], -) diff --git a/bazel/stamp/genstamp.c b/bazel/stamp/genstamp.c deleted file mode 100644 index 4fba8e6f..00000000 --- a/bazel/stamp/genstamp.c +++ /dev/null @@ -1,121 +0,0 @@ -#include -#include -#include - -static void write_escaped(FILE *out, const char *value) { - for (const unsigned char *p = (const unsigned char *)value; *p != '\0'; ++p) { - switch (*p) { - case '\\': - fputs("\\\\", out); - break; - case '"': - fputs("\\\"", out); - break; - case '\n': - fputs("\\n", out); - break; - case '\r': - fputs("\\r", out); - break; - case '\t': - fputs("\\t", out); - break; - default: - fputc(*p, out); - break; - } - } -} - -static int process_stream(FILE *out, FILE *in, const char *path) { - char *line = NULL; - size_t cap = 0; - ssize_t len; - - while ((len = getline(&line, &cap, in)) != -1) { - while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) { - line[--len] = '\0'; - } - - if (len == 0) { - continue; - } - - char *space = strchr(line, ' '); - if (space == NULL) { - continue; - } - - *space = '\0'; - const char *name = line; - const char *value = space + 1; - - if (*name == '\0') { - continue; - } - - fputs("const char ", out); - fputs(name, out); - fputs("[] = \"", out); - write_escaped(out, value); - fputs("\";\n", out); - } - - if (ferror(in)) { - fprintf(stderr, "genstamp: failed while reading %s\n", path); - free(line); - return 1; - } - - free(line); - return 0; -} - -int main(int argc, char **argv) { - if (argc < 3) { - fprintf(stderr, "usage: %s [ ...]\n", argv[0]); - return 1; - } - - const char *output_path = argv[1]; - FILE *out = fopen(output_path, "w"); - if (out == NULL) { - fprintf(stderr, "genstamp: failed to open %s for writing\n", output_path); - return 1; - } - - for (int i = 2; i < argc; ++i) { - const char *path = argv[i]; - FILE *in = fopen(path, "r"); - if (in == NULL) { - fprintf(stderr, "genstamp: failed to open %s\n", path); - fclose(out); - return 1; - } - - int rc = process_stream(out, in, path); - if (fclose(in) != 0) { - fprintf(stderr, "genstamp: failed to close %s\n", path); - fclose(out); - return 1; - } - - if (rc != 0) { - fclose(out); - return rc; - } - } - - if (fflush(out) != 0) { - fprintf(stderr, "genstamp: failed to flush %s\n", output_path); - fclose(out); - return 1; - } - - if (fclose(out) != 0) { - fprintf(stderr, "genstamp: failed to close %s\n", output_path); - return 1; - } - - return 0; -} \ No newline at end of file diff --git a/bazel/stamp/stamp.zig b/bazel/stamp/stamp.zig deleted file mode 100644 index 0e60e2c7..00000000 --- a/bazel/stamp/stamp.zig +++ /dev/null @@ -1,5 +0,0 @@ -const std = @import("std"); - -pub fn stamp(comptime name: []const u8) []const u8 { - return std.mem.span(@extern([*c]const u8, .{ .name = name })); -} diff --git a/bazel/swig.bzl b/bazel/swig.bzl index cd4f994f..d81310fb 100644 --- a/bazel/swig.bzl +++ b/bazel/swig.bzl @@ -1,7 +1,6 @@ load("@rules_cc//cc:action_names.bzl", "C_COMPILE_ACTION_NAME") load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_cc//cc/common:cc_common.bzl", "cc_common") -load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load("@rules_cc//cc:defs.bzl", "CcInfo", "cc_common") load("@rules_cc//cc:find_cc_toolchain.bzl", "find_cc_toolchain", "use_cc_toolchain") def _swig_cc_library_impl(ctx): @@ -61,7 +60,6 @@ def _swig_cc_library_impl(ctx): ] args.add("-o", output_cpp) args.add("-w-305") - args.add("-w-524") args.add(ctx.file.interface) inputs = depset(ctx.attr.srcs, transitive = [ diff --git a/bazel/zig_index.json b/bazel/zig_index.json index b8cf9b9e..da8c0455 100644 --- a/bazel/zig_index.json +++ b/bazel/zig_index.json @@ -1,518 +1,133 @@ { - "0.17.0": { - "version": "0.17.0", - "date": "2026-10-01", - "docs": "https://ziglang.org/documentation/0.17.0/", - "stdDocs": "https://ziglang.org/documentation/0.17.0/std/", - "notes": "https://ziglang.org/download/0.17.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.17.0/zig-0.17.0.tar.xz", - "shasum": "b6c7f1728f043700d6529bac980800792f824256a9d2f1839b3d62beed0b8abd", - "size": "23049960" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.17.0/zig-bootstrap-0.17.0.tar.xz", - "shasum": "1e9e9b8e3c753b35dfb1d9ea48f097579fadbd3e5c1e724ab3fc5bce05477b71", - "size": "57030388" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-macos-0.17.0.tar.xz", - "shasum": "4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6", - "size": "59317572" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-macos-0.17.0.tar.xz", - "shasum": "b607e9b9234790a008116ae5bdb71c6243b84b9fb42a53a9e70fde41c06c536a", - "size": "53985220" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-linux-0.17.0.tar.xz", - "shasum": "1cbe9df9f27e6b78d14ccbca43b6703a404ef79ef1c463de901d7f088d4e2026", - "size": "57332648" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-linux-0.17.0.tar.xz", - "shasum": "9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8", - "size": "52877280" - }, - "arm-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-arm-linux-0.17.0.tar.xz", - "shasum": "53f0045cdef7ba06da70a12b2ef654e4c5283b0139bc441aa7d84f364b46de43", - "size": "53841896" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-linux-0.17.0.tar.xz", - "shasum": "18ff032ac6cecf746a84259f6463264f13160ff88bc3b80722ce06f9479b82ed", - "size": "57116976" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-linux-0.17.0.tar.xz", - "shasum": "731e2887e66fff41fdc34f7fd9d8afa814d86d7426f5827805f8854b2977684b", - "size": "56990076" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86-linux-0.17.0.tar.xz", - "shasum": "55e39e175cd5b3098afc29ec47b2590134f5b874175369fb9ae7b8e836d7531a", - "size": "59823964" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-loongarch64-linux-0.17.0.tar.xz", - "shasum": "5a6000b866201c3b3820b563fcfd29641185393e7d35039e50235db418d8367c", - "size": "54257324" - }, - "s390x-linux": { - "tarball": "https://ziglang.org/download/0.17.0/zig-s390x-linux-0.17.0.tar.xz", - "shasum": "13905f11989b65566c544d191d4bc5bd5d6f858e303208cf1d2f090e5d91b512", - "size": "57136832" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-windows-0.17.0.zip", - "shasum": "b5663f69581dcf391293fbf16c06cb80d81d806545ce618b4d0bab7f0eb8c428", - "size": "100268137" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-windows-0.17.0.zip", - "shasum": "0a59d91fa1cb40cf068e9b0954434ce973500c7a2ea749f1e01af62cdab52d26", - "size": "95992853" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86-windows-0.17.0.zip", - "shasum": "c7b83507549c7bcd42cd58c4910b47037f4150ed7778e11b3684f130620969d1", - "size": "101891323" - }, - "aarch64-freebsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-freebsd-0.17.0.tar.xz", - "shasum": "3cf53d07b3587a26727b196a6eb7918704b907d951520daa68e4e1820fc6f48e", - "size": "52860244" - }, - "arm-freebsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-arm-freebsd-0.17.0.tar.xz", - "shasum": "e0c3ecf21c186a48d3b8e8d0b4459f9dc67ff8024445401506a7e970782da5f8", - "size": "54273908" - }, - "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-powerpc64le-freebsd-0.17.0.tar.xz", - "shasum": "94e127c2a4beb5151a8c58e8af4b82dc3649ed95f343ffbbfa381e2afba50234", - "size": "57027652" - }, - "riscv64-freebsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-freebsd-0.17.0.tar.xz", - "shasum": "4ab73eb01a0cb910dfa23d8efc88d38dcdd5ebb2174537cf531738ba520354f1", - "size": "57199160" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-freebsd-0.17.0.tar.xz", - "shasum": "a314f1cc92d098bd39caf9e5607a0de2a8fd0d00064776cb3fd5b3bee38a67c9", - "size": "57468884" - }, - "aarch64-netbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-netbsd-0.17.0.tar.xz", - "shasum": "421fefe3910dedc8b8a6ae9a8905ea9200ecb0760499bcf0f56af7a5ba4930b5", - "size": "52769844" - }, - "arm-netbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-arm-netbsd-0.17.0.tar.xz", - "shasum": "2537d66f6dbd57438355d100180d09f19640c98895d4052e4e2994a02cac320a", - "size": "55345456" - }, - "riscv64-netbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-netbsd-0.17.0.tar.xz", - "shasum": "687c92430ab3e7a5329e941e95f0ec4368d8d8ac0a4048f61c2dd714e675ca12", - "size": "57157568" - }, - "x86-netbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86-netbsd-0.17.0.tar.xz", - "shasum": "2025c508390cc78d9c469675d2a0602f2a975c92ec197af51faf09a565b27ee3", - "size": "60380332" - }, - "x86_64-netbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-netbsd-0.17.0.tar.xz", - "shasum": "10f71cc89efbafc6e22447f23c44047d53c5d2061c82b1c32efd1ef55ea2d9b6", - "size": "57363300" - }, - "aarch64-openbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-aarch64-openbsd-0.17.0.tar.xz", - "shasum": "1cf6c33a0cdd9a58a0b40c7ee86d8de9c190ed7bd8d47ad61687908955ecac10", - "size": "53265892" - }, - "arm-openbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-arm-openbsd-0.17.0.tar.xz", - "shasum": "068d5200b027541c2b0a2e16efc3fb367f1fae78f9c3954e57512868c9a0ecc2", - "size": "54024240" - }, - "riscv64-openbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-riscv64-openbsd-0.17.0.tar.xz", - "shasum": "4858839a9e4d96fb50d6e4894d10270f5377a087b2929d238b1a12b8293e274a", - "size": "57490512" - }, - "x86_64-openbsd": { - "tarball": "https://ziglang.org/download/0.17.0/zig-x86_64-openbsd-0.17.0.tar.xz", - "shasum": "5e9606fc322ef4c346c199e162d26e4c293358a36ea3ca72e877d42ef71e24be", - "size": "58612444" - } - }, - "0.17.0-dev.1936+5a625d5f3": { - "version": "0.17.0-dev.1936+5a625d5f3", - "docs": "https://ziglang.org/documentation/master/", - "stdDocs": "https://ziglang.org/documentation/master/std/", - "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", - "shasum": "8ab9f0d015abb75c0c9c039f7191711a7a2fc72c984458f553926725794791dd", - "size": "59162280" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1936+5a625d5f3.tar.xz", - "shasum": "e08d4485c96d2d31a8ce3ae80a58badec202920b675149ad6b0c3b974ec7c005", - "size": "53867184" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", - "shasum": "38620c06ca57b175f7dab8f65c8001f9a67b0ff1987c344d4e64ccb72b57174f", - "size": "57180268" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1936+5a625d5f3.tar.xz", - "shasum": "6c559ff4bca3e84fbbf7b6ad8521ce70bb3e460371b3d8803112353fb8ce70cc", - "size": "52741876" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1936+5a625d5f3.zip", - "shasum": "e0d62a00e067de41a0a34af2af73694e00692e045ca244748965a7b3729633a1", - "size": "100793891" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1936+5a625d5f3.zip", - "shasum": "e75f340afee0772fe4daed61d92b999534c8460601175b66fe890703cf752850", - "size": "96526396" - } - }, - "0.17.0-dev.1509+bb296ab9b": { - "version": "0.17.0-dev.1509+bb296ab9b", - "date": "2026-07-29", - "docs": "https://ziglang.org/documentation/master/", - "stdDocs": "https://ziglang.org/documentation/master/std/", - "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", - "shasum": "087aad38fead6c0ba7a52b5db3c405e63708c4765eda643d4144b9e4933d45e5", - "size": "59064872" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.17.0-dev.1509+bb296ab9b.tar.xz", - "shasum": "24bd83c1d435b8ab6192f58afcc22f4e5252077a938ffbc35eaa4ca97c5be709", - "size": "53773824" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", - "shasum": "48cc865b8b410ec84eaa97e50c2bd7a657871802ce3aaaf04dd1da2294d4b28a", - "size": "57056988" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.17.0-dev.1509+bb296ab9b.tar.xz", - "shasum": "7f17e09b675cfe39805c5b551b251151f7750b9df8a168538007890c1c00dc8e", - "size": "52649416" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.17.0-dev.1509+bb296ab9b.zip", - "shasum": "a4f537761ca1f5274c9bb0e7da598c6e82205a4ea78f329690a26eade555d7e8", - "size": "100743514" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.17.0-dev.1509+bb296ab9b.zip", - "shasum": "1fa7a2b5f5442628287fead0f7451b8cb93ed2ca5dd6c79f8026fc8ca83580d2", - "size": "96467368" - } - }, "master": { - "version": "0.16.0-dev.3153+d6f43caad", - "date": "2026-04-12", + "version": "0.16.0-dev.2611+f996d2866", + "date": "2026-02-14", "docs": "https://ziglang.org/documentation/master/", "stdDocs": "https://ziglang.org/documentation/master/std/", "src": { - "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "cb598f754d00a71af7586d437f490056b084eb6e54587fc2bb9f55148757fd49", - "size": "22477252" + "tarball": "https://ziglang.org/builds/zig-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "cf085b0b63b3520cda55aa7d98a8c86b28a5c9739bffe4a0253c58880ded4c7b", + "size": "22442124" }, "bootstrap": { - "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "a2ccc95aaed569b3c25d8693f01187144f09e08a2a9b828bad3e5f2c19a4c2bc", - "size": "55240588" + "tarball": "https://ziglang.org/builds/zig-bootstrap-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "b87fcb4aa966c9b7f07ceca302b15eb587936938a65b5861188200d5b3616c0d", + "size": "55205844" }, "x86_64-macos": { - "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "0251a8414f98513a33a7a7bd1f6cc13fb59f391168a742eee47b8c685e9827a0", - "size": "57381556" + "tarball": "https://ziglang.org/builds/zig-x86_64-macos-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "5ba0c08bd2e9c74d18c6ea25483226a8f8922f4e50d09d9219be842894b41334", + "size": "57619112" }, "aarch64-macos": { - "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "9121e3f03515a3434ef9f362f8c541540968316d33ed6485b2d8a31dc7332fe6", - "size": "52217412" + "tarball": "https://ziglang.org/builds/zig-aarch64-macos-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "1eb526bb948c1c1bffdb5be318c044fc0216457be28b0d92aa2d92294b9d32c9", + "size": "52470216" }, "x86_64-linux": { - "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "afb3016a94a8b5608d364e58ec7b01111af775a020d5f93f69eaebde2053fe77", - "size": "55445192" + "tarball": "https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "0188a70a3f84fe1918ac391a8b64bd241eb4c028c44ea023e853743cbecd3009", + "size": "55505692" }, "aarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "e46037f7747492b405a42d40bb9bf3ca3f28c71cf9c1ccdf165a4de74cb484bb", - "size": "51193908" + "tarball": "https://ziglang.org/builds/zig-aarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "d4219908cded6efb97a7326e4159f82607e3956d6d56efb1580187bf0872603e", + "size": "51276904" }, "arm-linux": { - "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "aec322b170a88c998c1c7b59c731e808543310008960b1654caab569567e7c95", - "size": "52093456" + "tarball": "https://ziglang.org/builds/zig-arm-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "b0b809695bee67798ce9c956be24d850beab5b2ca4f3bf3fc65552baea0214bb", + "size": "52142540" }, "riscv64-linux": { - "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "ad83132048c075edd1031a1e2fc9b29921e342f6b1d58142923e60e203a5b53a", - "size": "55303468" + "tarball": "https://ziglang.org/builds/zig-riscv64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "9244e44969386bd35477079b42db7c7dd5b636ed6ddd21cbcb35debf1db5f285", + "size": "55318440" }, "powerpc64le-linux": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "1866a073fdcd0773397d345b21f98221a20126f7e1cec298c16ff0f4ca3fb78f", - "size": "55241328" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "fd9f8b35403ac5c78d40c979a0f467413ddbb0294c2157590bb8e48635c18df5", + "size": "55350924" }, "x86-linux": { - "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "6572c8768de14c0a0bbaf1a0401e19849681840594744a875ade93e4dc040edd", - "size": "58171580" + "tarball": "https://ziglang.org/builds/zig-x86-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "24d2fbc05b7d92a63e228292c693f7fdfa974655510e8c7089e09ce8b9f41fd4", + "size": "58210984" }, "loongarch64-linux": { - "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "9bdf0f91187ede6b418b898d178521c4ce2065171b8bcfaa68cae0adf1d0a8f0", - "size": "52511048" + "tarball": "https://ziglang.org/builds/zig-loongarch64-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "1448d4e780a1257d042c836c237807c6ecaaba36c03961fef1940a842c22cd6d", + "size": "52566124" }, "s390x-linux": { - "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "c893299d0e486337390262ef480615d4f573da158f73a647eb104e10a2811f0d", - "size": "54926720" + "tarball": "https://ziglang.org/builds/zig-s390x-linux-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "d5bd0131aa1e1a40434acc703afd48f0d235ba9d1e988c944148d03df4dce967", + "size": "54992952" }, "x86_64-windows": { - "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.3153+d6f43caad.zip", - "shasum": "ab31039adca06ae87c95c11b3ff8ed82808335866d58bc6e508cd3174d2d7e3d", - "size": "97965399" + "tarball": "https://ziglang.org/builds/zig-x86_64-windows-0.16.0-dev.2611+f996d2866.zip", + "shasum": "3ec5cac4b74b5fe6e04f092ffb6e025993538ecf07970a428919d638efafa542", + "size": "98104739" }, "aarch64-windows": { - "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.3153+d6f43caad.zip", - "shasum": "b25c7046c111f5a4e614fae1a1b33c8db73fbc3de4c135acabb5740455a3b6fc", - "size": "93857928" + "tarball": "https://ziglang.org/builds/zig-aarch64-windows-0.16.0-dev.2611+f996d2866.zip", + "shasum": "42a612662453e47ad4001eb19c1d219dec6f254913988e2ebe6c95e4b5da3e82", + "size": "94105952" }, "x86-windows": { - "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.3153+d6f43caad.zip", - "shasum": "51d579299617bd5fae4d0333d6f2b06a8078c1a6f2f6db2dd08289236fad70d4", - "size": "99767726" + "tarball": "https://ziglang.org/builds/zig-x86-windows-0.16.0-dev.2611+f996d2866.zip", + "shasum": "ccc288a50431d732b4b8dcc55c6e76ec79b406cbb73c794864cd3c538a77ee0a", + "size": "99920751" }, "aarch64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "fe2a97ed29641cc214d58fa2c054fb3ad3a7120b7b43241e6b833c0978187bb6", - "size": "51125424" + "tarball": "https://ziglang.org/builds/zig-aarch64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "c0c4b7bee7d67c4fb44692a96068bde05ba88e4bbe431928540db5bb29a8ab34", + "size": "51220656" }, "arm-freebsd": { - "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "0aa627f73f0bc9c100394d512a375188afb8d33168409050d07b0c8b87adbc0b", - "size": "52702132" + "tarball": "https://ziglang.org/builds/zig-arm-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "9f470b7304d70e59febfb23e14cacdbf1f81c2839e181f290009c3ccb04bbd4e", + "size": "52814796" + }, + "powerpc64-freebsd": { + "tarball": "https://ziglang.org/builds/zig-powerpc64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "7caa39c909cee4342876f0623a40ad91f364b509a14bdcdcdd86629071a1dfbb", + "size": "53925668" }, "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "18500e3b2e742c07541b2d35377e3dc770758ea41e9a3bf09535eacad5266583", - "size": "55254720" + "tarball": "https://ziglang.org/builds/zig-powerpc64le-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "4a0ea164b83b6bac83764c5b34c64b85ecbae1b685dc07a0fa59410efdde32c5", + "size": "55367852" }, "riscv64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "593c19b8feb76a2e66c6d49f3a96390eed22fc4e2c780a96429d202bf4912afc", - "size": "55393696" + "tarball": "https://ziglang.org/builds/zig-riscv64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "49f2e4403aad2256b921a2952a7e4e262c1077f3eb3bb0bd7c3e69a3154fbf68", + "size": "55478468" }, "x86_64-freebsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "b043fa6731e205f208d2e50fa84d1d79c7060e04688a4f5e3bff6dace337d5f8", - "size": "55587512" + "tarball": "https://ziglang.org/builds/zig-x86_64-freebsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "169314c88df8e98c4b5c2db9805d160dcf332013fcd1f8a0df09f81a5eb3425a", + "size": "55617364" }, "aarch64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "c1625244d6fe4008feb27d8bc7bf49739da98d04bc961295124a4e8732cf3b18", - "size": "51092376" + "tarball": "https://ziglang.org/builds/zig-aarch64-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "a091fd4f7128e0098915d6693eb720b1a4d1740dcb552e99a0957d99c3fda3c4", + "size": "51174248" }, "arm-netbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "582116aeee90ea3d38ede6901f93138e543a07c6dc32d2645bf0b4e6fee813c6", - "size": "53732592" + "tarball": "https://ziglang.org/builds/zig-arm-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "e399f1751fc25e3e04c2b7411a5fd9b9c316233b9370fc2fdf7fb45e8b020922", + "size": "53840612" }, "x86-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "eba2eb8e8efe0748820fcedb4066233a96406f97c3fa7b326e220f7c3a9e07f3", - "size": "58765376" + "tarball": "https://ziglang.org/builds/zig-x86-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "8fae56f0683a8f9d2767ad89adc156971f0a872fb485b4c70966e6b113bcad68", + "size": "58800484" }, "x86_64-netbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "281d172d159326167a75f2bc87fbe18afe4e37a26b5a1440cd65c8eef952626a", - "size": "55515836" - }, - "aarch64-openbsd": { - "tarball": "https://ziglang.org/builds/zig-aarch64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "1d750bb3ed57ff4a83b1411a07df7e38f86a0a611123fcbede1ed84a48937585", - "size": "51551144" - }, - "arm-openbsd": { - "tarball": "https://ziglang.org/builds/zig-arm-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "2bd936eff1c2fadd5aa842e30f35f8509ee07f980bcb42a407940b1ae8fe5f29", - "size": "52270724" - }, - "riscv64-openbsd": { - "tarball": "https://ziglang.org/builds/zig-riscv64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "c2067369e07e991f2e1cfe8f657c9185d726487a4ab4cbc1fde9768c8fae363d", - "size": "55713660" - }, - "x86_64-openbsd": { - "tarball": "https://ziglang.org/builds/zig-x86_64-openbsd-0.16.0-dev.3153+d6f43caad.tar.xz", - "shasum": "01122836baaa88627de2e82addf7cdcdcfb3ba104ae9d051d92e36aaad4f4e27", - "size": "56927908" - } - }, - "0.16.0": { - "version": "0.16.0", - "date": "2026-04-13", - "docs": "https://ziglang.org/documentation/0.16.0/", - "stdDocs": "https://ziglang.org/documentation/0.16.0/std/", - "notes": "https://ziglang.org/download/0.16.0/release-notes.html", - "src": { - "tarball": "https://ziglang.org/download/0.16.0/zig-0.16.0.tar.xz", - "shasum": "43186959edc87d5c7a1be7b7d2a25efffd22ce5807c7af99067f86f99641bfdf", - "size": "22503260" - }, - "bootstrap": { - "tarball": "https://ziglang.org/download/0.16.0/zig-bootstrap-0.16.0.tar.xz", - "shasum": "2a8266a4205772ef40838c8cbdf14875855a515ff3adf89b49c2d2ae93613d10", - "size": "55245980" - }, - "x86_64-macos": { - "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-macos-0.16.0.tar.xz", - "shasum": "0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7", - "size": "57396836" - }, - "aarch64-macos": { - "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-macos-0.16.0.tar.xz", - "shasum": "b23d70deaa879b5c2d486ed3316f7eaa53e84acf6fc9cc747de152450d401489", - "size": "52238004" - }, - "x86_64-linux": { - "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz", - "shasum": "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00", - "size": "55478392" - }, - "aarch64-linux": { - "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-linux-0.16.0.tar.xz", - "shasum": "ea4b09bfb22ec6f6c6ceac57ab63efb6b46e17ab08d21f69f3a48b38e1534f17", - "size": "51211944" - }, - "arm-linux": { - "tarball": "https://ziglang.org/download/0.16.0/zig-arm-linux-0.16.0.tar.xz", - "shasum": "f85116bf2f9189bb6ae280c7f92f03b89c2551a88e17881c0c2df86bf4e42c50", - "size": "52122000" - }, - "riscv64-linux": { - "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-linux-0.16.0.tar.xz", - "shasum": "bc069b0f2f568f54bafbdfc1d65b12fd386ed6a652044a37aee6a4f72f14076e", - "size": "55313616" - }, - "powerpc64le-linux": { - "tarball": "https://ziglang.org/download/0.16.0/zig-powerpc64le-linux-0.16.0.tar.xz", - "shasum": "18800b45c08bf40b335ca5ab79aea70aca287ca969036e938155772becaeebeb", - "size": "55242728" - }, - "x86-linux": { - "tarball": "https://ziglang.org/download/0.16.0/zig-x86-linux-0.16.0.tar.xz", - "shasum": "4e34e279a9f856358de420490b531974c3d37f8f3707eef9f0342e92c14c301f", - "size": "58196012" - }, - "loongarch64-linux": { - "tarball": "https://ziglang.org/download/0.16.0/zig-loongarch64-linux-0.16.0.tar.xz", - "shasum": "2503be8ecc5965f1f7962471d267d9f83fcb3cc2f7ff78ac34093b9722bbea93", - "size": "52515080" - }, - "s390x-linux": { - "tarball": "https://ziglang.org/download/0.16.0/zig-s390x-linux-0.16.0.tar.xz", - "shasum": "14db4884b31414bd637879036c1e6e335f3ba44ccac59df104163728a0cab30e", - "size": "54995144" - }, - "x86_64-windows": { - "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-windows-0.16.0.zip", - "shasum": "68659eb5f1e4eb1437a722f1dd889c5a322c9954607f5edcf337bc3684a75a7e", - "size": "97217739" - }, - "aarch64-windows": { - "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-windows-0.16.0.zip", - "shasum": "aee38316ee4111717900f45dd3130145c39289e105541d737eb8c5ed653c78ef", - "size": "93109828" - }, - "x86-windows": { - "tarball": "https://ziglang.org/download/0.16.0/zig-x86-windows-0.16.0.zip", - "shasum": "8aee7e8a8deb998ba96cb95d89aa5fcdf32933fdc67de51d280d9e4d7396f1a0", - "size": "99015159" - }, - "aarch64-freebsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-freebsd-0.16.0.tar.xz", - "shasum": "0a441f50696a34cfb9f9a0b3c510fd49a56b18b364e03c26525cb7999959c969", - "size": "51137568" - }, - "arm-freebsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-arm-freebsd-0.16.0.tar.xz", - "shasum": "78b2a7bfa74f504075dd4b5c00418347a21b893f0eae580cf4e56c553cf181f6", - "size": "52741336" - }, - "powerpc64le-freebsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-powerpc64le-freebsd-0.16.0.tar.xz", - "shasum": "46b3fa0239be42669d86ce8394f8de2d608a4f51e3e65cb8222bfd543b95e310", - "size": "55232224" - }, - "riscv64-freebsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-freebsd-0.16.0.tar.xz", - "shasum": "aefbaea44f32bde07118f5522b7bb21de9967a6d09183e3e37b1f5be2c78f948", - "size": "55416052" - }, - "x86_64-freebsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-freebsd-0.16.0.tar.xz", - "shasum": "390379bfaf6b89b001c6bb4035b8fcbc5c12193fd9fd5048c3a0fe39e5d1cf72", - "size": "55591888" - }, - "aarch64-netbsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-netbsd-0.16.0.tar.xz", - "shasum": "4767425d72c779275ebbc5a6319f5a3c5d109664d3d8c880125d550e87d9d46a", - "size": "51104512" - }, - "arm-netbsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-arm-netbsd-0.16.0.tar.xz", - "shasum": "07aaa403c18e9fa33687a3eb3109bb484a96aece95a4f6792a911d2eefac07c5", - "size": "53772868" - }, - "x86-netbsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-x86-netbsd-0.16.0.tar.xz", - "shasum": "aa36c02caad52d92e6354baec16dd866642e64b7c30e4d515806c8296d65586f", - "size": "58792260" - }, - "x86_64-netbsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-netbsd-0.16.0.tar.xz", - "shasum": "e235ca96f63034a009aca831c69394a1c14498a8e06ff5f358fc3fb4bed73dfb", - "size": "55533332" - }, - "aarch64-openbsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-aarch64-openbsd-0.16.0.tar.xz", - "shasum": "f3c868ccc1bc30e2d2b68128186b2153d86c3efc41190f4170a43024a3cd7809", - "size": "51564888" - }, - "arm-openbsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-arm-openbsd-0.16.0.tar.xz", - "shasum": "33786d6487aaeca9f300e5a932d5ee5365d9254be0add44b3af8f6db8441ca2f", - "size": "52301840" - }, - "riscv64-openbsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-riscv64-openbsd-0.16.0.tar.xz", - "shasum": "ffb74aefa8184a4b886f8c0a9105db00ac9562d502330f4e764a7a49171e4f8e", - "size": "55734756" - }, - "x86_64-openbsd": { - "tarball": "https://ziglang.org/download/0.16.0/zig-x86_64-openbsd-0.16.0.tar.xz", - "shasum": "25b72946e8b2dde5c93bc03cbebeb17e4d90eea8040acce8bb78cefdeee97337", - "size": "56962688" + "tarball": "https://ziglang.org/builds/zig-x86_64-netbsd-0.16.0-dev.2611+f996d2866.tar.xz", + "shasum": "0063013d3d306b4979b16f87db42fa1c7d94d5eac9275979cf40afa944328a56", + "size": "55551904" } }, "0.15.2": { diff --git a/bazel/zig_options.bzl b/bazel/zig_options.bzl deleted file mode 100644 index 85938a0c..00000000 --- a/bazel/zig_options.bzl +++ /dev/null @@ -1,170 +0,0 @@ -"""Build settings for Zig options.""" - -load("@bazel_skylib//lib:types.bzl", "types") -load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") -load("@rules_zig//zig:defs.bzl", "zig_library") - -ZigOption = provider( - doc = "A Zig option's configured value and allowed string values.", - fields = { - "name": "The flag's target name, or this option's target name for explicit values.", - "module_name": "The Bazel module declaring the flag, or empty when unavailable.", - "value": "The configured build setting or explicit value.", - "type": "The Zig integer type, or None for other flag types.", - "allowed_values": "Allowed values for string flags; an empty list allows any string. None for other flag types.", - "enum_name": "Exported enum type name for a constrained string flag; empty keeps the option as a string.", - "nullable": "Whether a string or enum emits null for an empty value.", - }, -) - -_FlagInfo = provider(fields = ["allowed_values"]) - -def _flag_info_impl(_target, ctx): - return [_FlagInfo(allowed_values = getattr(ctx.rule.attr, "values", []))] - -_flag_info = aspect(implementation = _flag_info_impl) - -def _zig_option_impl(ctx): - flag = ctx.attr.flag - if ctx.attr.value_source == "flag": - if flag == None: - fail("flag must resolve to a build setting") - value = flag[BuildSettingInfo].value - else: - value = getattr(ctx.attr, ctx.attr.value_source) - if not (types.is_bool(value) or types.is_int(value) or types.is_string(value)): - fail("Zig options require an integer, boolean, or string value") - allowed_values = None - if types.is_string(value): - allowed_values = flag[_FlagInfo].allowed_values if flag else [] - if ctx.attr.type and not types.is_int(value): - fail("type is only supported for integer options") - if ctx.attr.nullable and not types.is_string(value): - fail("nullable is only supported for string options") - if ctx.attr.enum_name and not allowed_values: - fail("enum_name requires a string flag with nonempty values") - return [ZigOption( - name = flag.label.name if flag else ctx.label.name, - module_name = "", - value = value, - type = (ctx.attr.type or "usize") if types.is_int(value) else None, - allowed_values = allowed_values, - enum_name = ctx.attr.enum_name, - nullable = ctx.attr.nullable, - )] - -def _zig_string(value): - # JSON and Zig differ in their control-character and Unicode escapes. - escaped = [] - for char in value.elems(): - part = json.encode(char)[1:-1] - if part.startswith("\\u"): - part = "\\u{" + part[2:] + "}" - escaped.append(part.replace("\\b", "\\x08").replace("\\f", "\\x0c")) - return '"' + "".join(escaped) + '"' - -def _zig_options_impl(ctx): - lines = [] - names = {} - for setting in ctx.attr.settings: - option = setting[ZigOption] - name = option.name - if name in names: - fail("Duplicate Zig declaration: {}".format(name)) - names[name] = True - value = option.value - if types.is_bool(value): - zig_type = "bool" - literal = "true" if value else "false" - elif types.is_int(value): - zig_type = option.type - literal = str(value) - elif option.enum_name: - enum_values = [v for v in option.allowed_values if not option.nullable or v != ""] - zig_type = "enum { " + ", ".join(["@" + _zig_string(v) for v in enum_values]) + " }" - if option.enum_name in names: - fail("Duplicate Zig declaration: {}".format(option.enum_name)) - names[option.enum_name] = True - lines.append("pub const @{} = {};".format(_zig_string(option.enum_name), zig_type)) - zig_type = "@" + _zig_string(option.enum_name) - if option.nullable: - zig_type = "?" + zig_type - literal = "null" if option.nullable and not value else ".@" + _zig_string(value) - else: - zig_type = "?[]const u8" if option.nullable else "[]const u8" - literal = "null" if option.nullable and not value else _zig_string(value) - lines.append("pub const @{}: {} = {};".format(_zig_string(name), zig_type, literal)) - ctx.actions.write(ctx.outputs.out, "\n".join(lines) + "\n") - return [DefaultInfo(files = depset([ctx.outputs.out]))] - -_zig_options_source = rule( - implementation = _zig_options_impl, - attrs = { - "out": attr.output(mandatory = True), - "settings": attr.label_list( - mandatory = True, - allow_files = False, - providers = [ZigOption], - ), - }, -) - -def zig_options(name, settings, **kwargs): - """Generate an options module and expose it as a zig_library. - - Args: - name: Library target name; the generated source is .zig. - settings: zig_option targets; flag names or explicit option names become quoted Zig constants. - **kwargs: Attributes forwarded to zig_library, except main which is generated. - """ - if "main" in kwargs: - fail("zig_options generates its own main source") - source_attrs = {key: kwargs[key] for key in ["testonly", "tags", "compatible_with", "restricted_to", "target_compatible_with"] if key in kwargs} - _zig_options_source( - name = name + "_source", - settings = settings, - out = name + ".zig", - visibility = ["//visibility:private"], - **source_attrs - ) - zig_library(name = name, main = ":" + name + "_source", **kwargs) - -_zig_option = rule( - implementation = _zig_option_impl, - attrs = { - "flag": attr.label( - providers = [BuildSettingInfo], - aspects = [_flag_info], - doc = "Boolean, integer, or string build setting. An aspect discovers allowed string values.", - ), - "bool": attr.bool(), - "string": attr.string(), - "int": attr.int(), - "value_source": attr.string(mandatory = True, values = ["flag", "bool", "string", "int"]), - "type": attr.string(doc = "Zig integer type; defaults to usize. Only valid for integer options."), - "nullable": attr.bool(doc = "Emit an optional string or enum; an empty value becomes null. String flags with values must include an empty string to allow null; it is excluded from the enum."), - "enum_name": attr.string(doc = "Emit a named Zig enum instead of a string; requires nonempty values. Defaults to emitting a string."), - }, - doc = "Expose a build setting or explicit value as a Zig option, named after the flag or this target respectively.", -) - -def zig_option(name, flag = None, bool = None, string = None, int = None, **kwargs): - """Expose exactly one build setting or explicit value as a Zig option. - - Args: - name: Target name; also the generated Zig constant name for explicit values. - flag: Boolean, integer, or string build setting to read. - bool: Explicit boolean value, which may use select(). - string: Explicit string value, which may use select(). - int: Explicit integer value, which may use select(). - **kwargs: Attributes forwarded to the rule, including type, nullable, and enum_name. - """ - sources = {key: value for key, value in { - "flag": flag, - "bool": bool, - "string": string, - "int": int, - }.items() if value != None} - if len(sources) != 1: - fail("zig_option requires exactly one of flag, bool, string, or int") - _zig_option(name = name, value_source = sources.keys()[0], **dict(sources, **kwargs)) diff --git a/bazel/zig_srcs.bzl b/bazel/zig_srcs.bzl index 96bebfab..ee4adb36 100644 --- a/bazel/zig_srcs.bzl +++ b/bazel/zig_srcs.bzl @@ -1,5 +1,4 @@ -load("@tar.bzl//tar:mtree.bzl", "mtree_spec") -load("@tar.bzl//tar:tar.bzl", "tar") +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") load("@rules_zig//zig:defs.bzl", "zig_static_library") def zig_srcs(name, zig_bin = "", zig_lib = ""): diff --git a/bin/zml-smi/BUILD.bazel b/bin/zml-smi/BUILD.bazel deleted file mode 100644 index 077f5b34..00000000 --- a/bin/zml-smi/BUILD.bazel +++ /dev/null @@ -1,222 +0,0 @@ -load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_lib//tools/release:hashes.bzl", "hashes") -load("@bazel_skylib//rules:run_binary.bzl", "run_binary") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load") -load("@rules_shell//shell:sh_binary.bzl", "sh_binary") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") -load("@tar.bzl//tar:mtree.bzl", "mtree_spec") -load("@tar.bzl//tar:tar.bzl", "tar") - -# zig libraries - -# smi - -run_binary( - name = "libnrt.stripped", - srcs = ["@aws-neuronx-runtime-lib//:libnrt.patchelf"], - outs = ["libnrt.so.1"], - args = [ - "--strip-debug", - "-o", - "$(location libnrt.so.1)", - "$(location @aws-neuronx-runtime-lib//:libnrt.patchelf)", - ], - target_compatible_with = ["@platforms//os:linux"], - tool = "@llvm//tools:llvm-strip", -) - -run_binary( - name = "libncfw.stripped", - srcs = ["@aws-neuronx-runtime-lib//:libncfw.patchelf"], - outs = ["libncfw.so.2"], - args = [ - "--strip-debug", - "-o", - "$(location libncfw.so.2)", - "$(location @aws-neuronx-runtime-lib//:libncfw.patchelf)", - ], - target_compatible_with = ["@platforms//os:linux"], - tool = "@llvm//tools:llvm-strip", -) - -copy_to_directory( - name = "sandbox", - srcs = select({ - "@platforms//os:macos": [], - "@llvm//platforms/config:linux_x86_64": [ - ":libncfw.stripped", - ":libnrt.stripped", - "//platforms/rocm:zmlxrocm", - "@amdrocm-amdsmi//:libamd_smi", - "@amdrocm-sysdeps//:system_libs", - "@libzml_rocm//:amdgpu_ids", - ], - "@llvm//platforms/config:linux_aarch64": [], - }), - include_external_repositories = ["**"], - replace_prefixes = { - "amdgpu.ids": "share/libdrm/amdgpu.ids", - "platforms/rocm": "lib", - "libnrt.so.1": "lib/libnrt.so.1", - "libncfw.so.2": "lib/libncfw.so.2", - }, -) - -run_binary( - name = "zml-smi.stripped", - srcs = [":zml-smi"], - outs = ["zml-smi.stripped"], - args = [ - "--strip-all", - "-o", - "$(location zml-smi.stripped)", - "$(location :zml-smi)", - ], - tool = "@llvm//tools:llvm-strip", -) - -zig_binary( - name = "zml-smi", - data = [":sandbox"], - main = "main.zig", - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/api", - "//bin/zml-smi/collector", - "//bin/zml-smi/csv", - "//bin/zml-smi/info", - "//bin/zml-smi/json", - "//bin/zml-smi/prometheus", - "//bin/zml-smi/tui", - "//bin/zml-smi/utils:sysfs", - "//stdx", - "@libvaxis//:vaxis", - "@rules_zig//zig/runfiles", - ] + select({ - "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], - "@platforms//os:linux": ["//bin/zml-smi/platforms/linux"], - }) + select({ - "@llvm//platforms/config:linux_x86_64": [ - "//bin/zml-smi/platforms/amdsmi", - "//bin/zml-smi/platforms/neuron", - "//bin/zml-smi/platforms/nvml", - "//bin/zml-smi/platforms/oneapi", - "//bin/zml-smi/platforms/tpu", - "//platforms/neuron:libnrt_headers", - ], - "@llvm//platforms/config:linux_aarch64": [ - "//bin/zml-smi/platforms/nvml", - ], - "//conditions:default": [], - }), -) - -# docker image - -mtree_spec( - name = "mtree_spec", - srcs = [ - ":zml-smi", - ":zml-smi.stripped", - ], - tags = ["manual"], -) - -# Use this genrule until mtree_mutate is fixed -genrule( - name = "mtree", - srcs = [":mtree_spec"], - outs = ["mtree.txt"], - cmd = """sed -e '/type=dir/d' -e '/zml-smi\\/zml-smi /d' -e 's#^bin/##g' -e 's#zml-smi\\.stripped#zml-smi#' -e '/stamp.c/d' $< > $@""", - tags = ["manual"], -) - -# mtree_mutate( -# name = "mtree", -# mtree = ":mtree_spec", -# strip_prefix = "bin", -# ) - -# Release archives, one per platform, correctly named for distribution -# Build one: bazel build --config=release //bin/zml-smi:zml-smi-linux-amd64 -# Build all: bazel build --config=release //bin/zml-smi:archives - -_RELEASE_PLATFORMS = [ - ("linux", "amd64"), - ("linux", "arm64"), - ("macos", "arm64"), -] - -[tar( - name = "archive_" + os + "_" + arch, - srcs = [ - ":zml-smi", - ":zml-smi.stripped", - ], - out = "zml-smi-" + os + "-" + arch + ".tar.zst", - args = [ - "--options", - ",".join([ - "zstd:compression-level=22", - "zstd:threads=16", - ]), - ], - compress = "zstd", - mtree = ":mtree", - tags = ["manual"], -) for os, arch in _RELEASE_PLATFORMS] - -[platform_transition_filegroup( - name = "zml-smi-" + os + "-" + arch, - srcs = [":archive_" + os + "_" + arch], - tags = ["manual"], - target_platform = "//platforms:" + os + "_" + arch, -) for os, arch in _RELEASE_PLATFORMS] - -[hashes( - name = "zml-smi-" + os + "-" + arch + ".sha256", - src = ":zml-smi-" + os + "-" + arch, - tags = ["manual"], -) for os, arch in _RELEASE_PLATFORMS] - -sh_binary( - name = "release", - srcs = ["scripts/release.sh"], - tags = ["manual"], -) - -filegroup( - name = "archives", - srcs = [":zml-smi-" + os + "-" + arch for os, arch in _RELEASE_PLATFORMS] + - [":zml-smi-" + os + "-" + arch + ".sha256" for os, arch in _RELEASE_PLATFORMS], - tags = ["manual"], -) - -oci_image( - name = "image_", - base = "@distroless_cc_debian12", - entrypoint = ["/zml-smi/zml-smi"], - tags = ["manual"], - target_compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], - tars = [ - ":archive_linux_amd64", - ], -) - -platform_transition_filegroup( - name = "image", - srcs = [":image_"], - tags = ["manual"], - target_platform = "//platforms:linux_amd64", -) - -oci_load( - name = "load", - image = ":image", - repo_tags = ["zmlai/zml-smi:latest"], - tags = ["manual"], -) diff --git a/bin/zml-smi/api/BUILD.bazel b/bin/zml-smi/api/BUILD.bazel deleted file mode 100644 index 1a09f597..00000000 --- a/bin/zml-smi/api/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "api", - import_name = "zml-smi/api", - main = "api.zig", - srcs = [ - "client.zig", - "server.zig", - ], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/json", - "//bin/zml-smi/utils:double_buffer", - ] + select({ - "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], - "//conditions:default": ["//bin/zml-smi/platforms/linux"], - }), -) diff --git a/bin/zml-smi/api/api.zig b/bin/zml-smi/api/api.zig deleted file mode 100644 index fb39ddd8..00000000 --- a/bin/zml-smi/api/api.zig +++ /dev/null @@ -1,2 +0,0 @@ -pub const Server = @import("server.zig").Server; -pub const addRemotes = @import("client.zig").addRemotes; diff --git a/bin/zml-smi/api/client.zig b/bin/zml-smi/api/client.zig deleted file mode 100644 index ddec6d2d..00000000 --- a/bin/zml-smi/api/client.zig +++ /dev/null @@ -1,138 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const pi = smi_info.process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Collector = @import("zml-smi/collector").Collector; - -fn setRemote(device: *DeviceInfo) void { - inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { - if (device.* == @field(smi_info.device_info.Target, field_name)) { - @field(device, field_name).values[0].remote = true; - @field(device, field_name).values[1].remote = true; - } - } -} - -pub fn addRemotes(collector: *Collector, hosts: []const u8) !void { - var it = std.mem.splitScalar(u8, hosts, ','); - while (it.next()) |host| { - addHost(collector, host) catch |err| { - std.log.err("{s}: {s}", .{ host, @errorName(err) }); - }; - } -} - -fn addHost(collector: *Collector, host: []const u8) !void { - const body = try httpGet(collector.gpa, collector.io, host); - defer collector.gpa.free(body); - - const parsed = try std.json.parseFromSlice(std.json.Value, collector.gpa, body, .{}); - defer parsed.deinit(); - - const root = parsed.value.object; - const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; - - const dev_offset: u16 = @intCast(collector.device_infos.items.len); - - var devices: std.ArrayList(*DeviceInfo) = .empty; - for (json_devices) |item| { - var device = std.json.parseFromValueLeaky(DeviceInfo, collector.arena, item, .{ .ignore_unknown_fields = true }) catch continue; - setRemote(&device); - const info = try collector.addDevice(device); - try devices.append(collector.arena, info); - } - - if (devices.items.len == 0) { - return; - } - - const processes = try collector.createProcessList(); - const str_arenas: [2]*std.heap.ArenaAllocator = .{ try collector.createPollArena(), try collector.createPollArena() }; - - const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; - const back_idx: usize = 1 - processes.current.load(.acquire); - const back = processes.back(); - for (json_processes) |item| { - var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, str_arenas[back_idx].allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; - proc.device_idx += dev_offset; - proc.remote = true; - back.append(collector.gpa, proc) catch continue; - } - processes.swap(); - - const url = try collector.arena.dupe(u8, host); - const poll_arena = try collector.createPollArena(); - try collector.spawnPoll(pollOnce, .{ poll_arena, collector.gpa, collector.io, url, devices.items, processes, dev_offset, str_arenas }); -} - -fn pollOnce(poll_arena: *std.heap.ArenaAllocator, gpa: std.mem.Allocator, io: std.Io, host: []const u8, devices: []const *DeviceInfo, processes: *ProcessDoubleBuffer, dev_offset: u16, str_arenas: [2]*std.heap.ArenaAllocator) void { - _ = poll_arena.reset(.retain_capacity); - - const body = httpGet(gpa, io, host) catch return; - defer gpa.free(body); - - const parsed = std.json.parseFromSlice(std.json.Value, gpa, body, .{}) catch return; - defer parsed.deinit(); - - const root = parsed.value.object; - const json_devices = if (root.get("devices")) |v| v.array.items else &.{}; - - for (json_devices, 0..) |item, i| { - if (i >= devices.len) { - std.log.warn("{s}: remote has more devices than expected ({d} > {d})", .{ host, json_devices.len, devices.len }); - break; - } - var device = std.json.parseFromValueLeaky(DeviceInfo, poll_arena.allocator(), item, .{ .ignore_unknown_fields = true }) catch continue; - setRemote(&device); - - inline for (@typeInfo(DeviceInfo).@"union".field_names) |field_name| { - if (devices[i].* == @field(smi_info.device_info.Target, field_name)) { - @field(devices[i], field_name).back().* = @field(device, field_name).front().*; - @field(devices[i], field_name).swap(); - } - } - } - - const json_processes = if (root.get("processes")) |v| v.array.items else &.{}; - const back_idx: usize = 1 - processes.current.load(.acquire); - _ = str_arenas[back_idx].reset(.retain_capacity); - const alloc = str_arenas[back_idx].allocator(); - const back = processes.back(); - back.clearRetainingCapacity(); - for (json_processes) |item| { - var proc = std.json.parseFromValueLeaky(pi.ProcessInfo, alloc, item, .{ .ignore_unknown_fields = true }) catch continue; - proc.device_idx += dev_offset; - proc.remote = true; - back.append(gpa, proc) catch continue; - } - processes.swap(); -} - -fn httpGet(allocator: std.mem.Allocator, io: std.Io, host: []const u8) ![]const u8 { - if (!std.mem.startsWith(u8, host, "http://") and !std.mem.startsWith(u8, host, "https://")) { - std.log.err("remote host must start with http:// or https://: {s}", .{host}); - return error.InvalidUrl; - } - - const url = try std.fmt.allocPrint(allocator, "{s}/metrics", .{host}); - defer allocator.free(url); - - var client: std.http.Client = .{ .allocator = allocator, .io = io }; - defer client.deinit(); - - var aw: std.Io.Writer.Allocating = .init(allocator); - errdefer aw.deinit(); - - const result = try client.fetch(.{ - .location = .{ .url = url }, - .response_writer = &aw.writer, - }); - - if (result.status != .ok) { - aw.deinit(); - return error.HttpError; - } - - return try aw.toOwnedSlice(); -} diff --git a/bin/zml-smi/api/server.zig b/bin/zml-smi/api/server.zig deleted file mode 100644 index e05aa7fe..00000000 --- a/bin/zml-smi/api/server.zig +++ /dev/null @@ -1,105 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const ProcessInfo = smi_info.process_info.ProcessInfo; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(ProcessInfo)); -const json = @import("zml-smi/json"); -const ProcessEnricher = if (builtin.os.tag == .macos) - @import("zml-smi/platforms/macos").process.ProcessEnricher -else - @import("zml-smi/platforms/linux").process.ProcessEnricher; - -pub const Server = struct { - allocator: std.mem.Allocator, - io: std.Io, - devices: []const *DeviceInfo, - process_lists: []const *const ProcessDoubleBuffer, - enricher: *ProcessEnricher, - tcp: std.Io.net.Server, - connection_group: std.Io.Group = .init, - - pub fn init(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) !Server { - const address: std.Io.net.IpAddress = .{ .ip4 = .{ - .bytes = .{ 0, 0, 0, 0 }, - .port = port, - } }; - var tcp = try address.listen(io, .{ .reuse_address = true }); - errdefer tcp.deinit(io); - - return .{ - .allocator = allocator, - .io = io, - .devices = devices, - .process_lists = process_lists, - .enricher = enricher, - .tcp = tcp, - }; - } - - pub fn run(allocator: std.mem.Allocator, io: std.Io, port: u16, devices: []const *DeviceInfo, process_lists: []const *const ProcessDoubleBuffer, enricher: *ProcessEnricher) void { - var self = init(allocator, io, port, devices, process_lists, enricher) catch |err| { - std.log.err("api server failed to start: {s}", .{@errorName(err)}); - return; - }; - defer self.deinit(); - self.serve(); - } - - pub fn deinit(self: *Server) void { - self.connection_group.await(self.io) catch {}; - self.tcp.deinit(self.io); - } - - pub fn serve(self: *Server) void { - while (true) { - const stream = self.tcp.accept(self.io) catch break; - self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; - } - } - - fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { - defer stream.close(self.io); - self.handleConn(stream) catch {}; - } - - fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { - var read_buf: [8192]u8 = undefined; - var reader = stream.reader(self.io, &read_buf); - - var write_buf: [4096]u8 = undefined; - var writer = stream.writer(self.io, &write_buf); - - var http_server: std.http.Server = .init(&reader.interface, &writer.interface); - - var procs: std.ArrayList(ProcessInfo) = .empty; - defer procs.deinit(self.allocator); - - while (true) { - var request = http_server.receiveHead() catch |err| switch (err) { - error.HttpConnectionClosing => return, - else => return err, - }; - - procs.clearRetainingCapacity(); - for (self.process_lists) |pl| { - procs.appendSlice(self.allocator, pl.front().items) catch {}; - } - self.enricher.enrich(self.io, procs.items); - - var body_buf: [4096]u8 = undefined; - var body_writer = try request.respondStreaming(&body_buf, .{ - .respond_options = .{ - .status = .ok, - .extra_headers = &.{ - .{ .name = "content-type", .value = "application/json" }, - .{ .name = "access-control-allow-origin", .value = "*" }, - }, - }, - }); - - try json.write(&body_writer.writer, self.devices, procs.items); - try body_writer.end(); - } - } -}; diff --git a/bin/zml-smi/collector/BUILD.bazel b/bin/zml-smi/collector/BUILD.bazel deleted file mode 100644 index 0bf5fa8e..00000000 --- a/bin/zml-smi/collector/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "collector", - import_name = "zml-smi/collector", - main = "collector.zig", - srcs = ["worker.zig"], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - ], -) diff --git a/bin/zml-smi/collector/collector.zig b/bin/zml-smi/collector/collector.zig deleted file mode 100644 index d98b204d..00000000 --- a/bin/zml-smi/collector/collector.zig +++ /dev/null @@ -1,93 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const pi = smi_info.process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Worker = @import("worker.zig").Worker; - -pub const Collector = struct { - device_infos: std.ArrayList(*DeviceInfo) = .empty, - process_lists: std.ArrayList(*ProcessDoubleBuffer) = .empty, - poll_arenas: std.ArrayList(*std.heap.ArenaAllocator) = .empty, - arena: std.mem.Allocator, - gpa: std.mem.Allocator, - worker: Worker, - io: std.Io, - poll_only: bool, - - pub fn init(arena: std.mem.Allocator, gpa: std.mem.Allocator, io: std.Io, options: struct { poll_interval_ms: u16, poll_only: bool = false }) Collector { - return .{ - .arena = arena, - .gpa = gpa, - .io = io, - .worker = .{ .poll_interval_ms = options.poll_interval_ms }, - .poll_only = options.poll_only, - }; - } - - pub fn addDevice(self: *Collector, initial: DeviceInfo) !*DeviceInfo { - const info = try self.arena.create(DeviceInfo); - info.* = initial; - try self.device_infos.append(self.arena, info); - - return info; - } - - pub fn createProcessList(self: *Collector) !*ProcessDoubleBuffer { - const list = try self.arena.create(ProcessDoubleBuffer); - list.* = .{ .values = .{ .empty, .empty } }; - try self.process_lists.append(self.arena, list); - - return list; - } - - pub fn createPollArena(self: *Collector) !*std.heap.ArenaAllocator { - const poll_arena = try self.arena.create(std.heap.ArenaAllocator); - poll_arena.* = std.heap.ArenaAllocator.init(self.gpa); - try self.poll_arenas.append(self.arena, poll_arena); - - return poll_arena; - } - - pub fn spawnPoll(self: *Collector, comptime pollOnce: anytype, args: std.meta.ArgsTuple(@TypeOf(pollOnce))) !void { - @call(.auto, pollOnce, args); - if (self.poll_only) return; - - const Args = std.meta.ArgsTuple(@TypeOf(pollOnce)); - try self.worker.spawn(self.io, struct { - fn f(io: std.Io, w: *const Worker, a: Args) void { - const interval: std.Io.Duration = .fromMilliseconds(w.poll_interval_ms); - while (w.isRunning()) { - const start: std.Io.Timestamp = .now(io, .awake); - - @call(.auto, pollOnce, a); - - const elapsed = start.untilNow(io, .awake); - if (elapsed.nanoseconds < interval.nanoseconds) { - io.sleep(.fromNanoseconds(interval.nanoseconds - elapsed.nanoseconds), .awake) catch {}; - } - } - } - }.f, .{ self.io, &self.worker, args }); - } - - pub fn deinit(self: *Collector) void { - self.worker.shutdown(self.io); - - for (self.device_infos.items) |info| { - self.arena.destroy(info); - } - self.device_infos.deinit(self.arena); - - for (self.process_lists.items) |list| { - list.values[0].deinit(self.gpa); - list.values[1].deinit(self.gpa); - } - self.process_lists.deinit(self.arena); - - for (self.poll_arenas.items) |poll_arena| { - poll_arena.deinit(); - } - self.poll_arenas.deinit(self.arena); - } -}; diff --git a/bin/zml-smi/collector/worker.zig b/bin/zml-smi/collector/worker.zig deleted file mode 100644 index 125440c5..00000000 --- a/bin/zml-smi/collector/worker.zig +++ /dev/null @@ -1,25 +0,0 @@ -const std = @import("std"); - -pub const Worker = struct { - poll_interval_ms: u16, - should_stop: std.atomic.Value(bool) = .init(false), - group: std.Io.Group = .init, - - pub fn shutdown(self: *Worker, io: std.Io) void { - self.should_stop.store(true, .release); - self.group.await(io) catch {}; - } - - pub fn isRunning(self: *const Worker) bool { - return !self.should_stop.load(.acquire); - } - - pub fn spawn( - self: *Worker, - io: std.Io, - comptime runFn: anytype, - args: std.meta.ArgsTuple(@TypeOf(runFn)), - ) !void { - try self.group.concurrent(io, runFn, args); - } -}; diff --git a/bin/zml-smi/csv/BUILD.bazel b/bin/zml-smi/csv/BUILD.bazel deleted file mode 100644 index a0cb3422..00000000 --- a/bin/zml-smi/csv/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "csv", - import_name = "zml-smi/csv", - main = "csv.zig", - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/info", - ], -) diff --git a/bin/zml-smi/csv/csv.zig b/bin/zml-smi/csv/csv.zig deleted file mode 100644 index dbd80dd9..00000000 --- a/bin/zml-smi/csv/csv.zig +++ /dev/null @@ -1,46 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; - -pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo) !void { - const info = @typeInfo(DeviceInfo).@"union"; - inline for (info.field_names, info.field_types) |type_name, DeviceType| { - const tag = @field(smi_info.Target, type_name); - var header_printed = false; - - for (devices, 0..) |dev, i| { - switch (dev.*) { - tag => |*db| { - if (!header_printed) { - try writer.writeAll("index,type"); - inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { - try writer.writeAll("," ++ field_name); - } - try writer.writeAll("\n"); - - header_printed = true; - } - - const val = db.front().*; - - try writer.print("{d},{s}", .{ i, type_name }); - inline for (@typeInfo(DeviceType.Value).@"struct".field_names) |field_name| { - try writer.writeAll(","); - try writeValue(writer, @field(val, field_name)); - } - try writer.writeAll("\n"); - }, - else => {}, - } - } - } -} - -fn writeValue(writer: *std.Io.Writer, field: anytype) !void { - if (@typeInfo(@TypeOf(field)) == .optional) { - if (field) |v| return writeValue(writer, v); - } else switch (@typeInfo(@TypeOf(field))) { - .pointer => return writer.writeAll(field), - else => return writer.print("{any}", .{field}), - } -} diff --git a/bin/zml-smi/info/BUILD.bazel b/bin/zml-smi/info/BUILD.bazel deleted file mode 100644 index d035b161..00000000 --- a/bin/zml-smi/info/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "info", - import_name = "zml-smi/info", - main = "info.zig", - srcs = [ - "device_info.zig", - "host_info.zig", - "poll_metrics.zig", - "process_info.zig", - ], - visibility = ["//visibility:public"], - deps = ["//bin/zml-smi/utils:double_buffer"], -) diff --git a/bin/zml-smi/info/device_info.zig b/bin/zml-smi/info/device_info.zig deleted file mode 100644 index 4b01c728..00000000 --- a/bin/zml-smi/info/device_info.zig +++ /dev/null @@ -1,95 +0,0 @@ -const std = @import("std"); -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; - -pub const Target = enum { - cuda, - rocm, - oneapi, - neuron, - tpu, -}; - -pub const DeviceInfo = union(Target) { - cuda: DoubleBuffer(GpuInfo), - rocm: DoubleBuffer(GpuInfo), - oneapi: DoubleBuffer(GpuInfo), - neuron: DoubleBuffer(NeuronInfo), - tpu: DoubleBuffer(TpuInfo), - - pub fn isRemote(self: *const DeviceInfo) bool { - switch (self.*) { - inline else => |*db| return db.front().remote, - } - } -}; - -pub const GpuInfo = struct { - remote: bool = false, - name: ?[]const u8 = null, - driver_version: ?[]const u8 = null, - cuda_driver_version: ?[]const u8 = null, - - // Utilization - util_percent: ?u64 = null, - encoder_util_percent: ?u64 = null, - decoder_util_percent: ?u64 = null, - - // Power - power_mw: ?u64 = null, - power_limit_mw: ?u64 = null, - // Thermal - temperature: ?u64 = null, - fan_speed_percent: ?u64 = null, - - // Clocks - clock_graphics_mhz: ?u64 = null, - clock_sm_mhz: ?u64 = null, - clock_soc_mhz: ?u64 = null, - clock_mem_mhz: ?u64 = null, - clock_graphics_max_mhz: ?u64 = null, - clock_mem_max_mhz: ?u64 = null, - - // Memory - mem_used_bytes: ?u64 = null, - mem_total_bytes: ?u64 = null, - mem_bus_width: ?u64 = null, - - // PCIe - pcie_tx_kbps: ?u64 = null, - pcie_rx_kbps: ?u64 = null, - pcie_bandwidth_mbps: ?u64 = null, - pcie_link_gen: ?u64 = null, - pcie_link_width: ?u64 = null, -}; - -pub const NeuronInfo = struct { - remote: bool = false, - name: ?[]const u8 = null, - driver_version: ?[]const u8 = null, - - util_percent: ?u64 = null, - mem_used_bytes: ?u64 = null, - mem_total_bytes: ?u64 = null, - - // Neuron per-core HBM breakdown (bytes) - nc_tensors: ?u64 = null, - nc_constants: ?u64 = null, - nc_model_code: ?u64 = null, - nc_shared_scratchpad: ?u64 = null, - nc_nonshared_scratchpad: ?u64 = null, - nc_runtime: ?u64 = null, - nc_driver: ?u64 = null, - nc_dma_rings: ?u64 = null, - nc_collectives: ?u64 = null, - nc_notifications: ?u64 = null, - nc_uncategorized: ?u64 = null, -}; - -pub const TpuInfo = struct { - remote: bool = false, - name: ?[]const u8 = null, - - util_percent: ?u64 = null, - mem_used_bytes: ?u64 = null, - mem_total_bytes: ?u64 = null, -}; diff --git a/bin/zml-smi/info/host_info.zig b/bin/zml-smi/info/host_info.zig deleted file mode 100644 index f33f6fd8..00000000 --- a/bin/zml-smi/info/host_info.zig +++ /dev/null @@ -1,16 +0,0 @@ -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; - -pub const HostInfo = DoubleBuffer(HostData); - -pub const HostData = struct { - hostname: ?[]const u8 = null, - kernel: ?[]const u8 = null, - cpu_name: ?[]const u8 = null, - cpu_cores: ?u64 = null, - mem_total_kib: ?u64 = null, - mem_available_kib: ?u64 = null, - load_1: ?f32 = null, - load_5: ?f32 = null, - load_15: ?f32 = null, - uptime_seconds: ?u64 = null, -}; diff --git a/bin/zml-smi/info/info.zig b/bin/zml-smi/info/info.zig deleted file mode 100644 index 42e1b1b3..00000000 --- a/bin/zml-smi/info/info.zig +++ /dev/null @@ -1,9 +0,0 @@ -const std = @import("std"); - -pub const device_info = @import("device_info.zig"); -pub const host_info = @import("host_info.zig"); -pub const poll_metrics = @import("poll_metrics.zig"); -pub const process_info = @import("process_info.zig"); - -pub const Target = device_info.Target; -pub const Targets = std.EnumSet(Target); diff --git a/bin/zml-smi/info/poll_metrics.zig b/bin/zml-smi/info/poll_metrics.zig deleted file mode 100644 index 62ed94a7..00000000 --- a/bin/zml-smi/info/poll_metrics.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); - -pub fn apply(comptime table: anytype, dest: anytype, src: anytype) void { - inline for (table) |m| { - @field(dest, m.field) = m.query(src) catch null; - } -} - -pub fn poll(comptime DB: type, comptime Dev: type, comptime table: anytype) fn (?*std.heap.ArenaAllocator, DB, Dev) void { - return struct { - fn f(arena: ?*std.heap.ArenaAllocator, db: DB, dev: Dev) void { - if (arena) |a| _ = a.reset(.retain_capacity); - - const back = db.back(); - back.* = db.front().*; - apply(table, back, dev); - db.swap(); - } - }.f; -} diff --git a/bin/zml-smi/info/process_info.zig b/bin/zml-smi/info/process_info.zig deleted file mode 100644 index 73504656..00000000 --- a/bin/zml-smi/info/process_info.zig +++ /dev/null @@ -1,12 +0,0 @@ -pub const ProcessInfo = struct { - pid: u32 = 0, - device_idx: u16 = 0, - dev_util_percent: ?u16 = null, - dev_mem_kib: ?u64 = null, - uid: u32 = 0, - username: []const u8 = "", - comm: []const u8 = "", - cpu_percent: u16 = 0, // *10, e.g. 125 = 12.5% - rss_kib: u64 = 0, - remote: bool = false, -}; diff --git a/bin/zml-smi/json/json.zig b/bin/zml-smi/json/json.zig deleted file mode 100644 index c32a16cf..00000000 --- a/bin/zml-smi/json/json.zig +++ /dev/null @@ -1,15 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const ProcessInfo = smi_info.process_info.ProcessInfo; - -pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, processes: []const ProcessInfo) !void { - var jw: std.json.Stringify = .{ .writer = writer }; - try jw.write(Response{ .devices = devices, .processes = processes }); - try writer.writeAll("\n"); -} - -const Response = struct { - devices: []const *DeviceInfo, - processes: []const ProcessInfo, -}; diff --git a/bin/zml-smi/main.zig b/bin/zml-smi/main.zig deleted file mode 100644 index f066dd34..00000000 --- a/bin/zml-smi/main.zig +++ /dev/null @@ -1,241 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -pub const std_options: std.Options = .{ - .log_scope_levels = &.{ - .{ .scope = .vaxis, .level = .warn }, - }, -}; - -const Collector = @import("zml-smi/collector").Collector; -const smi_info = @import("zml-smi/info"); -const HostInfo = smi_info.host_info.HostInfo; -const platform = if (builtin.os.tag == .macos) - @import("zml-smi/platforms/macos") -else - @import("zml-smi/platforms/linux"); -const ProcessEnricher = platform.process.ProcessEnricher; -const host = platform.metrics; -const c = @import("c"); -const csv = @import("zml-smi/csv"); -const json = @import("zml-smi/json"); -const api = @import("zml-smi/api"); -const prometheus = @import("zml-smi/prometheus"); -const sysfs = @import("zml-smi/sysfs"); -const smi_tui = @import("zml-smi/tui"); -const tui = smi_tui.top; -const static_print = smi_tui.print; -const data = smi_tui.data; - -const CliArgs = struct { - top: bool = false, - csv: bool = false, - json: bool = false, - api: bool = false, - api_port: u16 = 9090, - remotes: ?[]const u8 = null, - prometheus_listen: ?[]const u8 = null, - tui_refresh_rate: u16 = 100, - poll_interval: u16 = 500, - - pub const help = - \\ zml-smi [--top] [--csv] [--json] [--prometheus-listen HOST:PORT] - \\ [--tui-refresh-rate MS] [--poll-interval MS] - \\ - \\ --top Interactive TUI mode - \\ --csv Output device metrics as CSV - \\ --json Output device metrics as JSON - \\ --prometheus-listen Expose metrics as Prometheus endpoint on HOST:PORT - \\ --tui-refresh-rate TUI refresh rate in ms (default: 100) - \\ --poll-interval Device polling interval in ms (default: 500) - \\ - ; - - // [--api] [--port PORT] [--remotes HOST,...] - // \\ --api Expose device metrics as HTTP JSON endpoint - // \\ --api-port API server port (default: 9090) - // \\ --remotes Add devices from remote hosts (comma-separated URLs) -}; - -const device_backends = switch (builtin.os.tag) { - .macos => .{}, - .linux => switch (builtin.cpu.arch) { - .x86_64 => .{ - .{ .cuda, @import("zml-smi/platforms/nvml") }, - .{ .rocm, @import("zml-smi/platforms/amdsmi") }, - .{ .oneapi, @import("zml-smi/platforms/oneapi") }, - .{ .neuron, @import("zml-smi/platforms/neuron") }, - .{ .tpu, @import("zml-smi/platforms/tpu") }, - }, - .aarch64 => .{ - .{ .cuda, @import("zml-smi/platforms/nvml") }, - }, - else => unreachable, - }, - else => unreachable, -}; - -pub fn main(init: std.process.Init) !void { - const gpa = init.gpa; - const arena = init.arena.allocator(); - const io = init.io; - const args = stdx.flags.parse(init.minimal.args, CliArgs); - - const targets = detect(gpa, io); - - var collector: Collector = .init(arena, gpa, io, .{ - .poll_interval_ms = args.poll_interval, - .poll_only = !args.top and args.prometheus_listen == null and !args.api, - }); - defer collector.deinit(); - - var host_info: HostInfo = .{ .values = .{ .{}, .{} } }; - try host.init(&collector, &host_info); - - var enricher: ProcessEnricher = try .init(gpa, io); - defer enricher.deinit(); - - if (args.api) { - std.log.info("serving api metrics on port {d}", .{args.api_port}); - } - - if (args.prometheus_listen) |listen| { - std.log.info("serving prometheus metrics on {s}", .{listen}); - } - - inline for (device_backends) |entry| { - const target, const backend = entry; - if (targets.contains(target)) { - backend.start(&collector) catch |err| { - std.log.err("{s} skipped: {s}", .{ @tagName(target), @errorName(err) }); - }; - } - } - - var api_group: std.Io.Group = .init; - - if (args.remotes) |hosts| { - try api.addRemotes(&collector, hosts); - } - - var state = try data.SystemState.init(.{ - .devices = collector.device_infos.items, - .host = &host_info, - .targets = targets, - .tui_refresh_rate = args.tui_refresh_rate, - .process_lists = collector.process_lists.items, - .enricher = &enricher, - .gpa = gpa, - .arena = arena, - .io = io, - }); - defer state.deinit(arena); - - if (args.prometheus_listen) |listen| { - try api_group.concurrent(io, prometheus.Server.run, .{ io, listen, collector.device_infos.items, &host_info }); - } - - if (args.api) { - try api_group.concurrent(io, api.Server.run, .{ gpa, io, args.api_port, collector.device_infos.items, collector.process_lists.items, &enricher }); - } - - if (args.prometheus_listen != null or args.api) { - try api_group.await(io); - } else if (args.csv) { - var csv_buf: [4096]u8 = undefined; - var csv_writer = std.Io.File.stdout().writer(io, &csv_buf); - - try csv.write(&csv_writer.interface, collector.device_infos.items); - try csv_writer.flush(); - } else if (args.json) { - var procs: std.ArrayList(smi_info.process_info.ProcessInfo) = .empty; - defer procs.deinit(gpa); - for (collector.process_lists.items) |pl| { - try procs.appendSlice(gpa, pl.front().items); - } - enricher.enrich(io, procs.items); - - var json_buf: [4096]u8 = undefined; - var json_writer = std.Io.File.stdout().writer(io, &json_buf); - - try json.write(&json_writer.interface, collector.device_infos.items, procs.items); - try json_writer.flush(); - } else if (args.top) { - try tui.run(gpa, io, init.environ_map, &state); - } else { - try static_print.run(arena, io, init.environ_map, &state); - } -} - -fn detect(allocator: std.mem.Allocator, io: std.Io) smi_info.Targets { - if (comptime builtin.os.tag == .macos) { - return .{}; - } - - var targets: smi_info.Targets = .{}; - - switch (builtin.os.tag) { - .macos => {}, - .linux => { - switch (builtin.cpu.arch) { - .x86_64 => { - if (hasDevice(io, "/dev/nvidiactl")) { - targets.insert(.cuda); - } - - if (hasDevice(io, "/dev/kfd")) { - targets.insert(.rocm); - } - - if (hasDrmRenderDevice(allocator, io, "0x8086")) { - targets.insert(.oneapi); - } - - if (hasDevice(io, c.NEURON_DEVICE_PREFIX ++ "0")) { - targets.insert(.neuron); - } - - if (hasDevice(io, "/dev/accel0") or hasDevice(io, "/dev/vfio/0")) { - targets.insert(.tpu); - } - }, - .aarch64 => { - if (hasDevice(io, "/dev/nvidiactl")) { - targets.insert(.cuda); - } - }, - else => unreachable, - } - }, - else => unreachable, - } - - return targets; -} - -fn hasDevice(io: std.Io, path: []const u8) bool { - std.Io.Dir.accessAbsolute(io, path, .{ .read = true }) catch return false; - return true; -} - -fn hasDrmRenderDevice(allocator: std.mem.Allocator, io: std.Io, comptime pci_vendor_id: []const u8) bool { - var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; - defer dir.close(io); - - var it = dir.iterate(); - while (it.next(io) catch null) |entry| { - if (!std.mem.startsWith(u8, entry.name, "renderD")) continue; - - var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const vendor_path = std.fmt.bufPrint(&vendor_path_buf, "/sys/class/drm/{s}/device/vendor", .{entry.name}) catch continue; - const vendor = sysfs.readString(allocator, io, vendor_path) catch continue; - defer allocator.free(vendor); - if (std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), pci_vendor_id)) { - return true; - } - } - - return false; -} diff --git a/bin/zml-smi/platforms/amdsmi/BUILD.bazel b/bin/zml-smi/platforms/amdsmi/BUILD.bazel deleted file mode 100644 index dac6b935..00000000 --- a/bin/zml-smi/platforms/amdsmi/BUILD.bazel +++ /dev/null @@ -1,22 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "amdsmi", - srcs = [ - "amdsmi.zig", - "process.zig", - ], - import_name = "zml-smi/platforms/amdsmi", - main = "metrics.zig", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//bin/zml-smi/utils:dynlib", - "//bin/zml-smi/utils:sandbox", - "//stdx", - "@amdrocm-amdsmi//:amdsmi", - ], -) diff --git a/bin/zml-smi/platforms/amdsmi/amdsmi.zig b/bin/zml-smi/platforms/amdsmi/amdsmi.zig deleted file mode 100644 index 2d89bc7c..00000000 --- a/bin/zml-smi/platforms/amdsmi/amdsmi.zig +++ /dev/null @@ -1,295 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const stdx = @import("stdx"); -const DynLib = @import("zml-smi/dynlib"); -const sandbox = @import("zml-smi/sandbox"); - -const AmdSmi = @This(); - -pub const Error = ReturnError || error{AmdSmiUnavailable}; -pub const Handle = c.amdsmi_processor_handle; -pub const ProcInfo = c.amdsmi_proc_info_t; - -lib: Fns, -gpu_handles: []c.amdsmi_processor_handle, - -const Fns = struct { - amdsmi_init: *const @TypeOf(c.amdsmi_init), - amdsmi_get_socket_handles: *const @TypeOf(c.amdsmi_get_socket_handles), - amdsmi_get_processor_handles: *const @TypeOf(c.amdsmi_get_processor_handles), - amdsmi_get_gpu_asic_info: *const @TypeOf(c.amdsmi_get_gpu_asic_info), - amdsmi_get_gpu_metrics_info: *const @TypeOf(c.amdsmi_get_gpu_metrics_info), - amdsmi_get_power_info: *const @TypeOf(c.amdsmi_get_power_info), - amdsmi_get_temp_metric: *const @TypeOf(c.amdsmi_get_temp_metric), - amdsmi_get_gpu_fan_speed: *const @TypeOf(c.amdsmi_get_gpu_fan_speed), - amdsmi_get_gpu_activity: *const @TypeOf(c.amdsmi_get_gpu_activity), - amdsmi_get_clock_info: *const @TypeOf(c.amdsmi_get_clock_info), - amdsmi_get_gpu_memory_total: *const @TypeOf(c.amdsmi_get_gpu_memory_total), - amdsmi_get_gpu_memory_usage: *const @TypeOf(c.amdsmi_get_gpu_memory_usage), - amdsmi_get_pcie_info: *const @TypeOf(c.amdsmi_get_pcie_info), - amdsmi_get_gpu_bdf_id: *const @TypeOf(c.amdsmi_get_gpu_bdf_id), - amdsmi_get_gpu_process_list: *const @TypeOf(c.amdsmi_get_gpu_process_list), - amdsmi_get_gpu_driver_info: *const @TypeOf(c.amdsmi_get_gpu_driver_info), -}; - -pub fn init(allocator: std.mem.Allocator) !AmdSmi { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = sandbox.path(&path_buf) orelse return error.AmdSmiUnavailable; - - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libamd_smi.so.27" }); - - var dynlib: std.DynLib = .{ .inner = .{ - .handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { - if (std.c.dlerror()) |err| std.log.err("amdsmi: dlopen: {s}", .{err}); - return error.AmdSmiUnavailable; - }, - } }; - const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.AmdSmiUnavailable; - - try check(fns.amdsmi_init(c.AMDSMI_INIT_AMD_GPUS)); - - var socket_count: u32 = 0; - try check(fns.amdsmi_get_socket_handles(&socket_count, null)); - - const sockets = try allocator.alloc(c.amdsmi_socket_handle, socket_count); - defer allocator.free(sockets); - try check(fns.amdsmi_get_socket_handles(&socket_count, @ptrCast(sockets.ptr))); - - var handle_list: std.ArrayList(c.amdsmi_processor_handle) = .empty; - defer handle_list.deinit(allocator); - - for (sockets[0..socket_count]) |socket| { - var proc_count: u32 = 0; - try check(fns.amdsmi_get_processor_handles(socket, &proc_count, null)); - - const procs = try allocator.alloc(c.amdsmi_processor_handle, proc_count); - defer allocator.free(procs); - - try check(fns.amdsmi_get_processor_handles(socket, &proc_count, @ptrCast(procs.ptr))); - - for (procs[0..proc_count]) |proc| { - try handle_list.append(allocator, proc); - } - } - - return .{ .lib = fns, .gpu_handles = try handle_list.toOwnedSlice(allocator) }; -} - -pub fn handleByIndex(self: AmdSmi, device_id: u32) Error!Handle { - if (device_id >= self.gpu_handles.len) { - return error.not_found; - } - return self.gpu_handles[device_id]; -} - -pub fn deviceCount(self: AmdSmi) u32 { - return @intCast(self.gpu_handles.len); -} - -pub const name_buf_len = c.AMDSMI_MAX_STRING_LENGTH; - -pub fn name(self: AmdSmi, handle: Handle, buf: *[c.AMDSMI_MAX_STRING_LENGTH]u8) Error![:0]const u8 { - var info: c.amdsmi_asic_info_t = undefined; - try check(self.lib.amdsmi_get_gpu_asic_info(handle, &info)); - @memcpy(buf, &info.market_name); - return std.mem.span(@as([*c]const u8, @ptrCast(buf))); -} - -pub fn powerUsage(self: AmdSmi, handle: Handle) Error!u32 { - var metrics: c.amdsmi_gpu_metrics_t = undefined; - try check(self.lib.amdsmi_get_gpu_metrics_info(handle, &metrics)); - const unsupported16 = std.math.maxInt(u16); - if (metrics.current_socket_power != unsupported16) { - return metrics.current_socket_power; - } - if (metrics.average_socket_power != unsupported16) { - return metrics.average_socket_power; - } - return error.not_supported; -} - -pub fn powerLimit(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_power_info_t = undefined; - try check(self.lib.amdsmi_get_power_info(handle, &info)); - return info.power_limit; -} - -pub fn temperature(self: AmdSmi, handle: Handle) Error!i64 { - var temp: i64 = 0; - try check(self.lib.amdsmi_get_temp_metric(handle, c.AMDSMI_TEMPERATURE_TYPE_EDGE, c.AMDSMI_TEMP_CURRENT, &temp)); - return temp; -} - -pub fn fanSpeed(self: AmdSmi, handle: Handle) Error!i64 { - var speed: i64 = 0; - try check(self.lib.amdsmi_get_gpu_fan_speed(handle, @as(u32, 0), &speed)); - return speed; -} - -pub fn gpuUtil(self: AmdSmi, handle: Handle) Error!u32 { - var usage: c.amdsmi_engine_usage_t = undefined; - try check(self.lib.amdsmi_get_gpu_activity(handle, &usage)); - return usage.gfx_activity; -} - -pub fn clockGraphics(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_clk_info_t = undefined; - try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); - return info.clk; -} - -pub fn maxClockGraphics(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_clk_info_t = undefined; - try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SYS, &info)); - return info.max_clk; -} - -pub fn clockMem(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_clk_info_t = undefined; - try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); - return info.clk; -} - -pub fn maxClockMem(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_clk_info_t = undefined; - try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_MEM, &info)); - return info.max_clk; -} - -pub fn clockSoc(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_clk_info_t = undefined; - try check(self.lib.amdsmi_get_clock_info(handle, c.AMDSMI_CLK_TYPE_SOC, &info)); - return info.clk; -} - -pub fn memTotal(self: AmdSmi, handle: Handle) Error!u64 { - var total: u64 = 0; - try check(self.lib.amdsmi_get_gpu_memory_total(handle, c.AMDSMI_MEM_TYPE_VRAM, &total)); - return total; -} - -pub fn memUsed(self: AmdSmi, handle: Handle) Error!u64 { - var used: u64 = 0; - try check(self.lib.amdsmi_get_gpu_memory_usage(handle, c.AMDSMI_MEM_TYPE_VRAM, &used)); - return used; -} - -pub fn pcieWidth(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_pcie_info_t = undefined; - try check(self.lib.amdsmi_get_pcie_info(handle, &info)); - return @intCast(info.pcie_metric.pcie_width); -} - -pub fn pcieBandwidth(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_pcie_info_t = undefined; - try check(self.lib.amdsmi_get_pcie_info(handle, &info)); - return info.pcie_metric.pcie_bandwidth; -} - -pub fn pcieLinkGen(self: AmdSmi, handle: Handle) Error!u32 { - var info: c.amdsmi_pcie_info_t = undefined; - try check(self.lib.amdsmi_get_pcie_info(handle, &info)); - return info.pcie_static.pcie_interface_version; -} - -pub fn bdfId(self: AmdSmi, handle: Handle) Error!u64 { - var bdf_id: u64 = 0; - try check(self.lib.amdsmi_get_gpu_bdf_id(handle, &bdf_id)); - return bdf_id; -} - -pub const driver_version_buf_len = @typeInfo(@TypeOf(@as(c.amdsmi_driver_info_t, undefined).driver_version)).array.len; - -pub fn driverVersion(self: AmdSmi, handle: Handle, buf: *[driver_version_buf_len]u8) Error![:0]const u8 { - var info: c.amdsmi_driver_info_t = std.mem.zeroes(c.amdsmi_driver_info_t); - try check(self.lib.amdsmi_get_gpu_driver_info(handle, &info)); - @memcpy(buf, &info.driver_version); - return std.mem.span(@as([*c]const u8, @ptrCast(buf))); -} - -pub fn processList(self: AmdSmi, allocator: std.mem.Allocator, handle: Handle) (Error || error{OutOfMemory})![]const ProcInfo { - var count: u32 = 0; - try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, null)); - if (count == 0) { - return &.{}; - } - const procs = try allocator.alloc(ProcInfo, count); - errdefer allocator.free(procs); - try check(self.lib.amdsmi_get_gpu_process_list(handle, &count, @ptrCast(procs.ptr))); - return procs; -} - -// Error handling - -pub const ReturnError = error{ - inval, - not_supported, - not_yet_implemented, - fail_load_module, - fail_load_symbol, - drm_error, - api_failed, - timeout, - retry, - no_perm, - interrupt, - io_error, - address_fault, - file_error, - out_of_resources, - internal_exception, - input_out_of_bounds, - init_error, - refcount_overflow, - directory_not_found, - busy, - not_found, - not_init, - no_slot, - driver_not_loaded, - no_data, - insufficient_size, - unexpected_size, - unexpected_data, - unknown_error, -}; - -fn check(ret: c.amdsmi_status_t) Error!void { - if (ret == c.AMDSMI_STATUS_SUCCESS) { - return; - } - - return switch (ret) { - c.AMDSMI_STATUS_INVAL => error.inval, - c.AMDSMI_STATUS_NOT_SUPPORTED => error.not_supported, - c.AMDSMI_STATUS_NOT_YET_IMPLEMENTED => error.not_yet_implemented, - c.AMDSMI_STATUS_FAIL_LOAD_MODULE => error.fail_load_module, - c.AMDSMI_STATUS_FAIL_LOAD_SYMBOL => error.fail_load_symbol, - c.AMDSMI_STATUS_DRM_ERROR => error.drm_error, - c.AMDSMI_STATUS_API_FAILED => error.api_failed, - c.AMDSMI_STATUS_TIMEOUT => error.timeout, - c.AMDSMI_STATUS_RETRY => error.retry, - c.AMDSMI_STATUS_NO_PERM => error.no_perm, - c.AMDSMI_STATUS_INTERRUPT => error.interrupt, - c.AMDSMI_STATUS_IO => error.io_error, - c.AMDSMI_STATUS_ADDRESS_FAULT => error.address_fault, - c.AMDSMI_STATUS_FILE_ERROR => error.file_error, - c.AMDSMI_STATUS_OUT_OF_RESOURCES => error.out_of_resources, - c.AMDSMI_STATUS_INTERNAL_EXCEPTION => error.internal_exception, - c.AMDSMI_STATUS_INPUT_OUT_OF_BOUNDS => error.input_out_of_bounds, - c.AMDSMI_STATUS_INIT_ERROR => error.init_error, - c.AMDSMI_STATUS_REFCOUNT_OVERFLOW => error.refcount_overflow, - c.AMDSMI_STATUS_DIRECTORY_NOT_FOUND => error.directory_not_found, - c.AMDSMI_STATUS_BUSY => error.busy, - c.AMDSMI_STATUS_NOT_FOUND => error.not_found, - c.AMDSMI_STATUS_NOT_INIT => error.not_init, - c.AMDSMI_STATUS_NO_SLOT => error.no_slot, - c.AMDSMI_STATUS_DRIVER_NOT_LOADED => error.driver_not_loaded, - c.AMDSMI_STATUS_NO_DATA => error.no_data, - c.AMDSMI_STATUS_INSUFFICIENT_SIZE => error.insufficient_size, - c.AMDSMI_STATUS_UNEXPECTED_SIZE => error.unexpected_size, - c.AMDSMI_STATUS_UNEXPECTED_DATA => error.unexpected_data, - else => error.unknown_error, - }; -} diff --git a/bin/zml-smi/platforms/amdsmi/metrics.zig b/bin/zml-smi/platforms/amdsmi/metrics.zig deleted file mode 100644 index 8aa0487c..00000000 --- a/bin/zml-smi/platforms/amdsmi/metrics.zig +++ /dev/null @@ -1,151 +0,0 @@ -const std = @import("std"); -const AmdSmi = @import("amdsmi.zig"); -const device_info = @import("zml-smi/info").device_info; -const DeviceInfo = device_info.DeviceInfo; -const GpuInfo = device_info.GpuInfo; -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; -const Collector = @import("zml-smi/collector").Collector; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const process = @import("process.zig"); - -pub fn start(collector: *Collector) !void { - const amdsmi = try collector.arena.create(AmdSmi); - amdsmi.* = try AmdSmi.init(collector.arena); - - const count = amdsmi.deviceCount(); - const dev_offset: u16 = @intCast(collector.device_infos.items.len); - - for (0..count) |i| { - const dev = Device.open(amdsmi, @intCast(i)) catch continue; - const initial: GpuInfo = .{ - .name = dev.name(collector.arena) catch null, - .driver_version = dev.driverVersion(collector.arena) catch null, - }; - const info = try collector.addDevice(.{ .rocm = .{ .values = .{ initial, initial } } }); - try collector.spawnPoll(pollOnce, .{ null, &info.rocm, dev }); - } - - const processes = try collector.createProcessList(); - try process.init(collector, processes, amdsmi, dev_offset); -} - -const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); - -const Device = struct { - amdsmi: *const AmdSmi, - handle: AmdSmi.Handle, - - pub fn open(amdsmi: *const AmdSmi, index: u32) !Device { - return .{ .amdsmi = amdsmi, .handle = try amdsmi.handleByIndex(index) }; - } - - fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { - var buf: [AmdSmi.name_buf_len]u8 = undefined; - const slice = try self.amdsmi.name(self.handle, &buf); - - return try arena.dupe(u8, slice); - } - - fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { - var buf: [AmdSmi.driver_version_buf_len]u8 = undefined; - const slice = try self.amdsmi.driverVersion(self.handle, &buf); - return try arena.dupe(u8, slice); - } - - // Power - pub fn powerUsage(self: Device) !u64 { - const pw = try self.amdsmi.powerUsage(self.handle); - return @as(u64, pw) * 1000; - } - - pub fn powerLimit(self: Device) !u64 { - const limit = try self.amdsmi.powerLimit(self.handle); - if (limit == std.math.maxInt(u32)) { - return error.not_supported; - } - - return @as(u64, limit) / 1000; - } - - // Thermal - pub fn temperature(self: Device) !u64 { - const temp = try self.amdsmi.temperature(self.handle); - return @intCast(temp); - } - - pub fn fanSpeed(self: Device) !u64 { - const speed = try self.amdsmi.fanSpeed(self.handle); - return @intCast(@divTrunc(speed * 100, 255)); - } - - // Utilization - pub fn gpuUtil(self: Device) !u64 { - return try self.amdsmi.gpuUtil(self.handle); - } - - // Clocks - pub fn clockGraphics(self: Device) !u64 { - return try self.amdsmi.clockGraphics(self.handle); - } - - pub fn clockSoc(self: Device) !u64 { - return try self.amdsmi.clockSoc(self.handle); - } - - pub fn clockMem(self: Device) !u64 { - return try self.amdsmi.clockMem(self.handle); - } - - pub fn maxClockGraphics(self: Device) !u64 { - return try self.amdsmi.maxClockGraphics(self.handle); - } - - pub fn maxClockMem(self: Device) !u64 { - return try self.amdsmi.maxClockMem(self.handle); - } - - // Memory - pub fn memUsed(self: Device) !u64 { - return self.amdsmi.memUsed(self.handle); - } - - pub fn memTotal(self: Device) !u64 { - return self.amdsmi.memTotal(self.handle); - } - - // PCIe - pub fn pcieBandwidth(self: Device) !u64 { - const bw = try self.amdsmi.pcieBandwidth(self.handle); - if (bw == std.math.maxInt(u32)) { - return error.not_supported; - } - - return bw; - } - - pub fn pcieLinkGen(self: Device) !u64 { - return try self.amdsmi.pcieLinkGen(self.handle); - } - - pub fn pcieLinkWidth(self: Device) !u64 { - return try self.amdsmi.pcieWidth(self.handle); - } -}; - -const metrics = .{ - .{ .field = "power_mw", .query = Device.powerUsage }, - .{ .field = "power_limit_mw", .query = Device.powerLimit }, - .{ .field = "temperature", .query = Device.temperature }, - .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, - .{ .field = "util_percent", .query = Device.gpuUtil }, - .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, - .{ .field = "clock_soc_mhz", .query = Device.clockSoc }, - .{ .field = "clock_mem_mhz", .query = Device.clockMem }, - .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, - .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, - .{ .field = "mem_used_bytes", .query = Device.memUsed }, - .{ .field = "mem_total_bytes", .query = Device.memTotal }, - .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, - .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, - .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, -}; diff --git a/bin/zml-smi/platforms/amdsmi/process.zig b/bin/zml-smi/platforms/amdsmi/process.zig deleted file mode 100644 index 8d1fdd01..00000000 --- a/bin/zml-smi/platforms/amdsmi/process.zig +++ /dev/null @@ -1,139 +0,0 @@ -const std = @import("std"); -const c = std.c; -const AmdSmi = @import("amdsmi.zig"); -const pi = @import("zml-smi/info").process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Collector = @import("zml-smi/collector").Collector; - -const bdf_len = "0000:00:00.0".len; - -pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16) !void { - const device_count = amdsmi.deviceCount(); - const pci_slots = try collector.arena.alloc(?[bdf_len]u8, device_count); - - for (pci_slots, 0..) |*slot, i| { - const handle = amdsmi.handleByIndex(@intCast(i)) catch { - slot.* = null; - continue; - }; - - slot.* = formatBdf(amdsmi.bdfId(handle) catch { - slot.* = null; - continue; - }); - } - - try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, amdsmi, dev_offset, device_count, pci_slots }); -} - -fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, amdsmi: *const AmdSmi, dev_offset: u16, device_count: u32, pci_slots: []?[bdf_len]u8) void { - const back = list.back(); - - back.clearRetainingCapacity(); - - for (0..device_count) |dev_idx| { - const handle = amdsmi.handleByIndex(@intCast(dev_idx)) catch continue; - - const procs = blk: { - const saved = c.dup(c.STDERR_FILENO); - const devnull = c.open("/dev/null", .{ .ACCMODE = .WRONLY }); - - if (devnull >= 0) { - if (saved >= 0) { - _ = c.dup2(devnull, c.STDERR_FILENO); - } - - _ = c.close(devnull); - } - - defer if (saved >= 0) { - _ = c.dup2(saved, c.STDERR_FILENO); - _ = c.close(saved); - }; - - break :blk amdsmi.processList(allocator, handle) catch continue; - }; - defer { - if (procs.len > 0) { - allocator.free(procs); - } - } - - const pci_slot = if (dev_idx < pci_slots.len) &(pci_slots[dev_idx] orelse continue) else continue; - - for (procs) |proc| { - back.append(allocator, .{ - .pid = proc.pid, - .device_idx = @intCast(dev_idx + dev_offset), - .dev_util_percent = @intCast(proc.engine_usage.gfx + proc.engine_usage.enc), - .dev_mem_kib = readProcessGpuVram(io, proc.pid, pci_slot), - }) catch break; - } - } - - list.swap(); -} - -/// Reads per-GPU VRAM from /proc//fdinfo/, filtered by PCI slot. -/// amdsmi reports global VRAM across all GPUs; fdinfo gives per-GPU values. -fn readProcessGpuVram(io: std.Io, pid: u32, pci_slot: *const [bdf_len]u8) u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const dir_path = std.fmt.bufPrint(&path_buf, "/proc/{d}/fdinfo", .{pid}) catch return 0; - - var dir = std.Io.Dir.openDirAbsolute(io, dir_path, .{ .iterate = true }) catch return 0; - defer dir.close(io); - - var total_vram_kib: u64 = 0; - var it = dir.iterate(); - while (it.next(io) catch null) |entry| { - var file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const file_path = std.fmt.bufPrint(&file_buf, "{s}/{s}", .{ dir_path, entry.name }) catch continue; - - total_vram_kib += parseFdinfoVram(io, file_path, pci_slot); - } - - return total_vram_kib; -} - -fn parseFdinfoVram(io: std.Io, path: []const u8, pci_slot: *const [bdf_len]u8) u64 { - var file = std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }) catch return 0; - defer file.close(io); - - var read_buf: [4096]u8 = undefined; - var reader = file.reader(io, &read_buf); - - var found_pci = false; - var vram_kib: ?u64 = null; - - while (true) { - const line = reader.interface.takeDelimiterExclusive('\n') catch break; - reader.interface.toss(1); - - if (std.mem.indexOf(u8, line, pci_slot) != null) found_pci = true; - - // Support both new and old fdinfo keys (see amdgpu_fdinfo.c) - inline for ([_][]const u8{ "drm-memory-vram:\t", "vram mem:\t" }) |key| { - if (std.mem.startsWith(u8, line, key)) { - const rest = line[key.len..]; - const end = std.mem.indexOfAny(u8, rest, " \t\n") orelse rest.len; - vram_kib = std.fmt.parseInt(u64, rest[0..end], 10) catch 0; - } - } - - if (found_pci and vram_kib != null) return vram_kib.?; - } - - return 0; -} - -fn formatBdf(bdf_id: u64) [bdf_len]u8 { - var buf: [bdf_len]u8 = undefined; - _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ - (bdf_id >> 16) & 0xFFFF, - (bdf_id >> 8) & 0xFF, - (bdf_id >> 3) & 0x1F, - bdf_id & 0x7, - }) catch unreachable; - - return buf; -} diff --git a/bin/zml-smi/platforms/linux/BUILD.bazel b/bin/zml-smi/platforms/linux/BUILD.bazel deleted file mode 100644 index 57bb96d7..00000000 --- a/bin/zml-smi/platforms/linux/BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "linux", - import_name = "zml-smi/platforms/linux", - main = "linux.zig", - srcs = [ - "metrics.zig", - "process.zig", - ], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:sysfs", - ], -) diff --git a/bin/zml-smi/platforms/linux/linux.zig b/bin/zml-smi/platforms/linux/linux.zig deleted file mode 100644 index ffab3ec4..00000000 --- a/bin/zml-smi/platforms/linux/linux.zig +++ /dev/null @@ -1,2 +0,0 @@ -pub const metrics = @import("metrics.zig"); -pub const process = @import("process.zig"); diff --git a/bin/zml-smi/platforms/linux/metrics.zig b/bin/zml-smi/platforms/linux/metrics.zig deleted file mode 100644 index 5a69663a..00000000 --- a/bin/zml-smi/platforms/linux/metrics.zig +++ /dev/null @@ -1,97 +0,0 @@ -const std = @import("std"); -const sysfs = @import("zml-smi/sysfs"); -const host_info = @import("zml-smi/info").host_info; -const HostInfo = host_info.HostInfo; -const HostData = host_info.HostData; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const Collector = @import("zml-smi/collector").Collector; - -pub fn init(collector: *Collector, info: *HostInfo) !void { - const poll_arena = try collector.createPollArena(); - const host: Host = .{ .io = collector.io, .arena = poll_arena }; - - // read once metrics - var initial = info.front().*; - initial.hostname = host.hostname(collector.arena) catch null; - initial.kernel = host.kernel(collector.arena) catch null; - initial.cpu_name = host.cpuName(collector.arena) catch null; - initial.cpu_cores = host.cpuCores() catch null; - initial.mem_total_kib = host.memTotal() catch null; - info.back().* = initial; - info.swap(); - - try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); -} - -const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); - -const Host = struct { - io: std.Io, - arena: *std.heap.ArenaAllocator, - - pub fn hostname(self: Host, allocator: std.mem.Allocator) ![]const u8 { - return sysfs.readString(allocator, self.io, "/proc/sys/kernel/hostname"); - } - - pub fn kernel(self: Host, allocator: std.mem.Allocator) ![]const u8 { - return sysfs.readString(allocator, self.io, "/proc/sys/kernel/osrelease"); - } - - pub fn cpuName(self: Host, allocator: std.mem.Allocator) ![]const u8 { - return sysfs.readFieldString(allocator, self.io, "/proc/cpuinfo", "model name"); - } - - pub fn cpuCores(self: Host) !u64 { - const data = try sysfs.readString(self.arena.allocator(), self.io, "/sys/devices/system/cpu/present"); - const dash = std.mem.indexOfScalar(u8, data, '-') orelse return 1; - - return (try std.fmt.parseInt(u64, data[dash + 1 ..], 10)) + 1; - } - - pub fn memTotal(self: Host) !u64 { - return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemTotal"); - } - - pub fn memAvailable(self: Host) !u64 { - return sysfs.readFieldInt(self.arena.allocator(), self.io, "/proc/meminfo", "MemAvailable"); - } - - fn parseLoadAvg(self: Host, comptime n: usize) !f32 { - const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/loadavg"); - var iter = std.mem.splitScalar(u8, data, ' '); - - var i: usize = 0; - while (iter.next()) |tok| : (i += 1) { - if (i == n) return std.fmt.parseFloat(f32, tok); - } - - return error.NotFound; - } - - pub fn load1(self: Host) !f32 { - return self.parseLoadAvg(0); - } - - pub fn load5(self: Host) !f32 { - return self.parseLoadAvg(1); - } - - pub fn load15(self: Host) !f32 { - return self.parseLoadAvg(2); - } - - pub fn uptime(self: Host) !u64 { - const data = try sysfs.readString(self.arena.allocator(), self.io, "/proc/uptime"); - const dot = std.mem.indexOfScalar(u8, data, '.') orelse data.len; - - return std.fmt.parseInt(u64, data[0..dot], 10); - } -}; - -const metrics = .{ - .{ .field = "mem_available_kib", .query = Host.memAvailable }, - .{ .field = "load_1", .query = Host.load1 }, - .{ .field = "load_5", .query = Host.load5 }, - .{ .field = "load_15", .query = Host.load15 }, - .{ .field = "uptime_seconds", .query = Host.uptime }, -}; diff --git a/bin/zml-smi/platforms/linux/process.zig b/bin/zml-smi/platforms/linux/process.zig deleted file mode 100644 index c57d1725..00000000 --- a/bin/zml-smi/platforms/linux/process.zig +++ /dev/null @@ -1,118 +0,0 @@ -const std = @import("std"); -const sysfs = @import("zml-smi/sysfs"); -const pi = @import("zml-smi/info").process_info; - -const ProcessInfo = pi.ProcessInfo; - -pub const ProcessEnricher = struct { - gpa: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - prev_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}, - prev_total_ticks: u64 = 0, - - pub fn init(gpa: std.mem.Allocator, io: std.Io) !ProcessEnricher { - var arena = std.heap.ArenaAllocator.init(gpa); - errdefer arena.deinit(); - - const prev_total_ticks = readTotalCpuTicks(arena.allocator(), io); - var enricher: ProcessEnricher = .{ - .gpa = gpa, - .arena = arena, - .prev_total_ticks = prev_total_ticks, - .prev_ticks = .{}, - }; - - try enricher.prev_ticks.ensureTotalCapacity(gpa, 64); - - return enricher; - } - - pub fn deinit(self: *ProcessEnricher) void { - self.prev_ticks.deinit(self.gpa); - self.arena.deinit(); - } - - pub fn enrich(self: *ProcessEnricher, io: std.Io, procs: []ProcessInfo) void { - _ = self.arena.reset(.retain_capacity); - const alloc = self.arena.allocator(); - - const curr_total = readTotalCpuTicks(alloc, io); - const delta_total = curr_total -| self.prev_total_ticks; - - var curr_ticks: std.AutoHashMapUnmanaged(u32, u64) = .{}; - curr_ticks.ensureTotalCapacity(self.gpa, @intCast(procs.len)) catch return; - - for (procs) |*info| { - if (info.remote) continue; - // Read Uid, VmRSS from /proc/{pid}/status - const status_path = std.fmt.allocPrint(alloc, "/proc/{d}/status", .{info.pid}) catch continue; - const uid: u32 = @intCast(sysfs.readFieldInt(alloc, io, status_path, "Uid") catch 0); - info.uid = uid; - info.username = lookupUsername(alloc, uid) catch continue; - info.rss_kib = sysfs.readFieldInt(alloc, io, status_path, "VmRSS") catch 0; - - // Read cmdline - const cmdline_path = std.fmt.allocPrint(alloc, "/proc/{d}/cmdline", .{info.pid}) catch continue; - info.comm = readCmdline(alloc, io, cmdline_path) catch continue; - - // CPU% delta - const ticks = readProcTicks(alloc, io, info.pid) orelse continue; - if (delta_total > 0) { - if (self.prev_ticks.get(info.pid)) |prev| { - const delta_proc = ticks -| prev; - info.cpu_percent = @intCast(@min(delta_proc *| 1000 / delta_total, std.math.maxInt(u16))); - } - } - - curr_ticks.getOrPutAssumeCapacity(info.pid).value_ptr.* = ticks; - } - - self.prev_ticks.deinit(self.gpa); - self.prev_ticks = curr_ticks; - self.prev_total_ticks = curr_total; - } -}; - -fn lookupUsername(allocator: std.mem.Allocator, uid: u32) ![]const u8 { - const pw = std.c.getpwuid(uid) orelse return error.NoUser; - const name = std.mem.span(pw.name orelse return error.NoName); - - return try allocator.dupe(u8, name); -} - -/// Extract utime + stime from /proc/{pid}/stat. -fn readProcTicks(allocator: std.mem.Allocator, io: std.Io, pid: u32) ?u64 { - const path = std.fmt.allocPrint(allocator, "/proc/{d}/stat", .{pid}) catch return null; - const data = sysfs.readFirstLine(allocator, io, path) catch return null; - - // Fields after ") ": state=0, ppid=1, ..., utime=11, stime=12 - const rest = data[(std.mem.lastIndexOfScalar(u8, data, ')') orelse return null) + 2 ..]; - - return sysfs.nthTokenInt(rest, 11) + sysfs.nthTokenInt(rest, 12); -} - -fn readCmdline(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { - const data = try sysfs.readFirstLine(allocator, io, path); - - for (data) |*c| { - if (c.* == 0) { - c.* = ' '; - } - } - - return std.mem.trimEnd(u8, data, &std.ascii.whitespace); -} - -fn readTotalCpuTicks(allocator: std.mem.Allocator, io: std.Io) u64 { - const data = sysfs.readFirstLine(allocator, io, "/proc/stat") catch return 0; - - var total: u64 = 0; - var iter = std.mem.tokenizeAny(u8, data, " \t"); - - _ = iter.next(); // skip "cpu" - while (iter.next()) |tok| { - total += std.fmt.parseInt(u64, tok, 10) catch 0; - } - - return total; -} diff --git a/bin/zml-smi/platforms/macos/BUILD.bazel b/bin/zml-smi/platforms/macos/BUILD.bazel deleted file mode 100644 index a92e298e..00000000 --- a/bin/zml-smi/platforms/macos/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "macos", - import_name = "zml-smi/platforms/macos", - main = "macos.zig", - srcs = [ - "metrics.zig", - "process.zig", - ], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - ], -) diff --git a/bin/zml-smi/platforms/macos/macos.zig b/bin/zml-smi/platforms/macos/macos.zig deleted file mode 100644 index ffab3ec4..00000000 --- a/bin/zml-smi/platforms/macos/macos.zig +++ /dev/null @@ -1,2 +0,0 @@ -pub const metrics = @import("metrics.zig"); -pub const process = @import("process.zig"); diff --git a/bin/zml-smi/platforms/macos/metrics.zig b/bin/zml-smi/platforms/macos/metrics.zig deleted file mode 100644 index 616168cd..00000000 --- a/bin/zml-smi/platforms/macos/metrics.zig +++ /dev/null @@ -1,161 +0,0 @@ -const std = @import("std"); -const host_info = @import("zml-smi/info").host_info; -const HostInfo = host_info.HostInfo; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const Collector = @import("zml-smi/collector").Collector; - -const c = std.c; - -const vm_statistics64 = extern struct { - free_count: c_uint, - active_count: c_uint, - inactive_count: c_uint, - wire_count: c_uint, - zero_fill_count: u64, - reactivations: u64, - pageins: u64, - pageouts: u64, - faults: u64, - cow_faults: u64, - lookups: u64, - hits: u64, - purges: u64, - purgeable_count: c_uint, - speculative_count: c_uint, - decompressions: u64, - compressions: u64, - swapins: u64, - swapouts: u64, - compressor_page_count: c_uint, - throttled_count: c_uint, - external_page_count: c_uint, - internal_page_count: c_uint, - total_uncompressed_pages_in_compressor: u64, -}; - -const HOST_VM_INFO64: c_int = 4; -const HOST_VM_INFO64_COUNT: c_uint = @sizeOf(vm_statistics64) / @sizeOf(c_int); -const KERN_SUCCESS: c_int = 0; - -extern "c" fn host_statistics64(host: c.mach_port_t, flavor: c_int, info: *anyopaque, count: *c_uint) c.kern_return_t; -extern "c" fn getloadavg(loadavg: *[3]f64, nelem: c_int) c_int; - -pub fn init(collector: *Collector, info: *HostInfo) !void { - const poll_arena = try collector.createPollArena(); - const host: Host = .{}; - - var initial = info.front().*; - initial.hostname = sysctlString(collector.arena, "kern.hostname") catch null; - initial.kernel = sysctlString(collector.arena, "kern.osrelease") catch null; - initial.cpu_name = sysctlString(collector.arena, "machdep.cpu.brand_string") catch null; - initial.cpu_cores = host.cpuCores() catch null; - initial.mem_total_kib = host.memTotal() catch null; - info.back().* = initial; - info.swap(); - - try collector.spawnPoll(pollOnce, .{ poll_arena, info, host }); -} - -const pollOnce = poll_metrics.poll(*HostInfo, Host, metrics); - -const Host = struct { - pub fn cpuCores(_: Host) !u64 { - return @intCast(try sysctlU32("hw.logicalcpu")); - } - - pub fn memTotal(_: Host) !u64 { - return (try sysctlU64("hw.memsize")) / 1024; - } - - pub fn memAvailable(_: Host) !u64 { - var vm_stat: vm_statistics64 = undefined; - var count: c_uint = HOST_VM_INFO64_COUNT; - - if (host_statistics64(c.mach_host_self(), HOST_VM_INFO64, &vm_stat, &count) != KERN_SUCCESS) { - return error.HostStatsFailed; - } - - var page_size: u64 = 0; - if (c._host_page_size(c.mach_host_self(), &page_size) != KERN_SUCCESS) { - return error.PageSizeFailed; - } - - const available_pages: u64 = @as(u64, vm_stat.free_count) + - @as(u64, vm_stat.inactive_count) + - @as(u64, vm_stat.purgeable_count); - - return (available_pages * page_size) / 1024; - } - - fn loadAvg() ![3]f64 { - var avg: [3]f64 = undefined; - if (getloadavg(&avg, 3) != 3) { - return error.LoadAvgFailed; - } - - return avg; - } - - pub fn load1(_: Host) !f32 { - return @floatCast((try loadAvg())[0]); - } - - pub fn load5(_: Host) !f32 { - return @floatCast((try loadAvg())[1]); - } - - pub fn load15(_: Host) !f32 { - return @floatCast((try loadAvg())[2]); - } - - pub fn uptime(_: Host) !u64 { - var boottime: c.timeval = undefined; - var len: usize = @sizeOf(c.timeval); - if (c.sysctlbyname("kern.boottime", std.mem.asBytes(&boottime), &len, null, 0) != 0) { - return error.SysctlFailed; - } - - var now: c.timeval = undefined; - _ = c.gettimeofday(&now, null); - - return @intCast(now.sec - boottime.sec); - } -}; - -fn sysctlString(allocator: std.mem.Allocator, name: [*:0]const u8) ![]const u8 { - var buf: [256]u8 = undefined; - var len: usize = buf.len; - if (c.sysctlbyname(name, &buf, &len, null, 0) != 0) { - return error.SysctlFailed; - } - - return allocator.dupe(u8, std.mem.span(@as([*:0]const u8, @ptrCast(buf[0..len])))); -} - -fn sysctlU64(name: [*:0]const u8) !u64 { - var value: u64 = 0; - var len: usize = @sizeOf(u64); - if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { - return error.SysctlFailed; - } - - return value; -} - -fn sysctlU32(name: [*:0]const u8) !u32 { - var value: u32 = 0; - var len: usize = @sizeOf(u32); - if (c.sysctlbyname(name, std.mem.asBytes(&value), &len, null, 0) != 0) { - return error.SysctlFailed; - } - - return value; -} - -const metrics = .{ - .{ .field = "mem_available_kib", .query = Host.memAvailable }, - .{ .field = "load_1", .query = Host.load1 }, - .{ .field = "load_5", .query = Host.load5 }, - .{ .field = "load_15", .query = Host.load15 }, - .{ .field = "uptime_seconds", .query = Host.uptime }, -}; diff --git a/bin/zml-smi/platforms/macos/process.zig b/bin/zml-smi/platforms/macos/process.zig deleted file mode 100644 index a48e9831..00000000 --- a/bin/zml-smi/platforms/macos/process.zig +++ /dev/null @@ -1,12 +0,0 @@ -const std = @import("std"); -const pi = @import("zml-smi/info").process_info; - -pub const ProcessEnricher = struct { - pub fn init(_: std.mem.Allocator, _: std.Io) !ProcessEnricher { - return .{}; - } - - pub fn deinit(_: *ProcessEnricher) void {} - - pub fn enrich(_: *ProcessEnricher, _: std.Io, _: []pi.ProcessInfo) void {} -}; diff --git a/bin/zml-smi/platforms/neuron/BUILD.bazel b/bin/zml-smi/platforms/neuron/BUILD.bazel deleted file mode 100644 index c7129ac7..00000000 --- a/bin/zml-smi/platforms/neuron/BUILD.bazel +++ /dev/null @@ -1,22 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "neuron", - import_name = "zml-smi/platforms/neuron", - main = "metrics.zig", - srcs = [ - "nrt.zig", - "process.zig", - ], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//bin/zml-smi/utils:dynlib", - "//bin/zml-smi/utils:sandbox", - "//bin/zml-smi/utils:sysfs", - "//platforms/neuron:libnrt_headers", - "//stdx", - ], -) diff --git a/bin/zml-smi/platforms/neuron/metrics.zig b/bin/zml-smi/platforms/neuron/metrics.zig deleted file mode 100644 index 0b5a5ddd..00000000 --- a/bin/zml-smi/platforms/neuron/metrics.zig +++ /dev/null @@ -1,201 +0,0 @@ -const std = @import("std"); -const sysfs = @import("zml-smi/sysfs"); -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; -const device_info = @import("zml-smi/info").device_info; -const DeviceInfo = device_info.DeviceInfo; -const NeuronInfo = device_info.NeuronInfo; -const Collector = @import("zml-smi/collector").Collector; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const Nrt = @import("nrt.zig"); -const process = @import("process.zig"); - -const base_path = "/sys/devices/virtual/neuron_device"; - -pub const UtilAtomic = std.atomic.Value(u64); - -pub fn start(collector: *Collector) !void { - const nrt = try collector.arena.create(Nrt); - nrt.* = try Nrt.init(collector.arena, collector.io); - - if (nrt.handles.len == 0) { - return; - } - - const total_nc = nrt.totalNcCount() catch return; - const nc_per_device = total_nc / @as(u32, @intCast(nrt.handles.len)); - - const total_cores = @as(usize, nrt.handles.len) * nc_per_device; - const util_buf = try collector.arena.alloc(UtilAtomic, total_cores); - for (util_buf) |*u| u.* = .init(0); - - const GiB: u64 = 1024 * 1024 * 1024; - var neuron_infos: std.ArrayList(*DeviceInfo) = .empty; - const dev_offset: u8 = @intCast(collector.device_infos.items.len); - - var ver_buf: [Nrt.version_buf_len]u8 = undefined; - const nrt_ver: ?[]const u8 = if (nrt.version(&ver_buf)) |v| - collector.arena.dupe(u8, v) catch null - else |_| - null; - - var flat_core: usize = 0; - for (nrt.handles, nrt.device_indexes) |dev_handle, device_idx| { - const device_type = Nrt.deviceType(dev_handle); - const mem_per_device: u64 = switch (device_type) { - .inf1 => 8 * GiB, - .inf2_trn1 => 32 * GiB, - .trn2 => 96 * GiB, - .trn3 => 144 * GiB, - else => 0, - }; - - for (0..nc_per_device) |ci| { - const poll_arena = try collector.createPollArena(); - - const dev: Device = .{ - .io = collector.io, - .arena = poll_arena, - .device_idx = @intCast(device_idx), - .core_idx = @intCast(ci), - .mem_per_device = mem_per_device, - .nc_per_device = nc_per_device, - .util_src = &util_buf[flat_core], - }; - - const initial: NeuronInfo = .{ - .name = dev.name(collector.arena) catch null, - .driver_version = nrt_ver, - }; - const info = try collector.addDevice(.{ .neuron = .{ .values = .{ initial, initial } } }); - try neuron_infos.append(collector.arena, info); - try collector.spawnPoll(pollOnce, .{ poll_arena, &info.neuron, dev }); - flat_core += 1; - } - } - - if (neuron_infos.items.len > 0) { - const processes = try collector.createProcessList(); - const warmup = collector.poll_only; - try process.init(collector, processes, nrt, nc_per_device, util_buf, dev_offset, warmup); - - if (warmup) { - for (neuron_infos.items, util_buf) |info, *u| { - const back = info.neuron.back(); - back.* = info.neuron.front().*; - back.util_percent = u.load(.acquire); - info.neuron.swap(); - } - } - } -} - -const pollOnce = poll_metrics.poll(*DoubleBuffer(NeuronInfo), Device, metrics); - -const Device = struct { - io: std.Io, - arena: *std.heap.ArenaAllocator, - device_idx: u32, - core_idx: u32, - mem_per_device: u64 = 0, - nc_per_device: u32 = 1, - util_src: *const UtilAtomic, - - pub fn utilPercent(self: Device) !u64 { - return self.util_src.load(.acquire); - } - - fn devicePath(self: Device, sub_path: []const u8) ![]const u8 { - return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/{s}", .{ self.device_idx, sub_path }); - } - - fn corePath(self: Device, sub_path: []const u8) ![]const u8 { - return std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/{s}", .{ self.device_idx, self.core_idx, sub_path }); - } - - fn name(self: Device, allocator: std.mem.Allocator) ![]const u8 { - return sysfs.readString(allocator, self.io, try self.devicePath("info/architecture/device_name")); - } - - fn readCoreMem(self: Device, comptime subdir: []const u8) !u64 { - return sysfs.readInt(self.arena.allocator(), self.io, try self.corePath("stats/memory_usage/device_mem/" ++ subdir ++ "/total")); - } - - pub fn memTotal(self: Device) !u64 { - if (self.mem_per_device == 0) { - return error.NrtUnavailable; - } - return self.mem_per_device; - } - - // Sysfs-based metrics — RAM is shared across cores, so sum all cores on the device. - pub fn memUsed(self: Device) !u64 { - var total: u64 = 0; - for (0..self.nc_per_device) |ci| { - const path = try std.fmt.allocPrint(self.arena.allocator(), base_path ++ "/neuron{d}/neuron_core{d}/stats/memory_usage/device_mem/total", .{ self.device_idx, ci }); - total += try sysfs.readInt(self.arena.allocator(), self.io, path); - } - - return total; - } - - pub fn tensors(self: Device) !u64 { - return self.readCoreMem("tensors"); - } - - pub fn constants(self: Device) !u64 { - return self.readCoreMem("constants"); - } - - pub fn modelCode(self: Device) !u64 { - return self.readCoreMem("model_code"); - } - - pub fn sharedScratchpad(self: Device) !u64 { - return self.readCoreMem("model_shared_scratchpad"); - } - - pub fn nonsharedScratchpad(self: Device) !u64 { - return self.readCoreMem("nonshared_scratchpad"); - } - - pub fn runtime(self: Device) !u64 { - return self.readCoreMem("runtime_memory"); - } - - pub fn driver(self: Device) !u64 { - return self.readCoreMem("driver_memory"); - } - - pub fn dmaRings(self: Device) !u64 { - return self.readCoreMem("dma_rings"); - } - - pub fn collectives(self: Device) !u64 { - return self.readCoreMem("collectives"); - } - - pub fn notifications(self: Device) !u64 { - return self.readCoreMem("notifications"); - } - - pub fn uncategorized(self: Device) !u64 { - return self.readCoreMem("uncategorized"); - } -}; - -const metrics = .{ - .{ .field = "util_percent", .query = Device.utilPercent }, - .{ .field = "mem_total_bytes", .query = Device.memTotal }, - .{ .field = "mem_used_bytes", .query = Device.memUsed }, - .{ .field = "nc_tensors", .query = Device.tensors }, - .{ .field = "nc_constants", .query = Device.constants }, - .{ .field = "nc_model_code", .query = Device.modelCode }, - .{ .field = "nc_shared_scratchpad", .query = Device.sharedScratchpad }, - .{ .field = "nc_nonshared_scratchpad", .query = Device.nonsharedScratchpad }, - .{ .field = "nc_runtime", .query = Device.runtime }, - .{ .field = "nc_driver", .query = Device.driver }, - .{ .field = "nc_dma_rings", .query = Device.dmaRings }, - .{ .field = "nc_collectives", .query = Device.collectives }, - .{ .field = "nc_notifications", .query = Device.notifications }, - .{ .field = "nc_uncategorized", .query = Device.uncategorized }, -}; diff --git a/bin/zml-smi/platforms/neuron/nrt.zig b/bin/zml-smi/platforms/neuron/nrt.zig deleted file mode 100644 index cd77b3c0..00000000 --- a/bin/zml-smi/platforms/neuron/nrt.zig +++ /dev/null @@ -1,230 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const elf = std.elf; -const stdx = @import("stdx"); -const DynLib = @import("zml-smi/dynlib"); -const sandbox = @import("zml-smi/sandbox"); - -const Nrt = @This(); - -pub const Error = error{ nrt_error, NrtUnavailable }; -pub const AppInfo = c.struct_neuron_app_info; -pub const DeviceType = enum(c_int) { - inf1 = 1, - inf2_trn1 = 2, - trn2 = 3, - trn3 = 4, - _, -}; - -lib: Fns, -private_lib: PrivateFns, -handles: []const *c.ndl_device_t, -device_indexes: []const c_int, - -const Fns = struct { - nrt_get_total_nc_count: *const @TypeOf(c.nrt_get_total_nc_count), - nrt_get_version: *const @TypeOf(c.nrt_get_version), -}; - -const PrivateFns = struct { - nds_open: *const @TypeOf(c.nds_open), - ndl_available_devices: *const @TypeOf(c.ndl_available_devices), - ndl_open_device: *const @TypeOf(c.ndl_open_device), - ndl_close_device: *const @TypeOf(c.ndl_close_device), - ndl_get_all_apps_info: *const @TypeOf(c.ndl_get_all_apps_info), - nds_close: *const @TypeOf(c.nds_close), - nds_get_nc_counter: *const @TypeOf(c.nds_get_nc_counter), -}; - -pub fn init(allocator: std.mem.Allocator, io: std.Io) !Nrt { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = sandbox.path(&path_buf) orelse { - std.log.err("neuron: sandbox path unavailable", .{}); - return error.NrtUnavailable; - }; - - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnrt.so.1" }) catch { - std.log.err("neuron: failed to construct libnrt.so.1 path", .{}); - return error.NrtUnavailable; - }; - - const dl_handle = std.c.dlopen(path, .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { - if (std.c.dlerror()) |err| std.log.err("neuron: dlopen: {s}", .{err}); - return error.NrtUnavailable; - }; - var dynlib: std.DynLib = .{ .inner = .{ .handle = dl_handle } }; - - const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NrtUnavailable; - const private_fns = resolveFromElf(dl_handle, io, "nrt_init") catch |err| { - std.log.err("neuron: ELF resolution failed: {s}", .{@errorName(err)}); - return error.NrtUnavailable; - }; - - var dev_index_buf: [c.MAX_NEURON_DEVICE_COUNT]c_int = undefined; - const count: usize = @intCast(@max(0, private_fns.ndl_available_devices(&dev_index_buf, c.MAX_NEURON_DEVICE_COUNT))); - - var handle_list: std.ArrayList(*c.ndl_device_t) = .empty; - var index_list: std.ArrayList(c_int) = .empty; - - for (dev_index_buf[0..count]) |device_idx| { - var dev: ?*c.ndl_device_t = null; - var t: c.struct_ndl_device_init_param = .{ - .initialize_device = false, - .num_dram_regions = 0, - .map_hbm = false, - .skip_copy_bufs = true, - }; - - if (private_fns.ndl_open_device(device_idx, &t, &dev) == 0) { - if (dev) |d| { - errdefer _ = private_fns.ndl_close_device(d); - try handle_list.append(allocator, d); - try index_list.append(allocator, device_idx); - } - } - } - - return .{ - .lib = fns, - .private_lib = private_fns, - .handles = try handle_list.toOwnedSlice(allocator), - .device_indexes = try index_list.toOwnedSlice(allocator), - }; -} - -pub fn deviceCount(self: Nrt) u32 { - return @intCast(self.handles.len); -} - -pub fn allAppsInfo(self: Nrt, dev: *c.ndl_device_t) Error!struct { ptr: ?[*]AppInfo, count: usize } { - var info: ?[*]AppInfo = null; - var count: usize = 0; - if (self.private_lib.ndl_get_all_apps_info(dev, &info, &count, c.APP_INFO_ALL) != 0) { - return error.nrt_error; - } - return .{ .ptr = info, .count = count }; -} - -pub fn ndsOpen(self: Nrt, dev: *c.ndl_device_t, pid: c.pid_t) Error!*c.nds_instance_t { - var inst: ?*c.nds_instance_t = null; - if (self.private_lib.nds_open(dev, pid, &inst) != 0) { - return error.nrt_error; - } - return inst orelse error.nrt_error; -} - -pub fn ndsClose(self: Nrt, inst: *c.nds_instance_t) void { - _ = self.private_lib.nds_close(inst); -} - -pub fn ncCounter(self: Nrt, inst: *c.nds_instance_t, pnc_index: c_int, counter_index: u32) Error!u64 { - var value: u64 = 0; - if (self.private_lib.nds_get_nc_counter(inst, pnc_index, counter_index, &value) != 0) { - return error.nrt_error; - } - return value; -} - -pub const version_buf_len = c.RT_VERSION_DETAIL_LEN; - -pub fn version(self: Nrt, buf: *[version_buf_len]u8) Error![:0]const u8 { - var ver: c.nrt_version_t = std.mem.zeroes(c.nrt_version_t); - if (self.lib.nrt_get_version(&ver, @sizeOf(c.nrt_version_t)) != 0) { - return error.nrt_error; - } - @memcpy(buf, &ver.rt_detail); - return std.mem.span(@as([*c]const u8, @ptrCast(buf))); -} - -pub fn totalNcCount(self: Nrt) Error!u32 { - var count: u32 = 0; - if (self.lib.nrt_get_total_nc_count(&count) != 0) { - return error.nrt_error; - } - return count; -} - -pub fn hbmSize(dev: *c.ndl_device_t) usize { - return dev.hbm_size; -} - -pub fn deviceType(dev: *c.ndl_device_t) DeviceType { - return @fromBackingInt(@intCast(dev.device_type)); -} - -fn resolveFromElf(handle: *anyopaque, io: std.Io, comptime known_sym: [:0]const u8) !PrivateFns { - const field_names = comptime std.meta.fieldNames(PrivateFns); - - const sym_addr = std.c.dlsym(handle, known_sym) orelse - return error.SymbolNotFound; - - var dl_info: DlInfo = undefined; - if (dladdr(sym_addr, &dl_info) == 0) return error.DladdrFailed; - - const elf_base = @intFromPtr(dl_info.dli_fbase); - const elf_path = std.mem.span(dl_info.dli_fname); - - const file = try std.Io.Dir.openFileAbsolute(io, elf_path, .{}); - defer file.close(io); - - const stat = try file.stat(io); - const mmap = try std.posix.mmap( - null, - @intCast(stat.size), - .{ .READ = true }, - .{ .TYPE = .PRIVATE }, - file.handle, - 0, - ); - defer std.posix.munmap(mmap); - - var result: PrivateFns = undefined; - var remaining: usize = field_names.len; - - const ehdr: *const elf.ElfN.Ehdr = @ptrCast(@alignCast(mmap)); - const shdrs: []const elf.ElfN.Shdr = @as([*]const elf.ElfN.Shdr, @ptrCast(@alignCast(mmap[ehdr.shoff..])))[0..ehdr.shnum]; - - for (shdrs) |sh| { - if (sh.type != .SYMTAB) { - continue; - } - - const strtab_sh = shdrs[sh.link]; - const strtab: []const u8 = mmap[strtab_sh.offset..][0..strtab_sh.size]; - - for (std.mem.bytesAsSlice(elf.ElfN.Sym, mmap[sh.offset..][0..sh.size])) |s| { - if (s.value == 0) { - continue; - } - - const sym_name = std.mem.span(@as([*c]const u8, @ptrCast(strtab[@as(usize, s.name)..]))); - - inline for (field_names) |field_name| { - if (std.mem.eql(u8, sym_name, field_name)) { - @field(result, field_name) = @ptrFromInt(elf_base + @as(usize, s.value)); - remaining -= 1; - } - } - - if (remaining == 0) { - return result; - } - } - } - - if (remaining > 0) { - return error.SymbolResolutionFailed; - } - return result; -} - -const DlInfo = extern struct { - dli_fname: [*:0]const u8, - dli_fbase: *anyopaque, - dli_sname: [*c]const u8, - dli_saddr: ?*anyopaque, -}; - -extern fn dladdr(addr: *anyopaque, info: *DlInfo) c_int; diff --git a/bin/zml-smi/platforms/neuron/process.zig b/bin/zml-smi/platforms/neuron/process.zig deleted file mode 100644 index 2dbf363c..00000000 --- a/bin/zml-smi/platforms/neuron/process.zig +++ /dev/null @@ -1,108 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const Nrt = @import("nrt.zig"); -const smi_info = @import("zml-smi/info"); -const pi = smi_info.process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Collector = @import("zml-smi/collector").Collector; -const UtilAtomic = @import("metrics.zig").UtilAtomic; - -const DeltaState = struct { - prev_total_us: u64 = 0, - prev_timestamp: ?std.Io.Timestamp = null, -}; - -pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, warmup: bool) !void { - if (nrt.handles.len == 0) return; - const delta_states = try collector.arena.alloc(DeltaState, util_buf.len); - @memset(delta_states, .{}); - const args: std.meta.ArgsTuple(@TypeOf(pollOnce)) = .{ collector.gpa, collector.io, list, nrt, nc_per_device, util_buf, dev_offset, delta_states }; - if (warmup) { - @call(.auto, pollOnce, args); - collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; - @call(.auto, pollOnce, args); - collector.io.sleep(.fromMilliseconds(500), .awake) catch {}; - } - - try collector.spawnPoll(pollOnce, args); -} - -fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, nrt: *const Nrt, nc_per_device: u32, util_buf: []UtilAtomic, dev_offset: u16, delta_states: []DeltaState) void { - const back = list.back(); - - back.clearRetainingCapacity(); - - for (nrt.handles, 0..) |handle, dev_i| { - const apps_result = nrt.allAppsInfo(handle) catch continue; - defer { - if (apps_result.ptr) |ptr| { - std.c.free(@ptrCast(ptr)); - } - } - - const apps = if (apps_result.ptr) |ptr| ptr[0..apps_result.count] else continue; - - var total_us_per_core: [c.MAX_NC_PER_DEVICE]u64 = @splat(0); - - for (apps) |*app| { - if (app.pid <= 0) { - continue; - } - - const nds = nrt.ndsOpen(handle, app.pid) catch continue; - defer nrt.ndsClose(nds); - - for (0..nc_per_device) |ci| { - const time_in_use = nrt.ncCounter(nds, @intCast(ci), c.NDS_NC_COUNTER_TIME_IN_USE) catch continue; - total_us_per_core[ci] += time_in_use; - - if (time_in_use == 0) { - continue; - } - - const dev_idx: u32 = @intCast(dev_i * nc_per_device + ci); - - const pid = std.math.cast(u32, app.pid) orelse continue; - var info: pi.ProcessInfo = .{ - .pid = pid, - .device_idx = @intCast(dev_idx + dev_offset), - .dev_mem_kib = @intCast(app.device_mem_size / 1024), - }; - - if (dev_idx < util_buf.len) { - info.dev_util_percent = @intCast(util_buf[dev_idx].load(.acquire)); - } - - if (info.dev_mem_kib != null and info.dev_mem_kib.? > 0) - back.append(allocator, info) catch break; - } - } - - const now: std.Io.Timestamp = .now(io, .awake); - - for (0..nc_per_device) |ci| { - const dev_idx = dev_i * nc_per_device + ci; - if (dev_idx >= delta_states.len) { - break; - } - - const ds = &delta_states[dev_idx]; - const prev_us = ds.prev_total_us; - ds.prev_total_us = total_us_per_core[ci]; - - if (ds.prev_timestamp) |prev| { - const elapsed = prev.untilNow(io, .awake); - const elapsed_us: u64 = @intCast(@divFloor(elapsed.nanoseconds, std.time.ns_per_us)); - - if (elapsed_us > 0) { - const delta_us = total_us_per_core[ci] -| prev_us; - util_buf[dev_idx].store(@min(100, delta_us * 100 / elapsed_us), .release); - } - } - - ds.prev_timestamp = now; - } - } - - list.swap(); -} diff --git a/bin/zml-smi/platforms/nvml/BUILD.bazel b/bin/zml-smi/platforms/nvml/BUILD.bazel deleted file mode 100644 index 0ab0fae1..00000000 --- a/bin/zml-smi/platforms/nvml/BUILD.bazel +++ /dev/null @@ -1,22 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "nvml", - srcs = [ - "nvml.zig", - "process.zig", - ], - import_name = "zml-smi/platforms/nvml", - main = "metrics.zig", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//bin/zml-smi/utils:dynlib", - ] + select({ - "@llvm//platforms/config:linux_x86_64": ["@cuda_nvml_dev_linux_x86_64//:nvml"], - "@llvm//platforms/config:linux_aarch64": ["@cuda_nvml_dev_linux_sbsa//:nvml"], - }), -) diff --git a/bin/zml-smi/platforms/nvml/metrics.zig b/bin/zml-smi/platforms/nvml/metrics.zig deleted file mode 100644 index 213dd253..00000000 --- a/bin/zml-smi/platforms/nvml/metrics.zig +++ /dev/null @@ -1,222 +0,0 @@ -const std = @import("std"); -const Nvml = @import("nvml.zig"); -const device_info = @import("zml-smi/info").device_info; -const GpuInfo = device_info.GpuInfo; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; -const Collector = @import("zml-smi/collector").Collector; -const process = @import("process.zig"); - -const pcie_every: u8 = 4; - -const Slot = struct { - info: *DoubleBuffer(GpuInfo), - dev: Device, -}; - -const Ctx = struct { - gpa: std.mem.Allocator, - nvml: *const Nvml, - slots: []Slot, - processes: *process.ProcessDoubleBuffer, - last_seen_ts: []u64, - dev_offset: u16, - device_count: u32, - tick: u8 = 0, -}; - -pub fn start(collector: *Collector) !void { - const nvml = try collector.arena.create(Nvml); - nvml.* = try Nvml.init(); - - const count = try nvml.deviceCount(); - const dev_offset: u16 = @intCast(collector.device_infos.items.len); - - var slots: std.ArrayList(Slot) = try .initCapacity(collector.arena, count); - for (0..count) |i| { - const dev = Device.open(nvml, @intCast(i)) catch continue; - const initial: GpuInfo = .{ - .name = dev.name(collector.arena) catch null, - .driver_version = dev.driverVersion(collector.arena) catch null, - .cuda_driver_version = dev.cudaDriverVersion(collector.arena) catch null, - }; - const info = try collector.addDevice(.{ .cuda = .{ .values = .{ initial, initial } } }); - slots.appendAssumeCapacity(.{ .info = &info.cuda, .dev = dev }); - } - - const last_seen_ts = try collector.arena.alloc(u64, count); - @memset(last_seen_ts, 0); - - const ctx = try collector.arena.create(Ctx); - ctx.* = .{ - .gpa = collector.gpa, - .nvml = nvml, - .slots = slots.items, - .processes = try collector.createProcessList(), - .last_seen_ts = last_seen_ts, - .dev_offset = dev_offset, - .device_count = count, - }; - try collector.spawnPoll(pollOnce, .{ctx}); -} - -fn pollOnce(ctx: *Ctx) void { - const pcie = ctx.tick % pcie_every == 0; - ctx.tick +%= 1; - - for (ctx.slots) |slot| { - pollDevice(slot.info, slot.dev, pcie); - } - process.pollOnce(ctx.gpa, ctx.processes, ctx.nvml, ctx.dev_offset, ctx.device_count, ctx.last_seen_ts); -} - -fn pollDevice(db: *DoubleBuffer(GpuInfo), dev: Device, pcie: bool) void { - const back = db.back(); - back.* = db.front().*; - poll_metrics.apply(metrics, back, dev); - if (pcie) poll_metrics.apply(pcie_metrics, back, dev); - db.swap(); -} - -const Device = struct { - nvml: *const Nvml, - handle: Nvml.Handle, - - pub fn open(nvml: *const Nvml, index: u32) !Device { - return .{ .nvml = nvml, .handle = try nvml.handleByIndex(index) }; - } - - fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { - var buf: [Nvml.name_buf_len]u8 = undefined; - const slice = try self.nvml.name(self.handle, &buf); - - return try arena.dupe(u8, slice); - } - - fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { - var buf: [Nvml.driver_version_buf_size]u8 = undefined; - const slice = try self.nvml.driverVersion(&buf); - - return try arena.dupe(u8, slice); - } - - fn cudaDriverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { - const v = try self.nvml.cudaDriverVersion(); - return try std.fmt.allocPrint(arena, "{}.{}", .{ - Nvml.cudaDriverVersionMajor(v), - Nvml.cudaDriverVersionMinor(v), - }); - } - - // Power - pub fn powerUsage(self: Device) !u64 { - return @intCast(try self.nvml.powerUsage(self.handle)); - } - - pub fn powerLimit(self: Device) !u64 { - return @intCast(try self.nvml.powerLimit(self.handle)); - } - - // Thermal - pub fn temperature(self: Device) !u64 { - return @intCast(try self.nvml.temperature(self.handle)); - } - - pub fn fanSpeed(self: Device) !u64 { - return @intCast(try self.nvml.fanSpeed(self.handle)); - } - - // Utilization - pub fn gpuUtil(self: Device) !u64 { - return @intCast(try self.nvml.utilizationGpu(self.handle)); - } - - pub fn encoderUtil(self: Device) !u64 { - return @intCast(try self.nvml.encoderUtil(self.handle)); - } - - pub fn decoderUtil(self: Device) !u64 { - return @intCast(try self.nvml.decoderUtil(self.handle)); - } - - // Clocks - pub fn clockGraphics(self: Device) !u64 { - return @intCast(try self.nvml.clockGraphics(self.handle)); - } - - pub fn clockSm(self: Device) !u64 { - return @intCast(try self.nvml.clockSm(self.handle)); - } - - pub fn clockMem(self: Device) !u64 { - return @intCast(try self.nvml.clockMem(self.handle)); - } - - pub fn maxClockGraphics(self: Device) !u64 { - return @intCast(try self.nvml.maxClockGraphics(self.handle)); - } - - pub fn maxClockMem(self: Device) !u64 { - return @intCast(try self.nvml.maxClockMem(self.handle)); - } - - // Memory - pub fn memUsed(self: Device) !u64 { - return self.nvml.memUsed(self.handle); - } - - pub fn memTotal(self: Device) !u64 { - return self.nvml.memTotal(self.handle); - } - - pub fn memBusWidth(self: Device) !u64 { - return @intCast(try self.nvml.memBusWidth(self.handle)); - } - - // PCIe - pub fn pcieTx(self: Device) !u64 { - return @intCast(try self.nvml.pcieTxKBps(self.handle)); - } - - pub fn pcieRx(self: Device) !u64 { - return @intCast(try self.nvml.pcieRxKBps(self.handle)); - } - - pub fn pcieBandwidth(self: Device) !u64 { - return @intCast(try self.nvml.pcieSpeed(self.handle)); - } - - pub fn pcieLinkGen(self: Device) !u64 { - return @intCast(try self.nvml.pcieLinkGen(self.handle)); - } - - pub fn pcieLinkWidth(self: Device) !u64 { - return @intCast(try self.nvml.pcieLinkWidth(self.handle)); - } -}; - -const metrics = .{ - .{ .field = "power_mw", .query = Device.powerUsage }, - .{ .field = "power_limit_mw", .query = Device.powerLimit }, - .{ .field = "temperature", .query = Device.temperature }, - .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, - .{ .field = "util_percent", .query = Device.gpuUtil }, - .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, - .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, - .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, - .{ .field = "clock_sm_mhz", .query = Device.clockSm }, - .{ .field = "clock_mem_mhz", .query = Device.clockMem }, - .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, - .{ .field = "clock_mem_max_mhz", .query = Device.maxClockMem }, - .{ .field = "mem_used_bytes", .query = Device.memUsed }, - .{ .field = "mem_total_bytes", .query = Device.memTotal }, - .{ .field = "mem_bus_width", .query = Device.memBusWidth }, - .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, - .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, - .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, -}; - -const pcie_metrics = .{ - .{ .field = "pcie_tx_kbps", .query = Device.pcieTx }, - .{ .field = "pcie_rx_kbps", .query = Device.pcieRx }, -}; diff --git a/bin/zml-smi/platforms/nvml/nvml.zig b/bin/zml-smi/platforms/nvml/nvml.zig deleted file mode 100644 index e2b36982..00000000 --- a/bin/zml-smi/platforms/nvml/nvml.zig +++ /dev/null @@ -1,349 +0,0 @@ -const std = @import("std"); -const c = @import("c"); -const DynLib = @import("zml-smi/dynlib"); - -const Nvml = @This(); - -pub const driver_version_buf_size = c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE; -pub const Error = ReturnError || error{NvmlUnavailable}; -pub const Handle = c.nvmlDevice_t; -pub const ProcessInfo_t = c.nvmlProcessInfo_t; - -lib: Fns, - -const Fns = struct { - nvmlInit_v2: *const @TypeOf(c.nvmlInit_v2), - nvmlDeviceGetHandleByIndex_v2: *const @TypeOf(c.nvmlDeviceGetHandleByIndex_v2), - nvmlDeviceGetCount_v2: *const @TypeOf(c.nvmlDeviceGetCount_v2), - nvmlDeviceGetName: *const @TypeOf(c.nvmlDeviceGetName), - nvmlDeviceGetPowerUsage: *const @TypeOf(c.nvmlDeviceGetPowerUsage), - nvmlDeviceGetTemperature: *const @TypeOf(c.nvmlDeviceGetTemperature), - nvmlDeviceGetUtilizationRates: *const @TypeOf(c.nvmlDeviceGetUtilizationRates), - nvmlDeviceGetClockInfo: *const @TypeOf(c.nvmlDeviceGetClockInfo), - nvmlDeviceGetMaxClockInfo: *const @TypeOf(c.nvmlDeviceGetMaxClockInfo), - nvmlDeviceGetMemoryInfo: *const @TypeOf(c.nvmlDeviceGetMemoryInfo), - nvmlDeviceGetFanSpeed: *const @TypeOf(c.nvmlDeviceGetFanSpeed), - nvmlDeviceGetEnforcedPowerLimit: *const @TypeOf(c.nvmlDeviceGetEnforcedPowerLimit), - nvmlDeviceGetPcieThroughput: *const @TypeOf(c.nvmlDeviceGetPcieThroughput), - nvmlDeviceGetEncoderUtilization: *const @TypeOf(c.nvmlDeviceGetEncoderUtilization), - nvmlDeviceGetDecoderUtilization: *const @TypeOf(c.nvmlDeviceGetDecoderUtilization), - nvmlDeviceGetCurrPcieLinkGeneration: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkGeneration), - nvmlDeviceGetCurrPcieLinkWidth: *const @TypeOf(c.nvmlDeviceGetCurrPcieLinkWidth), - nvmlDeviceGetMemoryBusWidth: *const @TypeOf(c.nvmlDeviceGetMemoryBusWidth), - nvmlDeviceGetPcieSpeed: *const @TypeOf(c.nvmlDeviceGetPcieSpeed), - nvmlDeviceGetComputeRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetComputeRunningProcesses_v3), - nvmlDeviceGetGraphicsRunningProcesses_v3: *const @TypeOf(c.nvmlDeviceGetGraphicsRunningProcesses_v3), - nvmlDeviceGetProcessUtilization: *const @TypeOf(c.nvmlDeviceGetProcessUtilization), - nvmlSystemGetCudaDriverVersion_v2: *const @TypeOf(c.nvmlSystemGetCudaDriverVersion_v2), - nvmlSystemGetDriverVersion: *const @TypeOf(c.nvmlSystemGetDriverVersion), -}; - -pub fn init() Error!Nvml { - var dynlib: std.DynLib = .{ .inner = .{ - .handle = std.c.dlopen("libnvidia-ml.so.1", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse - std.c.dlopen("libnvidia-ml.so", .{ .LAZY = true, .GLOBAL = true, .NODELETE = true }) orelse { - if (std.c.dlerror()) |err| std.log.err("nvml: dlopen: {s}", .{err}); - return error.NvmlUnavailable; - }, - } }; - const fns = DynLib.lookupStruct(&dynlib, Fns) catch return error.NvmlUnavailable; - try check(fns.nvmlInit_v2()); - return .{ .lib = fns }; -} - -pub fn handleByIndex(self: Nvml, device_id: u32) Error!c.nvmlDevice_t { - var handle: c.nvmlDevice_t = undefined; - try check(self.lib.nvmlDeviceGetHandleByIndex_v2(device_id, &handle)); - return handle; -} - -pub fn deviceCount(self: Nvml) Error!u32 { - var count: c_uint = 0; - try check(self.lib.nvmlDeviceGetCount_v2(&count)); - return @intCast(count); -} - -pub const name_buf_len = c.NVML_DEVICE_NAME_V2_BUFFER_SIZE; - -pub fn name(self: Nvml, handle: c.nvmlDevice_t, buf: *[c.NVML_DEVICE_NAME_V2_BUFFER_SIZE]u8) Error![:0]const u8 { - try check(self.lib.nvmlDeviceGetName(handle, buf, buf.len)); - return std.mem.span(@as([*c]const u8, @ptrCast(buf))); -} - -pub fn powerUsage(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var power_mw: c_uint = 0; - try check(self.lib.nvmlDeviceGetPowerUsage(handle, &power_mw)); - return power_mw; -} - -pub fn temperature(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var temp: c_uint = 0; - try check(self.lib.nvmlDeviceGetTemperature(handle, c.NVML_TEMPERATURE_GPU, &temp)); - return temp; -} - -pub fn utilizationGpu(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var util: c.nvmlUtilization_t = undefined; - try check(self.lib.nvmlDeviceGetUtilizationRates(handle, &util)); - return util.gpu; -} - -pub fn clockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var clock: c_uint = 0; - try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); - return clock; -} - -pub fn clockSm(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var clock: c_uint = 0; - try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_SM, &clock)); - return clock; -} - -pub fn clockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var clock: c_uint = 0; - try check(self.lib.nvmlDeviceGetClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); - return clock; -} - -pub fn maxClockGraphics(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var clock: c_uint = 0; - try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_GRAPHICS, &clock)); - return clock; -} - -pub fn maxClockMem(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var clock: c_uint = 0; - try check(self.lib.nvmlDeviceGetMaxClockInfo(handle, c.NVML_CLOCK_MEM, &clock)); - return clock; -} - -pub fn memTotal(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { - var mem: c.nvmlMemory_t = undefined; - try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); - return @intCast(mem.total); -} - -pub fn memUsed(self: Nvml, handle: c.nvmlDevice_t) Error!u64 { - var mem: c.nvmlMemory_t = undefined; - try check(self.lib.nvmlDeviceGetMemoryInfo(handle, &mem)); - return @intCast(mem.used); -} - -pub fn fanSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var speed: c_uint = 0; - try check(self.lib.nvmlDeviceGetFanSpeed(handle, &speed)); - return speed; -} - -pub fn powerLimit(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var limit: c_uint = 0; - try check(self.lib.nvmlDeviceGetEnforcedPowerLimit(handle, &limit)); - return limit; -} - -pub fn pcieTxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var value: c_uint = 0; - try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_TX_BYTES, &value)); - return value; -} - -pub fn pcieRxKBps(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var value: c_uint = 0; - try check(self.lib.nvmlDeviceGetPcieThroughput(handle, c.NVML_PCIE_UTIL_RX_BYTES, &value)); - return value; -} - -pub fn encoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var util: c_uint = 0; - var sampling: c_uint = 0; - try check(self.lib.nvmlDeviceGetEncoderUtilization(handle, &util, &sampling)); - return util; -} - -pub fn decoderUtil(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var util: c_uint = 0; - var sampling: c_uint = 0; - try check(self.lib.nvmlDeviceGetDecoderUtilization(handle, &util, &sampling)); - return util; -} - -pub fn pcieLinkGen(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var gen: c_uint = 0; - try check(self.lib.nvmlDeviceGetCurrPcieLinkGeneration(handle, &gen)); - return gen; -} - -pub fn pcieLinkWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var width: c_uint = 0; - try check(self.lib.nvmlDeviceGetCurrPcieLinkWidth(handle, &width)); - return width; -} - -pub fn pcieSpeed(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var speed: c_uint = 0; - try check(self.lib.nvmlDeviceGetPcieSpeed(handle, &speed)); - return speed; -} - -pub fn driverVersion(self: Nvml, buf: *[c.NVML_SYSTEM_DRIVER_VERSION_BUFFER_SIZE]u8) Error![:0]const u8 { - try check(self.lib.nvmlSystemGetDriverVersion(buf, buf.len)); - return std.mem.span(@as([*c]const u8, @ptrCast(buf))); -} - -pub fn cudaDriverVersionMajor(v: c_int) c_int { - return @divTrunc(v, 1000); -} - -pub fn cudaDriverVersionMinor(v: c_int) c_int { - return @divTrunc(@mod(v, 1000), 10); -} - -pub fn cudaDriverVersion(self: Nvml) Error!c_int { - var version: c_int = 0; - try check(self.lib.nvmlSystemGetCudaDriverVersion_v2(&version)); - return version; -} - -pub fn memBusWidth(self: Nvml, handle: c.nvmlDevice_t) Error!c_uint { - var width: c_uint = 0; - try check(self.lib.nvmlDeviceGetMemoryBusWidth(handle, &width)); - return width; -} - -const max_query = 4096; - -fn sizedCount(count: c_uint) ?c_uint { - if (count == 0 or count > max_query) return null; - return count; -} - -pub fn computeRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]c.nvmlProcessInfo_t { - var count: c_uint = 0; - const ret = self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, null); - if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { - if (ret == c.NVML_SUCCESS) { - return &.{}; - } - try check(ret); - } - count = sizedCount(count) orelse return &.{}; - const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); - errdefer allocator.free(infos); - try check(self.lib.nvmlDeviceGetComputeRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); - return infos; -} - -pub fn graphicsRunningProcesses(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t) (Error || error{OutOfMemory})![]const c.nvmlProcessInfo_t { - var count: c_uint = 0; - const ret = self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, null); - - if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { - if (ret == c.NVML_SUCCESS) { - return &.{}; - } - try check(ret); - } - - count = sizedCount(count) orelse return &.{}; - const infos = try allocator.alloc(c.nvmlProcessInfo_t, count); - errdefer allocator.free(infos); - - try check(self.lib.nvmlDeviceGetGraphicsRunningProcesses_v3(handle, &count, @ptrCast(infos.ptr))); - return infos; -} - -pub fn processUtilization(self: Nvml, allocator: std.mem.Allocator, handle: c.nvmlDevice_t, last_seen: u64) (Error || error{OutOfMemory})![]const c.nvmlProcessUtilizationSample_t { - var count: c_uint = 0; - const ret = self.lib.nvmlDeviceGetProcessUtilization(handle, null, &count, last_seen); - - if (ret != c.NVML_ERROR_INSUFFICIENT_SIZE) { - if (ret == c.NVML_SUCCESS) { - return &.{}; - } - try check(ret); - } - - count = sizedCount(count) orelse return &.{}; - const samples = try allocator.alloc(c.nvmlProcessUtilizationSample_t, count); - errdefer allocator.free(samples); - - try check(self.lib.nvmlDeviceGetProcessUtilization(handle, @ptrCast(samples.ptr), &count, last_seen)); - return samples; -} - -// Error handling - -pub const ReturnError = error{ - error_uninitialized, - error_invalid_argument, - error_not_supported, - error_no_permission, - error_already_initialized, - error_not_found, - error_insufficient_size, - error_insufficient_power, - error_driver_not_loaded, - error_timeout, - error_irq_issue, - error_library_not_found, - error_function_not_found, - error_corrupted_inforom, - error_gpu_is_lost, - error_reset_required, - error_operating_system, - error_lib_rm_version_mismatch, - error_in_use, - error_memory, - error_no_data, - error_vgpu_ecc_not_supported, - error_insufficient_resources, - error_freq_not_supported, - error_argument_version_mismatch, - error_deprecated, - error_not_ready, - error_gpu_not_found, - error_invalid_state, - error_reset_type_not_supported, - error_unknown, -}; - -fn check(ret: c_uint) Error!void { - if (ret == c.NVML_SUCCESS) { - return; - } - return switch (ret) { - c.NVML_ERROR_UNINITIALIZED => error.error_uninitialized, - c.NVML_ERROR_INVALID_ARGUMENT => error.error_invalid_argument, - c.NVML_ERROR_NOT_SUPPORTED => error.error_not_supported, - c.NVML_ERROR_NO_PERMISSION => error.error_no_permission, - c.NVML_ERROR_ALREADY_INITIALIZED => error.error_already_initialized, - c.NVML_ERROR_NOT_FOUND => error.error_not_found, - c.NVML_ERROR_INSUFFICIENT_SIZE => error.error_insufficient_size, - c.NVML_ERROR_INSUFFICIENT_POWER => error.error_insufficient_power, - c.NVML_ERROR_DRIVER_NOT_LOADED => error.error_driver_not_loaded, - c.NVML_ERROR_TIMEOUT => error.error_timeout, - c.NVML_ERROR_IRQ_ISSUE => error.error_irq_issue, - c.NVML_ERROR_LIBRARY_NOT_FOUND => error.error_library_not_found, - c.NVML_ERROR_FUNCTION_NOT_FOUND => error.error_function_not_found, - c.NVML_ERROR_CORRUPTED_INFOROM => error.error_corrupted_inforom, - c.NVML_ERROR_GPU_IS_LOST => error.error_gpu_is_lost, - c.NVML_ERROR_RESET_REQUIRED => error.error_reset_required, - c.NVML_ERROR_OPERATING_SYSTEM => error.error_operating_system, - c.NVML_ERROR_LIB_RM_VERSION_MISMATCH => error.error_lib_rm_version_mismatch, - c.NVML_ERROR_IN_USE => error.error_in_use, - c.NVML_ERROR_MEMORY => error.error_memory, - c.NVML_ERROR_NO_DATA => error.error_no_data, - c.NVML_ERROR_VGPU_ECC_NOT_SUPPORTED => error.error_vgpu_ecc_not_supported, - c.NVML_ERROR_INSUFFICIENT_RESOURCES => error.error_insufficient_resources, - c.NVML_ERROR_FREQ_NOT_SUPPORTED => error.error_freq_not_supported, - c.NVML_ERROR_ARGUMENT_VERSION_MISMATCH => error.error_argument_version_mismatch, - c.NVML_ERROR_DEPRECATED => error.error_deprecated, - c.NVML_ERROR_NOT_READY => error.error_not_ready, - c.NVML_ERROR_GPU_NOT_FOUND => error.error_gpu_not_found, - c.NVML_ERROR_INVALID_STATE => error.error_invalid_state, - c.NVML_ERROR_RESET_TYPE_NOT_SUPPORTED => error.error_reset_type_not_supported, - c.NVML_ERROR_UNKNOWN => error.error_unknown, - else => error.error_unknown, - }; -} diff --git a/bin/zml-smi/platforms/nvml/process.zig b/bin/zml-smi/platforms/nvml/process.zig deleted file mode 100644 index 5e18cdcb..00000000 --- a/bin/zml-smi/platforms/nvml/process.zig +++ /dev/null @@ -1,54 +0,0 @@ -const std = @import("std"); -const Nvml = @import("nvml.zig"); -const pi = @import("zml-smi/info").process_info; - -pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); - -pub fn pollOnce(allocator: std.mem.Allocator, list: *ProcessDoubleBuffer, nvml: *const Nvml, dev_offset: u16, device_count: u32, last_seen_ts: []u64) void { - const back = list.back(); - - back.clearRetainingCapacity(); - - for (0..device_count) |dev_idx| { - const handle = nvml.handleByIndex(@intCast(dev_idx)) catch continue; - const idx: u16 = @intCast(dev_idx + dev_offset); - - const compute = nvml.computeRunningProcesses(allocator, handle) catch &.{}; - defer if (compute.len > 0) allocator.free(compute); - collectFromQuery(allocator, back, idx, compute); - - const graphics = nvml.graphicsRunningProcesses(allocator, handle) catch &.{}; - defer if (graphics.len > 0) allocator.free(graphics); - collectFromQuery(allocator, back, idx, graphics); - - const last_ts = last_seen_ts[dev_idx]; - const utils = nvml.processUtilization(allocator, handle, last_ts) catch &.{}; - defer if (utils.len > 0) allocator.free(utils); - - for (utils) |sample| { - if (sample.smUtil > 100 or sample.timeStamp <= last_ts) { - continue; - } - - last_seen_ts[dev_idx] = @max(last_seen_ts[dev_idx], sample.timeStamp); - - for (back.items) |*entry| { - if (entry.pid == sample.pid and entry.device_idx == idx) { - entry.dev_util_percent = @intCast(sample.smUtil); - } - } - } - } - - list.swap(); -} - -fn collectFromQuery(allocator: std.mem.Allocator, back: *std.ArrayList(pi.ProcessInfo), dev_idx: u16, procs: []const Nvml.ProcessInfo_t) void { - for (procs) |gp| { - back.append(allocator, .{ - .pid = gp.pid, - .device_idx = dev_idx, - .dev_mem_kib = @intCast(gp.usedGpuMemory / 1024), - }) catch return; - } -} diff --git a/bin/zml-smi/platforms/oneapi/BUILD.bazel b/bin/zml-smi/platforms/oneapi/BUILD.bazel deleted file mode 100644 index c0b27d84..00000000 --- a/bin/zml-smi/platforms/oneapi/BUILD.bazel +++ /dev/null @@ -1,24 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "oneapi", - import_name = "zml-smi/platforms/oneapi", - main = "metrics.zig", - srcs = [ - "monitor.zig", - "oneapi.zig", - "process.zig", - ], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//bin/zml-smi/utils:sysfs", - ], -) - -zig_test( - name = "test", - deps = [":oneapi"], -) diff --git a/bin/zml-smi/platforms/oneapi/metrics.zig b/bin/zml-smi/platforms/oneapi/metrics.zig deleted file mode 100644 index ae2a86a3..00000000 --- a/bin/zml-smi/platforms/oneapi/metrics.zig +++ /dev/null @@ -1,134 +0,0 @@ -const std = @import("std"); -const OneApi = @import("oneapi.zig"); -const device_info = @import("zml-smi/info").device_info; -const GpuInfo = device_info.GpuInfo; -const DoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer; -const Collector = @import("zml-smi/collector").Collector; -const poll_metrics = @import("zml-smi/info").poll_metrics; -const process = @import("process.zig"); - -pub fn start(collector: *Collector) !void { - const oneapi = try collector.arena.create(OneApi); - oneapi.* = try OneApi.init(collector.arena, collector.io); - - const count = oneapi.monitor.devices.len; - const dev_offset: u16 = @intCast(collector.device_infos.items.len); - - for (0..count) |i| { - const poll_arena = try collector.createPollArena(); - const dev = Device.open(oneapi, i, poll_arena) catch continue; - const initial: GpuInfo = .{ - .name = dev.name(collector.arena) catch null, - .driver_version = dev.driverVersion(collector.arena) catch null, - .power_limit_mw = dev.powerLimit() catch null, - .clock_graphics_max_mhz = dev.maxClockGraphics() catch null, - .mem_total_bytes = dev.memTotal() catch null, - .pcie_link_gen = dev.pcieLinkGen() catch null, - .pcie_link_width = dev.pcieLinkWidth() catch null, - .pcie_bandwidth_mbps = dev.pcieBandwidth() catch null, - }; - const info = try collector.addDevice(.{ .oneapi = .{ .values = .{ initial, initial } } }); - - try collector.spawnPoll(pollOnce, .{ poll_arena, &info.oneapi, dev }); - } - - const processes = try collector.createProcessList(); - try process.init(collector, processes, oneapi, dev_offset); -} - -const pollOnce = poll_metrics.poll(*DoubleBuffer(GpuInfo), Device, metrics); - -const Device = struct { - oneapi: *OneApi, - arena: *std.heap.ArenaAllocator, - handle: OneApi.Handle, - - fn open(oneapi: *OneApi, index: usize, arena: *std.heap.ArenaAllocator) !Device { - return .{ - .oneapi = oneapi, - .arena = arena, - .handle = try oneapi.handleByIndex(index), - }; - } - - fn name(self: Device, arena: std.mem.Allocator) ![]const u8 { - return self.oneapi.name(arena, self.handle); - } - - fn driverVersion(self: Device, arena: std.mem.Allocator) ![]const u8 { - return self.oneapi.driverVersion(arena); - } - - pub fn powerUsage(self: Device) !u64 { - return self.oneapi.powerUsage(self.arena.allocator(), self.handle); - } - - pub fn temperature(self: Device) !u64 { - return self.oneapi.temperature(self.arena.allocator(), self.handle); - } - - pub fn fanSpeed(self: Device) !u64 { - return self.oneapi.fanSpeed(self.arena.allocator(), self.handle); - } - - pub fn gpuUtil(self: Device) !u64 { - return self.oneapi.gpuUtil(self.arena.allocator(), self.handle); - } - - pub fn encoderUtil(self: Device) !u64 { - return self.oneapi.encoderUtil(self.arena.allocator(), self.handle); - } - - pub fn decoderUtil(self: Device) !u64 { - return self.oneapi.decoderUtil(self.arena.allocator(), self.handle); - } - - pub fn powerLimit(self: Device) !u64 { - return self.oneapi.powerLimit(self.arena.allocator(), self.handle); - } - - pub fn clockGraphics(self: Device) !u64 { - return self.oneapi.clockGraphics(self.arena.allocator(), self.handle); - } - - pub fn maxClockGraphics(self: Device) !u64 { - return self.oneapi.maxClockGraphics(self.arena.allocator(), self.handle); - } - - pub fn memUsed(self: Device) !u64 { - return self.oneapi.memUsed(self.arena.allocator(), self.handle); - } - - pub fn memTotal(self: Device) !u64 { - return self.oneapi.memTotal(self.arena.allocator(), self.handle); - } - - pub fn pcieLinkGen(self: Device) !u64 { - return self.oneapi.pcieLinkGen(self.arena.allocator(), self.handle); - } - - pub fn pcieLinkWidth(self: Device) !u64 { - return self.oneapi.pcieLinkWidth(self.arena.allocator(), self.handle); - } - - pub fn pcieBandwidth(self: Device) !u64 { - return self.oneapi.pcieBandwidth(self.arena.allocator(), self.handle); - } -}; - -const metrics = .{ - .{ .field = "power_mw", .query = Device.powerUsage }, - .{ .field = "power_limit_mw", .query = Device.powerLimit }, - .{ .field = "temperature", .query = Device.temperature }, - .{ .field = "fan_speed_percent", .query = Device.fanSpeed }, - .{ .field = "util_percent", .query = Device.gpuUtil }, - .{ .field = "encoder_util_percent", .query = Device.encoderUtil }, - .{ .field = "decoder_util_percent", .query = Device.decoderUtil }, - .{ .field = "clock_graphics_mhz", .query = Device.clockGraphics }, - .{ .field = "clock_graphics_max_mhz", .query = Device.maxClockGraphics }, - .{ .field = "mem_used_bytes", .query = Device.memUsed }, - .{ .field = "mem_total_bytes", .query = Device.memTotal }, - .{ .field = "pcie_bandwidth_mbps", .query = Device.pcieBandwidth }, - .{ .field = "pcie_link_gen", .query = Device.pcieLinkGen }, - .{ .field = "pcie_link_width", .query = Device.pcieLinkWidth }, -}; diff --git a/bin/zml-smi/platforms/oneapi/monitor.zig b/bin/zml-smi/platforms/oneapi/monitor.zig deleted file mode 100644 index dd5f97a9..00000000 --- a/bin/zml-smi/platforms/oneapi/monitor.zig +++ /dev/null @@ -1,877 +0,0 @@ -const std = @import("std"); -const smi_sysfs = @import("zml-smi/sysfs"); -const Monitor = @This(); - -pub const bus_device_identifier_len = "0000:00:00.0".len; -pub const Handle = enum(u32) { _ }; - -const Device = struct { - render_idx: usize, - drm_path: []const u8, - dev_path: []const u8, - pcie_path: []const u8, - hwmon_path: ?[]const u8, - bus_device_identifier: ?[bus_device_identifier_len]u8, - activity_prev: ?EngineSample = null, - encoder_prev: ?EngineSample = null, - decoder_prev: ?EngineSample = null, - energy_prev: ?EnergySample = null, -}; - -const EnergySample = struct { - micro_joules: u64, - timestamp_ns: u64, -}; - -pub const EngineSample = struct { - engine_ns: u64, - timestamp_ns: u64, -}; - -const DeviceSample = struct { - mem_kib: u64 = 0, - engine: ?EngineSample = null, - encoder: ?EngineSample = null, - decoder: ?EngineSample = null, -}; - -pub const ProcessSample = struct { - pid: u32, - device_idx: u16, - mem_kib: ?u64 = null, - engine: ?EngineSample = null, - encoder: ?EngineSample = null, - decoder: ?EngineSample = null, -}; - -pub const ProcessUsage = std.AutoHashMapUnmanaged(u64, ProcessSample); -pub const ProcessPrevious = std.AutoHashMapUnmanaged(u64, EngineSample); -const DeviceUsage = std.AutoHashMapUnmanaged(u16, DeviceSample); - -allocator: std.mem.Allocator, -io: std.Io, -devices: []Device, - -pub fn init(allocator: std.mem.Allocator, io: std.Io) !Monitor { - var monitor: Monitor = .{ - .allocator = allocator, - .io = io, - .devices = try discoverIntelDevices(allocator, io), - }; - try monitor.seedDeviceMetrics(); - return monitor; -} - -pub fn deviceId(self: *const Monitor, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { - const dev = try self.device(handle); - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/device", .{dev.dev_path}); - const raw = try smi_sysfs.readString(allocator, self.io, path); - return std.mem.trim(u8, raw, &std.ascii.whitespace); -} - -inline fn seedDeviceMetrics(self: *Monitor) !void { - var device_usage = collectDeviceUsage(self.allocator, self.io, self.devices) catch DeviceUsage{}; - defer device_usage.deinit(self.allocator); - - for (self.devices, 0..) |*dev, idx| { - dev.energy_prev = readEnergy(self.allocator, self.io, dev) catch null; - if (device_usage.get(@intCast(idx))) |sample| { - dev.activity_prev = sample.engine; - dev.encoder_prev = sample.encoder; - dev.decoder_prev = sample.decoder; - } - } -} - -pub fn saveProcessPrevious(allocator: std.mem.Allocator, previous: *ProcessPrevious, usage: *const ProcessUsage) !void { - previous.clearRetainingCapacity(); - try previous.ensureTotalCapacity(allocator, @intCast(usage.count())); - - var it = usage.iterator(); - while (it.next()) |entry| { - if (entry.value_ptr.engine) |engine| { - previous.putAssumeCapacity(entry.key_ptr.*, engine); - } - } -} - -inline fn discoverIntelDevices(allocator: std.mem.Allocator, io: std.Io) ![]Device { - var devices: std.ArrayList(Device) = .empty; - errdefer devices.deinit(allocator); - - var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch |err| switch (err) { - error.FileNotFound => return &.{}, - else => |e| return e, - }; - defer dir.close(io); - - var it = dir.iterate(); - while (it.next(io) catch null) |entry| { - const render_idx = parseRenderIndex(entry.name) orelse continue; - const dev = openDevice(allocator, io, render_idx) catch continue; - try devices.append(allocator, dev); - } - - std.mem.sort(Device, devices.items, {}, struct { - fn lessThan(_: void, lhs: Device, rhs: Device) bool { - return lhs.render_idx < rhs.render_idx; - } - }.lessThan); - return devices.toOwnedSlice(allocator); -} - -inline fn openDevice(allocator: std.mem.Allocator, io: std.Io, render_idx: usize) !Device { - const drm_path = try std.fmt.allocPrint(allocator, "/sys/class/drm/renderD{d}", .{render_idx}); - errdefer allocator.free(drm_path); - - const dev_path = try std.fmt.allocPrint(allocator, "{s}/device", .{drm_path}); - errdefer allocator.free(dev_path); - - var vendor_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const vendor_path = try std.fmt.bufPrint(&vendor_path_buf, "{s}/vendor", .{dev_path}); - const vendor = try smi_sysfs.readString(allocator, io, vendor_path); - if (!std.mem.eql(u8, std.mem.trim(u8, vendor, &std.ascii.whitespace), "0x8086")) { - return error.not_intel; - } - - const pcie_path = try findPcieLinkPath(allocator, io, dev_path); - errdefer if (pcie_path.ptr != dev_path.ptr) allocator.free(pcie_path); - - return .{ - .render_idx = render_idx, - .drm_path = drm_path, - .dev_path = dev_path, - .pcie_path = pcie_path, - .hwmon_path = findHwmon(allocator, io, dev_path) catch null, - .bus_device_identifier = readBusDeviceId(allocator, io, dev_path) catch null, - }; -} - -pub fn device(self: *const Monitor, handle: Handle) !*const Device { - const idx = @backingInt(handle); - if (idx >= self.devices.len) return error.NotFound; - return &self.devices[idx]; -} - -pub fn readTemperature(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - const hwmon = dev.hwmon_path orelse return error.NotFound; - const milli_c = readHwmonInput(allocator, io, hwmon, "temp", "pkg") catch - readHwmonInput(allocator, io, hwmon, "temp", "gpu") catch - readNumberedInput(allocator, io, hwmon, "temp", 2) catch - try readNumberedInput(allocator, io, hwmon, "temp", 1); - return (milli_c + 500) / 1000; -} - -pub fn readEnergy(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !EnergySample { - const hwmon = dev.hwmon_path orelse return error.NotFound; - const micro_joules = readHwmonInput(allocator, io, hwmon, "energy", "card") catch - readHwmonInput(allocator, io, hwmon, "energy", "pkg") catch - try readNumberedInput(allocator, io, hwmon, "energy", 1); - return .{ - .micro_joules = micro_joules, - .timestamp_ns = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds), - }; -} -pub fn readPowerLimit(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - const hwmon = dev.hwmon_path orelse return error.NotFound; - const microwatts = readNumberedSuffix(allocator, io, hwmon, "power", 1, "cap") catch - try readNumberedSuffix(allocator, io, hwmon, "power", 1, "max"); - return microwatts / 1000; -} - -pub fn readFanSpeed(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - const hwmon = dev.hwmon_path orelse return error.NotFound; - return readPwmFanSpeed(allocator, io, hwmon) catch readRpmFanSpeedPercent(allocator, io, hwmon); -} - -pub fn readClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - return readFreq(allocator, io, dev, "act_freq") catch readFreq(allocator, io, dev, "cur_freq"); -} - -pub fn readMaxClockGraphics(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - return readFreq(allocator, io, dev, "max_freq"); -} - -pub fn readMemTotal(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/resource", .{dev.dev_path}); - return readLargestResource(allocator, io, path); -} - -pub fn readPcieLinkGen(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_speed", .{dev.pcie_path}); - const raw = try smi_sysfs.readString(allocator, io, path); - return pcieGenFromSpeed(raw); -} - -pub fn readPcieLinkWidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/current_link_width", .{dev.pcie_path}); - return smi_sysfs.readInt(allocator, io, path); -} - -pub fn readPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev: *const Device) !u64 { - const gen = try readPcieLinkGen(allocator, io, dev); - const width = try readPcieLinkWidth(allocator, io, dev); - return pcieBandwidthFromLink(gen, width) orelse error.NotFound; -} - -pub fn collectDeviceUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !DeviceUsage { - var processes = try collectProcessUsage(allocator, io, devices); - errdefer processes.deinit(allocator); - - var result: DeviceUsage = .{}; - errdefer result.deinit(allocator); - - var it = processes.iterator(); - while (it.next()) |entry| { - const sample = entry.value_ptr.*; - const gop = try result.getOrPut(allocator, sample.device_idx); - if (!gop.found_existing) gop.value_ptr.* = .{}; - if (sample.mem_kib) |mem| gop.value_ptr.mem_kib += mem; - addEngineSample(&gop.value_ptr.engine, sample.engine); - addEngineSample(&gop.value_ptr.encoder, sample.encoder); - addEngineSample(&gop.value_ptr.decoder, sample.decoder); - } - - return result; -} - -pub fn collectProcessUsage(allocator: std.mem.Allocator, io: std.Io, devices: []const Device) !ProcessUsage { - var result: ProcessUsage = .{}; - errdefer result.deinit(allocator); - - var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; - defer seen_device_clients.deinit(allocator); - - var proc_dir = try std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }); - defer proc_dir.close(io); - - const now_ns: u64 = @intCast(std.Io.Timestamp.now(io, .awake).nanoseconds); - var proc_it = proc_dir.iterate(); - while (proc_it.next(io) catch null) |proc_entry| { - const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; - - var fdinfo_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const fdinfo_path = std.fmt.bufPrint(&fdinfo_path_buf, "/proc/{d}/fdinfo", .{pid}) catch continue; - var fdinfo_dir = std.Io.Dir.openDirAbsolute(io, fdinfo_path, .{ .iterate = true }) catch continue; - defer fdinfo_dir.close(io); - - var fd_it = fdinfo_dir.iterate(); - while (fd_it.next(io) catch null) |fd_entry| { - var file_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const file_path = std.fmt.bufPrint(&file_path_buf, "{s}/{s}", .{ fdinfo_path, fd_entry.name }) catch continue; - const parsed = parseFdinfoFile(io, file_path, now_ns) catch continue; - const bus_device_identifier = parsed.bus_device_identifier orelse continue; - const dev_idx = matchDevice(devices, bus_device_identifier) orelse continue; - - // Skip fd if DRM client has been seen for this device. - // Summing after dup() / fork() over-reports memory and compute. - if (parsed.drm_client_id) |cid| { - if ((try seen_device_clients.getOrPut(allocator, .{ dev_idx, cid })).found_existing) continue; - } - - const key = processKey(pid, dev_idx); - const gop = try result.getOrPut(allocator, key); - if (!gop.found_existing) { - gop.value_ptr.* = .{ .pid = pid, .device_idx = dev_idx }; - } - mergeProcessSample(gop.value_ptr, parsed.sample); - } - } - - return result; -} - -pub fn processUtil(previous: ?EngineSample, current: ?EngineSample) ?u16 { - const prev = previous orelse return null; - const cur = current orelse return null; - if (cur.timestamp_ns <= prev.timestamp_ns or cur.engine_ns < prev.engine_ns) return null; - const engine_delta = cur.engine_ns - prev.engine_ns; - const time_delta = cur.timestamp_ns - prev.timestamp_ns; - if (time_delta == 0) return null; - const pct = @min(engine_delta * 100 / time_delta, 100); - return @intCast(if (pct == 0 and engine_delta > 0) 0 else pct); -} - -pub fn powerMilliwatts(previous: ?EnergySample, current: ?EnergySample) ?u64 { - const prev = previous orelse return null; - const cur = current orelse return null; - return milliwattsFromEnergyDelta(prev.micro_joules, cur.micro_joules, prev.timestamp_ns, cur.timestamp_ns); -} - -inline fn milliwattsFromEnergyDelta(prev_uj: u64, cur_uj: u64, prev_ns: u64, cur_ns: u64) ?u64 { - if (cur_uj < prev_uj or cur_ns <= prev_ns) return null; - const energy_delta = cur_uj - prev_uj; - const time_delta = cur_ns - prev_ns; - if (time_delta == 0) return null; - return energy_delta * 1_000_000 / time_delta; -} - -inline fn readFreq(allocator: std.mem.Allocator, io: std.Io, dev: *const Device, file: []const u8) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/tile0/gt0/freq0/{s}", .{ dev.dev_path, file }); - return smi_sysfs.readInt(allocator, io, path); -} - -inline fn parseRenderIndex(name: []const u8) ?usize { - if (!std.mem.startsWith(u8, name, "renderD")) return null; - return std.fmt.parseInt(usize, name["renderD".len..], 10) catch null; -} - -inline fn findHwmon(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !?[]const u8 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const hwmon_root = try std.fmt.bufPrint(&path_buf, "{s}/hwmon", .{dev_path}); - var dir = std.Io.Dir.openDirAbsolute(io, hwmon_root, .{ .iterate = true }) catch return null; - defer dir.close(io); - - var it = dir.iterate(); - while (it.next(io) catch null) |entry| { - if (!std.mem.startsWith(u8, entry.name, "hwmon")) continue; - const candidate = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ hwmon_root, entry.name }); - var name_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const name_path = try std.fmt.bufPrint(&name_buf, "{s}/name", .{candidate}); - const name = smi_sysfs.readString(allocator, io, name_path) catch return candidate; - const trimmed = std.mem.trim(u8, name, &std.ascii.whitespace); - if (std.mem.eql(u8, trimmed, "xe") or std.mem.eql(u8, trimmed, "i915")) return candidate; - } - return null; -} - -inline fn readBusDeviceId(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![bus_device_identifier_len]u8 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/uevent", .{dev_path}); - const raw = try smi_sysfs.readFieldString(allocator, io, path, "PCI_SLOT_NAME="); - const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); - if (trimmed.len != bus_device_identifier_len) return error.NotFound; - var out: [bus_device_identifier_len]u8 = undefined; - @memcpy(&out, trimmed); - return out; -} - -inline fn findPcieLinkPath(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) ![]const u8 { - var candidate = dev_path; - var best_path: ?[]const u8 = null; - - var best_bandwidth: u64 = 0; - var depth: usize = 0; - while (depth < 8) : (depth += 1) { - if (!try isIntelPciDevice(allocator, io, candidate)) break; - - const bandwidth = readMaxPcieBandwidth(allocator, io, candidate) catch 0; - if (bandwidth > best_bandwidth) { - best_path = candidate; - best_bandwidth = bandwidth; - } - - const parent = try std.fmt.allocPrint(allocator, "{s}/..", .{candidate}); - candidate = parent; - } - - return best_path orelse dev_path; -} - -inline fn isIntelPciDevice(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !bool { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const vendor_path = try std.fmt.bufPrint(&path_buf, "{s}/vendor", .{dev_path}); - const vendor_raw = smi_sysfs.readString(allocator, io, vendor_path) catch return false; - - const vendor = std.mem.trim(u8, vendor_raw, &std.ascii.whitespace); - return std.mem.eql(u8, vendor, "0x8086"); -} - -inline fn readMaxPcieBandwidth(allocator: std.mem.Allocator, io: std.Io, dev_path: []const u8) !u64 { - var speed_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const speed_path = try std.fmt.bufPrint(&speed_path_buf, "{s}/max_link_speed", .{dev_path}); - const raw_speed = try smi_sysfs.readString(allocator, io, speed_path); - - var width_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const width_path = try std.fmt.bufPrint(&width_path_buf, "{s}/max_link_width", .{dev_path}); - const width = try smi_sysfs.readInt(allocator, io, width_path); - const gen = try pcieGenFromSpeed(raw_speed); - return pcieBandwidthFromLink(gen, width) orelse error.NotFound; -} - -inline fn readHwmonInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, label: []const u8) !u64 { - for (1..16) |idx| { - var label_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const label_path = try std.fmt.bufPrint(&label_path_buf, "{s}/{s}{d}_label", .{ hwmon, prefix, idx }); - const raw = smi_sysfs.readString(allocator, io, label_path) catch continue; - const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); - if (!std.ascii.eqlIgnoreCase(trimmed, label)) continue; - return readNumberedInput(allocator, io, hwmon, prefix, idx); - } - return error.NotFound; -} - -inline fn readNumberedInput(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { - return readNumberedSuffix(allocator, io, hwmon, prefix, idx, "input"); -} - -inline fn readNumberedSuffix(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize, suffix: []const u8) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}_{s}", .{ hwmon, prefix, idx, suffix }); - return smi_sysfs.readInt(allocator, io, path); -} - -inline fn readNumberedRaw(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8, comptime prefix: []const u8, idx: usize) !u64 { - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "{s}/{s}{d}", .{ hwmon, prefix, idx }); - return smi_sysfs.readInt(allocator, io, path); -} - -inline fn readPwmFanSpeed(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { - for (1..16) |idx| { - const pwm = readNumberedRaw(allocator, io, hwmon, "pwm", idx) catch continue; - return pwmToPercent(pwm); - } - return error.NotFound; -} - -inline fn readRpmFanSpeedPercent(allocator: std.mem.Allocator, io: std.Io, hwmon: []const u8) !u64 { - for (1..16) |idx| { - const input = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "input") catch continue; - const max = readNumberedSuffix(allocator, io, hwmon, "fan", idx, "max") catch continue; - if (max == 0) continue; - return @min(input * 100 / max, 100); - } - return error.NotFound; -} - -inline fn pwmToPercent(pwm: u64) u64 { - return @min((@min(pwm, 255) * 100 + 127) / 255, 100); -} - -inline fn readLargestResource(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { - var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); - defer file.close(io); - - var read_buf: [4096]u8 = undefined; - var reader = file.reader(io, &read_buf); - var line_writer: std.Io.Writer.Allocating = .init(allocator); - defer line_writer.deinit(); - - var largest: u64 = 0; - while (true) { - line_writer.clearRetainingCapacity(); - _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch |err| switch (err) { - error.EndOfStream => break, - else => |e| return e, - }; - reader.interface.toss(1); - largest = @max(largest, resourceLineSize(line_writer.written()) orelse 0); - } - if (largest == 0) return error.NotFound; - return largest; -} - -inline fn resourceLineSize(line: []const u8) ?u64 { - var it = std.mem.tokenizeAny(u8, line, " \t"); - const base = parseHex(it.next() orelse return null) orelse return null; - const end = parseHex(it.next() orelse return null) orelse return null; - if (end < base) return null; - const size = end - base + 1; - return if (size >= 256 * 1024 * 1024) size else null; -} - -inline fn parseHex(raw: []const u8) ?u64 { - const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); - const digits = if (std.mem.startsWith(u8, trimmed, "0x")) trimmed[2..] else trimmed; - return std.fmt.parseInt(u64, digits, 16) catch null; -} - -inline fn pcieGenFromSpeed(raw: []const u8) !u64 { - const trimmed = std.mem.trim(u8, raw, &std.ascii.whitespace); - if (std.mem.startsWith(u8, trimmed, "2.5")) return 1; - if (std.mem.startsWith(u8, trimmed, "5.0") or std.mem.startsWith(u8, trimmed, "5 ")) return 2; - if (std.mem.startsWith(u8, trimmed, "8.0") or std.mem.startsWith(u8, trimmed, "8 ")) return 3; - if (std.mem.startsWith(u8, trimmed, "16.0") or std.mem.startsWith(u8, trimmed, "16 ")) return 4; - if (std.mem.startsWith(u8, trimmed, "32.0") or std.mem.startsWith(u8, trimmed, "32 ")) return 5; - if (std.mem.startsWith(u8, trimmed, "64.0") or std.mem.startsWith(u8, trimmed, "64 ")) return 6; - return error.NotFound; -} - -inline fn pcieBandwidthFromLink(gen: u64, width: u64) ?u64 { - const lane_mb_s: u64 = switch (gen) { - 1 => 250, - 2 => 500, - 3 => 985, - 4 => 1969, - 5 => 3938, - 6 => 7563, - else => return null, - }; - return lane_mb_s * width; -} - -const FdinfoSample = struct { - mem_kib: ?u64 = null, - engine: ?EngineSample = null, - encoder: ?EngineSample = null, - decoder: ?EngineSample = null, -}; - -const ParsedFdinfo = struct { - bus_device_identifier: ?[bus_device_identifier_len]u8 = null, - drm_client_id: ?u64 = null, - sample: FdinfoSample = .{}, - resident_vram_kib: u64 = 0, - total_vram_kib: u64 = 0, - resident_memory_kib: u64 = 0, - total_memory_kib: u64 = 0, - cycle_counter: u64 = 0, - total_cycle_counter: u64 = 0, - engine_time_seen: bool = false, -}; - -inline fn parseFdinfoFile(io: std.Io, path: []const u8, timestamp_ns: u64) !ParsedFdinfo { - var read_buf: [8192]u8 = undefined; - const data = std.Io.Dir.readFile(.cwd(), io, path, &read_buf) catch return error.FileUnavailable; - - var parsed: ParsedFdinfo = .{}; - - var lines = std.mem.tokenizeScalar(u8, data, '\n'); - while (lines.next()) |line| { - parseFdinfoLine(&parsed, line, timestamp_ns); - } - - finishFdinfo(&parsed); - return parsed; -} - -inline fn finishFdinfo(self: *ParsedFdinfo) void { - if (self.resident_vram_kib > 0) { - self.sample.mem_kib = self.resident_vram_kib; - } else if (self.total_vram_kib > 0) { - self.sample.mem_kib = self.total_vram_kib; - } else if (self.resident_memory_kib > 0) { - self.sample.mem_kib = self.resident_memory_kib; - } else if (self.total_memory_kib > 0) { - self.sample.mem_kib = self.total_memory_kib; - } - - if (!self.engine_time_seen and self.total_cycle_counter > 0) { - self.sample.engine = .{ - .engine_ns = self.cycle_counter, - .timestamp_ns = self.total_cycle_counter, - }; - } -} - -inline fn parseFdinfoLine(parsed: *ParsedFdinfo, line: []const u8, timestamp_ns: u64) void { - if (std.mem.startsWith(u8, line, "drm-pdev:")) { - parsed.bus_device_identifier = parseBdf(line["drm-pdev:".len..]); - return; - } - - if (std.mem.startsWith(u8, line, "drm-client-id:")) { - parsed.drm_client_id = firstInt(line["drm-client-id:".len..]); - return; - } - - if (std.mem.startsWith(u8, line, "vram mem:")) { - parsed.total_vram_kib += memoryKiB(line["vram mem:".len..]); - return; - } - - if (std.mem.startsWith(u8, line, "drm-cycles-")) { - const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; - parsed.cycle_counter += firstInt(line[colon + 1 ..]); - return; - } - - if (std.mem.startsWith(u8, line, "drm-total-cycles-")) { - const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; - parsed.total_cycle_counter = @max(parsed.total_cycle_counter, firstInt(line[colon + 1 ..])); - return; - } - - if (std.mem.startsWith(u8, line, "drm-engine-capacity-")) { - return; - } - - if (std.mem.startsWith(u8, line, "drm-engine-")) { - const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; - const engine_label = line["drm-engine-".len..colon]; - const value = firstInt(line[colon + 1 ..]); - parsed.engine_time_seen = true; - addEngineTime(&parsed.sample.engine, value, timestamp_ns); - if (containsAsciiIgnoreCase(engine_label, "encode")) { - addEngineTime(&parsed.sample.encoder, value, timestamp_ns); - } else if (containsAsciiIgnoreCase(engine_label, "decode")) { - addEngineTime(&parsed.sample.decoder, value, timestamp_ns); - } - return; - } - - if (std.mem.startsWith(u8, line, "drm-")) { - const colon = std.mem.indexOfScalar(u8, line, ':') orelse return; - const key = line[0..colon]; - const value = memoryKiB(line[colon + 1 ..]); - if (value == 0) return; - - if (std.mem.startsWith(u8, key, "drm-resident-vram")) { - parsed.resident_vram_kib += value; - return; - } - if (std.mem.startsWith(u8, key, "drm-total-vram") or std.mem.startsWith(u8, key, "drm-memory-vram")) { - parsed.total_vram_kib += value; - return; - } - if (std.mem.startsWith(u8, key, "drm-resident-")) { - parsed.resident_memory_kib += value; - return; - } - if (std.mem.startsWith(u8, key, "drm-total-")) { - parsed.total_memory_kib += value; - return; - } - if (std.mem.eql(u8, key, "drm-resident-memory")) { - parsed.resident_memory_kib += value; - return; - } - if (std.mem.eql(u8, key, "drm-total-memory")) { - parsed.total_memory_kib += value; - return; - } - } -} - -fn mergeProcessSample(dst: *ProcessSample, src: FdinfoSample) void { - if (src.mem_kib) |mem| dst.mem_kib = (dst.mem_kib orelse 0) + mem; - addEngineSample(&dst.engine, src.engine); - addEngineSample(&dst.encoder, src.encoder); - addEngineSample(&dst.decoder, src.decoder); -} - -fn addEngineSample(dst: *?EngineSample, src: ?EngineSample) void { - if (src) |engine| addEngineTime(dst, engine.engine_ns, engine.timestamp_ns); -} - -fn addEngineTime(dst: *?EngineSample, engine_ns: u64, timestamp_ns: u64) void { - const current = dst.* orelse EngineSample{ .engine_ns = 0, .timestamp_ns = timestamp_ns }; - dst.* = .{ - .engine_ns = current.engine_ns + engine_ns, - .timestamp_ns = timestamp_ns, - }; -} - -fn matchDevice(devices: []const Device, bus_device_identifier: [bus_device_identifier_len]u8) ?u16 { - for (devices, 0..) |dev, idx| { - if (dev.bus_device_identifier) |device_bus_device_identifier| { - if (std.mem.eql(u8, &device_bus_device_identifier, &bus_device_identifier)) return @intCast(idx); - } - } - return null; -} - -fn processKey(pid: u32, device_idx: u16) u64 { - return (@as(u64, device_idx) << 32) | pid; -} - -pub const BdfParts = struct { - domain: u32, - bus: u32, - device: u32, - function: u32, -}; - -pub fn formatBdf(parts: BdfParts) [bus_device_identifier_len]u8 { - var buf: [bus_device_identifier_len]u8 = undefined; - _ = std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>2}:{x:0>2}.{x}", .{ - parts.domain, - parts.bus, - parts.device, - parts.function, - }) catch unreachable; - return buf; -} - -inline fn parseBdf(raw: []const u8) ?[bus_device_identifier_len]u8 { - const trimmed = std.mem.trim(u8, raw, " \t\n"); - var out: [bus_device_identifier_len]u8 = undefined; - if (trimmed.len == bus_device_identifier_len) { - @memcpy(&out, trimmed[0..bus_device_identifier_len]); - return out; - } - if (trimmed.len == "00:00.0".len) { - _ = std.fmt.bufPrint(&out, "0000:{s}", .{trimmed}) catch return null; - return out; - } - return null; -} - -inline fn firstInt(raw: []const u8) u64 { - var iter = std.mem.tokenizeAny(u8, raw, " \t"); - return std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch 0; -} - -inline fn memoryKiB(raw: []const u8) u64 { - var iter = std.mem.tokenizeAny(u8, raw, " \t"); - const value = std.fmt.parseInt(u64, iter.next() orelse return 0, 10) catch return 0; - const unit = iter.next() orelse return value; - if (std.ascii.eqlIgnoreCase(unit, "KiB")) return value; - if (std.ascii.eqlIgnoreCase(unit, "MiB")) return value * 1024; - if (std.ascii.eqlIgnoreCase(unit, "GiB")) return value * 1024 * 1024; - if (std.ascii.eqlIgnoreCase(unit, "B")) return value / 1024; - return value; -} - -inline fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { - if (needle.len == 0) return true; - if (needle.len > haystack.len) return false; - for (0..haystack.len - needle.len + 1) |i| { - if (std.ascii.eqlIgnoreCase(haystack[i .. i + needle.len], needle)) return true; - } - return false; -} - -test "oneAPI fdinfo parser" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); - parseFdinfoLine(&parsed, "drm-client-id:\t42", 10); - parseFdinfoLine(&parsed, "drm-total-memory:\t2048 KiB", 10); - parseFdinfoLine(&parsed, "drm-engine-render:\t100 ns", 10); - parseFdinfoLine(&parsed, "drm-engine-compute:\t50 ns", 10); - finishFdinfo(&parsed); - - try std.testing.expect(parsed.bus_device_identifier != null); - try std.testing.expectEqualSlices(u8, "0000:03:00.0", &parsed.bus_device_identifier.?); - try std.testing.expectEqual(@as(?u64, 42), parsed.drm_client_id); - try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); - try std.testing.expectEqual(@as(u64, 150), parsed.sample.engine.?.engine_ns); - try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); - try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); -} - -test "oneAPI fdinfo parser classifies explicit media engines" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-engine-video-decode:\t100 ns", 10); - parseFdinfoLine(&parsed, "drm-engine-media-encode:\t50 ns", 10); - parseFdinfoLine(&parsed, "drm-engine-VIDEO-DECODE1:\t25 ns", 10); - finishFdinfo(&parsed); - - try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); - try std.testing.expectEqual(@as(u64, 50), parsed.sample.encoder.?.engine_ns); - try std.testing.expectEqual(@as(u64, 125), parsed.sample.decoder.?.engine_ns); -} - -test "oneAPI fdinfo parser leaves generic video engines unclassified" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-engine-video:\t100 ns", 10); - parseFdinfoLine(&parsed, "drm-engine-vcs0:\t50 ns", 10); - parseFdinfoLine(&parsed, "drm-engine-vecs0:\t25 ns", 10); - finishFdinfo(&parsed); - - try std.testing.expectEqual(@as(u64, 175), parsed.sample.engine.?.engine_ns); - try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.encoder); - try std.testing.expectEqual(@as(?EngineSample, null), parsed.sample.decoder); -} - -test "oneAPI fdinfo parser cycle counters" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); - parseFdinfoLine(&parsed, "drm-cycles-rcs:\t10", 10); - parseFdinfoLine(&parsed, "drm-total-cycles-rcs:\t100", 10); - parseFdinfoLine(&parsed, "drm-engine-capacity-vcs:\t2", 10); - parseFdinfoLine(&parsed, "drm-cycles-ccs:\t30", 10); - parseFdinfoLine(&parsed, "drm-total-cycles-ccs:\t100", 10); - finishFdinfo(&parsed); - - try std.testing.expectEqual(@as(u64, 40), parsed.sample.engine.?.engine_ns); - try std.testing.expectEqual(@as(u64, 100), parsed.sample.engine.?.timestamp_ns); -} - -test "oneAPI fdinfo parser xe vram regions" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-pdev:\t0000:03:00.0", 10); - parseFdinfoLine(&parsed, "drm-total-system:\t184 MiB", 10); - parseFdinfoLine(&parsed, "drm-total-vram0:\t4096 KiB", 10); - parseFdinfoLine(&parsed, "drm-resident-vram0:\t3072 KiB", 10); - finishFdinfo(&parsed); - - try std.testing.expectEqual(@as(?u64, 3072), parsed.sample.mem_kib); -} - -test "oneAPI fdinfo parser falls back to total vram" { - var parsed: ParsedFdinfo = .{}; - parseFdinfoLine(&parsed, "drm-total-vram0:\t2 MiB", 10); - finishFdinfo(&parsed); - - try std.testing.expectEqual(@as(?u64, 2048), parsed.sample.mem_kib); -} - -test "oneAPI DRM clients dedup per device" { - var seen_device_clients: std.AutoHashMapUnmanaged(struct { u16, u64 }, void) = .{}; - defer seen_device_clients.deinit(std.testing.allocator); - - try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); - try std.testing.expectEqual(true, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 42 })).found_existing); - try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 1, 42 })).found_existing); - try std.testing.expectEqual(false, (try seen_device_clients.getOrPut(std.testing.allocator, .{ 0, 43 })).found_existing); -} - -test "oneAPI process utilization delta" { - try std.testing.expectEqual(@as(?u16, 50), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 600, .timestamp_ns = 2000 })); - try std.testing.expectEqual(@as(?u16, 1), processUtil(.{ .engine_ns = 100, .timestamp_ns = 1000 }, .{ .engine_ns = 101, .timestamp_ns = 2000 })); - try std.testing.expectEqual(@as(?u16, null), processUtil(null, .{ .engine_ns = 600, .timestamp_ns = 2000 })); -} - -test "oneAPI PWM fan speed conversion" { - try std.testing.expectEqual(@as(u64, 0), pwmToPercent(0)); - try std.testing.expectEqual(@as(u64, 50), pwmToPercent(128)); - try std.testing.expectEqual(@as(u64, 100), pwmToPercent(255)); - try std.testing.expectEqual(@as(u64, 100), pwmToPercent(300)); -} - -test "oneAPI process previous keeps engine samples" { - var usage: ProcessUsage = .{}; - defer usage.deinit(std.testing.allocator); - - const engine_key = processKey(10, 1); - const mem_only_key = processKey(20, 1); - try usage.put(std.testing.allocator, engine_key, .{ - .pid = 10, - .device_idx = 1, - .engine = .{ .engine_ns = 100, .timestamp_ns = 1000 }, - }); - try usage.put(std.testing.allocator, mem_only_key, .{ - .pid = 20, - .device_idx = 1, - .mem_kib = 2048, - }); - - var previous: ProcessPrevious = .{}; - defer previous.deinit(std.testing.allocator); - - try saveProcessPrevious(std.testing.allocator, &previous, &usage); - - try std.testing.expectEqual(@as(u32, 1), previous.count()); - try std.testing.expectEqual(@as(?EngineSample, .{ .engine_ns = 100, .timestamp_ns = 1000 }), previous.get(engine_key)); - try std.testing.expectEqual(@as(?EngineSample, null), previous.get(mem_only_key)); -} - -test "oneAPI power delta from hwmon energy" { - try std.testing.expectEqual(@as(?u64, 20_000), milliwattsFromEnergyDelta(1_000, 11_000, 1_000, 501_000)); - try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(11_000, 1_000, 1_000, 501_000)); - try std.testing.expectEqual(@as(?u64, null), milliwattsFromEnergyDelta(1_000, 11_000, 501_000, 1_000)); -} - -test "oneAPI resource line chooses large memory BARs" { - try std.testing.expectEqual(@as(?u64, 34_359_738_368), resourceLineSize("0x0000048800000000 0x0000048fffffffff 0x000000000014220c")); - try std.testing.expectEqual(@as(?u64, null), resourceLineSize("0x00000000f6c00000 0x00000000f6dfffff 0x0000000000046200")); -} - -test "oneAPI PCIe speed maps to generation" { - try std.testing.expectEqual(@as(u64, 1), try pcieGenFromSpeed("2.5 GT/s PCIe")); - try std.testing.expectEqual(@as(u64, 4), try pcieGenFromSpeed("16.0 GT/s PCIe")); - try std.testing.expectError(error.NotFound, pcieGenFromSpeed("Unknown")); -} - -test "oneAPI PCIe bandwidth derives from generation and width" { - try std.testing.expectEqual(@as(?u64, 250), pcieBandwidthFromLink(1, 1)); - try std.testing.expectEqual(@as(?u64, 63_008), pcieBandwidthFromLink(5, 16)); - try std.testing.expectEqual(@as(?u64, null), pcieBandwidthFromLink(0, 16)); -} diff --git a/bin/zml-smi/platforms/oneapi/oneapi.zig b/bin/zml-smi/platforms/oneapi/oneapi.zig deleted file mode 100644 index 2e1e5deb..00000000 --- a/bin/zml-smi/platforms/oneapi/oneapi.zig +++ /dev/null @@ -1,248 +0,0 @@ -const std = @import("std"); -const Monitor = @import("monitor.zig"); -const smi_sysfs = @import("zml-smi/sysfs"); - -const OneApi = @This(); - -pub const Handle = Monitor.Handle; - -monitor: Monitor, -io: std.Io, - -pub fn init(allocator: std.mem.Allocator, io: std.Io) !OneApi { - return .{ - .monitor = try Monitor.init(allocator, io), - .io = io, - }; -} - -pub fn handleByIndex(self: *const OneApi, device_id: usize) !Handle { - if (device_id >= self.monitor.devices.len) return error.NotFound; - return @fromBackingInt(@intCast(@as(u32, @intCast(device_id)))); -} - -pub fn name(self: *const OneApi, allocator: std.mem.Allocator, handle: Handle) ![]const u8 { - if (self.monitor.device(handle)) |dev| { - if (dev.bus_device_identifier) |bdf| { - if (try lspciName(allocator, self.io, bdf[0..])) |display_name| { - return display_name; - } - } - } else |_| {} - - const id = try self.monitor.deviceId(allocator, handle); - return try std.fmt.allocPrint(allocator, "Intel GPU {s}", .{id}); -} - -fn lspciName(allocator: std.mem.Allocator, io: std.Io, bdf: []const u8) !?[]const u8 { - const result = std.process.run(allocator, io, .{ - .argv = &.{ "lspci", "-Dnn", "-s", bdf }, - .stdout_limit = .limited(4096), - .stderr_limit = .limited(1024), - .reserve_amount = 256, - }) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - else => return null, - }; - defer allocator.free(result.stdout); - defer allocator.free(result.stderr); - - switch (result.term) { - .exited => |code| if (code != 0) return null, - else => return null, - } - - return parseLspciName(allocator, result.stdout); -} - -fn parseLspciName(allocator: std.mem.Allocator, output: []const u8) !?[]const u8 { - var lines = std.mem.tokenizeScalar(u8, output, '\n'); - while (lines.next()) |line| { - const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace); - const header_end = std.mem.indexOf(u8, trimmed, "]: ") orelse continue; - const header = trimmed[0 .. header_end + 1]; - if (!containsAsciiIgnoreCase(header, "VGA") and !containsAsciiIgnoreCase(header, "Display")) continue; - - const name_start = header_end + "]: ".len; - const name_end = std.mem.lastIndexOf(u8, trimmed[name_start..], " [8086:") orelse - std.mem.indexOf(u8, trimmed[name_start..], " (rev ") orelse - trimmed[name_start..].len; - const raw_name = std.mem.trim(u8, trimmed[name_start .. name_start + name_end], &std.ascii.whitespace); - if (raw_name.len == 0 or std.mem.startsWith(u8, raw_name, "Intel Corporation Device ")) return null; - return try allocator.dupe(u8, raw_name); - } - return null; -} - -fn containsAsciiIgnoreCase(haystack: []const u8, needle: []const u8) bool { - if (needle.len == 0) return true; - if (needle.len > haystack.len) return false; - for (0..haystack.len - needle.len + 1) |idx| { - if (std.ascii.eqlIgnoreCase(haystack[idx .. idx + needle.len], needle)) return true; - } - return false; -} - -test "parse lspci display name" { - const output = - \\0000:84:00.0 VGA compatible controller [0300]: Intel Corporation Battlemage G31 [Arc Pro B70] [8086:e223] (rev 04) - ; - const name_ = (try parseLspciName(std.testing.allocator, output)).?; - defer std.testing.allocator.free(name_); - try std.testing.expectEqualStrings("Intel Corporation Battlemage G31 [Arc Pro B70]", name_); -} - -test "parse lspci display name skips generic device ids" { - const output = - \\0000:84:00.0 Display controller [0380]: Intel Corporation Device e223 [8086:e223] (rev 04) - ; - try std.testing.expect(try parseLspciName(std.testing.allocator, output) == null); -} - -pub fn driverVersion(self: *const OneApi, allocator: std.mem.Allocator) ![]const u8 { - const version = smi_sysfs.readString(allocator, self.io, "/sys/module/xe/version") catch - smi_sysfs.readString(allocator, self.io, "/sys/module/xe/srcversion") catch - smi_sysfs.readString(allocator, self.io, "/sys/module/i915/version") catch - try smi_sysfs.readString(allocator, self.io, "/sys/module/i915/srcversion"); - const trimmed = std.mem.trim(u8, version, &std.ascii.whitespace); - if (trimmed.len == 0) return error.NotFound; - return trimmed; -} - -pub fn powerUsage(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = block: { - const idx = @backingInt(handle); - if (idx >= self.monitor.devices.len) return error.NotFound; - break :block &self.monitor.devices[idx]; - }; - const current = Monitor.readEnergy(allocator, self.io, dev) catch |err| { - dev.energy_prev = null; - return err; - }; - const power = Monitor.powerMilliwatts(dev.energy_prev, current) orelse { - dev.energy_prev = current; - return error.NotFound; - }; - dev.energy_prev = current; - return power; -} - -pub fn powerLimit(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readPowerLimit(allocator, self.io, dev); -} - -pub fn temperature(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readTemperature(allocator, self.io, dev); -} - -pub fn fanSpeed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readFanSpeed(allocator, self.io, dev); -} - -pub fn gpuUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = block: { - const idx = @backingInt(handle); - if (idx >= self.monitor.devices.len) return error.NotFound; - break :block &self.monitor.devices[idx]; - }; - - var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch { - dev.activity_prev = null; - return 0; - }; - defer usage.deinit(allocator); - - const dev_idx: u16 = @intCast(@backingInt(handle)); - const current = if (usage.get(dev_idx)) |sample| sample.engine else null; - const util = Monitor.processUtil(dev.activity_prev, current) orelse 0; - dev.activity_prev = current; - return util; -} - -pub fn encoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = block: { - const idx = @backingInt(handle); - if (idx >= self.monitor.devices.len) return error.NotFound; - break :block &self.monitor.devices[idx]; - }; - - var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { - dev.encoder_prev = null; - return err; - }; - defer usage.deinit(allocator); - - const dev_idx: u16 = @intCast(@backingInt(handle)); - const current = if (usage.get(dev_idx)) |sample| sample.encoder else null; - const util = Monitor.processUtil(dev.encoder_prev, current) orelse { - dev.encoder_prev = current; - return error.NotFound; - }; - dev.encoder_prev = current; - return util; -} - -pub fn decoderUtil(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = block: { - const idx = @backingInt(handle); - if (idx >= self.monitor.devices.len) return error.NotFound; - break :block &self.monitor.devices[idx]; - }; - - var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch |err| { - dev.decoder_prev = null; - return err; - }; - defer usage.deinit(allocator); - - const dev_idx: u16 = @intCast(@backingInt(handle)); - const current = if (usage.get(dev_idx)) |sample| sample.decoder else null; - const util = Monitor.processUtil(dev.decoder_prev, current) orelse { - dev.decoder_prev = current; - return error.NotFound; - }; - dev.decoder_prev = current; - return util; -} - -pub fn clockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readClockGraphics(allocator, self.io, dev); -} - -pub fn maxClockGraphics(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readMaxClockGraphics(allocator, self.io, dev); -} - -pub fn memUsed(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev_idx: u16 = @intCast(@backingInt(handle)); - - var usage = Monitor.collectDeviceUsage(allocator, self.io, self.monitor.devices) catch return 0; - defer usage.deinit(allocator); - - return if (usage.get(dev_idx)) |sample| sample.mem_kib * 1024 else 0; -} - -pub fn memTotal(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readMemTotal(allocator, self.io, dev); -} - -pub fn pcieLinkGen(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readPcieLinkGen(allocator, self.io, dev); -} - -pub fn pcieLinkWidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readPcieLinkWidth(allocator, self.io, dev); -} - -pub fn pcieBandwidth(self: *OneApi, allocator: std.mem.Allocator, handle: Handle) !u64 { - const dev = try self.monitor.device(handle); - return Monitor.readPcieBandwidth(allocator, self.io, dev); -} diff --git a/bin/zml-smi/platforms/oneapi/process.zig b/bin/zml-smi/platforms/oneapi/process.zig deleted file mode 100644 index c4d0c719..00000000 --- a/bin/zml-smi/platforms/oneapi/process.zig +++ /dev/null @@ -1,45 +0,0 @@ -const std = @import("std"); -const OneApi = @import("oneapi.zig"); -const Monitor = @import("monitor.zig"); -const pi = @import("zml-smi/info").process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const Collector = @import("zml-smi/collector").Collector; - -pub fn init(collector: *Collector, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16) !void { - const state = try collector.arena.create(Monitor.ProcessPrevious); - state.* = .{}; - - const previous_usage: ?Monitor.ProcessUsage = Monitor.collectProcessUsage(collector.gpa, collector.io, oneapi.monitor.devices) catch null; - if (previous_usage) |usage_value| { - var usage = usage_value; - defer usage.deinit(collector.gpa); - Monitor.saveProcessPrevious(collector.arena, state, &usage) catch {}; - } - - try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.arena, collector.io, list, oneapi, dev_offset, state }); -} - -fn pollOnce(allocator: std.mem.Allocator, arena: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, oneapi: *OneApi, dev_offset: u16, state: *Monitor.ProcessPrevious) void { - const back = list.back(); - back.clearRetainingCapacity(); - - var usage = Monitor.collectProcessUsage(allocator, io, oneapi.monitor.devices) catch { - list.swap(); - return; - }; - defer usage.deinit(allocator); - - var it = usage.iterator(); - while (it.next()) |entry| { - const sample = entry.value_ptr.*; - back.append(allocator, .{ - .pid = sample.pid, - .device_idx = @intCast(sample.device_idx + dev_offset), - .dev_mem_kib = sample.mem_kib, - .dev_util_percent = Monitor.processUtil(state.get(entry.key_ptr.*), sample.engine), - }) catch break; - } - - Monitor.saveProcessPrevious(arena, state, &usage) catch {}; - list.swap(); -} diff --git a/bin/zml-smi/platforms/tpu/BUILD.bazel b/bin/zml-smi/platforms/tpu/BUILD.bazel deleted file mode 100644 index a65377b3..00000000 --- a/bin/zml-smi/platforms/tpu/BUILD.bazel +++ /dev/null @@ -1,30 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") - -cc_library( - name = "tpuinfo", - srcs = ["tpuinfo.cc"], - hdrs = ["tpuinfo.h"], - deps = [ - "@cloud_accelerator_diagnostics//:tpu_metric_service_cc_grpc", - "@grpc//:grpc++_unsecure", - ], -) - -zig_library( - name = "tpu", - import_name = "zml-smi/platforms/tpu", - main = "metrics.zig", - srcs = [ - "process.zig", - "tpuinfo.zig", - ], - visibility = ["//visibility:public"], - deps = [ - ":tpuinfo", - "//bin/zml-smi/collector", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//bin/zml-smi/utils:sysfs", - ], -) diff --git a/bin/zml-smi/platforms/tpu/metrics.zig b/bin/zml-smi/platforms/tpu/metrics.zig deleted file mode 100644 index 9410b3f8..00000000 --- a/bin/zml-smi/platforms/tpu/metrics.zig +++ /dev/null @@ -1,168 +0,0 @@ -const std = @import("std"); -const sysfs = @import("zml-smi/sysfs"); -const tpuinfo = @import("tpuinfo.zig"); -const device_info = @import("zml-smi/info").device_info; -const DeviceInfo = device_info.DeviceInfo; -const TpuInfo = device_info.TpuInfo; -const Collector = @import("zml-smi/collector").Collector; -const tpu_process = @import("process.zig"); - -const address = "localhost:8431"; - -pub fn start(collector: *Collector) !void { - const chip = scanPciChips(collector.arena, collector.io) orelse return error.TpuUnavailable; - const device_count = chip.chip_count * chip.devices_per_chip; - var tpu_infos: std.ArrayList(*DeviceInfo) = .empty; - const dev_offset: u8 = @intCast(collector.device_infos.items.len); - - for (0..device_count) |_| { - const chip_name = try collector.arena.dupe(u8, chip.name); - const initial: TpuInfo = .{ .name = chip_name }; - const info = try collector.addDevice(.{ .tpu = .{ .values = .{ initial, initial } } }); - try tpu_infos.append(collector.arena, info); - } - - try collector.spawnPoll(pollAllDevices, .{tpu_infos.items}); - - if (tpu_infos.items.len > 0) { - const processes = try collector.createProcessList(); - try tpu_process.init(collector, chip.devices_per_chip, tpu_infos.items, processes, dev_offset); - } -} - -const MetricKind = enum { int, double }; - -fn pollAllDevices(devs: []*DeviceInfo) void { - var mem_used: [tpuinfo.max_devices]?u64 = undefined; - var mem_total: [tpuinfo.max_devices]?u64 = undefined; - var util: [tpuinfo.max_devices]?u64 = undefined; - - for (devs, 0..) |info, i| { - const front = info.tpu.front(); - mem_used[i] = front.mem_used_bytes; - mem_total[i] = front.mem_total_bytes; - util[i] = front.util_percent; - } - - queryRaw("tpu.runtime.hbm.memory.usage.bytes", .int, mem_used[0..devs.len]); - queryRaw("tpu.runtime.hbm.memory.total.bytes", .int, mem_total[0..devs.len]); - queryRaw("tpu.runtime.tensorcore.dutycycle.percent", .double, util[0..devs.len]); - - for (devs, 0..) |info, i| { - const back = info.tpu.back(); - back.* = info.tpu.front().*; - back.mem_used_bytes = mem_used[i]; - back.mem_total_bytes = mem_total[i]; - back.util_percent = util[i]; - info.tpu.swap(); - } -} - -fn queryRaw(metric_name: [:0]const u8, comptime kind: MetricKind, out: []?u64) void { - var ids: [tpuinfo.max_devices]c_longlong = undefined; - switch (kind) { - .int => { - var vals: [tpuinfo.max_devices]c_longlong = undefined; - const n = tpuinfo.queryInt(address, metric_name, &ids, &vals) catch { - @memset(out, null); - return; - }; - - for (ids[0..n], vals[0..n]) |id, val| { - if (id >= 0 and id < out.len) out[@intCast(id)] = @intCast(val); - } - }, - .double => { - var vals: [tpuinfo.max_devices]f64 = undefined; - const n = tpuinfo.queryDouble(address, metric_name, &ids, &vals) catch { - @memset(out, null); - return; - }; - - for (ids[0..n], vals[0..n]) |id, val| { - if (id >= 0 and id < out.len and std.math.isFinite(val)) - out[@intCast(id)] = @intFromFloat(@round(val)); - } - }, - } -} - -// -- Discover available chips -// We need to scan /sys/bus/pci/devices for Google TPU chips. -// From: https://github.com/AI-Hypercomputer/cloud-accelerator-diagnostics/blob/c94f463dd3bf99eaecf4f5af46f87cc2a5ff1b22/tpu_info/tpu_info/device.py#L43-L92 - -const ChipInfo = struct { - name: []const u8, - devices_per_chip: u32, - chip_count: u32, -}; - -const google_pci_vendor_id = "0x1ae0"; -const pci_base = "/sys/bus/pci/devices"; - -const ChipEntry = struct { - device_id: []const u8, - subsystem_id: ?[]const u8 = null, - name: []const u8, - devices_per_chip: u32, -}; - -const chip_table = [_]ChipEntry{ - .{ .device_id = "0x0027", .subsystem_id = "0x004e", .name = "TPU v2", .devices_per_chip = 2 }, - .{ .device_id = "0x0027", .subsystem_id = "0x004f", .name = "TPU v3", .devices_per_chip = 2 }, - .{ .device_id = "0x005e", .name = "TPU v4", .devices_per_chip = 1 }, - .{ .device_id = "0x0063", .name = "TPU v5e", .devices_per_chip = 1 }, - .{ .device_id = "0x0062", .name = "TPU v5p", .devices_per_chip = 1 }, - .{ .device_id = "0x006f", .name = "TPU v6e", .devices_per_chip = 1 }, - .{ .device_id = "0x0076", .name = "TPU7x", .devices_per_chip = 2 }, -}; - -fn scanPciChips(allocator: std.mem.Allocator, io: std.Io) ?ChipInfo { - var pci_dir = std.Io.Dir.openDir(.cwd(), io, pci_base, .{ .iterate = true }) catch return null; - defer pci_dir.close(io); - - var it = pci_dir.iterate(); - var chip_count: u32 = 0; - var result: ChipInfo = undefined; - - while (it.next(io) catch null) |entry| { - const vendor = readSysfs(allocator, io, entry.name, "vendor") catch continue; - if (!std.mem.eql(u8, vendor, google_pci_vendor_id)) { - continue; - } - - const device_raw = readSysfs(allocator, io, entry.name, "device") catch continue; - const subsystem_raw: ?[]const u8 = readSysfs(allocator, io, entry.name, "subsystem_device") catch null; - - for (chip_table) |chip| { - const dev_match = std.mem.eql(u8, device_raw, chip.device_id); - const sub_match = if (chip.subsystem_id) |expected_sub| - if (subsystem_raw) |actual_sub| std.mem.eql(u8, actual_sub, expected_sub) else false - else - true; - - if (dev_match and sub_match) { - result = .{ - .name = chip.name, - .devices_per_chip = chip.devices_per_chip, - .chip_count = undefined, - }; - - chip_count += 1; - break; - } - } - } - - if (chip_count > 0) { - result.chip_count = chip_count; - return result; - } - - return null; -} - -fn readSysfs(allocator: std.mem.Allocator, io: std.Io, slot: []const u8, file: []const u8) ![]const u8 { - const path = try std.fmt.allocPrint(allocator, pci_base ++ "/{s}/{s}", .{ slot, file }); - return sysfs.readString(allocator, io, path); -} diff --git a/bin/zml-smi/platforms/tpu/process.zig b/bin/zml-smi/platforms/tpu/process.zig deleted file mode 100644 index cca8f6a1..00000000 --- a/bin/zml-smi/platforms/tpu/process.zig +++ /dev/null @@ -1,73 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const pi = smi_info.process_info; -const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const Collector = @import("zml-smi/collector").Collector; - -pub fn init(collector: *Collector, devices_per_chip: u32, device_infos: []*DeviceInfo, list: *ProcessDoubleBuffer, dev_offset: u16) !void { - try collector.spawnPoll(pollOnce, .{ collector.gpa, collector.io, list, devices_per_chip, device_infos, dev_offset }); -} - -fn pollOnce(allocator: std.mem.Allocator, io: std.Io, list: *ProcessDoubleBuffer, devices_per_chip: u32, device_infos: []*DeviceInfo, dev_offset: u16) void { - scan(allocator, io, list.back(), devices_per_chip, device_infos, dev_offset); - list.swap(); -} - -fn scan(allocator: std.mem.Allocator, io: std.Io, back: *std.ArrayList(pi.ProcessInfo), devices_per_chip: u32, infos: []*DeviceInfo, dev_offset: u16) void { - back.clearRetainingCapacity(); - - var proc_dir = std.Io.Dir.openDirAbsolute(io, "/proc", .{ .iterate = true }) catch return; - defer proc_dir.close(io); - - var proc_it = proc_dir.iterate(); - while (proc_it.next(io) catch null) |proc_entry| { - const pid = std.fmt.parseInt(u32, proc_entry.name, 10) catch continue; - - var fd_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const fd_sub = std.fmt.bufPrint(&fd_path_buf, "{d}/fd", .{pid}) catch continue; - var fd_dir = proc_dir.openDir(io, fd_sub, .{ .iterate = true }) catch continue; - defer fd_dir.close(io); - - var fd_it = fd_dir.iterate(); - while (fd_it.next(io) catch null) |fd_entry| { - var link_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const len = fd_dir.readLink(io, fd_entry.name, &link_buf) catch continue; - const target = link_buf[0..len]; - - if (parseChipIndex(target)) |chip_idx| { - const base: u16 = @intCast(chip_idx * devices_per_chip); - for (0..devices_per_chip) |d| { - const local_idx: u16 = @intCast(base + d); - var info: pi.ProcessInfo = .{ .pid = pid, .device_idx = local_idx + dev_offset }; - - if (local_idx < infos.len) { - const tpu = infos[local_idx].tpu.front().*; - - if (tpu.mem_used_bytes) |mem| { - info.dev_mem_kib = @intCast(mem / 1024); - } - - if (tpu.util_percent) |util| { - info.dev_util_percent = @intCast(util); - } - } - - back.append(allocator, info) catch return; - } - } - } - } -} - -fn parseChipIndex(target: []const u8) ?u32 { - if (std.mem.startsWith(u8, target, "/dev/accel")) { - return std.fmt.parseInt(u32, target["/dev/accel".len..], 10) catch null; - } - - if (std.mem.startsWith(u8, target, "/dev/vfio/")) { - return std.fmt.parseInt(u32, target["/dev/vfio/".len..], 10) catch null; - } - - return null; -} diff --git a/bin/zml-smi/platforms/tpu/tpuinfo.cc b/bin/zml-smi/platforms/tpu/tpuinfo.cc deleted file mode 100644 index a1e3fbda..00000000 --- a/bin/zml-smi/platforms/tpu/tpuinfo.cc +++ /dev/null @@ -1,84 +0,0 @@ -#include "tpuinfo.h" - -#include -#include -#include -#include - -#include - -#include "tpu_metric_service.grpc.pb.h" -#include "tpu_metric_service.pb.h" - -using tpu::monitoring::runtime::MetricRequest; -using tpu::monitoring::runtime::MetricResponse; -using tpu::monitoring::runtime::RuntimeMetricService; - -static MetricResponse FetchMetric(const char* address, const char* metric_name, - bool* ok) { - auto channel = - grpc::CreateChannel(address, grpc::InsecureChannelCredentials()); - auto stub = RuntimeMetricService::NewStub(channel); - - grpc::ClientContext ctx; - ctx.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(1)); - - MetricRequest req; - req.set_metric_name(metric_name); - - MetricResponse resp; - auto status = stub->GetRuntimeMetric(&ctx, req, &resp); - *ok = status.ok(); - - return resp; -} - -extern "C" int tpu_query_int(const char* address, const char* metric_name, - long long* device_ids, long long* values, - int max_n) { - bool ok; - auto resp = FetchMetric(address, metric_name, &ok); - if (!ok) return -1; - - std::map sorted; - for (const auto& m : resp.metric().metrics()) { - int id = static_cast(m.attribute().value().int_attr()); - sorted[id] = m.gauge().as_int(); - } - - int i = 0; - for (const auto& [id, val] : sorted) { - if (i >= max_n) break; - - device_ids[i] = id; - values[i] = val; - ++i; - } - - return i; -} - -extern "C" int tpu_query_double(const char* address, const char* metric_name, - long long* device_ids, double* values, - int max_n) { - bool ok; - auto resp = FetchMetric(address, metric_name, &ok); - if (!ok) return -1; - - std::map sorted; - for (const auto& m : resp.metric().metrics()) { - int id = static_cast(m.attribute().value().int_attr()); - sorted[id] = m.gauge().as_double(); - } - - int i = 0; - for (const auto& [id, val] : sorted) { - if (i >= max_n) break; - - device_ids[i] = id; - values[i] = val; - ++i; - } - - return i; -} diff --git a/bin/zml-smi/platforms/tpu/tpuinfo.h b/bin/zml-smi/platforms/tpu/tpuinfo.h deleted file mode 100644 index f2dc7db6..00000000 --- a/bin/zml-smi/platforms/tpu/tpuinfo.h +++ /dev/null @@ -1,20 +0,0 @@ -#ifndef TPUINFO_H_ -#define TPUINFO_H_ - -#define TPUINFO_MAX_DEVICES 64 - -#ifdef __cplusplus -extern "C" { -#endif - -int tpu_query_int(const char* address, const char* metric_name, - long long* device_ids, long long* values, int max_n); - -int tpu_query_double(const char* address, const char* metric_name, - long long* device_ids, double* values, int max_n); - -#ifdef __cplusplus -} -#endif - -#endif /* TPUINFO_H_ */ diff --git a/bin/zml-smi/platforms/tpu/tpuinfo.zig b/bin/zml-smi/platforms/tpu/tpuinfo.zig deleted file mode 100644 index 1baf62df..00000000 --- a/bin/zml-smi/platforms/tpu/tpuinfo.zig +++ /dev/null @@ -1,37 +0,0 @@ -const std = @import("std"); -const c = @import("c"); - -pub const max_devices: usize = c.TPUINFO_MAX_DEVICES; - -pub const Error = error{ - TpuUnavailable, - QueryFailed, -}; - -pub fn queryInt( - address: [:0]const u8, - metric_name: [:0]const u8, - device_ids: []c_longlong, - values: []c_longlong, -) Error!u32 { - const n = c.tpu_query_int(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); - if (n < 0) { - return error.QueryFailed; - } - - return @intCast(n); -} - -pub fn queryDouble( - address: [:0]const u8, - metric_name: [:0]const u8, - device_ids: []c_longlong, - values: []f64, -) Error!u32 { - const n = c.tpu_query_double(address.ptr, metric_name.ptr, device_ids.ptr, values.ptr, @intCast(device_ids.len)); - if (n < 0) { - return error.QueryFailed; - } - - return @intCast(n); -} diff --git a/bin/zml-smi/prometheus/BUILD.bazel b/bin/zml-smi/prometheus/BUILD.bazel deleted file mode 100644 index c00a26b5..00000000 --- a/bin/zml-smi/prometheus/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "prometheus", - import_name = "zml-smi/prometheus", - main = "prometheus.zig", - srcs = [ - "server.zig", - "exposition.zig", - ], - visibility = ["//visibility:public"], - deps = [ - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - ], -) diff --git a/bin/zml-smi/prometheus/exposition.zig b/bin/zml-smi/prometheus/exposition.zig deleted file mode 100644 index 54d48cb6..00000000 --- a/bin/zml-smi/prometheus/exposition.zig +++ /dev/null @@ -1,159 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const HostInfo = smi_info.host_info.HostInfo; - -const device_metrics = .{ - // Shared across all device types - .{ .field = "util_percent", .metric = "zml_device_utilization_percent", .help = "Device utilization percentage" }, - .{ .field = "mem_used_bytes", .metric = "zml_device_memory_used_bytes", .help = "Device memory used in bytes" }, - .{ .field = "mem_total_bytes", .metric = "zml_device_memory_total_bytes", .help = "Device memory total in bytes" }, - // GPU (CUDA/ROCm/oneAPI) - .{ .field = "encoder_util_percent", .metric = "zml_device_encoder_utilization_percent", .help = "Encoder utilization percentage" }, - .{ .field = "decoder_util_percent", .metric = "zml_device_decoder_utilization_percent", .help = "Decoder utilization percentage" }, - .{ .field = "power_mw", .metric = "zml_device_power_watts", .help = "Power draw in watts", .fmt = fmtMilliwattsAsWatts }, - .{ .field = "power_limit_mw", .metric = "zml_device_power_limit_watts", .help = "Power limit in watts", .fmt = fmtMilliwattsAsWatts }, - .{ .field = "temperature", .metric = "zml_device_temperature_celsius", .help = "Device temperature in celsius" }, - .{ .field = "fan_speed_percent", .metric = "zml_device_fan_speed_percent", .help = "Fan speed percentage" }, - .{ .field = "clock_graphics_mhz", .metric = "zml_device_clock_graphics_mhz", .help = "Graphics clock frequency in MHz" }, - .{ .field = "clock_sm_mhz", .metric = "zml_device_clock_sm_mhz", .help = "SM clock frequency in MHz" }, - .{ .field = "clock_mem_mhz", .metric = "zml_device_clock_memory_mhz", .help = "Memory clock frequency in MHz" }, - .{ .field = "pcie_tx_kbps", .metric = "zml_device_pcie_tx_bytes_per_second", .help = "PCIe TX throughput in bytes per second", .fmt = fmtScaled(128) }, - .{ .field = "pcie_rx_kbps", .metric = "zml_device_pcie_rx_bytes_per_second", .help = "PCIe RX throughput in bytes per second", .fmt = fmtScaled(128) }, - // Neuron - .{ .field = "nc_tensors", .metric = "zml_device_neuron_tensors_bytes", .help = "Neuron core tensor memory in bytes" }, - .{ .field = "nc_constants", .metric = "zml_device_neuron_constants_bytes", .help = "Neuron core constants memory in bytes" }, - .{ .field = "nc_model_code", .metric = "zml_device_neuron_model_code_bytes", .help = "Neuron core model code memory in bytes" }, - .{ .field = "nc_shared_scratchpad", .metric = "zml_device_neuron_shared_scratchpad_bytes", .help = "Neuron core shared scratchpad memory in bytes" }, - .{ .field = "nc_nonshared_scratchpad", .metric = "zml_device_neuron_nonshared_scratchpad_bytes", .help = "Neuron core non-shared scratchpad memory in bytes" }, - .{ .field = "nc_runtime", .metric = "zml_device_neuron_runtime_bytes", .help = "Neuron core runtime memory in bytes" }, - .{ .field = "nc_driver", .metric = "zml_device_neuron_driver_bytes", .help = "Neuron core driver memory in bytes" }, - .{ .field = "nc_dma_rings", .metric = "zml_device_neuron_dma_rings_bytes", .help = "Neuron core DMA rings memory in bytes" }, - .{ .field = "nc_collectives", .metric = "zml_device_neuron_collectives_bytes", .help = "Neuron core collectives memory in bytes" }, - .{ .field = "nc_notifications", .metric = "zml_device_neuron_notifications_bytes", .help = "Neuron core notifications memory in bytes" }, - .{ .field = "nc_uncategorized", .metric = "zml_device_neuron_uncategorized_bytes", .help = "Neuron core uncategorized memory in bytes" }, -}; - -const host_metrics = .{ - .{ .field = "cpu_cores", .metric = "zml_host_cpu_cores", .help = "Number of CPU cores" }, - .{ .field = "mem_total_kib", .metric = "zml_host_memory_total_bytes", .help = "Total host memory in bytes", .fmt = fmtScaled(1024) }, - .{ .field = "mem_available_kib", .metric = "zml_host_memory_available_bytes", .help = "Available host memory in bytes", .fmt = fmtScaled(1024) }, - .{ .field = "load_1", .metric = "zml_host_load_1", .help = "1-minute load average" }, - .{ .field = "load_5", .metric = "zml_host_load_5", .help = "5-minute load average" }, - .{ .field = "load_15", .metric = "zml_host_load_15", .help = "15-minute load average" }, - .{ .field = "uptime_seconds", .metric = "zml_host_uptime_seconds", .help = "System uptime in seconds" }, -}; - -pub fn write(writer: *std.Io.Writer, devices: []const *DeviceInfo, host: *const HostInfo) !void { - try writer.writeAll("# HELP zml_device_info Device metadata\n"); - try writer.writeAll("# TYPE zml_device_info gauge\n"); - for (devices, 0..) |dev, i| { - inline for (@typeInfo(DeviceInfo).@"union".field_names) |type_name| { - const tag = @field(smi_info.device_info.Target, type_name); - - switch (dev.*) { - tag => |*db| { - const val = db.front().*; - - try writer.print("zml_device_info{{device=\"{d}\",type=\"{s}\"", .{ i, type_name }); - - inline for (.{ - .{ "name", "name" }, - .{ "driver_version", "driver" }, - .{ "cuda_driver_version", "cuda_version" }, - }) |label| { - if (@hasField(@TypeOf(val), label[0])) { - if (@field(val, label[0])) |v| { - try writer.print(",{s}=\"", .{label[1]}); - try std.zig.stringEscape(v, writer); - try writer.writeAll("\""); - } - } - } - - try writer.writeAll("} 1\n"); - }, - else => {}, - } - } - } - try writer.writeAll("\n"); - - try writeAllDeviceMetrics(writer, device_metrics, devices); - try writeHostMetrics(writer, host.front().*); -} - -fn writeAllDeviceMetrics( - writer: *std.Io.Writer, - comptime metrics: anytype, - devices: []const *DeviceInfo, -) !void { - inline for (metrics) |m| { - var header_written = false; - - for (devices, 0..) |dev, i| { - switch (dev.*) { - inline else => |*db| { - const val = db.front().*; - if (@hasField(@TypeOf(val), m.field)) { - if (@field(val, m.field)) |raw| { - if (!header_written) { - try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n", .{ m.metric, m.help, m.metric }); - header_written = true; - } - - const device_type = @tagName(std.meta.activeTag(dev.*)); - try writer.print("{s}{{device=\"{d}\",type=\"{s}\",name=\"", .{ m.metric, i, device_type }); - try std.zig.stringEscape(val.name orelse "", writer); - try writer.writeAll("\"} "); - try fmtValue(m, writer, raw); - try writer.writeAll("\n"); - } - } - }, - } - } - - if (header_written) { - try writer.writeAll("\n"); - } - } -} - -fn writeHostMetrics(writer: *std.Io.Writer, val: anytype) !void { - inline for (host_metrics) |m| { - if (@field(val, m.field)) |raw| { - try writer.print("# HELP {s} {s}\n# TYPE {s} gauge\n{s} ", .{ m.metric, m.help, m.metric, m.metric }); - try fmtValue(m, writer, raw); - try writer.writeAll("\n\n"); - } - } -} - -fn fmtValue(comptime m: anytype, writer: *std.Io.Writer, raw: anytype) !void { - if (@hasField(@TypeOf(m), "fmt")) { - try m.fmt(writer, raw); - } else { - try writeValue(writer, raw); - } -} - -fn writeValue(writer: *std.Io.Writer, val: anytype) std.Io.Writer.Error!void { - switch (@typeInfo(@TypeOf(val))) { - .float, .comptime_float => try writer.print("{d:.2}", .{val}), - .pointer => try writer.writeAll(val), - else => try writer.print("{d}", .{val}), - } -} - -fn fmtMilliwattsAsWatts(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { - try writer.print("{d}.{d:0>3}", .{ val / 1000, val % 1000 }); -} - -fn fmtScaled(comptime scale: u64) fn (*std.Io.Writer, u64) std.Io.Writer.Error!void { - return struct { - fn f(writer: *std.Io.Writer, val: u64) std.Io.Writer.Error!void { - try writer.print("{d}", .{val * scale}); - } - }.f; -} diff --git a/bin/zml-smi/prometheus/prometheus.zig b/bin/zml-smi/prometheus/prometheus.zig deleted file mode 100644 index cfa2fc28..00000000 --- a/bin/zml-smi/prometheus/prometheus.zig +++ /dev/null @@ -1,2 +0,0 @@ -pub const Server = @import("server.zig").Server; -pub const exposition = @import("exposition.zig"); diff --git a/bin/zml-smi/prometheus/server.zig b/bin/zml-smi/prometheus/server.zig deleted file mode 100644 index 9517ecac..00000000 --- a/bin/zml-smi/prometheus/server.zig +++ /dev/null @@ -1,87 +0,0 @@ -const std = @import("std"); -const smi_info = @import("zml-smi/info"); -const DeviceInfo = smi_info.device_info.DeviceInfo; -const HostInfo = smi_info.host_info.HostInfo; -const exposition = @import("exposition.zig"); - -pub const Server = struct { - io: std.Io, - devices: []const *DeviceInfo, - host: *const HostInfo, - tcp: std.Io.net.Server, - connection_group: std.Io.Group = .init, - - pub fn init(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host: *const HostInfo) !Server { - const address = try std.Io.net.IpAddress.parseLiteral(listen); - var tcp = try address.listen(io, .{ .reuse_address = true }); - errdefer tcp.deinit(io); - - return .{ - .io = io, - .devices = devices, - .host = host, - .tcp = tcp, - }; - } - - pub fn run(io: std.Io, listen: []const u8, devices: []const *DeviceInfo, host_info: *const HostInfo) void { - var self = init(io, listen, devices, host_info) catch |err| { - std.log.err("prometheus server failed to start: {s}", .{@errorName(err)}); - return; - }; - defer self.deinit(); - self.serve(); - } - - pub fn deinit(self: *Server) void { - self.connection_group.await(self.io) catch {}; - self.tcp.deinit(self.io); - } - - pub fn serve(self: *Server) void { - while (true) { - const stream = self.tcp.accept(self.io) catch break; - self.connection_group.concurrent(self.io, Server.onConnection, .{ self, stream }) catch break; - } - } - - fn onConnection(self: *Server, stream: std.Io.net.Stream) std.Io.Cancelable!void { - defer stream.close(self.io); - self.handleConn(stream) catch {}; - } - - fn handleConn(self: *Server, stream: std.Io.net.Stream) !void { - var read_buf: [8192]u8 = undefined; - var reader = stream.reader(self.io, &read_buf); - - var write_buf: [4096]u8 = undefined; - var writer = stream.writer(self.io, &write_buf); - - var http_server: std.http.Server = .init(&reader.interface, &writer.interface); - - while (true) { - var request = http_server.receiveHead() catch |err| switch (err) { - error.HttpConnectionClosing => return, - else => return err, - }; - - if (request.head.method != .GET) { - try request.respond("Method Not Allowed\n", .{ .status = .method_not_allowed }); - continue; - } - - var body_buf: [4096]u8 = undefined; - var body_writer = try request.respondStreaming(&body_buf, .{ - .respond_options = .{ - .status = .ok, - .extra_headers = &.{ - .{ .name = "content-type", .value = "text/plain" }, - }, - }, - }); - - try exposition.write(&body_writer.writer, self.devices, self.host); - try body_writer.end(); - } - } -}; diff --git a/bin/zml-smi/scripts/release.sh b/bin/zml-smi/scripts/release.sh deleted file mode 100755 index d1e89902..00000000 --- a/bin/zml-smi/scripts/release.sh +++ /dev/null @@ -1,395 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# --- UI --- - -if [ -t 1 ]; then - DIM='\033[2m' - RESET='\033[0m' - BLUE='\033[1;38;2;140;180;255m' - GREEN='\033[1;32m' - RED='\033[1;31m' - YELLOW='\033[1;33m' - CYAN='\033[1;36m' - WHITE='\033[1;37m' -else - DIM='' RESET='' BLUE='' GREEN='' RED='' YELLOW='' CYAN='' WHITE='' -fi - -info() { printf "${BLUE} │${RESET} %b\n" "$*"; } -success() { printf "${GREEN} ✓${RESET} %b\n" "$*"; } -warn() { printf "${YELLOW} !${RESET} %b\n" "$*"; } -fail() { printf "${RED} ✗ %b${RESET}\n" "$*" >&2; exit 1; } -step() { printf "\n${WHITE} ▸ %b${RESET}\n" "$*"; } - -confirm() { - if [ "$YES" = true ]; then - return 0 - fi - printf "${WHITE} ? %b ${DIM}[y/N]${RESET} " "$*" - local answer - read -r answer - case "$answer" in - y|Y|yes|Yes) return 0 ;; - *) return 1 ;; - esac -} - -usage() { - printf >&2 " - ${WHITE}Usage${RESET} - bazel run //bin/zml-smi:release -- [flags] - - ${WHITE}Arguments${RESET} - ${CYAN}version${RESET} Version string, e.g. ${DIM}v1.0.0${RESET} (tagged as ${DIM}zml-smi-v1.0.0${RESET}) - - ${WHITE}Flags${RESET} - ${CYAN}--dry-run${RESET} Build only — skip tagging, uploading, and publishing - ${CYAN}--verify${RESET} Verify uploaded files by comparing sha256 against the CDN - ${CYAN}--publish${RESET} Push git tag and create a GitHub release - ${CYAN}-y, --yes${RESET} Skip all confirmation prompts - ${CYAN}-h, --help${RESET} Show this help - - ${WHITE}Environment${RESET} - ${CYAN}BASE_URL${RESET} Public base URL serving the bucket - ${CYAN}RCLONE_REMOTE${RESET} Rclone remote name ${DIM}(default: r2)${RESET} - - ${DIM}If rclone is not configured, you can set the remote via env vars:${RESET} - ${DIM} RCLONE_CONFIG_R2_TYPE=s3${RESET} - ${DIM} RCLONE_CONFIG_R2_PROVIDER=Cloudflare${RESET} - ${DIM} RCLONE_CONFIG_R2_ACCESS_KEY_ID=your_access_key${RESET} - ${DIM} RCLONE_CONFIG_R2_SECRET_ACCESS_KEY=your_secret_key${RESET} - ${DIM} RCLONE_CONFIG_R2_ENDPOINT=https://.r2.cloudflarestorage.com${RESET} - - ${WHITE}Examples${RESET} - ${DIM}# Interactive, local only${RESET} - bazel run //bin/zml-smi:release -- 0.4.0 - - ${DIM}# Full release, non-interactive${RESET} - bazel run //bin/zml-smi:release -- --publish --verify --yes 0.4.0 - -" - exit 0 -} - -# --- Checks --- - -check_env() { - if [ -z "${BUILD_WORKSPACE_DIRECTORY:-}" ]; then - fail "Must be run via 'bazel run //bin/zml-smi:release -- '" - fi - if [ -z "${BASE_URL:-}" ]; then - fail "Missing required environment variable: ${CYAN}BASE_URL" - fi -} - -# --- Steps --- - -changelog() { - local prev_tag - prev_tag="$(git describe --tags --abbrev=0 --match 'zml-smi-*' 2>/dev/null || true)" - local range="${prev_tag:+${prev_tag}..HEAD}" - - git log ${range:-HEAD} --pretty=format:"- %s" -- bin/zml-smi/ | grep -vi WIP || true -} - -tag() { - step "Tagging ${CYAN}${TAG}${WHITE}" - - # Cache changelog before tagging, otherwise the new tag makes the range empty - CACHED_CHANGELOG="$(changelog)" - - if [ -z "$CACHED_CHANGELOG" ]; then - warn "No changes since last release" - fi - - if git rev-parse "$TAG" &>/dev/null; then - warn "Tag ${TAG} already exists, skipping" - return - fi - - if [ "$DRY_RUN" = true ]; then - warn "Dry run — skipping tag creation" - elif confirm "Create tag ${CYAN}${TAG}${RESET}${WHITE}?"; then - git tag -a "$TAG" -m "$(printf '%s\n\n%s' "$TAG" "$CACHED_CHANGELOG")" - success "Created tag ${CYAN}${TAG}" - else - warn "Skipped tagging" - return - fi - printf "\n" - - printf "${DIM}" - echo "$CACHED_CHANGELOG" - printf "${RESET}" -} - -build() { - step "Building release archives" - - if ! confirm "Build all platforms?"; then - warn "Skipped build" - return 1 - fi - - printf "${DIM}%s${RESET}\n" "────────────────────────────────────────" - - bazel build --config=release //bin/zml-smi:archives - - printf "${DIM}%s${RESET}\n" "────────────────────────────────────────" - FILES=() - while IFS= read -r line; do - FILES+=("$line") - done < <(bazel cquery --config=release --output=files //bin/zml-smi:archives 2>/dev/null) - success "Built ${CYAN}${#FILES[@]}${RESET}${GREEN} artifacts" -} - -upload() { - local rclone_remote="${RCLONE_REMOTE:-r2}" - local install_url="${BASE_URL}/zml-smi/${VERSION}" - - step "Patching install script" - - INSTALL_SCRIPT="$(mktemp)" - sed -e "s|^BASE_URL=.*|BASE_URL=\"${install_url}\"|" \ - -e "s|^VERSION=.*|VERSION=\"${VERSION}\"|" \ - "${BUILD_WORKSPACE_DIRECTORY}/bin/zml-smi/scripts/remote-install.sh" > "$INSTALL_SCRIPT" - success "BASE_URL=${CYAN}${install_url}" - - if [ "$DRY_RUN" = true ]; then - step "Uploading to ${CYAN}${rclone_remote}" - warn "Dry run — skipping upload" - return - fi - - step "Uploading to ${CYAN}${rclone_remote}" - - if ! confirm "Upload artifacts?"; then - warn "Skipped upload" - return - fi - - for f in "${FILES[@]}"; do - if [ ! -f "$f" ]; then - warn "SKIP $(basename "$f") ${DIM}(not built for this host)${RESET}" - continue - fi - local name remote_name - name="$(basename "$f")" - remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" - rclone copyto "$f" "${rclone_remote}:zml-smi/${VERSION}/${remote_name}" - success "${remote_name}" - done - - rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:zml-smi/${VERSION}/install.sh" - success "zml-smi/${VERSION}/install.sh" - - if confirm "Set ${CYAN}${VERSION}${RESET}${WHITE} as the default install version?"; then - rclone copyto "$INSTALL_SCRIPT" "${rclone_remote}:zml-smi/install.sh" - success "zml-smi/install.sh" - DEFAULT_INSTALL=true - else - warn "Skipped default install script" - DEFAULT_INSTALL=false - fi -} - -verify() { - local install_url="${BASE_URL}/zml-smi/${VERSION}" - - step "Verifying uploads" - - for f in "${FILES[@]}"; do - if [ ! -f "$f" ]; then - continue - fi - local name remote_name - name="$(basename "$f")" - remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" - local local_sha remote_sha - local_sha="$(shasum -a 256 "$f" | awk '{print $1}')" - remote_sha="$(curl -fsSL "${install_url}/${remote_name}" | shasum -a 256 | awk '{print $1}')" - if [ "$local_sha" = "$remote_sha" ]; then - success "${remote_name}" - else - fail "${remote_name} — sha256 mismatch (local: ${local_sha}, remote: ${remote_sha})" - fi - done - - local local_sha remote_sha - local_sha="$(shasum -a 256 "$INSTALL_SCRIPT" | awk '{print $1}')" - remote_sha="$(curl -fsSL "${install_url}/install.sh" | shasum -a 256 | awk '{print $1}')" - if [ "$local_sha" = "$remote_sha" ]; then - success "install.sh" - else - fail "install.sh — sha256 mismatch (local: ${local_sha}, remote: ${remote_sha})" - fi -} - -release_notes() { - local install_url="${BASE_URL}/zml-smi/${VERSION}" - local log="$CACHED_CHANGELOG" - - local downloads="" - for f in "${FILES[@]}"; do - [ -f "$f" ] || continue - local name remote_name - name="$(basename "$f")" - [[ "$name" == *.sha256 ]] && continue - local sha_file="" - for sf in "${FILES[@]}"; do - if [ "$(basename "$sf")" = "${name}.sha256" ]; then - sha_file="$sf" - break - fi - done - if [ -z "$sha_file" ] || [ ! -f "$sha_file" ]; then - fail "Missing sha256 sidecar for ${name}" - fi - local sha - sha="$(awk '{print $1}' "$sha_file")" - local actual - actual="$(shasum -a 256 "$f" | awk '{print $1}')" - if [ "$sha" != "$actual" ]; then - fail "${name} sha256 mismatch — sidecar: ${sha}, actual: ${actual}" - fi - remote_name="${name/zml-smi-/zml-smi-${VERSION}-}" - downloads="${downloads}| [${remote_name}](${install_url}/${remote_name}) | \`${sha}\` |"$'\n' - done - - cat </dev/null && [ -w "$INSTALL_DIR" ] && - mkdir -p "$BIN_DIR" 2>/dev/null && [ -w "$BIN_DIR" ]; then - return - fi - - warn "Cannot write to /opt or /usr/local/bin — falling back to ~/.local" - INSTALL_DIR="${HOME}/.local/zml-smi" - BIN_DIR="${HOME}/.local/bin" -} - -if [ -t 1 ]; then - DIM='\033[2m' - RESET='\033[0m' - BLUE='\033[1;38;2;140;180;255m' - GREEN='\033[1;32m' - RED='\033[1;31m' - YELLOW='\033[1;33m' - CYAN='\033[1;36m' - WHITE='\033[1;37m' -else - DIM='' RESET='' BLUE='' GREEN='' RED='' YELLOW='' CYAN='' WHITE='' -fi - -info() { printf "${BLUE} │${RESET} %b\n" "$*"; } -success() { printf "${GREEN} ✓${RESET} %b\n" "$*"; } -warn() { printf "${YELLOW} !${RESET} %b\n" "$*" >&2; } -fail() { printf "${RED} ✗ %b${RESET}\n" "$*" >&2; exit 1; } -step() { printf "\n${WHITE} ▸ %b${RESET}\n" "$*"; } - -print_logo() { - printf "\n" - printf "${BLUE} █████╗███╗ ███╗██╗ ${RESET}\n" - printf "${BLUE} ╚═██╔╝████╗ ████║██║ ${RESET}\n" - printf "${BLUE} ██╔╝ ██╔████╔██║██║ ${RESET}\n" - printf "${BLUE} █████╗██║ ╚═╝ ██║████╗${RESET}\n" - printf "${BLUE} ╚════╝╚═╝ ╚═╝╚═══╝${RESET}\n" -} - -detect_platform() { - step "Detecting platform" - - OS="$(uname -s)" - ARCH="$(uname -m)" - - case "$OS" in - Linux) OS_LABEL="linux" ;; - Darwin) OS_LABEL="macos" ;; - MINGW*|MSYS*|CYGWIN*) - fail "Windows is not supported. Use WSL2 instead." ;; - *) - fail "Unsupported operating system: ${OS}" ;; - esac - - case "$ARCH" in - x86_64|amd64) ARCH_LABEL="amd64" ;; - aarch64|arm64) ARCH_LABEL="arm64" ;; - *) - fail "Unsupported architecture: ${ARCH}" ;; - esac - - if [ "$OS_LABEL" = "macos" ] && [ "$ARCH_LABEL" = "amd64" ]; then - fail "macOS x86_64 is not supported. Only Apple Silicon (arm64) is supported." - fi - - success "${OS_LABEL} ${ARCH_LABEL}" - - DOWNLOAD_URL="${BASE_URL}/${BINARY_NAME}-${VERSION}-${OS_LABEL}-${ARCH_LABEL}.tar.zst" -} - -check_deps() { - step "Checking dependencies" - local missing=() - - if command -v curl &>/dev/null; then - DOWNLOADER="curl" - success "curl" - elif command -v wget &>/dev/null; then - DOWNLOADER="wget" - success "wget" - else - missing+=("curl or wget") - fi - - if command -v zstd &>/dev/null; then - success "zstd" - else - missing+=("zstd") - fi - - if command -v tar &>/dev/null; then - success "tar" - else - missing+=("tar") - fi - - if [ ${#missing[@]} -gt 0 ]; then - fail "Missing required tools: ${missing[*]}" - fi -} - -download_and_install() { - step "Downloading & extracting" - - rm -rf "$INSTALL_DIR" - mkdir -p "$INSTALL_DIR" - - if [ "$DOWNLOADER" = "curl" ]; then - if ! curl -fsSL "$DOWNLOAD_URL" | tar --zstd -xf - --strip-components=1 -C "$INSTALL_DIR"; then - fail "Download failed — is this OS/arch supported?" - fi - else - if ! wget -q -O- "$DOWNLOAD_URL" | tar --zstd -xf - --strip-components=1 -C "$INSTALL_DIR"; then - fail "Download failed — is this OS/arch supported?" - fi - fi - - chmod 755 "${INSTALL_DIR}/${BINARY_NAME}" - success "Installed to ${INSTALL_DIR}/" - - mkdir -p "$BIN_DIR" - ln -sf "${INSTALL_DIR}/${BINARY_NAME}" "${BIN_DIR}/${BINARY_NAME}" - success "Symlink at ${BIN_DIR}/${BINARY_NAME}" -} - -check_path() { - if ! echo "$PATH" | tr ':' '\n' | grep -qx "$BIN_DIR"; then - printf "\n" - warn "${BIN_DIR} is not in your PATH" - step "Add to your shell profile:" - info "${CYAN}export PATH=\"${BIN_DIR}:\$PATH\"" - printf "\n" - fi -} - -main() { - print_logo - resolve_dirs - detect_platform - check_deps - - info "Install to: ${CYAN}${INSTALL_DIR}${RESET}" - info "Bin dir: ${CYAN}${BIN_DIR}${RESET}" - - download_and_install - check_path - - printf "\n" - printf " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n" - printf " ${GREEN} Installation complete!${RESET}\n" - printf " ${DIM} Run '${BINARY_NAME}' to get started.${RESET}\n" - printf " ${GREEN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}\n" - printf "\n" -} - -main diff --git a/bin/zml-smi/tui/BUILD.bazel b/bin/zml-smi/tui/BUILD.bazel deleted file mode 100644 index 29f4824c..00000000 --- a/bin/zml-smi/tui/BUILD.bazel +++ /dev/null @@ -1,59 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "tui", - import_name = "zml-smi/tui", - main = "tui.zig", - srcs = [ - "data.zig", - "image_cache.zig", - "lib/compose.zig", - "lib/image.zig", - "lib/ui.zig", - "lib/utils.zig", - "pages/detail.zig", - "pages/detail/detail_common.zig", - "pages/detail/gpu_detail.zig", - "pages/detail/neuron_detail.zig", - "pages/detail/tpu_detail.zig", - "pages/overview.zig", - "print.zig", - "theme.zig", - "top.zig", - "widgets/braille_chart.zig", - "widgets/chart.zig", - "widgets/column_layout.zig", - "widgets/device_card.zig", - "widgets/gauge.zig", - "widgets/info_lines.zig", - "widgets/logo.zig", - "widgets/metric_card.zig", - "widgets/process_table.zig", - "widgets/status_line.zig", - "widgets/titled_border.zig", - ], - extra_srcs = [ - "assets/amd.png", - "assets/intel.png", - "assets/logo.png", - "assets/neuron.png", - "assets/nvidia.png", - "assets/tpu.png", - ], - visibility = ["//visibility:public"], - deps = [ - "//bazel/stamp:zig", - "//bin/zml-smi/info", - "//bin/zml-smi/utils:double_buffer", - "//zml:logo", - "@libvaxis//:vaxis", - ] + select({ - "@platforms//os:macos": ["//bin/zml-smi/platforms/macos"], - "//conditions:default": ["//bin/zml-smi/platforms/linux"], - }), -) - -zig_test( - name = "test", - deps = [":tui"], -) diff --git a/bin/zml-smi/tui/assets/amd.png b/bin/zml-smi/tui/assets/amd.png deleted file mode 100644 index 1de65ece..00000000 Binary files a/bin/zml-smi/tui/assets/amd.png and /dev/null differ diff --git a/bin/zml-smi/tui/assets/intel.png b/bin/zml-smi/tui/assets/intel.png deleted file mode 100644 index 1f71fcd7..00000000 Binary files a/bin/zml-smi/tui/assets/intel.png and /dev/null differ diff --git a/bin/zml-smi/tui/assets/logo.png b/bin/zml-smi/tui/assets/logo.png deleted file mode 100644 index 77ceba76..00000000 Binary files a/bin/zml-smi/tui/assets/logo.png and /dev/null differ diff --git a/bin/zml-smi/tui/assets/neuron.png b/bin/zml-smi/tui/assets/neuron.png deleted file mode 100644 index ea57a753..00000000 Binary files a/bin/zml-smi/tui/assets/neuron.png and /dev/null differ diff --git a/bin/zml-smi/tui/assets/nvidia.png b/bin/zml-smi/tui/assets/nvidia.png deleted file mode 100644 index 6c2a474c..00000000 Binary files a/bin/zml-smi/tui/assets/nvidia.png and /dev/null differ diff --git a/bin/zml-smi/tui/assets/tpu.png b/bin/zml-smi/tui/assets/tpu.png deleted file mode 100644 index 97a6ae3c..00000000 Binary files a/bin/zml-smi/tui/assets/tpu.png and /dev/null differ diff --git a/bin/zml-smi/tui/data.zig b/bin/zml-smi/tui/data.zig deleted file mode 100644 index bc6171b7..00000000 --- a/bin/zml-smi/tui/data.zig +++ /dev/null @@ -1,155 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const smi_info = @import("zml-smi/info"); -const di = smi_info.device_info; -pub const DeviceInfo = di.DeviceInfo; -pub const GpuInfo = di.GpuInfo; -pub const NeuronInfo = di.NeuronInfo; -pub const TpuInfo = di.TpuInfo; -pub const Target = di.Target; -pub const Targets = smi_info.Targets; -const hi = smi_info.host_info; -pub const HostInfo = hi.HostInfo; -pub const HostData = hi.HostData; -pub const pi = smi_info.process_info; -pub const ProcessDoubleBuffer = @import("zml-smi/double_buffer").DoubleBuffer(std.ArrayList(pi.ProcessInfo)); -pub const ProcessEnricher = if (builtin.os.tag == .macos) - @import("zml-smi/platforms/macos").process.ProcessEnricher -else - @import("zml-smi/platforms/linux").process.ProcessEnricher; - -pub const history_len: usize = 500; - -pub fn RingBuffer(comptime T: type, comptime capacity: usize) type { - return struct { - const Self = @This(); - - buf: [capacity]T = @splat(0), - head: usize = 0, - len: usize = 0, - - pub fn push(self: *Self, value: T) void { - self.buf[self.head] = value; - self.head = (self.head + 1) % capacity; - if (self.len < capacity) { - self.len += 1; - } - } - - pub fn sliceLast(self: *const Self, arena: std.mem.Allocator, n: usize) std.mem.Allocator.Error![]const T { - const count = @min(n, self.len); - if (count == 0) { - return &.{}; - } - const out = try arena.alloc(T, count); - const start = (self.head + capacity - count) % capacity; - for (0..count) |i| { - out[i] = self.buf[(start + i) % capacity]; - } - return out; - } - }; -} - -pub const HistoryBuffers = struct { - util: []RingBuffer(u64, history_len) = &.{}, - mem_util: []RingBuffer(u64, history_len) = &.{}, - temp: []RingBuffer(u64, history_len) = &.{}, - power: []RingBuffer(u64, history_len) = &.{}, - - pub fn init(allocator: std.mem.Allocator, count: usize) !HistoryBuffers { - const h: HistoryBuffers = .{ - .util = try allocator.alloc(RingBuffer(u64, history_len), count), - .mem_util = try allocator.alloc(RingBuffer(u64, history_len), count), - .temp = try allocator.alloc(RingBuffer(u64, history_len), count), - .power = try allocator.alloc(RingBuffer(u64, history_len), count), - }; - - @memset(h.util, .{}); - @memset(h.mem_util, .{}); - @memset(h.temp, .{}); - @memset(h.power, .{}); - - return h; - } - - pub fn deinit(self: *HistoryBuffers, allocator: std.mem.Allocator) void { - allocator.free(self.util); - allocator.free(self.mem_util); - allocator.free(self.temp); - allocator.free(self.power); - } -}; - -pub const SystemState = struct { - devices: []*DeviceInfo, - host: *HostInfo, - history: HistoryBuffers = .{}, - targets: Targets, - process_lists: []*ProcessDoubleBuffer, - enricher: *ProcessEnricher, - gpa: std.mem.Allocator, - arena: std.mem.Allocator, - io: std.Io, - tui_refresh_rate: u16, - - pub const Config = struct { - devices: []*DeviceInfo, - host: *HostInfo, - targets: Targets, - tui_refresh_rate: u16, - process_lists: []*ProcessDoubleBuffer, - enricher: *ProcessEnricher, - gpa: std.mem.Allocator, - arena: std.mem.Allocator, - io: std.Io, - }; - - pub fn init(cfg: Config) !SystemState { - return .{ - .devices = cfg.devices, - .host = cfg.host, - .history = try HistoryBuffers.init(cfg.arena, cfg.devices.len), - .targets = cfg.targets, - .tui_refresh_rate = cfg.tui_refresh_rate, - .process_lists = cfg.process_lists, - .enricher = cfg.enricher, - .gpa = cfg.gpa, - .arena = cfg.arena, - .io = cfg.io, - }; - } - - pub fn deinit(self: *SystemState, allocator: std.mem.Allocator) void { - self.history.deinit(allocator); - } - - pub fn deviceCount(self: *const SystemState) usize { - return self.devices.len; - } - - pub fn recordHistory(self: *SystemState) void { - for (0..self.deviceCount()) |i| { - switch (self.devices[i].*) { - .cuda, .rocm, .oneapi => |*sv| { - const gpu = sv.front().*; - self.history.util[i].push(gpu.util_percent orelse 0); - const used = gpu.mem_used_bytes orelse 0; - const total = gpu.mem_total_bytes orelse 0; - self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); - self.history.temp[i].push(gpu.temperature orelse 0); - self.history.power[i].push(gpu.power_mw orelse 0); - }, - inline else => |*sv| { - const info = sv.front().*; - self.history.util[i].push(info.util_percent orelse 0); - const used = info.mem_used_bytes orelse 0; - const total = info.mem_total_bytes orelse 0; - self.history.mem_util[i].push(if (total > 0) used * 100 / total else 0); - self.history.temp[i].push(0); - self.history.power[i].push(0); - }, - } - } - } -}; diff --git a/bin/zml-smi/tui/image_cache.zig b/bin/zml-smi/tui/image_cache.zig deleted file mode 100644 index 1aea4778..00000000 --- a/bin/zml-smi/tui/image_cache.zig +++ /dev/null @@ -1,37 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); - -const ImageCache = @This(); - -pub var global: ImageCache = .{}; - -map: std.StringHashMapUnmanaged(vaxis.Image) = .empty, - -pub fn deinit(self: *ImageCache, allocator: std.mem.Allocator) void { - self.map.deinit(allocator); -} - -pub fn load( - self: *ImageCache, - vx: *vaxis.Vaxis, - allocator: std.mem.Allocator, - writer: *std.Io.Writer, - key: []const u8, - data: []const u8, -) void { - const image = vx.loadImage(allocator, writer, .{ .mem = data }) catch return; - self.map.put(allocator, key, image) catch return; -} - -pub fn loadAll(self: *ImageCache, vx: *vaxis.Vaxis, allocator: std.mem.Allocator, writer: *std.Io.Writer) void { - self.load(vx, allocator, writer, "logo", @embedFile("assets/logo.png")); - self.load(vx, allocator, writer, "gpu_cuda", @embedFile("assets/nvidia.png")); - self.load(vx, allocator, writer, "gpu_rocm", @embedFile("assets/amd.png")); - self.load(vx, allocator, writer, "gpu_oneapi", @embedFile("assets/intel.png")); - self.load(vx, allocator, writer, "gpu_neuron", @embedFile("assets/neuron.png")); - self.load(vx, allocator, writer, "gpu_tpu", @embedFile("assets/tpu.png")); -} - -pub fn get(self: *const ImageCache, key: []const u8) ?vaxis.Image { - return self.map.get(key); -} diff --git a/bin/zml-smi/tui/lib/compose.zig b/bin/zml-smi/tui/lib/compose.zig deleted file mode 100644 index e3438762..00000000 --- a/bin/zml-smi/tui/lib/compose.zig +++ /dev/null @@ -1,46 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; - -// ── SurfaceBuilder ───────────────────────────────────────────────── - -pub const SurfaceBuilder = struct { - arena: std.mem.Allocator, - children: std.ArrayList(vxfw.SubSurface), - - pub fn add(self: *SurfaceBuilder, row: i17, col: i17, surface: vxfw.Surface) !void { - try self.addZ(row, col, surface, 0); - } - - pub fn addZ(self: *SurfaceBuilder, row: i17, col: i17, surface: vxfw.Surface, z_index: u8) !void { - try self.children.append(self.arena, .{ .origin = .{ .row = row, .col = col }, .surface = surface, .z_index = z_index }); - } - - pub fn finish(self: *SurfaceBuilder, size: vxfw.Size, wgt: vxfw.Widget) vxfw.Surface { - return .{ .size = size, .widget = wgt, .buffer = &.{}, .children = self.children.items }; - } -}; - -pub fn surfaceBuilder(arena: std.mem.Allocator) SurfaceBuilder { - return .{ .arena = arena, .children = .empty }; -} - -// ── Fluent widget wrappers ───────────────────────────────────────── - -pub fn center(arena: std.mem.Allocator, child: vxfw.Widget) std.mem.Allocator.Error!vxfw.Widget { - const c = try arena.create(vxfw.Center); - c.* = .{ .child = child }; - return c.widget(); -} - -pub fn sized(arena: std.mem.Allocator, child: vxfw.Widget, size: vxfw.Size) std.mem.Allocator.Error!vxfw.Widget { - const s = try arena.create(vxfw.SizedBox); - s.* = .{ .child = child, .size = size }; - return s.widget(); -} - -pub fn pad(arena: std.mem.Allocator, child: vxfw.Widget, padding: vxfw.Padding.PadValues) std.mem.Allocator.Error!vxfw.Widget { - const p = try arena.create(vxfw.Padding); - p.* = .{ .child = child, .padding = padding }; - return p.widget(); -} diff --git a/bin/zml-smi/tui/lib/image.zig b/bin/zml-smi/tui/lib/image.zig deleted file mode 100644 index a365ac4d..00000000 --- a/bin/zml-smi/tui/lib/image.zig +++ /dev/null @@ -1,84 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; - -const ui = @import("ui.zig"); -const Allocator = std.mem.Allocator; - -const Image = @This(); - -/// The loaded image to display. -image: vaxis.Image, -/// Desired height in cell rows. Width is computed to maintain aspect ratio. -/// If null, uses max constraint height (or 1 if unbounded). -rows: ?u16 = null, - -pub fn draw(self: *const Image, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surface { - const target_rows = self.rows orelse ctx.max.height orelse 1; - if (target_rows == 0 or self.image.width == 0 or self.image.height == 0) { - return .{ - .size = .{ .width = 0, .height = 0 }, - .widget = ui.widget(self), - .buffer = &.{}, - .children = &.{}, - }; - } - - // Compute cell columns from image aspect ratio and terminal cell pixel size - const cell_h: u32 = if (ctx.cell_size.height > 0) ctx.cell_size.height else 20; - const cell_w: u32 = if (ctx.cell_size.width > 0) ctx.cell_size.width else 10; - const height_px = @as(u32, target_rows) * cell_h; - const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(self.image.height)); - const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(self.image.width)) * scale_f); - const cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); - - const size: vxfw.Size = .{ .width = cols, .height = target_rows }; - var surf = try vxfw.Surface.init(ctx.arena, ui.widget(self), size); - surf.writeCell(0, 0, .{ - .image = .{ - .img_id = self.image.id, - .options = .{ - // Emit both rows and cols so that target bounds are unambiguous. - // Rows alone lets some terminals fall back to native pixel size. - .size = .{ .rows = target_rows, .cols = cols }, - }, - }, - }); - return surf; -} - -test Image { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - vxfw.DrawContext.init(.unicode); - - const img: Image = .{ - .image = .{ .id = 1, .width = 100, .height = 50 }, - .rows = 5, - }; - const w = ui.widget(&img); - - const draw_ctx: vxfw.DrawContext = .{ - .arena = arena.allocator(), - .min = .{}, - .max = .{ .width = 40, .height = 10 }, - .cell_size = .{ .width = 10, .height = 20 }, - }; - - const surface = try w.draw(draw_ctx); - try std.testing.expectEqual(@as(u16, 5), surface.size.height); - // 5 rows * 20px = 100px height. scale = 100/50 = 2.0. width = 100*2 = 200px. cols = 200/10 = 20 - try std.testing.expectEqual(@as(u16, 20), surface.size.width); - // Image cell should be at (0, 0) - const cell = surface.buffer[0]; - try std.testing.expect(cell.image != null); - try std.testing.expectEqual(@as(u32, 1), cell.image.?.img_id); - // Both rows and cols must be emitted so terminals don't fall back to native pixel size. - const size = cell.image.?.options.size.?; - try std.testing.expectEqual(@as(?u16, 5), size.rows); - try std.testing.expectEqual(@as(?u16, 20), size.cols); -} - -test "refAllDecls" { - std.testing.refAllDecls(@This()); -} diff --git a/bin/zml-smi/tui/lib/ui.zig b/bin/zml-smi/tui/lib/ui.zig deleted file mode 100644 index 6008d1e0..00000000 --- a/bin/zml-smi/tui/lib/ui.zig +++ /dev/null @@ -1,87 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; - -/// Creates a vxfw.Widget from any pointer whose type has a `draw` method -/// (and optionally a `handleEvent` method). -pub fn widget(ptr: anytype) vxfw.Widget { - const Ptr = @TypeOf(ptr); - const T = @typeInfo(Ptr).pointer.child; - return .{ - .userdata = @ptrCast(@constCast(ptr)), - .drawFn = struct { - fn f(p: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const self: *T = @ptrCast(@alignCast(p)); - return self.draw(ctx); - } - }.f, - .eventHandler = if (@hasDecl(T, "handleEvent")) struct { - fn f(p: *anyopaque, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { - const self: *T = @ptrCast(@alignCast(p)); - return self.handleEvent(ctx, event); - } - }.f else null, - }; -} - -/// Creates a vxfw.Widget that calls a specific draw function (for non-standard widget patterns -/// like MetricCard.drawContent or Chart.drawContent). -pub fn drawWidget(ptr: anytype, comptime drawFn: anytype) vxfw.Widget { - const Ptr = @TypeOf(ptr); - const T = @typeInfo(Ptr).pointer.child; - return .{ - .userdata = @ptrCast(@constCast(ptr)), - .drawFn = struct { - fn f(p: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const self: *T = @ptrCast(@alignCast(p)); - return @call(.auto, drawFn, .{ self, ctx }); - } - }.f, - }; -} - -// ── DrawContext constraint helpers ────────────────────────────────── - -/// min = max = {w, h} — exact size. -pub fn fixedSize(ctx: vxfw.DrawContext, w: u16, h: u16) vxfw.DrawContext { - return ctx.withConstraints(.{ .width = w, .height = h }, .{ .width = w, .height = h }); -} - -/// Width fixed (min = max = w), height unbounded. -pub fn fixedWidth(ctx: vxfw.DrawContext, w: u16) vxfw.DrawContext { - return ctx.withConstraints(.{ .width = w }, .{ .width = w, .height = null }); -} - -/// max = {w, h}, min = default — bounded but unconstrained from below. -pub fn maxSize(ctx: vxfw.DrawContext, w: u16, h: u16) vxfw.DrawContext { - return ctx.withConstraints(.{}, .{ .width = w, .height = h }); -} - -// ── Drawing helpers ──────────────────────────────────────────────── - -/// Draw a single-line RichText with softwrap=false, overflow=clip, max width. -pub fn drawRichLine(ctx: vxfw.DrawContext, segments: []const vaxis.Cell.Segment, max_w: u16) std.mem.Allocator.Error!vxfw.Surface { - const rich: vxfw.RichText = .{ - .text = segments, - .softwrap = false, - .overflow = .clip, - }; - return rich.draw(ctx.withConstraints(.{}, .{ .width = max_w, .height = 1 })); -} - -pub const ImageCellSize = struct { - cols: u16, - rows: u16, -}; - -/// Compute the cell dimensions for an image scaled to a target row count, -/// maintaining aspect ratio based on the terminal's cell pixel size. -pub fn imageCellSize(img: vaxis.Image, target_rows: u16, cell_size: vxfw.Size) ImageCellSize { - const cell_h: u32 = if (cell_size.height > 0) cell_size.height else 20; - const cell_w: u32 = if (cell_size.width > 0) cell_size.width else 10; - const height_px = @as(u32, target_rows) * cell_h; - const scale_f = @as(f64, @floatFromInt(height_px)) / @as(f64, @floatFromInt(img.height)); - const width_px: u32 = @intFromFloat(@as(f64, @floatFromInt(img.width)) * scale_f); - const img_cols: u16 = @intCast((width_px + cell_w - 1) / cell_w); - return .{ .cols = img_cols, .rows = target_rows }; -} diff --git a/bin/zml-smi/tui/lib/utils.zig b/bin/zml-smi/tui/lib/utils.zig deleted file mode 100644 index 993703c9..00000000 --- a/bin/zml-smi/tui/lib/utils.zig +++ /dev/null @@ -1,30 +0,0 @@ -const std = @import("std"); - -pub fn bytesToMb(val: ?u64) u64 { - return if (val) |v| v / (1024 * 1024) else 0; -} - -pub fn repeatStr(arena: std.mem.Allocator, char: []const u8, count: u16) std.mem.Allocator.Error![]const u8 { - const buf = try arena.alloc(u8, char.len * count); - for (0..count) |i| { - @memcpy(buf[i * char.len ..][0..char.len], char); - } - return buf; -} - -/// Maps values from [min_val, max_val] to [0, 100], clamped. -pub fn normalizeRange(arena: std.mem.Allocator, raw: []const u64, min_val: u64, max_val: u64) std.mem.Allocator.Error![]u8 { - const result = try arena.alloc(u8, raw.len); - const range = max_val -| min_val; - for (raw, 0..) |v, i| { - result[i] = if (v <= min_val) - 0 - else if (v >= max_val) - 100 - else if (range > 0) - @intCast((v - min_val) * 100 / range) - else - 0; - } - return result; -} diff --git a/bin/zml-smi/tui/pages/detail.zig b/bin/zml-smi/tui/pages/detail.zig deleted file mode 100644 index 86d4b7a3..00000000 --- a/bin/zml-smi/tui/pages/detail.zig +++ /dev/null @@ -1,28 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const data = @import("../data.zig"); -const ui = @import("../lib/ui.zig"); -const ProcessTable = @import("../widgets/process_table.zig"); -const gpu_detail = @import("detail/gpu_detail.zig"); -const neuron_detail = @import("detail/neuron_detail.zig"); -const tpu_detail = @import("detail/tpu_detail.zig"); - -const Detail = @This(); - -state: *const data.SystemState, -device_id: u16, -process_table: *ProcessTable = undefined, - -pub fn draw(self: *const Detail, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const w = ctx.max.width orelse 80; - const id = self.device_id; - const dev = self.state.devices[id]; - const wgt = ui.widget(self); - - return switch (dev.*) { - .cuda, .rocm, .oneapi => |*sv| try gpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), - .neuron => |*sv| try neuron_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), - .tpu => |*sv| try tpu_detail.draw(self.state, self.process_table, ctx, w, id, sv.front().*, wgt), - }; -} diff --git a/bin/zml-smi/tui/pages/detail/detail_common.zig b/bin/zml-smi/tui/pages/detail/detail_common.zig deleted file mode 100644 index 93e8addb..00000000 --- a/bin/zml-smi/tui/pages/detail/detail_common.zig +++ /dev/null @@ -1,55 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const data = @import("../../data.zig"); -const theme = @import("../../theme.zig"); -const utils = @import("../../lib/utils.zig"); -const ui = @import("../../lib/ui.zig"); -const compose = @import("../../lib/compose.zig"); -const Chart = @import("../../widgets/chart.zig"); -const ColumnLayout = @import("../../widgets/column_layout.zig"); - -pub fn headerText(arena: std.mem.Allocator, id: u16, name: []const u8) std.mem.Allocator.Error!vxfw.Text { - return .{ - .text = try std.fmt.allocPrint(arena, "Device {d}: {s}", .{ id, name }), - .style = theme.header_style, - .softwrap = false, - .overflow = .clip, - }; -} - -/// Creates a Chart pre-filled with history data and standard defaults. -pub fn historyChart( - ctx: vxfw.DrawContext, - state: *const data.SystemState, - id: u16, - history: anytype, - opts: struct { - title: []const u8, - value_label: []const u8, - info_line: ?[]const u8 = null, - y_min: u32 = 0, - y_max: u32 = 100, - y_unit: []const u8 = "%", - }, -) std.mem.Allocator.Error!Chart { - return .{ - .title = opts.title, - .data = try utils.normalizeRange(ctx.arena, try history[id].sliceLast(ctx.arena, data.history_len), opts.y_min, opts.y_max), - .value_label = opts.value_label, - .info_line = opts.info_line, - .y_min = opts.y_min, - .y_max = opts.y_max, - .y_unit = opts.y_unit, - .chart_height = 8, - .tui_refresh_rate = state.tui_refresh_rate, - }; -} - -/// Draws a page-level layout with standard margin (top=1, left/right=2) inside a full-width surface. -pub fn pageFrame(ctx: vxfw.DrawContext, layout: ColumnLayout, w: u16, parent_widget: vxfw.Widget) std.mem.Allocator.Error!vxfw.Surface { - const padded = try compose.pad(ctx.arena, ui.widget(&layout), .{ .top = 1, .left = 2, .right = 2 }); - var surf = try padded.draw(ui.fixedWidth(ctx, w)); - surf.widget = parent_widget; - return surf; -} diff --git a/bin/zml-smi/tui/pages/detail/gpu_detail.zig b/bin/zml-smi/tui/pages/detail/gpu_detail.zig deleted file mode 100644 index 7884d6f8..00000000 --- a/bin/zml-smi/tui/pages/detail/gpu_detail.zig +++ /dev/null @@ -1,160 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const data = @import("../../data.zig"); -const utils = @import("../../lib/utils.zig"); -const ui = @import("../../lib/ui.zig"); -const Chart = @import("../../widgets/chart.zig"); -const ColumnLayout = @import("../../widgets/column_layout.zig"); -const MetricCard = @import("../../widgets/metric_card.zig"); -const ProcessTable = @import("../../widgets/process_table.zig"); -const common = @import("detail_common.zig"); - -pub fn draw( - state: *const data.SystemState, - process_table: *ProcessTable, - ctx: vxfw.DrawContext, - w: u16, - id: u16, - gpu: data.GpuInfo, - parent_widget: vxfw.Widget, -) std.mem.Allocator.Error!vxfw.Surface { - // ── Header ────────────────────────────────────────────── - const name_suffix: []const u8 = blk: { - const drv = gpu.driver_version orelse ""; - const cuda = gpu.cuda_driver_version orelse ""; - break :blk if (drv.len > 0 and cuda.len > 0) - try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) - else if (drv.len > 0) - try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) - else if (cuda.len > 0) - try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) - else - @as([]const u8, ""); - }; - const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ gpu.name orelse "Unknown", name_suffix })); - - // ── GPU Utilization + Memory charts ───────────────────── - const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ - .title = "GPU Utilization", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{gpu.util_percent orelse 0}), - .info_line = if (gpu.encoder_util_percent != null or gpu.decoder_util_percent != null) - try std.fmt.allocPrint(ctx.arena, "Enc {d}% Dec {d}%", .{ - gpu.encoder_util_percent orelse 0, gpu.decoder_util_percent orelse 0, - }) - else - null, - }); - const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ - .title = "Memory", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (gpu.mem_total_bytes) |total| if (total > 0) (gpu.mem_used_bytes orelse 0) * 100 / total else 0 else 0}), - .info_line = if (gpu.mem_bus_width) |bw| - try std.fmt.allocPrint(ctx.arena, "VRAM {d} / {d} MB ({d}-bit)", .{ - utils.bytesToMb(gpu.mem_used_bytes), utils.bytesToMb(gpu.mem_total_bytes), bw, - }) - else - try std.fmt.allocPrint(ctx.arena, "VRAM {d} / {d} MB", .{ - utils.bytesToMb(gpu.mem_used_bytes), utils.bytesToMb(gpu.mem_total_bytes), - }), - }); - - // ── Temperature + Power charts ────────────────────────── - const temp_chart = try common.historyChart(ctx, state, id, state.history.temp, .{ - .title = "Temperature", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C", .{gpu.temperature orelse 0}), - .info_line = try std.fmt.allocPrint(ctx.arena, "Fan {d}%", .{gpu.fan_speed_percent orelse 0}), - .y_min = 20, - .y_max = 100, - .y_unit = "\u{00b0}C", - }); - - const power_limit_mw = gpu.power_limit_mw orelse 0; - const power_limit_w = power_limit_mw / 1000; - const power_raw = try state.history.power[id].sliceLast(ctx.arena, data.history_len); - const power_watts = try ctx.arena.alloc(u64, power_raw.len); - for (power_raw, 0..) |p, j| { - power_watts[j] = p / 1000; - } - const y_max = if (power_limit_w > 0) power_limit_w else 1; - const power_chart: Chart = .{ - .title = "Power", - .data = try utils.normalizeRange(ctx.arena, power_watts, 0, y_max), - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}W", .{(gpu.power_mw orelse 0) / 1000}), - .info_line = if (power_limit_w > 0) - try std.fmt.allocPrint(ctx.arena, "Limit {d}W", .{power_limit_w}) - else - "N/A", - .y_min = 0, - .y_max = @intCast(@min(y_max, std.math.maxInt(u32))), - .y_unit = "W", - .chart_height = 8, - .tui_refresh_rate = state.tui_refresh_rate, - }; - - const charts_flow: ColumnLayout = .{ - .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart), ui.widget(&temp_chart), ui.widget(&power_chart) }, - .min_child_width = 60, - .gap = 1, - }; - - // ── Clocks + PCIe ─────────────────────────────────────── - var clock_lines: std.ArrayList(MetricCard.LineEntry) = .empty; - try clock_lines.append(ctx.arena, .{ .label = "Graphics ", .value = try formatClockPair(ctx.arena, gpu.clock_graphics_mhz, gpu.clock_graphics_max_mhz) }); - if (gpu.clock_sm_mhz) |sm| { - try clock_lines.append(ctx.arena, .{ .label = "SM ", .value = try std.fmt.allocPrint(ctx.arena, " {d} MHz", .{sm}) }); - } - if (gpu.clock_soc_mhz) |soc| { - try clock_lines.append(ctx.arena, .{ .label = "SOC ", .value = try std.fmt.allocPrint(ctx.arena, " {d} MHz", .{soc}) }); - } - try clock_lines.append(ctx.arena, .{ .label = "Memory ", .value = try formatClockPair(ctx.arena, gpu.clock_mem_mhz, gpu.clock_mem_max_mhz) }); - const clocks_card: MetricCard = .{ .title = "Clocks", .lines = clock_lines.items }; - - var pcie_lines: std.ArrayList(MetricCard.LineEntry) = .empty; - try pcie_lines.append(ctx.arena, .{ .label = "Link ", .value = try formatPcieLink(ctx.arena, gpu.pcie_link_gen, gpu.pcie_link_width) }); - if (gpu.pcie_tx_kbps != null or gpu.pcie_rx_kbps != null) { - try pcie_lines.append(ctx.arena, .{ .label = "TX ", .value = try formatBandwidth(ctx.arena, gpu.pcie_tx_kbps orelse 0) }); - try pcie_lines.append(ctx.arena, .{ .label = "RX ", .value = try formatBandwidth(ctx.arena, gpu.pcie_rx_kbps orelse 0) }); - } - if (gpu.pcie_bandwidth_mbps) |bw| { - try pcie_lines.append(ctx.arena, .{ .label = "BW ", .value = try std.fmt.allocPrint(ctx.arena, " {d} Mb/s", .{bw}) }); - } - const pcie_card: MetricCard = .{ .title = "PCIe", .lines = pcie_lines.items }; - - const cards_flow: ColumnLayout = .{ - .children = &.{ ui.widget(&clocks_card), ui.widget(&pcie_card) }, - .min_child_width = 30, - }; - - // ── Compose vertical layout ───────────────────────────── - return common.pageFrame(ctx, .{ - .children = &.{ - header.widget(), - ui.widget(&charts_flow), - ui.widget(&cards_flow), - ui.widget(process_table), - }, - .gap = 1, - }, w, parent_widget); -} - -fn formatClockPair(arena: std.mem.Allocator, current: ?u64, max: ?u64) std.mem.Allocator.Error![]const u8 { - if (current == null and max == null) return " N/A"; - return std.fmt.allocPrint(arena, " {d} / {d} MHz", .{ current orelse 0, max orelse 0 }); -} - -fn formatPcieLink(arena: std.mem.Allocator, gen: ?u64, width: ?u64) std.mem.Allocator.Error![]const u8 { - if (gen == null and width == null) return " N/A"; - return std.fmt.allocPrint(arena, " Gen{d} x{d}", .{ gen orelse 0, width orelse 0 }); -} - -fn formatBandwidth(arena: std.mem.Allocator, kbps: u64) std.mem.Allocator.Error![]const u8 { - if (kbps >= 1_000_000) { - const gbps = @as(f64, @floatFromInt(kbps)) / 1_000_000.0; - return std.fmt.allocPrint(arena, " {d:.1} GB/s", .{gbps}); - } else if (kbps >= 1_000) { - const mbps = @as(f64, @floatFromInt(kbps)) / 1_000.0; - return std.fmt.allocPrint(arena, " {d:.1} MB/s", .{mbps}); - } else { - return std.fmt.allocPrint(arena, " {d} KB/s", .{kbps}); - } -} diff --git a/bin/zml-smi/tui/pages/detail/neuron_detail.zig b/bin/zml-smi/tui/pages/detail/neuron_detail.zig deleted file mode 100644 index e0dc3bcd..00000000 --- a/bin/zml-smi/tui/pages/detail/neuron_detail.zig +++ /dev/null @@ -1,83 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const data = @import("../../data.zig"); -const utils = @import("../../lib/utils.zig"); -const ui = @import("../../lib/ui.zig"); -const ColumnLayout = @import("../../widgets/column_layout.zig"); -const MetricCard = @import("../../widgets/metric_card.zig"); -const ProcessTable = @import("../../widgets/process_table.zig"); -const common = @import("detail_common.zig"); - -pub fn draw( - state: *const data.SystemState, - process_table: *ProcessTable, - ctx: vxfw.DrawContext, - w: u16, - id: u16, - nc: data.NeuronInfo, - parent_widget: vxfw.Widget, -) std.mem.Allocator.Error!vxfw.Surface { - // ── Header ────────────────────────────────────────────── - const name_suffix: []const u8 = if (nc.driver_version) |v| - try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) - else - ""; - const header = try common.headerText(ctx.arena, id, try std.fmt.allocPrint(ctx.arena, "{s}{s}", .{ nc.name orelse "Unknown", name_suffix })); - - // ── Utilization + Memory charts ───────────────────────── - const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ - .title = "Core Utilization", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{nc.util_percent orelse 0}), - }); - - const mem_used = nc.mem_used_bytes orelse 0; - const mem_total = nc.mem_total_bytes orelse 0; - const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ - .title = "Memory", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) mem_used * 100 / mem_total else 0}), - .info_line = try std.fmt.allocPrint(ctx.arena, "HBM {d} / {d} MB", .{ - utils.bytesToMb(nc.mem_used_bytes), utils.bytesToMb(nc.mem_total_bytes), - }), - }); - - const charts_flow: ColumnLayout = .{ - .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart) }, - .min_child_width = 40, - .gap = 1, - }; - - // ── HBM Breakdown ─────────────────────────────────────── - const fmt_mb = struct { - fn f(arena: std.mem.Allocator, val: ?u64) std.mem.Allocator.Error![]const u8 { - return std.fmt.allocPrint(arena, " {d} MB", .{utils.bytesToMb(val)}); - } - }.f; - const hbm_card: MetricCard = .{ - .title = "HBM Breakdown", - .lines = &.{ - .{ .label = "Tensors ", .value = try fmt_mb(ctx.arena, nc.nc_tensors) }, - .{ .label = "Constants ", .value = try fmt_mb(ctx.arena, nc.nc_constants) }, - .{ .label = "Model Code ", .value = try fmt_mb(ctx.arena, nc.nc_model_code) }, - .{ .label = "Shared Scratch", .value = try fmt_mb(ctx.arena, nc.nc_shared_scratchpad) }, - .{ .label = "Nonshared ", .value = try fmt_mb(ctx.arena, nc.nc_nonshared_scratchpad) }, - .{ .label = "Runtime ", .value = try fmt_mb(ctx.arena, nc.nc_runtime) }, - .{ .label = "Driver ", .value = try fmt_mb(ctx.arena, nc.nc_driver) }, - .{ .label = "DMA Rings ", .value = try fmt_mb(ctx.arena, nc.nc_dma_rings) }, - .{ .label = "Collectives ", .value = try fmt_mb(ctx.arena, nc.nc_collectives) }, - .{ .label = "Notifications ", .value = try fmt_mb(ctx.arena, nc.nc_notifications) }, - .{ .label = "Uncategorized ", .value = try fmt_mb(ctx.arena, nc.nc_uncategorized) }, - }, - }; - - // ── Compose vertical layout ───────────────────────────── - return common.pageFrame(ctx, .{ - .children = &.{ - header.widget(), - ui.widget(&charts_flow), - ui.widget(&hbm_card), - ui.widget(process_table), - }, - .gap = 1, - }, w, parent_widget); -} diff --git a/bin/zml-smi/tui/pages/detail/tpu_detail.zig b/bin/zml-smi/tui/pages/detail/tpu_detail.zig deleted file mode 100644 index ca6d3b66..00000000 --- a/bin/zml-smi/tui/pages/detail/tpu_detail.zig +++ /dev/null @@ -1,54 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const data = @import("../../data.zig"); -const utils = @import("../../lib/utils.zig"); -const ui = @import("../../lib/ui.zig"); -const ColumnLayout = @import("../../widgets/column_layout.zig"); -const ProcessTable = @import("../../widgets/process_table.zig"); -const common = @import("detail_common.zig"); - -pub fn draw( - state: *const data.SystemState, - process_table: *ProcessTable, - ctx: vxfw.DrawContext, - w: u16, - id: u16, - tp: data.TpuInfo, - parent_widget: vxfw.Widget, -) std.mem.Allocator.Error!vxfw.Surface { - // ── Header ────────────────────────────────────────────── - const header = try common.headerText(ctx.arena, id, tp.name orelse "Unknown"); - - // ── Utilization + Memory charts ───────────────────────── - const util_chart = try common.historyChart(ctx, state, id, state.history.util, .{ - .title = "Duty Cycle", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{tp.util_percent orelse 0}), - }); - - const mem_used = tp.mem_used_bytes orelse 0; - const mem_total = tp.mem_total_bytes orelse 0; - const mem_chart = try common.historyChart(ctx, state, id, state.history.mem_util, .{ - .title = "Memory", - .value_label = try std.fmt.allocPrint(ctx.arena, "{d}%", .{if (mem_total > 0) mem_used * 100 / mem_total else 0}), - .info_line = try std.fmt.allocPrint(ctx.arena, "HBM {d} / {d} MB", .{ - utils.bytesToMb(tp.mem_used_bytes), utils.bytesToMb(tp.mem_total_bytes), - }), - }); - - const charts_flow: ColumnLayout = .{ - .children = &.{ ui.widget(&util_chart), ui.widget(&mem_chart) }, - .min_child_width = 40, - .gap = 1, - }; - - // ── Compose vertical layout ───────────────────────────── - return common.pageFrame(ctx, .{ - .children = &.{ - header.widget(), - ui.widget(&charts_flow), - ui.widget(process_table), - }, - .gap = 1, - }, w, parent_widget); -} diff --git a/bin/zml-smi/tui/pages/overview.zig b/bin/zml-smi/tui/pages/overview.zig deleted file mode 100644 index e91253bd..00000000 --- a/bin/zml-smi/tui/pages/overview.zig +++ /dev/null @@ -1,119 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const data = @import("../data.zig"); -const ui = @import("../lib/ui.zig"); -const compose = @import("../lib/compose.zig"); -const image_cache = @import("../image_cache.zig"); -const ColumnLayout = @import("../widgets/column_layout.zig"); -const common = @import("detail/detail_common.zig"); -const DeviceCard = @import("../widgets/device_card.zig"); -const Logo = @import("../widgets/logo.zig"); -const InfoLines = @import("../widgets/info_lines.zig"); -const ProcessTable = @import("../widgets/process_table.zig"); - -const Overview = @This(); - -const two_col_breakpoint: u16 = 140; -const narrow_breakpoint: u16 = 80; -const max_info_width: u16 = 50; -/// Total width of the wide banner: logo box (incl. left margin) + info lines + right margin. -pub const host_line_width: u16 = (Logo.logo_width + 6) + max_info_width + 2; - -state: *const data.SystemState, -device_cards: []DeviceCard = &.{}, -process_table: ?*ProcessTable = null, // maybe null if we decide not to print the process table in print mode -viewing_device: *?u16 = undefined, -use_braille: bool = false, - -pub fn init(allocator: std.mem.Allocator, state: *const data.SystemState, process_table: ?*ProcessTable, viewing_device: *?u16) !Overview { - const count = state.deviceCount(); - const cards = try allocator.alloc(DeviceCard, count); - for (cards, 0..) |*card, i| { - card.* = .{ - .device_id = @intCast(i), - .state = state, - }; - } - return .{ - .state = state, - .device_cards = cards, - .process_table = process_table, - .viewing_device = viewing_device, - }; -} - -pub fn deinit(self: *Overview, allocator: std.mem.Allocator) void { - allocator.free(self.device_cards); -} - -fn drawNarrowBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { - const logo: Logo = .{ .image = image_cache.global.get("logo") }; - const info_lines: InfoLines = .{ .state = self.state }; - - const children = [2]vxfw.Widget{ - try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&logo)), .{ .width = content_w, .height = Logo.logo_height }), - ui.widget(&info_lines), - }; - const layout: ColumnLayout = .{ .children = &children, .gap = 1 }; - return ui.widget(&layout).draw(ui.fixedWidth(ctx, content_w)); -} - -fn drawWideBanner(self: *const Overview, ctx: vxfw.DrawContext, content_w: u16) !vxfw.Surface { - const logo: Logo = .{ .image = image_cache.global.get("logo") }; - const info_lines: InfoLines = .{ .state = self.state }; - - const logo_box_w = Logo.logo_width + 6; // +4 centering + 2 absorbed page margin - const info_max_w = @min(content_w -| logo_box_w, max_info_width); - const banner_h = @max(Logo.logo_height, InfoLines.entry_count); - - const flex_items = [2]vxfw.FlexItem{ - .{ .widget = try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&logo)), .{ .width = logo_box_w, .height = banner_h }), .flex = 0 }, - .{ .widget = try compose.sized(ctx.arena, try compose.center(ctx.arena, ui.widget(&info_lines)), .{ .width = info_max_w, .height = banner_h }), .flex = 0 }, - }; - const sized_banner = try compose.sized(ctx.arena, (vxfw.FlexRow{ .children = &flex_items }).widget(), .{ .width = content_w, .height = banner_h }); - return sized_banner.draw(ui.fixedWidth(ctx, content_w)); -} - -pub fn draw(self: *Overview, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const w = ctx.max.width orelse 80; - const narrow = w < narrow_breakpoint; - const content_w: u16 = w -| 4; - const content_ctx = ui.fixedWidth(ctx, content_w); - - var sb = compose.surfaceBuilder(ctx.arena); - var row: i17 = 1; // top margin - - // ── Banner (Logo + Info Lines) ───────────────────────── - const banner_surf = try if (narrow) self.drawNarrowBanner(ctx, content_w) else self.drawWideBanner(ctx, content_w + 2); - try sb.add(row, if (narrow) @as(i17, 2) else 0, banner_surf); - row += @intCast(banner_surf.size.height + 1); - - // ── Device Grid ───────────────────────────────────────── - for (self.device_cards) |*card| { - card.use_braille = self.use_braille; - card.viewing_device = self.viewing_device; - } - const card_widgets = try ctx.arena.alloc(vxfw.Widget, self.device_cards.len); - for (self.device_cards, card_widgets) |*card, *cw| { - cw.* = ui.widget(card); - } - const device_grid: ColumnLayout = .{ - .children = card_widgets, - .min_child_width = two_col_breakpoint / 2, - .col_gap = 2, - .gap = 1, - }; - const grid_surf = try ui.widget(&device_grid).draw(content_ctx); - try sb.add(row, 2, grid_surf); - row += @intCast(grid_surf.size.height + 1); - - // ── Process Table ──────────────────────────────────────── - if (self.process_table) |pt| { - const pt_surf = try ui.widget(pt).draw(content_ctx); - try sb.add(row, 2, pt_surf); - row += @intCast(pt_surf.size.height); - } - - return sb.finish(.{ .width = w, .height = @intCast(row) }, ui.widget(self)); -} diff --git a/bin/zml-smi/tui/print.zig b/bin/zml-smi/tui/print.zig deleted file mode 100644 index 238534cd..00000000 --- a/bin/zml-smi/tui/print.zig +++ /dev/null @@ -1,91 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const data = @import("data.zig"); -const ui = @import("lib/ui.zig"); -const image_cache = @import("image_cache.zig"); -const Overview = @import("pages/overview.zig"); - -fn silentResize(vx: *vaxis.Vaxis, allocator: std.mem.Allocator, winsize: vaxis.Winsize) !void { - vx.screen.deinit(allocator); - vx.screen = try vaxis.Screen.init(allocator, winsize); - vx.screen.width_method = vx.caps.unicode; - vx.screen_last.deinit(allocator); - vx.screen_last = try vaxis.AllocatingScreen.init(allocator, winsize.cols, winsize.rows); -} - -pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { - var tty_buffer: [4096]u8 = undefined; - var tty = try vaxis.Tty.init(io, &tty_buffer); - defer tty.deinit(); - - var vx = try vaxis.init(io, allocator, env_map, .{}); - defer { - // Reset terminal modes without clearing screen content. - vx.resetModes(tty.writer()) catch {}; - var buf: [256]u8 = undefined; - var discarding: std.Io.Writer.Discarding = .init(&buf); - vx.deinit(allocator, &discarding.writer); - } - - const ws = try tty.getWinsize(); - try silentResize(&vx, allocator, ws); - - { - const EventLoop = vaxis.Loop(vxfw.Event); - var loop: EventLoop = .init(io, &tty, &vx); - try loop.start(); - defer loop.stop(); - - // queryTerminal sends cursor home (\x1b[H) for capability detection. - // Save/restore cursor so we stay at the current position. - const writer = tty.writer(); - try writer.writeAll("\x1b7"); // DECSC: save cursor - try writer.flush(); - try vx.queryTerminal(writer, .fromSeconds(1)); - try writer.writeAll("\x1b8"); // DECRC: restore cursor - try writer.flush(); - } - - image_cache.global.loadAll(&vx, allocator, tty.writer()); - defer image_cache.global.deinit(allocator); - - vxfw.DrawContext.init(vx.screen.width_method); - - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - const content_w: u16 = @min(@as(u16, @intCast(vx.screen.width)), Overview.host_line_width + 2); - - var viewing_device: ?u16 = null; - var overview = try Overview.init(allocator, state, null, &viewing_device); - defer overview.deinit(allocator); - - const draw_ctx: vxfw.DrawContext = .{ - .arena = arena.allocator(), - .min = .{ .width = content_w }, - .max = .{ .width = content_w, .height = null }, - .cell_size = .{ - .width = if (vx.screen.width > 0) vx.screen.width_pix / vx.screen.width else 0, - .height = if (vx.screen.height > 0) vx.screen.height_pix / vx.screen.height else 0, - }, - }; - - const surface = try overview.draw(draw_ctx); - - try silentResize(&vx, allocator, .{ - .rows = surface.size.height, - .cols = content_w, - .x_pixel = ws.x_pixel, - .y_pixel = ws.y_pixel, - }); - - const win = vx.window(); - const root_win = win.child(.{ - .width = surface.size.width, - .height = surface.size.height, - }); - surface.render(root_win, ui.widget(&overview)); - - try vx.prettyPrint(tty.writer()); -} diff --git a/bin/zml-smi/tui/theme.zig b/bin/zml-smi/tui/theme.zig deleted file mode 100644 index e037f67f..00000000 --- a/bin/zml-smi/tui/theme.zig +++ /dev/null @@ -1,50 +0,0 @@ -const vaxis = @import("vaxis"); - -const Color = vaxis.Cell.Color; -const Style = vaxis.Cell.Style; - -/// Returns a color on a green → yellow → red gradient based on percentage (0-100). -pub fn colorForPercent(pct: u8) Color { - if (pct <= 50) { - // Green (#00cc66) → Yellow (#ffcc00) - const t: u16 = @as(u16, pct) * 255 / 50; - return .{ .rgb = .{ @intCast(t), 204, @intCast(102 -| (t * 102 / 255)) } }; - } else if (pct <= 80) { - // Yellow (#ffcc00) → Orange (#ff6600) - const t: u16 = (@as(u16, pct) - 50) * 255 / 30; - return .{ .rgb = .{ 255, @intCast(204 -| (t * 102 / 255)), 0 } }; - } else { - // Orange (#ff6600) → Red (#ff2200) - const t: u16 = (@as(u16, pct) - 80) * 255 / 20; - return .{ .rgb = .{ 255, @intCast(102 -| (t * 68 / 255)), 0 } }; - } -} - -/// Returns a color for temperature (20-100°C range). -pub fn colorForTemp(temp_c: u16) Color { - if (temp_c <= 20) { - return colorForPercent(0); - } - if (temp_c >= 100) { - return colorForPercent(100); - } - const pct: u8 = @intCast((@as(u32, temp_c) - 20) * 100 / 80); - return colorForPercent(pct); -} - -// ── Style Constants ────────────────────────────────────────────── - -pub const accent: Color = .{ .rgb = .{ 120, 180, 255 } }; -pub const accent_secondary: Color = .{ .rgb = .{ 200, 160, 255 } }; -pub const dim: Color = .{ .rgb = .{ 100, 100, 120 } }; -pub const text_primary: Color = .{ .rgb = .{ 230, 230, 240 } }; -pub const text_secondary: Color = .{ .rgb = .{ 160, 160, 175 } }; -pub const surface_border: Color = .{ .rgb = .{ 70, 70, 90 } }; -pub const gauge_empty: Color = .{ .rgb = .{ 50, 50, 65 } }; - -pub const header_style: Style = .{ .bold = true, .fg = accent }; -pub const title_style: Style = .{ .bold = true, .fg = accent_secondary }; -pub const label_style: Style = .{ .fg = text_secondary }; -pub const value_style: Style = .{ .bold = true, .fg = text_primary }; -pub const border_style: Style = .{ .fg = surface_border }; -pub const dim_style: Style = .{ .fg = dim }; diff --git a/bin/zml-smi/tui/top.zig b/bin/zml-smi/tui/top.zig deleted file mode 100644 index 74b3796b..00000000 --- a/bin/zml-smi/tui/top.zig +++ /dev/null @@ -1,168 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; - -const data = @import("data.zig"); -const ui = @import("lib/ui.zig"); -const compose = @import("lib/compose.zig"); -const image_cache = @import("image_cache.zig"); -const Logo = @import("widgets/logo.zig"); -const ProcessTable = @import("widgets/process_table.zig"); -const Overview = @import("pages/overview.zig"); -const Detail = @import("pages/detail.zig"); -const StatusLine = @import("widgets/status_line.zig"); - -const min_width: u16 = 45; -const min_height: u16 = 24; - -const ScrollState = struct { - row: i17 = 0, - col: i17 = 0, - - pub fn reset(self: *ScrollState) void { - self.row = 0; - self.col = 0; - } - - pub fn scrollBy(self: *ScrollState, dr: i17, dc: i17) void { - self.row = @max(0, self.row + dr); - self.col = @max(0, self.col + dc); - } - - pub fn clamp(self: *ScrollState, content_h: u16, viewport_h: u16, content_w: u16, viewport_w: u16) void { - const max_row: i17 = @max(0, @as(i17, content_h) - @as(i17, viewport_h)); - const max_col: i17 = @max(0, @as(i17, content_w) - @as(i17, viewport_w)); - self.row = std.math.clamp(self.row, 0, max_row); - self.col = std.math.clamp(self.col, 0, max_col); - } -}; - -const Model = struct { - allocator: std.mem.Allocator, - state: *data.SystemState, - viewing_device: ?u16 = null, - prev_viewing_device: ?u16 = null, - scroll: ScrollState = .{}, - vx: *vaxis.Vaxis, - tty: *vaxis.Tty, - process_table: ProcessTable = .{}, - overview: Overview = undefined, - - fn deinit(self: *Model) void { - self.process_table.deinit(self.allocator); - self.overview.deinit(self.allocator); - image_cache.global.deinit(self.allocator); - } - - pub fn handleEvent(self: *Model, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { - switch (event) { - .init => { - self.overview = try Overview.init(self.allocator, self.state, &self.process_table, &self.viewing_device); - self.overview.use_braille = true; - image_cache.global.loadAll(self.vx, self.allocator, self.tty.writer()); - try ctx.tick(1000, ui.widget(self)); - }, - .tick => { - self.state.recordHistory(); - try ctx.tick(self.state.tui_refresh_rate, ui.widget(self)); - ctx.redraw = true; - }, - .key_press => |key| { - if (key.matches('q', .{}) or key.matches('c', .{ .ctrl = true })) { - ctx.quit = true; - return; - } - if (self.viewing_device != null) { - if (key.matches(vaxis.Key.escape, .{}) or key.matches(vaxis.Key.backspace, .{})) { - self.viewing_device = null; - return ctx.consumeAndRedraw(); - } - } else { - if (key.matches('v', .{})) { - self.overview.use_braille = !self.overview.use_braille; - return ctx.consumeAndRedraw(); - } - } - if (key.matches(vaxis.Key.down, .{})) { - self.scroll.scrollBy(1, 0); - } else if (key.matches(vaxis.Key.up, .{})) { - self.scroll.scrollBy(-1, 0); - } else if (key.matches(vaxis.Key.right, .{})) { - self.scroll.scrollBy(0, 1); - } else if (key.matches(vaxis.Key.left, .{})) { - self.scroll.scrollBy(0, -1); - } else return; - return ctx.consumeAndRedraw(); - }, - .mouse => |mouse| { - switch (mouse.button) { - .wheel_up => self.scroll.scrollBy(-3, 0), - .wheel_down => self.scroll.scrollBy(3, 0), - .wheel_left => self.scroll.scrollBy(0, 3), - .wheel_right => self.scroll.scrollBy(0, -3), - else => return, - } - return ctx.consumeAndRedraw(); - }, - else => {}, - } - } - - pub fn draw(self: *Model, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const screen = ctx.max.size(); - const content_w = @max(screen.width, min_width); - - if (self.viewing_device != self.prev_viewing_device) { - self.scroll.reset(); - self.process_table.resetScroll(); - self.prev_viewing_device = self.viewing_device; - } - - self.process_table.prepare(self.state, self.viewing_device); - - var content_surf: vxfw.Surface = undefined; - - if (self.viewing_device) |dev_id| { - const detail: Detail = .{ - .state = self.state, - .device_id = dev_id, - .process_table = &self.process_table, - }; - content_surf = try detail.draw(ui.fixedWidth(ctx, content_w)); - } else { - content_surf = try self.overview.draw(ui.fixedWidth(ctx, content_w)); - } - - content_surf.size.height = @max(content_surf.size.height, min_height); - - const content_h = screen.height -| 1; - self.scroll.clamp(content_surf.size.height, content_h, content_surf.size.width, screen.width); - - const status_line: StatusLine = .{ - .viewing_device = self.viewing_device, - .use_braille = self.overview.use_braille, - }; - const status_surf = try status_line.draw(ui.fixedSize(ctx, screen.width, 1)); - - var sb = compose.surfaceBuilder(ctx.arena); - try sb.add(-self.scroll.row, -self.scroll.col, content_surf); - try sb.addZ(@intCast(content_h), 0, status_surf, 1); - return sb.finish(screen, ui.widget(self)); - } -}; - -pub fn run(allocator: std.mem.Allocator, io: std.Io, env_map: *std.process.Environ.Map, state: *data.SystemState) !void { - var tty_buffer: [4096]u8 = undefined; - var app = try vxfw.App.init(io, allocator, env_map, &tty_buffer); - defer app.deinit(); - - var model: Model = .{ - .allocator = allocator, - .state = state, - .vx = &app.vx, - .tty = &app.tty, - }; - defer model.deinit(); - - try app.run(ui.widget(&model), .{}); -} diff --git a/bin/zml-smi/tui/tui.zig b/bin/zml-smi/tui/tui.zig deleted file mode 100644 index 015e3ce2..00000000 --- a/bin/zml-smi/tui/tui.zig +++ /dev/null @@ -1,3 +0,0 @@ -pub const data = @import("data.zig"); -pub const print = @import("print.zig"); -pub const top = @import("top.zig"); diff --git a/bin/zml-smi/tui/widgets/braille_chart.zig b/bin/zml-smi/tui/widgets/braille_chart.zig deleted file mode 100644 index 498739d1..00000000 --- a/bin/zml-smi/tui/widgets/braille_chart.zig +++ /dev/null @@ -1,76 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const theme = @import("../theme.zig"); - -const ui = @import("../lib/ui.zig"); - -const BrailleChart = @This(); - -values: []const u8, // 0-100 -height: u16 = 2, - -pub const chart_bg: vaxis.Cell.Color = .{ .rgb = .{ 30, 30, 40 } }; - -/// Braille dot bit positions: [row_in_cell (0-3)][col_in_cell (0-1)] -/// Each braille character is a 2x4 dot grid encoded as U+2800 + pattern byte. -pub const dot_bits = [4][2]u8{ - .{ 0x01, 0x08 }, // row 0 - .{ 0x02, 0x10 }, // row 1 - .{ 0x04, 0x20 }, // row 2 - .{ 0x40, 0x80 }, // row 3 -}; - -pub fn draw(self: *const BrailleChart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const w = ctx.max.width orelse 40; - var surface = try vxfw.Surface.init(ctx.arena, ui.widget(self), .{ .width = w, .height = self.height }); - try self.renderTo(ctx.arena, &surface, 0, 0, w); - return surface; -} - -/// Render braille data into an existing surface at the given offset. -pub fn renderTo(self: *const BrailleChart, arena: std.mem.Allocator, surface: *vxfw.Surface, col_offset: u16, row_offset: u16, w: u16) std.mem.Allocator.Error!void { - const h = self.height; - const dot_rows: u32 = @as(u32, h) * 4; - const dot_cols: u32 = @as(u32, w) * 2; - const data_len: u32 = @intCast(self.values.len); - - for (0..w) |cell_col| { - var max_val: u8 = 0; - for (0..2) |dx| { - const dot_x: u32 = @as(u32, @intCast(cell_col)) * 2 + @as(u32, @intCast(dx)); - if (dot_x + data_len >= dot_cols) { - max_val = @max(max_val, self.values[@intCast(dot_x + data_len - dot_cols)]); - } - } - const color = theme.colorForPercent(max_val); - - for (0..h) |cell_row| { - var pattern: u8 = 0; - for (0..2) |dx| { - const dot_x: u32 = @as(u32, @intCast(cell_col)) * 2 + @as(u32, @intCast(dx)); - if (dot_x + data_len >= dot_cols) { - const val = self.values[@intCast(dot_x + data_len - dot_cols)]; - const fill: u32 = @as(u32, val) * dot_rows / 100; - const threshold: u32 = dot_rows - fill; - for (0..4) |dr| { - const abs_row: u32 = @as(u32, @intCast(cell_row)) * 4 + @as(u32, @intCast(dr)); - if (abs_row >= threshold) { - pattern |= dot_bits[dr][dx]; - } - } - } - } - - const buf = try arena.alloc(u8, 3); - buf[0] = 0xE2; - buf[1] = 0xA0 + (pattern >> 6); - buf[2] = 0x80 + (pattern & 0x3F); - - surface.writeCell(col_offset + @as(u16, @intCast(cell_col)), row_offset + @as(u16, @intCast(cell_row)), .{ - .char = .{ .grapheme = buf, .width = 1 }, - .style = .{ .fg = color, .bg = chart_bg }, - }); - } - } -} diff --git a/bin/zml-smi/tui/widgets/chart.zig b/bin/zml-smi/tui/widgets/chart.zig deleted file mode 100644 index 5421cd1f..00000000 --- a/bin/zml-smi/tui/widgets/chart.zig +++ /dev/null @@ -1,141 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const theme = @import("../theme.zig"); -const ui = @import("../lib/ui.zig"); -const TitledBorder = @import("titled_border.zig"); -const BrailleChart = @import("braille_chart.zig"); - -const Chart = @This(); - -// Layout constants for rows below the chart data area -const x_axis_h: u16 = 1; // x-axis line -const time_labels_h: u16 = 1; // time labels below x-axis -const info_line_h: u16 = 1; // info text below time labels - -title: []const u8, -data: []const u8, // 0-100 normalized -value_label: ?[]const u8 = null, -info_line: ?[]const u8 = null, -y_min: u32 = 0, -y_max: u32 = 100, -y_unit: []const u8 = "%", -chart_height: u16 = 6, -tui_refresh_rate: u16, - -pub fn draw(self: *const Chart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const tb: TitledBorder = .{ - .child = ui.drawWidget(self, drawContent), - .title = self.title, - .value_label = self.value_label, - }; - return tb.draw(ctx); -} - -fn drawContent(self: *const Chart, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const max_w = ctx.max.width orelse 40; - const content_h: u16 = self.chart_height + x_axis_h + time_labels_h + info_line_h; - - // ── Y-axis label widths ────────────────────────────────── - const y_labels = [3]struct { row: u16, val: u32 }{ - .{ .row = 0, .val = self.y_max }, - .{ .row = self.chart_height / 2, .val = (self.y_min + self.y_max) / 2 }, - .{ .row = self.chart_height, .val = self.y_min }, - }; - var y_strs: [3][]const u8 = undefined; - var y_str_widths: [3]u16 = undefined; - var y_label_w: u16 = 0; - for (&y_strs, &y_str_widths, y_labels) |*s, *sw, lbl| { - s.* = try std.fmt.allocPrint(ctx.arena, "{d}{s}", .{ lbl.val, self.y_unit }); - sw.* = @intCast(ctx.stringWidth(s.*)); - y_label_w = @max(y_label_w, sw.*); - } - - // ── Column layout ──────────────────────────────────────── - const chart_start: u16 = y_label_w + 1; - const chart_w: u16 = if (max_w > chart_start) max_w - chart_start else 0; - - var surface = try vxfw.Surface.init(ctx.arena, ui.drawWidget(self, drawContent), .{ .width = max_w, .height = content_h }); - if (chart_w == 0) { - return surface; - } - - // ── Y-axis labels + axis line ──────────────────────────── - for (y_strs, y_str_widths, y_labels) |text, sw, lbl| { - writeStr(&surface, y_label_w - sw, lbl.row, text, theme.dim_style); - } - for (0..self.chart_height + 1) |r| { - const row: u16 = @intCast(r); - const ch: []const u8 = if (row == self.chart_height) - "\u{2514}" - else - "\u{2502}"; - surface.writeCell(y_label_w, row, .{ - .char = .{ .grapheme = ch, .width = 1 }, - .style = theme.dim_style, - }); - } - - // ── Braille data area ──────────────────────────────────── - const braille: BrailleChart = .{ .values = self.data, .height = self.chart_height }; - try braille.renderTo(ctx.arena, &surface, chart_start, 0, chart_w); - - // ── X-axis line ────────────────────────────────────────── - const tick_mid: u16 = chart_w / 2; - const tick_right: u16 = chart_w -| 1; - for (0..chart_w) |c| { - const col: u16 = @intCast(c); - const is_tick = (col == 0 or col == tick_mid or col == tick_right); - surface.writeCell(chart_start + col, self.chart_height, .{ - .char = .{ .grapheme = if (is_tick) "\u{252C}" else "\u{2500}", .width = 1 }, - .style = theme.dim_style, - }); - } - - // ── X-axis time labels ─────────────────────────────────── - const total_ms: u32 = @as(u32, chart_w) * 2 * @as(u32, self.tui_refresh_rate); - const total_s: u16 = @intCast(std.math.divCeil(u32, total_ms, 1000) catch unreachable); - const mid_s: u16 = @intCast(std.math.divCeil(u32, total_ms / 2, 1000) catch unreachable); - const label_row = self.chart_height + x_axis_h; - const left_str = try fmtTime(ctx.arena, total_s); - const mid_str = try fmtTime(ctx.arena, mid_s); - - writeStr(&surface, chart_start, label_row, left_str, theme.dim_style); - - const mid_w: u16 = @intCast(ctx.stringWidth(mid_str)); - writeStr(&surface, chart_start + tick_mid -| (mid_w / 2), label_row, mid_str, theme.dim_style); - - const right_str: []const u8 = "now"; - const right_w: u16 = @intCast(ctx.stringWidth(right_str)); - writeStr(&surface, chart_start + tick_right + 1 -| right_w, label_row, right_str, theme.dim_style); - - // ── Info line ──────────────────────────────────────────── - if (self.info_line) |info| { - writeStr(&surface, chart_start, self.chart_height + x_axis_h + time_labels_h, info, theme.dim_style); - } - - return surface; -} - -fn writeStr(surface: *vxfw.Surface, col: u16, row: u16, text: []const u8, style: vaxis.Cell.Style) void { - var c = col; - var i: usize = 0; - while (i < text.len) { - const len = std.unicode.utf8ByteSequenceLength(text[i]) catch 1; - const end = @min(i + len, text.len); - surface.writeCell(c, row, .{ - .char = .{ .grapheme = text[i..end], .width = 1 }, - .style = style, - }); - c += 1; - i = end; - } -} - -fn fmtTime(arena: std.mem.Allocator, seconds: u16) std.mem.Allocator.Error![]const u8 { - if (seconds >= 120) { - return std.fmt.allocPrint(arena, "-{d}m", .{seconds / 60}); - } else { - return std.fmt.allocPrint(arena, "-{d}s", .{seconds}); - } -} diff --git a/bin/zml-smi/tui/widgets/column_layout.zig b/bin/zml-smi/tui/widgets/column_layout.zig deleted file mode 100644 index f307d40c..00000000 --- a/bin/zml-smi/tui/widgets/column_layout.zig +++ /dev/null @@ -1,64 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const ui = @import("../lib/ui.zig"); -const compose = @import("../lib/compose.zig"); - -const ColumnLayout = @This(); - -children: []const vxfw.Widget, -gap: u16 = 0, -col_gap: u16 = 0, -/// When set, children wrap into a grid with this minimum column width. -/// When null (default), behaves as a simple vertical stack. -min_child_width: ?u16 = null, - -pub fn draw(self: *const ColumnLayout, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const n = self.children.len; - if (n == 0) { - return vxfw.Surface.init(ctx.arena, ui.widget(self), ctx.min); - } - - const available_w = ctx.max.width orelse 80; - const cols: u16 = if (self.min_child_width) |mcw| - @intCast(@max(1, @min(n, available_w / @max(mcw, 1)))) - else - 1; - - var sb = compose.surfaceBuilder(ctx.arena); - var row_y: u16 = 0; - var max_width: u16 = 0; - var i: usize = 0; - - while (i < n) { - if (i > 0) { - row_y += self.gap; - } - - const row_end = @min(i + cols, n); - var row_h: u16 = 0; - - for (i..row_end) |j| { - const col_idx: u16 = @intCast(j - i); - - const col_x, const w = if (cols > 1) blk: { - const total_col_gaps = @as(u16, cols - 1) * self.col_gap; - const usable_w = available_w -| total_col_gaps; - const child_w = usable_w / cols; - const x = col_idx * (child_w + self.col_gap); - break :blk .{ x, if (col_idx == cols - 1) available_w - x else child_w }; - } else .{ 0, available_w }; - - const surf = try self.children[j].draw(ui.fixedWidth(ctx, w)); - try sb.add(row_y, col_x, surf); - max_width = @max(max_width, col_x + surf.size.width); - row_h = @max(row_h, surf.size.height); - } - - row_y += row_h; - i = row_end; - } - - const result_w = if (ctx.max.width) |mw| @min(@max(max_width, ctx.min.width), mw) else @max(max_width, ctx.min.width); - return sb.finish(.{ .width = result_w, .height = row_y }, ui.widget(self)); -} diff --git a/bin/zml-smi/tui/widgets/device_card.zig b/bin/zml-smi/tui/widgets/device_card.zig deleted file mode 100644 index c7f7ac38..00000000 --- a/bin/zml-smi/tui/widgets/device_card.zig +++ /dev/null @@ -1,148 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const data = @import("../data.zig"); -const theme = @import("../theme.zig"); -const utils = @import("../lib/utils.zig"); -const ui = @import("../lib/ui.zig"); -const image_cache = @import("../image_cache.zig"); -const MetricCard = @import("metric_card.zig"); - -const DeviceCard = @This(); - -device_id: u16 = 0, -highlighted: bool = false, - -state: *const data.SystemState = undefined, -use_braille: bool = false, -viewing_device: *?u16 = undefined, - -pub fn handleEvent(self: *DeviceCard, ctx: *vxfw.EventContext, event: vxfw.Event) anyerror!void { - switch (event) { - .mouse => |mouse| { - if (mouse.type == .press and mouse.button == .left) { - self.viewing_device.* = self.device_id; - return ctx.consumeAndRedraw(); - } - }, - .mouse_enter => { - self.highlighted = true; - try ctx.setMouseShape(.pointer); - return ctx.consumeAndRedraw(); - }, - .mouse_leave => { - self.highlighted = false; - try ctx.setMouseShape(.default); - ctx.redraw = true; - }, - else => {}, - } -} - -pub fn draw(self: *DeviceCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const i = self.device_id; - const dev = self.state.devices[i]; - const target: data.Target = std.meta.activeTag(dev.*); - const cf = switch (dev.*) { - inline else => |*sv| blk: { - const f = sv.front().*; - break :blk .{ - .name = f.name orelse "Unknown", - .util_percent = @as(u8, @intCast(@min(f.util_percent orelse 0, 100))), - .mem_used = f.mem_used_bytes orelse 0, - .mem_total = f.mem_total_bytes orelse 0, - }; - }, - }; - const mem_pct: u8 = if (cf.mem_total > 0) @intCast(@min(cf.mem_used * 100 / cf.mem_total, 100)) else 0; - - const device_label, const util_label = switch (target) { - .cuda, .rocm, .oneapi => .{ "GPU", "GPU" }, - .neuron => .{ "NC", "Core" }, - .tpu => .{ "TPU", "Duty" }, - }; - const driver_ver: []const u8 = switch (dev.*) { - .cuda => |*sv| blk: { - const gpu = sv.front(); - const drv = gpu.driver_version orelse ""; - const cuda = gpu.cuda_driver_version orelse ""; - break :blk if (drv.len > 0 and cuda.len > 0) - try std.fmt.allocPrint(ctx.arena, " ({s}, CUDA {s})", .{ drv, cuda }) - else if (drv.len > 0) - try std.fmt.allocPrint(ctx.arena, " ({s})", .{drv}) - else if (cuda.len > 0) - try std.fmt.allocPrint(ctx.arena, " (CUDA {s})", .{cuda}) - else - @as([]const u8, ""); - }, - .rocm, .oneapi => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", - .neuron => |*sv| if (sv.front().driver_version) |v| try std.fmt.allocPrint(ctx.arena, " ({s})", .{v}) else "", - else => "", - }; - const card_title = try std.fmt.allocPrint(ctx.arena, "{s} {d}: {s}{s}", .{ device_label, i, cf.name, driver_ver }); - - const dev_suffix = switch (dev.*) { - .cuda, .rocm, .oneapi => |*sv| blk: { - const gpu = sv.front().*; - const power = (gpu.power_mw orelse 0) / 1000; - const power_limit = (gpu.power_limit_mw orelse 0) / 1000; - - const power_limit_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power_limit}); - const power_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{power}); - - const pad = try utils.repeatStr(ctx.arena, " ", @intCast(power_limit_str.len -| power_str.len)); - break :blk try std.fmt.allocPrint(ctx.arena, "{d}\u{00b0}C | {s}{s} W", .{ - gpu.temperature orelse 0, pad, power_str, - }); - }, - else => "", - }; - - const mem_used_mb = utils.bytesToMb(cf.mem_used); - const mem_total_mb = utils.bytesToMb(cf.mem_total); - const mem_total_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_total_mb}); - const mem_used_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{mem_used_mb}); - - const mem_pad = try utils.repeatStr(ctx.arena, " ", @intCast(mem_total_str.len -| mem_used_str.len)); - const mem_suffix = try std.fmt.allocPrint(ctx.arena, "{s}{s} / {s} MB", .{ - mem_pad, mem_used_str, mem_total_str, - }); - - var card: MetricCard = .{ - .title = card_title, - .title_image = image_cache.global.get(switch (dev.*) { - .cuda => "gpu_cuda", - .rocm => "gpu_rocm", - .oneapi => "gpu_oneapi", - .neuron => "gpu_neuron", - .tpu => "gpu_tpu", - }), - .title_right = if (self.state.devices[i].isRemote()) "remote" else null, - .cell_size = ctx.cell_size, - .highlighted = self.highlighted, - }; - - const util_hist = try self.state.history.util[i].sliceLast(ctx.arena, data.history_len); - const mem_hist = try self.state.history.mem_util[i].sliceLast(ctx.arena, data.history_len); - const util_chart_data = try utils.normalizeRange(ctx.arena, util_hist, 0, 100); - const mem_chart_data = try utils.normalizeRange(ctx.arena, mem_hist, 0, 100); - - const chart_entries = [2]MetricCard.ChartEntry{ - .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix, .data = util_chart_data }, - .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix, .data = mem_chart_data }, - }; - const gauge_entries = [2]MetricCard.GaugeEntry{ - .{ .label = util_label, .value = cf.util_percent, .suffix = dev_suffix }, - .{ .label = "MEM", .value = mem_pct, .suffix = mem_suffix }, - }; - - if (self.use_braille) { - card.charts = &chart_entries; - } else { - card.gauges = &gauge_entries; - } - - var card_surf = try card.draw(ctx); - card_surf.widget = ui.widget(self); - return card_surf; -} diff --git a/bin/zml-smi/tui/widgets/gauge.zig b/bin/zml-smi/tui/widgets/gauge.zig deleted file mode 100644 index 865a45f5..00000000 --- a/bin/zml-smi/tui/widgets/gauge.zig +++ /dev/null @@ -1,103 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const theme = @import("../theme.zig"); -const utils = @import("../lib/utils.zig"); -const ui = @import("../lib/ui.zig"); - -const RichText = vxfw.RichText; - -const Gauge = @This(); - -value: u8, // 0-100 -label: ?[]const u8 = null, -suffix: ?[]const u8 = null, -/// Minimum label space to reserve (for aligning gauges with different label widths). -label_reserve: u16 = 0, -/// Minimum suffix space to reserve (for aligning gauges with different suffix widths). -suffix_reserve: u16 = 0, - -pub const sep_w: u16 = 3; // " " before suffix - -pub fn draw(self: *const Gauge, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const max_w = ctx.max.width orelse 30; - const width: u16 = @max(ctx.min.width, max_w); - - // Label width (use reserve for alignment if set) - var label_w: u16 = 0; - if (self.label) |label| { - label_w = @intCast(ctx.stringWidth(label)); - label_w += 1; // trailing space - } - const effective_label_w = @max(label_w, self.label_reserve); - - // Percentage text: " XX%" - const pct_str = try std.fmt.allocPrint(ctx.arena, "{d:>3}%", .{self.value}); - const pct_width: u16 = @intCast(ctx.stringWidth(pct_str)); - - // Suffix area (text + separator); use reserve for bar calculation, actual for rendering - const actual_suffix_area: u16 = if (self.suffix) |s| @as(u16, @intCast(ctx.stringWidth(s))) + sep_w else 0; - const effective_suffix_area = @max(actual_suffix_area, self.suffix_reserve); - - // Reserved: label + "[" + "]" + pct + suffix_area - const reserved = effective_label_w + pct_width + 2 + effective_suffix_area; // 2 for [ ] - const bar_width = width -| reserved; - const filled: u16 = @intCast(@as(u32, bar_width) * @as(u32, @min(self.value, 100)) / 100); - const empty_count = bar_width - filled; - - var spans: std.ArrayList(vaxis.Cell.Segment) = .empty; - - // Label (padded to effective_label_w for alignment) - if (self.label) |label| { - try spans.append(ctx.arena, .{ .text = label, .style = theme.label_style }); - const pad = effective_label_w -| label_w; - const pad_str = try utils.repeatStr(ctx.arena, " ", pad + 1); // +1 for trailing space - try spans.append(ctx.arena, .{ .text = pad_str, .style = theme.label_style }); - } - - // Opening bracket - try spans.append(ctx.arena, .{ .text = "[", .style = theme.dim_style }); - - // Filled blocks — per-block gradient color - for (0..filled) |i| { - const pos_pct: u8 = if (bar_width > 1) - @intCast(@as(u32, @intCast(i)) * 100 / (@as(u32, bar_width) - 1)) - else - 0; - try spans.append(ctx.arena, .{ - .text = "\u{2588}", - .style = .{ .fg = theme.colorForPercent(pos_pct) }, - }); - } - - // Empty blocks - if (empty_count > 0) { - const empty_str = try utils.repeatStr(ctx.arena, "\u{2591}", empty_count); - try spans.append(ctx.arena, .{ .text = empty_str, .style = .{ .fg = theme.gauge_empty } }); - } - - // Closing bracket (no space — pct% collides with bar) - try spans.append(ctx.arena, .{ .text = "]", .style = theme.dim_style }); - - // Percentage text - try spans.append(ctx.arena, .{ .text = pct_str, .style = .{ .bold = true, .fg = theme.colorForPercent(self.value) } }); - - // Suffix (right-aligned) - if (self.suffix) |suffix| { - const suffix_text_w: u16 = @intCast(ctx.stringWidth(suffix)); - const pad = effective_suffix_area -| suffix_text_w; - if (pad > 0) { - const pad_str = try utils.repeatStr(ctx.arena, " ", pad); - try spans.append(ctx.arena, .{ .text = pad_str, .style = theme.dim_style }); - } - try spans.append(ctx.arena, .{ .text = suffix, .style = theme.dim_style }); - } - - const rich: RichText = .{ - .text = spans.items, - .softwrap = false, - .overflow = .clip, - .width_basis = .parent, - }; - return rich.draw(ui.fixedSize(ctx, width, 1)); -} diff --git a/bin/zml-smi/tui/widgets/info_lines.zig b/bin/zml-smi/tui/widgets/info_lines.zig deleted file mode 100644 index fcf1ae0d..00000000 --- a/bin/zml-smi/tui/widgets/info_lines.zig +++ /dev/null @@ -1,112 +0,0 @@ -const std = @import("std"); - -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; - -const data = @import("../data.zig"); -const compose = @import("../lib/compose.zig"); -const ui = @import("../lib/ui.zig"); -const theme = @import("../theme.zig"); -const stamp = @import("bazel/stamp"); - -const InfoLines = @This(); - -state: *const data.SystemState, - -const labels = [_][]const u8{ - "zml-smi ", - "Hostname ", - "Kernel ", - "CPU ", - "Cores ", - "Memory ", - "Uptime ", - "Load ", - "Devices ", -}; - -pub const entry_count: u16 = labels.len; - -const Entry = struct { - label: []const u8, - value: []const u8, -}; - -pub fn draw(self: *const InfoLines, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const w = ctx.max.width orelse 80; - const host = self.state.host.front().*; - - const hostname_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.hostname orelse "Unknown"}); - const kernel_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.kernel orelse "Unknown"}); - const cpu_str = try std.fmt.allocPrint(ctx.arena, "{s}", .{host.cpu_name orelse "Unknown"}); - const uptime_str = try formatUptime(ctx.arena, host.uptime_seconds orelse 0); - const cores_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{host.cpu_cores orelse 0}); - const load_str = try std.fmt.allocPrint(ctx.arena, "{d:.2} / {d:.2} / {d:.2}", .{ - host.load_1 orelse 0, host.load_5 orelse 0, host.load_15 orelse 0, - }); - const total_kib = host.mem_total_kib orelse 0; - const avail_kib = host.mem_available_kib orelse 0; - const used_gb = (total_kib -| avail_kib) / (1024 * 1024); - const total_gb = total_kib / (1024 * 1024); - const mem_str = try std.fmt.allocPrint(ctx.arena, "{d} / {d} GB", .{ used_gb, total_gb }); - const devices_str = try std.fmt.allocPrint(ctx.arena, "{d}", .{self.state.deviceCount()}); - const values = [entry_count][]const u8{ - stamp.stamp("STABLE_ZML_SMI_VERSION"), - hostname_str, - kernel_str, - cpu_str, - cores_str, - mem_str, - uptime_str, - load_str, - devices_str, - }; - var entries: [entry_count]Entry = undefined; - for (&entries, labels, values) |*e, label, value| { - e.* = .{ .label = label, .value = value }; - } - - var sb = compose.surfaceBuilder(ctx.arena); - - for (entries, 0..) |entry, i| { - if (w == 0) { - continue; - } - - const total_len = entry.label.len + entry.value.len; - const segments = if (total_len < w) blk: { - const pad_len = w - total_len; - const pad = try ctx.arena.alloc(u8, pad_len); - @memset(pad, '-'); - pad[pad_len - 1] = ' '; - break :blk try ctx.arena.dupe(vaxis.Cell.Segment, &.{ - .{ .text = entry.label, .style = theme.header_style }, - .{ .text = pad, .style = theme.dim_style }, - .{ .text = entry.value, .style = theme.value_style }, - }); - } else try ctx.arena.dupe(vaxis.Cell.Segment, &.{ - .{ .text = entry.label, .style = theme.header_style }, - .{ .text = entry.value, .style = theme.value_style }, - }); - - const text_surf = try ui.drawRichLine(ctx, segments, w); - - try sb.add(@intCast(i), 0, text_surf); - } - - const total_h: u16 = @intCast(entries.len); - return sb.finish(.{ .width = w, .height = total_h }, ui.widget(self)); -} - -fn formatUptime(arena: std.mem.Allocator, seconds: u64) std.mem.Allocator.Error![]const u8 { - const days = seconds / 86400; - const hours = (seconds % 86400) / 3600; - const mins = (seconds % 3600) / 60; - if (days > 0) { - return std.fmt.allocPrint(arena, "{d}d {d}h {d}m", .{ days, hours, mins }); - } else if (hours > 0) { - return std.fmt.allocPrint(arena, "{d}h {d}m", .{ hours, mins }); - } else { - return std.fmt.allocPrint(arena, "{d}m", .{mins}); - } -} diff --git a/bin/zml-smi/tui/widgets/logo.zig b/bin/zml-smi/tui/widgets/logo.zig deleted file mode 100644 index e6970485..00000000 --- a/bin/zml-smi/tui/widgets/logo.zig +++ /dev/null @@ -1,63 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const ui = @import("../lib/ui.zig"); -const compose = @import("../lib/compose.zig"); -const Image = @import("../lib/image.zig"); -const zml_logo = @import("zml/logo"); - -const Logo = @This(); - -style: vaxis.Cell.Style = .{}, -image: ?vaxis.Image = null, - -pub const logo_width: u16 = 28; -pub const logo_height: u16 = @intCast(zml_logo.zml_art_blocks[0].rows.len); - -pub fn draw(self: *const Logo, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - if (self.image) |img| { - const img_w: Image = .{ .image = img, .rows = logo_height }; - return img_w.draw(ctx); - } - - var sb = compose.surfaceBuilder(ctx.arena); - for (0..logo_height, 0..) |logo_row, row| { - try sb.add(@intCast(row), 0, try self.drawLogoRow(ctx, logo_row)); - } - return sb.finish(.{ .width = logo_width, .height = logo_height }, ui.widget(self)); -} - -fn drawLogoRow(self: *const Logo, ctx: vxfw.DrawContext, row_index: usize) std.mem.Allocator.Error!vxfw.Surface { - var segments: std.ArrayList(vaxis.Cell.Segment) = .empty; - for (zml_logo.zml_art_blocks[0..]) |block| { - const row = block.rows[row_index]; - if (row.text.len == 0) continue; - - var view: std.unicode.Utf8View = .initUnchecked(row.text); - var iter = view.iterator(); - while (iter.nextCodepointSlice()) |codepoint| { - const shiny = zml_logo.isShineGlyph(codepoint); - try segments.append(ctx.arena, .{ - .text = codepoint, - .style = .{ - .fg = switch (if (shiny) row.shine else row.color) { - .reset => .default, - .cyan => .{ .rgb = .{ 0, 255, 255 } }, - .sky => .{ .rgb = .{ 0, 221, 255 } }, - .blue => .{ .rgb = .{ 135, 143, 255 } }, - .violet => .{ .rgb = .{ 215, 90, 219 } }, - .pink => .{ .rgb = .{ 242, 48, 174 } }, - .shine_cyan => .{ .rgb = .{ 140, 255, 255 } }, - .shine_sky => .{ .rgb = .{ 128, 244, 255 } }, - .shine_blue => .{ .rgb = .{ 205, 210, 255 } }, - .shine_violet => .{ .rgb = .{ 255, 170, 255 } }, - .shine_pink => .{ .rgb = .{ 255, 150, 220 } }, - }, - .bg = self.style.bg, - .bold = shiny, - }, - }); - } - } - return ui.drawRichLine(ctx, segments.items, logo_width); -} diff --git a/bin/zml-smi/tui/widgets/metric_card.zig b/bin/zml-smi/tui/widgets/metric_card.zig deleted file mode 100644 index 6ceb9069..00000000 --- a/bin/zml-smi/tui/widgets/metric_card.zig +++ /dev/null @@ -1,165 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const theme = @import("../theme.zig"); -const ui = @import("../lib/ui.zig"); -const compose = @import("../lib/compose.zig"); -const Gauge = @import("gauge.zig"); -const BrailleChart = @import("braille_chart.zig"); -const TitledBorder = @import("titled_border.zig"); - -const MetricCard = @This(); - -title: []const u8, -title_image: ?vaxis.Image = null, -title_right: ?[]const u8 = null, -cell_size: vxfw.Size = .{}, -/// Lines of "label: value" pairs to display. -lines: []const LineEntry = &.{}, -/// Optional gauge bars. -gauges: []const GaugeEntry = &.{}, -/// Optional braille mini-charts. -charts: []const ChartEntry = &.{}, -/// Whether this card is highlighted (e.g. hovered). -highlighted: bool = false, - -pub const LineEntry = struct { - label: []const u8, - value: []const u8, - style: vaxis.Cell.Style = theme.value_style, -}; - -pub const GaugeEntry = struct { - label: []const u8, - value: u8, - suffix: ?[]const u8 = null, -}; - -pub const ChartEntry = struct { - label: []const u8, - value: u8, - suffix: ?[]const u8 = null, - data: []const u8, // history values 0-100 - chart_height: u16 = 3, -}; - -pub fn draw(self: *const MetricCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const bstyle: vaxis.Cell.Style = if (self.highlighted) - .{ .fg = theme.accent, .bold = true } - else - theme.border_style; - const tb: TitledBorder = .{ - .child = ui.drawWidget(self, drawContent), - .title = self.title, - .title_image = self.title_image, - .value_label = self.title_right, - .cell_size = self.cell_size, - .border_style = bstyle, - }; - return tb.draw(ctx); -} - -fn maxFieldWidth(ctx: vxfw.DrawContext, entries: anytype, comptime field: []const u8) u16 { - var max_w: u16 = 0; - for (entries) |entry| { - const raw = @field(entry, field); - const value: []const u8 = if (@typeInfo(@TypeOf(raw)) == .optional) raw orelse continue else raw; - max_w = @max(max_w, @as(u16, @intCast(ctx.stringWidth(value)))); - } - return max_w; -} - -fn drawContent(self: *const MetricCard, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const inner_w = ctx.max.width orelse 40; - - var sb = compose.surfaceBuilder(ctx.arena); - var row: i17 = 0; - - // Charts (gauge-like layout: label [braille] pct% suffix, braille spans multiple rows) - { - const max_chart_label_w = maxFieldWidth(ctx, self.charts, "label") + 1; // +1 trailing space - const max_chart_suffix_w = maxFieldWidth(ctx, self.charts, "suffix") + Gauge.sep_w; - - for (self.charts, 0..) |chart, ci| { - if (ci > 0) { - row += 1; - } - - const ch_h = chart.chart_height; - - const pct_str = try std.fmt.allocPrint(ctx.arena, "{d:>3}%", .{chart.value}); - const pct_w: u16 = @intCast(ctx.stringWidth(pct_str)); - - const reserved = max_chart_label_w + pct_w + 1 + max_chart_suffix_w; - const braille_w: u16 = if (reserved < inner_w) inner_w - reserved else 0; - const text_row: i17 = row + @as(i17, @intCast(ch_h / 2)); - - // Label (vertically centered, col 0) - const label_surf = try ui.drawRichLine(ctx, &.{ - .{ .text = chart.label, .style = theme.label_style }, - }, max_chart_label_w); - try sb.add(text_row, 0, label_surf); - - // Braille chart (all rows, after label) - if (braille_w > 0) { - const braille: BrailleChart = .{ .values = chart.data, .height = ch_h }; - const braille_surf = try braille.draw(ui.fixedSize(ctx, braille_w, ch_h)); - try sb.add(row, @intCast(max_chart_label_w), braille_surf); - } - - // Pct + suffix (vertically centered, after braille) - const right_col: i17 = @intCast(max_chart_label_w + braille_w); - const pct_surf = try ui.drawRichLine(ctx, &.{ - .{ .text = " ", .style = .{} }, - .{ .text = pct_str, .style = .{ .bold = true, .fg = theme.colorForPercent(chart.value) } }, - }, pct_w + 1); - try sb.add(text_row, right_col, pct_surf); - - if (chart.suffix) |suffix| { - const suffix_text_w: u16 = @intCast(ctx.stringWidth(suffix)); - const suffix_col: i17 = @intCast(inner_w -| suffix_text_w); - const suffix_surf = try ui.drawRichLine(ctx, &.{ - .{ .text = suffix, .style = theme.dim_style }, - }, suffix_text_w); - try sb.add(text_row, suffix_col, suffix_surf); - } - - row += @intCast(ch_h); - } - } - - // Gap between charts and gauges - if (self.charts.len > 0 and self.gauges.len > 0) { - row += 1; - } - - // Gauges (with 1-row gap between consecutive gauges) - const max_label_w = maxFieldWidth(ctx, self.gauges, "label") + 1; // +1 trailing space - const max_suffix_w = maxFieldWidth(ctx, self.gauges, "suffix") + Gauge.sep_w; - for (self.gauges, 0..) |ge, gi| { - const gauge: Gauge = .{ - .value = ge.value, - .label = ge.label, - .suffix = ge.suffix, - .label_reserve = max_label_w, - .suffix_reserve = max_suffix_w, - }; - const gauge_surf = try gauge.draw(ui.fixedSize(ctx, inner_w, 1)); - try sb.add(row, 0, gauge_surf); - row += 1; - if (gi + 1 < self.gauges.len) row += 1; // gap between gauges - } - - // Text lines - for (self.lines) |line| { - const text_surf = try ui.drawRichLine(ctx, &.{ - .{ .text = line.label, .style = theme.label_style }, - .{ .text = line.value, .style = line.style }, - }, inner_w); - try sb.add(row, 0, text_surf); - row += 1; - } - - const content_h: u16 = @intCast(@max(row, 1)); - return sb.finish(.{ .width = inner_w, .height = content_h }, ui.drawWidget(self, drawContent)); -} diff --git a/bin/zml-smi/tui/widgets/process_table.zig b/bin/zml-smi/tui/widgets/process_table.zig deleted file mode 100644 index 58b20d51..00000000 --- a/bin/zml-smi/tui/widgets/process_table.zig +++ /dev/null @@ -1,183 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const data = @import("../data.zig"); -const theme = @import("../theme.zig"); -const ui = @import("../lib/ui.zig"); -const compose = @import("../lib/compose.zig"); -const TitledBorder = @import("titled_border.zig"); -const ColumnLayout = @import("column_layout.zig"); -const pi = @import("zml-smi/info").process_info; -const ProcessInfo = pi.ProcessInfo; - -const RichText = vxfw.RichText; -const Segment = vaxis.Cell.Segment; - -const ProcessTable = @This(); - -const max_visible_rows: u16 = 20; - -scroll_bars: vxfw.ScrollBars = .{ - .scroll_view = .{ - .children = .{ .builder = .{ .userdata = undefined, .buildFn = buildRow } }, - .draw_cursor = false, - }, - .draw_horizontal_scrollbar = false, -}, -merged: std.ArrayList(ProcessInfo) = .empty, - -pub fn deinit(self: *ProcessTable, allocator: std.mem.Allocator) void { - self.merged.deinit(allocator); -} - -pub fn prepare(self: *ProcessTable, state: *data.SystemState, device_id: ?u16) void { - self.scroll_bars.scroll_view.children.builder.userdata = self; - self.merged.clearRetainingCapacity(); - - for (state.process_lists) |pl| { - const pl_items = pl.front().items; - if (device_id) |did| { - for (pl_items) |entry| { - if (entry.device_idx != did) { - continue; - } - self.merged.append(state.gpa, entry) catch break; - } - } else { - self.merged.appendSlice(state.gpa, pl_items) catch break; - } - } - - // Enrich with host data (uid, username, cpu%, rss, cmdline) - state.enricher.enrich(state.io, self.merged.items); - - // Sort by dev_mem descending - std.sort.insertion(ProcessInfo, self.merged.items, {}, struct { - fn cmp(_: void, a: ProcessInfo, b: ProcessInfo) bool { - return (a.dev_mem_kib orelse 0) > (b.dev_mem_kib orelse 0); - } - }.cmp); - - const count: u32 = @intCast(self.merged.items.len); - self.scroll_bars.estimated_content_height = count; - self.scroll_bars.scroll_view.item_count = count; -} - -pub fn resetScroll(self: *ProcessTable) void { - self.scroll_bars.scroll_view.scroll = .{}; - self.scroll_bars.scroll_view.cursor = 0; -} - -pub fn draw(self: *ProcessTable, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const w = ctx.max.width orelse 80; - const row_count = self.merged.items.len; - - var header_rt: RichText = .{ .text = &header_segs, .softwrap = false, .overflow = .clip }; - const empty_rt: RichText = .{ - .text = &.{.{ .text = " No process data available", .style = theme.dim_style }}, - .softwrap = false, - .overflow = .clip, - }; - - var children: std.ArrayList(vxfw.Widget) = .empty; - try children.append(ctx.arena, header_rt.widget()); - - if (row_count == 0) { - try children.append(ctx.arena, try compose.sized( - ctx.arena, - try compose.pad(ctx.arena, empty_rt.widget(), .{ .top = 1 }), - .{ .width = w, .height = 2 }, - )); - } else { - const scroll_h: u16 = @min(@as(u16, @intCast(row_count)), max_visible_rows); - try children.append(ctx.arena, try compose.sized(ctx.arena, self.scroll_bars.widget(), .{ .width = w, .height = scroll_h })); - } - - const content: ColumnLayout = .{ .children = children.items }; - const tb: TitledBorder = .{ - .child = ui.widget(&content), - .title = "Processes", - }; - return tb.draw(ui.fixedWidth(ctx, w)); -} - -/// Builds only data rows (no header) for the scroll view. -fn buildRow(ptr: *const anyopaque, idx: usize, _: usize) ?vxfw.Widget { - const self: *ProcessTable = @ptrCast(@alignCast(@constCast(ptr))); - if (idx >= self.merged.items.len) { - return null; - } - return .{ .userdata = @ptrCast(&self.merged.items[idx]), .drawFn = drawProcessRow }; -} - -// Column format strings — shared between header (comptime) and row (runtime). -const col_fmt = struct { - const pid = " {s: >7} "; - const user = "{s: <10} "; - const dev = "{s: <5} "; - const util = "{s: >6} "; - const mem = "{s: >8} "; - const cpu = "{s: >6} "; - const host_mem = "{s: >8} "; -}; - -const header_segs = blk: { - const s = theme.label_style; - break :blk [_]Segment{ - .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.pid, .{"PID"}) }, - .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.user, .{"USER"}) }, - .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.dev, .{"DEV"}) }, - .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.util, .{"UTIL"}) }, - .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.mem, .{"DEV MEM"}) }, - .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.cpu, .{"CPU"}) }, - .{ .style = s, .text = std.fmt.comptimePrint(col_fmt.host_mem, .{"HOST MEM"}) }, - .{ .style = s, .text = "Command" }, - }; -}; - -/// Format a value into a column: col(arena, col_fmt.pid, "{d}", .{pid}) -fn col(arena: std.mem.Allocator, comptime column: []const u8, comptime fmt: []const u8, args: anytype) std.mem.Allocator.Error![]const u8 { - return std.fmt.allocPrint(arena, column, .{try std.fmt.allocPrint(arena, fmt, args)}); -} - -fn drawProcessRow(ptr: *anyopaque, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const info: *const ProcessInfo = @ptrCast(@alignCast(ptr)); - const val: vaxis.Style = .{ .fg = theme.text_primary }; - const dim: vaxis.Style = .{ .fg = theme.dim }; - - const segs = try ctx.arena.alloc(Segment, 8); - segs[0] = .{ .style = val, .text = try col(ctx.arena, col_fmt.pid, "{d}", .{info.pid}) }; - segs[1] = .{ .style = val, .text = try col(ctx.arena, col_fmt.user, "{s}", .{info.username[0..@min(info.username.len, 10)]}) }; - segs[2] = .{ .style = val, .text = try col(ctx.arena, col_fmt.dev, "{d}", .{info.device_idx}) }; - segs[3] = if (info.dev_util_percent) |util| - .{ .style = val, .text = try col(ctx.arena, col_fmt.util, "{d}%", .{util}) } - else - .{ .style = dim, .text = try col(ctx.arena, col_fmt.util, "-", .{}) }; - segs[4] = if (info.dev_mem_kib) |mem| - .{ .style = val, .text = try col(ctx.arena, col_fmt.mem, "{s}", .{try fmtMem(ctx.arena, mem)}) } - else - .{ .style = dim, .text = try col(ctx.arena, col_fmt.mem, "-", .{}) }; - segs[5] = .{ .style = val, .text = try col(ctx.arena, col_fmt.cpu, "{d}.{d}%", .{ info.cpu_percent / 10, info.cpu_percent % 10 }) }; - segs[6] = .{ .style = val, .text = try col(ctx.arena, col_fmt.host_mem, "{s}", .{try fmtMem(ctx.arena, info.rss_kib)}) }; - segs[7] = .{ .style = val, .text = info.comm }; - - const rich: RichText = .{ .text = segs, .softwrap = false, .overflow = .clip }; - return rich.draw(ctx.withConstraints( - .{ .width = ctx.min.width, .height = 1 }, - .{ .width = ctx.max.width, .height = 1 }, - )); -} - -fn fmtMem(arena: std.mem.Allocator, kib: u64) std.mem.Allocator.Error![]const u8 { - if (kib >= 1024 * 1024) { - const whole = kib / (1024 * 1024); - const frac = (kib % (1024 * 1024)) * 10 / (1024 * 1024); - return std.fmt.allocPrint(arena, "{d}.{d}G", .{ whole, frac }); - } else if (kib >= 1024) { - const whole = kib / 1024; - const frac = (kib % 1024) * 10 / 1024; - return std.fmt.allocPrint(arena, "{d}.{d}M", .{ whole, frac }); - } else { - return std.fmt.allocPrint(arena, "{d}K", .{kib}); - } -} diff --git a/bin/zml-smi/tui/widgets/status_line.zig b/bin/zml-smi/tui/widgets/status_line.zig deleted file mode 100644 index fb32541e..00000000 --- a/bin/zml-smi/tui/widgets/status_line.zig +++ /dev/null @@ -1,77 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const theme = @import("../theme.zig"); -const ui = @import("../lib/ui.zig"); - -const StatusLine = @This(); - -viewing_device: ?u16, -use_braille: bool, - -const Binding = struct { - key: []const u8, - desc: []const u8, -}; - -pub fn draw(self: *const StatusLine, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const w = ctx.max.width orelse 80; - - const key_style: vaxis.Style = .{ .bold = true, .fg = .{ .rgb = .{ 30, 30, 40 } }, .bg = theme.accent }; - const desc_style: vaxis.Style = .{ .fg = theme.text_secondary }; - const sep_style: vaxis.Style = .{ .fg = theme.dim }; - const bg_style: vaxis.Style = .{ .bg = .{ .rgb = .{ 30, 30, 45 } } }; - - const chart_label = if (self.use_braille) "gauges" else "charts"; - - var bindings_buf: [8]Binding = undefined; - var n: usize = 0; - - if (self.viewing_device != null) { - bindings_buf[n] = .{ .key = "Esc", .desc = "back" }; - n += 1; - } else { - bindings_buf[n] = .{ .key = "v", .desc = chart_label }; - n += 1; - bindings_buf[n] = .{ .key = "click", .desc = "device details" }; - n += 1; - } - bindings_buf[n] = .{ .key = "q", .desc = "quit" }; - n += 1; - - const bindings = bindings_buf[0..n]; - - // Build segments: " key desc | key desc | ... " - var segments: std.ArrayList(vaxis.Cell.Segment) = .empty; - - // Leading space - try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); - - for (bindings, 0..) |b, i| { - try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); - try segments.append(ctx.arena, .{ .text = b.key, .style = key_style }); - try segments.append(ctx.arena, .{ .text = " ", .style = bg_style }); - try segments.append(ctx.arena, .{ .text = b.desc, .style = desc_style }); - if (i + 1 < bindings.len) { - try segments.append(ctx.arena, .{ .text = " \u{2502}", .style = sep_style }); - } - } - - const rich: vxfw.RichText = .{ - .text = segments.items, - .softwrap = false, - .overflow = .clip, - }; - const surf = try rich.draw(ui.fixedSize(ctx, w, 1)); - - // Fill background across full width - if (surf.buffer.len > 0) { - for (surf.buffer) |*cell| { - if (cell.style.bg == .default) { - cell.style.bg = bg_style.bg; - } - } - } - - return surf; -} diff --git a/bin/zml-smi/tui/widgets/titled_border.zig b/bin/zml-smi/tui/widgets/titled_border.zig deleted file mode 100644 index d9c16171..00000000 --- a/bin/zml-smi/tui/widgets/titled_border.zig +++ /dev/null @@ -1,112 +0,0 @@ -const std = @import("std"); -const vaxis = @import("vaxis"); -const vxfw = vaxis.vxfw; -const theme = @import("../theme.zig"); -const ui = @import("../lib/ui.zig"); -const Image = @import("../lib/image.zig"); - -const Text = vxfw.Text; - -const TitledBorder = @This(); - -child: vxfw.Widget, -title: []const u8 = "", -title_style: vaxis.Cell.Style = theme.title_style, -title_image: ?vaxis.Image = null, -cell_size: vxfw.Size = .{}, -value_label: ?[]const u8 = null, -value_label_style: vaxis.Cell.Style = theme.value_style, -border_style: vaxis.Cell.Style = theme.border_style, -padding: vxfw.Padding.PadValues = .{ .left = 1, .right = 1, .top = 1, .bottom = 1 }, - -pub fn draw(self: *const TitledBorder, ctx: vxfw.DrawContext) std.mem.Allocator.Error!vxfw.Surface { - const pad: vxfw.Padding = .{ - .child = self.child, - .padding = self.padding, - }; - const border: vxfw.Border = .{ - .child = pad.widget(), - .style = self.border_style, - }; - var surf = try border.draw(ctx); - - // Count overlay children needed - var overlay_count: usize = 0; - var title_col: u16 = 2; - var has_image = false; - - if (self.title_image) |img| { - const icon = ui.imageCellSize(img, 1, self.cell_size); - has_image = true; - title_col += icon.cols; - overlay_count += 1; - } - if (self.title.len > 0) { - overlay_count += 1; - } - if (self.value_label != null) { - overlay_count += 1; - } - - if (overlay_count == 0) { - return surf; - } - - const new_children = try ctx.arena.alloc(vxfw.SubSurface, surf.children.len + overlay_count); - @memcpy(new_children[0..surf.children.len], surf.children); - var idx = surf.children.len; - - // Image icon overlay - if (has_image) { - const img = self.title_image.?; - const img_widget: Image = .{ .image = img, .rows = 1 }; - const img_surf = try img_widget.draw(ctx); - new_children[idx] = .{ - .origin = .{ .row = 0, .col = 2 }, - .surface = img_surf, - .z_index = 1, - }; - idx += 1; - } - - // Title text overlay (top-left) - if (self.title.len > 0) { - const title_text = try std.fmt.allocPrint(ctx.arena, " {s} ", .{self.title}); - const title_widget: Text = .{ - .text = title_text, - .style = self.title_style, - .softwrap = false, - .overflow = .clip, - }; - const max_title_w = surf.size.width -| (title_col + 1); - const title_surf = try title_widget.draw(ui.maxSize(ctx, max_title_w, 1)); - new_children[idx] = .{ - .origin = .{ .row = 0, .col = @intCast(title_col) }, - .surface = title_surf, - .z_index = 1, - }; - idx += 1; - } - - // Value label overlay (top-right) - if (self.value_label) |vlabel| { - const label_text = try std.fmt.allocPrint(ctx.arena, " {s} ", .{vlabel}); - const label_widget: Text = .{ - .text = label_text, - .style = self.value_label_style, - .softwrap = false, - .overflow = .clip, - }; - const right = surf.size.width -| 1; - const lw: u16 = @intCast(@min(ctx.stringWidth(label_text), right -| 2)); - const label_surf = try label_widget.draw(ui.maxSize(ctx, lw, 1)); - new_children[idx] = .{ - .origin = .{ .row = 0, .col = @intCast(right -| lw) }, - .surface = label_surf, - .z_index = 1, - }; - } - - surf.children = new_children; - return surf; -} diff --git a/bin/zml-smi/utils/BUILD.bazel b/bin/zml-smi/utils/BUILD.bazel deleted file mode 100644 index a154556c..00000000 --- a/bin/zml-smi/utils/BUILD.bazel +++ /dev/null @@ -1,30 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "double_buffer", - import_name = "zml-smi/double_buffer", - main = "double_buffer.zig", - visibility = ["//visibility:public"], -) - -zig_library( - name = "dynlib", - import_name = "zml-smi/dynlib", - main = "dynlib.zig", - visibility = ["//visibility:public"], -) - -zig_library( - name = "sandbox", - import_name = "zml-smi/sandbox", - main = "sandbox.zig", - visibility = ["//visibility:public"], - deps = ["@rules_zig//zig/runfiles"], -) - -zig_library( - name = "sysfs", - import_name = "zml-smi/sysfs", - main = "sysfs.zig", - visibility = ["//visibility:public"], -) diff --git a/bin/zml-smi/utils/double_buffer.zig b/bin/zml-smi/utils/double_buffer.zig deleted file mode 100644 index add3037b..00000000 --- a/bin/zml-smi/utils/double_buffer.zig +++ /dev/null @@ -1,32 +0,0 @@ -const std = @import("std"); - -pub fn DoubleBuffer(comptime T: type) type { - return struct { - const Self = @This(); - pub const Value = T; - - values: [2]T, - current: std.atomic.Value(u8) = std.atomic.Value(u8).init(0), - - pub fn front(self: *const Self) *const T { - return &self.values[self.current.load(.acquire)]; - } - - pub fn back(self: *Self) *T { - return &self.values[1 - self.current.load(.acquire)]; - } - - pub fn swap(self: *Self) void { - self.current.store(1 - self.current.load(.acquire), .release); - } - - pub fn jsonStringify(self: *const Self, jw: *std.json.Stringify) !void { - try jw.write(self.front().*); - } - - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !Self { - const val = try std.json.innerParseFromValue(T, allocator, source, options); - return .{ .values = .{ val, val } }; - } - }; -} diff --git a/bin/zml-smi/utils/dynlib.zig b/bin/zml-smi/utils/dynlib.zig deleted file mode 100644 index 4a478107..00000000 --- a/bin/zml-smi/utils/dynlib.zig +++ /dev/null @@ -1,10 +0,0 @@ -const std = @import("std"); - -pub fn lookupStruct(dynlib: *std.DynLib, comptime VTable: type) !VTable { - var result: VTable = undefined; - inline for (comptime std.meta.fieldNames(VTable), comptime std.meta.fieldTypes(VTable)) |field_name, Field| { - @field(result, field_name) = dynlib.lookup(Field, field_name) orelse return error.SymbolResolutionFailed; - } - - return result; -} diff --git a/bin/zml-smi/utils/sandbox.zig b/bin/zml-smi/utils/sandbox.zig deleted file mode 100644 index d4e268bd..00000000 --- a/bin/zml-smi/utils/sandbox.zig +++ /dev/null @@ -1,44 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const bazel_runfiles = @import("runfiles"); -const bazel_builtin = @import("bazel_builtin"); - -var runfiles_global: bazel_runfiles.Runfiles = undefined; - -const init_array_section = switch (builtin.object_format) { - .macho => "__DATA,__mod_init_func", - .elf => ".init_array", - else => "", -}; - -export const _ align(@alignOf(usize)) linksection(init_array_section) = &struct { - fn call(argc: c_int, argv: [*c][*:0]u8, envp: [*:null]?[*:0]u8) callconv(.c) void { - _ = argc; - _ = argv; - _ = envp; - var threaded: std.Io.Threaded = .init_single_threaded; - const io = threaded.io(); - - var exe_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const exe_size = std.process.executablePath(io, &exe_path_buf) catch - std.debug.panic("Unable to get executable path", .{}); - - runfiles_global = bazel_runfiles.Runfiles.create(.{ - .allocator = std.heap.c_allocator, - .io = io, - .argv0 = exe_path_buf[0..exe_size], - .directory = if (std.c.getenv("RUNFILES_DIR")) |directory| std.mem.span(directory) else null, - .manifest = if (std.c.getenv("RUNFILES_MANIFEST_FILE")) |manifest| std.mem.span(manifest) else null, - }) catch { - std.debug.panic("Unable to find runfiles", .{}); - } orelse { - std.debug.panic("Unable to initialize runfiles", .{}); - }; - } -}.call; - -pub fn path(buf: *[std.Io.Dir.max_path_bytes]u8) ?[]const u8 { - const with_repo = runfiles_global.withSourceRepo(bazel_builtin.current_repository); - - return with_repo.rlocation("zml/bin/zml-smi/sandbox", buf) catch null; -} diff --git a/bin/zml-smi/utils/sysfs.zig b/bin/zml-smi/utils/sysfs.zig deleted file mode 100644 index 636883b2..00000000 --- a/bin/zml-smi/utils/sysfs.zig +++ /dev/null @@ -1,70 +0,0 @@ -const std = @import("std"); - -pub fn readInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !u64 { - const data = try readFirstLine(allocator, io, path); - return std.fmt.parseInt(u64, data, 10); -} - -pub fn readString(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]const u8 { - return readFirstLine(allocator, io, path); -} - -pub fn readFieldInt(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) !u64 { - return nthTokenInt(try findField(allocator, io, path, key), 0); -} - -pub fn nthTokenInt(data: []const u8, n: usize) u64 { - var i: usize = 0; - - var iter = std.mem.tokenizeAny(u8, data, " \t"); - while (iter.next()) |tok| : (i += 1) { - if (i == n) { - return std.fmt.parseInt(u64, tok, 10) catch 0; - } - } - - return 0; -} - -pub fn readFieldString(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { - const value = try findField(allocator, io, path, key); - return std.mem.trimEnd(u8, value, &std.ascii.whitespace); -} - -fn findField(allocator: std.mem.Allocator, io: std.Io, path: []const u8, comptime key: []const u8) ![]const u8 { - var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); - defer file.close(io); - - var read_buf: [4096]u8 = undefined; - var reader = file.reader(io, &read_buf); - - var line_writer: std.Io.Writer.Allocating = .init(allocator); - errdefer line_writer.deinit(); - - while (true) { - line_writer.clearRetainingCapacity(); - _ = reader.interface.streamDelimiter(&line_writer.writer, '\n') catch return error.NotFound; - reader.interface.toss(1); - - const line = line_writer.written(); - if (std.mem.startsWith(u8, line, key)) { - const owned = line_writer.toOwnedSlice() catch return error.NotFound; - return std.mem.trimStart(u8, owned[key.len..], " :\t"); - } - } -} - -pub fn readFirstLine(allocator: std.mem.Allocator, io: std.Io, path: []const u8) ![]u8 { - var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); - defer file.close(io); - - var read_buf: [4096]u8 = undefined; - var reader = file.reader(io, &read_buf); - var writer: std.Io.Writer.Allocating = .init(allocator); - errdefer writer.deinit(); - _ = reader.interface.streamDelimiter(&writer.writer, '\n') catch |err| switch (err) { - error.EndOfStream => 0, - else => |e| return e, - }; - return writer.toOwnedSlice(); -} diff --git a/devenv.lock b/devenv.lock deleted file mode 100644 index 6c3be332..00000000 --- a/devenv.lock +++ /dev/null @@ -1,65 +0,0 @@ -{ - "nodes": { - "devenv": { - "locked": { - "dir": "src/modules", - "lastModified": 1793199252, - "narHash": "sha256-LRgdAdMHL3uIX+kix/oyMzG+gMxzSYOzLrrRaoZthog=", - "owner": "cachix", - "repo": "devenv", - "rev": "0d6c556e5f862c14a75a543e2ee4fdb7acc005ad", - "type": "github" - }, - "original": { - "dir": "src/modules", - "owner": "cachix", - "repo": "devenv", - "type": "github" - } - }, - "nixpkgs": { - "inputs": { - "nixpkgs-src": "nixpkgs-src" - }, - "locked": { - "lastModified": 1793043970, - "narHash": "sha256-m14odYTdkBUEkJOiEYs2zjGjn8fcjvdCJsm45+k084U=", - "owner": "cachix", - "repo": "devenv-nixpkgs", - "rev": "d97234e58d4e90e3bf77c082fc3b546e51200938", - "type": "github" - }, - "original": { - "owner": "cachix", - "ref": "rolling", - "repo": "devenv-nixpkgs", - "type": "github" - } - }, - "nixpkgs-src": { - "flake": false, - "locked": { - "lastModified": 1792812153, - "narHash": "sha256-T2+XBfNbf9pecLUWS4+RtuVNZ3W8K5taG5QdV71mWlk=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "674a2028661e550a97400c1cc992b9cca9a668a1", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "devenv": "devenv", - "nixpkgs": "nixpkgs" - } - } - }, - "root": "root", - "version": 7 -} \ No newline at end of file diff --git a/devenv.nix b/devenv.nix deleted file mode 100644 index cafd0f90..00000000 --- a/devenv.nix +++ /dev/null @@ -1,20 +0,0 @@ -{ - pkgs, - config, - ... -}: { - packages = with pkgs; [ - bazelisk - python3 - uv - zig - ]; - env.ZLS_CMD = "${config.devenv.root}/tools/zls.sh"; - scripts.bazel.exec = ''bazelisk "$@"''; - - enterShell = pkgs.lib.optionalString pkgs.stdenv.isDarwin '' - # Use the "real" XCode rather than Nixpkg's one. This does assume you have it installed. - export DEVELOPER_DIR="/Applications/Xcode.app/Contents/Developer" - export SDKROOT="/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk"; - ''; -} diff --git a/devenv.yaml b/devenv.yaml deleted file mode 100644 index 116a2adb..00000000 --- a/devenv.yaml +++ /dev/null @@ -1,15 +0,0 @@ -# yaml-language-server: $schema=https://devenv.sh/devenv.schema.json -inputs: - nixpkgs: - url: github:cachix/devenv-nixpkgs/rolling - -# If you're using non-OSS software, you can set allowUnfree to true. -# allowUnfree: true - -# If you're willing to use a package that's vulnerable -# permittedInsecurePackages: -# - "openssl-1.1.1w" - -# If you have more than one devenv you can merge them -#imports: -# - ./backend diff --git a/docs/README.md b/docs/README.md index 03369120..eedfdd9b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,9 +17,9 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## How to ? - [HuggingFace Authentication](./howtos/huggingface_access_token.md) - [Port Pytorch models to ZML](./howtos/howto_torch2zml.md) +- [Add Weights Files to your projects](./howtos/add_weights.md) - [Cross Compile and Deploy Models on a Server](./howtos/deploy_on_server.md) - [Dockerize Models](./howtos/dockerize_models.md) -- [Profile ZML Programs](./howtos/profiling.md) ## Learn more... @@ -28,3 +28,4 @@ Ready to write some code? Try starting with [your first model in ZML](./tutorial ## Contribute - [Style Guide](./misc/style_guide.md) + diff --git a/docs/howtos/add_weights.md b/docs/howtos/add_weights.md new file mode 100644 index 00000000..2caf2fc9 --- /dev/null +++ b/docs/howtos/add_weights.md @@ -0,0 +1,69 @@ + +# Adding Weights Files + +Our [first model](../tutorials/write_first_model.md) did not need any weights files. +We just created weights and biases at runtime. + +But real-world models typically need weights files, and maybe some other +supporting files. + +We recommend, for easy deployments, you upload those files. In many instances, +you will use a site like [🤗 Hugging Face](https://huggingface.co). + +We also recommend to add a `weights.bzl` file to your project root directory, so +you don't "pollute" your build file with long URLs and SHAs: + +```python +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def _weights_impl(mctx): + http_archive( + name = "mnist", + sha256 = "075905e433ea0cce13c3fc08832448ab86225d089b5d412be67f59c29388fb19", + url = "https://mirror.zml.ai/data/mnist.tar.zst", + build_file_content = """exports_files(glob(["**"]), visibility = ["//visibility:public"])""", + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +weights = module_extension( + implementation = _weights_impl, +) +``` + +The above `weights.bzl` shows how we load files for MNIST: + +- `mnist.pt` (model weights) +- `mnist.ylc` (dataset for picking sample images) + +Then, in your `BUILD.bazel`, you can refer to the files you defined above, in +the following way: + +```python +zig_binary( + name = "mnist", + args = [ + "$(location @mnist//:mnist.pt)", + "$(location @mnist//:t10k-images.idx3-ubyte)", + ], + data = [ + "@mnist//:mnist.pt", + "@mnist//:t10k-images.idx3-ubyte", + ], + main = "mnist.zig", + deps = [ + "//async", + "//zml", + ], +) +``` + +See how: + +- we use `data = [ ... ]` to reference the files in `weights.bzl` +- we use `args = [ ... ]` to pass the files as command-line arguments to the + MNIST executable at runtime, automatically. diff --git a/docs/howtos/deploy_on_server.md b/docs/howtos/deploy_on_server.md index a8a6452e..05a88889 100644 --- a/docs/howtos/deploy_on_server.md +++ b/docs/howtos/deploy_on_server.md @@ -24,11 +24,7 @@ So, to run the Llama model from above **on your development machine** housing an NVIDIA GPU, run the following: ``` -bazel run --config=release //examples/llm \ - --@zml//platforms:cuda=true \ - -- \ - --model=hf://meta-llama/Llama-3.2-1B-Instruct \ - --prompt="What is the capital of France?" +bazel run --config=release //examples/llama --@zml//platforms:cuda=true -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct ``` @@ -44,7 +40,7 @@ architectures: As an example, here is how you build above Llama for CUDA on Linux X86_64: ``` -bazel build --config=release //examples/llm \ +bazel build --config=release //examples/llama \ --@zml//platforms:cuda=true \ --@zml//platforms:cpu=false \ --platforms=@zml//platforms:linux_amd64 @@ -61,8 +57,7 @@ If not present already, add an "archive" target to the model's `BUILD.bazel`, like this: ```python -load("@tar.bzl//tar:mtree.bzl", "mtree_spec") -load("@tar.bzl//tar:tar.bzl", "tar") +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") # Manifest, required for building the tar archive mtree_spec( @@ -86,15 +81,15 @@ tar( ... and then build the TAR archive: ``` -bazel build --config=release //examples/mnist:archive \ +bazel build --config=release //mnist:archive \ --@zml//platforms:cuda=true \ --@zml//platforms:cpu=false \ --platforms=@zml//platforms:linux_amd64 ``` -Note the `//examples/mnist:archive` notation. +Note the `//mnist:archive` notation. -The resulting tar file will be in `bazel-bin/examples/mnist/archive.tar.zst`. +The resulting tar file will be in `bazel-bin/mnist/archive.tar.zst`. ### Run it on the server @@ -103,35 +98,36 @@ and run it: ```bash # on your machine -scp bazel-bin/examples/mnist/archive.tar.zst destination-server: +scp bazel-bin/mnist/archive.tar.zst destination-server: ssh destination-server # to enter the server # ... on the server tar xvf archive.tar.zst -./examples/mnist/mnist \ - $PWD/examples/mnist/mnist.runfiles/+non_module_deps+mnist/mnist.safetensors \ - $PWD/examples/mnist/mnist.runfiles/+non_module_deps+mnist/t10k-images.idx3-ubyte +./mnist \ + 'mnist.runfiles/_main~_repo_rules~com_github_ggerganov_ggml_mnist/file/mnist.pt' \ + 'mnist.runfiles/_main~_repo_rules~com_github_ggerganov_ggml_mnist_data/file/mnist.ylc' ``` The easiest way to figure out the commandline arguments of an example model is to consult the model's `BUILD.bazel` and check out its `args` section. It will -reference e.g. weights files that are defined in the same `BUILD.bazel`. +reference e.g. weights files that are defined either in the same `BUILD.bazel` +file or in a `weights.bzl` file. You can also consult the console output when running your model locally: ```bash -bazel run //examples/mnist +bazel run //mnist -INFO: Analyzed target //examples/mnist:mnist (0 packages loaded, 0 targets configured). +INFO: Analyzed target //mnist:mnist (0 packages loaded, 0 targets configured). INFO: Found 1 target... -Target //examples/mnist:mnist up-to-date: - bazel-bin/examples/mnist/mnist -INFO: Elapsed time: 0.114s, Critical Path: 0.00s -INFO: 1 process: 1 internal. -INFO: Build completed successfully, 1 total action -INFO: Running command line: bazel-bin/examples/mnist/mnist ../+non_module_deps+mnist/mnist.safetensors ../+non_module_deps+mnist/t10k-images.idx3-ubyte. -# .. +Target //mnist:mnist up-to-date: + bazel-bin/mnist/mnist +INFO: Elapsed time: 0.302s, Critical Path: 0.00s +INFO: 3 processes: 3 internal. +INFO: Build completed successfully, 3 total actions +INFO: Running command line: bazel-bin/mnist/mnist ../_main~_repo_rules~com_github_ggerganov_ggml_mnist/file/mnist.pt ../_main~_repo_rules~com_github_ggerganov_ggml_mnist_data/file/mnist.ylc +# ... ``` You see the command line right up there. On the server, you just need to replace -`../` with the absolute path to the file. +`../` with the 'runfiles' directory of your TAR. diff --git a/docs/howtos/dockerize_models.md b/docs/howtos/dockerize_models.md index 7414b985..ac35990c 100644 --- a/docs/howtos/dockerize_models.md +++ b/docs/howtos/dockerize_models.md @@ -38,16 +38,16 @@ We need to add a few "imports" at the beginning of our `BUILD.bazel` so we can use their rules to define our 5 additional targets: ```python -load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_zig//zig:defs.bzl", "zig_binary") -load("@tar.bzl//tar:mtree.bzl", "mtree_spec") -load("@tar.bzl//tar:tar.bzl", "tar") zig_binary( name = "simple_layer", main = "main.zig", deps = [ + "@zml//async", "@zml//zml", ], ) @@ -135,11 +135,11 @@ And that's almost it! You can already build the image: ``` bazel build --config=release //examples/simple_layer:image -INFO: Analyzed target //examples/simple_layer:image (0 packages loaded, 0 targets configured). +INFO: Analyzed target //simple_layer:image (1 packages loaded, 8 targets configured). INFO: Found 1 target... -Target //examples/simple_layer:image up-to-date: - bazel-out/linux_amd64-opt/bin/examples/simple_layer/image_ -INFO: Elapsed time: 0.285s, Critical Path: 0.00s +Target //simple_layer:image up-to-date: + bazel-out/k8-dbg-ST-f832ad0148ae/bin/simple_layer/image_ +INFO: Elapsed time: 0.279s, Critical Path: 0.00s INFO: 1 process: 1 internal. INFO: Build completed successfully, 1 total action ``` @@ -168,7 +168,7 @@ oci_load( ... then we can load the image and run it with the following commands: ``` -bazel run --config=release //examples/simple_layer:load +bazel run --config=release //simple_layer:load docker run --rm distroless/simple_layer:latest ``` @@ -193,7 +193,7 @@ This will push the `simple_layer` image with the tag `latest` (you can add more) to the docker registry: ``` -bazel run --config=release //examples/simple_layer:push +bazel run --config=release //simple_layer:push ``` When dealing with maybe a public and a private container registry - or if you @@ -201,7 +201,7 @@ just want to try it out **right now**, you can always override the repository on the command line: ``` -bazel run --config=release //examples/simple_layer:push -- --repository my.server.com/org/image +bazel run --config=release //simple_layer:push -- --repository my.server.com/org/image ``` @@ -215,8 +215,8 @@ We'll use the [MNIST example](https://github.com/zml/zml/tree/master/examples/mnist) to illustrate how to build Docker images that also contain data files. -You can `bazel run --config=release //examples/mnist:push -- --repository -index.docker.io/my_org/zml_mnist` if you want to try +You can `bazel run --config=release //mnist:push -- --repository +index.docker.io/my_org/zml_mnist` in the `./examples` folder if you want to try it out. **Note: Please add one more of the following parameters to specify all the @@ -231,7 +231,7 @@ platforms your containerized model should support.** **Example:** ``` -bazel run //examples/mnist:push --config=release --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist +bazel run //mnist:push --config=release --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist ``` @@ -244,27 +244,26 @@ same. Let's start with creating the manifest and archive: ```python -load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@aspect_bazel_lib//lib:expand_template.bzl", "expand_template") +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") load("@rules_zig//zig:defs.bzl", "zig_binary") -load("@tar.bzl//tar:mtree.bzl", "mtree_spec") -load("@tar.bzl//tar:tar.bzl", "tar") # The executable zig_binary( name = "mnist", args = [ - "$(location @mnist//:mnist.safetensors)", - "$(location @mnist//:t10k-images.idx3-ubyte)", + "$(location @com_github_ggerganov_ggml_mnist//file)", + "$(location @com_github_ggerganov_ggml_mnist_data//file)", ], data = [ - "@mnist//:mnist.safetensors", - "@mnist//:t10k-images.idx3-ubyte", + "@com_github_ggerganov_ggml_mnist//file", + "@com_github_ggerganov_ggml_mnist_data//file", ], main = "mnist.zig", - visibility = ["//visibility:public"], deps = [ - "@zml//bazel", + "@zml//async", "@zml//zml", ], ) @@ -297,24 +296,22 @@ simple string interpolation will not be enough. For this reason, we use the `expand_template` rule, like this: ```python -load("@bazel_lib//lib:expand_template.bzl", "expand_template") - # A convenience template for creating the "command line" for the entrypoint expand_template( name = "entrypoint", data = [ ":mnist", - "@mnist//:mnist.safetensors", - "@mnist//:t10k-images.idx3-ubyte", + "@com_github_ggerganov_ggml_mnist//file", + "@com_github_ggerganov_ggml_mnist_data//file", ], substitutions = { - ":model": "$(rlocationpath @mnist//:mnist.safetensors)", - ":data": "$(rlocationpath @mnist//:t10k-images.idx3-ubyte)", + ":model": "$(rlocationpath @com_github_ggerganov_ggml_mnist//file)", + ":data": "$(rlocationpath @com_github_ggerganov_ggml_mnist_data//file)", }, template = [ "./{}/mnist".format(package_name()), - "/{}/mnist.runfiles/:model".format(package_name()), - "/{}/mnist.runfiles/:data".format(package_name()), + "./{}/mnist.runfiles/:model".format(package_name()), + "./{}/mnist.runfiles/:data".format(package_name()), ], ) ``` @@ -375,5 +372,5 @@ And that's it! With one simple bazel command, you can push a neatly packaged MNIST model, including weights and dataset, to the docker registry: ``` -bazel run //examples/mnist:push --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist +bazel run //mnist:push --@zml//platforms:cuda=true -- --repository index.docker.io/my_org/zml_mnist ``` diff --git a/docs/howtos/howto_torch2zml.md b/docs/howtos/howto_torch2zml.md index 3d246f11..5c751c53 100644 --- a/docs/howtos/howto_torch2zml.md +++ b/docs/howtos/howto_torch2zml.md @@ -67,15 +67,14 @@ package, so the simplest way to use it, is to copy it next to your python script. Then wrap the model/pipeline in a `zml_utils.ActivationCollector`. The collector wraps the given model, and returns the original results AND the activations in a dict of `torch.Tensor` when it's being called. After that, you -can save those activations to a `.safetensors` file. +can save those activations to a `.pt` file. ```python import torch import transformers -from safetensors.torch import save_file import zml_utils -model_path = "meta-llama/Llama-3.2-1B-Instruct" +model_path = "meta-llama/Meta-Llama-3-8B" pipeline = transformers.pipeline( "text-generation", @@ -98,8 +97,8 @@ if output: print(output) # Save activations to a file. -filename = model_path.split("/")[-1] + ".activations.safetensors" -save_file(activations, filename) +filename = model_path.split("/")[-1] + ".activations.pt" +torch.save(activations, filename) print(f"Saved {len(activations)} activations to {filename}") ``` @@ -119,27 +118,36 @@ model. Put the following in `my_project/torch2zml.zig`. const std = @import("std"); const log = std.log; +const async = @import("async"); const zml = @import("zml"); -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - const io = init.io; +pub fn main() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + try async.AsyncThread.main(gpa.allocator(), asyncMain, .{}); +} + +pub fn asyncMain() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + const args = try std.process.argsAlloc(allocator); + defer std.process.argsFree(allocator, args); - const args = try init.minimal.args.toSlice(allocator); - defer allocator.free(args); const model_path, const activations_path = args[1..3].*; - var activations_registry = try zml.safetensors.TensorRegistry.fromPath(allocator, io, activations_path); - defer activations_registry.deinit(); - log.info("Found {} activations in {s}", .{ activations_registry.tensors.count(), activations_path }); + const activations = try zml.aio.torch.open(allocator, activations_path); + defer activations.deinit(); + log.info("Found {} activations in {s}", .{ activations.buffers.count(), activations_path }); - var model_registry = try zml.safetensors.TensorRegistry.fromPath(allocator, io, model_path); - defer model_registry.deinit(); - log.info("Found {} activations in {s}", .{ model_registry.tensors.count(), model_path }); + const model_weights = try zml.aio.detectFormatAndOpen(allocator, model_path); + defer model_weights.deinit(); + log.info("Found {} model layers in {s}", .{ model_weights.buffers.count(), activations_path }); } ``` -And add a `zig_binary` target in `my_project/BUILD.bazel`: +And add a `zig_cc_binary` target in `my_project/BUILD.bazel`: ```python load("@rules_zig//zig:defs.bzl", "zig_binary") @@ -148,6 +156,7 @@ zig_binary( name = "torch2zml", main = "torch2zml.zig", deps = [ + "@zml//async", "@zml//zml", ], ) @@ -157,16 +166,19 @@ Now check that the weights can be loaded correctly using the bazel CLI. ```bash bazel build //my_project:torch2zml -./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Llama-3.2-1B-Instruct.activations.safetensors +./bazel-bin/my_project/torch2zml /path/to/my/model.safetensors.index.json ./my_project/Meta-Llama-3-8B.activations.pt -info: Found 564 activations in /Users/corendos/dev/zml/python/temp/Llama-3.2-1B-Instruct.activations.safetensors -debug(zml/safetensors): Added new metadata key=format with value=pt -info: Found 146 activations in /Users/corendos/models/Llama-3.2-1B-Instruct/model.safetensors +info: Found 1108 activations in /Users/guw/Documents/zml/models/torch2zml/Meta-Llama-3-8B.activations.pt +debug(zml_io): Loading shard: model-00004-of-00004.safetensors +debug(zml_io): Loading shard: model-00001-of-00004.safetensors +debug(zml_io): Loading shard: model-00002-of-00004.safetensors +debug(zml_io): Loading shard: model-00003-of-00004.safetensors +info: Found 291 model layers in /Users/guw/Documents/zml/models/torch2zml/Meta-Llama-3-8B.activations.pt ``` -## Initializing an individual layer +## Loading an individual layer -In the above Zig code, the `model_registry` struct is a wrapper around a flat +In the above Zig code, the `model_weights` struct is a wrapper around a flat dictionary, containing an entry for each tensor in the model (similar to a "state dict"). Manipulating a dictionary is generally not very convenient, so let's convert it to a Zig struct. @@ -184,70 +196,26 @@ const Mlp = struct { The `zml.nn.Linear` is the equivalent of `torch.nn.Linear` and is defined by its `weight` and optional `bias` tensors. -To create such a struct from our `model_registry` dictionary, we first need to -convert it to a `TensorStore`: +To create such a struct from our `model_weights` dictionary, we can use the +`zml.aio.populateModelWithPrefix` helper: ```zig -pub fn main(init: std.process.Init) !void { +pub fn asyncMain() !void { ... - var model_store: zml.io.TensorStore = .fromRegistry(allocator, &model_registry, &.{}); - defer model_store.deinit(); -} -``` - -Then, we can initialize our MLP: - -```zig -pub fn main(init: std.process.Init) !void { - ... - - const mlp_view = model_store.view().withPrefix("model.layers.0.mlp"); - - const mlp: Mlp = .{ - .up_proj = .init( - mlp_view.createReplicatedTensor("up_proj.weight", .{ .dout, .d }), - mlp_view.maybeCreateReplicatedTensor("up_proj.bias", .{.dout}), - .d, - ), - .gate_proj = .init( - mlp_view.createReplicatedTensor("gate_proj.weight", .{ .dout, .d }), - mlp_view.maybeCreateReplicatedTensor("gate_proj.bias", .{.dout}), - .d, - ), - .down_proj = .init( - mlp_view.createReplicatedTensor("down_proj.weight", .{ .d, .dout }), - mlp_view.maybeCreateReplicatedTensor("down_proj.bias", .{.d}), - .dout, - ), - }; + const mlp_shape = try zml.aio.populateModelWithPrefix(Mlp, allocator, model_weights, "model.layers.0.mlp"); + log.info("layer.0.mlp: {}", .{mlp_shape}); } ``` Build and run, using previous commands. -Typical errors are due to the naming of layers in Zig not matching the naming -in the file. +Typical errors are of the form _"Layer not found: ..."_. This is typically due +to the naming of layers in Zig not matching the naming in the file. Double-check everything and don't hesitate to print more things, e.g. in the Python script. Alternatively, Huggingface's web-interface allows to peek into `.safetensor` files. -## Loading an individual layer - -Currently, the layer is initialized but the weights are still on the disk. -ZML provides a convenience to automatically load weights from an initialized model: - -```zig -pub fn main(init: std.process.Init) !void { - ... - - var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator); - var mlp_weights = try zml.io.load(Mlp, &mlp, allocator, io, platform, &model_store, .auto); -} -``` - - ## Testing an individual layer Finally, we are going to write the actual math code for our `MLP` layer. @@ -258,39 +226,33 @@ const Mlp = struct { gate_proj: zml.nn.Linear, down_proj: zml.nn.Linear, - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const proj = self.up_proj.forward(x, x.dtype()); - var output = self.gate_proj.forward(x, x.dtype()); + pub fn forward(self: Mlp, x: Tensor) Tensor { + const proj = zml.call(self.up_proj, .forward, .{x}); + var output = zml.call(self.gate_proj, .forward, .{x}); output = output.silu().mul(proj); - return self.down_proj.forward(output, x.dtype()); + return zml.call(self.down_proj, .forward, .{output}); } }; ``` -Our MLP layer is now almost ready to be tested with the `zml.testing.testLayer` utility. -As you may have noticed, ZML makes heavy-use of tagged tensors. In the MLP, we expect -to do a dot product on the `.d` dimension. But the activations we load from the -activations files are not tagged. One solution could be to tag the tensors before saving -them to file, but a much simpler solution is to intercept the inputs before providing -them to the layer we want to test, using a wrapper. +Note that we use `zml.call` instead of directly calling +`self.up_proj.forward(x)`. Calling `forward` directly results in the same +computation happening at runtime; but going through `zml.call` allows ZML to +generate an MLIR representation that is closer to the Zig code and therefore +easier to read. + +We can test the MLP layer with the `zml.testing.testLayer` utility: ```zig -pub const Wrapper = struct { - mlp: Mlp, - - pub fn forward(wrapper: Wrapper, input: zml.Tensor) zml.Tensor { - const tagged_input = input.withTags(.{.b, .s, .d}); - return wrapper.mlp.forward(tagged_input); - } -}; - -pub fn main(init: std.process.Init) !void { +pub fn asyncMain() !void { ... - var activations_store: zml.io.TensorStore = .fromRegistry(allocator, &activations_registry, &.{}); - defer activations_store.deinit(); + var ctx = try zml.Context.init(); + defer ctx.deinit(); + const platform = ctx.autoPlatform(.{}); + const mlp_weights = try zml.aio.loadModelBuffers(Mlp, mlp_shape, model_weights, allocator, platform); - try zml.testing.testLayer(allocator, io, platform, Wrapper{ .mlp = mlp }, .forward, activations_store.view(), "model.model.layers.0.mlp", .{ .mlp = mlp_weights }, &.{}, .{}); + zml.testing.testLayer(platform, activations, "model.layers.0.mlp", mlp_shape, mlp_weights, 1e-3); } ``` diff --git a/docs/howtos/huggingface_access_token.md b/docs/howtos/huggingface_access_token.md index cc11ffbb..797c6dfb 100644 --- a/docs/howtos/huggingface_access_token.md +++ b/docs/howtos/huggingface_access_token.md @@ -3,7 +3,7 @@ Some models have restrictions and may require some sort of approval or agreement process, which, by consequence, **requires token-authentication with Huggingface**. -The easiest way might be to use the `hf auth login` command. +The easiest way might be to use the `huggingface-cli login` command. Alternatively, here is how you can generate a **"read-only public repositories"** access token to log into your account on Huggingface, directly from `bazel`, in order to download models. @@ -29,6 +29,7 @@ Now you're ready to download a gated model like `Meta-Llama-3.2-1b`! ``` # requires token in $HOME/.cache/huggingface/token, as created by the -# `hf auth login` command, or the `HUGGINGFACE_TOKEN` environment variable. -bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" +# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. +bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` diff --git a/docs/howtos/profiling.md b/docs/howtos/profiling.md deleted file mode 100644 index f98a7dc2..00000000 --- a/docs/howtos/profiling.md +++ /dev/null @@ -1,375 +0,0 @@ -# Profiling ZML Programs - -ZML has two profiling layers: - -- host tracing through `zml.tracer.scope(...)` -- backend profiling through `zml.Platform.profiler(...)` - -The host tracing path is always based on `TraceMe`. Depending on the target -platform, the tracing bridge also emits device-visible annotations: - -- CUDA on Linux: NVTX ranges -- ROCm on Linux: ROCTx ranges -- macOS: `os_signpost` intervals for Instruments - -When backend profiling is enabled, ZML writes both the raw `XSpace` protobuf -and a streamed Perfetto / trace-viewer JSON file. - -## Profiling architecture - -ZML profiling is split into four layers: - -1. `zml.tracer.scope(...)` in Zig marks lexical regions of interest -2. `zml/profiling/traceme_lib.cc` bridges those scopes into platform-specific - tracing backends -3. `zml.Platform.profiler(...)` manages a profiling session and collects profile - data from PJRT -4. `tools/xspace_to_perfetto` converts the final `XSpace` protobuf into a - streamed trace-event JSON file - -The important architectural split is: - -- `zml.tracer.scope(...)` is for instrumentation inside your code -- `zml.Platform.profiler(...)` is for session lifecycle and artifact generation - -You can use scopes without a profiler session, but you only get exported -`profiling.xplane.pb` and `profiling.trace.json` when a profiler session is -started. - -### Scope instrumentation path - -`zml.tracer.scope(...)` is the lowest-level API you use directly in Zig code. -Each scope creates a `TraceMe` region, and the C bridge adds extra -platform-visible annotations when supported: - -- CUDA/Linux: NVTX ranges -- ROCm/Linux: ROCTx ranges -- macOS: `os_signpost` intervals - -Metadata passed to `zml.tracer.scope(...)` is encoded into the `TraceMe` scope -name. That is why the API only accepts small scalar or string-like fields: it is -designed to produce compact trace labels rather than structured protobuf -payloads. - -### Profiler session path - -`zml.Platform.profiler(...)` builds a `tensorflow.ProfileOptions` protobuf from -`zml.Platform.ProfilerOptions` and asks the active PJRT backend for a profiler -session. - -At `start()` time, ZML does two distinct things: - -- it tries to start the backend PJRT profiler session -- it configures local host-side `TraceMeRecorder` capture when enabled - -At `stop()` time, ZML: - -1. stops the PJRT profiler session if one exists -2. collects the backend `XSpace` protobuf -3. merges locally recorded host `TraceMe` events into the host plane -4. writes the final protobuf to disk -5. streams trace JSON from that final protobuf - -This separation is important because host scopes and backend profiler output are -not produced by the same subsystem. ZML joins them only at the end of the -session. - -### Export path - -The final export path is intentionally one-way: - -- source data format: `XSpace` protobuf -- derived export format: trace-event JSON for Perfetto / trace viewer - -ZML keeps `profiling.xplane.pb` as the source of truth and generates -`profiling.trace.json` from it afterwards. The converter is implemented in Zig -and streams JSON directly from the protobuf instead of materializing the whole -JSON document in memory first. - -## The generated artifacts - -By default, a profiling session is written under: - -```text -/tmp/xprof/plugins/profile/profiling/ -``` - -The relevant files are: - -- `profiling.xplane.pb`: raw `XSpace` protobuf -- `profiling.trace.json`: Perfetto / trace-viewer JSON export - -These paths come from `zml/profiling/profiler.zig` and can be changed through -`zml.Platform.ProfilerOptions`. - -## Add trace scopes in Zig - -Use `zml.tracer.scope(name, metadata)` to bracket the code you want to inspect. - -```zig -const zml = @import("zml"); - -var trace = try zml.tracer.scope("llm.compile", .{ - .seqlen = args.seqlen, - .interactive = args.prompt == null, -}); -defer trace.end(); - -try models.LoadedModel.compile(...); -``` - -Use `.{}` when there is no metadata: - -```zig -var trace = try zml.tracer.scope("llm.load_buffers", .{}); -defer trace.end(); -``` - -Metadata is encoded into the `TraceMe` scope name, so keep it small and -high-signal. - -## Record a ZML profile from code - -If you want ZML to collect a backend profile and export `profiling.trace.json`, -create a profiler from the current platform: - -```zig -const platform: *zml.Platform = try .auto(allocator, io, .{}); - -var profiler = try platform.profiler(allocator, io, .defaults); -defer profiler.deinit(); - -try profiler.start(); -defer { - if ((profiler.stop() catch unreachable)) |profile| { - std.log.info("Profile dumped: {s} and {s}", .{ - profile.protobuf_path, - profile.perfetto_path, - }); - } -} -``` - -This is the normal ZML flow: - -1. start the PJRT backend profiler -2. record local `TraceMe` scopes -3. merge the local host trace into the backend `XSpace` -4. write `profiling.xplane.pb` -5. stream `profiling.trace.json` - -If the backend profiler is unavailable, ZML still keeps the local host trace -path and logs a warning. - -## CUDA on Linux with Nsight Systems - -ZML provides a Bazel run config for `nsys`: - -```bash -bazel run --config=nsys //examples/benchmark:benchmark -``` - -Or for a real model: - -```bash -bazel run --config=nsys //examples/llm -- \ - --model=/path/to/model \ - --prompt="Write a story about a cat" -``` - -The wrapper target is `//tools:nsys`. By default it runs: - -```text -sudo -E nsys profile ... -``` - -with CUDA, cuBLAS, cuSPARSE, cuDNN, NVTX, syscall, and OS runtime tracing -enabled. - -If you need to install Nsight Systems manually, use NVIDIA's official download -page: . For the server -or target machine where `nsys` will run, choose the `Linux CLI only` package. -For the client machine where you open and inspect reports, choose the full host -package for your OS. NVIDIA's installation guide is here: -. - -Useful overrides: - -- `ZML_PROFILE_NO_SUDO=1`: run `nsys` without `sudo` -- `ZML_NSYS_ARGS='...'`: replace the default `nsys` arguments - -Example: - -```bash -ZML_PROFILE_NO_SUDO=1 bazel run --config=nsys //examples/benchmark:benchmark -``` - -### Important interaction with `zml.profiler` - -The `nsys` wrapper sets: - -```text -SKIP_PJRT_PROFILER=true -``` - -That tells `zml/profiling/profiler.zig` to skip nested PJRT backend profiling. This is -intentional. Under `nsys`, you normally want: - -- the external Nsight session -- local `TraceMe` / NVTX annotations from your scopes - -and not a second backend profile session layered on top. - -## ROCm on Linux with `rocprofv3` - -ZML also provides a Bazel run config for ROCm: - -```bash -bazel run --config=rocprofv3 //examples/benchmark:benchmark -``` - -Or: - -```bash -bazel run --config=rocprofv3 //examples/llm -- \ - --model=/path/to/model \ - --prompt="Write a story about a cat" -``` - -The wrapper target is `//tools:rocprofv3`. - -This path is intentionally sandboxed inside the repository's ROCm packaging: - -- it launches the Bazel-runfiles copy of `rocprofv3` -- it passes `--rocm-root` pointing at the sandboxed ROCm runtime - -So it does not depend on a host-installed `rocprofv3` layout. - -The default wrapper enables system, runtime, HIP, kernel, memory copy, -scratch-memory, HSA core, and marker tracing, and writes `pftrace` output. - -Useful overrides: - -- `ZML_ROCPROFV3_ARGS='...'`: replace the default `rocprofv3` arguments - -### Important interaction with `zml.profiler` - -Like the `nsys` wrapper, the `rocprofv3` wrapper sets: - -```text -SKIP_PJRT_PROFILER=true -``` - -That means: - -- the external `rocprofv3` session owns backend/device profiling -- ZML still records local host `TraceMe` scopes -- ROCTx ranges from `zml.tracer.scope(...)` stay visible as device annotations - -## Neuron Profiling - -Use `--config=neuron-profile` with the command you already run. The wrapper -prints the profile directory when the program exits. - -```bash -NEURON_RT_VISIBLE_CORES=0,1 \ -bazel run --config=neuron-profile //examples/llm -- \ - --model=/path/to/model \ - --prompt="Explain mixture-of-experts routing in one paragraph." \ - --seqlen=128 \ - --backend=nki \ - --topk=1 -``` - -The default profile root is `/tmp/zml-neuron-profile`. Each run contains: - -```text -/tmp/zml-neuron-profile// - compile/ - execution/ -``` - -Start the Neuron Explorer server in one terminal: - -```bash -bazel run //tools/neuron:server -``` - -Open . When working through SSH, forward ports `3001` -and `3002`. - -With the server running, ingest the run from another terminal: - -```bash -bazel run //tools/neuron:ingest -- /tmp/zml-neuron-profile/ -``` - -Create a quick summary from one run when you want a file artifact: - -```bash -bazel run //tools/neuron:summary-json -- /tmp/zml-neuron-profile/ -bazel run //tools/neuron:summary-txt -- /tmp/zml-neuron-profile/ -bazel run //tools/neuron:summary-perfetto -- /tmp/zml-neuron-profile/ -``` - -Each summary command writes beside the run and prints the output path: -`summary.json`, `summary.txt`, or `summary.pftrace`. - -## macOS Instruments - -On macOS, `zml.tracer.scope(...)` emits `os_signpost` intervals. These show up -in Instruments, especially in the `Points of Interest` template. - -Build the binary first: - -```bash -bazel build //examples/llm:llm -``` - -Then either launch it from the Instruments UI or use `xctrace`: - -```bash -xctrace record \ - --template 'Points of Interest' \ - --output /tmp/llm.trace \ - --launch -- \ - /home/you/zml/bazel-bin/examples/llm/llm \ - --model=/path/to/model \ - --prompt="Write a story about a cat" -``` - -Use full Xcode for `xctrace`. Command Line Tools alone are not enough. - -## Open the exported trace - -The `profiling.trace.json` file can be opened in Perfetto or in tools that still -understand the trace-event JSON format. - -Typical flow: - -1. run a ZML program with `zml.Platform.profiler(...)` -2. wait for `profiling.trace.json` -3. open the JSON in Perfetto - -The converter is implemented in Zig and streams JSON directly from `XSpace`, -which keeps large traces much more manageable than the old in-memory path. - -## Platform summary - -- Plain ZML profiling: backend `XSpace` profile plus merged local host trace -- CUDA + `--config=nsys`: external Nsight session plus local `TraceMe` / NVTX -- ROCm + `--config=rocprofv3`: external `rocprofv3` session plus local `TraceMe` / ROCTx -- macOS Instruments: local `TraceMe` scopes exposed as `os_signpost` - -## Troubleshooting - -- If you run under `--config=nsys`, `--config=rocprofv3`, or - `--config=neuron-profile`, PJRT profiling is skipped on purpose because the - wrappers set `SKIP_PJRT_PROFILER=true`. -- If you do not see device annotations, first check whether you are on a - supported platform: - - CUDA/Linux for NVTX - - ROCm/Linux for ROCTx - - macOS for Instruments signposts -- If you only need host scopes, `zml.tracer.scope(...)` works independently of - whether backend profiling succeeds. diff --git a/docs/learn/concepts.md b/docs/learn/concepts.md index 1a3cea05..de14e15b 100644 --- a/docs/learn/concepts.md +++ b/docs/learn/concepts.md @@ -8,8 +8,8 @@ particulary Neural Networks (NN). The lifecycle of a model is implemented in the following steps: -1. Open the model file and read the shapes of the weights, but don't load the weights - yet. +1. Open the model file and read the shapes of the weights, but leave the + weights on the disk. 2. Using the loaded shapes and optional metadata, instantiate a model struct with `Tensor`s, representing the shape and layout of each layer of the NN. @@ -20,21 +20,28 @@ The lifecycle of a model is implemented in the following steps: 4. Load the model weights from disk, onto the accelerator memory. -5. Load some user inputs, and copy them to the accelerator. +5. Bind the model weights to the executable. -6. Call the executable using the weights and the user inputs. +6. Load some user inputs, and copy them to the accelerator. -7. Fetch the returned model output from accelerator into host memory, and +7. Call the executable on the user inputs. + +8. Fetch the returned model output from accelerator into host memory, and finally present it to the user. -8. When all user inputs have been processed, free the executable resources, weights and - inputs. +9. When all user inputs have been processed, free the executable resources and + the associated weights. **Some details:** Note that the compilation and weight loading steps are both bottlenecks to your -model startup time, but they can be done in parallel thanks to Zig's `std.Io` interface. +model startup time, but they can be done in parallel. **ZML provides +asynchronous primitives** to make that easy. + +The **compilation can be cached** across runs, and if you're always using the +same model architecture with the same shapes, it's possible to by-pass it +entirely. The accelerator is typically a GPU, but can be another chip, or even the CPU itself, churning vector instructions. @@ -44,7 +51,7 @@ itself, churning vector instructions. In ZML, we leverage Zig's static type system to differentiate between a few concepts, hence we not only have a `Tensor` to work with, like other ML -frameworks, but also `Buffer`, `Slice`, and `Shape`. +frameworks, but also `Buffer`, `HostBuffer`, and `Shape`. Let's explain all that. @@ -58,19 +65,25 @@ Let's explain all that. `Shape` struct can also represent a regular number, aka a scalar: `Shape.init(.{}, .i32)` represents a 32-bit signed integer. -* `Slice`: _is_ the combination of a `Shape` and raw bytes (bytes that are **on the CPU**). - - can own the underlying memory - but can also accomodate non-owned memory. +* `HostBuffer`: _is_ a multi-dimensional array, whose memory is allocated **on + the CPU**. + - points to the slice of memory containing the array + - typically owns the underlying memory - but has a flag to remember when it + doesn't. * `Buffer`: _is_ a multi-dimension array, whose memory is allocated **on an accelerator**. - contains a handle that the ZML runtime can use to convert it into a physical address, but there is no guarantee this address is visible from the CPU. - - can be created from a `Slice` by calling `Buffer.fromSlice(...)`. + - can be created by loading weights from disk directly to the device via + `zml.aio.loadBuffers` + - can be created by calling `HostBuffer.toDevice(accelerator)`. * `Tensor`: is a mathematical object representing an intermediary result of a - computation or an input to an executable (including weights). - - is basically a `Shape` with an attached MLIR value when in the context of a compilation. + computation. + - is basically a `Shape` with an attached MLIR value representing the + mathematical operation that produced this `Tensor`. ## The model struct @@ -124,22 +137,25 @@ struct of `Model` but with a `Buffer` replacing every `Tensor`. Let's look at the model life cycle again, but this time annotated with the corresponding types. -1. Open the model file and read the shapes of the weights -> `zml.Shape` +1. Open the model file and read the shapes of the weights -> `zml.HostBuffer` + (using memory mapping, no actual copies happen yet) 2. Instantiate a model struct -> `Model` struct (with `zml.Tensor` inside) 3. Compile the model struct and its `forward` function into an executable. `foward` is a `Tensor -> Tensor` function, executable is a - `zml.Executable` + `zml.FnExe(Model.forward)` 4. Load the model weights from disk, onto accelerator memory -> `zml.Bufferized(Model)` struct (with `zml.Buffer` inside) -5. Load some user inputs (custom struct), encode them into arrays of numbers - (`zml.Slice`), and copy them to the accelerator (`zml.Buffer`). +5. Bind the model weights to the executable `zml.ModuleExe(Model.forward)` -6. Call the executable on the user inputs. `exe.call` accepts `zml.Buffer` +6. Load some user inputs (custom struct), encode them into arrays of numbers + (`zml.HostBuffer`), and copy them to the accelerator (`zml.Buffer`). + +7. Call the executable on the user inputs. `module.call` accepts `zml.Buffer` arguments and returns `zml.Buffer` -7. Return the model output (`zml.Buffer`) to the host (`zml.Slice`), +8. Return the model output (`zml.Buffer`) to the host (`zml.HostBuffer`), decode it (custom struct) and finally return to the user. diff --git a/docs/misc/style_guide.md b/docs/misc/style_guide.md index a112377c..165555ce 100644 --- a/docs/misc/style_guide.md +++ b/docs/misc/style_guide.md @@ -1,7 +1,7 @@ # ZML Style Guide -We prefer to keep it simple and adhere to the [Zig Style Guide](https://ziglang.org/documentation/0.15.1/#Style-Guide). +We prefer to keep it simple and adhere to the [Zig Style Guide](https://ziglang.org/documentation/0.13.0/#Style-Guide). We use ZLS to auto-format code. diff --git a/docs/tutorials/getting_started.md b/docs/tutorials/getting_started.md index ababeb15..31f8b4db 100644 --- a/docs/tutorials/getting_started.md +++ b/docs/tutorials/getting_started.md @@ -21,14 +21,14 @@ through `bazelisk`, a version manager for `bazel`. **macOs:** ``` -brew install bazelisk + brew install bazelisk ``` **Linux:** ``` -curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.28.1/bazelisk-linux-amd64' -chmod +x /usr/local/bin/bazel + curl -L -o /usr/local/bin/bazel 'https://github.com/bazelbuild/bazelisk/releases/download/v1.25.0/bazelisk-linux-amd64' + chmod +x /usr/local/bin/bazel ``` @@ -49,7 +49,7 @@ compile it, and classify a randomly picked example from the test dataset. On the command line: ``` -bazel run --config=release //examples/mnist +bazel run --config=release //mnist ``` ### Llama @@ -71,10 +71,11 @@ Once you've been granted access, you're ready to download a gated model like `Meta-Llama-3.1-8B-Instruct`! ``` -# Authenticate on Huggingface. -bazel run //tools/hf -- auth login -# Run the model, loading directly from HF. -bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" +# requires token in $HOME/.cache/huggingface/token, as created by the +# `huggingface-cli login` command, or the `HUGGINGFACE_TOKEN` environment variable. +bazel run @zml//tools:hf -- download meta-llama/Llama-3.1-8B-Instruct --local-dir $HOME/Llama-3.1-8B-Instruct --exclude='*.pth' +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct +bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.1-8B-Instruct --prompt="What is the capital of France?" ``` You can also try `Llama-3.1-70B-Instruct` if you have enough memory. @@ -85,8 +86,9 @@ Like the 8B model above, this model also requires approval. See [here](https://huggingface.co/meta-llama/Llama-3.2-1B-Instruct) for access requirements. ``` -bazel run //tools/hf -- auth login -bazel run --config=release //examples/llm -- --model=hf://meta-llama/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" +bazel run @zml//tools:hf -- download meta-llama/Llama-3.2-1B-Instruct --local-dir $HOME/Llama-3.2-1B-Instruct --exclude='*.pth' +bazel run --config=release //examples/llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct +bazel run --config=release //llama -- --hf-model-path=$HOME/Llama-3.2-1B-Instruct --prompt="What is the capital of France?" ``` For a larger 3.2 model, you can also try `Llama-3.2-3B-Instruct`. @@ -112,15 +114,13 @@ following arguments to the command line when compiling or running a model: The latter, avoiding compilation for CPU, cuts down compilation time. -So, to run the Llama model from above on your host sporting an NVIDIA GPU, +So, to run the OpenLLama model from above on your host sporting an NVIDIA GPU, run the following: ``` -bazel run --config=release //examples/llm \ - --@zml//platforms:cuda=true \ - -- \ - --model=hf://meta-llama/Llama-3.1-8B-Instruct \ - --prompt="What is the capital of France?" +bazel run --config=release //examples/llama:Llama-3.2-1B-Instruct \ + --@zml//platforms:cuda=true \ + -- --prompt="What is the capital of France?" ``` diff --git a/docs/tutorials/write_first_model.md b/docs/tutorials/write_first_model.md index 312c9afc..ebaf6c2f 100644 --- a/docs/tutorials/write_first_model.md +++ b/docs/tutorials/write_first_model.md @@ -42,6 +42,13 @@ touch simple_layer/main.zig touch simple_layer/BUILD.bazel ``` +By the way, you can access the complete source code of this walkthrough here: + +- [main.zig](https://github.com/zml/zml/tree/master/examples/simple_layer/main.zig) +- [BUILD.bazel](https://github.com/zml/zml/tree/master/examples/simple_layer/BUILD.bazel) + + + ## The high-level Overview Before firing up our editor, let's quickly talk about a few basic ZML @@ -50,31 +57,35 @@ fundamentals. In ZML, we describe a _Module_, which represents our AI model, as a Zig `struct`. That struct can contain Tensor fields that are used for computation, e.g. weights and biases. In the _forward_ function of a Module, we describe the -computation by calling tensor operations like _mul_, _add_, _dot_, +computation by calling tensor operations like _mul_, _add_, _dotGeneral_, _conv2D_, etc., or even nested Modules. ZML creates an MLIR representation of the computation when we compile the -_Module_. For compilation, only the _Tensors_ are required. No actual tensor data -is needed at this step. This is important for large models: we can compile them -while the actual weight data is being fetched from disk. +Module. For compilation, only the _Shapes_ of all tensors must be known. No +actual tensor data is needed at this step. This is important for large models: +we can compile them while the actual weight data is being fetched from disk. -To accomplish this, ZML uses a _TensorStore_. The _TensorStore_ loads everything -that is required to make _Tensors_ and later materialize them to _Buffers_. In our -example though, we won't even use a _TensorStore_ as it's optional. We will directly -create tensors using their shape and data type. +To accomplish this, ZML uses a _BufferStore_. The _BufferStore_ knows how to +only load shapes and when to load actual tensor data. In our example, we will +fake the _BufferStore_ a bit: we won't load from disk; we'll use float arrays +instead. -After compilation is done, we get what we call an _Executable_. From this _Executable_ -we can create _Args_ and _Results_, two structs that will store respectively the inputs -and outputs of a computation done with an _Executable_. +After compilation is done (and the _BufferStore_ has finished loading weights), +we can send the weights from the _BufferStore_ to our computation device. That +produces an _executable_ module which we can call with different _inputs_. + +In our example, we then copy the result from the computation device to CPU +memory and print it. **So the steps for us are:** - describe the computation as ZML _Module_, using tensor operations -- initialize the _Module_ -- compile the _Module_ to produce an _Executable_ -- make a "bufferized" version of the _Module_, containing the actual data on the - computation device -- prepare the _Args_ and _Results_ of a computation and call the _Executable_ +- create a _BufferStore_ that provides _Shapes_ and data of weights and bias + (ca. 5 lines of code). +- compile the _Module_ **asynchronously** +- make the compiled _Module_ send the weights (and bias) to the computation + device utilizing the _BufferStore_, producing an _executable_ module +- prepare input tensor and call the _executable_ module. - get the result back to CPU memory and print it If you like to read more about the underlying concepts of the above, please see @@ -88,9 +99,13 @@ Let's start by writing some Zig code, importing ZML and often-used modules: ```zig const std = @import("std"); const zml = @import("zml"); +const async = @import("async"); ``` -You will use above lines probably in all ZML projects. +You will use above lines probably in all ZML projects. Also, note that **ZML is +async** and comes with its own async runtime, thanks to +[zigcoro](https://github.com/rsepassi/zigcoro). + ### Defining our Model @@ -122,139 +137,205 @@ There are more things to observe: - more advanced use-cases are passing in / returning structs or tuples, like `struct { Tensor, Tensor }` as an example for a tuple of two tensors. You can see such use-cases, for example in the - [LLM CLI](https://github.com/zml/zml/tree/master/examples/llm) + [Llama Model](https://github.com/zml/zml/tree/master/examples/llama) - in the model, tensors may be optional. As is the case with `bias`. ### Adding a main() function -```zig -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - const io = init.io; +ZML code is async. Hence, We need to provide an async main function. It works +like this: - var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator, io); - +```zig +pub fn main() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + try async.AsyncThread.main(gpa.allocator(), asyncMain); +} + + +pub fn asyncMain() !void { + // ... +``` + +The above `main()` function only creates an allocator and an async main thread +that executes our `asyncMain()` function by calling it with no (`.{}`) +arguments. + +So, let's start with the async main function: + +```zig +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + // Arena allocator for BufferStore etc. + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // Create ZML context + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); ... } ``` -The above `main()` function takes advantage of Zig's "fat main" to avoid having to write -boilerplate code for allocations and io. +This is boilerplate code that provides a general-purpose allocator and, for +convenience, an arena allocator that we will use later. The advantage of arena +allocators is that you don't need to deallocate individual allocations; you +simply call `.deinit()` to deinitialize the entire arena instead! -We also get our ZML CPU platform `platform` automatically. +We also initialize the ZML context `context` and get our CPU `platform` +automatically. -### Initializing the Module -Next, we need to initialize the module so that we can compile it. +### The BufferStore + +Next, we need to set up the concrete weight and bias tensors for our model. +Typically, we would load them from disk. But since our example works without +stored weights, we are going to create a BufferStore manually, containing +_HostBuffers_ (buffers on the CPU) for both the `weight` and the `bias` tensor. + +A BufferStore basically contains a dictionary with string keys that match the +name of the struct fields of our `Layer` struct. So, let's create this +dictionary: ```zig -const layer: Layer = .{ - .weight = zml.Tensor.init(.{3}, .f16), - .bias = zml.Tensor.init(.{3}, .f16), +// Our weights and bias to use +var weights = [3]f16{ 2.0, 2.0, 2.0 }; +var bias = [3]f16{ 1.0, 2.0, 3.0 }; +const input_shape = zml.Shape.init(.{3}, .f16); + +// We manually produce a BufferStore. You would not normally do that. +// A BufferStore is usually created by loading model data from a file. +var buffers: zml.aio.BufferStore.Buffers = .{}; +try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); +try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); + +// the actual BufferStore +const bs: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, }; ``` -As you can see, it's pretty simple. You can create _Tensors_ from their shape and data type. -Alternatively, you are free to add an arbitrary `.init()` function to your _Module_ if the -initialization code is complex. +Our weights are `{2.0, 2.0, 2.0}`, and our bias is just `{1.0, 2.0, 3.0}`. The +shape of the weight and bias tensors is `{3}`, and because of that, the **shape +of the input tensor** is also going to be `{3}`! + +Note that `zml.Shape` always takes the data type associated with the tensor. In +our example, that is `f16`, expressed as the enum value `.f16`. + ### Compiling our Module for the accelerator We're only going to use the CPU for our simple model, but we need to compile the -`forward()` function nonetheless: +`forward()` function nonetheless. This compilation is usually done +asynchronously. That means, we can continue doing other things while the module +is compiling: ```zig -// Our computation require an input tensor -const input: zml.Tensor = .init(.{3}, .f16); +// A clone of our model, consisting of shapes. We only need shapes for compiling. +// We use the BufferStore to infer the shapes. +const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); -var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{}); -defer executable.deinit(); +// Start compiling. This uses the inferred shapes from the BufferStore. +// The shape of the input tensor, we have to pass in manually. +var compilation = try async.async( + zml.compileModel, + .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }, +); + +// Produce a bufferized weights struct from the fake BufferStore. +// This is like the inferred shapes, but with actual values. +// We will need to send those to the computation device later. +var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); +defer zml.aio.unloadBuffers(&model_weights); // for good practice + +// Wait for compilation to finish +const compiled = try compilation.await(); ``` -You might wonder what this `sharding` variable is for ? ZML supports sharding tensors across multiple devices, -and the `compile()` function needs to know how the tensors are sharded in order to compile the module correctly. -For this simple example, we just replicate the tensors across all devices for simplicity. +Compiling is happening in the background via the `async` function. We call +`async` with the `zml.compileModel` function and its arguments +separately. The arguments themselves are basically the shapes of the weights in +the BufferStore, the `.forward` function name in order to compile +`Layer.forward`, the shape of the input tensor(s), and the platform for which to +compile (we used auto platform). -### Creating the "bufferized" Model -Now that we have compiled the module, we can take care of loading the actual tensor data. -In our case, we'll create the _Buffers_ from the raw data directly (but keep in mind that ZML -provides ways to have extra-low loading times). +### Creating the Executable Model + +Now that we have compiled the module utilizing the shapes, we turn it into an +executable. ```zig - -const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 2.0, 3.0})); -const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{1.0, 1.0, 1.0})); -var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), -}; -defer layer_buffers.weight.deinit(); -defer layer_buffers.bias.?.deinit(); +// pass the model weights to the compiled module to create an executable module +// all required memory has been allocated in `compile`. +var executable = compiled.prepare(model_weights); +defer executable.deinit(); ``` ### Calling / running the Model -Before being able to run our _Executable_ there is still two steps: -- create the input buffer -- create and fill the _Args_ and _Results_ structs +The executable can now be invoked with an input of our choice. -To create the `input` we already know how to do it, as we did it for the _Module_ -buffers. +To create the `input`, we directly use `zml.Buffer` by calling +`zml.Buffer.fromArray()`. It's important to note that `Buffer`s reside in +_accelerator_ (or _device_) memory, which is precisely where the input needs to +be for the executable to process it on the device. + +For clarity, let's recap the distinction: `HostBuffer`s are located in standard +_host_ memory, which is accessible by the CPU. When we initialized the weights, +we used `HostBuffers` to set up the `BufferStore`. This is because the +`BufferStore` typically loads weights from disk into `HostBuffer`s, and then +converts them into `Buffer`s when we call `loadBuffers()`. + +However, for inputs, we bypass the `BufferStore` and create `Buffer`s directly +in device memory. -To create the _Args_ and _Results_, the _Executable_ exposes two conveniences. ```zig -// create the input buffer -const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); -var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); +// prepare an input buffer +// Here, we use zml.HostBuffer.fromSlice to show how you would create a +// HostBuffer with a specific shape from an array. +// For situations where e.g. you have an [4]f16 array but need a .{2, 2} input +// shape. +var input = [3]f16{ 5.0, 5.0, 5.0 }; +var input_buffer = try zml.Buffer.from( + platform, + zml.HostBuffer.fromSlice(input_shape, &input), +); defer input_buffer.deinit(); -// create the Args and Results structs -var args = try executable.args(allocator); -defer args.deinit(allocator); - -var results = try executable.results(allocator); -defer results.deinit(allocator); - -// fill the Args -args.set(.{ layer_buffers, input_buffer }); - -// call our executable -executable.call(io, args, &results, .{}); - -// Retrieve the resulting buffer -var result = results.get(zml.Buffer); +// call our executable module +var result: zml.Buffer = executable.call(.{input_buffer}); defer result.deinit(); // fetch the result buffer to CPU memory -const result_slice = try result.toSliceAlloc(allocator, io); -defer result_slice.free(allocator); - +const cpu_result = try result.toHostAlloc(arena); std.debug.print( "\n\nThe result of {d} * {d} + {d} = {d}\n", - .{ weight_slice, input_slice, bias_slice, result_slice }, + .{ &weights, &input, &bias, cpu_result.items(f16) }, ); ``` Note that the result of a computation is usually residing in the memory of the -computation device, so with `.toSliceAlloc()` we bring it back to CPU memory in -the form of a `Slice`. After that, we can print it. +computation device, so with `.toHostAlloc()` we bring it back to CPU memory in +the form of a `HostBuffer`. After that, we can print it. -`call(io, args, &results, .{})` enqueues execution and returns immediately. Use -`call(io, args, &results, .{ .wait = true })` to wait for execution to finish before returning. -Here, `toSliceAlloc()` waits for the result before reading it. - -Raw calls leave input buffer handles owned by the caller, including inputs donated -with `reuseBuffer()`. Deinit their original handles after calling. Runners and -`eval` destroy donated input handles automatically. With runners, use output -destinations pointing to those buffers to replace them with the returned outputs. +In order to print it, we need to tell the host buffer how to interpret the +memory. We do that by calling `.items(f16)`, making it cast the memory to `f16` +items. And that's it! Now, let's have a look at building and actually running this example! @@ -274,18 +355,20 @@ zig_binary( name = "simple_layer", main = "main.zig", deps = [ + "@zml//async", "@zml//zml", ], ) ``` -To produce an executable, we import `zig_binary` from the zig rules, and -pass it a name and the zig file we just wrote. The dependency in `deps` is -what's needed for a basic ZML executable and correlate with our import at the +To produce an executable, we import `zig_cc_binary` from the zig rules, and +pass it a name and the zig file we just wrote. The dependencies in `deps` are +what's needed for a basic ZML executable and correlate with our imports at the top of the Zig file: ```zig const zml = @import("zml"); +const async = @import("async"); ``` @@ -298,44 +381,41 @@ With everything in place now, running the model is easy: bazel run --config=release //examples/simple_layer # compile and run debug version -bazel run //examples/simple_layer +bazel run //simple_layer ``` And voila! Here's the output: ``` -bazel run --config=release //examples/simple_layer -INFO: Analyzed target //examples/simple_layer:simple_layer (0 packages loaded, 0 targets configured). +bazel run --config=release //simple_layer +INFO: Analyzed target //simple_layer:simple_layer (0 packages loaded, 0 targets configured). INFO: Found 1 target... -Target //examples/simple_layer:simple_layer up-to-date: - bazel-bin/examples/simple_layer/simple_layer +Target //simple_layer:simple_layer up-to-date: + bazel-bin/simple_layer/simple_layer INFO: Elapsed time: 0.120s, Critical Path: 0.00s INFO: 1 process: 1 internal. INFO: Build completed successfully, 1 total action -INFO: Running command line: bazel-bin/examples/simple_layer/simple_layer -info(pjrt): Loaded library: libzml_cpu.dylib -debug(zml/module): -******** ZML generated MLIR ******** -module @zml attributes {mhlo.num_partitions = 4 : i32, mhlo.num_replicas = 1 : i32} { - sdy.mesh @replicated = <["bus"=4]> - func.func @main(%arg0: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg1: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}, %arg2: tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) -> (tensor<3xf16> {sdy.sharding = #sdy.sharding<@replicated, [{}], replicated={"bus"}>}) { - %0 = stablehlo.multiply %arg1, %arg2 : tensor<3xf16> - %1 = stablehlo.add %0, %arg0 : tensor<3xf16> - return %1 : tensor<3xf16> - } -} +INFO: Running command line: bazel-bin/simple_layer/simple_layer +info(pjrt): Loaded library: libpjrt_cpu.dylib +info(zml_module): Compiling main.Layer.forward with { Shape({3}, dtype=.f16) } - - -The result of {1,2,3} * {5,5,5} + {1,1,1} = {6,11,16} +The result of { 2, 2, 2 } * { 5, 5, 5 } + { 1, 2, 3 } = { 11, 12, 13 } ``` +--- + +You can access the complete source code of this walkthrough here: + +- [main.zig](https://github.com/zml/zml/tree/master/examples/simple_layer/main.zig) +- [BUILD.bazel](https://github.com/zml/zml/tree/master/examples/simple_layer/BUILD.bazel) + ## The complete example ```zig const std = @import("std"); const zml = @import("zml"); +const async = @import("async"); /// Model definition const Layer = struct { @@ -351,68 +431,96 @@ const Layer = struct { } }; -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - const io = init.io; +pub fn main() !void { + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + try async.AsyncThread.main(gpa.allocator(), asyncMain); +} - var platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator, io); +pub fn asyncMain() !void { + // Short lived allocations + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); - const layer: Layer = .{ - .weight = zml.Tensor.init(.{3}, .f16), - .bias = zml.Tensor.init(.{3}, .f16), + // Arena allocator for BufferStore etc. + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + // Create ZML context + var context = try zml.Context.init(); + defer context.deinit(); + + const platform = context.autoPlatform(.{}); + + // Our weights and bias to use + var weights = [3]f16{ 2.0, 2.0, 2.0 }; + var bias = [3]f16{ 1.0, 2.0, 3.0 }; + const input_shape = zml.Shape.init(.{3}, .f16); + + // We manually produce a BufferStore. You would not normally do that. + // A BufferStore is usually created by loading model data from a file. + var buffers: zml.aio.BufferStore.Buffers = .{}; + try buffers.put(arena, "weight", zml.HostBuffer.fromArray(&weights)); + try buffers.put(arena, "bias", zml.HostBuffer.fromArray(&bias)); + + // the actual BufferStore + const bs: zml.aio.BufferStore = .{ + .arena = arena_state, + .buffers = buffers, }; - // Our computation require an input tensor - const input: zml.Tensor = .init(.{3}, .f16); + // A clone of our model, consisting of shapes. We only need shapes for + // compiling. We use the BufferStore to infer the shapes. + const model_shapes = try zml.aio.populateModel(Layer, allocator, bs); - var executable = try platform.compile(allocator, io, layer, .forward, .{input}, .{}); + // Start compiling. This uses the inferred shapes from the BufferStore. + // The shape of the input tensor, we have to pass in manually. + var compilation = try async.async(zml.compileModel, .{ allocator, Layer.forward, model_shapes, .{input_shape}, platform }); + + // Produce a bufferized weights struct from the fake BufferStore. + // This is like the inferred shapes, but with actual values. + // We will need to send those to the computation device later. + var model_weights = try zml.aio.loadBuffers(Layer, .{}, bs, arena, platform); + defer zml.aio.unloadBuffers(&model_weights); // for good practice + + // Wait for compilation to finish + const compiled = try compilation.await(); + + // pass the model weights to the compiled module to create an executable + // module + var executable = compiled.prepare(model_weights); defer executable.deinit(); - const weight_slice: zml.Slice = .init(layer.weight.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 2.0, 3.0 })); - const bias_slice: zml.Slice = .init(layer.bias.?.shape(), std.mem.sliceAsBytes(&[3]f16{ 1.0, 1.0, 1.0 })); - var layer_buffers: zml.Bufferized(Layer) = .{ - .weight = try zml.Buffer.fromSlice(io, platform, weight_slice, .replicated), - .bias = try zml.Buffer.fromSlice(io, platform, bias_slice, .replicated), - }; - defer layer_buffers.weight.deinit(); - defer layer_buffers.bias.?.deinit(); - - // create the input buffer - const input_slice: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[3]f16{ 5.0, 5.0, 5.0 })); - var input_buffer: zml.Buffer = try .fromSlice(io, platform, input_slice, .replicated); + // prepare an input buffer + // Here, we use zml.HostBuffer.fromSlice to show how you would create a + // HostBuffer with a specific shape from an array. + // For situations where e.g. you have an [4]f16 array but need a .{2, 2} + // input shape. + var input = [3]f16{ 5.0, 5.0, 5.0 }; + var input_buffer = try zml.Buffer.from( + platform, + zml.HostBuffer.fromSlice(input_shape, &input), + ); defer input_buffer.deinit(); - // create the Args and Results structs - var args = try executable.args(allocator); - defer args.deinit(allocator); - - var results = try executable.results(allocator); - defer results.deinit(allocator); - - // fill the Args - args.set(.{ layer_buffers, input_buffer }); - - // call our executable - executable.call(io, args, &results, .{}); - - // Retrieve the resulting buffer - var result = results.get(zml.Buffer); + // call our executable module + var result: zml.Buffer = executable.call(.{input_buffer}); defer result.deinit(); - // fetch the result buffer to CPU memory - const result_slice = try result.toSliceAlloc(allocator, io); - defer result_slice.free(allocator); - + // fetch the result to CPU memory + const cpu_result = try result.toHostAlloc(arena); std.debug.print( "\n\nThe result of {d} * {d} + {d} = {d}\n", - .{ weight_slice, input_slice, bias_slice, result_slice }, + .{ &weights, &input, &bias, cpu_result.items(f16) }, ); } ``` ## Where to go from here +- [Add some weights files to your model](../howtos/add_weights.md) - [Run the model on GPU](../tutorials/getting_started.md) - [Deploy the model on a server](../howtos/deploy_on_server.md) - [Dockerize this model](../howtos/dockerize_models.md) diff --git a/dsl/common/BUILD.bazel b/dsl/common/BUILD.bazel deleted file mode 100644 index 1476dcfd..00000000 --- a/dsl/common/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "builder", - srcs = [ - "args.zig", - "control_flow.zig", - ], - import_name = "dsl/common/builder", - main = "builder.zig", - visibility = ["//visibility:public"], - deps = [ - "//mlir", - "//mlir/dialects/scf", - ], -) - -zig_test( - name = "test", - deps = [":builder"], -) diff --git a/dsl/common/args.zig b/dsl/common/args.zig deleted file mode 100644 index 84d57d49..00000000 --- a/dsl/common/args.zig +++ /dev/null @@ -1,4 +0,0 @@ -pub fn NamedArgs(comptime Spec: type, comptime ValueT: type) type { - const field_names = @typeInfo(Spec).@"struct".field_names; - return @Struct(.auto, null, field_names, &@splat(ValueT), &@splat(.{})); -} diff --git a/dsl/common/builder.zig b/dsl/common/builder.zig deleted file mode 100644 index 1addd259..00000000 --- a/dsl/common/builder.zig +++ /dev/null @@ -1,35 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const scf = @import("mlir/dialects/scf"); - -pub const control_flow = @import("control_flow.zig"); -pub const args = @import("args.zig"); - -pub const arity = control_flow.arity; -pub const BlockStack = control_flow.BlockStack; -pub const forBounds = control_flow.forBounds; -pub const ForOpts = scf.ForOpts; -pub const openFor = control_flow.openFor; -pub const openIf = control_flow.openIf; -pub const openIfElse = control_flow.openIfElse; -pub const openWhile = control_flow.openWhile; -pub const innerValues = control_flow.innerValues; -pub const resultTypes = control_flow.resultTypes; -pub const ForScope = control_flow.ForScope; -pub const IfOnlyScope = control_flow.IfOnlyScope; -pub const IfScope = control_flow.IfScope; -pub const WhileScope = control_flow.WhileScope; - -pub const NamedArgs = args.NamedArgs; - -/// Verifies `module`, logging it when it fails. -pub fn verify(module: *mlir.Module, comptime target: []const u8) error{InvalidMlir}!void { - if (module.operation().verify()) return; - std.log.err(target ++ ": the module does not verify:\n{f}", .{module.operation().fmt(.{ .debug_info = false })}); - return error.InvalidMlir; -} - -test { - std.testing.refAllDecls(@This()); -} diff --git a/dsl/common/control_flow.zig b/dsl/common/control_flow.zig deleted file mode 100644 index e557e24b..00000000 --- a/dsl/common/control_flow.zig +++ /dev/null @@ -1,364 +0,0 @@ -//! `scf` scope wrappers, generic over the frontend's `BuilderT` and `ValueT`, -//! and `BlockStack`, the blocks a builder emits into. -//! -//! `BuilderT` must expose `ctx`, `loc()`, `pushBlock`, `popBlock` and -//! `currentBlock`. The openers also call its `lift` on literal inits and its -//! `scalarTy` on dtype result types, and `forBounds` its `constantLike` on -//! literal bounds. -//! `ValueT` must have an `inner: *const mlir.Value` field and a `kernel: ?*BuilderT` field. - -const std = @import("std"); -const mlir = @import("mlir"); -const scf = @import("mlir/dialects/scf"); - -/// The blocks a builder emits into: the function's entry block at the bottom, -/// the innermost scope's on top. -pub const BlockStack = struct { - blocks: std.ArrayList(*mlir.Block) = .empty, - - pub fn push(self: *BlockStack, allocator: std.mem.Allocator, block: *mlir.Block) void { - self.blocks.append(allocator, block) catch @panic("BlockStack.push: out of memory"); - } - - /// Closes the innermost scope. - pub fn pop(self: *BlockStack) void { - if (self.blocks.items.len < 2) @panic("BlockStack.pop: no scope open"); // the entry block always stays - _ = self.blocks.pop(); - } - - /// Drops every block, keeping the memory for the next function. - pub fn clear(self: *BlockStack) void { - self.blocks.clearRetainingCapacity(); - } - - /// Returns the top block. - pub fn current(self: *const BlockStack) *mlir.Block { - return self.blocks.getLastOrNull() orelse @panic("BlockStack.current: no block, call declareArgs first"); - } - - /// Replaces the top block with `block`. - pub fn replaceTop(self: *BlockStack, block: *mlir.Block) void { - if (self.blocks.items.len == 0) @panic("BlockStack.replaceTop: no block, call declareArgs first"); - self.blocks.items[self.blocks.items.len - 1] = block; - } -}; - -/// The length of `T`, a tuple like `.{ v1, v2 }` or an array. -pub fn arity(comptime T: type, comptime what: []const u8) comptime_int { - return switch (@typeInfo(T)) { - .array => |a| a.len, - .@"struct" => |s| if (s.is_tuple) s.field_names.len else @compileError(what ++ " must be a tuple like `.{ v1, v2 }` or an array"), - else => @compileError(what ++ " must be a tuple like `.{ v1, v2 }` or an array"), - }; -} - -/// The MLIR values of `values`, `N` `ValueT`s. -pub fn innerValues(comptime ValueT: type, comptime N: usize, values: anytype, comptime what: []const u8) [N]*const mlir.Value { - if (arity(@TypeOf(values), what) != N) - @compileError(what ++ ": the arity must match the scope's"); - var buf: [N]*const mlir.Value = undefined; - inline for (0..N) |i| { - if (@TypeOf(values[i]) != ValueT) @compileError(what ++ ": every element must be a Value"); - buf[i] = values[i].inner; - } - return buf; -} - -/// `values` as `ValueT`s: values as they are, anything else through `kernel.lift`. -fn liftValues(kernel: anytype, comptime ValueT: type, values: anytype, comptime what: []const u8) [arity(@TypeOf(values), what)]ValueT { - var out: [arity(@TypeOf(values), what)]ValueT = undefined; - inline for (0..out.len) |i| out[i] = if (@TypeOf(values[i]) == ValueT) values[i] else kernel.lift(values[i]); - return out; -} - -/// The types `types` names: `*const mlir.Type`s, `ValueT`s for their types, -/// or dtypes (`.f32`) for `kernel.scalarTy`. -pub fn resultTypes(kernel: anytype, comptime ValueT: type, types: anytype, comptime what: []const u8) [arity(@TypeOf(types), what)]*const mlir.Type { - var out: [arity(@TypeOf(types), what)]*const mlir.Type = undefined; - inline for (0..out.len) |i| { - const T = @TypeOf(types[i]); - out[i] = if (T == *const mlir.Type) types[i] else if (T == ValueT) types[i].inner.type_() else kernel.scalarTy(types[i]); - } - return out; -} - -fn scfYield(comptime ValueT: type, comptime N: usize, kernel: anytype, block: *mlir.Block, values: anytype, comptime what: []const u8) void { - const buf = innerValues(ValueT, N, values, what); - _ = scf.yield(kernel.ctx, &buf, kernel.loc()).appendTo(block); -} - -pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { - return struct { - kernel: *BuilderT, - body: *mlir.Block, - lb_inner: *const mlir.Value, - ub_inner: *const mlir.Value, - step_inner: *const mlir.Value, - inits_inner: [N]*const mlir.Value, - iv: ValueT, - carried: [N]ValueT, - results: [N]ValueT = undefined, - opts: scf.ForOpts = .{}, - - const Self = @This(); - - pub fn yield(self: *Self, values: anytype) void { - if (N == 0) @compileError("ForScope.yield: the loop carries nothing, end it with close()"); - self.end(values); - } - - pub fn close(self: *Self) void { - if (N != 0) @compileError("ForScope.close: the loop carries values, end it with yield()"); - self.end(.{}); - } - - fn end(self: *Self, values: anytype) void { - const k = self.kernel; - scfYield(ValueT, N, k, self.body, values, "ForScope.yield"); - k.popBlock(); - const for_op = scf.for_( - k.ctx, - self.lb_inner, - self.ub_inner, - self.step_inner, - &self.inits_inner, - self.body, - self.opts, - k.loc(), - ); - _ = for_op.appendTo(k.currentBlock()); - for (0..N) |i| self.results[i] = .{ .inner = for_op.result(i), .kernel = k }; - } - }; -} - -pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { - return struct { - kernel: *BuilderT, - cond_inner: *const mlir.Value, - then_block: *mlir.Block, - - const Self = @This(); - - pub fn close(self: *Self) void { - const k = self.kernel; - scfYield(ValueT, 0, k, self.then_block, .{}, "IfOnlyScope.close"); - k.popBlock(); - const if_op = scf.if_(k.ctx, self.cond_inner, &.{}, self.then_block, .{}, k.loc()); - _ = if_op.appendTo(k.currentBlock()); - } - }; -} - -pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { - return struct { - kernel: *BuilderT, - cond_inner: *const mlir.Value, - then_block: *mlir.Block, - else_block: *mlir.Block, - result_types: [N]*const mlir.Type, - results: [N]ValueT = undefined, - - const Self = @This(); - - pub fn yieldThen(self: *Self, values: anytype) void { - if (N == 0) @compileError("IfScope.yieldThen: the if has no results, switch branches with elseBranch()"); - self.endThen(values); - } - - pub fn yieldElse(self: *Self, values: anytype) void { - if (N == 0) @compileError("IfScope.yieldElse: the if has no results, end it with close()"); - self.endElse(values); - } - - pub fn elseBranch(self: *Self) void { - if (N != 0) @compileError("IfScope.elseBranch: the if has results, switch branches with yieldThen()"); - self.endThen(.{}); - } - - pub fn close(self: *Self) void { - if (N != 0) @compileError("IfScope.close: the if has results, end it with yieldElse()"); - self.endElse(.{}); - } - - fn endThen(self: *Self, values: anytype) void { - const k = self.kernel; - scfYield(ValueT, N, k, self.then_block, values, "IfScope.yieldThen"); - k.popBlock(); - k.pushBlock(self.else_block); - } - - fn endElse(self: *Self, values: anytype) void { - const k = self.kernel; - scfYield(ValueT, N, k, self.else_block, values, "IfScope.yieldElse"); - k.popBlock(); - const if_op = scf.if_(k.ctx, self.cond_inner, &self.result_types, self.then_block, .{ .else_block = self.else_block }, k.loc()); - _ = if_op.appendTo(k.currentBlock()); - for (0..N) |i| self.results[i] = .{ .inner = if_op.result(i), .kernel = k }; - } - }; -} - -pub fn WhileScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize, comptime M: usize) type { - return struct { - kernel: *BuilderT, - before_block: *mlir.Block, - after_block: *mlir.Block, - inits_inner: [N]*const mlir.Value, - after_types: [M]*const mlir.Type, - before_carried: [N]ValueT, - after_carried: [M]ValueT = undefined, - results: [M]ValueT = undefined, - - const Self = @This(); - - pub fn yieldBefore(self: *Self, cond: ValueT, forwarded: anytype) void { - const k = self.kernel; - const buf = innerValues(ValueT, M, forwarded, "WhileScope.yieldBefore"); - _ = scf.condition(k.ctx, cond.inner, &buf, k.loc()).appendTo(self.before_block); - k.popBlock(); - k.pushBlock(self.after_block); - for (0..M) |i| self.after_carried[i] = .{ .inner = self.after_block.argument(i), .kernel = k }; - } - - pub fn yieldAfter(self: *Self, values: anytype) void { - const k = self.kernel; - scfYield(ValueT, N, k, self.after_block, values, "WhileScope.yieldAfter"); - k.popBlock(); - const w = scf.while_( - k.ctx, - &self.inits_inner, - &self.after_types, - self.before_block, - self.after_block, - k.loc(), - ); - _ = w.appendTo(k.currentBlock()); - for (0..M) |i| self.results[i] = .{ .inner = w.result(i), .kernel = k }; - } - }; -} - -/// `openFor`'s bounds as values. A bound that isn't a value takes the type of -/// the first one that is (`kernel.constantLike`); with none, `lower` becomes -/// `defaultLower(kernel, lower)` and the others take its type. -pub fn forBounds( - kernel: anytype, - comptime ValueT: type, - lower: anytype, - upper: anytype, - step: anytype, - comptime defaultLower: anytype, -) [3]ValueT { - const bounds: [3]ValueT = if (@TypeOf(lower) != ValueT and @TypeOf(upper) != ValueT and @TypeOf(step) != ValueT) blk: { - const lb = defaultLower(kernel, lower); - break :blk .{ lb, kernel.constantLike(upper, lb), kernel.constantLike(step, lb) }; - } else blk: { - const first: ValueT = if (@TypeOf(lower) == ValueT) lower else if (@TypeOf(upper) == ValueT) upper else step; - break :blk .{ kernel.constantLike(lower, first), kernel.constantLike(upper, first), kernel.constantLike(step, first) }; - }; - const t = bounds[0].inner.type_(); - if (!bounds[1].inner.type_().eql(t) or !bounds[2].inner.type_().eql(t)) - std.debug.panic("openFor: bounds of types {f}, {f} and {f}", .{ t, bounds[1].inner.type_(), bounds[2].inner.type_() }); - return bounds; -} - -/// Opens `for iv in [bounds[0], bounds[1]) step bounds[2]` carrying `inits`, -/// values or literals `kernel.lift` takes. -pub fn openFor( - kernel: anytype, - comptime ValueT: type, - bounds: [3]ValueT, - inits: anytype, - opts: scf.ForOpts, -) ForScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(inits), "openFor: inits")) { - const N = comptime arity(@TypeOf(inits), "openFor: inits"); - const values = liftValues(kernel, ValueT, inits, "openFor: inits"); - var types: [N + 1]*const mlir.Type = undefined; - types[0] = bounds[0].inner.type_(); - var inits_inner: [N]*const mlir.Value = undefined; - for (values, 0..) |v, i| { - types[i + 1] = v.inner.type_(); - inits_inner[i] = v.inner; - } - const locs: [N + 1]*const mlir.Location = @splat(kernel.loc()); - const body = mlir.Block.init(&types, &locs); - kernel.pushBlock(body); - var carried: [N]ValueT = undefined; - for (&carried, 0..) |*c, i| c.* = .{ .inner = body.argument(i + 1), .kernel = kernel }; - return .{ - .kernel = kernel, - .body = body, - .lb_inner = bounds[0].inner, - .ub_inner = bounds[1].inner, - .step_inner = bounds[2].inner, - .inits_inner = inits_inner, - .iv = .{ .inner = body.argument(0), .kernel = kernel }, - .carried = carried, - .opts = opts, - }; -} - -pub fn openIf(kernel: anytype, comptime ValueT: type, cond: ValueT) IfOnlyScope(@TypeOf(kernel.*), ValueT) { - const then_block = mlir.Block.init(&.{}, &.{}); - kernel.pushBlock(then_block); - return .{ .kernel = kernel, .cond_inner = cond.inner, .then_block = then_block }; -} - -/// Opens an `if` with an `else`, of results of `result_types` (see `resultTypes`). -pub fn openIfElse( - kernel: anytype, - comptime ValueT: type, - cond: ValueT, - result_types: anytype, -) IfScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(result_types), "openIfElse: result_types")) { - const then_block = mlir.Block.init(&.{}, &.{}); - const else_block = mlir.Block.init(&.{}, &.{}); - kernel.pushBlock(then_block); - return .{ - .kernel = kernel, - .cond_inner = cond.inner, - .then_block = then_block, - .else_block = else_block, - .result_types = resultTypes(kernel, ValueT, result_types, "openIfElse: result_types"), - }; -} - -/// Opens an `scf.while` carrying `inits` into its condition region, which -/// forwards values of `after_types` (see `resultTypes`) to its body. -pub fn openWhile( - kernel: anytype, - comptime ValueT: type, - inits: anytype, - after_types: anytype, -) WhileScope(@TypeOf(kernel.*), ValueT, arity(@TypeOf(inits), "openWhile: inits"), arity(@TypeOf(after_types), "openWhile: after_types")) { - const N = comptime arity(@TypeOf(inits), "openWhile: inits"); - const M = comptime arity(@TypeOf(after_types), "openWhile: after_types"); - const values = liftValues(kernel, ValueT, inits, "openWhile: inits"); - var before_types: [N]*const mlir.Type = undefined; - var inits_inner: [N]*const mlir.Value = undefined; - for (values, 0..) |v, i| { - before_types[i] = v.inner.type_(); - inits_inner[i] = v.inner; - } - const before_locs: [N]*const mlir.Location = @splat(kernel.loc()); - const before_block = mlir.Block.init(&before_types, &before_locs); - const after_types_ = resultTypes(kernel, ValueT, after_types, "openWhile: after_types"); - const after_locs: [M]*const mlir.Location = @splat(kernel.loc()); - const after_block = mlir.Block.init(&after_types_, &after_locs); - kernel.pushBlock(before_block); - var before_carried: [N]ValueT = undefined; - for (&before_carried, 0..) |*c, i| c.* = .{ .inner = before_block.argument(i), .kernel = kernel }; - return .{ - .kernel = kernel, - .before_block = before_block, - .after_block = after_block, - .inits_inner = inits_inner, - .after_types = after_types_, - .before_carried = before_carried, - }; -} - -test { - std.testing.refAllDecls(@This()); -} diff --git a/dsl/cuda_tile/BUILD.bazel b/dsl/cuda_tile/BUILD.bazel deleted file mode 100644 index 527c53e4..00000000 --- a/dsl/cuda_tile/BUILD.bazel +++ /dev/null @@ -1,24 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "builder", - srcs = [ - "control_flow.zig", - "dtype.zig", - ], - import_name = "dsl/cuda_tile/builder", - main = "builder.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//dsl/common:builder", - "//mlir", - "//mlir/dialects/cuda_tile", - "//stdx", - ], -) - -zig_test( - name = "test", - deps = [":builder"], -) diff --git a/dsl/cuda_tile/README.md b/dsl/cuda_tile/README.md deleted file mode 100644 index d8c5fdb8..00000000 --- a/dsl/cuda_tile/README.md +++ /dev/null @@ -1,137 +0,0 @@ -# CUDA Tile IR kernels in Zig - -`dsl/cuda_tile` is a Zig DSL that emits NVIDIA's `cuda_tile` MLIR dialect -(CUDA Tile IR). It is the third instance of the `dsl/triton` / -`dsl/mosaic_tpu` pattern: a `Builder` that owns an MLIR module and appends -ops to a block stack, a `Value` with fluent arithmetic, and a -`zml.kernel.cuda_tile.Kernel` wrapper that turns a `run` function into a -`stablehlo.custom_call` XLA compiles with `tileiras`. - -The dialect binding lives in `mlir/dialects/cuda_tile`: one function per op -(all 100 of them, in `Ops.td` order), every type through the dialect's C API, -every enum attribute through its C getter. - -## A kernel - -```zig -const zml = @import("zml"); -const cut = zml.kernel.cuda_tile; - -const Cfg = struct { n: i64 }; - -pub const AddOne = cut.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = run, -}); - -fn run(b: *cut.Builder, cfg: Cfg) cut.FinishError!void { - // Every parameter is a rank-0 tile>; views are built inside. - const a = try b.declareArgsOpts(.{ - .x = .{ .ptr = .f32 }, - .out = .{ .ptr = .f32 }, - }, .{ .hints = &.{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }} }); - - // pointer -> tensor_view -> partition_view; the view pads the ragged edge. - const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); - const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); - - const bid = b.tileBlockId(); - _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); -} - -// In a model: -const out = AddOne.call(.{ .x = x }, .{ .out = x.shape() }, .{ - .cfg = .{ .n = 1024 }, - .grid = .{ 8, 1, 1 }, -}).out; -``` - -Put each scope's body in a Zig `{ ... }` block right after `openFor`, `openIf` -or `openLoop`, ending with its `yield` (or `close()`). `openIfElse` gets one -block per branch, ending with `yieldThen`/`yieldElse` (`elseBranch()`/`close()` -without results). - -## What is different from the Triton DSL - -* **Rank 0 is the scalar.** Every value is a `tile<...>`; `entry` parameters - must be rank 0, so buffers arrive as `tile>`. -* **Memory is a view stack.** `tensorView` → `partitionView` (or - `stridedView`, `gatherScatterView`) → tile-space `load`/`store`. A partial - edge tile reads the view's `padding` and its store is masked; there is no - mask tile. The pointer-tile arm (`loadPtr`/`storePtr` with `offset`) exists - for gathers but cannot use TMA. -* **Every load and store returns a token.** The dialect has no barrier or - fence; `loadOpts`/`storeOpts` take and return tokens, `load`/`store` drop - them. -* **Float ops carry a rounding mode.** `addf` defaults to `nearest_even`; - `addfOpts` sets it. Integer ops carry none. -* **Loop bodies end with `continue`, `if` bodies with `yield`, and `loop` - exits with `break`** — `ForScope`, `IfScope`, `LoopScope` do this for you. -* **No `num_warps`/`num_stages`.** Warp and CTA tuning is - `optimization_hints` on the entry, per architecture (`Opts.hints`), and - `allow_tma`/`latency` on view loads and stores (`LoadOpts.hints`). -* **Tile dims are powers of two**, at most 2^24 elements; `tileTy` and - `partitionView` check this in Zig. -* **Some conversions are one-way.** `ftof` to `f8e8m0fnu` (the MX scale - type) rounds toward zero by default (`CastOpts.rounding = .positive_inf` - is the other legal choice); an integer reaches it through `f32`, which - `cast`/`to` do for you. `ftoi` always truncates toward zero. `tanh` takes - only `approx`/`full`. `Value.cdiv` is integer-only. -* **`i4` is a tile element type, not a pointee.** Declare a 4-bit buffer as - `.i8` and `unpack` it in the kernel. -* **`atomicRedView` is relaxed-only** on an unpadded view and takes no mask; - use the pointer atomics (`atomicRmw`, `atomicCas`) for masked updates. -* **Reductions over several tiles at once** (argmax, an online softmax's - (max, sum)) go through `reduceMulti`/`scanMulti`. -* **A scalar argument is a pointer**: `.{ .n = .{ .ptr = .i32 } }` then - `b.loadPtr(a.n)` gives the rank-0 tile; XLA passes device addresses only. - -## The custom call - -`ops.cudaTile` emits `__gpu$xla.gpu.cuda_tile` with a printed dictionary: -`name`, `kernel_type = "cuda_tile"`, `ir` (the textual module), `grid_x/y/z`, -optional `ir_version`. XLA parses the module, renames the entry, checks the -argument count against the operands + result leaves, scrubs locations, -serializes bytecode, runs `tileiras`, and launches over `grid` tile blocks with -a `(1,1,1)` block and no shared memory — cuda-tile's launch ABI. Every operand -and result is one raw device pointer in the default layout, operands first; -XLA rejects an entry whose parameters are not all `tile>`. - -To see what XLA received, run with `--xla_dump_to=`: it writes -`.cuda_tile.mlir` (post-rename) and `.tilebc` (the exact bytes -handed to `tileiras`). - -## Finding what exists - -There is one function per op and no hidden API, so grep is the index: - -```sh -# every op of the dialect, in Ops.td order (115 constructors) -grep -n '^pub fn ' mlir/dialects/cuda_tile/cuda_tile.zig - -# the types, enums and attribute helpers they take -grep -n '^pub const ' mlir/dialects/cuda_tile/cuda_tile.zig - -# the DSL: Value's fluent methods (add, mul, lt, to, sum, reshape, ...) -awk '/^pub const Value = struct/,/^};/' dsl/cuda_tile/builder.zig | grep 'pub fn ' - -# the DSL: everything on Builder (views, load/store, math, casts, reductions, -# atomics, control flow) -- the `*Opts` variant of a call takes the options -awk '/^pub const Builder = struct/,0' dsl/cuda_tile/builder.zig | grep ' pub fn ' - -# what each option struct holds, and its defaults -grep -n '^pub const [A-Za-z]*Opts = struct' -A 8 dsl/cuda_tile/builder.zig \ - dsl/cuda_tile/control_flow.zig - -# for / if / loop scopes and their terminators -grep -n 'pub fn ' dsl/cuda_tile/control_flow.zig - -# element types -grep -n 'pub const DType = enum' -A 30 dsl/cuda_tile/dtype.zig -``` - -Every op also has a test: `grep -n '^test "' dsl/cuda_tile/builder.zig` -lists them, and each one prints IR that round-trips through the verifier, so -they double as usage examples. diff --git a/dsl/cuda_tile/builder.zig b/dsl/cuda_tile/builder.zig deleted file mode 100644 index a8414acc..00000000 --- a/dsl/cuda_tile/builder.zig +++ /dev/null @@ -1,2218 +0,0 @@ -const std = @import("std"); - -const common = @import("dsl/common/builder"); -const cuda_tile = @import("mlir/dialects/cuda_tile"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -const cf = @import("control_flow.zig"); -const dtypes = @import("dtype.zig"); -pub const DType = dtypes.DType; -const isFloatDtype = dtypes.isFloatDtype; -const dtypeBitwidth = dtypes.dtypeBitwidth; -const intBitwidth = dtypes.intBitwidth; - -pub const RoundingMode = cuda_tile.RoundingMode; -pub const Signedness = cuda_tile.Signedness; -pub const IntegerOverflow = cuda_tile.IntegerOverflow; -pub const ComparisonPredicate = cuda_tile.ComparisonPredicate; -pub const ComparisonOrdering = cuda_tile.ComparisonOrdering; -pub const AtomicRMWMode = cuda_tile.AtomicRMWMode; -pub const MemoryScope = cuda_tile.MemoryScope; -pub const MemoryOrdering = cuda_tile.MemoryOrdering; -pub const PaddingValue = cuda_tile.PaddingValue; -pub const Arch = cuda_tile.Arch; -pub const EntryHint = cuda_tile.EntryHint; -pub const LoadStoreHint = cuda_tile.LoadStoreHint; -pub const PtrLoadStoreHint = cuda_tile.PtrLoadStoreHint; - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Builder); - std.testing.refAllDecls(Value); -} - -pub fn registerDialects(registry: *mlir.DialectRegistry) void { - inline for (.{cuda_tile}) |dialect| dialect.registerDialect(registry); -} - -pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; - -pub const Value = struct { - inner: *const mlir.Value, - kernel: ?*Builder = null, - - pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); - - pub fn type_(self: Value) *const mlir.Type { - return self.inner.type_(); - } - - fn kern(self: Value) *Builder { - return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); - } - - pub fn tile(self: Value) *const cuda_tile.TileType { - return self.type_().isA(cuda_tile.TileType) orelse std.debug.panic("Value {f} is not a tile", .{self.type_()}); - } - - pub fn isTile(self: Value) bool { - return self.type_().isA(cuda_tile.TileType) != null; - } - - pub fn elemType(self: Value) *const mlir.Type { - return self.tile().elementType(); - } - - pub fn rank(self: Value) usize { - return self.tile().rank(); - } - - pub fn isScalar(self: Value) bool { - return self.rank() == 0; - } - - pub fn dim(self: Value, i: usize) i64 { - return self.tile().dimension(i); - } - - pub fn shape(self: Value) Value.Shape { - var out: Value.Shape = .empty; - const t = self.tile(); - for (0..t.rank()) |i| out.appendAssumeCapacity(t.dimension(i)); - return out; - } - - pub fn isFloatElem(self: Value) bool { - const et = self.elemType(); - inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |f_name| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, f_name))) != null) return true; - } - return false; - } - - pub fn isIntElem(self: Value) bool { - return self.elemType().isA(mlir.IntegerType) != null; - } - - pub fn isPtrElem(self: Value) bool { - return self.elemType().isA(cuda_tile.PointerType) != null; - } - - pub fn add(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); - } - - pub fn sub(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); - } - - pub fn mul(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); - } - - pub fn div(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.divf(l, r) else k.divi(l, r, .signed); - } - - pub fn rem(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.remf(l, r) else k.remi(l, r, .signed); - } - - pub fn cdiv(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - if (l.isFloatElem()) @panic("Value.cdiv is integer-only; use div (and ceil) on floats"); - return k.diviOpts(l, r, .{ .signedness = .signed, .rounding = .positive_inf }); - } - - pub fn neg(self: Value) Value { - const k = self.kern(); - return if (self.isFloatElem()) k.negf(self) else k.negi(self); - } - - pub fn abs(self: Value) Value { - const k = self.kern(); - return if (self.isFloatElem()) k.absf(self) else k.absi(self); - } - - pub fn bitAnd(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.andi(l, r); - } - - pub fn bitOr(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.ori(l, r); - } - - pub fn bitXor(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.xori(l, r); - } - - pub fn shl(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.shli(l, r); - } - - pub fn shr(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.shri(l, r, .signed); - } - - pub fn minimum(self: Value, rhs: anytype) Value { - return self.kern().minimum(self, rhs); - } - - pub fn maximum(self: Value, rhs: anytype) Value { - return self.kern().maximum(self, rhs); - } - - fn compare(self: Value, rhs: anytype, pred: ComparisonPredicate) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.cmpf(pred, l, r) else k.cmpi(pred, l, r, .signed); - } - - pub fn lt(self: Value, rhs: anytype) Value { - return self.compare(rhs, .less_than); - } - pub fn le(self: Value, rhs: anytype) Value { - return self.compare(rhs, .less_than_or_equal); - } - pub fn gt(self: Value, rhs: anytype) Value { - return self.compare(rhs, .greater_than); - } - pub fn ge(self: Value, rhs: anytype) Value { - return self.compare(rhs, .greater_than_or_equal); - } - pub fn eq(self: Value, rhs: anytype) Value { - return self.compare(rhs, .equal); - } - pub fn ne(self: Value, rhs: anytype) Value { - return self.compare(rhs, .not_equal); - } - - pub fn to(self: Value, dtype: DType) Value { - return self.kern().cast(self, dtype); - } - - pub fn bitCast(self: Value, dtype: DType) Value { - return self.kern().bitCast(self, dtype); - } - - pub fn reshape(self: Value, shape_: []const i64) Value { - return self.kern().reshape(self, shape_); - } - - pub fn broadcastTo(self: Value, shape_: []const i64) Value { - return self.kern().broadcastTo(self, shape_); - } - - pub fn expandDims(self: Value, axis: usize) Value { - return self.kern().expandDims(self, axis); - } - - pub fn permute(self: Value, order: []const i32) Value { - return self.kern().permute(self, order); - } - - pub fn sum(self: Value, axis: usize) Value { - return self.kern().sum(self, axis); - } - - pub fn max(self: Value, axis: usize) Value { - return self.kern().max(self, axis); - } - - pub fn min(self: Value, axis: usize) Value { - return self.kern().min(self, axis); - } - - pub fn cumsum(self: Value, axis: usize) Value { - return self.kern().cumsum(self, axis); - } - - pub fn offset(self: Value, off: anytype) Value { - const k = self.kern(); - const o: Value = if (@TypeOf(off) == Value) off else k.lift(off); - const p, const o2 = k.coerceShapes(self, o); - return k.offset(p, o2); - } -}; - -pub const Loaded = struct { - tile: Value, - token: Value, -}; - -pub const BlockId = struct { x: Value, y: Value, z: Value }; - -pub const ArgSpec = struct { - name: []const u8, - kind: Kind, - - /// XLA passes one raw device pointer per custom-call argument, so every - /// parameter is a rank-0 `tile>` and views are built inside the - /// kernel. A scalar reaches the kernel through a pointer (`loadPtr` on a - /// rank-0 pointer tile) or as a comptime constant in `cfg`. - pub const Kind = union(enum) { - ptr: DType, - ptr_opts: PtrOpts, - }; - - /// `div_by` becomes a `cuda_tile.assume #cuda_tile.div_by` at the top - /// of the entry, and `arg(i)` returns the refined value. XLA guarantees - /// only 16 bytes on the pointer it passes. - pub const PtrOpts = struct { - dtype: DType, - div_by: ?u32 = 16, - }; -}; - -pub const Opts = struct { - hints: []const EntryHint = &.{}, -}; - -pub const FloatOpts = struct { - rounding: RoundingMode = .nearest_even, - flush_to_zero: bool = false, -}; - -pub const MinMaxOpts = struct { - propagate_nan: bool = false, - flush_to_zero: bool = false, -}; - -pub const IntOpts = struct { - overflow: IntegerOverflow = .none, -}; - -pub const DivIOpts = struct { - signedness: Signedness = .signed, - rounding: ?RoundingMode = null, -}; - -/// `rounding` null picks the target's legal default: `nearest_even`, except -/// `f8e8m0fnu`, which the dialect only lets you reach with `zero` or -/// `positive_inf`. -pub const CastOpts = struct { - rounding: ?RoundingMode = null, - signedness: Signedness = .signed, -}; - -/// `ftoi` admits exactly one rounding mode (`nearest_int_to_zero`), so only -/// the signedness is a choice. -pub const FToIOpts = struct { - signedness: Signedness = .signed, -}; - -pub const MmaOpts = struct { - /// 13.3 only. - fast_acc: bool = false, -}; - -pub const PartitionOpts = struct { - padding: ?PaddingValue = .zero, - dim_map: []const i32 = &.{}, -}; - -pub const LoadOpts = struct { - ordering: MemoryOrdering = .weak, - scope: ?MemoryScope = null, - token: ?Value = null, - hints: []const LoadStoreHint = &.{}, -}; - -pub const StoreOpts = LoadOpts; - -/// The pointer arm takes `latency` hints only; `allow_tma` is a view-op hint. -pub const LoadPtrOpts = struct { - mask: ?Value = null, - padding: ?Value = null, - ordering: MemoryOrdering = .weak, - scope: ?MemoryScope = null, - token: ?Value = null, - hints: []const PtrLoadStoreHint = &.{}, -}; - -pub const StorePtrOpts = struct { - mask: ?Value = null, - ordering: MemoryOrdering = .weak, - scope: ?MemoryScope = null, - token: ?Value = null, - hints: []const PtrLoadStoreHint = &.{}, -}; - -pub const AtomicOpts = struct { - ordering: MemoryOrdering = .acq_rel, - scope: MemoryScope = .device, - mask: ?Value = null, - token: ?Value = null, -}; - -/// `atomic_red_view_tko` is relaxed-only, scoped to the block or device, and -/// has no mask operand — the view's padding decides what is touched. -pub const RedViewOpts = struct { - scope: MemoryScope = .device, - token: ?Value = null, -}; - -pub const AllocaOpts = struct { - alignment: i64 = 16, - global: bool = false, -}; - -pub const GlobalOpts = struct { - alignment: ?i64 = null, - constant: bool = false, - visibility: ?cuda_tile.SymbolVisibility = null, -}; - -pub const ScanOpts = struct { - reverse: bool = false, -}; - -pub fn ForScope(comptime N: usize) type { - return cf.ForScope(Builder, Value, N); -} - -pub const ForOpts = cf.ForOpts; - -pub const IfOnlyScope = cf.IfOnlyScope(Builder, Value); - -pub fn IfScope(comptime N: usize) type { - return cf.IfScope(Builder, Value, N); -} - -pub fn LoopScope(comptime N: usize) type { - return cf.LoopScope(Builder, Value, N); -} - -/// A custom reduction: `combine(builder, element, accumulator, ctx)` runs on -/// rank-0 tiles of `src`'s element type; `identity` is the element attribute -/// the dialect requires (see `Builder.identityAttr`). -pub fn ReduceArgs(comptime CtxT: type) type { - return struct { - src: Value, - axis: usize, - identity: *const mlir.Attribute, - combine: *const fn (*Builder, Value, Value, CtxT) Value, - }; -} - -pub fn ScanArgs(comptime CtxT: type) type { - return struct { - src: Value, - axis: usize, - reverse: bool = false, - identity: *const mlir.Attribute, - combine: *const fn (*Builder, Value, Value, CtxT) Value, - }; -} - -/// A reduction over several same-shaped tiles at once (argmax, an online -/// softmax's (max, sum)): `combine(builder, elements, accumulators, ctx)` -/// gets one rank-0 tile per source in each slice and returns the new -/// accumulators, one per source; `identities` is one element attribute per -/// source. -pub fn ReduceMultiArgs(comptime CtxT: type) type { - return struct { - srcs: []const Value, - axis: usize, - identities: []const *const mlir.Attribute, - combine: *const fn (*Builder, []const Value, []const Value, CtxT) []const Value, - }; -} - -pub fn ScanMultiArgs(comptime CtxT: type) type { - return struct { - srcs: []const Value, - axis: usize, - reverse: bool = false, - identities: []const *const mlir.Attribute, - combine: *const fn (*Builder, []const Value, []const Value, CtxT) []const Value, - }; -} - -pub const Builder = struct { - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - ctx: *mlir.Context, - module: *mlir.Module, - name: []const u8, - func_op: ?*mlir.Operation, - entry_block: ?*mlir.Block, - block_stack: common.BlockStack = .{}, - ct_module: ?*mlir.Operation = null, - ct_body: ?*mlir.Block = null, - refined_args: []?Value = &.{}, - - pub fn init( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - name: []const u8, - args: []const ArgSpec, - ) !Builder { - return initOpts(allocator, ctx, name, args, .{}); - } - - pub fn initOpts( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - name: []const u8, - args: []const ArgSpec, - opts: Opts, - ) !Builder { - var b = try open(allocator, ctx, name); - errdefer b.deinit(); - try b.declareArgsLowOpts(args, opts); - return b; - } - - pub fn open( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - name: []const u8, - ) !Builder { - const unknown_loc: *const mlir.Location = .unknown(ctx); - const module: *mlir.Module = .init(unknown_loc); - errdefer module.deinit(); - - var arena: std.heap.ArenaAllocator = .init(allocator); - errdefer arena.deinit(); - - return .{ - .allocator = allocator, - .arena = arena, - .ctx = ctx, - .module = module, - .name = name, - .func_op = null, - .entry_block = null, - .block_stack = .{}, - }; - } - - pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { - return self.declareArgsOpts(spec, .{}); - } - - pub fn declareArgsOpts(self: *Builder, spec: anytype, opts: Opts) !common.NamedArgs(@TypeOf(spec), Value) { - const Spec = @TypeOf(spec); - const field_names = @typeInfo(Spec).@"struct".field_names; - - var arg_specs: [field_names.len]ArgSpec = undefined; - inline for (0.., field_names) |i, field_name| { - const raw = @field(spec, field_name); - const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; - const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); - const inner = @field(raw, variant); - break :blk switch (tag) { - .ptr => .{ .ptr = inner }, - .ptr_opts => .{ .ptr_opts = .{ - .dtype = inner.dtype, - .div_by = if (@hasField(@TypeOf(inner), "div_by")) inner.div_by else 16, - } }, - }; - }; - arg_specs[i] = .{ .name = field_name, .kind = kind }; - } - - try self.declareArgsLowOpts(&arg_specs, opts); - - var named: common.NamedArgs(Spec, Value) = undefined; - inline for (0.., field_names) |i, field_name| { - @field(named, field_name) = self.arg(i); - } - return named; - } - - fn declareArgsLowOpts(self: *Builder, args: []const ArgSpec, opts: Opts) !void { - std.debug.assert(self.entry_block == null); - const ctx = self.ctx; - const unknown_loc: *const mlir.Location = .unknown(ctx); - const scratch = self.arena.allocator(); - - const arg_types = try scratch.alloc(*const mlir.Type, args.len); - const arg_locs = try scratch.alloc(*const mlir.Location, args.len); - for (arg_types, arg_locs, args) |*ty, *a_loc, a| { - ty.* = switch (a.kind) { - .ptr => |dt| self.ptrTy(dt), - .ptr_opts => |p| self.ptrTy(p.dtype), - }; - a_loc.* = unknown_loc; - } - - const entry = mlir.Block.init(arg_types, arg_locs); - const ct_body = mlir.Block.init(&.{}, &.{}); - const ct_module = cuda_tile.module(ctx, self.name, ct_body, unknown_loc); - _ = ct_module.appendTo(self.module.body()); - - const func_op = cuda_tile.entry(ctx, .{ - .name = self.name, - .block = entry, - .optimization_hints = if (opts.hints.len > 0) cuda_tile.optimizationHintsAttr(ctx, opts.hints) else null, - .location = unknown_loc, - }); - _ = func_op.appendTo(ct_body); - - self.func_op = func_op; - self.entry_block = entry; - self.block_stack.push(self.arena.allocator(), entry); - self.ct_module = ct_module; - self.ct_body = ct_body; - - self.refined_args = try scratch.alloc(?Value, args.len); - for (self.refined_args, args, 0..) |*refined, a, i| { - refined.* = switch (a.kind) { - .ptr => self.assumeDivBy(self.rawArg(i), 16), - .ptr_opts => |p| if (p.div_by) |n| self.assumeDivBy(self.rawArg(i), n) else null, - }; - } - } - - pub fn deinit(self: *Builder) void { - self.module.deinit(); - self.arena.deinit(); - } - - fn rawArg(self: *Builder, i: usize) Value { - const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); - return .{ .inner = eb.argument(i), .kernel = self }; - } - - /// Parameter `i`, refined by its `assume` when one was declared. The - /// owner is re-stamped: `initOpts` fills `refined_args` through a Builder - /// that is then moved out of its frame. - pub fn arg(self: *Builder, i: usize) Value { - if (i < self.refined_args.len) { - if (self.refined_args[i]) |v| return .{ .inner = v.inner, .kernel = self }; - } - return self.rawArg(i); - } - - pub fn pushBlock(self: *Builder, b: *mlir.Block) void { - self.block_stack.push(self.arena.allocator(), b); - } - - pub fn popBlock(self: *Builder) void { - self.block_stack.pop(); - } - - pub fn currentBlock(self: *Builder) *mlir.Block { - return self.block_stack.current(); - } - - fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { - const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); - for (values, 0..) |v, i| out[i] = v.inner; - return out; - } - - fn innerOpt(v: ?Value) ?*const mlir.Value { - return if (v) |x| x.inner else null; - } - - pub fn emit(self: *Builder, op: *mlir.Operation) Value { - _ = op.appendTo(self.currentBlock()); - return .{ .inner = op.result(0), .kernel = self }; - } - - pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { - _ = op.appendTo(self.currentBlock()); - const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); - for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; - return out; - } - - pub fn yield(self: *Builder, values: anytype) []const Value { - const T = @TypeOf(values); - const info = @typeInfo(T); - if (info != .@"struct" or !info.@"struct".is_tuple) - @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".field_names.len; - const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".field_names, 0..) |f, i| { - if (@FieldType(T, f) != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(@FieldType(T, f))); - out[i] = @field(values, f); - } - return out; - } - - pub fn loc(self: *const Builder) *const mlir.Location { - return .unknown(self.ctx); - } - - /// `tile`; `&.{}` for a scalar. - pub fn tileTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { - if (!dtypes.tileLegal(shape)) { - std.debug.panic("Builder.tileTy: {any} is not a legal tile shape (power-of-two dims, at most 2^24 elements)", .{shape}); - } - return cuda_tile.tileType(self.ctx, shape, dtype.toMlir(self.ctx)); - } - - pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { - return self.tileTy(&.{}, dtype); - } - - /// `tile>` — the type of a buffer parameter. - pub fn ptrTy(self: *const Builder, dtype: DType) *const mlir.Type { - return self.ptrTileTy(&.{}, dtype); - } - - /// `tile>`. `i4` is a tile element type but not a pointee - /// (Types.td: CudaTile_NumberType excludes Int4), so a 4-bit buffer is - /// declared as `.i8` and `unpack`ed in the kernel. - pub fn ptrTileTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { - if (dtype == .i4) @panic("Builder: i4 is a tile element type only, never a pointee; declare the buffer as .i8 and unpack in-kernel"); - return cuda_tile.tileType(self.ctx, shape, cuda_tile.pointerType(self.ctx, dtype.toMlir(self.ctx))); - } - - pub fn tokenTy(self: *const Builder) *const mlir.Type { - return cuda_tile.tokenType(self.ctx); - } - - fn tileLike(self: *const Builder, v: Value, elem: *const mlir.Type) *const mlir.Type { - return cuda_tile.tileType(self.ctx, v.shape().constSlice(), elem); - } - - fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { - return dtypes.mlirElemToDType(self.ctx, elem); - } - - fn pointee(v: Value) *const mlir.Type { - const p = v.elemType().isA(cuda_tile.PointerType) orelse std.debug.panic("expected a pointer tile, got {f}", .{v.type_()}); - return p.pointee(); - } - - fn elemAttr(self: *Builder, dtype: DType, value: anytype) *const mlir.Attribute { - const elem = dtype.toMlir(self.ctx); - const T = @TypeOf(value); - if (isFloatDtype(dtype)) { - const v: f64 = switch (@typeInfo(T)) { - .comptime_int, .int => @floatFromInt(value), - .comptime_float, .float => @floatCast(value), - else => @compileError("Builder: unsupported scalar " ++ @typeName(T)), - }; - return cuda_tile.floatElemAttr(self.ctx, elem, v); - } - const v: i128 = switch (@typeInfo(T)) { - .comptime_int, .int => value, - .bool => @intFromBool(value), - .comptime_float, .float => @intFromFloat(value), - else => @compileError("Builder: unsupported scalar " ++ @typeName(T)), - }; - return cuda_tile.intElemAttr(elem, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); - } - - /// `value` as the `i64` an integer constant of `bits` bits is built from. - /// It may be signed or an unsigned bit pattern, so it must lie in - /// [-2^(bits-1), 2^bits). - fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { - if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) - std.debug.panic("Builder: {d} does not fit {s}", .{ value, dtype }); - return @truncate(value); - } - - fn splat(self: *Builder, tile_ty: *const mlir.Type, dtype: DType, value: anytype) Value { - return self.emit(cuda_tile.constant(self.ctx, cuda_tile.splatAttr(tile_ty, self.elemAttr(dtype, value)), tile_ty, self.loc())); - } - - /// `value` as a rank-0 constant of `dtype`; a Value passes through. An - /// integer is signed or an unsigned bit pattern of `dtype`'s width; - /// anything else panics. - pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { - if (@TypeOf(value) == Value) return value; - return self.splat(self.scalarTy(dtype), dtype, value); - } - - pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { - return self.splat(self.tileTy(shape, dtype), dtype, value); - } - - pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { - return self.full(shape, 0, dtype); - } - - pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { - return self.full(shape, 1, dtype); - } - - /// A dense constant from a slice of Zig scalars in row-major order. MLIR - /// stores every element in whole bytes except `i1`, which it bit-packs; - /// the slice must hold exactly that many bytes (use `full` for a splat). - pub fn dense(self: *Builder, shape: []const i64, dtype: DType, values: anytype) Value { - checkDenseBytes("Builder.dense", shape, dtype, values); - const ty = self.tileTy(shape, dtype); - return self.emit(cuda_tile.constant(self.ctx, cuda_tile.denseAttr(ty, values), ty, self.loc())); - } - - fn checkDenseBytes(comptime who: []const u8, shape: []const i64, dtype: DType, values: anytype) void { - var elems: usize = 1; - for (shape) |d| elems *= @intCast(d); - const bits = dtypeBitwidth(dtype); - const expected: usize = if (bits == 1) (elems + 7) / 8 else ((bits + 7) / 8) * elems; - const got = std.mem.sliceAsBytes(values).len; - if (got != expected) { - std.debug.panic("{s}: {d} bytes given for {any} x {s}, {d} expected", .{ who, got, shape, @tagName(dtype), expected }); - } - } - - pub fn iota(self: *Builder, n: i64, dtype: DType) Value { - return self.emit(cuda_tile.iota(self.ctx, self.tileTy(&.{n}, dtype), self.loc())); - } - - /// The element attribute `reduce`/`scan` need as an identity. - pub fn identityAttr(self: *Builder, dtype: DType, value: anytype) *const mlir.Attribute { - return self.elemAttr(dtype, value); - } - - /// Lift a Zig scalar to a rank-0 constant. Dtype follows the source type; - /// unsigned ints are bit-cast. Use `constant` for an explicit dtype. - pub fn lift(self: *Builder, value: anytype) Value { - const T = @TypeOf(value); - if (T == Value) return value; - return switch (@typeInfo(T)) { - .comptime_int => blk: { - if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) - break :blk self.constant(.i32, value); - break :blk self.constant(.i64, value); - }, - .int => |info| if (info.bits > 32) self.constant(.i64, value) else self.constant(.i32, value), - .bool => self.constant(.i1, value), - .comptime_float => self.constant(.f32, value), - .float => |info| switch (info.bits) { - 16 => self.constant(.f16, value), - 32 => self.constant(.f32, value), - 64 => self.constant(.f64, value), - else => @compileError("Builder.lift: unsupported float bitwidth"), - }, - else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), - }; - } - - fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { - const T = @TypeOf(value); - if (T == Value) return value; - return switch (@typeInfo(T)) { - .comptime_int, .comptime_float => self.constant(self.mlirElemToDType(ref_elem), value), - else => self.lift(value), - }; - } - - /// `value` as a constant of `like`'s element type; a Value passes through. - pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { - if (@TypeOf(value) == Value) return value; - return self.constMatching(value, like.type_()); - } - - fn i32Bound(self: *Builder, value: anytype) Value { - return self.constant(.i32, value); - } - - fn constMatching(self: *Builder, value: anytype, ref: *const mlir.Type) Value { - const elem = if (ref.isA(cuda_tile.TileType)) |t| t.elementType() else ref; - return self.constant(self.mlirElemToDType(elem), value); - } - - pub fn broadcastLike(self: *Builder, value: anytype, ref: Value) Value { - const v = self.liftMatching(value, ref.elemType()); - return self.expandTo(v, ref.shape().constSlice()); - } - - fn expandTo(self: *Builder, v: Value, target: []const i64) Value { - const cur = v.shape(); - if (std.mem.eql(i64, cur.constSlice(), target)) return v; - var aligned = v; - if (cur.len != target.len) { - std.debug.assert(cur.len <= target.len); - var padded: Value.Shape = .empty; - for (0..target.len - cur.len) |_| padded.appendAssumeCapacity(1); - padded.appendSliceAssumeCapacity(cur.constSlice()); - aligned = self.reshape(v, padded.constSlice()); - } - if (std.mem.eql(i64, aligned.shape().constSlice(), target)) return aligned; - return self.broadcastTo(aligned, target); - } - - fn coerceShapes(self: *Builder, a: Value, b: Value) struct { Value, Value } { - const a_sh = a.shape(); - const b_sh = b.shape(); - if (std.mem.eql(i64, a_sh.constSlice(), b_sh.constSlice())) return .{ a, b }; - const n = @max(a_sh.len, b_sh.len); - var target: Value.Shape = .empty; - for (0..n) |i| { - const ai: i64 = if (i + a_sh.len >= n) a_sh.get(i + a_sh.len - n) else 1; - const bi: i64 = if (i + b_sh.len >= n) b_sh.get(i + b_sh.len - n) else 1; - if (ai == bi or bi == 1) { - target.appendAssumeCapacity(ai); - } else if (ai == 1) { - target.appendAssumeCapacity(bi); - } else { - std.debug.panic("Builder.coerce: incompatible shapes {any} vs {any} at axis {d}", .{ a_sh.constSlice(), b_sh.constSlice(), i }); - } - } - return .{ self.expandTo(a, target.constSlice()), self.expandTo(b, target.constSlice()) }; - } - - pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { - const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; - const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; - var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); - var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); - - if (av.isIntElem() and bv.isIntElem()) { - const a_dt = self.mlirElemToDType(av.elemType()); - const b_dt = self.mlirElemToDType(bv.elemType()); - const aw = intBitwidth(a_dt); - const bw = intBitwidth(b_dt); - if (aw < bw) av = av.to(b_dt); - if (bw < aw) bv = bv.to(a_dt); - } - - return self.coerceShapes(av, bv); - } - - pub fn tileBlockId(self: *Builder) BlockId { - const r = self.emitMulti(cuda_tile.get_tile_block_id(self.ctx, self.loc()), 3); - return .{ .x = r[0], .y = r[1], .z = r[2] }; - } - - pub fn numTileBlocks(self: *Builder) BlockId { - const r = self.emitMulti(cuda_tile.get_num_tile_blocks(self.ctx, self.loc()), 3); - return .{ .x = r[0], .y = r[1], .z = r[2] }; - } - - pub fn assumeDivBy(self: *Builder, v: Value, n: u64) Value { - return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.divByAttr(self.ctx, n, .{}), self.loc())); - } - - pub fn assumeDivByEvery(self: *Builder, v: Value, n: u64, every: i64, along: i64) Value { - return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.divByAttr(self.ctx, n, .{ .every = every, .along = along }), self.loc())); - } - - pub fn assumeSameElements(self: *Builder, v: Value, values: []const i64) Value { - return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.sameElementsAttr(self.ctx, values), self.loc())); - } - - pub fn assumeBounded(self: *Builder, v: Value, lb: ?i64, ub: ?i64) Value { - return self.emit(cuda_tile.assume(self.ctx, v.inner, cuda_tile.boundedAttr(self.ctx, .{ .lb = lb, .ub = ub }), self.loc())); - } - - pub fn tensorView(self: *Builder, base: Value, shape: []const i64, strides: []const i64) Value { - const ty = cuda_tile.tensorViewType(self.ctx, pointee(base), shape, strides); - return self.emit(cuda_tile.make_tensor_view(self.ctx, base.inner, &.{}, &.{}, ty, self.loc())); - } - - pub const Dim = union(enum) { - static: i64, - dynamic: Value, - }; - - pub fn tensorViewDyn(self: *Builder, base: Value, shape: []const Dim, strides: []const Dim) Value { - const scratch = self.arena.allocator(); - var static_shape: Value.Shape = .empty; - var static_strides: Value.Shape = .empty; - var dyn_shape = std.ArrayList(*const mlir.Value).empty; - var dyn_strides = std.ArrayList(*const mlir.Value).empty; - for (shape) |d| switch (d) { - .static => |s| static_shape.appendAssumeCapacity(s), - .dynamic => |v| { - static_shape.appendAssumeCapacity(cuda_tile.TensorViewType.dynamic()); - dyn_shape.append(scratch, v.inner) catch @panic("OOM"); - }, - }; - for (strides) |d| switch (d) { - .static => |s| static_strides.appendAssumeCapacity(s), - .dynamic => |v| { - static_strides.appendAssumeCapacity(cuda_tile.TensorViewType.dynamic()); - dyn_strides.append(scratch, v.inner) catch @panic("OOM"); - }, - }; - const ty = cuda_tile.tensorViewType(self.ctx, pointee(base), static_shape.constSlice(), static_strides.constSlice()); - return self.emit(cuda_tile.make_tensor_view(self.ctx, base.inner, dyn_shape.items, dyn_strides.items, ty, self.loc())); - } - - fn i32Slice(self: *Builder, values: []const i64) []const i32 { - const out = self.arena.allocator().alloc(i32, values.len) catch @panic("OOM"); - for (values, 0..) |v, i| out[i] = @intCast(v); - return out; - } - - /// Tiles `tv` into `tile`-shaped pieces addressed by tile-space indices. - /// `padding` fills the ragged edge on load; stores past the edge are - /// masked. That is why this DSL has no mask tile. - pub fn partitionView(self: *Builder, tv: Value, tile: []const i64, opts: PartitionOpts) Value { - if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.partitionView: {any} is not a legal tile shape", .{tile}); - const ty: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(self.ctx, .{ .tile_shape = self.i32Slice(tile), .tensor_view = tv.type_(), .dim_map = opts.dim_map, .padding = opts.padding })); - return self.emit(cuda_tile.make_partition_view(self.ctx, tv.inner, ty, self.loc())); - } - - /// 13.3: like `partitionView`, stepping by `traversal_strides` elements - /// per index instead of a whole tile. - pub fn stridedView(self: *Builder, tv: Value, tile: []const i64, traversal_strides: []const i64, opts: PartitionOpts) Value { - if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.stridedView: {any} is not a legal tile shape", .{tile}); - const ty: *const mlir.Type = @ptrCast(cuda_tile.StridedViewType.get(self.ctx, .{ - .tile_shape = self.i32Slice(tile), - .traversal_strides = self.i32Slice(traversal_strides), - .tensor_view = tv.type_(), - .dim_map = opts.dim_map, - .padding = opts.padding, - })); - return self.emit(cuda_tile.make_strided_view(self.ctx, tv.inner, ty, self.loc())); - } - - /// 13.3: a view whose `sparse_dim` is indexed by a rank-1 tile of - /// `tile[sparse_dim]` indices (every other index stays rank 0). - pub fn gatherScatterView(self: *Builder, tv: Value, tile: []const i64, sparse_dim: u32, padding: ?PaddingValue) Value { - if (!dtypes.tileLegal(tile)) std.debug.panic("Builder.gatherScatterView: {any} is not a legal tile shape", .{tile}); - const ty = cuda_tile.gatherScatterViewType(self.ctx, tile, tv.type_(), sparse_dim, .{ .padding = padding }); - return self.emit(cuda_tile.make_gather_scatter_view(self.ctx, tv.inner, ty, self.loc())); - } - - fn i32Scalars(self: *Builder, n: usize) []const *const mlir.Type { - const out = self.arena.allocator().alloc(*const mlir.Type, n) catch @panic("OOM"); - for (out) |*t| t.* = self.scalarTy(.i32); - return out; - } - - pub fn indexSpaceShape(self: *Builder, view: Value) []Value { - const n = cuda_tile.indexRankOfView(view.type_()); - return self.emitMulti(cuda_tile.get_index_space_shape(self.ctx, view.inner, self.i32Scalars(n), self.loc()), n); - } - - pub fn tensorShape(self: *Builder, tv: Value) []Value { - const t = tv.type_().isA(cuda_tile.TensorViewType) orelse std.debug.panic("expected a tensor_view, got {f}", .{tv.type_()}); - const n = t.rank(); - return self.emitMulti(cuda_tile.get_tensor_shape(self.ctx, tv.inner, self.i32Scalars(n), self.loc()), n); - } - - fn memOpts(self: *Builder, ordering: MemoryOrdering, scope: ?MemoryScope, token: ?Value, hints: anytype) cuda_tile.MemOpts { - return .{ - .ordering = ordering, - .scope = scope, - .token = innerOpt(token), - .optimization_hints = if (hints.len > 0) cuda_tile.optimizationHintsAttr(self.ctx, hints) else null, - }; - } - - pub fn load(self: *Builder, view: Value, index: []const Value) Value { - return self.loadOpts(view, index, .{}).tile; - } - - pub fn loadOpts(self: *Builder, view: Value, index: []const Value, opts: LoadOpts) Loaded { - const expected = cuda_tile.indexRankOfView(view.type_()); - if (index.len != expected) std.debug.panic("Builder.load: view takes {d} indices, got {d}", .{ expected, index.len }); - const r = self.emitMulti(cuda_tile.load_view_tko(self.ctx, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc()), 2); - return .{ .tile = r[0], .token = r[1] }; - } - - pub fn store(self: *Builder, tile: Value, view: Value, index: []const Value) Value { - return self.storeOpts(tile, view, index, .{}); - } - - pub fn storeOpts(self: *Builder, tile: Value, view: Value, index: []const Value, opts: StoreOpts) Value { - const expected = cuda_tile.indexRankOfView(view.type_()); - if (index.len != expected) std.debug.panic("Builder.store: view takes {d} indices, got {d}", .{ expected, index.len }); - return self.emit(cuda_tile.store_view_tko(self.ctx, tile.inner, view.inner, self.innerSlice(index), self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), self.loc())); - } - - /// Gather through a tile of pointers, dropping the token. This arm cannot - /// use TMA; prefer views for dense access. - pub fn loadPtr(self: *Builder, ptrs: Value) Value { - return self.loadPtrOpts(ptrs, .{}).tile; - } - - pub fn loadPtrOpts(self: *Builder, ptrs: Value, opts: LoadPtrOpts) Loaded { - const result_ty = self.tileLike(ptrs, pointee(ptrs)); - const mask: ?Value = if (opts.mask) |m| self.expandTo(m, ptrs.shape().constSlice()) else null; - const r = self.emitMulti(cuda_tile.load_ptr_tko(self.ctx, ptrs.inner, result_ty, .{ - .mask = innerOpt(mask), - .padding = innerOpt(opts.padding), - .mem = self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), - }, self.loc()), 2); - return .{ .tile = r[0], .token = r[1] }; - } - - pub fn storePtr(self: *Builder, ptrs: Value, value: Value) Value { - return self.storePtrOpts(ptrs, value, .{}); - } - - pub fn storePtrOpts(self: *Builder, ptrs: Value, value: Value, opts: StorePtrOpts) Value { - const mask: ?Value = if (opts.mask) |m| self.expandTo(m, ptrs.shape().constSlice()) else null; - return self.emit(cuda_tile.store_ptr_tko(self.ctx, ptrs.inner, value.inner, .{ - .mask = innerOpt(mask), - .mem = self.memOpts(opts.ordering, opts.scope, opts.token, opts.hints), - }, self.loc())); - } - - pub fn makeToken(self: *Builder) Value { - return self.emit(cuda_tile.make_token(self.ctx, self.loc())); - } - - pub fn joinTokens(self: *Builder, tokens: []const Value) Value { - return self.emit(cuda_tile.join_tokens(self.ctx, self.innerSlice(tokens), self.loc())); - } - - pub fn print(self: *Builder, str: []const u8, args: []const Value) Value { - return self.printOpts(str, args, null); - } - - pub fn printOpts(self: *Builder, str: []const u8, args: []const Value, token: ?Value) Value { - return self.emit(cuda_tile.print_tko(self.ctx, str, self.innerSlice(args), .{ .token = innerOpt(token) }, self.loc())); - } - - fn atomicOpts(opts: AtomicOpts) cuda_tile.AtomicOpts { - return .{ .ordering = opts.ordering, .scope = opts.scope, .mask = innerOpt(opts.mask), .token = innerOpt(opts.token) }; - } - - pub fn atomicCas(self: *Builder, ptrs: Value, cmp: Value, val: Value, opts: AtomicOpts) Loaded { - const r = self.emitMulti(cuda_tile.atomic_cas_tko(self.ctx, ptrs.inner, cmp.inner, val.inner, atomicOpts(opts), self.loc()), 2); - return .{ .tile = r[0], .token = r[1] }; - } - - pub fn atomicRmw(self: *Builder, mode: AtomicRMWMode, ptrs: Value, operand: Value, opts: AtomicOpts) Loaded { - const r = self.emitMulti(cuda_tile.atomic_rmw_tko(self.ctx, ptrs.inner, mode, operand.inner, atomicOpts(opts), self.loc()), 2); - return .{ .tile = r[0], .token = r[1] }; - } - - /// 13.3: an atomic reduction into an unpadded tiled view; returns the - /// token. `xchg` is not a reduction and the dialect rejects it. - pub fn atomicRedView(self: *Builder, mode: AtomicRMWMode, view: Value, index: []const Value, value: Value, opts: RedViewOpts) Value { - return self.emit(cuda_tile.atomic_red_view_tko(self.ctx, view.inner, self.innerSlice(index), mode, value.inner, .{ - .scope = opts.scope, - .token = innerOpt(opts.token), - }, self.loc())); - } - - /// 13.3: `num_elem` elements of `dtype`, as a rank-0 pointer tile. - pub fn alloca(self: *Builder, dtype: DType, num_elem: i64, opts: AllocaOpts) Value { - return self.emit(cuda_tile.alloca(self.ctx, num_elem, opts.alignment, opts.global, self.ptrTy(dtype), self.loc())); - } - - pub fn global(self: *Builder, name: []const u8, shape: []const i64, dtype: DType, value: anytype, opts: GlobalOpts) void { - const ty = self.tileTy(shape, dtype); - const attr = cuda_tile.splatAttr(ty, self.elemAttr(dtype, value)); - self.appendGlobal(cuda_tile.global(self.ctx, name, attr, .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); - } - - /// A module-level dense buffer from a slice of Zig scalars (see `dense` - /// for the byte-count rule). - pub fn globalDense(self: *Builder, name: []const u8, shape: []const i64, dtype: DType, values: anytype, opts: GlobalOpts) void { - checkDenseBytes("Builder.globalDense", shape, dtype, values); - const ty = self.tileTy(shape, dtype); - self.appendGlobal(cuda_tile.global(self.ctx, name, cuda_tile.denseAttr(ty, values), .{ .alignment = opts.alignment, .constant = opts.constant, .visibility = opts.visibility }, self.loc())); - } - - fn appendGlobal(self: *Builder, op: *mlir.Operation) void { - const body = self.ct_body orelse @panic("Builder.global called before declareArgs"); - const first = body.firstOperation() orelse { - body.appendOwnedOperation(op); - return; - }; - body.insertOwnedOperationBefore(first, op); - } - - pub fn getGlobal(self: *Builder, name: []const u8, dtype: DType) Value { - return self.emit(cuda_tile.get_global(self.ctx, name, self.ptrTy(dtype), self.loc())); - } - - pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.addfOpts(lhs, rhs, .{}); - } - pub fn addfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { - return self.emit(cuda_tile.addf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); - } - pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.subfOpts(lhs, rhs, .{}); - } - pub fn subfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { - return self.emit(cuda_tile.subf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); - } - pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.mulfOpts(lhs, rhs, .{}); - } - pub fn mulfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { - return self.emit(cuda_tile.mulf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); - } - pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.divfOpts(lhs, rhs, .{}); - } - pub fn divfOpts(self: *Builder, lhs: Value, rhs: Value, opts: FloatOpts) Value { - return self.emit(cuda_tile.divf(self.ctx, lhs.inner, rhs.inner, opts.rounding, opts.flush_to_zero, self.loc())); - } - pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(cuda_tile.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn negf(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.negf(self.ctx, x.inner, self.loc())); - } - pub fn absf(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.absf(self.ctx, x.inner, self.loc())); - } - pub fn fma(self: *Builder, a: Value, b: Value, acc: Value) Value { - return self.fmaOpts(a, b, acc, .{}); - } - pub fn fmaOpts(self: *Builder, a: Value, b: Value, acc: Value, opts: FloatOpts) Value { - return self.emit(cuda_tile.fma(self.ctx, a.inner, b.inner, acc.inner, opts.rounding, opts.flush_to_zero, self.loc())); - } - pub fn maxf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.maxfOpts(lhs, rhs, .{}); - } - pub fn maxfOpts(self: *Builder, lhs: Value, rhs: Value, opts: MinMaxOpts) Value { - return self.emit(cuda_tile.maxf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); - } - pub fn minf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.minfOpts(lhs, rhs, .{}); - } - pub fn minfOpts(self: *Builder, lhs: Value, rhs: Value, opts: MinMaxOpts) Value { - return self.emit(cuda_tile.minf(self.ctx, lhs.inner, rhs.inner, opts.propagate_nan, opts.flush_to_zero, self.loc())); - } - pub fn pow(self: *Builder, x: Value, exponent: Value) Value { - return self.emit(cuda_tile.pow(self.ctx, x.inner, exponent.inner, self.loc())); - } - pub fn atan2(self: *Builder, x: Value, y: Value) Value { - return self.emit(cuda_tile.atan2(self.ctx, x.inner, y.inner, self.loc())); - } - pub fn exp(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.exp(self.ctx, x.inner, .{}, self.loc())); - } - /// 13.3: `exp` with an explicit rounding mode. - pub fn expRound(self: *Builder, x: Value, rounding: RoundingMode) Value { - return self.emit(cuda_tile.exp(self.ctx, x.inner, .{ .rounding = rounding }, self.loc())); - } - pub fn exp2(self: *Builder, x: Value) Value { - return self.exp2Opts(x, false); - } - pub fn exp2Opts(self: *Builder, x: Value, flush_to_zero: bool) Value { - return self.emit(cuda_tile.exp2(self.ctx, x.inner, flush_to_zero, self.loc())); - } - pub fn log(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.log(self.ctx, x.inner, self.loc())); - } - pub fn log2(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.log2(self.ctx, x.inner, self.loc())); - } - pub fn sqrt(self: *Builder, x: Value) Value { - return self.sqrtOpts(x, .{}); - } - pub fn sqrtOpts(self: *Builder, x: Value, opts: FloatOpts) Value { - return self.emit(cuda_tile.sqrt(self.ctx, x.inner, opts.rounding, opts.flush_to_zero, self.loc())); - } - pub fn rsqrt(self: *Builder, x: Value) Value { - return self.rsqrtOpts(x, false); - } - pub fn rsqrtOpts(self: *Builder, x: Value, flush_to_zero: bool) Value { - return self.emit(cuda_tile.rsqrt(self.ctx, x.inner, flush_to_zero, self.loc())); - } - pub fn sin(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.sin(self.ctx, x.inner, self.loc())); - } - pub fn cos(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.cos(self.ctx, x.inner, self.loc())); - } - pub fn tan(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.tan(self.ctx, x.inner, self.loc())); - } - pub fn sinh(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.sinh(self.ctx, x.inner, self.loc())); - } - pub fn cosh(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.cosh(self.ctx, x.inner, self.loc())); - } - pub fn tanh(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.tanh(self.ctx, x.inner, .{}, self.loc())); - } - /// 13.2: `tanh` with an explicit rounding mode. - pub fn tanhRound(self: *Builder, x: Value, rounding: RoundingMode) Value { - return self.emit(cuda_tile.tanh(self.ctx, x.inner, .{ .rounding = rounding }, self.loc())); - } - pub fn ceil(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.ceil(self.ctx, x.inner, self.loc())); - } - pub fn floor(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.floor(self.ctx, x.inner, self.loc())); - } - - pub fn cmpf(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value) Value { - return self.cmpfOpts(predicate, lhs, rhs, .ordered); - } - pub fn cmpfOpts(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value, ordering: ComparisonOrdering) Value { - return self.emit(cuda_tile.cmpf(self.ctx, predicate, ordering, lhs.inner, rhs.inner, self.loc())); - } - - pub fn mmaf(self: *Builder, a: Value, b: Value, acc: Value) Value { - return self.mmafOpts(a, b, acc, .{}); - } - pub fn mmafOpts(self: *Builder, a: Value, b: Value, acc: Value, opts: MmaOpts) Value { - return self.emit(cuda_tile.mmaf(self.ctx, a.inner, b.inner, acc.inner, opts.fast_acc, self.loc())); - } - /// 13.3: block-scaled MMA. Scales are plain tiles in logical layout — - /// `a_scale` is `[M, K/V]`, `b_scale` is `[K/V, N]` — never pre-swizzled. - pub fn mmafScaled(self: *Builder, a: Value, b: Value, acc: Value, a_scale: Value, b_scale: Value) Value { - return self.emit(cuda_tile.mmaf_scaled(self.ctx, a.inner, b.inner, acc.inner, a_scale.inner, b_scale.inner, self.loc())); - } - pub fn mmai(self: *Builder, a: Value, b: Value, acc: Value, a_signedness: Signedness, b_signedness: Signedness) Value { - return self.emit(cuda_tile.mmai(self.ctx, a.inner, b.inner, acc.inner, a_signedness, b_signedness, self.loc())); - } - - pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.addiOpts(lhs, rhs, .{}); - } - pub fn addiOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { - return self.emit(cuda_tile.addi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); - } - pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.subiOpts(lhs, rhs, .{}); - } - pub fn subiOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { - return self.emit(cuda_tile.subi(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); - } - pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { - return self.muliOpts(lhs, rhs, .{}); - } - pub fn muliOpts(self: *Builder, lhs: Value, rhs: Value, opts: IntOpts) Value { - return self.emit(cuda_tile.muli(self.ctx, lhs.inner, rhs.inner, opts.overflow, self.loc())); - } - pub fn mulhii(self: *Builder, x: Value, y: Value) Value { - return self.emit(cuda_tile.mulhii(self.ctx, x.inner, y.inner, self.loc())); - } - pub fn divi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { - return self.diviOpts(lhs, rhs, .{ .signedness = signedness }); - } - pub fn diviOpts(self: *Builder, lhs: Value, rhs: Value, opts: DivIOpts) Value { - return self.emit(cuda_tile.divi(self.ctx, lhs.inner, rhs.inner, opts.signedness, .{ .rounding = opts.rounding }, self.loc())); - } - pub fn remi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { - return self.emit(cuda_tile.remi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); - } - pub fn maxi(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { - return self.emit(cuda_tile.maxi(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); - } - pub fn mini(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { - return self.emit(cuda_tile.mini(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); - } - pub fn negi(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.negi(self.ctx, x.inner, .none, self.loc())); - } - pub fn absi(self: *Builder, x: Value) Value { - return self.emit(cuda_tile.absi(self.ctx, x.inner, self.loc())); - } - pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(cuda_tile.shli(self.ctx, lhs.inner, rhs.inner, .none, self.loc())); - } - pub fn shri(self: *Builder, lhs: Value, rhs: Value, signedness: Signedness) Value { - return self.emit(cuda_tile.shri(self.ctx, lhs.inner, rhs.inner, signedness, self.loc())); - } - pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(cuda_tile.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(cuda_tile.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(cuda_tile.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn cmpi(self: *Builder, predicate: ComparisonPredicate, lhs: Value, rhs: Value, signedness: Signedness) Value { - return self.emit(cuda_tile.cmpi(self.ctx, predicate, signedness, lhs.inner, rhs.inner, self.loc())); - } - - pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { - const l, const r = self.coerce(a, b); - return if (l.isFloatElem()) self.maxf(l, r) else self.maxi(l, r, .signed); - } - - pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { - const l, const r = self.coerce(a, b); - return if (l.isFloatElem()) self.minf(l, r) else self.mini(l, r, .signed); - } - - pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { - return self.emit(cuda_tile.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); - } - - pub fn where(self: *Builder, cond: Value, x: anytype, y: anytype) Value { - var xv, var yv = self.coerce(x, y); - var cv = cond; - cv, xv = self.coerceShapes(cv, xv); - xv, yv = self.coerceShapes(xv, yv); - cv, _ = self.coerceShapes(cv, xv); - return self.select(cv, xv, yv); - } - - pub fn bitCast(self: *Builder, src: Value, dtype: DType) Value { - return self.emit(cuda_tile.bitcast(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); - } - pub fn exti(self: *Builder, src: Value, dtype: DType, signedness: Signedness) Value { - return self.emit(cuda_tile.exti(self.ctx, src.inner, signedness, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); - } - pub fn trunci(self: *Builder, src: Value, dtype: DType) Value { - return self.emit(cuda_tile.trunci(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), .none, self.loc())); - } - pub fn ftof(self: *Builder, src: Value, dtype: DType) Value { - return self.ftofOpts(src, dtype, .{}); - } - pub fn ftofOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { - const rounding: RoundingMode = opts.rounding orelse (if (dtype == .f8e8m0fnu) .zero else .nearest_even); - return self.emit(cuda_tile.ftof(self.ctx, src.inner, self.tileLike(src, dtype.toMlir(self.ctx)), .{ .rounding = rounding }, self.loc())); - } - /// Float -> integer, truncating toward zero (the only rounding `ftoi` - /// admits). - pub fn ftoi(self: *Builder, src: Value, dtype: DType) Value { - return self.ftoiOpts(src, dtype, .{}); - } - pub fn ftoiOpts(self: *Builder, src: Value, dtype: DType, opts: FToIOpts) Value { - return self.emit(cuda_tile.ftoi(self.ctx, src.inner, opts.signedness, .nearest_int_to_zero, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); - } - /// Integer -> float. `f8e8m0fnu` is not a legal target; go through `f32`. - pub fn itof(self: *Builder, src: Value, dtype: DType) Value { - return self.itofOpts(src, dtype, .{}); - } - pub fn itofOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { - if (dtype == .f8e8m0fnu) @panic("Builder.itof: the dialect refuses integer -> f8e8m0fnu; convert to .f32 first (Builder.cast does)"); - return self.emit(cuda_tile.itof(self.ctx, src.inner, opts.signedness, opts.rounding orelse .nearest_even, self.tileLike(src, dtype.toMlir(self.ctx)), self.loc())); - } - pub fn intToPtr(self: *Builder, src: Value, pointee_dtype: DType) Value { - return self.emit(cuda_tile.int_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); - } - pub fn ptrToInt(self: *Builder, src: Value) Value { - return self.emit(cuda_tile.ptr_to_int(self.ctx, src.inner, self.tileLike(src, DType.i64.toMlir(self.ctx)), self.loc())); - } - pub fn ptrToPtr(self: *Builder, src: Value, pointee_dtype: DType) Value { - return self.emit(cuda_tile.ptr_to_ptr(self.ctx, src.inner, self.ptrTileTy(src.shape().constSlice(), pointee_dtype), self.loc())); - } - /// 13.3: a sub-byte tile packed into `tile`. - pub fn pack(self: *Builder, src: Value, shape: []const i64) Value { - return self.emit(cuda_tile.pack(self.ctx, src.inner, self.tileTy(shape, .i8), self.loc())); - } - /// 13.3: an i8 tile unpacked into `tile` of a sub-byte `dtype`. - pub fn unpack(self: *Builder, src: Value, shape: []const i64, dtype: DType) Value { - return self.emit(cuda_tile.unpack(self.ctx, src.inner, self.tileTy(shape, dtype), self.loc())); - } - - /// Numeric cast with auto-dispatch: ftof / ftoi / itof / exti / trunci / - /// bitcast by element kind and width. Integers are signed; i1 is not. - pub fn cast(self: *Builder, src: Value, dtype: DType) Value { - const cur_elem = src.elemType(); - const tgt_elem = dtype.toMlir(self.ctx); - if (cur_elem.eql(tgt_elem)) return src; - const cur_dtype = self.mlirElemToDType(cur_elem); - const cur_is_float = src.isFloatElem(); - const tgt_is_float = isFloatDtype(dtype); - if (cur_is_float and tgt_is_float) return self.ftof(src, dtype); - if (cur_is_float) return self.ftoi(src, dtype); - if (tgt_is_float) { - const signedness: Signedness = if (cur_dtype == .i1) .unsigned else .signed; - if (dtype == .f8e8m0fnu) return self.ftof(self.itofOpts(src, .f32, .{ .signedness = signedness }), dtype); - return self.itofOpts(src, dtype, .{ .signedness = signedness }); - } - const cur_bw = dtypeBitwidth(cur_dtype); - const tgt_bw = dtypeBitwidth(dtype); - if (tgt_bw > cur_bw) return self.exti(src, dtype, if (cur_dtype == .i1) .unsigned else .signed); - if (tgt_bw < cur_bw) return self.trunci(src, dtype); - return self.bitCast(src, dtype); - } - - pub fn reshape(self: *Builder, src: Value, shape: []const i64) Value { - return self.emit(cuda_tile.reshape(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); - } - - fn tileTyOf(self: *const Builder, shape: []const i64, elem: *const mlir.Type) *const mlir.Type { - if (!dtypes.tileLegal(shape)) std.debug.panic("Builder: {any} is not a legal tile shape", .{shape}); - return cuda_tile.tileType(self.ctx, shape, elem); - } - - pub fn broadcastTo(self: *Builder, src: Value, shape: []const i64) Value { - return self.emit(cuda_tile.broadcast(self.ctx, src.inner, self.tileTyOf(shape, src.elemType()), self.loc())); - } - - pub fn expandDims(self: *Builder, src: Value, axis: usize) Value { - const in_shape = src.shape(); - var out: Value.Shape = .empty; - for (0..in_shape.len + 1) |i| { - if (i == axis) out.appendAssumeCapacity(1) else out.appendAssumeCapacity(in_shape.get(if (i < axis) i else i - 1)); - } - return self.reshape(src, out.constSlice()); - } - - pub fn cat(self: *Builder, lhs: Value, rhs: Value, dim: usize) Value { - var out = lhs.shape(); - out.set(dim, out.get(dim) + rhs.dim(dim)); - return self.emit(cuda_tile.cat(self.ctx, lhs.inner, rhs.inner, @intCast(dim), self.tileTyOf(out.constSlice(), lhs.elemType()), self.loc())); - } - - pub fn extract(self: *Builder, src: Value, indices: []const Value, shape: []const i64) Value { - return self.emit(cuda_tile.extract(self.ctx, src.inner, self.innerSlice(indices), self.tileTyOf(shape, src.elemType()), self.loc())); - } - - pub fn permute(self: *Builder, src: Value, order: []const i32) Value { - const src_shape = src.shape(); - std.debug.assert(order.len == src_shape.len); - var out: Value.Shape = .empty; - for (order) |i| out.appendAssumeCapacity(src_shape.get(@intCast(i))); - return self.emit(cuda_tile.permute(self.ctx, src.inner, order, self.tileTyOf(out.constSlice(), src.elemType()), self.loc())); - } - - pub fn transpose(self: *Builder, src: Value) Value { - std.debug.assert(src.rank() == 2); - return self.permute(src, &.{ 1, 0 }); - } - - pub fn offset(self: *Builder, ptrs: Value, off: Value) Value { - return self.emit(cuda_tile.offset(self.ctx, ptrs.inner, off.inner, self.loc())); - } - - fn reducedShape(src: Value, axis: usize) Value.Shape { - if (axis >= src.rank()) std.debug.panic("Builder.reduce: axis {d} out of range for rank {d}", .{ axis, src.rank() }); - var out: Value.Shape = .empty; - const s = src.shape(); - for (0..s.len) |i| if (i != axis) out.appendAssumeCapacity(s.get(i)); - return out; - } - - fn checkAxis(src: Value, axis: usize) void { - if (axis >= src.rank()) std.debug.panic("Builder.scan: axis {d} out of range for rank {d}", .{ axis, src.rank() }); - } - - fn combineBody(self: *Builder, ctx: anytype, elem: *const mlir.Type, combine: anytype) *mlir.Block { - const scalar = cuda_tile.tileType(self.ctx, &.{}, elem); - const body = mlir.Block.init(&.{ scalar, scalar }, &.{ self.loc(), self.loc() }); - self.pushBlock(body); - const element: Value = .{ .inner = body.argument(0), .kernel = self }; - const acc: Value = .{ .inner = body.argument(1), .kernel = self }; - const combined: Value = combine(self, element, acc, ctx); - _ = cuda_tile.yield(self.ctx, &.{combined.inner}, self.loc()).appendTo(body); - self.popBlock(); - return body; - } - - pub fn reduce(self: *Builder, ctx: anytype, args: ReduceArgs(@TypeOf(ctx))) Value { - const result_ty = self.tileTyOf(reducedShape(args.src, args.axis).constSlice(), args.src.elemType()); - const body = self.combineBody(ctx, args.src.elemType(), args.combine); - return self.emit(cuda_tile.reduce(self.ctx, &.{args.src.inner}, @intCast(args.axis), &.{args.identity}, body, &.{result_ty}, self.loc())); - } - - pub fn scan(self: *Builder, ctx: anytype, args: ScanArgs(@TypeOf(ctx))) Value { - checkAxis(args.src, args.axis); - const body = self.combineBody(ctx, args.src.elemType(), args.combine); - return self.emit(cuda_tile.scan(self.ctx, &.{args.src.inner}, @intCast(args.axis), args.reverse, &.{args.identity}, body, &.{args.src.type_()}, self.loc())); - } - - /// The `2N`-argument body of a variadic `reduce`/`scan`: arguments come in - /// (element, accumulator) pairs, one pair per source. - fn combineBodyMulti(self: *Builder, ctx: anytype, srcs: []const Value, combine: anytype) *mlir.Block { - const scratch = self.arena.allocator(); - const n = srcs.len; - const types = scratch.alloc(*const mlir.Type, 2 * n) catch @panic("OOM"); - const locs = scratch.alloc(*const mlir.Location, 2 * n) catch @panic("OOM"); - for (srcs, 0..) |s, i| { - const scalar = cuda_tile.tileType(self.ctx, &.{}, s.elemType()); - types[2 * i] = scalar; - types[2 * i + 1] = scalar; - locs[2 * i] = self.loc(); - locs[2 * i + 1] = self.loc(); - } - const body = mlir.Block.init(types, locs); - self.pushBlock(body); - const elements = scratch.alloc(Value, n) catch @panic("OOM"); - const accs = scratch.alloc(Value, n) catch @panic("OOM"); - for (0..n) |i| { - elements[i] = .{ .inner = body.argument(2 * i), .kernel = self }; - accs[i] = .{ .inner = body.argument(2 * i + 1), .kernel = self }; - } - const out: []const Value = combine(self, elements, accs, ctx); - if (out.len != n) std.debug.panic("Builder.reduceMulti: combine returned {d} values for {d} sources", .{ out.len, n }); - _ = cuda_tile.yield(self.ctx, self.innerSlice(out), self.loc()).appendTo(body); - self.popBlock(); - return body; - } - - pub fn reduceMulti(self: *Builder, ctx: anytype, args: ReduceMultiArgs(@TypeOf(ctx))) []Value { - std.debug.assert(args.srcs.len == args.identities.len); - const result_types = self.arena.allocator().alloc(*const mlir.Type, args.srcs.len) catch @panic("OOM"); - for (args.srcs, 0..) |s, i| result_types[i] = self.tileTyOf(reducedShape(s, args.axis).constSlice(), s.elemType()); - const body = self.combineBodyMulti(ctx, args.srcs, args.combine); - return self.emitMulti(cuda_tile.reduce(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.identities, body, result_types, self.loc()), args.srcs.len); - } - - pub fn scanMulti(self: *Builder, ctx: anytype, args: ScanMultiArgs(@TypeOf(ctx))) []Value { - std.debug.assert(args.srcs.len == args.identities.len); - const result_types = self.arena.allocator().alloc(*const mlir.Type, args.srcs.len) catch @panic("OOM"); - for (args.srcs, 0..) |s, i| { - checkAxis(s, args.axis); - result_types[i] = s.type_(); - } - const body = self.combineBodyMulti(ctx, args.srcs, args.combine); - return self.emitMulti(cuda_tile.scan(self.ctx, self.innerSlice(args.srcs), @intCast(args.axis), args.reverse, args.identities, body, result_types, self.loc()), args.srcs.len); - } - - const ReduceKind = enum { sum, max, min, prod }; - - fn combineKind(k: *Builder, element: Value, acc: Value, kind: ReduceKind) Value { - const is_float = element.isFloatElem(); - return switch (kind) { - .sum => if (is_float) k.addf(element, acc) else k.addi(element, acc), - .prod => if (is_float) k.mulf(element, acc) else k.muli(element, acc), - .max => if (is_float) k.maxf(element, acc) else k.maxi(element, acc, .signed), - .min => if (is_float) k.minf(element, acc) else k.mini(element, acc, .signed), - }; - } - - fn kindIdentity(self: *Builder, src: Value, kind: ReduceKind) *const mlir.Attribute { - const dt = self.mlirElemToDType(src.elemType()); - if (isFloatDtype(dt)) { - return self.identityAttr(dt, switch (kind) { - .sum => @as(f64, 0), - .prod => @as(f64, 1), - .max => -std.math.inf(f64), - .min => std.math.inf(f64), - }); - } - const bits: u32 = intBitwidth(dt); - const min_int: i64 = if (bits == 64) std.math.minInt(i64) else -(@as(i64, 1) << @intCast(bits - 1)); - const max_int: i64 = if (bits == 64) std.math.maxInt(i64) else (@as(i64, 1) << @intCast(bits - 1)) - 1; - return self.identityAttr(dt, switch (kind) { - .sum => @as(i64, 0), - .prod => @as(i64, 1), - .max => min_int, - .min => max_int, - }); - } - - fn reduceKind(self: *Builder, src: Value, axis: usize, comptime kind: ReduceKind) Value { - const combine = struct { - fn c(k: *Builder, element: Value, acc: Value, _: void) Value { - return combineKind(k, element, acc, kind); - } - }.c; - return self.reduce({}, .{ .src = src, .axis = axis, .identity = self.kindIdentity(src, kind), .combine = combine }); - } - - pub fn sum(self: *Builder, src: Value, axis: usize) Value { - return self.reduceKind(src, axis, .sum); - } - pub fn max(self: *Builder, src: Value, axis: usize) Value { - return self.reduceKind(src, axis, .max); - } - pub fn min(self: *Builder, src: Value, axis: usize) Value { - return self.reduceKind(src, axis, .min); - } - pub fn prod(self: *Builder, src: Value, axis: usize) Value { - return self.reduceKind(src, axis, .prod); - } - - fn scanKind(self: *Builder, src: Value, axis: usize, opts: ScanOpts, comptime kind: ReduceKind) Value { - const combine = struct { - fn c(k: *Builder, element: Value, acc: Value, _: void) Value { - return combineKind(k, element, acc, kind); - } - }.c; - return self.scan({}, .{ .src = src, .axis = axis, .reverse = opts.reverse, .identity = self.kindIdentity(src, kind), .combine = combine }); - } - - pub fn cumsum(self: *Builder, src: Value, axis: usize) Value { - return self.scanKind(src, axis, .{}, .sum); - } - pub fn cumsumOpts(self: *Builder, src: Value, axis: usize, opts: ScanOpts) Value { - return self.scanKind(src, axis, opts, .sum); - } - pub fn cumprod(self: *Builder, src: Value, axis: usize) Value { - return self.scanKind(src, axis, .{}, .prod); - } - pub fn cummax(self: *Builder, src: Value, axis: usize) Value { - return self.scanKind(src, axis, .{}, .max); - } - pub fn cummin(self: *Builder, src: Value, axis: usize) Value { - return self.scanKind(src, axis, .{}, .min); - } - - /// `for iv in [lower, upper) step` carrying `inits`; close with - /// `scope.yield(.{...})` (`scope.close()` without inits), then read - /// `scope.results`. - pub fn openFor( - self: *Builder, - lower: anytype, - upper: anytype, - step: anytype, - inits: anytype, - ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { - return self.openForOpts(lower, upper, step, inits, .{}); - } - - /// `openFor` with `cuda_tile.for` options. - pub fn openForOpts( - self: *Builder, - lower: anytype, - upper: anytype, - step: anytype, - inits: anytype, - opts: ForOpts, - ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { - const N = comptime common.arity(@TypeOf(inits), "openFor: inits"); - - // Literals take the type of the first bound that is a Value, else - // `i32`; all three must agree (`AllTypesMatch` on the op). - const lb_v, const ub_v, const step_v = common.forBounds(self, Value, lower, upper, step, i32Bound); - - var block_types: [N + 1]*const mlir.Type = undefined; - block_types[0] = lb_v.type_(); - var inits_inner: [N]*const mlir.Value = undefined; - inline for (0..N) |i| { - const v: Value = if (@TypeOf(inits[i]) == Value) inits[i] else self.lift(inits[i]); - block_types[i + 1] = v.type_(); - inits_inner[i] = v.inner; - } - const block_locs: [N + 1]*const mlir.Location = @splat(self.loc()); - const body = mlir.Block.init(&block_types, &block_locs); - self.pushBlock(body); - - var carried: [N]Value = undefined; - for (0..N) |i| carried[i] = .{ .inner = body.argument(i + 1), .kernel = self }; - - return .{ - .kernel = self, - .body = body, - .lb_inner = lb_v.inner, - .ub_inner = ub_v.inner, - .step_inner = step_v.inner, - .inits_inner = inits_inner, - .iv = .{ .inner = body.argument(0), .kernel = self }, - .carried = carried, - .opts = opts, - }; - } - - pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { - const then_block = mlir.Block.init(&.{}, &.{}); - self.pushBlock(then_block); - return .{ .kernel = self, .cond_inner = cond.inner, .then_block = then_block }; - } - - pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { - const then_block = mlir.Block.init(&.{}, &.{}); - const else_block = mlir.Block.init(&.{}, &.{}); - self.pushBlock(then_block); - return .{ - .kernel = self, - .cond_inner = cond.inner, - .then_block = then_block, - .else_block = else_block, - .result_types = common.resultTypes(self, Value, result_types, "openIfElse: result_types"), - }; - } - - /// An unstructured loop carrying `inits` (`scope.close()` ends a body that - /// carries nothing, in place of `scope.yield`); leave with `scope.breakWith` - /// inside an `if`, and close the body with `scope.yield`. - pub fn openLoop(self: *Builder, inits: anytype) LoopScope(common.arity(@TypeOf(inits), "openLoop: inits")) { - const N = comptime common.arity(@TypeOf(inits), "openLoop: inits"); - var block_types: [N]*const mlir.Type = undefined; - var inits_inner: [N]*const mlir.Value = undefined; - inline for (0..N) |i| { - const v: Value = if (@TypeOf(inits[i]) == Value) inits[i] else self.lift(inits[i]); - block_types[i] = v.type_(); - inits_inner[i] = v.inner; - } - const block_locs: [N]*const mlir.Location = @splat(self.loc()); - const body = mlir.Block.init(&block_types, &block_locs); - self.pushBlock(body); - - var carried: [N]Value = undefined; - for (0..N) |i| carried[i] = .{ .inner = body.argument(i), .kernel = self }; - - return .{ - .kernel = self, - .body = body, - .inits_inner = inits_inner, - .result_types = block_types, - .carried = carried, - }; - } - - pub fn deviceAssert(self: *Builder, condition: Value, message: []const u8) void { - _ = cuda_tile.assert_(self.ctx, condition.inner, message, self.loc()).appendTo(self.currentBlock()); - } - - /// Append `return`, verify the module, and serialize it to a NUL-terminated - /// string owned by the kernel's allocator. `results` must be empty: an - /// `entry` returns nothing; outputs reach the host through pointers. - pub fn finish(self: *Builder, results: []const Value) FinishError![:0]const u8 { - std.debug.assert(results.len == 0); - const current = self.currentBlock(); - if (current.terminator() == null) { - _ = cuda_tile.return_(self.ctx, &.{}, self.loc()).appendTo(current); - } - - try common.verify(self.module, "dsl/cuda_tile"); - - var al: std.Io.Writer.Allocating = .init(self.allocator); - defer al.deinit(); - - try al.writer.print("{f}", .{self.module.operation()}); - - return try self.allocator.dupeSentinel(u8, al.written(), 0); - } -}; - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - registerDialects(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -fn expectRoundTrip(ctx: *mlir.Context, ir: [:0]const u8) !void { - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "empty entry round-trips" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "empty"); - defer b.deinit(); - _ = try b.declareArgs(.{ .p = .{ .ptr = .f32 } }); - - const ir = try b.finish(&.{}); - defer std.testing.allocator.free(ir); - - try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.module @empty") != null); - try std.testing.expect(std.mem.find(u8, ir, "entry @empty(") != null); - try std.testing.expect(std.mem.find(u8, ir, "assume") != null); - try std.testing.expect(std.mem.find(u8, ir, "div_by<16>") != null); - try std.testing.expect(std.mem.find(u8, ir, "cuda_tile.entry") == null); - try std.testing.expect(std.mem.find(u8, ir, "return") != null); - try expectRoundTrip(ctx, ir); -} - -test "add_one over pointer tiles matches the XLA milestone kernel" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "add_one"); - defer b.deinit(); - const a = try b.declareArgsOpts(.{ - .in = .{ .ptr_opts = .{ .dtype = .f32, .div_by = null } }, - .out = .{ .ptr_opts = .{ .dtype = .f32, .div_by = null } }, - }, .{}); - - const offsets = b.iota(128, .i32); - const loaded = b.loadPtrOpts(a.in.offset(offsets), .{}); - const sum = loaded.tile.add(1.0); - _ = b.storePtrOpts(a.out.offset(offsets), sum, .{ .token = loaded.token }); - - const ir = try b.finish(&.{}); - defer std.testing.allocator.free(ir); - - try std.testing.expect(std.mem.find(u8, ir, "iota : tile<128xi32>") != null); - try std.testing.expect(std.mem.find(u8, ir, "reshape") != null); - try std.testing.expect(std.mem.find(u8, ir, "broadcast") != null); - try std.testing.expect(std.mem.find(u8, ir, "offset") != null); - try std.testing.expect(std.mem.find(u8, ir, "load_ptr_tko weak") != null); - try std.testing.expect(std.mem.find(u8, ir, "addf") != null); - try std.testing.expect(std.mem.find(u8, ir, "store_ptr_tko weak") != null); - try std.testing.expect(std.mem.find(u8, ir, "token = ") != null); - try expectRoundTrip(ctx, ir); -} - -test "vector_add over partition views" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "vector_add"); - defer b.deinit(); - const a = try b.declareArgsOpts(.{ - .x = .{ .ptr = .f32 }, - .y = .{ .ptr = .f32 }, - .out = .{ .ptr = .f32 }, - }, .{ .hints = &.{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }} }); - - const shape = [_]i64{ 8192, 128 }; - const strides = [_]i64{ 128, 1 }; - const tile = [_]i64{ 64, 64 }; - const vx = b.partitionView(b.tensorView(a.x, &shape, &strides), &tile, .{}); - const vy = b.partitionView(b.tensorView(a.y, &shape, &strides), &tile, .{}); - const vo = b.partitionView(b.tensorView(a.out, &shape, &strides), &tile, .{}); - - const bid = b.tileBlockId(); - const idx = [_]Value{ bid.x, bid.y }; - const xt = b.load(vx, &idx); - const yt = b.load(vy, &idx); - _ = b.store(xt.add(yt), vo, &idx); - - const ir = try b.finish(&.{}); - defer std.testing.allocator.free(ir); - - try std.testing.expect(std.mem.find(u8, ir, "optimization_hints=") != null); - try std.testing.expect(std.mem.find(u8, ir, "make_tensor_view") != null); - try std.testing.expect(std.mem.find(u8, ir, "partition_view") != null or std.mem.find(u8, ir, "addf") != null); - try std.testing.expect(std.mem.find(u8, ir, "store_view_tko weak") != null); - try expectRoundTrip(ctx, ir); -} - -test "gemm with a for loop and mmaf" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "gemm"); - defer b.deinit(); - const a = try b.declareArgs(.{ .a = .{ .ptr = .bf16 }, .bm = .{ .ptr = .bf16 }, .c = .{ .ptr = .f32 } }); - - const m: i64 = 1024; - const n: i64 = 1024; - const k: i64 = 512; - const pa = b.partitionView(b.tensorView(a.a, &.{ m, k }, &.{ k, 1 }), &.{ 128, 64 }, .{}); - const pb = b.partitionView(b.tensorView(a.bm, &.{ k, n }, &.{ n, 1 }), &.{ 64, 128 }, .{}); - const pc = b.partitionView(b.tensorView(a.c, &.{ m, n }, &.{ n, 1 }), &.{ 128, 128 }, .{}); - - const bid = b.tileBlockId(); - const acc0 = b.zeros(&.{ 128, 128 }, .f32); - - var loop = b.openFor(0, @divExact(k, 64), 1, .{acc0}); - { - const kt = loop.iv; - const acc = loop.carried[0]; - const at = b.load(pa, &.{ bid.x, kt }); - const bt = b.load(pb, &.{ kt, bid.y }); - loop.yield(.{b.mmaf(at, bt, acc)}); - } - _ = b.store(loop.results[0], pc, &.{ bid.x, bid.y }); - - const ir = try b.finish(&.{}); - defer std.testing.allocator.free(ir); - - try std.testing.expect(std.mem.find(u8, ir, "for ") != null); - try std.testing.expect(std.mem.find(u8, ir, "iter_values") != null); - try std.testing.expect(std.mem.find(u8, ir, "mmaf") != null); - try std.testing.expect(std.mem.find(u8, ir, "continue ") != null); - try expectRoundTrip(ctx, ir); -} - -test "if, if-else, loop with break, reduce, scan, cast, select" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "control"); - defer b.deinit(); - // A scalar reaches the kernel through a rank-0 pointer. - const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .n = .{ .ptr = .i32 } }); - const n = b.loadPtr(a.n); - try std.testing.expect(n.isScalar()); - - const v = b.partitionView(b.tensorView(a.x, &.{ 256, 64 }, &.{ 64, 1 }), &.{ 8, 64 }, .{}); - const bid = b.tileBlockId(); - const t = b.load(v, &.{ bid.x, bid.y }); - - // Reductions and scans over the tile. - const row_sum = b.sum(t, 1); - const col_max = b.max(t, 0); - const running = b.cumsum(t, 1); - try std.testing.expectEqual(@as(usize, 1), row_sum.rank()); - try std.testing.expectEqual(@as(i64, 8), row_sum.dim(0)); - try std.testing.expectEqual(@as(i64, 64), col_max.dim(0)); - try std.testing.expectEqual(@as(usize, 2), running.rank()); - - // A variadic reduce: argmax over axis 1 as (max, index). - const cols = b.broadcastTo(b.reshape(b.iota(64, .i32), &.{ 1, 64 }), &.{ 8, 64 }); - const argmax = b.reduceMulti({}, .{ - .srcs = &.{ t, cols }, - .axis = 1, - .identities = &.{ b.identityAttr(.f32, -std.math.inf(f64)), b.identityAttr(.i32, 0) }, - .combine = struct { - fn c(k: *Builder, elements: []const Value, accs: []const Value, _: void) []const Value { - const better = elements[0].gt(accs[0]); - return k.yield(.{ k.select(better, elements[0], accs[0]), k.select(better, elements[1], accs[1]) }); - } - }.c, - }); - try std.testing.expectEqual(@as(usize, 2), argmax.len); - try std.testing.expectEqual(@as(i64, 8), argmax[1].dim(0)); - - // Casts and select. - const as_i32 = t.to(.i32); - const back = as_i32.to(.f32); - const picked = b.where(t.gt(0.0), back, t.neg()); - - // if / if-else carrying a value. - const is_first = bid.x.eq(0); - var only = b.openIf(is_first); - { - _ = b.store(picked, v, &.{ bid.x, bid.y }); - only.close(); - } - - var branch = b.openIfElse(is_first, .{b.tileTy(&.{ 8, 64 }, .f32)}); - { - branch.yieldThen(.{picked}); - } - { - branch.yieldElse(.{t}); - } - const chosen = branch.results[0]; - - // loop: count up to n, break when reached. - var lp = b.openLoop(.{b.constant(.i32, 0)}); - { - const i = lp.carried[0]; - var done = b.openIf(i.ge(n)); - { - lp.breakWith(.{i}); - done.close(); - } - lp.yield(.{i.add(1)}); - } - _ = lp.results[0]; - - // a loop carrying nothing ends with close(). - var once = b.openLoop(.{}); - { - var first = b.openIf(is_first); - { - once.breakWith(.{}); - first.close(); - } - once.close(); - } - - // for with an early `continue` inside an `if`, and i64 bounds from a Value. - var skip = b.openFor(b.constant(.i64, 0), 16, 1, .{b.constant(.i64, 0)}); - { - const acc = skip.carried[0]; - var odd = b.openIf(skip.iv.bitAnd(1).eq(1)); - { - skip.continueWith(.{acc}); - odd.close(); - } - skip.yield(.{acc.add(skip.iv)}); - } - try std.testing.expect(skip.results[0].isScalar()); - - _ = b.store(chosen, v, &.{ bid.x, bid.y }); - - const ir = try b.finish(&.{}); - defer std.testing.allocator.free(ir); - - try std.testing.expect(std.mem.find(u8, ir, "reduce ") != null); - try std.testing.expect(std.mem.find(u8, ir, "scan ") != null); - try std.testing.expect(std.mem.find(u8, ir, "ftoi") != null); - try std.testing.expect(std.mem.find(u8, ir, "itof") != null); - try std.testing.expect(std.mem.find(u8, ir, "select") != null); - try std.testing.expect(std.mem.find(u8, ir, "loop ") != null); - try std.testing.expect(std.mem.find(u8, ir, "break ") != null); - try std.testing.expect(std.mem.find(u8, ir, "yield") != null); - try expectRoundTrip(ctx, ir); -} - -test "atomics, print, globals, alloca, dynamic views, strided view, tokens" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "misc"); - defer b.deinit(); - const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .cnt = .{ .ptr = .i32 }, .rows = .{ .ptr = .i32 } }); - b.global("lut", &.{16}, .f32, 1.5, .{ .constant = true }); - - const rows = b.loadPtr(a.rows); - const tv = b.tensorViewDyn(a.x, &.{ .{ .dynamic = rows }, .{ .static = 64 } }, &.{ .{ .static = 64 }, .{ .static = 1 } }); - const extents = b.tensorShape(tv); - try std.testing.expectEqual(@as(usize, 2), extents.len); - const v = b.partitionView(tv, &.{ 8, 64 }, .{ .padding = .nan }); - const tiles = b.indexSpaceShape(v); - try std.testing.expectEqual(@as(usize, 2), tiles.len); - const sv = b.stridedView(tv, &.{ 8, 64 }, &.{ 2, 1 }, .{}); - _ = b.load(sv, &.{ b.constant(.i32, 0), b.constant(.i32, 0) }); - - const bid = b.tileBlockId(); - const loaded = b.loadOpts(v, &.{ bid.x, bid.y }, .{ .hints = &.{.{ .arch = .sm_120, .allow_tma = true }} }); - const tok = b.print("tile %f\n", &.{loaded.tile}); - - const counter = a.cnt.offset(b.iota(8, .i32)); - const old = b.atomicRmw(.add, counter, b.full(&.{8}, 1, .i32), .{ .token = tok }); - const cas = b.atomicCas(counter, old.tile, b.full(&.{8}, 0, .i32), .{ .token = old.token }); - const joined = b.joinTokens(&.{ loaded.token, cas.token, b.makeToken() }); - // Atomic reductions refuse a padded view; the default options must verify. - const unpadded = b.partitionView(tv, &.{ 8, 64 }, .{ .padding = null }); - _ = b.atomicRedView(.addf, unpadded, &.{ bid.x, bid.y }, loaded.tile, .{ .token = joined }); - _ = b.atomicRedView(.addf, unpadded, &.{ bid.x, bid.y }, loaded.tile, .{}); - - // A gather/scatter view: the sparse index is a rank-1 tile of row ids. - const gsv = b.gatherScatterView(tv, &.{ 8, 64 }, 0, .zero); - const gathered = b.load(gsv, &.{ b.iota(8, .i32), b.constant(.i32, 0) }); - try std.testing.expectEqual(@as(i64, 8), gathered.dim(0)); - try std.testing.expectEqual(@as(i64, 64), gathered.dim(1)); - _ = b.store(gathered, gsv, &.{ b.iota(8, .i32), b.constant(.i32, 0) }); - - const lut = b.getGlobal("lut", .f32); - const scratch = b.alloca(.f32, 64, .{}); - _ = b.loadPtr(lut.offset(b.iota(16, .i32))); - _ = b.storePtr(scratch.offset(b.iota(64, .i32)), b.zeros(&.{64}, .f32)); - - const ir = try b.finish(&.{}); - defer std.testing.allocator.free(ir); - - try std.testing.expect(std.mem.find(u8, ir, "global @lut") != null); - try std.testing.expect(std.mem.find(u8, ir, "get_global @lut") != null); - try std.testing.expect(std.mem.find(u8, ir, "tensor_view") != null); - try expectRoundTrip(ctx, ir); -} diff --git a/dsl/cuda_tile/control_flow.zig b/dsl/cuda_tile/control_flow.zig deleted file mode 100644 index 26979e0a..00000000 --- a/dsl/cuda_tile/control_flow.zig +++ /dev/null @@ -1,185 +0,0 @@ -//! `cuda_tile` scope wrappers, like `dsl/common/control_flow.zig` but with -//! the dialect's terminators: `continue` ends a `for`/`loop` body, `yield` an -//! `if`, `break` leaves a `loop`. An already-terminated branch is left alone. - -const common = @import("dsl/common/builder"); -const std = @import("std"); -const cuda_tile = @import("mlir/dialects/cuda_tile"); -const mlir = @import("mlir"); - -fn terminate(comptime ValueT: type, comptime N: usize, kernel: anytype, block: *mlir.Block, comptime mnemonic: enum { yield, continue_, break_ }, values: anytype, comptime what: []const u8) void { - if (block.terminator() != null) return; - const buf = common.innerValues(ValueT, N, values, what); - const op = switch (mnemonic) { - .yield => cuda_tile.yield(kernel.ctx, &buf, kernel.loc()), - .continue_ => cuda_tile.continue_(kernel.ctx, &buf, kernel.loc()), - .break_ => cuda_tile.break_(kernel.ctx, &buf, kernel.loc()), - }; - _ = op.appendTo(block); -} - -pub const ForOpts = struct { - /// Compare the induction variable unsigned. - unsigned_cmp: bool = false, -}; - -/// `for %iv in (lb to ub, step) iter_values(...)`; `yield` closes it, or -/// `close()` when it carries nothing. -pub fn ForScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { - return struct { - kernel: *BuilderT, - body: *mlir.Block, - lb_inner: *const mlir.Value, - ub_inner: *const mlir.Value, - step_inner: *const mlir.Value, - inits_inner: [N]*const mlir.Value, - iv: ValueT, - carried: [N]ValueT, - results: [N]ValueT = undefined, - opts: ForOpts = .{}, - - const Self = @This(); - - /// `continue` in the current block, inside a zero-result `if`: skips - /// the rest of the iteration and terminates that branch. - pub fn continueWith(self: *Self, values: anytype) void { - const k = self.kernel; - terminate(ValueT, N, k, k.currentBlock(), .continue_, values, "ForScope.continueWith"); - } - - pub fn yield(self: *Self, values: anytype) void { - if (N == 0) @compileError("ForScope.yield: the loop carries nothing, end it with close()"); - self.end(values); - } - - pub fn close(self: *Self) void { - if (N != 0) @compileError("ForScope.close: the loop carries values, end it with yield()"); - self.end(.{}); - } - - fn end(self: *Self, values: anytype) void { - const k = self.kernel; - terminate(ValueT, N, k, self.body, .continue_, values, "ForScope.yield"); - k.popBlock(); - const for_op = cuda_tile.for_(k.ctx, self.lb_inner, self.ub_inner, self.step_inner, &self.inits_inner, self.body, self.opts.unsigned_cmp, k.loc()); - _ = for_op.appendTo(k.currentBlock()); - for (0..N) |i| self.results[i] = .{ .inner = for_op.result(i), .kernel = k }; - } - }; -} - -pub fn IfOnlyScope(comptime BuilderT: type, comptime ValueT: type) type { - return struct { - kernel: *BuilderT, - cond_inner: *const mlir.Value, - then_block: *mlir.Block, - - const Self = @This(); - - pub fn close(self: *Self) void { - const k = self.kernel; - terminate(ValueT, 0, k, self.then_block, .yield, .{}, "IfOnlyScope.close"); - k.popBlock(); - _ = cuda_tile.if_(k.ctx, self.cond_inner, &.{}, self.then_block, .{}, k.loc()).appendTo(k.currentBlock()); - } - }; -} - -pub fn IfScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { - return struct { - kernel: *BuilderT, - cond_inner: *const mlir.Value, - then_block: *mlir.Block, - else_block: *mlir.Block, - result_types: [N]*const mlir.Type, - results: [N]ValueT = undefined, - - const Self = @This(); - - pub fn yieldThen(self: *Self, values: anytype) void { - if (N == 0) @compileError("IfScope.yieldThen: the if has no results, switch branches with elseBranch()"); - self.endThen(values); - } - - pub fn yieldElse(self: *Self, values: anytype) void { - if (N == 0) @compileError("IfScope.yieldElse: the if has no results, end it with close()"); - self.endElse(values); - } - - pub fn elseBranch(self: *Self) void { - if (N != 0) @compileError("IfScope.elseBranch: the if has results, switch branches with yieldThen()"); - self.endThen(.{}); - } - - pub fn close(self: *Self) void { - if (N != 0) @compileError("IfScope.close: the if has results, end it with yieldElse()"); - self.endElse(.{}); - } - - fn endThen(self: *Self, values: anytype) void { - const k = self.kernel; - terminate(ValueT, N, k, self.then_block, .yield, values, "IfScope.yieldThen"); - k.popBlock(); - k.pushBlock(self.else_block); - } - - fn endElse(self: *Self, values: anytype) void { - const k = self.kernel; - terminate(ValueT, N, k, self.else_block, .yield, values, "IfScope.yieldElse"); - k.popBlock(); - const if_op = cuda_tile.if_(k.ctx, self.cond_inner, &self.result_types, self.then_block, .{ .else_block = self.else_block }, k.loc()); - _ = if_op.appendTo(k.currentBlock()); - for (0..N) |i| self.results[i] = .{ .inner = if_op.result(i), .kernel = k }; - } - }; -} - -/// `loop iter_values(...)`: the body ends with `continue` (`yield`, or -/// `close()` when it carries nothing) and leaves through `break` (`breakWith`, -/// inside an `if`). `results` are the broken. -pub fn LoopScope(comptime BuilderT: type, comptime ValueT: type, comptime N: usize) type { - return struct { - kernel: *BuilderT, - body: *mlir.Block, - inits_inner: [N]*const mlir.Value, - result_types: [N]*const mlir.Type, - carried: [N]ValueT, - results: [N]ValueT = undefined, - - const Self = @This(); - - /// `break` in the current block; terminates the enclosing `if` branch. - pub fn breakWith(self: *Self, values: anytype) void { - const k = self.kernel; - terminate(ValueT, N, k, k.currentBlock(), .break_, values, "LoopScope.breakWith"); - } - - pub fn continueWith(self: *Self, values: anytype) void { - const k = self.kernel; - terminate(ValueT, N, k, k.currentBlock(), .continue_, values, "LoopScope.continueWith"); - } - - pub fn yield(self: *Self, values: anytype) void { - if (N == 0) @compileError("LoopScope.yield: the loop carries nothing, end it with close()"); - self.end(values); - } - - pub fn close(self: *Self) void { - if (N != 0) @compileError("LoopScope.close: the loop carries values, end it with yield()"); - self.end(.{}); - } - - fn end(self: *Self, values: anytype) void { - const k = self.kernel; - terminate(ValueT, N, k, self.body, .continue_, values, "LoopScope.yield"); - k.popBlock(); - const loop_op = cuda_tile.loop(k.ctx, &self.inits_inner, &self.result_types, self.body, k.loc()); - _ = loop_op.appendTo(k.currentBlock()); - for (0..N) |i| self.results[i] = .{ .inner = loop_op.result(i), .kernel = k }; - } - }; -} - -test { - std.testing.refAllDecls(@This()); -} diff --git a/dsl/cuda_tile/dtype.zig b/dsl/cuda_tile/dtype.zig deleted file mode 100644 index a9193839..00000000 --- a/dsl/cuda_tile/dtype.zig +++ /dev/null @@ -1,103 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -/// Every element type `cuda_tile.tile` accepts (Types.td: -/// CudaTile_TileElementType). `i4`, `f4e2m1fn` need 13.3; `f8e8m0fnu` needs -/// 13.2. `i4` is a tile element only — a pointer cannot point at it. -pub const DType = enum { - i1, - i4, - i8, - i16, - i32, - i64, - f16, - bf16, - f32, - tf32, - f64, - f8e4m3fn, - f8e5m2, - f8e8m0fnu, - f4e2m1fn, - - pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { - return switch (self) { - .i1 => .int(ctx, .i1), - .i4 => .int(ctx, .i4), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .f16 => .float(ctx, .f16), - .bf16 => .float(ctx, .bf16), - .f32 => .float(ctx, .f32), - .tf32 => .float(ctx, .tf32), - .f64 => .float(ctx, .f64), - .f8e4m3fn => .float(ctx, .f8e4m3fn), - .f8e5m2 => .float(ctx, .f8e5m2), - .f8e8m0fnu => .float(ctx, .f8e8m0fnu), - .f4e2m1fn => .float(ctx, .f4e2m1fn), - }; - } -}; - -pub fn isFloatDtype(dt: DType) bool { - return switch (dt) { - .f16, .bf16, .f32, .tf32, .f64, .f8e4m3fn, .f8e5m2, .f8e8m0fnu, .f4e2m1fn => true, - else => false, - }; -} - -pub fn dtypeBitwidth(dt: DType) usize { - return switch (dt) { - .i1 => 1, - .i4, .f4e2m1fn => 4, - .i8, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => 8, - .i16, .f16, .bf16 => 16, - .i32, .f32, .tf32 => 32, - .i64, .f64 => 64, - }; -} - -pub fn intBitwidth(dt: DType) u32 { - return switch (dt) { - .i1 => 1, - .i4 => 4, - .i8 => 8, - .i16 => 16, - .i32 => 32, - .i64 => 64, - else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), - }; -} - -pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (comptime std.meta.fieldNames(DType)) |f_name| { - const dt = @field(DType, f_name); - if (elem.eql(dt.toMlir(ctx))) return dt; - } - @panic("element type not a recognized CUDA Tile IR DType"); -} - -/// SharedVerifiers.h verifyTileSize: every dimension a power of two and the -/// tile at most 2^24 elements. Checked in Zig so the failure names the shape. -pub fn tileLegal(shape: []const i64) bool { - var elems: i64 = 1; - for (shape) |d| { - if (d <= 0 or (d & (d - 1)) != 0) return false; - elems *= d; - } - return elems <= (1 << 24); -} - -test { - std.testing.refAllDecls(@This()); -} - -test "tileLegal" { - try std.testing.expect(tileLegal(&.{})); - try std.testing.expect(tileLegal(&.{ 64, 128 })); - try std.testing.expect(!tileLegal(&.{3})); - try std.testing.expect(!tileLegal(&.{ 4096, 4096, 2 })); -} diff --git a/dsl/cute/BUILD.bazel b/dsl/cute/BUILD.bazel deleted file mode 100644 index ef5fc6d9..00000000 --- a/dsl/cute/BUILD.bazel +++ /dev/null @@ -1,34 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "builder", - srcs = [ - "dtype.zig", - ], - import_name = "dsl/cute/builder", - main = "builder.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//dsl/common:builder", - "//mlir", - "//mlir/dialects/arith", - "//mlir/dialects/cf", - "//mlir/dialects/cuda", - "//mlir/dialects/cute_ir", - "//mlir/dialects/cute_nvgpu", - "//mlir/dialects/func", - "//mlir/dialects/gpu", - "//mlir/dialects/llvm", - "//mlir/dialects/math", - "//mlir/dialects/nvvm", - "//mlir/dialects/scf", - "//mlir/dialects/vector", - "//stdx", - ], -) - -zig_test( - name = "test", - deps = [":builder"], -) diff --git a/dsl/cute/README.md b/dsl/cute/README.md deleted file mode 100644 index 946c542d..00000000 --- a/dsl/cute/README.md +++ /dev/null @@ -1,186 +0,0 @@ -# CuTe DSL kernels in Zig - -`dsl/cute` is a Zig front end for NVIDIA's CuTe DSL, shaped after the -Python `cutlass.cute` API: a kernel takes tensors, indexes them with -coordinates, and reads `thread_idx()` and friends from `cute.arch`. It is the -fourth instance of the `dsl/triton` / `dsl/mosaic_tpu` / -`dsl/cuda_tile` pattern: a `Builder` that owns an MLIR module and appends -ops to a block stack, a `Value` with fluent scalar arithmetic, and a -`zml.kernel.cute.Kernel` wrapper that turns a `run` function into a -`stablehlo.custom_call` XLA compiles in process with `libcute_compiler.so`. - -The dialect bindings live in `mlir/dialects/cute_ir` (`cute`), -`mlir/dialects/cute_nvgpu`, and `mlir/dialects/nvvm` and `mlir/dialects/cuda` -for NVIDIA's `nvvm` and `cuda` (operation builders under `.ops`), with types -and attributes through each dialect's C API. - -## A kernel - -The Python notebook's `naive_elementwise_add_kernel`, in Zig: - -```zig -const zml = @import("zml"); -const cute = zml.kernel.cute; - -const Cfg = struct { m: i64, n: i64 }; - -pub const NaiveElementwiseAdd = cute.Kernel(Cfg, .{ - .name = "naive_elementwise_add_kernel", - .inputs = &.{ "gA", "gB" }, - .outputs = &.{"gC"}, - .run = run, -}); - -fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { - // A `tensor` argument is XLA's device pointer composed with a static - // layout; the Python kernel takes `cute.Tensor` the same way. - const t = try b.declareArgs(.{ - .gA = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, - .gB = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, - .gC = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.m, cfg.n } } }, - }); - - const tidx = b.threadIdx().x; - const bidx = b.blockIdx().x; - const bdim = b.blockDim().x; - const thread_idx = bidx.mul(bdim).add(tidx); - - const n = t.gA.dim(1); - const ni = thread_idx.rem(n); - const mi = thread_idx.div(n); - - // gC[mi, ni] = gA[mi, ni] + gB[mi, ni] - t.gC.set(.{ mi, ni }, t.gA.get(.{ mi, ni }).add(t.gB.get(.{ mi, ni }))); -} - -// In a model (`kernel.launch(grid=..., block=...)`): -const c = NaiveElementwiseAdd.call(.{ .gA = a, .gB = b }, .{ .gC = a.shape() }, .{ - .cfg = .{ .m = 64, .n = 128 }, - .grid = .{ 32, 1, 1 }, - .block = .{ 256, 1, 1 }, -}).gC; -``` - -## Python to Zig - -Put each control-flow region in a Zig `{ ... }` block immediately after -`openFor`, `openIf`, or `openWhile`, with its corresponding `yield` call (or -`close()` for a scope without values) inside the block. For `openIfElse`, use -separate blocks ending in `yieldThen` and `yieldElse` (`elseBranch()` and -`close()` when it has no results); likewise, give `openWhile` separate -`yieldBefore` and `yieldAfter` blocks. Keep the scope variable outside these -blocks so its `.results` remain accessible afterward. The braces make the -generated control flow visible and keep region-local values from escaping -their scope; the builder's `open` and `yield` calls still create and close -the actual IR regions. - -| `cutlass.cute` | `dsl/cute` | -|--------------------------------------------|-------------------------------------------------| -| `gA: cute.Tensor` kernel argument | `.gA = .{ .tensor = .{ .dtype, .shape } }` | -| `gA[mi, ni]` / `gC[mi, ni] = v` | `gA.get(.{ mi, ni })` / `gC.set(.{ mi, ni }, v)`| -| `cute.make_layout(shape, stride=...)` | `b.makeLayout(shape, .{ .stride = ... })` | -| `cute.make_tensor(ptr, layout)` | `b.makeTensor(ptr, layout)` | -| `cute.arch.thread_idx()` → `(x, y, z)` | `b.threadIdx().x` | -| `block_idx()`, `block_dim()`, `grid_dim()` | `blockIdx()`, `blockDim()`, `gridDim()` | -| `cute.arch.alloc_smem(T, n, align)` | `b.allocSmem(.f32, n, align)` | -| `cute.arch.sync_threads()` | `b.syncThreads()` | -| `cute.arch.shuffle_sync_bfly(v, lane)` | `b.shuffleXor(v, lane)` | -| `cute.arch.griddepcontrol_launch_dependents()` | `b.launchDependents()` | -| `x.to(Float32)` | `x.to(.f32)` | -| `if cond:` / `for i in range(...)` | `b.openIf(cond)` / `b.openFor(0, n, 1, .{})` | -| `cpasync.make_tiled_tma_atom(...)` | `b.makeTiledTmaAtom(...)` | -| `cpasync.tma_partition(...)` | `b.tmaPartition(...)` | -| `cute.copy(atom, src, dst)` | `b.copy(atom, src, dst, .{})` | -| `cute.make_tiled_mma(atom)` | `b.makeTiledMma(atom, result_type)` | -| `tiled_mma.make_fragment_A/B/C(...)` | `b.mmaMakeFragment(...)` | -| `cute.gemm(...)` | `b.gemm(...)` | -| `cute.arch.alloc_tmem(...)` | `b.allocTmem(...)` | -| `cute.arch.retrieve_tmem_ptr(...)` | `b.retrieveTmemPtr(...)` | - -* **Layouts accept static and dynamic extents.** `makeLayout` takes Zig slices - for a tracked static `Layout`, or nested tuples of integers and SSA `Value`s - for a `View`. Dynamic shape and stride leaves retain their integer widths. - The tracked `Tensor` keeps its shape and stride in Zig; use `makeTensorView` - with dynamic layouts. Integers stay static in coordinate types. -* **Stride defaults differ by origin.** `makeLayout` defaults to CuTe's - compact left-most stride like `cute.make_layout`; a `tensor` argument - defaults to XLA's row-major layout, which is what the buffer holds. -* **Scalars are `i32` and `f32` by default**, like `Int32`/`Float32`; - `constant(dtype, v)` picks another type and `coerce` lifts literals to the other - operand's type. -* **`nvvm` and `cuda` are the compiler's.** The context registers NVIDIA's - `nvvm` and `cuda` dialects (`nvvm.ops`, `cuda.ops`, from the - `mlir/dialects/nvvm` and `mlir/dialects/cuda` packages) and upstream's - `llvm`, so every operation is - verified as it is built; nothing is left unregistered. -* **The module is just the kernel.** `finish` prints a module holding only - `func.func @name(...)`; the CuTe compiler makes every public function a - kernel entry, so there is no `gpu.module` or host launch function to write. -* **Generic form.** `finish` and `finishProgram` print the generic operation - form: NVIDIA's compiler parses with its own MLIR revision, whose custom - assembly differs from ours (`llvm.load`/`llvm.store` alignment, for one). -* **Explicit result types:** Zig covers local tiling, TMA partitioning, tiled - copies, tensor-memory fragments, and SM100 block-scaled MMA. CuTe derives - their layouts in the compiler, so callers spell out the resulting CuTe type - with `View` instead of relying on Python's dynamic type objects. -* **TMA tilers follow Python.** `makeTiledTmaAtom` accepts a shape tuple, tile, - or layout, composes it with the GMEM identity layout, and removes the SMEM - stage mode when present. The lower-level `makeTiledTmaLoadAtom` and - `makeTiledTmaStoreAtom` take an already composed coordinate map. `copy` - makes store atoms executable automatically; loads require `tma_bar_ptr`. -* **Typed result types.** Operations whose result type CuTe derives - (`localTile`, `slice`, `tmaPartition`, ...) take an `*mlir.Type` assembled - with `layoutSpec`, `layoutType`, `memrefType`, `coordTensorType`, - `smemDescViewType` or `tiledCopyType`; nothing is parsed from text. -* **Architecture sections.** `Builder` groups TMA (`cute.nvgpu.cpasync`, - SM90+) and tcgen05 / tensor memory (`cute.nvgpu.tcgen05`, SM100) in their - own sections; mbarriers, fences and bulk-copy groups sit with `cute.arch`. - Each doc comment names its minimum architecture when it is above SM80. - -## Programs with a host launch - -Kernels that need launch-time CuTe objects (TMA descriptors, tiled MMA atoms, -scheduler parameters) use `zml.kernel.cute.Program` instead of `Kernel`. Its -`run` callback emits one `gpu.module` with the `cuda.kernel`s -(`b.beginFunction(name, .cuda_kernel)`), then a public host `func.func` -(`b.beginFunction(name, .host)`) that builds those objects from the XLA -buffers and calls `b.launchEx`. The custom call carries no grid or block: -XLA runs the host function with the real buffers and launches (or records in -a command buffer) whatever it launched. `CallOpts.scalars` are passed to the -host function after the buffers. `zml/kernels/moe/cute/persistent_mxfp4.zig` -is the reference: a warp-specialized SM100 block-scaled GEMM whose host -function builds four TMA tensor maps and a PDL launch. - -## The custom call - -`ops.cute` emits `__gpu$xla.gpu.cute` with a printed dictionary: `name`, -`kernel_type = "cute"`, `ir` (the textual module), `grid` and `block` -(3-element arrays), optional `zeroed_outputs`. XLA compiles it in process -(`lib/libcute_compiler.so`, which pulls in `lib/cutlass_ir.so`, both from the -sandbox pointed at by `--xla_gpu_cuda_data_dir`), takes the kernel's -dynamic shared memory requirement from the compiler's metadata, and launches -the cubin over `grid` × `block`. Every operand and result is one raw device -pointer in the default layout, operands first; the count is not checked -against the kernel signature, so `inputs`/`outputs` must match `declareArgs`. -A module that carries its own `gpu.module` and host launch still works; -it then owns the launch configuration and `grid`/`block` must be left out. - -To see what XLA received, run with `--xla_dump_to=`: it writes -`.cute.mlir`, `.cute.json` and `.cubin`. - -## Finding what exists - -```sh -# the DSL: Value's fluent methods (add, mul, lt, to, exp, ...) -awk '/^pub const Value = struct/,/^};/' dsl/cute/builder.zig | grep 'pub fn ' - -# the DSL: everything on Tensor and Builder -awk '/^pub const Tensor = struct/,/^};/' dsl/cute/builder.zig | grep 'pub fn ' -awk '/^pub const Builder = struct/,0' dsl/cute/builder.zig | grep ' pub fn ' - -# every bound op of the dialect -grep -n '^pub fn ' mlir/dialects/cute_ir/cute_ir.zig mlir/dialects/cute_nvgpu/cute_nvgpu.zig -``` - -`grep -n '^test "' dsl/cute/builder.zig` lists the tests; each prints a -kernel that round-trips through the verifier. diff --git a/dsl/cute/builder.zig b/dsl/cute/builder.zig deleted file mode 100644 index 8f791f80..00000000 --- a/dsl/cute/builder.zig +++ /dev/null @@ -1,4953 +0,0 @@ -const std = @import("std"); - -const arith = @import("mlir/dialects/arith"); -const cf = @import("mlir/dialects/cf"); -const common = @import("dsl/common/builder"); -const cuda = @import("mlir/dialects/cuda"); -const cute_ir = @import("mlir/dialects/cute_ir"); -const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); -const func = @import("mlir/dialects/func"); -const gpu = @import("mlir/dialects/gpu"); -const llvm = @import("mlir/dialects/llvm"); -const math = @import("mlir/dialects/math"); -const nvvm = @import("mlir/dialects/nvvm"); -const scf = @import("mlir/dialects/scf"); -const vector = @import("mlir/dialects/vector"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -/// `CUtensorMapDataType`. -pub const TensorMapDataType = enum(u32) { - uint8 = 0, - uint16 = 1, - uint32 = 2, - int32 = 3, - uint64 = 4, - int64 = 5, - float16 = 6, - float32 = 7, - float64 = 8, - bfloat16 = 9, - float32_ftz = 10, - tfloat32 = 11, - tfloat32_ftz = 12, - @"16u4_align8b" = 13, - @"16u4_align16b" = 14, - @"16u6_align16b" = 15, -}; - -/// `CUtensorMapSwizzle`. -pub const TensorMapSwizzle = enum(u32) { none = 0, @"32b" = 1, @"64b" = 2, @"128b" = 3, @"128b_atom_32b" = 4 }; -/// `CUtensorMapL2promotion`. -pub const TensorMapL2Promotion = enum(u32) { none = 0, @"64b" = 1, @"128b" = 2, @"256b" = 3 }; -/// `CUtensorMapInterleave`. -pub const TensorMapInterleave = enum(u32) { none = 0, @"16b" = 1, @"32b" = 2 }; -/// `CUtensorMapFloatOOBfill`. -pub const TensorMapOobFill = enum(u32) { none = 0, nan_request_zero_fma = 1 }; - -pub const TensorMapSpec = struct { - data_type: TensorMapDataType, - rank: u32, - global_dims: [5]u64 = @splat(1), - /// Byte strides of dimensions 1..rank-1. - global_strides: [4]u64 = @splat(0), - box_dims: [5]u32 = @splat(1), - element_strides: [5]u32 = @splat(1), - interleave: TensorMapInterleave = .none, - swizzle: TensorMapSwizzle, - l2_promotion: TensorMapL2Promotion = .none, - oob_fill: TensorMapOobFill = .none, -}; - -const dtypes = @import("dtype.zig"); -pub const DType = dtypes.DType; -const isFloatDtype = dtypes.isFloatDtype; -const dtypeBitwidth = dtypes.dtypeBitwidth; - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Builder); - std.testing.refAllDecls(Value); - std.testing.refAllDecls(Tensor); - std.testing.refAllDecls(View); - std.testing.refAllDecls(Atom); -} - -/// `nvvm` and `cuda` are NVIDIA's (`nvvm.ops`, `cuda.ops`, handles `cute_nvvm` -/// and `cuda`); `llvm` is upstream's, whose types and operations NVIDIA's -/// compiler reads as its own. -pub fn registerDialects(registry: *mlir.DialectRegistry) void { - inline for (.{ func, gpu, cute_ir, cute_nvgpu, nvvm, cuda, llvm, arith, scf, math, cf, vector }) |dialect| dialect.registerDialect(registry); -} - -pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; - -pub const MAX_RANK = 8; -pub const Dims = stdx.BoundedArray(i64, MAX_RANK); - -pub const MemorySpace = cute_ir.MemorySpace; - -/// A CuTe algebra expression assembled from Zig values. Kernel authors use -/// `Builder.expr`, `Builder.basis`, and `Builder.layoutSpec`; the DSL builds -/// the algebra attribute of the kind each use needs. -pub const AlgebraExpr = union(enum) { - static: i64, - /// A dynamic leaf known to be a multiple of this, 1 when nothing is known. - dynamic: i64, - /// A dynamic integer with an explicit bit width and divisibility. - dynamic_int: struct { width: u8, divisible_by: i64 = 1 }, - all, - basis: ScaledBasis, - tuple: []const AlgebraExpr, -}; - -pub const AlgebraToken = enum { - /// CuTe's slice placeholder (`_`). - all, - /// A dynamic algebra leaf (`?`). - dynamic, -}; - -pub const ScaledBasis = struct { - numerator: i64, - denominator: i64 = 1, - modes: [MAX_RANK]u8 = @splat(0), - mode_count: u8, -}; - -/// A dynamic CuTe integer whose runtime value is known to be divisible by a -/// static amount. CuTe carries this fact in dependent coordinate-tensor types -/// after tiling and slicing, for example `?{div=128}`. -pub const ConstrainedDynamic = struct { - divisible_by: u64, -}; - -pub const LayoutSpec = struct { - shape: AlgebraExpr, - stride: AlgebraExpr, -}; - -/// A scalar SSA value (`Int32`, `Float32`, ... in the Python DSL). -pub const Value = struct { - inner: *const mlir.Value, - kernel: ?*Builder = null, - - pub fn type_(self: Value) *const mlir.Type { - return self.inner.type_(); - } - - fn kern(self: Value) *Builder { - return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); - } - - pub fn dtype(self: Value) DType { - return dtypes.mlirToDType(self.kern().ctx, self.type_()); - } - - pub fn isFloat(self: Value) bool { - const t = self.type_(); - inline for (@typeInfo(mlir.FloatTypes).@"enum".field_names) |f| { - if (t.isA(mlir.FloatType(@field(mlir.FloatTypes, f))) != null) return true; - } - return false; - } - - pub fn isInt(self: Value) bool { - return self.type_().isA(mlir.IntegerType) != null; - } - - pub fn isPtr(self: Value) bool { - return self.type_().isA(cute_ir.PtrType) != null; - } - - pub fn add(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(if (l.isFloat()) arith.addf(k.ctx, l.inner, r.inner, k.loc()) else arith.addi(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn sub(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(if (l.isFloat()) arith.subf(k.ctx, l.inner, r.inner, k.loc()) else arith.subi(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn mul(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(if (l.isFloat()) arith.mulf(k.ctx, l.inner, r.inner, k.loc()) else arith.muli(k.ctx, l.inner, r.inner, k.loc())); - } - - /// Signed division on integers (Python `//` on Int32). - pub fn div(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(if (l.isFloat()) arith.divf(k.ctx, l.inner, r.inner, k.loc()) else arith.divsi(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn rem(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(if (l.isFloat()) arith.remf(k.ctx, l.inner, r.inner, k.loc()) else arith.remsi(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn cdiv(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - if (l.isFloat()) @panic("Value.cdiv is integer-only"); - return k.emit(arith.ceildivsi(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn neg(self: Value) Value { - const k = self.kern(); - if (self.isFloat()) return k.emit(arith.negf(k.ctx, self.inner, k.loc())); - return k.constant(self.dtype(), 0).sub(self); - } - - pub fn abs(self: Value) Value { - const k = self.kern(); - return k.emit(if (self.isFloat()) math.absf(k.ctx, self.inner, k.loc()) else math.absi(k.ctx, self.inner, k.loc())); - } - - pub fn minimum(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(if (l.isFloat()) arith.minimumf(k.ctx, l.inner, r.inner, k.loc()) else arith.minsi(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn maximum(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(if (l.isFloat()) arith.maximumf(k.ctx, l.inner, r.inner, k.loc()) else arith.maxsi(k.ctx, l.inner, r.inner, k.loc())); - } - - /// Float max that ignores NaN operands (`arith.maxnumf`, a single `max.f32`), unlike - /// the NaN-propagating `maximum`. - pub fn maxnum(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(arith.maxnumf(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn bitAnd(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(arith.andi(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn bitOr(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(arith.ori(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn bitXor(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(arith.xori(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn shl(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(arith.shli(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn shr(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(arith.shrsi(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn shrLogical(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.emit(arith.shrui(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn bitCast(self: Value, dt: DType) Value { - const k = self.kern(); - if (dtypeBitwidth(self.dtype()) != dtypeBitwidth(dt)) @panic("Value.bitCast requires equal bit widths"); - return k.emit(arith.bitcast(k.ctx, self.inner, dt.toMlir(k.ctx), k.loc())); - } - - pub fn lt(self: Value, rhs: anytype) Value { - return self.kern().cmp(self, rhs, .slt, .olt); - } - - pub fn le(self: Value, rhs: anytype) Value { - return self.kern().cmp(self, rhs, .sle, .ole); - } - - pub fn gt(self: Value, rhs: anytype) Value { - return self.kern().cmp(self, rhs, .sgt, .ogt); - } - - pub fn ge(self: Value, rhs: anytype) Value { - return self.kern().cmp(self, rhs, .sge, .oge); - } - - pub fn eq(self: Value, rhs: anytype) Value { - return self.kern().cmp(self, rhs, .eq, .oeq); - } - - pub fn ne(self: Value, rhs: anytype) Value { - return self.kern().cmp(self, rhs, .ne, .une); - } - - pub fn exp(self: Value) Value { - const k = self.kern(); - return k.emit(math.exp(k.ctx, self.inner, k.loc())); - } - - pub fn log(self: Value) Value { - const k = self.kern(); - return k.emit(math.log(k.ctx, self.inner, k.loc())); - } - - pub fn log2(self: Value) Value { - const k = self.kern(); - return k.emit(math.log2(k.ctx, self.inner, k.loc())); - } - - pub fn exp2(self: Value) Value { - const k = self.kern(); - return k.emit(math.exp2(k.ctx, self.inner, k.loc())); - } - - pub fn ceil(self: Value) Value { - const k = self.kern(); - if (!self.isFloat()) @panic("Value.ceil is float-only"); - return k.emit(math.ceil(k.ctx, self.inner, k.loc())); - } - - pub fn sqrt(self: Value) Value { - const k = self.kern(); - return k.emit(math.sqrt(k.ctx, self.inner, k.loc())); - } - - pub fn rsqrt(self: Value) Value { - const k = self.kern(); - return k.emit(math.rsqrt(k.ctx, self.inner, k.loc())); - } - - pub fn tanh(self: Value) Value { - const k = self.kern(); - return k.emit(math.tanh(k.ctx, self.inner, k.loc())); - } - - /// Python `x.to(T)`: int <-> int is signed, int <-> float goes through - /// `sitofp`/`fptosi`, same-width floats through `convertf`. - pub fn to(self: Value, dt: DType) Value { - const k = self.kern(); - const src = self.dtype(); - if (src == dt) return self; - const ty = dt.toMlir(k.ctx); - const src_f = isFloatDtype(src); - const dst_f = isFloatDtype(dt); - const sw = dtypeBitwidth(src); - const dw = dtypeBitwidth(dt); - if (src_f and dst_f) { - if (sw == dw) return k.emit(arith.convertf(k.ctx, self.inner, ty, .{}, k.loc())); - return k.emit(if (dw > sw) arith.extf(k.ctx, self.inner, ty, k.loc()) else arith.truncf(k.ctx, self.inner, ty, k.loc())); - } - if (src_f) return k.emit(arith.fptosi(k.ctx, self.inner, ty, k.loc())); - if (dst_f) return k.emit(arith.sitofp(k.ctx, self.inner, ty, k.loc())); - if (sw == dw) return self; - if (src == .i1) return k.emit(arith.extui(k.ctx, self.inner, ty, k.loc())); - return k.emit(if (dw > sw) arith.extsi(k.ctx, self.inner, ty, k.loc()) else arith.trunci(k.ctx, self.inner, ty, k.loc())); - } -}; - -/// `cute.Layout`: a static shape and stride, carried in Zig so coordinates -/// and element counts need no MLIR queries. -pub const Layout = struct { - inner: *const mlir.Value, - shape: Dims, - stride: Dims, - - pub fn rank(self: Layout) usize { - return self.shape.len; - } - - pub fn size(self: Layout) i64 { - var n: i64 = 1; - for (self.shape.constSlice()) |d| n *= d; - return n; - } - - pub fn dim(self: Layout, i: usize) i64 { - return self.shape.get(i); - } -}; - -/// `cute.Tensor`: an engine (pointer) composed with a layout. `get`/`set` -/// are the Python `t[coord]` / `t[coord] = v`. -pub const Tensor = struct { - inner: *const mlir.Value, - kernel: *Builder, - layout: Layout, - dtype: DType, - space: MemorySpace, - - pub fn shape(self: Tensor) []const i64 { - return self.layout.shape.constSlice(); - } - - pub fn rank(self: Tensor) usize { - return self.layout.rank(); - } - - pub fn size(self: Tensor) i64 { - return self.layout.size(); - } - - pub fn dim(self: Tensor, i: usize) i64 { - return self.layout.dim(i); - } - - /// `t[coord]`: `coord` is a tuple of `Value`s and ints, one per mode. - pub fn get(self: Tensor, coord: anytype) Value { - const k = self.kernel; - const c = k.makeCoord(coord); - return k.emit(cute_ir.memref_load(k.ctx, self.inner, c, self.dtype.toMlir(k.ctx), k.loc())); - } - - /// `t[coord] = value`; the value is converted to the element type. - pub fn set(self: Tensor, coord: anytype, value_: anytype) void { - const k = self.kernel; - const c = k.makeCoord(coord); - const v = k.constant(self.dtype, value_).to(self.dtype); - _ = cute_ir.memref_store(k.ctx, self.inner, c, v.inner, k.loc()).appendTo(k.currentBlock()); - } - - /// Erase Zig's tracked layout while retaining the MLIR memref. CuTe - /// layout algebra derives the layouts returned by tiling/partitioning; - /// those values are represented by `View` because Zig cannot recompute - /// their dependent types. - pub fn view(self: Tensor) View { - return .{ .inner = self.inner, .kernel = self.kernel }; - } - - pub fn value(self: Tensor) Value { - return .{ .inner = self.inner, .kernel = self.kernel }; - } -}; - -/// A CuTe tensor or pointer with a layout derived by the CuTe compiler. -pub const View = struct { - inner: *const mlir.Value, - kernel: *Builder, - - pub fn type_(self: View) *const mlir.Type { - return self.inner.type_(); - } - - pub fn value(self: View) Value { - return .{ .inner = self.inner, .kernel = self.kernel }; - } - - pub fn get(self: View, coord: anytype, result_dtype: DType) Value { - const c = self.kernel.makeCoord(coord); - return self.kernel.emit(cute_ir.memref_load(self.kernel.ctx, self.inner, c, result_dtype.toMlir(self.kernel.ctx), self.kernel.loc())); - } - - pub fn set(self: View, coord: anytype, value_: Value) void { - self.kernel.emitVoid(cute_ir.memref_store(self.kernel.ctx, self.inner, self.kernel.makeCoord(coord), value_.inner, self.kernel.loc())); - } -}; - -/// A copy/MMA atom or a tiled atom. Atom fields are immutable SSA values; -/// `setField` returns the atom carrying the updated runtime field. -pub const Atom = struct { - inner: *const mlir.Value, - kernel: *Builder, - - pub fn type_(self: Atom) *const mlir.Type { - return self.inner.type_(); - } - - pub fn value(self: Atom) Value { - return .{ .inner = self.inner, .kernel = self.kernel }; - } - - pub fn setField(self: Atom, field: []const u8, value_: anytype) Atom { - const T = @TypeOf(value_); - const new_value = if (T == Value or T == Tensor or T == View or T == Atom or T == Layout) - self.kernel.asValue(value_) - else - self.kernel.lift(value_); - const result = self.kernel.emit(cute_nvgpu.atom_set_value( - self.kernel.ctx, - self.inner, - new_value.inner, - self.type_(), - .string(self.kernel.ctx, field), - self.kernel.loc(), - )); - return .{ .inner = result.inner, .kernel = self.kernel }; - } -}; - -pub const TmaDescriptor = struct { - atom: Atom, - tensor: View, -}; - -pub const TmaPartition = struct { - smem: View, - targets: []const View, -}; - -/// Element encoding of a TMA transfer when it differs from the tensor dtype. -pub const TmaFormat = enum { - default, - /// Packed 4-bit elements, two per byte in shared memory (NVFP4 operands - /// of the `mxf4nvf4` MMA). - u4, - /// 4-bit elements expanded to one per byte in shared memory (FP4 operands - /// of the `mxf8f6f4` MMA). - u4_unpack_u8, - u16, -}; - -/// CuTe's 64-bit TMA `.L2::cache_hint` policies for SM90. -pub const CacheHintSm90 = enum(u64) { - evict_normal = 0x1000000000000000, - evict_first = 0x12F0000000000000, - evict_last = 0x14F0000000000000, -}; - -/// CuTe's 64-bit TMA `.L2::cache_hint` policies for SM100. -pub const CacheHintSm100 = enum(u64) { - evict_normal = 0x1000000000000000, - evict_first = 0x12F0000000000000, - evict_last = 0x14F0000000000000, -}; - -pub const CtaGroup = enum(u8) { - one = 1, - two = 2, - - fn toNvvm(self: CtaGroup) nvvm.CtaGroup { - return switch (self) { - .one => .cta_1, - .two => .cta_2, - }; - } -}; -pub const MBarrierScope = nvvm.MBarrierScope; -pub const OperandMajorMode = enum(u1) { k = 0, mn = 1 }; - -/// Number of CTAs cooperating on one instruction, as in CuTe's CtaGroup.value. -pub fn ctaCount(group: CtaGroup) u8 { - return @backingInt(group); -} - -pub const Tcgen05SmemSwizzle = enum(u3) { - none = 0, - swizzle_128b_atom_32b = 1, - swizzle_128b = 2, - swizzle_64b = 4, - swizzle_32b = 6, -}; - -/// CuTe's packed SM100 shared-memory descriptor. All offsets are in bytes. -pub const Tcgen05SmemDesc = struct { - pub const Options = struct { - leading_byte_offset: u32 = 0, - stride_byte_offset: u32 = 0, - version: u2 = 1, - base_offset: u3 = 0, - layout: Tcgen05SmemSwizzle = .none, - }; - - pub fn build(b: *Builder, start_address: Value, opts: Options) Value { - std.debug.assert(opts.leading_byte_offset % 16 == 0 and opts.leading_byte_offset < 0x40000); - std.debug.assert(opts.stride_byte_offset % 16 == 0 and opts.stride_byte_offset < 0x40000); - const fields = (@as(u64, opts.leading_byte_offset >> 4) << 16) | - (@as(u64, opts.stride_byte_offset >> 4) << 32) | - (@as(u64, opts.version) << 46) | - (@as(u64, opts.base_offset) << 49) | - (@as(u64, @backingInt(opts.layout)) << 61); - const address = if (start_address.isInt()) start_address else b.ptrToInt(start_address, .i32); - return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.constant(.i64, @as(i64, @bitCast(fields)))); - } -}; - -/// Dense SM100 instruction descriptor; the MMA kind disambiguates dtype encodings. -pub const Tcgen05InstrDesc = struct { - pub const Options = struct { - a_dtype: DType, - b_dtype: DType, - c_dtype: DType = .f32, - m_dim: u16, - n_dim: u16, - a_major: OperandMajorMode = .k, - b_major: OperandMajorMode = .k, - a_negate: bool = false, - b_negate: bool = false, - }; - - pub fn build(comptime opts: Options) u32 { - comptime std.debug.assert(opts.m_dim > 0 and opts.m_dim <= 256 and opts.m_dim % 16 == 0); - comptime std.debug.assert(opts.n_dim > 0 and opts.n_dim <= 256 and opts.n_dim % 8 == 0); - const half_inputs = (opts.a_dtype == .f16 or opts.a_dtype == .bf16) and (opts.b_dtype == .f16 or opts.b_dtype == .bf16); - const fp8_inputs = (opts.a_dtype == .f8e4m3fn or opts.a_dtype == .f8e5m2) and (opts.b_dtype == .f8e4m3fn or opts.b_dtype == .f8e5m2); - const valid = (half_inputs and opts.c_dtype == .f32) or - (opts.a_dtype == .f16 and opts.b_dtype == .f16 and opts.c_dtype == .f16) or - (opts.a_dtype == .f32 and opts.b_dtype == .f32 and opts.c_dtype == .f32) or - (fp8_inputs and (opts.c_dtype == .f16 or opts.c_dtype == .f32)) or - (opts.a_dtype == .i8 and opts.b_dtype == .i8 and opts.c_dtype == .i32); - if (!valid) @compileError("unsupported dense tcgen05 dtype combination"); - const c: u32 = switch (opts.c_dtype) { - .f16 => 0, - .f32 => 1, - .i32 => 2, - else => @compileError("unsupported tcgen05 accumulator dtype"), - }; - return (c << 4) | (inputFormat(opts.a_dtype) << 7) | (inputFormat(opts.b_dtype) << 10) | - (@as(u32, @intFromBool(opts.a_negate)) << 13) | - (@as(u32, @intFromBool(opts.b_negate)) << 14) | - (@as(u32, @backingInt(opts.a_major)) << 15) | - (@as(u32, @backingInt(opts.b_major)) << 16) | - (@as(u32, opts.n_dim >> 3) << 17) | (@as(u32, opts.m_dim >> 4) << 24); - } - - fn inputFormat(comptime dtype: DType) u32 { - return switch (dtype) { - .f16, .f8e4m3fn => 0, - .bf16, .f8e5m2, .i8 => 1, - .f32 => 2, - .f4e2m1fn => 5, - else => @compileError("unsupported tcgen05 input dtype"), - }; - } -}; - -/// Typed description of a non-multicast tiled TMA atom: the element type, -/// bits moved per copy, basis of one box, and coordinate layout of the tensor. -pub const TmaConfig = struct { - dtype: DType, - copy_bits: u32, - global_basis: LayoutSpec, - coordinate_layout: LayoutSpec, - coordinate_rank: u8, - format: TmaFormat = .default, - /// Load completion group: with `.two`, each CTA's load completes on - /// the mbarrier of the pair's even CTA. Ignored by TMA stores. - cta_group: CtaGroup = .one, - mode: enum { tiled, gather4 } = .tiled, -}; - -pub const CopyBulkTensorTileG2SOp = struct { cta_group: CtaGroup = .one }; -pub const CopyBulkTensor2DGather4G2SOp = struct { cta_group: CtaGroup = .one }; -pub const CopyBulkTensorTileS2GOp = struct {}; - -pub const BlockScaledMma = struct { - atom: Atom, - tiled: Atom, -}; - -/// SS, K-major SM100 block-scaled MMA instruction with FP4 weights as A. -/// The instruction `kind` fixes B, the scales and K. -pub const BlockScaledMmaConfig = struct { - kind: Kind = .mxfp4_mxfp8, - /// M is 128 for one CTA, or 128 or 256 for a CTA pair. - m: u16 = 128, - n: u16, - cta_group: CtaGroup = .one, - - pub const Kind = enum { - /// MXFP4 weights x MXFP8 activations: E8M0 scales every 32 - /// elements, K = 32 (PTX `kind::mxf8f6f4`). - mxfp4_mxfp8, - /// NVFP4 x NVFP4: E4M3 scales every 16 elements, K = 64 (PTX - /// `kind::mxf4nvf4.block16`). - nvfp4_nvfp4, - }; - - pub fn k(self: BlockScaledMmaConfig) u16 { - return switch (self.kind) { - .mxfp4_mxfp8 => 32, - .nvfp4_nvfp4 => 64, - }; - } - - fn bType(self: BlockScaledMmaConfig) DType { - return switch (self.kind) { - .mxfp4_mxfp8 => .f8e4m3fn, - .nvfp4_nvfp4 => .f4e2m1fn, - }; - } - - fn scaleType(self: BlockScaledMmaConfig) DType { - return switch (self.kind) { - .mxfp4_mxfp8 => .f8e8m0fnu, - .nvfp4_nvfp4 => .f8e4m3fn, - }; - } - - fn vecSize(self: BlockScaledMmaConfig) u8 { - return switch (self.kind) { - .mxfp4_mxfp8 => 32, - .nvfp4_nvfp4 => 16, - }; - } - - fn validate(self: BlockScaledMmaConfig) void { - const m_ok = self.m == 128 or (self.cta_group == .two and self.m == 256); - const n_multiple: u16 = if (self.cta_group == .two) 16 else 8; - if (!m_ok or self.n < n_multiple or self.n > 256 or self.n % n_multiple != 0) { - std.debug.panic("unsupported SM100 block-scaled MMA shape {d}x{d}x{d} on {d} CTAs", .{ self.m, self.n, self.k(), ctaCount(self.cta_group) }); - } - } -}; - -fn tmaLoadKind(config: TmaConfig) cute_nvgpu.TiledTmaLoad { - return switch (config.cta_group) { - .one => .sm_90, - .two => .sm_100_2sm, - }; -} - -fn tmaDataFormat(format: TmaFormat, dtype: DType) cute_nvgpu.TmaDataFormat { - return switch (format) { - .u4 => .U4, - .u4_unpack_u8 => .U4_UNPACK_U8, - .u16 => .U16, - .default => switch (dtype) { - .f8e4m3fn => .U8, - .f32 => .F32_RN, - else => @panic("TMA format must be specified for this dtype"), - }, - }; -} - -fn tmaFormatAttribute(b: *Builder, config: TmaConfig) ?*const mlir.Attribute { - if (config.format == .default) return null; - const attr = cute_nvgpu.TmaDataFormatAttr.get(b.ctx, .{ .value = tmaDataFormat(config.format, config.dtype) }) catch @panic("invalid TMA format"); - return attr.attribute(); -} - -pub const Index3 = struct { x: Value, y: Value, z: Value }; - -pub const LaunchConfig = struct { - inner: *const mlir.Value, - kernel: *Builder, - - pub fn value(self: LaunchConfig) Value { - return .{ .inner = self.inner, .kernel = self.kernel }; - } -}; - -pub const ArgSpec = struct { - name: []const u8, - kind: Kind, - - /// XLA passes one raw device pointer per custom-call argument. A `tensor` - /// argument is that pointer composed with a static layout at the top of - /// the kernel, the way the Python DSL passes `cute.Tensor` arguments. - pub const Kind = union(enum) { - ptr: DType, - tensor: TensorSpec, - /// A type constructed through a dialect/type API. - mlir_type: MlirTypeSpec, - }; - - pub const MlirTypeSpec = struct { - type_: *const mlir.Type, - grid_constant: bool = false, - cuda_grid_constant: bool = false, - }; - - pub const TensorSpec = struct { - dtype: DType, - shape: []const i64, - /// Defaults to XLA's row-major layout. - stride: ?[]const i64 = null, - alignment: u64 = 16, - }; -}; - -pub const LayoutOpts = struct { - /// Defaults to CuTe's compact left-most stride, like `cute.make_layout`. - stride: ?[]const i64 = null, -}; - -pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); - -pub fn IfScope(comptime N: usize) type { - return common.IfScope(Builder, Value, N); -} - -pub fn ForScope(comptime N: usize) type { - return common.ForScope(Builder, Value, N); -} - -pub fn WhileScope(comptime N: usize, comptime M: usize) type { - return common.WhileScope(Builder, Value, N, M); -} - -pub const Builder = struct { - pub const FunctionKind = enum { func_only_kernel, cuda_kernel, host }; - - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - ctx: *mlir.Context, - module: *mlir.Module, - gpu_body: ?*mlir.Block = null, - name: []const u8, - function_kind: FunctionKind = .func_only_kernel, - func_op: ?*mlir.Operation = null, - entry_block: ?*mlir.Block = null, - block_stack: common.BlockStack = .{}, - args: []const ArgSpec = &.{}, - tensors: []?Tensor = &.{}, - - pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { - const module: *mlir.Module = .init(.unknown(ctx)); - errdefer module.deinit(); - return .{ - .allocator = allocator, - .arena = .init(allocator), - .ctx = ctx, - .module = module, - .name = name, - }; - } - - /// Open a complete CuTe program. Unlike `open`, which emits the compact - /// func.func-only ABI consumed directly by XLA, a program owns a - /// `gpu.module` for one or more `cuda.kernel` operations and can append a - /// host `func.func` that constructs CuTe launch objects and calls - /// `cuda.launch_ex`. - pub fn openProgram(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { - var self = try open(allocator, ctx, name); - self.module.operation().setAttribute(gpu.containerModule(ctx)); - const body = mlir.Block.init(&.{}, &.{}); - const gpu_module = gpu.module(ctx, "kernels", body, self.loc()); - _ = gpu_module.appendTo(self.module.body()); - self.gpu_body = body; - self.function_kind = .cuda_kernel; - return self; - } - - /// Start another function in a complete program after `endFunction`. - pub fn beginFunction(self: *Builder, name: []const u8, kind: FunctionKind) void { - std.debug.assert(self.entry_block == null and self.func_op == null); - self.name = name; - self.function_kind = kind; - } - - pub fn deinit(self: *Builder) void { - self.module.deinit(); - self.arena.deinit(); - } - - /// One field per kernel argument: `.{ .a = .{ .tensor = .{ .dtype = .f32, .shape = &.{n} } } }` - /// yields a `Tensor`, `.{ .p = .{ .ptr = .f32 } }` a pointer `Value`. - pub fn declareArgs(self: *Builder, spec: anytype) FinishError!ArgsOf(@TypeOf(spec)) { - const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".field_names; - - const arg_specs = try self.arena.allocator().alloc(ArgSpec, fields.len); - inline for (fields, 0..) |f, i| { - const raw = @field(spec, f); - const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; - const inner = @field(raw, variant); - arg_specs[i] = .{ .name = f, .kind = switch (@field(std.meta.Tag(ArgSpec.Kind), variant)) { - .ptr => .{ .ptr = inner }, - .tensor => .{ .tensor = .{ - .dtype = inner.dtype, - .shape = inner.shape, - .stride = if (@hasField(@TypeOf(inner), "stride")) inner.stride else null, - .alignment = if (@hasField(@TypeOf(inner), "alignment")) inner.alignment else 16, - } }, - .mlir_type => .{ .mlir_type = .{ - .type_ = inner.type_, - .grid_constant = if (@hasField(@TypeOf(inner), "grid_constant")) inner.grid_constant else false, - .cuda_grid_constant = if (@hasField(@TypeOf(inner), "cuda_grid_constant")) inner.cuda_grid_constant else false, - } }, - } }; - } - try self.declareArgsLow(arg_specs); - - var named: ArgsOf(Spec) = undefined; - inline for (fields, 0..) |f, i| { - @field(named, f) = switch (@FieldType(ArgsOf(Spec), f)) { - Tensor => self.tensors[i].?, - else => self.arg(i), - }; - } - return named; - } - - fn ArgsOf(comptime Spec: type) type { - const in = @typeInfo(Spec).@"struct".field_names; - comptime var names: [in.len][]const u8 = undefined; - comptime var types: [in.len]type = undefined; - inline for (in, 0..) |f, i| { - names[i] = f; - types[i] = if (@hasField(@FieldType(Spec, f), "tensor")) Tensor else Value; - } - return @Struct(.auto, null, &names, &types, &@splat(.{})); - } - - pub fn declareArgsLow(self: *Builder, args: []const ArgSpec) FinishError!void { - std.debug.assert(self.entry_block == null); - const ctx = self.ctx; - const scratch = self.arena.allocator(); - - const arg_types = try scratch.alloc(*const mlir.Type, args.len); - const arg_locs = try scratch.alloc(*const mlir.Location, args.len); - for (arg_types, arg_locs, args) |*ty, *l, a| { - ty.* = switch (a.kind) { - .ptr => |dt| self.ptrTy(dt, .gmem, 16), - .tensor => |t| self.ptrTy(t.dtype, .gmem, t.alignment), - .mlir_type => |typed| typed.type_, - } catch return error.InvalidMlir; - l.* = self.loc(); - } - - const entry = mlir.Block.init(arg_types, arg_locs); - const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); - const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); - var any_arg_attr = false; - for (arg_attrs, args) |*attr, a| { - attr.* = switch (a.kind) { - .mlir_type => |typed| if (typed.cuda_grid_constant) .dict(ctx, &.{.named(ctx, "cuda.grid_constant", .unit(ctx))}) else if (typed.grid_constant) .dict(ctx, &.{cute_nvgpu.gridConstant(ctx)}) else empty_dict, - else => empty_dict, - }; - if (attr.* != empty_dict) any_arg_attr = true; - } - - const func_op = switch (self.function_kind) { - .func_only_kernel => func.func(ctx, .{ - .name = self.name, - .block = entry, - .args_attributes = if (any_arg_attr) arg_attrs else null, - .results = &.{}, - .location = self.loc(), - .verify = false, - .extra_attributes = &.{ - cute_ir.kernel(ctx), - gpu.kernel(ctx), - }, - }), - .cuda_kernel => cuda.kernel(ctx, .{ - .name = self.name, - .function_type = .function(ctx, arg_types, &.{}), - .body = entry, - .arg_attrs = .array(ctx, arg_attrs), - .extra_attributes = &.{ - cute_ir.kernel(ctx), - gpu.kernel(ctx), - }, - .location = self.loc(), - }), - .host => func.func(ctx, .{ - .name = self.name, - .block = entry, - .args_attributes = if (any_arg_attr) arg_attrs else null, - .results = &.{.int(ctx, .i32)}, - .location = self.loc(), - .verify = false, - .extra_attributes = &.{llvm.emitCInterface(ctx)}, - }), - }; - _ = func_op.appendTo(if (self.function_kind == .cuda_kernel) - self.gpu_body orelse @panic("cuda_kernel requires Builder.openProgram") - else - self.module.body()); - - self.func_op = func_op; - self.entry_block = entry; - self.block_stack.push(self.arena.allocator(), entry); - self.args = args; - - self.tensors = try scratch.alloc(?Tensor, args.len); - for (self.tensors, args, 0..) |*t, a, i| { - t.* = switch (a.kind) { - .ptr => null, - .tensor => |spec| self.makeTensor(self.arg(i), self.makeLayout(spec.shape, .{ - .stride = spec.stride orelse self.rowMajor(spec.shape), - })), - .mlir_type => null, - }; - } - } - - pub fn arg(self: *Builder, i: usize) Value { - const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); - return .{ .inner = eb.argument(i), .kernel = self }; - } - - pub fn pushBlock(self: *Builder, b: *mlir.Block) void { - self.block_stack.push(self.arena.allocator(), b); - } - - pub fn popBlock(self: *Builder) void { - self.block_stack.pop(); - } - - pub fn currentBlock(self: *Builder) *mlir.Block { - return self.block_stack.current(); - } - - pub fn emit(self: *Builder, op: *mlir.Operation) Value { - _ = op.appendTo(self.currentBlock()); - return .{ .inner = op.result(0), .kernel = self }; - } - - /// Append an operation which does not return an SSA value. - pub fn emitVoid(self: *Builder, op: *mlir.Operation) void { - std.debug.assert(op.numResults() == 0); - _ = op.appendTo(self.currentBlock()); - } - - /// Append a multi-result operation and return all of its SSA values. - /// - /// Most of the small CuTe kernels only need one-result operations, but - /// TMA descriptor construction and TMA partitioning return an atom and a - /// coordinate tensor together. Keeping both results from one operation - /// is required: rebuilding either value changes the descriptor identity. - pub fn emitResults(self: *Builder, comptime N: usize, op: *mlir.Operation) [N]Value { - std.debug.assert(op.numResults() == N); - _ = op.appendTo(self.currentBlock()); - var result: [N]Value = undefined; - for (&result, 0..) |*value, i| value.* = .{ .inner = op.result(i), .kernel = self }; - return result; - } - - /// Convert any DSL handle into a scalar wrapper without changing its - /// MLIR type. This keeps the low-level generated bindings usable while - /// callers work with `Tensor`, `View`, and `Atom` handles. - pub fn asValue(self: *Builder, value_: anytype) Value { - return switch (@TypeOf(value_)) { - Value => value_, - Tensor => value_.value(), - View => value_.value(), - Atom => value_.value(), - LaunchConfig => value_.value(), - Layout => .{ .inner = value_.inner, .kernel = self }, - else => @compileError("expected a CuTe DSL handle, got " ++ @typeName(@TypeOf(value_))), - }; - } - - fn innerSlice(self: *Builder, values: anytype) []const *const mlir.Value { - const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("OOM"); - switch (@typeInfo(@TypeOf(values))) { - .@"struct" => { - inline for (values, 0..) |value, i| out[i] = self.asValue(value).inner; - }, - else => { - for (values, out) |value, *dst| dst.* = self.asValue(value).inner; - }, - } - return out; - } - - pub fn loc(self: *const Builder) *const mlir.Location { - return .unknown(self.ctx); - } - - /// `#cuda.dev_max_shared_memory_optin`, the device's opt-in maximum of - /// shared memory per block, for a kernel's `cu_attrs`. - pub fn devMaxSharedMemoryOptin(self: *Builder) *const mlir.Attribute { - return (cuda.ops.DevMaxSharedMemoryOptinAttr.get(self.ctx, .{}) catch @panic("invalid CUDA attribute")).attribute(); - } - - pub fn setFunctionAttribute(self: *Builder, attribute: mlir.NamedAttribute) void { - const function = self.func_op orelse @panic("setFunctionAttribute called before declareArgs"); - function.setAttribute(attribute); - } - - // ==================== cuda host launch ==================== - - /// `@kernels::@`, a kernel of the nested GPU module. - fn kernelSymbol(self: *Builder, kernel_name: []const u8) *const mlir.Attribute { - return .symbolRef(self.ctx, "kernels", &.{.init(self.ctx, kernel_name)}); - } - - pub fn kernelSmemSize(self: *Builder, kernel_name: []const u8) Value { - return self.emit(cute_ir.kernel_smem_size( - self.ctx, - .int(self.ctx, .i64), - self.kernelSymbol(kernel_name), - self.loc(), - )); - } - - /// Convert the stream placeholder used by a generated host entry into a - /// CUDA stream. The native CuTe host bridge substitutes XLA's current - /// stream when it executes the wrapper. - pub fn cudaStream(self: *Builder) Value { - return self.emit(cuda.cast(self.ctx, cuda.streamType(self.ctx), self.constant(.i64, 0).inner, self.loc())); - } - - pub const LaunchOptions = struct { - grid: [3]Value, - block: [3]Value, - dynamic_smem: Value, - stream: Value, - cluster: ?[3]Value = null, - cooperative: bool = false, - use_pdl: bool = false, - }; - - /// Build the mutable CUDA launch configuration used by `cuda.launch_ex`. - pub fn makeLaunchConfig(self: *Builder, opts: LaunchOptions) LaunchConfig { - const config = self.emit(cuda.launch_cfg_create(self.ctx, .{ - .max_attrs = 17, - .block = .{ opts.block[0].inner, opts.block[1].inner, opts.block[2].inner }, - .dynamic_smem = opts.dynamic_smem.inner, - .grid = .{ opts.grid[0].inner, opts.grid[1].inner, opts.grid[2].inner }, - .stream = opts.stream.inner, - .location = self.loc(), - })); - if (opts.cluster) |cluster| { - self.emitVoid(cuda.launch_cfg_cluster_dim(self.ctx, config.inner, .{ cluster[0].inner, cluster[1].inner, cluster[2].inner }, self.loc())); - } - self.emitVoid(cuda.launch_cfg_cooperative(self.ctx, config.inner, self.constant(.i32, @intFromBool(opts.cooperative)).inner, self.loc())); - self.emitVoid(cuda.launch_cfg_programmatic_stream_serialization_allowed(self.ctx, config.inner, self.constant(.i32, @intFromBool(opts.use_pdl)).inner, self.loc())); - return .{ .inner = config.inner, .kernel = self }; - } - - /// Request a CUDA kernel launch from a generated host function. The host - /// bridge records this request; XLA performs or command-buffer-records the - /// actual launch on its stream. - pub fn launchEx(self: *Builder, kernel_name: []const u8, config: LaunchConfig, arguments: anytype) Value { - const values = self.innerSlice(arguments); - return self.emit(cuda.launch_ex(self.ctx, .{ - .config = config.inner, - .inputs = values, - .callee = self.kernelSymbol(kernel_name), - .assume_kernel_attr = cuda.assumeKernelAttr(self.ctx, true), - .location = self.loc(), - })); - } - - /// `!cuda.tensor_map`: the kernel argument type of an explicitly encoded map. - pub fn tensorMapType(self: *Builder) *const mlir.Type { - return (cuda.ops.TensorMapType.get(self.ctx, .{}) catch unreachable).type_(); - } - - /// A host stack array of `values` (u64 -> i64 slots, u32 -> i32 slots): a `cuuint64_t*` / - /// `cuuint32_t*` argument of `cuTensorMapEncodeTiled`. - fn tensorMapHostArray(self: *Builder, comptime T: type, values: []const T) Value { - const ctx = self.ctx; - const elem: *const mlir.Type = if (T == u64) .int(ctx, .i64) else .int(ctx, .i32); - const ptr_type = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic); - const array = self.emit(mlir.Operation.make(ctx, "llvm.alloca", .{ - .operands = .{ .flat = &.{self.constant(.i32, @as(i32, @intCast(@max(values.len, 1)))).inner} }, - .results = .{ .flat = &.{ptr_type} }, - .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "alignment", .int(ctx, .i64, 8)) }, - .location = self.loc(), - })); - for (values, 0..) |v, i| { - const slot = self.emit(mlir.Operation.make(ctx, "llvm.getelementptr", .{ - .operands = .{ .flat = &.{array.inner} }, - .results = .{ .flat = &.{ptr_type} }, - .attributes = &.{ .named(ctx, "elem_type", .typeAttr(elem)), .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, &[_]i32{@intCast(i)})) }, - .location = self.loc(), - })); - const value = if (T == u64) self.constant(.i64, @as(i64, @intCast(v))) else self.constant(.i32, @as(i32, @intCast(v))); - self.emitVoid(llvm.store(ctx, value.inner, slot.inner, .{ .alignment = @sizeOf(T) }, self.loc())); - } - return array; - } - - /// Encode a tiled map in a host function and return both the map and CUDA status. - /// The caller supplies the final device address and handles the returned status. - pub fn encodeTensorMap(self: *Builder, spec: TensorMapSpec, address: Value) struct { map: Value, result: Value } { - const ctx = self.ctx; - // The `cuda` dialect's address operand is a `ptr` dialect pointer, which zml does not register. - ctx.setAllowUnregisteredDialects(true); - const addr = self.emit(cuda.cast(ctx, (mlir.Type.parse(self.ctx, "!ptr.ptr<#ptr.generic_space>") catch @panic("invalid tensor-map IR")), address.inner, self.loc())); - const results = self.emitResults(2, cuda.ops.tensor_map_encode_tiled( - ctx, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.data_type)))).inner, - self.constant(.i32, @as(i32, @intCast(spec.rank))).inner, - addr.inner, - self.tensorMapHostArray(u64, spec.global_dims[0..spec.rank]).inner, - self.tensorMapHostArray(u64, spec.global_strides[0 .. spec.rank - 1]).inner, - self.tensorMapHostArray(u32, spec.box_dims[0..spec.rank]).inner, - self.tensorMapHostArray(u32, spec.element_strides[0..spec.rank]).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.interleave)))).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.swizzle)))).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.l2_promotion)))).inner, - self.constant(.i32, @as(i32, @intCast(@backingInt(spec.oob_fill)))).inner, - cuda.resultType(ctx), - self.tensorMapType(), - self.loc(), - )); - return .{ .map = results[1], .result = results[0] }; - } - - /// Device side: the generic address (i64) of the `!cuda.tensor_map` kernel argument `arg` - /// (`&tensor_map` for a `__grid_constant__` parameter). - pub fn tensorMapAddress(self: *Builder, descriptor: Value) Value { - const ctx = self.ctx; - const llvm_ptr = self.emit(cuda.cast(ctx, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), descriptor.inner, self.loc())); - return self.emit(mlir.Operation.make(ctx, "llvm.ptrtoint", .{ - .operands = .{ .flat = &.{llvm_ptr.inner} }, - .results = .{ .flat = &.{DType.i64.toMlir(ctx)} }, - .location = self.loc(), - })); - } - - pub fn cudaResultStatus(self: *Builder, result: Value) Value { - return self.emit(cuda.cast(self.ctx, .int(self.ctx, .i32), result.inner, self.loc())); - } - - pub fn returnHostStatus(self: *Builder, status: Value) void { - self.emitVoid(func.returns(self.ctx, &.{status.inner}, self.loc())); - } - - /// Finish the current function but retain the surrounding module so a - /// host launch function can be emitted next. - pub fn endFunction(self: *Builder, block: ?[3]i32) void { - const current = self.currentBlock(); - if (current.terminator() == null) { - const terminator = if (self.function_kind == .cuda_kernel) - cuda.return_(self.ctx, &.{}, self.loc()) - else - func.returns(self.ctx, &.{}, self.loc()); - _ = terminator.appendTo(current); - } - if (block) |dims| { - const function = self.func_op orelse @panic("endFunction before declareArgs"); - function.setAttribute(nvvm.reqntid(self.ctx, dims)); - } - self.func_op = null; - self.entry_block = null; - self.block_stack.clear(); - self.args = &.{}; - self.tensors = &.{}; - } - - pub fn finishProgram(self: *Builder) FinishError![:0]const u8 { - std.debug.assert(self.entry_block == null and self.func_op == null); - try common.verify(self.module, "dsl/cute"); - return self.renderModule(); - } - - fn renderModule(self: *Builder) FinishError![:0]const u8 { - var printed: std.Io.Writer.Allocating = .init(self.allocator); - defer printed.deinit(); - // Always the generic form: NVIDIA's compiler parses the module with its - // own MLIR revision, whose custom assembly differs from ours (e.g. - // `cute.make_composed_layout`, or `llvm.load`/`llvm.store` printing - // their alignment as a `` property). - try printed.writer.print("{f}", .{self.module.operation().fmt(.{ .print_generic_op_form = true })}); - return try self.allocator.dupeSentinel(u8, printed.written(), 0); - } - - // ==================== types ==================== - - pub fn ptrTy(self: *Builder, dt: DType, space: MemorySpace, alignment: u64) !*const mlir.Type { - return cute_ir.pointerType(self.ctx, space, alignment, .{ .element_type = dt.toMlir(self.ctx) }); - } - - pub fn swizzledPtrTy(self: *Builder, dt: DType, space: MemorySpace, alignment: u64, swizzle: Swizzle) !*const mlir.Type { - return (try cute_ir.PtrType.get(self.ctx, .{ - .valueType = dt.toMlir(self.ctx), - .addressSpace = space, - .alignment = alignment, - .swizzle = (try self.swizzleAttr(swizzle)).attribute(), - })).type_(); - } - - fn swizzleAttr(self: *Builder, swizzle: Swizzle) !*const cute_ir.SwizzleAttr { - return cute_ir.SwizzleAttr.get(self.ctx, .{ - .bits = std.math.cast(u32, swizzle.bits) orelse return error.InvalidMlir, - .base = std.math.cast(u32, swizzle.base) orelse return error.InvalidMlir, - .shift = std.math.cast(i32, swizzle.shift) orelse return error.InvalidMlir, - }); - } - - pub fn intToPtr(self: *Builder, address: anytype, dt: DType, space: MemorySpace, alignment: u64) Value { - const ptr_type = self.ptrTy(dt, space, alignment) catch @panic("invalid CuTe pointer type"); - return self.emit(cute_ir.inttoptr(self.ctx, self.lift(address).inner, ptr_type, self.loc())); - } - - /// `cute.ptrtoint`: the integer address of a pointer (`i32` for shared - /// memory, `i64` for global memory), for hand-written PTX operands. - pub fn ptrToInt(self: *Builder, pointer: anytype, dt: DType) Value { - const ptr = self.asValue(pointer); - return self.emit(if (ptr.isPtr()) cute_ir.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc()) else llvm.ptrtoint(self.ctx, ptr.inner, dt.toMlir(self.ctx), self.loc())); - } - - /// Reinterpret a pointer without changing its address. This is the typed - /// counterpart of Python CuTe's `recast_ptr`; packed MXFP4 buffers use it - /// to expose their byte storage to a scalar correctness implementation. - pub fn recastPointer(self: *Builder, pointer: anytype, dt: DType, space: MemorySpace, alignment: u64) Value { - const result_type = self.ptrTy(dt, space, alignment) catch @panic("invalid recast pointer type"); - return self.emit(cute_ir.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); - } - - pub fn recastSwizzledPointer(self: *Builder, pointer: anytype, dt: DType, space: MemorySpace, alignment: u64, swizzle: Swizzle) Value { - const result_type = self.swizzledPtrTy(dt, space, alignment, swizzle) catch @panic("invalid swizzled pointer type"); - return self.emit(cute_ir.recast_iter(self.ctx, self.asValue(pointer).inner, result_type, self.loc())); - } - - /// The scalar of a single extent, otherwise the tuple of `dims`. - fn dimsExpr(self: *Builder, dims: []const i64) AlgebraExpr { - if (dims.len == 1) return .{ .static = dims[0] }; - const elements = self.arena.allocator().alloc(AlgebraExpr, dims.len) catch @panic("OOM"); - for (dims, elements) |d, *e| e.* = .{ .static = d }; - return .{ .tuple = elements }; - } - - /// Build a CuTe algebra expression from integers, nested Zig tuples, - /// `AlgebraToken`s, `ConstrainedDynamic`s and `ScaledBasis` values. - pub fn expr(self: *Builder, value: anytype) AlgebraExpr { - const T = @TypeOf(value); - if (T == AlgebraExpr) return value; - if (T == AlgebraToken) return switch (value) { - .all => .all, - .dynamic => .{ .dynamic = 1 }, - }; - if (T == ScaledBasis) return .{ .basis = value }; - if (T == ConstrainedDynamic) { - if (value.divisible_by == 0) @panic("CuTe constrained dynamic divisor must be positive"); - return .{ .dynamic = @intCast(value.divisible_by) }; - } - switch (@typeInfo(T)) { - .comptime_int, .int => return .{ .static = @intCast(value) }, - .@"struct" => |info| { - const elements = self.arena.allocator().alloc(AlgebraExpr, info.field_names.len) catch @panic("OOM"); - inline for (info.field_names, 0..) |field, i| elements[i] = self.expr(@field(value, field)); - return .{ .tuple = elements }; - }, - else => @compileError("unsupported CuTe algebra value " ++ @typeName(T)), - } - } - - /// The algebra attribute `A` (`cute_ir.IntTupleAttr`, `ShapeAttr`, ...) - /// of `e`. - fn algebraAttr(self: *Builder, comptime A: type, e: AlgebraExpr) *const A { - const result = switch (e) { - .static => |v| if (@hasDecl(A, "getStatic")) A.getStatic(self.ctx, v) else std.debug.panic("{s} has no integer leaf", .{@typeName(A)}), - .dynamic => |divisibility| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, 32, divisibility) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), - .dynamic_int => |v| if (@hasDecl(A, "getDynamic")) A.getDynamic(self.ctx, v.width, v.divisible_by) else std.debug.panic("{s} has no dynamic leaf", .{@typeName(A)}), - .all => if (@hasDecl(A, "getUnderscore")) A.getUnderscore(self.ctx) else std.debug.panic("{s} has no `_` leaf", .{@typeName(A)}), - .basis => |b| if (@hasDecl(A, "getBasis")) blk: { - const modes = self.arena.allocator().alloc(i32, b.mode_count) catch @panic("OOM"); - for (modes, b.modes[0..b.mode_count]) |*mode, m| mode.* = m; - break :blk A.getBasis(self.ctx, b.numerator, b.denominator, modes); - } else std.debug.panic("{s} has no scaled-basis leaf", .{@typeName(A)}), - .tuple => |elements| blk: { - const attrs = self.arena.allocator().alloc(*const A, elements.len) catch @panic("OOM"); - for (elements, attrs) |element, *attr| attr.* = self.algebraAttr(A, element); - break :blk A.getTuple(self.ctx, attrs); - }, - }; - return result catch std.debug.panic("invalid CuTe algebra for {s}", .{@typeName(A)}); - } - - fn layoutAttr(self: *Builder, spec: LayoutSpec) *const cute_ir.LayoutAttr { - return cute_ir.LayoutAttr.get(self.ctx, .{ - .shape = self.algebraAttr(cute_ir.ShapeAttr, spec.shape), - .stride = self.algebraAttr(cute_ir.StrideAttr, spec.stride), - }) catch @panic("invalid CuTe layout"); - } - - fn intTupleType(self: *Builder, e: AlgebraExpr) *const mlir.Type { - return (cute_ir.IntTupleType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.IntTupleAttr, e).attribute() }) catch @panic("invalid CuTe integer tuple")).type_(); - } - - /// `numerator/denominator` along `modes`, the mode path outermost first like - /// Python's `ScaledBasis(value, modes)`. CuTe prints it innermost first: - /// `basis(1, 2, .{ 0, 1 })` is `1/2@1@0`. - pub fn basis(self: *Builder, numerator: i64, denominator: i64, modes: anytype) ScaledBasis { - _ = self; - if (denominator <= 0) @panic("CuTe scaled-basis denominator must be positive"); - const fields = @typeInfo(@TypeOf(modes)).@"struct".field_names; - if (fields.len == 0 or fields.len > MAX_RANK) @panic("CuTe scaled basis needs 1..MAX_RANK modes"); - var result: ScaledBasis = .{ .numerator = numerator, .denominator = denominator, .mode_count = fields.len }; - inline for (fields, 0..) |field, i| result.modes[i] = @intCast(@field(modes, field)); - return result; - } - - pub fn layoutSpec(self: *Builder, shape: anytype, stride: anytype) LayoutSpec { - return .{ .shape = self.expr(shape), .stride = self.expr(stride) }; - } - - pub fn layoutType(self: *Builder, spec: LayoutSpec) *const mlir.Type { - return (cute_ir.LayoutType.get(self.ctx, .{ .attr = self.layoutAttr(spec).attribute() }) catch @panic("invalid CuTe layout")).type_(); - } - - pub fn composedLayoutType(self: *Builder, swizzle: Swizzle, offset: anytype, outer: LayoutSpec) *const mlir.Type { - const attr = cute_ir.ComposedLayoutAttr.get(self.ctx, .{ - .inner = .{ .swizzle = self.swizzleAttr(swizzle) catch @panic("invalid CuTe swizzle") }, - .offset = self.algebraAttr(cute_ir.IntTupleAttr, self.expr(offset)), - .outer = self.layoutAttr(outer), - }) catch @panic("invalid CuTe composed layout"); - return (cute_ir.ComposedLayoutType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("invalid CuTe composed layout")).type_(); - } - - pub fn memrefType(self: *Builder, dtype: DType, space: MemorySpace, alignment: u64, layout_type: *const mlir.Type) *const mlir.Type { - const pointer = self.ptrTy(dtype, space, alignment) catch @panic("invalid CuTe pointer type"); - return (cute_ir.MemRefType.get(self.ctx, .{ .ptr = pointer, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); - } - - pub fn smemDescType(self: *Builder) *const mlir.Type { - return (cute_nvgpu.SmemDescType.get(self.ctx, .{}) catch @panic("invalid shared-memory descriptor")).type_(); - } - - /// `!cute_nvgpu.smem_desc_view` of a shared-memory descriptor over `layout`. - pub fn smemDescViewType(self: *Builder, layout: LayoutSpec) *const mlir.Type { - return (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ - .desc = self.smemDescType(), - .layout = self.layoutAttr(layout).attribute(), - }) catch @panic("invalid shared-memory descriptor view")).type_(); - } - - /// `!cute.tiled_copy` of `copy_atom` over `layout_copy_tv`, tiled by the - /// modes of `tiler_mn`. - pub fn tiledCopyType(self: *Builder, copy_atom: *const mlir.Type, layout_copy_tv: LayoutSpec, tiler_mn: []const LayoutSpec) *const mlir.Type { - return (cute_ir.TiledCopyType.get(self.ctx, .{ - .copyAtom = copy_atom, - .layoutCopyTv = self.layoutAttr(layout_copy_tv).attribute(), - .tilerMn = self.tileAttr(tiler_mn).attribute(), - }) catch @panic("invalid tiled copy")).type_(); - } - - pub fn memrefTypeFromPointer(self: *Builder, pointer_type: *const mlir.Type, layout_type: *const mlir.Type) *const mlir.Type { - return (cute_ir.MemRefType.get(self.ctx, .{ .ptr = pointer_type, .layout = layout_type }) catch @panic("invalid CuTe memref type")).type_(); - } - - /// `cute.static` of an integer tuple such as `128` or `.{ 0, 4 }`. - pub fn staticIntTuple(self: *Builder, value: anytype) View { - const result = self.emit(cute_ir.static(self.ctx, self.intTupleType(self.expr(value)), self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn staticLayout(self: *Builder, spec: LayoutSpec) View { - const result = self.emit(cute_ir.static(self.ctx, self.layoutType(spec), self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub const Swizzle = struct { bits: i64, base: i64, shift: i64 }; - - pub fn staticSwizzle(self: *Builder, spec: Swizzle) View { - const attr = self.swizzleAttr(spec) catch @panic("invalid CuTe swizzle"); - const ty = (cute_ir.SwizzleType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("invalid CuTe swizzle")).type_(); - const result = self.emit(cute_ir.static(self.ctx, ty, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Construct `swizzle o offset o outer_layout` through the CuTe dialect. - pub fn makeComposedLayout(self: *Builder, swizzle: Swizzle, offset: anytype, outer: LayoutSpec) View { - const inner = self.staticSwizzle(swizzle); - const offset_value = self.emit(cute_ir.make_int_tuple(self.ctx, &.{}, self.intTupleType(self.expr(offset)), self.loc())); - const outer_value = self.staticLayout(outer); - const result_ty = self.composedLayoutType(swizzle, offset, outer); - const result = self.emit(cute_ir.make_composed_layout(self.ctx, inner.inner, offset_value.inner, outer_value.inner, result_ty, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Compose a raw pointer and any CuTe layout value. The memref result type - /// is obtained through the dialect C API rather than parsed from assembly. - pub fn makeTensorView(self: *Builder, pointer: anytype, layout: anytype) View { - const p = self.asValue(pointer); - const l = self.asValue(layout); - const result_ty = (cute_ir.MemRefType.get(self.ctx, .{ .ptr = p.type_(), .layout = l.type_() }) catch @panic("invalid CuTe tensor view")).type_(); - const result = self.emit(cute_ir.make_view(self.ctx, p.inner, result_ty, .{ .layout = l.inner }, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn coordTensorType(self: *Builder, rank: usize, layout: LayoutSpec) *const mlir.Type { - if (rank == 0 or rank > MAX_RANK) @panic("coordTensorType rank must be in 1..MAX_RANK"); - const zeros: [MAX_RANK]i64 = @splat(0); - return self.coordTensorTypeWithTuple(self.dimsExpr(zeros[0..rank]), layout); - } - - /// Coordinate-tensor type with an explicit arithmetic-tuple expression, - /// such as `.{ 0, ConstrainedDynamic{ .divisible_by = 128 }, AlgebraToken.dynamic }`. - /// This represents the divisibility information produced by CuTe tiling - /// without requiring kernel code to spell textual MLIR types. - pub fn coordTensorTypeWithTuple(self: *Builder, tuple: anytype, layout: LayoutSpec) *const mlir.Type { - return (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = self.intTupleType(self.expr(tuple)), .layout = self.layoutType(layout) }) catch @panic("invalid coordinate tensor")).type_(); - } - - fn rowMajor(self: *Builder, shape: []const i64) []const i64 { - const s = self.arena.allocator().alloc(i64, shape.len) catch @panic("OOM"); - var acc: i64 = 1; - var i = shape.len; - while (i > 0) : (i -= 1) { - s[i - 1] = acc; - acc *= shape[i - 1]; - } - return s; - } - - fn leftMost(self: *Builder, shape: []const i64) []const i64 { - const s = self.arena.allocator().alloc(i64, shape.len) catch @panic("OOM"); - var acc: i64 = 1; - for (shape, 0..) |d, i| { - s[i] = acc; - acc *= d; - } - return s; - } - - // ==================== cute.core ==================== - - fn layoutResult(comptime Shape: type) type { - return switch (@typeInfo(Shape)) { - .pointer => Layout, - else => View, - }; - } - - /// `cute.make_layout(shape, stride=...)`. Slices keep the tracked static - /// Layout API; nested tuples accept mixed integer and SSA shape/stride leaves. - pub fn makeLayout(self: *Builder, shape: anytype, opts: anytype) layoutResult(@TypeOf(shape)) { - if (comptime layoutResult(@TypeOf(shape)) == Layout) return self.makeStaticLayout(shape, .{ - .stride = if (@hasField(@TypeOf(opts), "stride")) opts.stride else null, - }); - var shape_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; - var stride_values: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; - const shape_expr = self.coordExpr(&shape_values, shape); - const stride_expr = if (@hasField(@TypeOf(opts), "stride") and @TypeOf(opts.stride) != @TypeOf(null)) self.coordExpr(&stride_values, opts.stride) else blk: { - var stride: AlgebraExpr = .{ .static = 1 }; - var stride_value: ?Value = null; - var index: usize = 0; - break :blk self.compactLayoutStride(shape_expr, shape_values.constSlice(), &index, &stride, &stride_value, &stride_values); - }; - const shape_value = self.emit(cute_ir.make_shape(self.ctx, shape_values.constSlice(), self.shapeType(shape_expr), self.loc())); - const stride_type = (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, stride_expr).attribute() }) catch @panic("invalid stride")).type_(); - const stride_value = self.emit(cute_ir.make_stride(self.ctx, stride_values.constSlice(), stride_type, self.loc())); - const result = self.emit(cute_ir.make_layout(self.ctx, self.layoutType(.{ .shape = shape_expr, .stride = stride_expr }), .{ .shape = shape_value.inner, .stride = stride_value.inner }, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - fn compactLayoutStride(self: *Builder, shape: AlgebraExpr, values: []const *const mlir.Value, index: *usize, stride: *AlgebraExpr, stride_value: *?Value, dynamic: *stdx.BoundedArray(*const mlir.Value, MAX_RANK)) AlgebraExpr { - if (shape == .tuple) { - const elements = self.arena.allocator().alloc(AlgebraExpr, shape.tuple.len) catch @panic("OOM"); - for (shape.tuple, elements) |extent, *element| element.* = self.compactLayoutStride(extent, values, index, stride, stride_value, dynamic); - return .{ .tuple = elements }; - } - if (shape == .static and shape.static == 1) return .{ .static = 0 }; - const result = stride.*; - if (stride_value.*) |v| dynamic.append(v.inner) catch @panic("too many stride leaves"); - if (shape == .static and stride.* == .static) { - stride.static *= shape.static; - } else { - const extent = if (shape == .dynamic or shape == .dynamic_int) blk: { - const v: Value = .{ .inner = values[index.*], .kernel = self }; - index.* += 1; - break :blk v; - } else self.lift(shape.static); - stride_value.* = (stride_value.* orelse self.lift(stride.static)).mul(extent); - stride.* = dynamicExpr(stride_value.*.?); - } - return result; - } - - fn makeStaticLayout(self: *Builder, shape: []const i64, opts: LayoutOpts) Layout { - if (shape.len == 0 or shape.len > MAX_RANK) std.debug.panic("makeLayout: rank {d} not in 1..{d}", .{ shape.len, MAX_RANK }); - const stride = opts.stride orelse self.leftMost(shape); - if (stride.len != shape.len) std.debug.panic("makeLayout: stride rank {d} != shape rank {d}", .{ stride.len, shape.len }); - const layout_ty = self.layoutType(.{ .shape = self.dimsExpr(shape), .stride = self.dimsExpr(stride) }); - const l = self.emit(cute_ir.static(self.ctx, layout_ty, self.loc())); - return .{ .inner = l.inner, .shape = Dims.fromSlice(shape) catch unreachable, .stride = Dims.fromSlice(stride) catch unreachable }; - } - - /// `cute.make_tensor(ptr, layout)`. - pub fn makeTensor(self: *Builder, ptr: Value, layout: Layout) Tensor { - const pt = ptr.type_().isA(cute_ir.PtrType) orelse std.debug.panic("makeTensor: {f} is not a !cute.ptr", .{ptr.type_()}); - const elem = pt.valueType() orelse @panic("makeTensor: untyped pointer"); - const memref_ty = (cute_ir.MemRefType.get(self.ctx, .{ .ptr = ptr.type_(), .layout = layout.inner.type_() }) catch @panic("bad memref")).type_(); - const v = self.emit(cute_ir.make_view(self.ctx, ptr.inner, memref_ty, .{ .layout = layout.inner }, self.loc())); - return .{ .inner = v.inner, .kernel = self, .layout = layout, .dtype = dtypes.mlirToDType(self.ctx, elem), .space = pt.addressSpace() }; - } - - /// The algebra of a coordinate whose `Value` leaves are dynamic and - /// appended to `dyn`. A one-element tuple is its element. - fn coordExpr(self: *Builder, dyn: *stdx.BoundedArray(*const mlir.Value, MAX_RANK), coord: anytype) AlgebraExpr { - const T = @TypeOf(coord); - if (T == Value) { - dyn.append(coord.inner) catch @panic("makeCoord: too many dynamic leaves"); - return dynamicExpr(coord); - } - if (T == AlgebraToken) return switch (coord) { - .all => .all, - .dynamic => @panic("makeCoord: AlgebraToken.dynamic needs an SSA value"), - }; - switch (@typeInfo(T)) { - .comptime_int, .int => return .{ .static = @intCast(coord) }, - .@"struct" => |info| { - if (info.field_names.len == 0) @compileError("makeCoord: empty tuple"); - if (info.field_names.len == 1) return self.coordExpr(dyn, @field(coord, info.field_names[0])); - const elements = self.arena.allocator().alloc(AlgebraExpr, info.field_names.len) catch @panic("OOM"); - inline for (info.field_names, 0..) |field, i| elements[i] = self.coordExpr(dyn, @field(coord, field)); - return .{ .tuple = elements }; - }, - else => @compileError("makeCoord: unsupported coordinate leaf " ++ @typeName(T)), - } - } - - fn dynamicExpr(value: Value) AlgebraExpr { - std.debug.assert(value.isInt()); - const width = dtypeBitwidth(value.dtype()); - return if (width == 32) .{ .dynamic = 1 } else .{ .dynamic_int = .{ .width = @intCast(width) } }; - } - - /// A `!cute.coord` from nested tuples of `Value`s, integers, and `.all` - /// placeholders. Integer leaves stay static in the dependent CuTe type. - pub fn makeCoord(self: *Builder, coord: anytype) *const mlir.Value { - if (@TypeOf(coord) == Value) return self.makeCoord(.{coord}); - var dyn: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; - const attr = self.algebraAttr(cute_ir.CoordAttr, self.coordExpr(&dyn, coord)); - const ty = (cute_ir.CoordType.get(self.ctx, .{ .attr = attr.attribute() }) catch @panic("bad coord")).type_(); - return self.emit(cute_ir.make_coord(self.ctx, dyn.constSlice(), ty, self.loc())).inner; - } - - /// A dynamic/static CuTe integer tuple, used for pointer arithmetic in - /// tensor and shared memory. It accepts the same nested Zig values as a - /// coordinate, but preserves integer-tuple semantics. - pub fn makeIntTuple(self: *Builder, values: anytype) View { - var dyn: stdx.BoundedArray(*const mlir.Value, MAX_RANK) = .empty; - const ty = self.intTupleType(self.coordExpr(&dyn, values)); - const value = self.emit(cute_ir.make_int_tuple(self.ctx, dyn.constSlice(), ty, self.loc())); - return .{ .inner = value.inner, .kernel = self }; - } - - /// A one-element integer tuple holding a value known to be a multiple of - /// `divisor`. Ops that derive alignment from an offset, such as - /// `cute.add_offset`, need that to keep the operand's alignment. - pub fn makeIntTupleDivBy(self: *Builder, value: anytype, comptime divisor: u32) View { - const constrained = self.assumeDivBy(value, divisor); - const tuple = self.emit(cute_ir.make_int_tuple(self.ctx, &.{constrained.inner}, self.intTupleType(.{ .dynamic = divisor }), self.loc())); - return .{ .inner = tuple.inner, .kernel = self }; - } - - /// Refine an integer SSA value with the divisibility information CuTe - /// derives while slicing a tiled coordinate tensor. - pub fn assumeDivBy(self: *Builder, value: anytype, comptime divisor: u32) Value { - if (divisor == 0) @compileError("assumeDivBy divisor must be non-zero"); - const constrained = cute_ir.constrainedIntType(self.ctx, .i32, divisor) catch @panic("invalid CuTe constrained integer"); - return self.emit(cute_ir.assume(self.ctx, self.lift(value).to(.i32).inner, constrained, self.loc())); - } - - /// Build a coordinate tensor from a runtime origin and a static layout. - /// This is the direct form of the `get_iter`/`make_arith_tuple_iter`/ - /// `make_view` sequence emitted by the Python CuTe front end after a tile - /// is selected from a larger coordinate tensor. - pub fn makeCoordTensor(self: *Builder, origin: anytype, layout: anytype) View { - const tuple = self.makeIntTuple(origin); - const layout_value = self.asValue(layout); - const iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = tuple.type_() }) catch @panic("invalid arithmetic tuple iterator")).type_(); - const iterator = self.emit(cute_ir.make_arith_tuple_iter(self.ctx, iterator_type, .{ .value_ = tuple.inner }, self.loc())); - const result_type = (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = tuple.type_(), .layout = layout_value.type_() }) catch @panic("invalid coordinate tensor")).type_(); - const result = self.emit(cute_ir.make_view(self.ctx, iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Rebuild a coordinate tensor from the arithmetic-tuple iterator of an - /// existing tensor. CuTe's Python front end emits this sequence after it - /// slices a TMA-partitioned coordinate tensor: - /// - /// get_iter -> deref_arith_tuple_iter -> get_leaves -> make_int_tuple - /// -> make_arith_tuple_iter -> make_view - /// - /// Keeping that sequence explicit matters for TMA. The iterator carries - /// the descriptor coordinates selected by `slice`; constructing a fresh - /// tuple from the scheduler indices would lose CuTe's coordinate algebra. - pub fn rebuildCoordTensorIterator( - self: *Builder, - comptime leaf_count: usize, - input: anytype, - tuple: anytype, - comptime leaf_divisibility: [leaf_count]u32, - layout: anytype, - ) View { - if (leaf_count == 0 or leaf_count > MAX_RANK) @panic("invalid coordinate tuple leaf count"); - const source = self.asValue(input); - const source_type = source.type_().isA(cute_ir.CoordTensorType) orelse - std.debug.panic("rebuildCoordTensorIterator: expected coordinate tensor, got {f}", .{source.type_()}); - const source_tuple_type = source_type.arithTuple(); - const source_iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = source_tuple_type }) catch - @panic("invalid source arithmetic tuple iterator")).type_(); - const source_iterator = self.emit(cute_ir.get_iter(self.ctx, source.inner, source_iterator_type, self.loc())); - const source_tuple = self.emit(cute_ir.deref_arith_tuple_iter( - self.ctx, - source_iterator.inner, - source_tuple_type, - self.loc(), - )); - - const leaf_types = self.arena.allocator().alloc(*const mlir.Type, leaf_count) catch @panic("OOM"); - // 0 marks a leaf the producer knows to be statically zero, 1 a leaf - // with no known divisibility. - inline for (leaf_divisibility, 0..) |divisor, i| { - leaf_types[i] = self.intTupleType(if (divisor == 0) .{ .static = 0 } else .{ .dynamic = divisor }); - } - const leaves = self.emitResults(leaf_count, cute_ir.get_leaves( - self.ctx, - source_tuple.inner, - leaf_types, - self.loc(), - )); - - const rebuilt_tuple_type = self.intTupleType(self.expr(tuple)); - const rebuilt_tuple = self.emit(cute_ir.make_int_tuple( - self.ctx, - self.innerSlice(leaves[0..]), - rebuilt_tuple_type, - self.loc(), - )); - const rebuilt_iterator_type = (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = rebuilt_tuple_type }) catch - @panic("invalid rebuilt arithmetic tuple iterator")).type_(); - const rebuilt_iterator = self.emit(cute_ir.make_arith_tuple_iter(self.ctx, rebuilt_iterator_type, .{ .value_ = rebuilt_tuple.inner }, self.loc())); - const layout_value = self.asValue(layout); - const result_type = (cute_ir.CoordTensorType.get(self.ctx, .{ - .arithTuple = rebuilt_tuple_type, - .layout = layout_value.type_(), - }) catch @panic("invalid rebuilt coordinate tensor")).type_(); - const result = self.emit(cute_ir.make_view(self.ctx, rebuilt_iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - /// `!cute.shape` of a CuTe algebra value such as `.{ .{ 128, 16 }, 1, 1, 2 }` - /// or `.{ AlgebraToken.dynamic, AlgebraToken.dynamic }`. - pub fn shapeType(self: *Builder, shape: anytype) *const mlir.Type { - return (cute_ir.ShapeType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.ShapeAttr, self.expr(shape)).attribute() }) catch @panic("invalid CuTe shape")).type_(); - } - - /// Static `cute.make_shape` of a CuTe algebra value. - pub fn makeShape(self: *Builder, shape: anytype) View { - const value = self.emit(cute_ir.make_shape(self.ctx, &.{}, self.shapeType(shape), self.loc())); - return .{ .inner = value.inner, .kernel = self }; - } - - /// Dynamic `cute.make_shape`. The result type describes which leaves are - /// runtime values (for example `!cute.shape<"(?,?,?)">`). Keeping these - /// leaves dynamic is significant to CuTe's TMA layout algebra when a - /// runtime extent happens to be one. - pub fn makeShapeValues(self: *Builder, values: anytype, result_type: *const mlir.Type) View { - const value = self.emit(cute_ir.make_shape(self.ctx, self.innerSlice(values), result_type, self.loc())); - return .{ .inner = value.inner, .kernel = self }; - } - - pub fn tileToShape(self: *Builder, input: anytype, shape: anytype, order: View, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.tile_to_shape( - self.ctx, - self.asValue(input).inner, - self.asValue(shape).inner, - order.inner, - result_type, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Static `cute.make_tile` from mode extents. - pub fn makeTile(self: *Builder, shape: []const i64) View { - const modes = self.arena.allocator().alloc(LayoutSpec, shape.len) catch @panic("OOM"); - for (shape, modes) |extent, *mode| mode.* = .{ .shape = .{ .static = extent }, .stride = .{ .static = 1 } }; - const value = self.emit(cute_ir.make_tile(self.ctx, &.{}, self.tileType(modes), self.loc())); - return .{ .inner = value.inner, .kernel = self }; - } - - /// `!cute.tile` whose modes are `layouts`, such as `[512:1;1:0;4:1]`. - pub fn tileType(self: *Builder, layouts: []const LayoutSpec) *const mlir.Type { - return (cute_ir.TileType.get(self.ctx, .{ .attr = self.tileAttr(layouts).attribute() }) catch @panic("invalid CuTe tile")).type_(); - } - - fn tileAttr(self: *Builder, layouts: []const LayoutSpec) *const cute_ir.TileAttr { - const modes = self.arena.allocator().alloc(*const cute_ir.TileAttr, layouts.len) catch @panic("OOM"); - for (layouts, modes) |layout, *mode| mode.* = cute_ir.TileAttr.getLayout(self.ctx, self.layoutAttr(layout)) catch @panic("invalid CuTe tile mode"); - return cute_ir.TileAttr.getTuple(self.ctx, modes) catch @panic("invalid CuTe tile"); - } - - /// Construct a target atom whose complete architecture payload is - /// encoded in `atom_type` (for example `!cute_nvgpu.sm100.mma_bs<...>`). - pub fn makeAtom(self: *Builder, atom_type: *const mlir.Type, values: anytype) Atom { - const result = self.emit(cute_ir.make_atom(self.ctx, self.innerSlice(values), atom_type, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn makeTiledMma(self: *Builder, atom: Atom, result_type: *const mlir.Type) Atom { - const result = self.emit(cute_ir.make_tiled_mma(self.ctx, atom.inner, result_type, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Allocate shared memory with a type assembled by the Zig DSL. - pub fn allocSmemView(self: *Builder, dtype: DType, alignment: u64, layout_type: *const mlir.Type) View { - const memref_type = self.memrefType(dtype, .smem, alignment, layout_type); - const result = self.emit(cute_ir.memref_alloc_smem(self.ctx, memref_type, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Allocate one compiler-visible shared-memory storage object. The field - /// descriptions use the CuTe host compiler's `name:size:offset` format; - /// they let its shared-memory sizing/layout pass reproduce a Python - /// `SharedStorage` struct while Zig keeps typed pointers to its fields. - pub fn allocSmemStorage( - self: *Builder, - dtype: DType, - alignment: u64, - layout_type: *const mlir.Type, - partition_id: i32, - fields: []const []const u8, - ) View { - const field_attrs = self.arena.allocator().alloc(*const mlir.Attribute, fields.len) catch @panic("OOM"); - for (fields, field_attrs) |field, *attr| attr.* = .string(self.ctx, field); - const memref_type = self.memrefType(dtype, .smem, alignment, layout_type); - const result = self.emit(mlir.Operation.make(self.ctx, "cute.memref.alloca", .{ - .results = .{ .flat = &.{memref_type} }, - .attributes = &.{ - .named(self.ctx, "smem.partition_id", .int(self.ctx, .i32, partition_id)), - .named(self.ctx, "smem.struct_fields", .array(self.ctx, field_attrs)), - }, - .location = self.loc(), - })); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Allocate a register-memory tensor whose layout is represented by a - /// normal CuTe layout value. - pub fn allocRmem(self: *Builder, dtype: DType, alignment: u64, layout: anytype) View { - const layout_value = self.asValue(layout); - const memref_type = self.memrefType(dtype, .rmem, alignment, layout_value.type_()); - const result = self.emit(cute_ir.memref_alloca(self.ctx, memref_type, .{ .layout = layout_value.inner }, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn makeView(self: *Builder, pointer: anytype, layout: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.make_view(self.ctx, self.asValue(pointer).inner, result_type, .{ .layout = self.asValue(layout).inner }, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn makeViewNoLayout(self: *Builder, pointer: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.make_view(self.ctx, self.asValue(pointer).inner, result_type, .{}, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - /// `projection`, like Python's `proj`, selects the modes of `tile` that - /// apply, such as `b.expr(.{ 1, AlgebraToken.all, 1 })`. - pub fn localTile(self: *Builder, input: anytype, tile: anytype, coord: anytype, result_type: ?*const mlir.Type, projection: ?AlgebraExpr) View { - if (result_type == null) return self.inferView("cute.local_tile", &.{ self.asValue(input).inner, self.asValue(tile).inner, if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord) }, if (projection) |p| &.{.named(self.ctx, "proj", self.algebraAttr(cute_ir.CoordAttr, p).attribute())} else &.{}); - const coord_value = if (@TypeOf(coord) == View) coord.inner else self.makeCoord(coord); - const result = self.emit(cute_ir.local_tile( - self.ctx, - self.asValue(input).inner, - self.asValue(tile).inner, - coord_value, - result_type.?, - .{ .proj = if (projection) |p| self.algebraAttr(cute_ir.CoordAttr, p).attribute() else null }, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - // A null result type on the algebra operations below requests dialect inference. - fn inferView(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attrs: []const mlir.NamedAttribute) View { - const value = self.emit(mlir.Operation.make(self.ctx, name, .{ .operands = .{ .flat = operands }, .attributes = attrs, .result_type_inference = true, .location = self.loc() })); - return .{ .inner = value.inner, .kernel = self }; - } - - pub fn localPartition(self: *Builder, input: anytype, tiler: anytype, index: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.local_partition( - self.ctx, - self.asValue(input).inner, - self.asValue(tiler).inner, - self.makeCoord(index), - result_type, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn slice(self: *Builder, input: anytype, coord: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.slice", &.{ self.asValue(input).inner, self.makeCoord(coord) }, &.{}); - const result = self.emit(cute_ir.slice( - self.ctx, - self.asValue(input).inner, - self.makeCoord(coord), - result_type.?, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Rebuild a tensor with the same iterator and a new layout. This is the - /// tensor form of CuTe's `group_modes`: grouping changes only the layout, - /// while the backing pointer or arithmetic-tuple iterator stays intact. - /// The local CUTLASS schema exposes `cute.group_modes` only for layouts, - /// so expressing the equivalent algebra this way also remains portable. - pub fn withLayout(self: *Builder, input: anytype, layout: anytype) View { - const value = self.asValue(input); - const layout_value = self.asValue(layout); - const input_type = value.type_(); - const result_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| - (cute_ir.MemRefType.get(self.ctx, .{ .ptr = memref.ptr_(), .layout = layout_value.type_() }) catch @panic("invalid regrouped memref")).type_() - else if (input_type.isA(cute_ir.CoordTensorType)) |coords| - (cute_ir.CoordTensorType.get(self.ctx, .{ .arithTuple = coords.arithTuple(), .layout = layout_value.type_() }) catch @panic("invalid regrouped coordinate tensor")).type_() - else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| - (cute_nvgpu.SmemDescViewType.get(self.ctx, .{ .desc = desc.desc(), .layout = layout_value.type_().isA(cute_ir.LayoutType).?.attr() }) catch @panic("invalid regrouped descriptor")).type_() - else - std.debug.panic("withLayout: expected memref, coordinate tensor or shared-memory descriptor view, got {f}", .{input_type}); - - const iterator_type: *const mlir.Type = if (input_type.isA(cute_ir.MemRefType)) |memref| - memref.ptr_() - else if (input_type.isA(cute_ir.CoordTensorType)) |coords| - (cute_ir.ArithTupleIteratorType.get(self.ctx, .{ .arithTuple = coords.arithTuple() }) catch @panic("invalid arithmetic tuple iterator")).type_() - else if (input_type.isA(cute_nvgpu.SmemDescViewType)) |desc| - desc.desc() - else - unreachable; - const iterator = self.emit(cute_ir.get_iter(self.ctx, value.inner, iterator_type, self.loc())); - const result = self.emit(cute_ir.make_view(self.ctx, iterator.inner, result_type, .{ .layout = layout_value.inner }, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn groupModes(self: *Builder, input: anytype, begin: u32, end: u32, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.group_modes", &.{self.asValue(input).inner}, &.{ .named(self.ctx, "begin", .int(self.ctx, .i32, begin)), .named(self.ctx, "end", .int(self.ctx, .i32, end)) }); - const result = self.emit(cute_ir.group_modes( - self.ctx, - self.asValue(input).inner, - result_type.?, - .int(self.ctx, .i32, begin), - .int(self.ctx, .i32, end), - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn tiledDivide(self: *Builder, input: anytype, tiler: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.tiled_divide( - self.ctx, - self.asValue(input).inner, - self.asValue(tiler).inner, - result_type, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn flatDivide(self: *Builder, input: anytype, tiler: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.flat_divide", &.{ self.asValue(input).inner, self.asValue(tiler).inner }, &.{}); - const result = self.emit(cute_ir.flat_divide( - self.ctx, - self.asValue(input).inner, - self.asValue(tiler).inner, - result_type.?, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn coalesce(self: *Builder, input: anytype, target_profile: ?View, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.coalesce( - self.ctx, - self.asValue(input).inner, - result_type, - .{ .target_profile = if (target_profile) |target| target.inner else null }, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn filterZeros(self: *Builder, input: anytype, target_profile: ?View, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.filter_zeros", if (target_profile) |target| &.{ self.asValue(input).inner, target.inner } else &.{self.asValue(input).inner}, &.{}); - const result = self.emit(cute_ir.filter_zeros( - self.ctx, - self.asValue(input).inner, - result_type.?, - .{ .target_profile = if (target_profile) |target| target.inner else null }, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn getLayout(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.get_layout", &.{self.asValue(input).inner}, &.{}); - const result = self.emit(cute_ir.get_layout(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - /// `cute.get_iter`: the iterator of `input`, of type `result_type`. - pub fn getIter(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.get_iter", &.{self.asValue(input).inner}, &.{}); - const result = self.emit(cute_ir.get_iter(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn getShape(self: *Builder, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.get_shape", &.{self.asValue(input).inner}, &.{}); - const result = self.emit(cute_ir.get_shape(self.ctx, self.asValue(input).inner, result_type.?, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn addOffset(self: *Builder, input: anytype, offset: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.add_offset", &.{ self.asValue(input).inner, self.asValue(offset).inner }, &.{}); - const result = self.emit(cute_ir.add_offset( - self.ctx, - self.asValue(input).inner, - self.asValue(offset).inner, - result_type.?, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn recastIter(self: *Builder, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.recast_iter(self.ctx, self.asValue(input).inner, result_type, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn appendToRank(self: *Builder, input: anytype, element: anytype, rank: u32, result_type: *const mlir.Type) View { - const result = self.emit(cute_ir.append_to_rank( - self.ctx, - self.asValue(input).inner, - self.asValue(element).inner, - result_type, - .int(self.ctx, .i32, rank), - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn makeTiledCopy(self: *Builder, atom: Atom, result_type: *const mlir.Type) Atom { - const result = self.emit(cute_ir.make_tiled_copy(self.ctx, atom.inner, result_type, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn tiledCopyPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, destination: bool, result_type: *const mlir.Type) View { - const op = if (destination) - cute_ir.tiled_copy_partition_D(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), result_type, self.loc()) - else - cute_ir.tiled_copy_partition_S(self.ctx, tiled.inner, self.asValue(input).inner, self.makeCoord(coord), result_type, self.loc()); - const result = self.emit(op); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn tiledCopyRetile(self: *Builder, tiled: Atom, input: anytype, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.tiled.copy.retile", &.{ tiled.inner, self.asValue(input).inner }, &.{}); - const result = self.emit(cute_ir.tiled_copy_retile( - self.ctx, - tiled.inner, - self.asValue(input).inner, - result_type.?, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn tiledMmaPartition(self: *Builder, tiled: Atom, input: anytype, coord: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.tiled.mma.partition", &.{ tiled.inner, self.asValue(input).inner, self.makeCoord(coord) }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); - const result = self.emit(cute_ir.tiled_mma_partition( - self.ctx, - tiled.inner, - self.asValue(input).inner, - self.makeCoord(coord), - result_type.?, - .int(self.ctx, .i32, operand_id), - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn mmaMakeFragment(self: *Builder, tiled: Atom, input: anytype, operand_id: u2, result_type: ?*const mlir.Type) View { - if (result_type == null) return self.inferView("cute.mma.make_fragment", &.{ tiled.inner, self.asValue(input).inner }, &.{.named(self.ctx, "operand_id", .int(self.ctx, .i32, operand_id))}); - const result = self.emit(cute_ir.mma_make_fragment( - self.ctx, - tiled.inner, - self.asValue(input).inner, - result_type.?, - .int(self.ctx, .i32, operand_id), - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub const CopyOptions = struct { pred: ?Value = null, tma_bar_ptr: ?Value = null, cache_policy: ?Value = null }; - - pub fn copy(self: *Builder, atom: Atom, src: anytype, dst: anytype, opts: CopyOptions) void { - var exec = atom; - if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaStoreType)) |ty| { - const exec_type = (cute_nvgpu.CopyAtomTmaStoreType.get(self.ctx, .{ - .valType = ty.valType(), - .copyBits = ty.copyBits(), - .mode = .tiled, - .gStride = self.emptyStrideType(), - .tmaGbasis = ty.tmaGbasis(), - }) catch @panic("invalid executable TMA store atom")).type_(); - exec = self.makeExecTma(atom, exec_type); - } - if (opts.tma_bar_ptr) |mbar| { - if (atom.type_().isA(cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType)) |ty| { - const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ - .valType = ty.valType(), - .copyBits = ty.copyBits(), - .mode = .gather4, - .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, - .gStride = self.emptyStrideType(), - .tmaGbasis = ty.tmaGbasis(), - }) catch @panic("invalid executable gather atom")).type_(); - exec = self.makeExecTma(atom, exec_type); - } else if (atom.type_().isA(cute_nvgpu.CopyAtomNonExecTiledTmaLoadType)) |ty| { - const exec_type = (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ - .valType = ty.valType(), - .copyBits = ty.copyBits(), - .mode = .tiled, - .numCta = if (ty.kind() == .sm_100_2sm) 2 else 1, - .gStride = self.emptyStrideType(), - .tmaGbasis = ty.tmaGbasis(), - }) catch @panic("invalid executable TMA atom")).type_(); - exec = self.makeExecTma(atom, exec_type); - } - exec = self.setTmaBarrier(exec, mbar); - } - if (opts.cache_policy) |policy| { - const field = if (exec.type_().isA(cute_nvgpu.CopyAtomTmaLoadType) != null) - (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() - else if (exec.type_().isA(cute_nvgpu.CopyAtomTmaStoreType) != null) - (cute_nvgpu.AtomCopyFieldTmaStoreAttr.get(self.ctx, .{ .value = .cache_policy }) catch unreachable).attribute() - else - @panic("cache_policy requires an executable TMA copy atom"); - const updated = self.emit(cute_nvgpu.atom_set_value(self.ctx, exec.inner, policy.to(.i64).inner, exec.type_(), field, self.loc())); - exec = .{ .inner = updated.inner, .kernel = self }; - } - self.emitVoid(cute_ir.copy(self.ctx, exec.inner, self.innerSlice(src), self.innerSlice(dst), .{ .pred = if (opts.pred) |p| p.inner else null }, self.loc())); - } - - pub fn gemm(self: *Builder, tiled: Atom, d: anytype, a: anytype, rhs: anytype, c: anytype) void { - self.emitVoid(cute_ir.gemm( - self.ctx, - tiled.inner, - self.asValue(d).inner, - self.innerSlice(a), - self.innerSlice(rhs), - self.asValue(c).inner, - self.loc(), - )); - } - - pub fn vectorExtract(self: *Builder, source: Value, lane: i64, dtype: DType) Value { - return self.emit(vector.extract( - self.ctx, - source.inner, - &.{lane}, - &.{}, - dtype.toMlir(self.ctx), - self.loc(), - )); - } - - // ==================== cute_nvgpu.cpasync: TMA (SM90+) ==================== - // The host function builds tiled TMA atoms (Python `make_tiled_tma_atom`); - // the kernel receives them as grid-constant arguments and makes them - // executable before issuing copies. - - pub const TmaAtomOptions = struct { gmem_coord_tensor: ?View = null }; - - fn inferValue(self: *Builder, name: []const u8, operands: []const *const mlir.Value, attributes: []const mlir.NamedAttribute) Value { - return self.emit(mlir.Operation.make(self.ctx, name, .{ - .operands = .{ .flat = operands }, - .attributes = attributes, - .result_type_inference = true, - .location = self.loc(), - })); - } - - fn valueShape(self: *Builder, value: Value) Value { - if (value.type_().isA(cute_ir.ShapeType) != null) return value; - return self.inferValue("cute.get_shape", &.{value.inner}, &.{}); - } - - fn shapeRank(shape: Value) usize { - return shape.type_().isA(cute_ir.ShapeType).?.attr().isA(cute_ir.ShapeAttr).?.rank(); - } - - /// Common CuTe TMA atom constructor. Gather uses the index tensor's layout - /// at construction and passes data/index tensors together to partition/copy. - pub fn makeTiledTmaAtom(self: *Builder, op: anytype, gmem: anytype, smem_layout: anytype, cta_tiler: anytype, config: TmaConfig, opts: TmaAtomOptions) TmaDescriptor { - const Tiler = @TypeOf(cta_tiler); - const tiler = if (Tiler == Value or Tiler == View or Tiler == Layout) - self.asValue(cta_tiler) - else if (@typeInfo(Tiler) == .pointer) - self.makeShape(self.dimsExpr(cta_tiler)).value() - else - self.makeShape(cta_tiler).value(); - var smem = self.asValue(smem_layout); - const tiler_rank = shapeRank(self.valueShape(tiler)); - const smem_rank = shapeRank(self.valueShape(smem)); - std.debug.assert(smem_rank == tiler_rank or smem_rank == tiler_rank + 1); - if (smem_rank == tiler_rank + 1) { - const modes = self.arena.allocator().alloc(i32, tiler_rank) catch @panic("OOM"); - for (modes, 0..) |*mode, i| mode.* = @intCast(i); - smem = self.inferValue("cute.select", &.{smem.inner}, &.{.named(self.ctx, "mode", .denseArray(self.ctx, .i32, modes))}); - } - const shape = self.valueShape(self.asValue(gmem)); - const identity = self.inferValue("cute.make_identity_layout", &.{shape.inner}, &.{}); - const cta_map = self.inferValue("cute.composition", &.{ identity.inner, tiler.inner }, &.{}); - var cfg = config; - cfg.mode = .tiled; - if (@TypeOf(op) == CopyBulkTensorTileS2GOp) return self.makeTiledTmaStoreAtom(gmem, smem, cta_map, cfg); - cfg.cta_group = op.cta_group; - if (@TypeOf(op) == CopyBulkTensorTileG2SOp) return self.makeTiledTmaLoadAtom(gmem, smem, cta_map, cfg); - if (@TypeOf(op) != CopyBulkTensor2DGather4G2SOp) @compileError("unsupported TMA copy operation"); - cfg.mode = .gather4; - const indices = opts.gmem_coord_tensor orelse @panic("gather4 requires gmem_coord_tensor"); - const index_type = indices.type_().isA(cute_ir.MemRefType) orelse @panic("gather4 indices must be a tensor"); - const layout = self.emit(cute_ir.get_layout(self.ctx, indices.inner, index_type.layout(), self.loc())); - const kind: cute_nvgpu.GatherScatterTmaLoad = if (cfg.cta_group == .two) .sm_100_2sm else .sm_100; - const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_2d_gather4_tma_load(self.ctx, self.asValue(gmem).inner, layout.inner, smem.inner, cta_map.inner, self.tmaLoadAtomType(cfg), self.coordTensorType(cfg.coordinate_rank, cfg.coordinate_layout), (cute_nvgpu.GatherScatterTmaLoadAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, cfg) }, self.loc())); - return .{ .atom = .{ .inner = results[0].inner, .kernel = self }, .tensor = .{ .inner = results[1].inner, .kernel = self } }; - } - - /// Build the host-visible TMA descriptor and its coordinate tensor, whose - /// types are assembled from `config`. - pub fn makeTiledTmaLoadAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { - std.debug.assert(config.mode == .tiled); - const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_load( - self.ctx, - self.asValue(gmem).inner, - self.asValue(smem_layout).inner, - self.asValue(cta_map).inner, - self.tmaLoadAtomType(config), - self.coordTensorType(config.coordinate_rank, config.coordinate_layout), - (cute_nvgpu.TiledTmaLoadAttr.get(self.ctx, .{ .value = tmaLoadKind(config) }) catch @panic("invalid tiled TMA load")).attribute(), - .{ .num_multicast = .int(self.ctx, .i32, 1), .tma_format = tmaFormatAttribute(self, config) }, - self.loc(), - )); - return .{ - .atom = .{ .inner = results[0].inner, .kernel = self }, - .tensor = .{ .inner = results[1].inner, .kernel = self }, - }; - } - - /// Host-visible tiled TMA store atom and its coordinate tensor. - pub fn makeTiledTmaStoreAtom(self: *Builder, gmem: anytype, smem_layout: anytype, cta_map: anytype, config: TmaConfig) TmaDescriptor { - const results = self.emitResults(2, cute_nvgpu.atom_make_non_exec_tiled_tma_store( - self.ctx, - self.asValue(gmem).inner, - self.asValue(smem_layout).inner, - self.asValue(cta_map).inner, - self.tmaStoreAtomType(config), - self.coordTensorType(config.coordinate_rank, config.coordinate_layout), - .{ .tma_format = tmaFormatAttribute(self, config) }, - self.loc(), - )); - return .{ - .atom = .{ .inner = results[0].inner, .kernel = self }, - .tensor = .{ .inner = results[1].inner, .kernel = self }, - }; - } - - /// Host-side tiled TMA load atom type described by `config`. - pub fn tmaLoadAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { - if (config.mode == .gather4) return (cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType.get(self.ctx, .{ - .kind = if (config.cta_group == .two) .sm_100_2sm else .sm_100, - .valType = config.dtype.toMlir(self.ctx), - .copyBits = @intCast(config.copy_bits), - .tmaGbasis = self.layoutType(config.global_basis), - .tmaFormat = if (config.format == .default) null else tmaDataFormat(config.format, config.dtype), - }) catch @panic("invalid gather TMA atom")).type_(); - return (cute_nvgpu.CopyAtomNonExecTiledTmaLoadType.get(self.ctx, .{ - .kind = tmaLoadKind(config), - .valType = config.dtype.toMlir(self.ctx), - .copyBits = @intCast(config.copy_bits), - .tmaGbasis = self.layoutType(config.global_basis), - .tmaFormat = tmaDataFormat(config.format, config.dtype), - }) catch @panic("invalid TMA load atom")).type_(); - } - - /// Executable device-side form of a tiled TMA load atom. `make_exec_tma` - /// preserves the tensor-map payload and adds the launch mode used by - /// `copy`. - pub fn tmaLoadExecAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { - return (cute_nvgpu.CopyAtomTmaLoadType.get(self.ctx, .{ - .valType = config.dtype.toMlir(self.ctx), - .copyBits = @intCast(config.copy_bits), - .mode = if (config.mode == .gather4) .gather4 else .tiled, - .numCta = ctaCount(config.cta_group), - .gStride = self.emptyStrideType(), - .tmaGbasis = self.layoutType(config.global_basis), - }) catch @panic("invalid TMA load atom")).type_(); - } - - /// Host-side tiled TMA store atom type described by `config`. - pub fn tmaStoreAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { - return (cute_nvgpu.CopyAtomNonExecTiledTmaStoreType.get(self.ctx, .{ - .valType = config.dtype.toMlir(self.ctx), - .copyBits = @intCast(config.copy_bits), - .tmaGbasis = self.layoutType(config.global_basis), - .tmaFormat = tmaDataFormat(config.format, config.dtype), - }) catch @panic("invalid TMA store atom")).type_(); - } - - /// Executable device-side form of a tiled TMA store atom. - pub fn tmaStoreExecAtomType(self: *Builder, config: TmaConfig) *const mlir.Type { - return (cute_nvgpu.CopyAtomTmaStoreType.get(self.ctx, .{ - .valType = config.dtype.toMlir(self.ctx), - .copyBits = @intCast(config.copy_bits), - .mode = .tiled, - .gStride = self.emptyStrideType(), - .tmaGbasis = self.layoutType(config.global_basis), - }) catch @panic("invalid TMA store atom")).type_(); - } - - /// The `()` global stride of tiled TMA atoms. - fn emptyStrideType(self: *Builder) *const mlir.Type { - return (cute_ir.StrideType.get(self.ctx, .{ .attr = self.algebraAttr(cute_ir.StrideAttr, .{ .tuple = &.{} }).attribute() }) catch @panic("invalid CuTe stride")).type_(); - } - - /// Generic address (i64) of the tensor map in an executable tiled TMA atom. - pub fn tmaDescriptorAddress(self: *Builder, descriptor: Atom) Value { - const ptr = self.emit(cute_nvgpu.get_tma_desc_addr(self.ctx, descriptor.inner, (cute_ir.PtrType.get(self.ctx, .{ .valueType = (cute_nvgpu.TmaDescriptorTiledType.get(self.ctx, .{}) catch unreachable).type_(), .addressSpace = .generic, .alignment = 64 }) catch unreachable).type_(), self.loc())); - return self.ptrToInt(ptr, .i64); - } - - pub fn prefetchTmaDesc(self: *Builder, descriptor: Atom) void { - self.emitVoid(cute_nvgpu.prefetch_tma_desc(self.ctx, descriptor.inner, self.loc())); - } - - pub fn makeExecTma(self: *Builder, descriptor: Atom, result_type: *const mlir.Type) Atom { - const result = self.emit(cute_nvgpu.atom_make_exec_tma(self.ctx, descriptor.inner, result_type, .{}, self.loc())); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Attach the mbarrier that an executable TMA load atom completes. - pub fn setTmaBarrier(self: *Builder, atom: Atom, mbar: Value) Atom { - const result = self.emit(cute_nvgpu.atom_set_value( - self.ctx, - atom.inner, - mbar.inner, - atom.type_(), - (cute_nvgpu.AtomCopyFieldTmaLoadAttr.get(self.ctx, .{ .value = .tma_bar }) catch @panic("invalid TMA load field")).attribute(), - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Partition `smem` and `targets` for the TMA atom `descriptor`; the - /// dependent result types are built by the caller. - pub fn tmaPartition( - self: *Builder, - descriptor: Atom, - cta_coord: anytype, - cta_layout: anytype, - smem: anytype, - targets: anytype, - smem_type: *const mlir.Type, - target_types: []const *const mlir.Type, - ) TmaPartition { - if (targets.len != target_types.len) @panic("tmaPartition: target count does not match result type count"); - const views = self.arena.allocator().alloc(View, target_types.len) catch @panic("OOM"); - const op = cute_nvgpu.atom_tma_partition( - self.ctx, - descriptor.inner, - self.makeCoord(cta_coord), - self.asValue(cta_layout).inner, - self.asValue(smem).inner, - self.innerSlice(targets), - smem_type, - target_types, - self.loc(), - ); - _ = op.appendTo(self.currentBlock()); - for (views, 0..) |*view, i| view.* = .{ .inner = op.result(i + 1), .kernel = self }; - return .{ .smem = .{ .inner = op.result(0), .kernel = self }, .targets = views }; - } - - // ==================== cute_nvgpu.tcgen05 (SM100) ==================== - - /// Tiled MMA type of the block-scaled atom described by `cfg`. - pub fn blockScaledMmaType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { - return (cute_ir.TiledMmaType.get(self.ctx, .{ - .mmaAtom = self.blockScaledAtomType(cfg), - .atomLayoutMNK = self.layoutAttr(self.layoutSpec(.{ 1, 1, 1 }, .{ 0, 0, 0 })).attribute(), - .permutationMNK = self.algebraAttr(cute_ir.TileAttr, self.expr(.{ AlgebraToken.all, AlgebraToken.all, AlgebraToken.all })).attribute(), - }) catch @panic("invalid block-scaled tiled MMA")).type_(); - } - - fn blockScaledAtomType(self: *Builder, cfg: BlockScaledMmaConfig) *const mlir.Type { - cfg.validate(); - return (cute_nvgpu.MmaAtomSM100UMMABlockScaledType.get(self.ctx, .{ - .shapeMnk = self.algebraAttr(cute_ir.ShapeAttr, self.expr(.{ cfg.m, cfg.n, cfg.k() })).attribute(), - .numCta = ctaCount(cfg.cta_group), - .aMajor = .k, - .bMajor = .k, - .aType = DType.f4e2m1fn.toMlir(self.ctx), - .bType = cfg.bType().toMlir(self.ctx), - .cType = DType.f32.toMlir(self.ctx), - .sfType = cfg.scaleType().toMlir(self.ctx), - .aFragKind = .smem_desc, - .vecSize = cfg.vecSize(), - }) catch @panic("invalid block-scaled MMA atom")).type_(); - } - - /// Block-scaled atom and its tiled MMA. Both scale-factor pointers start - /// at `scale_ptr` and are patched after tensor-memory allocation. - pub fn makeBlockScaledMma(self: *Builder, cfg: BlockScaledMmaConfig, scale_ptr: Value) BlockScaledMma { - const disabled = self.constant(.i1, false); - const atom = self.makeAtom(self.blockScaledAtomType(cfg), .{ disabled, disabled, disabled, scale_ptr, scale_ptr }); - const tiled = self.emit(cute_ir.make_tiled_mma(self.ctx, atom.inner, self.blockScaledMmaType(cfg), self.loc())); - return .{ .atom = atom, .tiled = .{ .inner = tiled.inner, .kernel = self } }; - } - - /// Enable (or disable) accumulation into C for the next block-scaled MMA. - pub fn setMmaAccumulate(self: *Builder, mma: Atom, enabled: Value) Atom { - const result = self.emit(cute_nvgpu.atom_set_value( - self.ctx, - mma.inner, - enabled.inner, - mma.type_(), - (cute_nvgpu.AtomMmaFieldSM100BlockScaledAttr.get(self.ctx, .{ .value = .accum_c }) catch @panic("invalid block-scaled MMA field")).attribute(), - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub const Tcgen05CommitOptions = struct { - /// Must match the group of the operations being tracked. - cta_group: CtaGroup = .one, - /// Arrive at the same barrier offset in each CTA whose rank bit is set. - /// Without a mask, arrive only on the issuing CTA's barrier. - multicast_mask: ?Value = null, - }; - - /// Track this thread's preceding tcgen05 MMA/copy operations of the - /// selected group, signaling one arrival per destination on completion. - /// The same elected thread that issues those operations must commit them. - /// For `.two`, either CTA in the pair can be the issuer. All tcgen05 - /// operations in a kernel must use the same group. - pub fn tcgen05Commit(self: *Builder, mbar: Value, opts: Tcgen05CommitOptions) void { - self.emitVoid(nvvm.tcgen05_commit( - self.ctx, - self.sharedBarrierPtr(mbar).inner, - opts.cta_group.toNvvm(), - if (opts.multicast_mask) |mask| mask.to(.i16).inner else null, - self.loc(), - )); - } - - /// Wait until asynchronous tensor-memory loads have reached registers. - pub fn fenceTmemLoad(self: *Builder) void { - self.emitVoid(nvvm.tcgen05_wait(self.ctx, .load, self.loc())); - } - - pub const TmemOptions = struct { cta_group: CtaGroup = .one, exclusive: bool = false }; - - pub fn allocTmem(self: *Builder, columns: u16, holding_ptr: anytype, opts: TmemOptions) void { - self.emitVoid(cute_nvgpu.arch_sm100_alloc_tmem( - self.ctx, - self.constant(.i32, columns).inner, - self.asValue(holding_ptr).inner, - .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, - self.loc(), - )); - } - - pub fn retrieveTmemPtr(self: *Builder, holding_ptr: anytype, dtype: DType, alignment: u64) Value { - const ptr_type = self.ptrTy(dtype, .tmem, alignment) catch @panic("invalid tensor-memory pointer type"); - return self.emit(cute_nvgpu.arch_sm100_retrieve_tmem_ptr(self.ctx, self.asValue(holding_ptr).inner, ptr_type, self.loc())); - } - - pub fn relinquishTmemAllocPermit(self: *Builder, opts: TmemOptions) void { - self.emitVoid(cute_nvgpu.arch_sm100_relinquish_tmem_alloc_permit(self.ctx, .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null }, self.loc())); - } - - pub fn deallocTmem(self: *Builder, pointer: anytype, columns: u16, opts: TmemOptions) void { - self.emitVoid(cute_nvgpu.arch_sm100_dealloc_tmem( - self.ctx, - self.asValue(pointer).inner, - self.constant(.i32, columns).inner, - .{ .is_two_cta = if (opts.cta_group == .two) .unit(self.ctx) else null, .exclusive = if (opts.exclusive) .unit(self.ctx) else null }, - self.loc(), - )); - } - - pub const TmemLoadOptions = struct { - num_dp: u16, - num_b: u16, - num_rep: u16, - pack_16: bool = false, - }; - - pub fn tmemLoad(self: *Builder, src: anytype, result_type: *const mlir.Type, opts: TmemLoadOptions) View { - const result = self.emit(cute_nvgpu.arch_copy_SM100_tmem_load( - self.ctx, - self.asValue(src).inner, - result_type, - .int(self.ctx, .i32, opts.num_dp), - .int(self.ctx, .i32, opts.num_b), - .int(self.ctx, .i32, opts.num_rep), - .{ .pack_16 = if (opts.pack_16) .boolean(self.ctx, true) else null }, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Direct tensor-memory load into an SSA vector. Higher-level tiled-copy - /// helpers remain preferable when their dependent layouts are available. - pub fn tmemLoadVector(self: *Builder, src: anytype, lanes: i64, opts: TmemLoadOptions) Value { - // The NVVM-facing operation transports raw 32-bit register words. - // Callers reinterpret each lane as the accumulator element type. - const result_type = mlir.Type.vector(&.{lanes}, DType.i32.toMlir(self.ctx)); - return self.emit(cute_nvgpu.arch_copy_SM100_tmem_load( - self.ctx, - self.asValue(src).inner, - result_type, - .int(self.ctx, .i32, opts.num_dp), - .int(self.ctx, .i32, opts.num_b), - .int(self.ctx, .i32, opts.num_rep), - .{ .pack_16 = if (opts.pack_16) .boolean(self.ctx, true) else null }, - self.loc(), - )); - } - - pub const S2tCopyConfig = struct { - num_dp: i32, - num_bit: i32, - num_cta: i32 = 1, - broadcast: cute_nvgpu.CopyS2TBroadcast = .none, - }; - - /// `!cute_nvgpu.atom.s2t_copy`, a `tcgen05.cp` atom moving `dtype`. - pub fn s2tCopyAtomType(self: *Builder, dtype: DType, cfg: S2tCopyConfig) *const mlir.Type { - return (cute_nvgpu.CopyAtomSM100CopyS2TType.get(self.ctx, .{ - .valType = dtype.toMlir(self.ctx), - .numDp = cfg.num_dp, - .numBit = cfg.num_bit, - .numCta = cfg.num_cta, - .broadcast = cfg.broadcast, - }) catch @panic("invalid S2T copy atom")).type_(); - } - - pub fn makeS2tCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: *const mlir.Type) Atom { - const result = self.emit(cute_nvgpu.atom_make_s2t_copy( - self.ctx, - atom.inner, - self.asValue(tmem).inner, - result_type, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn makeTmemCopy(self: *Builder, atom: Atom, tmem: anytype, result_type: *const mlir.Type) Atom { - const result = self.emit(cute_nvgpu.atom_make_tmem_copy( - self.ctx, - atom.inner, - self.asValue(tmem).inner, - result_type, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - pub fn s2tSmemDescriptor(self: *Builder, atom: Atom, input: anytype, result_type: *const mlir.Type) View { - const result = self.emit(cute_nvgpu.atom_get_copy_s2t_smem_desc_view( - self.ctx, - atom.inner, - self.asValue(input).inner, - result_type, - self.loc(), - )); - return .{ .inner = result.inner, .kernel = self }; - } - - /// Raw SM100 TMA load. The descriptor address, destinations and - /// coordinates come from the executable atom and partition operations. - pub fn tmaLoad( - self: *Builder, - descriptor_addr: Value, - data_addr: Value, - barrier_addr: Value, - coords: []const Value, - multicast_mask: ?Value, - offsets: []const Value, - cache_policy: ?Value, - mode: *const mlir.Attribute, - num_cta: u8, - ) void { - self.emitVoid(cute_nvgpu.arch_copy_SM100_tma_load( - self.ctx, - descriptor_addr.inner, - data_addr.inner, - barrier_addr.inner, - self.innerSlice(coords), - self.innerSlice(offsets), - mode, - .int(self.ctx, .i32, num_cta), - .{ .multicast_mask = if (multicast_mask) |v| v.inner else null, .cache_policy = if (cache_policy) |v| v.inner else null }, - self.loc(), - )); - } - - /// Raw SM100 TMA store. - pub fn tmaStore( - self: *Builder, - descriptor_addr: Value, - data_addr: Value, - coords: []const Value, - cache_policy: ?Value, - mode: *const mlir.Attribute, - ) void { - self.emitVoid(cute_nvgpu.arch_copy_SM100_tma_store( - self.ctx, - descriptor_addr.inner, - data_addr.inner, - self.innerSlice(coords), - mode, - .{ .cache_policy = if (cache_policy) |v| v.inner else null }, - self.loc(), - )); - } - - // ==================== cute.arch ==================== - - fn sreg(self: *Builder, comptime reg: []const u8) Index3 { - var out: [3]Value = undefined; - inline for (.{ "x", "y", "z" }, 0..) |axis, i| { - out[i] = self.emit(nvvm.read_sreg(self.ctx, reg ++ "." ++ axis, self.loc())); - } - return .{ .x = out[0], .y = out[1], .z = out[2] }; - } - - pub fn threadIdx(self: *Builder) Index3 { - return self.sreg("tid"); - } - - pub fn blockIdx(self: *Builder) Index3 { - return self.sreg("ctaid"); - } - - pub fn blockDim(self: *Builder) Index3 { - return self.sreg("ntid"); - } - - pub fn gridDim(self: *Builder) Index3 { - return self.sreg("nctaid"); - } - - /// Mark a scalar as identical in every lane of its warp. CuTe uses this - /// fact when lowering role branches that contain warp-collective TMA and - /// tensor-memory operations. - pub fn makeWarpUniform(self: *Builder, value: anytype) Value { - const scalar = self.asValue(value); - // The CuTe Python frontend legalizes predicates through an i32: - // extui i1 -> i32; make_warp_uniform i32; cmpi ne 0 - // `arch.make_warp_uniform` on i1 parses, but the native SM100 - // pipeline does not lower it all the way to NVVM. - const uniform_input = if (scalar.dtype() == .i1) scalar.to(.i32) else scalar; - const uniform = self.emit(cute_nvgpu.arch_make_warp_uniform( - self.ctx, - uniform_input.inner, - uniform_input.type_(), - self.loc(), - )); - return if (scalar.dtype() == .i1) uniform.ne(0) else uniform; - } - - /// `cute.arch.sync_threads()`. - pub fn syncThreads(self: *Builder) void { - _ = nvvm.barrier(self.ctx, self.loc()).appendTo(self.currentBlock()); - } - - /// `bar.sync id, participants` (`barrier.cta.sync.aligned`). - /// `id` is an integer or an i32 Value in [0, 15]. `participants` must be - /// a positive warp multiple, and every participating thread must execute - /// the same barrier id. The aligned form requires CTA-uniform control flow. - pub fn barrierCtaSyncAligned(self: *Builder, id: anytype, participants: anytype) void { - if (@TypeOf(participants) == ?Value) { - if (participants) |count| return self.barrierCtaSyncAligned(id, count); - return self.barrierCtaSyncAligned(id, null); - } - if (@TypeOf(participants) != Value and @TypeOf(participants) != @TypeOf(null)) { - std.debug.assert(participants > 0 and participants <= 1024 and @mod(participants, 32) == 0); - } - if (@TypeOf(id) == Value) { - std.debug.assert(id.dtype() == .i32); - } else { - const immediate: i32 = id; - std.debug.assert(immediate >= 0 and immediate < 16); - } - // The LLVM intrinsic preserves convergence and both operands across - // the embedded compiler's NVVM dialect version boundary. - if (@TypeOf(participants) == @TypeOf(null)) { - _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned", &.{self.lift(id).to(.i32)}, null); - } else { - _ = self.callIntrinsic("llvm.nvvm.barrier.cta.sync.aligned.count", &.{ self.lift(id).to(.i32), self.lift(participants).to(.i32) }, null); - } - } - - pub const BarrierOptions = struct { barrier_id: ?Value = null, number_of_threads: ?Value = null }; - - pub fn barrier(self: *Builder, opts: BarrierOptions) void { - self.barrierCtaSyncAligned(opts.barrier_id orelse self.constant(.i32, 0), opts.number_of_threads); - } - - /// Build the CuTe first-class divisor used by CUTLASS's persistent tile - /// scheduler. Its arithmetic is lowered to the same multiply/shift - /// sequence as Python CuTe DSL rather than an integer div/rem pair. - pub fn fastDivmodCreate(self: *Builder, divisor_: anytype) Value { - const divisor = self.lift(divisor_).to(.i32); - return self.emit(cute_ir.fast_divmod_create_divisor( - self.ctx, - divisor.inner, - (cute_ir.FastDivmodDivisorType.get(self.ctx, .{ .width = 32, .isPow2 = false }) catch @panic("invalid fast divmod divisor")).type_(), - self.loc(), - )); - } - - /// Return `{ quotient, remainder }` for a CUTLASS FastDivmod divisor. - pub fn fastDivmod(self: *Builder, dividend_: anytype, divisor: Value) [2]Value { - const dividend = self.lift(dividend_).to(.i32); - return self.emitResults(2, cute_ir.fast_divmod_compute( - self.ctx, - dividend.inner, - divisor.inner, - DType.i32.toMlir(self.ctx), - DType.i32.toMlir(self.ctx), - self.loc(), - )); - } - - /// Elect one lane from the active warp. Used by TMA issue, tensor-memory - /// allocation, and tcgen05 commit operations. - pub fn electSync(self: *Builder) Value { - return self.emit(nvvm.elect_sync(self.ctx, self.loc())); - } - - /// `cute.arch.shuffle_sync_bfly`: exchange a scalar with the lane at - /// `lane_id ^ offset` in the current warp. - /// Warp-wide maximum of a non-negative float, as one `redux.sync` instead - /// of a shuffle reduction: IEEE bit patterns of non-negative floats order - /// like unsigned integers. - pub fn warpMaxNonNegative(self: *Builder, value: Value) Value { - const dtype = value.dtype(); - if (dtypeBitwidth(dtype) != 32) @panic("warpMaxNonNegative requires a 32-bit scalar"); - const bits = if (dtype == .i32) value else value.bitCast(.i32); - const reduced = self.emit(llvm.inline_asm( - self.ctx, - "redux.sync.max.u32 $0, $1, 0xffffffff;", - &.{bits.inner}, - .{ .constraints = "=r,r", .result_type = .int(self.ctx, .i32) }, - self.loc(), - )); - return if (dtype == .i32) reduced else reduced.bitCast(dtype); - } - - pub fn shuffleXor(self: *Builder, value: Value, offset: i32) Value { - const dtype = value.dtype(); - if (dtypeBitwidth(dtype) != 32) @panic("shuffleXor currently requires a 32-bit scalar"); - const bits = if (dtype == .i32) value else value.bitCast(.i32); - const shuffled = self.emit(llvm.inline_asm( - self.ctx, - "shfl.sync.bfly.b32 $0, $1, $2, 31, 0xffffffff;", - &.{ bits.inner, self.constant(.i32, offset).inner }, - .{ .constraints = "=r,r,r", .result_type = .int(self.ctx, .i32) }, - self.loc(), - )); - return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); - } - - /// `cute.arch.shuffle_sync`: read a scalar from lane `lane` of the - /// current warp. Every lane must participate. - pub fn shuffleIdx(self: *Builder, value: Value, lane: anytype) Value { - const dtype = value.dtype(); - if (dtypeBitwidth(dtype) != 32) @panic("shuffleIdx currently requires a 32-bit scalar"); - const bits = if (dtype == .i32) value else value.bitCast(.i32); - const shuffled = self.emit(llvm.inline_asm( - self.ctx, - "shfl.sync.idx.b32 $0, $1, $2, 31, 0xffffffff;", - &.{ bits.inner, self.constant(.i32, lane).inner }, - .{ .constraints = "=r,r,r", .result_type = .int(self.ctx, .i32) }, - self.loc(), - )); - return if (dtype == .i32) shuffled else shuffled.bitCast(dtype); - } - - /// Four FP32 values rounded to FP8 E4M3 (saturating) and packed into one - /// 32-bit word, `a` in the lowest byte. - pub fn packFp8x4(self: *Builder, a: Value, b: Value, c: Value, d: Value) Value { - return self.emit(llvm.inline_asm( - self.ctx, - "{ .reg .b16 lo, hi; cvt.rn.satfinite.e4m3x2.f32 lo, $2, $1; cvt.rn.satfinite.e4m3x2.f32 hi, $4, $3; mov.b32 $0, {lo, hi}; }", - &.{ a.to(.f32).inner, b.to(.f32).inner, c.to(.f32).inner, d.to(.f32).inner }, - .{ .constraints = "=r,f,f,f,f", .result_type = .int(self.ctx, .i32) }, - self.loc(), - )); - } - - /// Eight FP32 values rounded to FP4 E2M1 (saturating, nearest-even) and - /// packed into one 32-bit word, `values[0]` in the lowest nibble: the - /// packed NVFP4 storage order. - pub fn packFp4x8(self: *Builder, values: [8]Value) Value { - var operands: [8]*const mlir.Value = undefined; - for (&operands, values) |*operand, value| operand.* = value.to(.f32).inner; - return self.emit(llvm.inline_asm( - self.ctx, - "{ .reg .b8 b0, b1, b2, b3; cvt.rn.satfinite.e2m1x2.f32 b0, $2, $1; cvt.rn.satfinite.e2m1x2.f32 b1, $4, $3; " ++ - "cvt.rn.satfinite.e2m1x2.f32 b2, $6, $5; cvt.rn.satfinite.e2m1x2.f32 b3, $8, $7; mov.b32 $0, {b0, b1, b2, b3}; }", - &operands, - .{ .constraints = "=r,f,f,f,f,f,f,f,f", .result_type = .int(self.ctx, .i32) }, - self.loc(), - )); - } - - /// Two FP32 values rounded to FP4 E2M1 (saturating, nearest-even) in the - /// low byte of a 32-bit word, `a` in the low nibble. - pub fn packFp4x2(self: *Builder, a: Value, b: Value) Value { - return self.emit(llvm.inline_asm( - self.ctx, - "{ .reg .b8 lo; cvt.rn.satfinite.e2m1x2.f32 lo, $2, $1; cvt.u32.u8 $0, lo; }", - &.{ a.to(.f32).inner, b.to(.f32).inner }, - .{ .constraints = "=r,f,f", .result_type = .int(self.ctx, .i32) }, - self.loc(), - )); - } - - /// The FP8 E4M3 byte at the low end of `bits` (an i32), decoded to FP32. - pub fn e4m3ToF32(self: *Builder, bits: Value) Value { - return self.emit(llvm.inline_asm( - self.ctx, - "{ .reg .b16 lo, hl, hh; .reg .b32 pair; cvt.u16.u32 lo, $1; cvt.rn.f16x2.e4m3x2 pair, lo; mov.b32 {hl, hh}, pair; cvt.f32.f16 $0, hl; }", - &.{bits.inner}, - .{ .constraints = "=f,r", .result_type = DType.f32.toMlir(self.ctx) }, - self.loc(), - )); - } - - /// One-operand FP32 PTX instruction, e.g. `ex2.approx.ftz.f32`. - pub fn unaryF32(self: *Builder, comptime instruction: []const u8, value: Value) Value { - return self.emit(llvm.inline_asm( - self.ctx, - instruction ++ " $0, $1;", - &.{value.to(.f32).inner}, - .{ .constraints = "=f,f", .result_type = DType.f32.toMlir(self.ctx) }, - self.loc(), - )); - } - - /// Signal programmatic dependent launches after this grid has consumed - /// its inputs. The caller must also launch the producer/consumer chain - /// with PDL enabled. - pub fn launchDependents(self: *Builder) void { - _ = llvm.inline_asm(self.ctx, "griddepcontrol.launch_dependents;", &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - /// Wait until the preceding grid has made its programmatic dependency - /// available. This is the consumer-side half of a PDL launch chain. - pub fn waitForDependency(self: *Builder) void { - _ = llvm.inline_asm(self.ctx, "griddepcontrol.wait;", &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - /// `cute.arch.alloc_smem(dtype, size_in_elems, alignment)`: a static - /// shared-memory pointer, sized at compile time. - pub fn allocSmem(self: *Builder, dt: DType, size_in_elems: i64, alignment: ?u64) Value { - const al = alignment orelse @max(1, dtypeBitwidth(dt) / 8); - const ty = self.ptrTy(dt, .smem, al) catch @panic("bad smem pointer"); - return self.emit(cute_nvgpu.arch_alloc_smem(self.ctx, ty, .int(self.ctx, .i32, size_in_elems), self.loc())); - } - - // mbarriers, async-proxy fences and bulk-copy groups. - - /// mbarrier operands are `!llvm.ptr<3>` for NVVM. A barrier is either a CuTe - /// shared-memory pointer or a raw `i32` shared-memory address. - fn sharedBarrierPtr(self: *Builder, mbar: Value) Value { - if (mbar.isInt()) return self.addressPtr(mbar, .shared); - return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{mbar.inner}, &.{llvm.pointerType(self.ctx, NVPTXAddressSpace.shared)}, self.loc())); - } - - /// Initialize a shared-memory mbarrier expecting `arrivals` per phase. - pub fn mbarrierInit(self: *Builder, mbar: Value, arrivals: u32) void { - self.emitVoid(nvvm.mbarrier_init(self.ctx, self.sharedBarrierPtr(mbar).inner, self.constant(.i32, arrivals).inner, self.loc())); - } - - /// Arrive and expect `bytes` of asynchronous transactions (SM90+), as a - /// TMA producer does before issuing its copies. - pub fn mbarrierArriveExpectTx(self: *Builder, mbar: Value, bytes: anytype) void { - self.emitVoid(nvvm.mbarrier_arrive_expect_tx(self.ctx, self.sharedBarrierPtr(mbar).inner, self.lift(bytes).to(.i32).inner, .cta, self.loc())); - } - - pub const MBarrierArriveOptions = struct { - peer_cta_rank_in_cluster: ?Value = null, - arrive_count: ?Value = null, - scope: MBarrierScope = .cta, - relaxed: bool = false, - }; - - /// Arrive locally or at the same offset in a peer CTA. Local arrivals - /// return the opaque state token; remote arrivals have no state result. - pub fn mbarrierArrive(self: *Builder, mbar: Value, opts: MBarrierArriveOptions) ?Value { - var ptr = self.sharedBarrierPtr(mbar); - if (opts.peer_cta_rank_in_cluster) |rank| { - ptr = self.emit(nvvm.mapa(self.ctx, ptr.inner, rank.inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); - } - const op = nvvm.ops.mbarrier_arrive(self.ctx, ptr.inner, .{ - .count = if (opts.arrive_count) |count| count.to(.i32).inner else null, - .scope = (nvvm.ops.MemScopeKindAttr.get(self.ctx, .{ .value = switch (opts.scope) { - .cta => .cta, - .cluster => .cluster, - } }) catch unreachable).attribute(), - .relaxed = .boolean(self.ctx, opts.relaxed), - .res_type = if (opts.peer_cta_rank_in_cluster == null) mlir.Type.int(self.ctx, .i64) else null, - }, self.loc()); - if (opts.peer_cta_rank_in_cluster == null) return self.emit(op); - self.emitVoid(op); - return null; - } - - pub const MBarrierTryWaitOptions = struct { - scope: MBarrierScope = .cta, - time_limit: ?Value = null, - }; - - /// Try an acquire wait for phase `phase` (i32 parity, 0 or 1). May suspend - /// briefly and return false before completion. `scope` selects which - /// threads this wait can synchronize with; the barrier remains local. - pub fn mbarrierTryWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierTryWaitOptions) Value { - const ptr = self.sharedBarrierPtr(mbar); - if (opts.time_limit) |ticks| { - return self.emit(nvvm.mbarrier_try_wait_parity_timelimit(self.ctx, ptr.inner, phase.inner, ticks.to(.i32).inner, opts.scope, self.loc())); - } - return self.emit(nvvm.mbarrier_wait_parity(self.ctx, ptr.inner, phase.inner, .@"try", opts.scope, self.loc())); - } - - pub const MBarrierWaitOptions = struct { - scope: MBarrierScope = .cta, - /// Hardware suspend hint in nanoseconds per attempt, not a total timeout. - time_limit: u32 = 10_000_000, - /// Probe before suspending, as in CuTe's pipeline consumer wait. - test_first: bool = true, - }; - - fn mbarrierWaitLoop(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { - const ptr = self.sharedBarrierPtr(mbar); - const ticks = self.constant(.i32, opts.time_limit); - switch (opts.scope) { - // Preserve the operation recognized by CuTe's pipeline compiler. - .cta => self.emitVoid(nvvm.mbarrier_try_wait_parity(self.ctx, ptr.inner, phase.inner, ticks.inner, self.loc())), - .cluster => { - var pending = self.openWhile(.{}, .{}); - const ready = self.mbarrierTryWait(mbar, phase, .{ .scope = opts.scope, .time_limit = ticks }); - pending.yieldBefore(ready.eq(false), .{}); - pending.yieldAfter(.{}); - }, - } - } - - /// Acquire a pipeline phase at `scope`: test without suspending, then - /// block only when the phase is still outstanding. The initial test matters when - /// a producer reaches a full ring and must yield to the consumer that - /// recycles its next stage. - pub fn mbarrierWait(self: *Builder, mbar: Value, phase: Value, opts: MBarrierWaitOptions) void { - if (opts.test_first) { - const ready = self.mbarrierTestWait(mbar, phase, .{ .scope = opts.scope }); - var pending = self.openIf(ready.eq(false)); - self.mbarrierWaitLoop(mbar, phase, opts); - pending.close(); - } else { - self.mbarrierWaitLoop(mbar, phase, opts); - } - } - - /// Make mbarrier initialization visible to the async proxy (SM90+). - pub fn fenceMbarrierInit(self: *Builder) void { - self.emitVoid(nvvm.fence_mbarrier_init(self.ctx, self.loc())); - } - - /// Publish ordinary shared-memory writes to the asynchronous (TMA) proxy (SM90+). - pub fn fenceProxyAsyncShared(self: *Builder) void { - self.emitVoid(nvvm.fence_proxy(self.ctx, .@"async.shared", .cta, self.loc())); - } - - /// Close the current group of bulk asynchronous copies, e.g. TMA stores (SM90+). - pub fn cpAsyncBulkCommitGroup(self: *Builder) void { - self.emitVoid(nvvm.cp_async_bulk_commit_group(self.ctx, self.loc())); - } - - /// Wait until at most `pending` bulk copy groups are in flight; with - /// `read`, only until their sources have been read (SM90+). - pub fn cpAsyncBulkWaitGroup(self: *Builder, pending: u32, read: bool) void { - self.emitVoid(nvvm.cp_async_bulk_wait_group(self.ctx, pending, read, self.loc())); - } - - // ==================== raw-address SM90/SM100 primitives ==================== - // Hand-scheduled kernels address shared memory with `i32` offsets in the - // shared window, global memory with `i64` addresses and tensor memory with - // `i32` TMEM addresses (lane << 16 | column). These helpers emit the NVVM / - // LLVM operations for them directly, using inline PTX where needed. - - pub const NVPTXAddressSpace = llvm.NVPTXAddressSpace; - - /// Typed LLVM pointer (`llvm.inttoptr`) for an integer address in `space`. - pub fn addressPtr(self: *Builder, address: Value, comptime space: NVPTXAddressSpace) Value { - return self.emit(llvm.inttoptr(self.ctx, address.inner, llvm.pointerType(self.ctx, space), self.loc())); - } - - fn wordsType(self: *Builder, comptime n: usize) *const mlir.Type { - const word = mlir.Type.int(self.ctx, .i32); - return if (n == 1) word else mlir.Type.vector(&.{n}, word); - } - - fn splitWords(self: *Builder, comptime n: usize, v: Value) [n]Value { - var out: [n]Value = undefined; - if (n == 1) { - out[0] = v; - } else { - for (0..n) |i| out[i] = self.vectorExtract(v, @intCast(i), .i32); - } - return out; - } - - fn joinWords(self: *Builder, comptime n: usize, words: [n]Value) Value { - if (n == 1) return words[0]; - var inner: [n]*const mlir.Value = undefined; - for (words, &inner) |w, *x| x.* = w.inner; - return self.emit(vector.from_elements(self.ctx, &inner, self.wordsType(n), self.loc())); - } - - pub const LoadOptions = struct { - /// Read-only for the whole kernel (global: non-coherent `ld.global.nc`). - invariant: bool = false, - }; - - /// Load `n` (1, 2 or 4) consecutive 32-bit words from a typed pointer. - pub fn loadWordsPtr(self: *Builder, comptime n: usize, ptr: Value, opts: LoadOptions) [n]Value { - const v = self.emit(llvm.load(self.ctx, ptr.inner, self.wordsType(n), .{ .alignment = 4 * n, .invariant = opts.invariant }, self.loc())); - return self.splitWords(n, v); - } - - /// Load `n` (1, 2 or 4) consecutive 32-bit words at an integer address. - pub fn loadWords(self: *Builder, comptime n: usize, address: Value, comptime space: NVPTXAddressSpace, opts: LoadOptions) [n]Value { - return self.loadWordsPtr(n, self.addressPtr(address, space), opts); - } - - /// Store `n` (1, 2 or 4) consecutive 32-bit words through a typed pointer. - pub fn storeWordsPtr(self: *Builder, comptime n: usize, ptr: Value, words: [n]Value) void { - self.emitVoid(llvm.store(self.ctx, self.joinWords(n, words).inner, ptr.inner, .{ .alignment = 4 * n }, self.loc())); - } - - /// Store `n` (1, 2 or 4) consecutive 32-bit words at an integer address. - pub fn storeWords(self: *Builder, comptime n: usize, address: Value, comptime space: NVPTXAddressSpace, words: [n]Value) void { - self.storeWordsPtr(n, self.addressPtr(address, space), words); - } - - /// Store the low 16 bits of an `i32`. - pub fn storeHalf(self: *Builder, address: Value, comptime space: NVPTXAddressSpace, word: Value) void { - self.emitVoid(llvm.store(self.ctx, word.to(.i16).inner, self.addressPtr(address, space).inner, .{ .alignment = 2 }, self.loc())); - } - - /// `bar.red.or.pred`: named barrier over `threads` threads that also returns whether - /// `predicate` (i1) is true in any of them. - pub fn barrierReduceOr(self: *Builder, id: i32, threads: i32, predicate: Value) Value { - return self.callIntrinsic("llvm.nvvm.barrier.cta.red.or.aligned.count", &.{ self.constant(.i32, id), self.constant(.i32, threads), predicate }, mlir.Type.int(self.ctx, .i1)).?; - } - - /// `bar.warp.sync 0xffffffff`. - pub fn syncWarp(self: *Builder) void { - self.emitVoid(nvvm.bar_warp_sync(self.ctx, self.constant(.i32, -1).inner, self.loc())); - } - - /// Non-blocking acquire test of mbarrier phase `parity` (i32, 0 or 1). - pub fn mbarrierTestWait(self: *Builder, mbar: Value, parity: Value, opts: MBarrierTryWaitOptions) Value { - std.debug.assert(opts.time_limit == null); - return self.emit(nvvm.mbarrier_wait_parity(self.ctx, self.sharedBarrierPtr(mbar).inner, parity.inner, .@"test", opts.scope, self.loc())); - } - - /// Call an LLVM intrinsic by name (e.g. an `llvm.nvvm.*` intrinsic the NVVM dialect of - /// the CuTe compiler does not model). - pub fn callIntrinsic(self: *Builder, comptime name: []const u8, operands: []const Value, result: ?*const mlir.Type) ?Value { - const alloc = self.arena.allocator(); - const inner = alloc.alloc(*const mlir.Value, operands.len) catch @panic("OOM"); - for (operands, inner) |o, *x| x.* = o.inner; - const op = llvm.call_intrinsic(self.ctx, name, inner, .{ .result_type = result }, self.loc()); - if (result == null) { - self.emitVoid(op); - return null; - } - return self.emit(op); - } - - /// 16-byte `cp.async.cg` global -> shared. With `valid` (i1) false, the destination is - /// zero-filled and `src` is not read (src-size 0). - pub fn cpAsync16(self: *Builder, dst: Value, src: Value, valid: ?Value) void { - const d = self.addressPtr(dst, .shared); - const g = self.addressPtr(src, .global); - if (valid) |v| { - _ = self.callIntrinsic("llvm.nvvm.cp.async.cg.shared.global.16.s", &.{ d, g, self.select(v, self.constant(.i32, 16), self.constant(.i32, 0)) }, null); - } else { - _ = self.callIntrinsic("llvm.nvvm.cp.async.cg.shared.global.16", &.{ d, g }, null); - } - } - - /// Arrive on `barrier` once this thread's prior cp.async copies have landed; with - /// `noinc`, the arrival is counted in the barrier's expected count. - pub fn cpAsyncMbarrierArrive(self: *Builder, mbar: Value, comptime noinc: bool) void { - _ = self.callIntrinsic(if (noinc) "llvm.nvvm.cp.async.mbarrier.arrive.noinc.shared" else "llvm.nvvm.cp.async.mbarrier.arrive.shared", &.{self.sharedBarrierPtr(mbar)}, null); - } - - /// Wait until all of this thread's cp.async copies have landed. - pub fn cpAsyncWaitAll(self: *Builder) void { - self.emitVoid(nvvm.cp_async_commit_group(self.ctx, self.loc())); - self.emitVoid(nvvm.cp_async_wait_group(self.ctx, 0, self.loc())); - } - - /// Prefetch the global line of `address` (i64) into L2 with evict-last priority. - pub fn prefetchL2EvictLast(self: *Builder, address: Value) void { - _ = self.callIntrinsic("llvm.nvvm.prefetch.global.L2.evict.last", &.{self.addressPtr(address, .global)}, null); - } - - /// Warp-wide TMEM allocation of `columns`; the base address is written to `holder` (i32 smem). - pub fn tmemAllocAt(self: *Builder, holder: Value, columns: u32) void { - self.emitVoid(nvvm.tcgen05_alloc(self.ctx, self.addressPtr(holder, .shared).inner, self.constant(.i32, columns).inner, self.loc())); - } - - pub fn tmemDeallocAt(self: *Builder, taddr: Value, columns: u32) void { - self.emitVoid(nvvm.tcgen05_dealloc(self.ctx, self.addressPtr(taddr, .tensor).inner, self.constant(.i32, columns).inner, self.loc())); - } - - pub const Tcgen05Fence = nvvm.Tcgen05FenceKind; - - /// `tcgen05.fence::{before,after}_thread_sync`. - pub fn tcgen05Fence(self: *Builder, comptime kind: Tcgen05Fence) void { - self.emitVoid(nvvm.tcgen05_fence(self.ctx, kind, self.loc())); - } - - /// `tcgen05.wait::st`. - pub fn fenceTmemStore(self: *Builder) void { - self.emitVoid(nvvm.tcgen05_wait(self.ctx, .store, self.loc())); - } - - /// `tcgen05.mma.cta_group::1.kind::f16`: D (TMEM at `d`) = A . B (+ D when - /// `accumulate`, i1), with A and B given by shared-memory descriptors (i64) and the - /// instruction descriptor `idesc` (i32). - pub fn tcgen05MmaF16(self: *Builder, d: Value, adesc: Value, bdesc: Value, idesc: Value, accumulate: Value) void { - self.emitVoid(nvvm.tcgen05_mma(self.ctx, self.addressPtr(d, .tensor).inner, adesc.inner, bdesc.inner, idesc.inner, accumulate.inner, .f16, .cta_1, self.loc())); - } - - /// `tcgen05.ld.32x32b.x{n}`: thread i of the warp gets TMEM lane (warp base + i), - /// `n` consecutive 32-bit columns from `taddr`. - pub fn tmemLoad32x32b(self: *Builder, comptime n: usize, taddr: Value) [n]Value { - const v = self.emit(nvvm.tcgen05_ld(self.ctx, self.addressPtr(taddr, .tensor).inner, self.wordsType(n), .shape_32x32b, self.loc())); - return self.splitWords(n, v); - } - - /// `tcgen05.st.32x32b.x{n}` (see `tmemLoad32x32b`). - pub fn tmemStore32x32b(self: *Builder, comptime n: usize, taddr: Value, words: [n]Value) void { - self.emitVoid(nvvm.tcgen05_st(self.ctx, self.addressPtr(taddr, .tensor).inner, self.joinWords(n, words).inner, .shape_32x32b, self.loc())); - } - - /// Rank of this CTA in its cluster. - pub fn clusterCtaRank(self: *Builder) Value { - return self.emit(nvvm.read_sreg(self.ctx, "cluster.ctarank", self.loc())); - } - - /// The same shared-memory location (i32 address) in CTA `rank` of the cluster, as a - /// `shared::cluster` pointer. - pub fn mapaShared(self: *Builder, address: Value, rank: Value) Value { - return self.emit(nvvm.mapa(self.ctx, self.addressPtr(address, .shared).inner, rank.inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); - } - - /// Full cluster barrier (arrive.release + wait.acquire, aligned). - pub fn clusterSync(self: *Builder) void { - self.emitVoid(nvvm.cluster_arrive(self.ctx, true, self.loc())); - self.emitVoid(nvvm.cluster_wait(self.ctx, true, self.loc())); - } - - /// Warp-wide maximum of an f32 (`redux.sync.max.f32`, SM100). - pub fn warpMaxF32(self: *Builder, value: Value) Value { - return self.callIntrinsic("llvm.nvvm.redux.sync.fmax", &.{ value, self.constant(.i32, -1) }, mlir.Type.float(self.ctx, .f32)).?; - } - - /// Fast approximate 2^x (`ex2.approx.ftz.f32`). - pub fn exp2Approx(self: *Builder, value: Value) Value { - return self.emit(nvvm.ex2(self.ctx, value.inner, true, self.loc())); - } - - /// Fast approximate 1/x (`rcp.approx.ftz.f32`). - pub fn rcpApprox(self: *Builder, value: Value) Value { - return self.emit(nvvm.rcp_approx_ftz(self.ctx, value.inner, self.loc())); - } - - /// `vector<2 x dt>`, e.g. a bf16x2 pair packed in 32 bits. - fn pairType(self: *Builder, comptime dt: DType) *const mlir.Type { - return mlir.Type.vector(&.{2}, dt.toMlir(self.ctx)); - } - - fn bitcastTo(self: *Builder, value: Value, ty: *const mlir.Type) Value { - return self.emit(llvm.bitcast(self.ctx, value.inner, ty, self.loc())); - } - - /// Two FP8 e4m3 (low 16 bits of `pair`, i32) -> bf16x2 packed in an i32. - pub fn e4m3x2ToBf16x2(self: *Builder, pair: Value) Value { - const v = self.callIntrinsic("llvm.nvvm.e4m3x2.to.bf16x2.rn", &.{pair.to(.i16)}, self.pairType(.bf16)).?; - return self.bitcastTo(v, mlir.Type.int(self.ctx, .i32)); - } - - /// Two FP4 e2m1 (the low byte of `pair`, the even element in the low nibble) - /// -> bf16x2 packed in an i32. - pub fn e2m1x2ToBf16x2(self: *Builder, pair: Value) Value { - const v = self.callIntrinsic("llvm.nvvm.e2m1x2.to.bf16x2.rn", &.{pair.to(.i16)}, self.pairType(.bf16)).?; - return self.bitcastTo(v, mlir.Type.int(self.ctx, .i32)); - } - - /// Eight FP4 e2m1 (one word, even elements in the low nibbles) -> four bf16x2 packed - /// in i32s. The pairs are read as the low bytes of the 16-bit halves of `word` and - /// `word >> 8`, which ptxas maps to operand selectors (one shift per word). - pub fn e2m1x8ToBf16x2x4(self: *Builder, word: Value) [4]Value { - const halves = self.pairType(.i16); - const even = self.bitcastTo(word, halves); - const odd = self.bitcastTo(word.shrLogical(8), halves); - const pairs = [4]Value{ - self.vectorExtract(even, 0, .i16), self.vectorExtract(odd, 0, .i16), - self.vectorExtract(even, 1, .i16), self.vectorExtract(odd, 1, .i16), - }; - var out: [4]Value = undefined; - for (pairs, &out) |pair, *o| o.* = self.e2m1x2ToBf16x2(pair); - return out; - } - - /// bf16x2 * bf16x2 (round to nearest), both packed in i32. - pub fn mulBf16x2(self: *Builder, a: Value, b: Value) Value { - const ty = self.pairType(.bf16); - const p = self.emit(llvm.fmul(self.ctx, self.bitcastTo(a, ty).inner, self.bitcastTo(b, ty).inner, ty, self.loc())); - return self.bitcastTo(p, mlir.Type.int(self.ctx, .i32)); - } - - /// Two f32 -> a packed pair of `dt` (bf16 or f16) in an i32, `lo` in the low half. - pub fn packHalf2(self: *Builder, lo: Value, hi: Value, comptime dt: DType) Value { - const pair = self.emit(vector.from_elements(self.ctx, &.{ lo.to(dt).inner, hi.to(dt).inner }, self.pairType(dt), self.loc())); - return self.bitcastTo(pair, mlir.Type.int(self.ctx, .i32)); - } - - /// A packed pair of `dt` (bf16 or f16) in an i32 -> two f32 (low half first). - pub fn unpackHalf2(self: *Builder, word: Value, comptime dt: DType) [2]Value { - const pair = self.bitcastTo(word, self.pairType(dt)); - return .{ self.vectorExtract(pair, 0, dt).to(.f32), self.vectorExtract(pair, 1, dt).to(.f32) }; - } - - /// f32 -> bf16 bits (round to nearest even) in the low 16 bits of an i32. - pub fn f32ToBf16Bits(self: *Builder, value: Value) Value { - return self.emit(arith.extui(self.ctx, value.to(.bf16).bitCast(.i16).inner, mlir.Type.int(self.ctx, .i32), self.loc())); - } - - /// An i32 equal to `value` at run time that the compiler cannot constant-fold - /// (scaled by `%nctaid.z`, which is 1 for 2-D grids): e.g. a loop bound that must - /// keep a loop rolled (small code, fewer instruction-cache misses). - pub fn opaqueI32(self: *Builder, value: i32) Value { - return self.gridDim().z.mul(value); - } - - /// SM100 `tile::gather4` TMA load from a tiled 2-D descriptor address `desc` (i64). - /// Gather four rows `r` (i32) at column `col` into 128-byte-aligned shared `dst` (i32), - /// completing their bytes on an 8-byte-aligned local mbarrier `bar` (i32). - /// The descriptor must have row box extent 1 and no interleaving. `col` and `r` - /// are element coordinates; row length and out-of-bounds fill come from the - /// descriptor (zero or OOB-NaN). Arm the barrier for all four rows' bytes before waiting. - pub fn tmaGather4(self: *Builder, desc: Value, dst: Value, bar: Value, col: Value, r: [4]Value, policy: CacheHintSm100) void { - std.debug.assert(desc.dtype() == .i64 and dst.dtype() == .i32 and bar.dtype() == .i32 and col.dtype() == .i32); - for (r) |row| std.debug.assert(row.dtype() == .i32); - _ = llvm.inline_asm(self.ctx, "cp.async.bulk.tensor.2d.shared::cta.global.tile::gather4.mbarrier::complete_tx::bytes.cta_group::1.L2::cache_hint [$0], [$1, {$2, $3, $4, $5, $6}], [$7], $8;", &.{ dst.inner, desc.inner, col.inner, r[0].inner, r[1].inner, r[2].inner, r[3].inner, bar.inner, self.constant(.i64, policy).inner }, .{ .constraints = "r,l,r,r,r,r,r,r,l,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - pub const BulkPrefetchOptions = struct { l2_cache_hint: ?Value = null }; - - /// Best-effort byte-range prefetch into L2; size must be a multiple of 16. - pub fn cpAsyncBulkPrefetch(self: *Builder, addr: Value, bytes: anytype, opts: BulkPrefetchOptions) void { - if (@TypeOf(bytes) != Value) std.debug.assert(bytes > 0 and bytes % 16 == 0); - const address = if (addr.isInt()) self.addressPtr(addr, .global) else self.llvmPointer(addr, .global); - self.emitVoid(nvvm.ops.cp_async_bulk_prefetch(self.ctx, address.inner, self.lift(bytes).to(.i32).inner, .{ - .l2CacheHint = if (opts.l2_cache_hint) |hint| hint.to(.i64).inner else null, - }, self.loc())); - } - - pub fn prefetchL1(self: *Builder, addr: Value) void { - const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); - _ = llvm.inline_asm(self.ctx, "prefetch.global.L1 [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - pub fn prefetchTensorMap(self: *Builder, desc: Value) void { - const address = if (desc.isInt()) desc else self.ptrToInt(desc, .i64); - _ = llvm.inline_asm(self.ctx, "prefetch.tensormap [$0];", &.{address.inner}, .{ .constraints = "l", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - pub const OperandSource = union(enum) { smem: Value, tmem: Value }; - pub const Tcgen05MmaWsOptions = struct { - collector_b_buffer: ?nvvm.ops.Tcgen05MMACollectorBBuffer = null, - collector_op: ?nvvm.ops.Tcgen05MMACollectorOp = null, - col_b_zero_mask: ?Value = null, - }; - - /// Single-thread, asynchronous weight-stationary MMA, as in Python primitives. - /// The caller elects the issuer and commits/waits before consuming D. - pub fn tcgen05MmaWs(self: *Builder, kind: nvvm.Tcgen05MmaKind, d: Value, a: OperandSource, bdesc: Value, idesc: anytype, enable_input_d: Value, opts: Tcgen05MmaWsOptions) void { - std.debug.assert(kind == .f16 or kind == .tf32 or kind == .f8f6f4 or kind == .i8); - const adesc = switch (a) { - .smem => |desc| self.smemDescriptorToInt(desc), - .tmem => |ptr| if (ptr.isInt()) self.addressPtr(ptr, .tensor) else self.llvmPointer(ptr, .tensor), - }; - const dest = if (d.isInt()) self.addressPtr(d, .tensor) else self.llvmPointer(d, .tensor); - self.emitVoid(nvvm.ops.tcgen05_mma_ws(self.ctx, dest.inner, adesc.inner, self.smemDescriptorToInt(bdesc).inner, self.lift(idesc).to(.i32).inner, enable_input_d.to(.i1).inner, (nvvm.ops.Tcgen05MMAKindAttr.get(self.ctx, .{ .value = kind }) catch unreachable).attribute(), .{ - .collectorBBuffer = if (opts.collector_b_buffer) |v| (nvvm.ops.Tcgen05MMACollectorBBufferAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, - .collectorOp = if (opts.collector_op) |v| (nvvm.ops.Tcgen05MMACollectorOpAttr.get(self.ctx, .{ .value = v }) catch unreachable).attribute() else null, - .zeroColMask = if (opts.col_b_zero_mask) |v| v.to(.i64).inner else null, - }, self.loc())); - } - - pub fn makeUmmaSmemDesc(self: *Builder, src: Value, layout: LayoutSpec, major: OperandMajorMode, next_src: ?Value) Value { - return self.emit(cute_nvgpu.make_umma_smem_desc(self.ctx, src.inner, self.smemDescType(), self.layoutAttr(layout).attribute(), (cute_nvgpu.MajorModeAttr.get(self.ctx, .{ .value = switch (major) { - .k => .k, - .mn => .mn, - } }) catch unreachable).attribute(), .{ .nextSrc = if (next_src) |v| v.inner else null }, self.loc())); - } - - pub fn intToSmemDescriptor(self: *Builder, value: anytype) Value { - return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{self.lift(value).to(.i64).inner}, &.{self.smemDescType()}, self.loc())); - } - - pub fn smemDescriptorToInt(self: *Builder, desc: Value) Value { - if (desc.isInt()) return desc.to(.i64); - return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{desc.inner}, &.{mlir.Type.int(self.ctx, .i64)}, self.loc())); - } - - pub fn setMaxRegisterIncrease(self: *Builder, comptime count: u32) void { - self.setMaxRegister(count, .increase); - } - - pub fn setMaxRegisterDecrease(self: *Builder, comptime count: u32) void { - self.setMaxRegister(count, .decrease); - } - - fn setMaxRegister(self: *Builder, comptime count: u32, comptime action: enum { increase, decrease }) void { - comptime std.debug.assert(count >= 24 and count <= 256 and count % 8 == 0); - const text = std.fmt.comptimePrint("setmaxnreg.{s}.sync.aligned.u32 {d};", .{ if (action == .increase) "inc" else "dec", count }); - _ = llvm.inline_asm(self.ctx, text, &.{}, .{ .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - /// Specialized busy wait for hand-scheduled kernels; never hardware-suspends. - pub fn mbarrierSpin(self: *Builder, bar: Value, parity: Value, scope: MBarrierScope) void { - std.debug.assert(bar.dtype() == .i32 and parity.dtype() == .i32); - _ = llvm.inline_asm(self.ctx, switch (scope) { - .cta => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cta.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", - .cluster => "{ .reg .pred p; spin${:uid}: mbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 p, [$0], $1; @!p bra spin${:uid}; }", - }, &.{ bar.inner, parity.inner }, .{ .constraints = "r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - /// Convert a CuTe or LLVM pointer to the LLVM pointer required by NVVM. - fn llvmPointer(self: *Builder, ptr: Value, comptime space: NVPTXAddressSpace) Value { - if (!ptr.isPtr()) return ptr; - return self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{ptr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); - } - - /// Map a local shared pointer/address to a typed pointer in peer CTA memory. - pub fn mapDsmemPtr(self: *Builder, smem: Value, peer_cta_rank: Value) Value { - const ptr = if (smem.isInt()) self.addressPtr(smem, .shared) else self.llvmPointer(smem, .shared); - const remote = self.emit(nvvm.mapa(self.ctx, ptr.inner, peer_cta_rank.to(.i32).inner, llvm.pointerType(self.ctx, NVPTXAddressSpace.shared_cluster), self.loc())); - if (smem.type_().isA(cute_ir.PtrType)) |ty| { - std.debug.assert(ty.addressSpace() == .smem); - const result_type = (cute_ir.PtrType.get(self.ctx, .{ - .valueType = ty.valueType(), - .addressSpace = .dsmem, - .alignment = ty.alignment(), - .swizzle = ty.swizzle(), - .bitlayout = ty.bitlayout(), - }) catch @panic("invalid DSMEM pointer")).type_(); - return self.emit(cute_ir.inttoptr(self.ctx, self.ptrToInt(remote, .i32).inner, result_type, self.loc())); - } - return remote; - } - - /// Store one, two, or four i32 words in a peer CTA, completing bytes on its - /// mbarrier. Both inputs name local shared memory; both are mapped to the peer. - pub fn storeAsyncDsmem(self: *Builder, smem: Value, value: anytype, mbar: Value, peer_cta_rank: Value) void { - const n = if (@TypeOf(value) == Value) 1 else common.arity(@TypeOf(value), "storeAsyncDsmem: value"); - comptime std.debug.assert(n == 1 or n == 2 or n == 4); - if (smem.type_().isA(cute_ir.PtrType)) |ty| { - std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % (n * 4) == 0); - } - if (mbar.type_().isA(cute_ir.PtrType)) |ty| { - std.debug.assert(ty.addressSpace() == .smem and ty.alignment() % 8 == 0); - } - const packed_value = if (@TypeOf(value) == Value) value else blk: { - var words: [n]Value = undefined; - inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); - break :blk self.joinWords(n, words); - }; - self.emitVoid(nvvm.ops.store_async(self.ctx, self.llvmPointer(self.mapDsmemPtr(smem, peer_cta_rank), .shared_cluster).inner, packed_value.inner, self.llvmPointer(self.mapDsmemPtr(mbar, peer_cta_rank), .shared_cluster).inner, self.loc())); - } - - /// Explicit aligned variants retain the convergence requirement of raw PTX. - pub fn barrierClusterArriveRelaxedAligned(self: *Builder) void { - _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed.aligned;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - pub fn barrierClusterWaitAligned(self: *Builder) void { - self.emitVoid(nvvm.cluster_wait(self.ctx, true, self.loc())); - } - - pub const ClusterArriveOptions = struct { aligned: bool = false }; - - pub fn clusterArriveRelaxed(self: *Builder, opts: ClusterArriveOptions) void { - if (opts.aligned) return self.barrierClusterArriveRelaxedAligned(); - _ = llvm.inline_asm(self.ctx, "barrier.cluster.arrive.relaxed;", &.{}, .{ .constraints = "~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - } - - pub fn clusterWait(self: *Builder) void { - self.emitVoid(nvvm.cluster_wait(self.ctx, false, self.loc())); - } - - /// A null mask selects CuTe's FULL_MASK; integers and SSA masks also work. - pub fn voteAnySync(self: *Builder, pred: Value, mask: anytype) Value { - const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); - return self.callIntrinsic("llvm.nvvm.vote.any.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i1)).?; - } - - /// A null mask selects CuTe's FULL_MASK. - pub fn voteBallotSync(self: *Builder, pred: Value, mask: anytype) Value { - const active = if (@TypeOf(mask) == @TypeOf(null)) self.constant(.i32, -1) else self.lift(mask).to(.i32); - return self.callIntrinsic("llvm.nvvm.vote.ballot.sync", &.{ active, pred }, mlir.Type.int(self.ctx, .i32)).?; - } - - pub const FmaxOptions = struct { abs: bool = false, nan: bool = false, ftz: bool = false }; - - pub fn fmax(self: *Builder, x: Value, y: Value, opts: FmaxOptions) Value { - if (!opts.abs and !opts.nan and !opts.ftz) return self.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, mlir.Type.float(self.ctx, .f32)).?; - return self.emit(nvvm.ops.fmax(self.ctx, x.inner, y.inner, mlir.Type.float(self.ctx, .f32), .{ - .abs = if (opts.abs) .unit(self.ctx) else null, - .nan = if (opts.nan) .unit(self.ctx) else null, - .ftz = if (opts.ftz) .unit(self.ctx) else null, - }, self.loc())); - } - - fn vec2(self: *Builder, x: [2]Value) Value { - const ty = mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)); - return self.emit(vector.from_elements(self.ctx, &.{ x[0].inner, x[1].inner }, ty, self.loc())); - } - - fn unvec2(self: *Builder, v: Value) [2]Value { - return .{ self.vectorExtract(v, 0, .f32), self.vectorExtract(v, 1, .f32) }; - } - - pub const PackedF32x2Options = struct { rnd: nvvm.ops.FPRoundingMode = .rn, ftz: bool = false }; - - fn packedF32x2Attrs(self: *Builder, opts: PackedF32x2Options) struct { rnd: ?*const mlir.Attribute, ftz: ?*const mlir.Attribute } { - return .{ - .rnd = (nvvm.ops.FPRoundingModeAttr.get(self.ctx, .{ .value = opts.rnd }) catch unreachable).attribute(), - .ftz = if (opts.ftz) .unit(self.ctx) else null, - }; - } - - pub fn fmaPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, c: [2]Value, opts: PackedF32x2Options) [2]Value { - return self.unvec2(self.emit(nvvm.ops.fma_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, self.vec2(c).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); - } - - pub fn addPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { - return self.unvec2(self.emit(nvvm.ops.add_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); - } - - pub fn mulPackedF32x2(self: *Builder, a: [2]Value, b: [2]Value, opts: PackedF32x2Options) [2]Value { - return self.unvec2(self.emit(nvvm.ops.mul_packed_f32x2(self.ctx, self.vec2(a).inner, self.vec2(b).inner, mlir.Type.vector(&.{2}, mlir.Type.float(self.ctx, .f32)), .{ .rnd = self.packedF32x2Attrs(opts).rnd, .ftz = self.packedF32x2Attrs(opts).ftz }, self.loc()))); - } - - pub const StoreOptions = struct { - /// Address space for raw integer addresses; typed pointers retain theirs. - space: NVPTXAddressSpace = .global, - level1_eviction_priority: nvvm.ops.EvictKind = .evict_normal, - }; - - /// Scalar/vector store. Eight i32 words with no_allocate keep STG.E.256. - pub fn store(self: *Builder, addr: Value, value: anytype, comptime opts: StoreOptions) void { - if (@TypeOf(value) != Value) { - const n = common.arity(@TypeOf(value), "store: value"); - comptime std.debug.assert(n == 1 or n == 2 or n == 4 or n == 8); - var words: [n]Value = undefined; - inline for (0..n) |i| words[i] = self.lift(value[i]).to(.i32); - const is_global = if (addr.type_().isA(cute_ir.PtrType)) |ty| ty.addressSpace() == .gmem else addr.isInt() and opts.space == .global; - if (n == 8 and is_global and opts.level1_eviction_priority == .evict_no_allocate) { - const address = if (addr.isInt()) addr else self.ptrToInt(addr, .i64); - var args: [9]*const mlir.Value = undefined; - args[0] = address.inner; - for (words, 0..) |word, i| args[i + 1] = word.inner; - _ = llvm.inline_asm(self.ctx, "st.global.L1::no_allocate.v8.b32 [$0], {$1, $2, $3, $4, $5, $6, $7, $8};", &args, .{ .constraints = "l,r,r,r,r,r,r,r,r,~{memory}", .has_side_effects = true }, self.loc()).appendTo(self.currentBlock()); - return; - } - self.store(addr, self.joinWords(n, words), opts); - return; - } - const ptr = if (addr.isInt()) self.addressPtr(addr, opts.space) else if (addr.type_().isA(cute_ir.PtrType)) |ty| blk: { - const space: NVPTXAddressSpace = @fromBackingInt(@intCast(@backingInt(ty.addressSpace()))); - break :blk self.emit(mlir.unrealized_conversion_cast(self.ctx, &.{addr.inner}, &.{llvm.pointerType(self.ctx, space)}, self.loc())); - } else addr; - self.emitVoid(nvvm.ops.store_ext(self.ctx, value.inner, ptr.inner, .{ - .evict = if (opts.level1_eviction_priority == .evict_normal) null else (nvvm.ops.EvictKindAttr.get(self.ctx, .{ .value = opts.level1_eviction_priority }) catch unreachable).attribute(), - }, self.loc())); - } - - pub fn fma(self: *Builder, x: Value, y: Value, z: Value) Value { - return self.callIntrinsic("llvm.fma.f32", &.{ x, y, z }, mlir.Type.float(self.ctx, .f32)).?; - } - - pub fn clock(self: *Builder) Value { - return self.emit(llvm.inline_asm(self.ctx, "mov.u32 $0, %clock;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i32), .constraints = "=r", .has_side_effects = true }, self.loc())); - } - - pub fn globalTimer(self: *Builder) Value { - return self.emit(llvm.inline_asm(self.ctx, "mov.u64 $0, %globaltimer;", &.{}, .{ .result_type = mlir.Type.int(self.ctx, .i64), .constraints = "=l", .has_side_effects = true }, self.loc())); - } - - // ==================== scalars ==================== - - /// `value` as a constant of `dt`; a Value passes through and enums use - /// their backing integers. An integer is signed or an unsigned bit - /// pattern of `dt`'s width; anything else panics. - pub fn constant(self: *Builder, dt: DType, value: anytype) Value { - if (@TypeOf(value) == Value) return value; - if (@typeInfo(@TypeOf(value)) == .@"enum") return self.constant(dt, @backingInt(value)); - const ctx = self.ctx; - const T = @TypeOf(value); - if (isFloatDtype(dt)) { - const f: f64 = switch (@typeInfo(T)) { - .comptime_int, .comptime_float => value, - .int => @floatFromInt(value), - .bool => @floatFromInt(@intFromBool(value)), - .float => @floatCast(value), - else => @compileError("Builder.constant: unsupported value type " ++ @typeName(T)), - }; - return self.emit(switch (dt) { - inline .f16, .bf16, .f32, .f64, .f4e2m1fn, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => |ft| arith.constant_float(ctx, f, @field(mlir.FloatTypes, @tagName(ft)), self.loc()), - else => unreachable, - }); - } - const i: i128 = switch (@typeInfo(T)) { - .comptime_int, .int => value, - .bool => @intFromBool(value), - .comptime_float, .float => @intFromFloat(value), - else => @compileError("Builder.constant: unsupported value type " ++ @typeName(T)), - }; - return self.emit(arith.constant_int(ctx, intBits(i, @intCast(dtypeBitwidth(dt)), @tagName(dt)), dt.toMlir(ctx), self.loc())); - } - - /// `value` as the `i64` an integer constant of `bits` bits is built from. - /// It may be signed or an unsigned bit pattern, so it must lie in - /// [-2^(bits-1), 2^bits). - fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { - if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) - std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); - return @truncate(value); - } - - fn fitsInt(i: i64, dt: DType) bool { - if (dt == .i1) return i == 0 or i == 1; - const bits = dtypeBitwidth(dt); - if (bits >= 64) return true; - const shift: u6 = @intCast(bits - 1); - return i >= -(@as(i64, 1) << shift) and i < (@as(i64, 1) << shift); - } - - fn literalFits(value: anytype, dt: DType) bool { - return switch (@typeInfo(@TypeOf(value))) { - .comptime_int, .int => fitsInt(@intCast(value), dt), - .bool => !isFloatDtype(dt), - else => isFloatDtype(dt), - }; - } - - /// A Zig scalar as a constant: ints become `i32` when they fit, `i64` - /// otherwise; floats `f32`, like the Python DSL's `Int32`/`Float32`. - pub fn lift(self: *Builder, value: anytype) Value { - const T = @TypeOf(value); - if (T == Value) return value; - return switch (@typeInfo(T)) { - .comptime_int, .int => if (literalFits(value, .i32)) self.constant(.i32, value) else self.constant(.i64, value), - .bool => self.constant(.i1, value), - .comptime_float => self.constant(.f32, value), - .float => |info| switch (info.bits) { - 16 => self.constant(.f16, value), - 32 => self.constant(.f32, value), - 64 => self.constant(.f64, value), - else => @compileError("Builder.lift: unsupported float bitwidth"), - }, - else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), - }; - } - - /// `value` as a constant of `like`'s element type; a Value passes through. - pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { - return self.constant(like.dtype(), value); - } - - fn liftLike(self: *Builder, value: anytype, ref: Value) Value { - const dt = ref.dtype(); - return if (literalFits(value, dt)) self.constant(dt, value) else self.lift(value); - } - - /// Lift a literal to the other operand's dtype when it fits, else to its - /// own width; then widen the narrower integer. - pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { - var av: Value = if (@TypeOf(a) == Value) a else if (@TypeOf(b) == Value) self.liftLike(a, b) else self.lift(a); - var bv: Value = if (@TypeOf(b) == Value) b else self.liftLike(b, av); - if (av.isInt() and bv.isInt()) { - const aw = dtypeBitwidth(av.dtype()); - const bw = dtypeBitwidth(bv.dtype()); - if (aw < bw) av = av.to(bv.dtype()); - if (bw < aw) bv = bv.to(av.dtype()); - } - return .{ av, bv }; - } - - fn cmp(self: *Builder, a: Value, b: anytype, ip: arith.CmpIPredicate, fp: arith.CmpFPredicate) Value { - const l, const r = self.coerce(a, b); - return self.emit(if (l.isFloat()) arith.cmpf(self.ctx, fp, l.inner, r.inner, self.loc()) else arith.cmpi(self.ctx, ip, l.inner, r.inner, self.loc())); - } - - pub fn select(self: *Builder, cond: Value, t: anytype, f: anytype) Value { - const tv, const fv = self.coerce(t, f); - return self.emit(arith.select(self.ctx, cond.inner, tv.inner, fv.inner, self.loc())); - } - - // ==================== scf ==================== - - pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { - return dtype.toMlir(self.ctx); - } - - pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { - return common.openIf(self, Value, cond); - } - - pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { - return common.openIfElse(self, Value, cond, result_types); - } - - /// `for iv in range(lower, upper, step)` with loop-carried `inits`. Literal - /// bounds take the type of a Value bound, else `lower` is lifted (`i32` - /// when it fits). - pub fn openFor( - self: *Builder, - lower: anytype, - upper: anytype, - step: anytype, - inits: anytype, - ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { - return self.openForOpts(lower, upper, step, inits, .{}); - } - - /// `openFor` with `scf.for` options. - pub fn openForOpts( - self: *Builder, - lower: anytype, - upper: anytype, - step: anytype, - inits: anytype, - opts: common.ForOpts, - ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { - return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.lift), inits, opts); - } - - /// `scf.while` with explicit loop-carried values. The before region sees - /// `inits`, calls `yieldBefore(condition, forwarded)`, then the after - /// region sees `forwarded` and calls `yieldAfter(next_inits)`. This is the - /// control-flow shape emitted by the Python CuTe persistent scheduler. - pub fn openWhile( - self: *Builder, - inits: anytype, - after_types: anytype, - ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { - return common.openWhile(self, Value, inits, after_types); - } - - // ==================== module ==================== - - /// The module the CuTe compiler takes: the verified kernel as a public - /// `func.func`, which the compiler turns into the kernel entry. Grid and - /// block travel in the custom call; `block` is also pinned as - /// `nvvm.reqntid`. - pub fn finish(self: *Builder, block: [3]i32) FinishError![:0]const u8 { - const current = self.currentBlock(); - if (current.terminator() == null) { - _ = func.returns(self.ctx, &.{}, self.loc()).appendTo(current); - } - const func_op = self.func_op orelse return error.InvalidMlir; - func_op.setAttribute(nvvm.reqntid(self.ctx, block)); - try common.verify(self.module, "dsl/cute"); - - return self.renderModule(); - } -}; - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - registerDialects(registry); - func.registerExtensions(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -fn expectContains(haystack: []const u8, needle: []const u8) !void { - if (std.mem.indexOf(u8, haystack, needle) == null) { - std.debug.print("missing {s} in:\n{s}\n", .{ needle, haystack }); - return error.TestExpectedContains; - } -} - -test "naive elementwise add matches the Python notebook kernel" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "naive_elementwise_add_kernel"); - defer b.deinit(); - const a = try b.declareArgs(.{ - .gA = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, - .gB = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, - .gC = .{ .tensor = .{ .dtype = .f16, .shape = &.{ 16, 8 } } }, - }); - - const tidx = b.threadIdx().x; - const bidx = b.blockIdx().x; - const bdim = b.blockDim().x; - const thread_idx = bidx.mul(bdim).add(tidx); - const n = a.gA.dim(1); - const ni = thread_idx.rem(n); - const mi = thread_idx.div(n); - a.gC.set(.{ mi, ni }, a.gA.get(.{ mi, ni }).add(a.gB.get(.{ mi, ni }))); - - // The module round-trips through the local dialects. - const ir = try b.finish(.{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try expectContains(ir, "\"func.func\"() <{function_type = (!cute.ptr>, !cute.ptr>, !cute.ptr>) -> (), sym_name = \"naive_elementwise_add_kernel\"}>"); - try expectContains(ir, "}) {cute.kernel, gpu.kernel, nvvm.reqntid = array} : () -> ()"); - try expectContains(ir, "\"nvvm.read.ptx.sreg.tid.x\"() : () -> i32"); - try expectContains(ir, "!cute.layout<\"(16,8):(8,1)\">"); - try expectContains(ir, "!cute.coord<\"(?,?)\">"); - try expectContains(ir, "cute.memref.load"); - try expectContains(ir, "cute.memref.store"); - try expectContains(ir, "arith.addf"); - try expectContains(ir, "\"func.return\"() : () -> ()\n })"); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "guard, shared memory, sync, layouts, for loop" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "reverse_blocks"); - defer b.deinit(); - const a = try b.declareArgs(.{ - .x = .{ .ptr = .f32 }, - .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{4096} } }, - }); - const gx = b.makeTensor(a.x, b.makeLayout(&.{4096}, .{})); - const smem = b.makeTensor(b.allocSmem(.f32, 128, 16), b.makeLayout(&.{128}, .{})); - - const tid = b.threadIdx().x; - const i = b.blockIdx().x.mul(128).add(tid); - smem.set(.{tid}, gx.get(.{i})); - b.syncThreads(); - b.barrierCtaSyncAligned(2, 128); - b.launchDependents(); - b.waitForDependency(); - const j = b.constant(.i32, 127).sub(tid); - var guard = b.openIf(i.lt(4096)); - { - var acc = b.openFor(0, 4, 1, .{b.constant(.f32, 0.0)}); - { - acc.yield(.{acc.carried[0].add(smem.get(.{j}).mul(acc.iv.to(.f32)))}); - } - a.out.set(.{i}, acc.results[0]); - guard.close(); - } - - const kernel = try b.finish(.{ 128, 1, 1 }); - defer std.testing.allocator.free(kernel); - try expectContains(kernel, "cute_nvgpu.arch.alloc_smem"); - try expectContains(kernel, "!cute.ptr>"); - try expectContains(kernel, "\"nvvm.barrier\"() <{operandSegmentSizes = array}> : () -> ()"); - try expectContains(kernel, "llvm.nvvm.barrier.cta.sync.aligned.count"); - try expectContains(kernel, "value = 2 : i32"); - try expectContains(kernel, "value = 128 : i32"); - try expectContains(kernel, "griddepcontrol.launch_dependents;"); - try expectContains(kernel, "griddepcontrol.wait;"); - try expectContains(kernel, "scf.if"); - try expectContains(kernel, "scf.for"); - try expectContains(kernel, "!cute.layout<\"4096:1\">"); - const parsed = try mlir.Module.parse(ctx, kernel); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "left-most default stride and static coordinate leaves" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "k"); - defer b.deinit(); - const a = try b.declareArgs(.{ .p = .{ .ptr = .f32 } }); - const t = b.makeTensor(a.p, b.makeLayout(&.{ 4, 4 }, .{})); - try std.testing.expectEqualSlices(i64, &.{ 1, 4 }, t.layout.stride.constSlice()); - try std.testing.expectEqual(@as(i64, 16), t.size()); - const v = t.get(.{ b.threadIdx().x, 3 }); - try std.testing.expect(v.isFloat()); - const kernel = try b.finish(.{ 1, 1, 1 }); - defer std.testing.allocator.free(kernel); - try expectContains(kernel, "!cute.layout<\"(4,4):(1,4)\">"); - try expectContains(kernel, "!cute.coord<\"(?,3)\">"); - const parsed = try mlir.Module.parse(ctx, kernel); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "casts, select, if-else, integer widening" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "k"); - defer b.deinit(); - const a = try b.declareArgs(.{ .p = .{ .tensor = .{ .dtype = .f16, .shape = &.{8} } } }); - const tid = b.threadIdx().x; - - // A literal that does not fit i32 widens the i32 Value instead of truncating. - const big: u32 = 3_000_000_000; - const wide = tid.add(big); - try std.testing.expectEqual(DType.i64, wide.dtype()); - const bound = tid.lt(@as(i64, 5_000_000_000)); - try std.testing.expectEqual(DType.i1, bound.dtype()); - - const f = tid.to(.f32); - const shuffled = b.shuffleXor(f, 16); - try std.testing.expectEqual(DType.f16, f.to(.f16).dtype()); - try std.testing.expectEqual(DType.f64, f.to(.f64).dtype()); - try std.testing.expectEqual(DType.bf16, f.to(.f16).to(.bf16).dtype()); - try std.testing.expectEqual(DType.i32, f.to(.i32).dtype()); - try std.testing.expectEqual(DType.i8, tid.to(.i8).dtype()); - try std.testing.expectEqual(DType.i32, bound.to(.i32).dtype()); - - const picked = b.select(bound, shuffled, 2.0); - var scope = b.openIfElse(tid.lt(4), .{.f16}); - { - scope.yieldThen(.{picked.to(.f16)}); - } - { - scope.yieldElse(.{b.constant(.f16, 1)}); - } - a.p.set(.{tid}, scope.results[0]); - - const kernel = try b.finish(.{ 1, 1, 1 }); - defer std.testing.allocator.free(kernel); - try expectContains(kernel, "arith.extsi"); - try expectContains(kernel, "\"arith.constant\"() <{value = 3000000000 : i64}> : () -> i64"); - try expectContains(kernel, "arith.sitofp"); - try expectContains(kernel, "arith.truncf"); - try expectContains(kernel, "arith.extf"); - try expectContains(kernel, "arith.convertf"); - try expectContains(kernel, "arith.fptosi"); - try expectContains(kernel, "arith.trunci"); - try expectContains(kernel, "arith.extui"); - try expectContains(kernel, "arith.select"); - try expectContains(kernel, "shfl.sync.bfly.b32"); - try expectContains(kernel, "\"scf.if\""); - try expectContains(kernel, "}, {"); - const parsed = try mlir.Module.parse(ctx, kernel); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "complete program emits a Zig-built CUDA host launch" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.openProgram(std.testing.allocator, ctx, "program_add"); - defer b.deinit(); - const device = try b.declareArgs(.{ - .input = .{ .tensor = .{ .dtype = .f32, .shape = &.{128} } }, - .output = .{ .tensor = .{ .dtype = .f32, .shape = &.{128} } }, - }); - const tid = b.threadIdx().x; - device.output.set(.{tid}, device.input.get(.{tid}).add(1.0)); - b.endFunction(.{ 128, 1, 1 }); - - b.beginFunction("program_add", .host); - _ = try b.declareArgs(.{ - .input = .{ .ptr = DType.f32 }, - .output = .{ .ptr = DType.f32 }, - }); - const one = b.constant(.i32, 1); - const config = b.makeLaunchConfig(.{ - .grid = .{ one, one, one }, - .block = .{ b.constant(.i32, 128), one, one }, - .dynamic_smem = b.kernelSmemSize("program_add"), - .stream = b.cudaStream(), - .use_pdl = true, - }); - const launched = b.launchEx("program_add", config, .{ b.arg(0), b.arg(1) }); - b.returnHostStatus(b.cudaResultStatus(launched)); - b.endFunction(null); - - const ir = try b.finishProgram(); - defer std.testing.allocator.free(ir); - try expectContains(ir, "\"gpu.module\""); - try expectContains(ir, "cuda.kernel"); - try expectContains(ir, "sym_name = \"program_add\""); - try expectContains(ir, "cuda.launch_ex"); -} - -test "SM100 block-scaled MMA atoms use the CuTe DSL instruction shapes" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - registerDialects(registry); - func.registerExtensions(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - - var b = try Builder.open(std.testing.allocator, ctx, "sm100_blockscaled_atoms"); - defer b.deinit(); - const tma: TmaConfig = .{ - .dtype = .f8e4m3fn, - .copy_bits = 16384, - .global_basis = b.layoutSpec(.{ 128, 16, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }), - .coordinate_layout = b.layoutSpec(.{ 16, 128, 8 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}), b.basis(1, 1, .{2}) }), - .coordinate_rank = 3, - }; - var pair_tma = tma; - pair_tma.cta_group = .two; - const args = try b.declareArgs(.{ - .single = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(tma), .grid_constant = true } }, - .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair_tma), .grid_constant = true } }, - }); - _ = b.makeExecTma(.{ .inner = args.single.inner, .kernel = &b }, b.tmaLoadExecAtomType(tma)); - _ = b.makeExecTma(.{ .inner = args.pair.inner, .kernel = &b }, b.tmaLoadExecAtomType(pair_tma)); - - const sf_ptr = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); - for ([_]CtaGroup{ .one, .two }) |group| { - _ = b.makeBlockScaledMma(.{ .n = 16, .cta_group = group }, sf_ptr); - _ = b.makeBlockScaledMma(.{ .n = 128, .cta_group = group }, sf_ptr); - } - _ = b.makeBlockScaledMma(.{ .m = 256, .n = 16, .cta_group = .two }, sf_ptr); - _ = b.makeBlockScaledMma(.{ .m = 256, .n = 128, .cta_group = .two }, sf_ptr); - const nvfp4_sf_ptr = b.intToPtr(0, .f8e4m3fn, .tmem, 1); - _ = b.makeBlockScaledMma(.{ .kind = .nvfp4_nvfp4, .n = 16 }, nvfp4_sf_ptr); - - const ir = try b.finish(.{ 192, 1, 1 }); - defer std.testing.allocator.free(ir); - try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 1"); - try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 1"); - try expectContains(ir, "sm100.mma_bs<128x16x32, num_cta = 2"); - try expectContains(ir, "sm100.mma_bs<128x128x32, num_cta = 2"); - try expectContains(ir, "sm100.mma_bs<256x16x32, num_cta = 2"); - try expectContains(ir, "sm100.mma_bs<256x128x32, num_cta = 2"); - try expectContains(ir, "sm100.mma_bs<128x16x64, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>"); - try expectContains(ir, "non_exec_tiled_tma_load"); - try expectContains(ir, "value = 3 : i32"); - try expectContains(ir, "value = 32 : i32"); - try expectContains(ir, "!llvm.ptr<7>"); - try expectContains(ir, "#nvvm.mem_scope"); - try expectContains(ir, "#nvvm.mem_scope"); - - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "mbarrier waits preserve acquire scope through polling and blocking retries" { - const ctx = try testContext(); - defer ctx.deinit(); - for ([_]MBarrierScope{ .cta, .cluster }) |scope| { - var b = try Builder.open(std.testing.allocator, ctx, "barrier_waits"); - defer b.deinit(); - _ = try b.declareArgs(.{}); - - const barrier = b.allocSmem(.i64, 1, 8); - const phase = b.constant(.i32, 0); - _ = b.mbarrierTestWait(barrier, phase, .{ .scope = scope }); - _ = b.mbarrierTryWait(barrier, phase, .{ .scope = scope }); - b.mbarrierWait(barrier, phase, .{ .scope = scope, .time_limit = 1234, .test_first = false }); - b.mbarrierWait(barrier, phase, .{ .scope = scope }); - - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mbar_wait")); - try std.testing.expectEqual(@as(usize, 1), std.mem.count(u8, ir, "#nvvm.mbar_wait")); - try expectContains(ir, "value = 1234 : i32"); - try expectContains(ir, "\"scf.if\""); - switch (scope) { - .cta => { - try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "#nvvm.mbar_scope")); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity\"")); - try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "\"scf.while\"")); - }, - .cluster => { - try std.testing.expectEqual(@as(usize, 5), std.mem.count(u8, ir, "#nvvm.mbar_scope")); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "#nvvm.mem_order")); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"nvvm.mbarrier.try_wait.parity.timelimit\"")); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.while\"")); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"scf.condition\"")); - try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "#nvvm.mbar_scope")); - }, - } - - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); - } -} - -test "SM100 raw primitives preserve operand types and round-trip through MLIR" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "sm100_raw_primitives"); - defer b.deinit(); - const cfg: TmaConfig = .{ - .dtype = .f8e4m3fn, - .copy_bits = 1024, - .global_basis = b.layoutSpec(.{ 128, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_layout = b.layoutSpec(.{ 1024, 16 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_rank = 2, - }; - const a = try b.declareArgs(.{ - .gmem = .{ .ptr = DType.i32 }, - .tma = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, - }); - const atom = b.makeExecTma(.{ .inner = a.tma.inner, .kernel = &b }, b.tmaLoadExecAtomType(cfg)); - const desc = b.tmaDescriptorAddress(atom); - const dst = b.constant(.i32, 1024); - const bar = b.constant(.i32, 0); - const zero = b.constant(.i32, 0); - const gmem = b.ptrToInt(a.gmem, .i64); - const pred = b.threadIdx().x.eq(0); - b.tmaGather4(desc, dst, bar, zero, .{ zero, zero.add(1), zero.add(2), zero.add(3) }, .evict_last); - b.cpAsyncBulkPrefetch(gmem, b.constant(.i32, 128), .{}); - b.prefetchL1(gmem); - b.prefetchTensorMap(desc); - const swizzled = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .layout = .swizzle_128b }); - const interleaved = Tcgen05SmemDesc.build(&b, dst, .{ .leading_byte_offset = 1024, .stride_byte_offset = 128 }); - b.tcgen05MmaWs(.f16, zero, .{ .tmem = dst }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k }), pred, .{}); - b.tcgen05MmaWs(.f16, zero, .{ .smem = interleaved }, swizzled, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn }), pred, .{}); - b.setMaxRegisterIncrease(128); - b.barrierCtaSyncAligned(0, 512); - b.setMaxRegisterDecrease(128); - b.barrierCtaSyncAligned(b.threadIdx().x.div(128), 128); - b.mbarrierSpin(bar, zero, .cta); - b.mbarrierSpin(bar, zero, .cluster); - const state = b.mbarrierArrive(bar, .{}).?; - const remote = b.mapDsmemPtr(dst, zero); - _ = remote; - _ = b.mbarrierArrive(bar, .{ .peer_cta_rank_in_cluster = zero, .arrive_count = b.constant(.i32, 1), .scope = .cluster }); - b.storeAsyncDsmem(dst, .{ zero, zero }, bar, zero); - b.storeAsyncDsmem(dst, .{ zero, zero, zero, zero }, bar, zero); - b.barrierClusterArriveRelaxedAligned(); - b.barrierClusterWaitAligned(); - const any = b.voteAnySync(pred, null); - const mask = b.voteBallotSync(pred, null); - const x = b.constant(.f32, 1.0); - const y = b.constant(.f32, 2.0); - const maximum = b.fmax(x, y, .{}); - const sum = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{}); - const product = b.mulPackedF32x2(sum, .{ x, y }, .{}); - const fused = b.fmaPackedF32x2(product, sum, .{ x, y }, .{}); - const scalar = b.fma(maximum, fused[0], fused[1]); - b.store(gmem, .{ scalar.bitCast(.i32), maximum.bitCast(.i32), mask, any.to(.i32), state.to(.i32), b.clock(), b.globalTimer().to(.i32), zero }, .{ .level1_eviction_priority = .evict_no_allocate }); - - const ir = try b.finish(.{ 512, 1, 1 }); - defer std.testing.allocator.free(ir); - for ([_][]const u8{ - "cute_nvgpu.get_tma_desc_addr", "tile::gather4", "nvvm.cp.async.bulk.prefetch", - "prefetch.global.L1", "prefetch.tensormap", "nvvm.tcgen05.mma.ws", - "nvvm.tcgen05.mma.ws", "setmaxnreg.inc", "setmaxnreg.dec", - "llvm.nvvm.barrier.cta.sync.aligned.count", "mbarrier.test_wait.parity.acquire.cta", "mbarrier.test_wait.parity.acquire.cluster", - "nvvm.mapa", "#nvvm.mem_scope", "nvvm.store.async", - "vector<4xi32>", "barrier.cluster.arrive.relaxed.aligned", "nvvm.cluster.wait", - "llvm.nvvm.vote.any.sync", "llvm.nvvm.vote.ballot.sync", "llvm.nvvm.fmax.f", - "nvvm.add.packed.f32x2", "nvvm.mul.packed.f32x2", "nvvm.fma.packed.f32x2", - "llvm.fma.f32", "st.global.L1::no_allocate.v8.b32", "%clock", - "%globaltimer", - }) |op| try expectContains(ir, op); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "SM100 BF16 weight-stationary descriptor encodes supported shapes and major modes" { - try std.testing.expectEqual(@as(u32, 0x04200490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128, .a_major = .k, .b_major = .k })); - try std.testing.expectEqual(@as(u32, 0x04410490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 256, .a_major = .k, .b_major = .mn })); - try std.testing.expectEqual(@as(u32, 0x08118490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 128, .n_dim = 64, .a_major = .mn, .b_major = .mn })); - try std.testing.expectEqual(@as(u32, 0x02100490), Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 32, .n_dim = 64, .a_major = .k, .b_major = .k })); -} - -test "SM100 shared descriptors preserve nonzero swizzle bases and boundary offsets" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "smem_descriptors"); - defer b.deinit(); - const args = try b.declareArgs(.{ .out = .{ .ptr = DType.i64 } }); - const addr = b.constant(.i32, 128); - const fields = [_]u64{ - 0x4000404000010000, // sw128, base 0, leading 16 bytes, stride 1024 bytes - 0x4002404000010000, // sw128, pattern at byte 128: base 1 - 0x400e404000010000, // sw128, pattern at byte 896: base 7 - 0x00007fff3fff0000, // unswizzled, largest representable offsets - }; - const descs = [_]Value{ - Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 0, .layout = .swizzle_128b }), - Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 1, .layout = .swizzle_128b }), - Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 16, .stride_byte_offset = 1024, .base_offset = 7, .layout = .swizzle_128b }), - Tcgen05SmemDesc.build(&b, addr, .{ .leading_byte_offset = 0x3fff0, .stride_byte_offset = 0x3fff0 }), - }; - const out = b.ptrToInt(args.out, .i64); - for (descs, 0..) |desc, i| b.storeWords(2, out.add(@as(i64, @intCast(i * 8))), .global, .{ desc.to(.i32), desc.shrLogical(32).to(.i32) }); - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - for (fields) |bits| { - const expected = try std.fmt.allocPrint(std.testing.allocator, "value = {d} : i64", .{bits}); - defer std.testing.allocator.free(expected); - try expectContains(ir, expected); - } - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "makeLayout preserves dynamic singleton and nested shape and stride leaves" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "mixed_layout"); - defer b.deinit(); - _ = try b.declareArgs(.{}); - const n = b.blockIdx().x.add(1); - const singleton = b.constant(.i32, 1); - const compact = b.makeLayout(.{ .{ 4, n }, singleton }, .{}); - const explicit = b.makeLayout(.{ n, 16 }, .{ .stride = .{ 1, n } }); - const static_nested = b.makeLayout(.{ .{ 4, 8 }, 16 }, .{}); - _ = compact; - _ = explicit; - _ = static_nested; - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try expectContains(ir, "((4,?),?):((1,4),?)"); - try expectContains(ir, "(?,16):(1,?)"); - try expectContains(ir, "((4,8),16):((1,4),32)"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "makeLayout preserves 64-bit shape and stride values" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "wide_layout"); - defer b.deinit(); - const args = try b.declareArgs(.{ .extent = .{ .mlir_type = .{ .type_ = DType.i64.toMlir(ctx) } } }); - const wide = args.extent.add(@as(i64, 1) << 32); - const explicit = b.makeLayout(.{ 8, 2 }, .{ .stride = .{ 1, wide } }); - const wide_leaf: AlgebraExpr = .{ .dynamic_int = .{ .width = 64 } }; - try std.testing.expect(explicit.type_().eql(b.layoutType(.{ - .shape = b.expr(.{ 8, 2 }), - .stride = .{ .tuple = &.{ .{ .static = 1 }, wide_leaf } }, - }))); - _ = b.makeLayout(.{ .{ 4, wide }, 1, 8 }, .{ .stride = null }); - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "arith.trunci") == null); - try expectContains(ir, "((4,?{i64}),1,8):((1,4),0,?{i64})"); - try expectContains(ir, "value = 4294967296 : i64"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "TMA constructors normalize tilers and staged layouts and copy executes store atoms" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "tma_stores"); - defer b.deinit(); - const cfg: TmaConfig = .{ - .dtype = .f32, - .copy_bits = 2048, - .global_basis = b.layoutSpec(.{ 16, 4 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_rank = 2, - }; - const args = try b.declareArgs(.{ - .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, - }); - const staged = b.makeLayout(.{ 16, 4, 2 }, .{ .stride = .{ 1, 16, 64 } }); - const store = b.makeTiledTmaAtom(CopyBulkTensorTileS2GOp{}, args.data, staged, .{ 16, 4 }, cfg, .{}); - const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), b.makeLayout(.{ 16, 4 }, .{})); - const tile = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.global_basis)); - b.copy(store.atom, .{smem}, .{tile}, .{}); - b.copy(store.atom, .{smem}, .{tile}, .{ .cache_policy = b.constant(.i64, CacheHintSm90.evict_last) }); - const swizzled = b.makeComposedLayout(.{ .bits = 1, .base = 4, .shift = 3 }, 0, b.layoutSpec(.{ 16, 4, 2 }, .{ 1, 16, 64 })); - _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, swizzled, b.makeTile(&.{ 16, 4 }), cfg, .{}); - _ = b.makeTiledTmaAtom(CopyBulkTensorTileG2SOp{}, args.data, staged, b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 128 } }), cfg, .{}); - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"cute_nvgpu.atom.make_exec_tma\"")); - try std.testing.expectEqual(@as(usize, 3), std.mem.count(u8, ir, "\"cute.select\"")); - try expectContains(ir, "(16,4):(1@0,1@1)"); - try expectContains(ir, "tma_store"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "CuTe options preserve runtime masks, timed try waits, descriptors and store widths" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "cute_options"); - defer b.deinit(); - const a = try b.declareArgs(.{ .out = .{ .ptr = DType.i32 } }); - const tid = b.threadIdx().x; - const mbar = b.allocSmem(.i64, 1, 8); - b.barrier(.{}); - b.barrier(.{ .barrier_id = tid.div(128), .number_of_threads = b.blockDim().x }); - b.tcgen05Commit(mbar, .{ .cta_group = .two, .multicast_mask = b.clusterCtaRank().shl(1).bitOr(1) }); - _ = b.mbarrierTryWait(mbar, b.constant(.i32, 0), .{ .time_limit = tid.add(100), .scope = .cluster }); - const state = b.mbarrierArrive(mbar, .{ .relaxed = true }).?; - b.storeAsyncDsmem(b.constant(.i32, 128), tid, mbar, b.clusterCtaRank().bitXor(1)); - b.clusterArriveRelaxed(.{}); - b.clusterWait(); - b.clusterArriveRelaxed(.{ .aligned = true }); - b.barrierClusterWaitAligned(); - _ = b.voteAnySync(tid.eq(0), tid.bitOr(1)); - _ = b.voteBallotSync(tid.eq(0), 0xff); - const x = tid.to(.f32); - const y = b.fmax(x, b.constant(.f32, 0), .{ .nan = true, .ftz = true, .abs = true }); - const pair = b.addPackedF32x2(.{ x, y }, .{ y, x }, .{ .rnd = .rz, .ftz = true }); - b.store(a.out, state.to(.i32), .{}); - b.store(a.out, .{ pair[0].bitCast(.i32), pair[1].bitCast(.i32) }, .{}); - const desc = b.makeUmmaSmemDesc(b.intToPtr(0, .bf16, .smem, 16), b.layoutSpec(.{ 64, 16 }, .{ 16, 1 }), .k, null); - const raw_desc = b.smemDescriptorToInt(desc); - _ = b.intToSmemDescriptor(raw_desc); - const mapped = b.mapDsmemPtr(b.intToPtr(128, .i32, .smem, 16), b.clusterCtaRank()); - const mapped_ty = mapped.type_().isA(cute_ir.PtrType).?; - try std.testing.expectEqual(MemorySpace.dsmem, mapped_ty.addressSpace()); - try std.testing.expectEqual(@as(u64, 16), mapped_ty.alignment()); - b.store(mapped, tid, .{}); - const swizzles = [_]Tcgen05SmemSwizzle{ .none, .swizzle_32b, .swizzle_64b, .swizzle_128b, .swizzle_128b_atom_32b }; - for (swizzles) |layout| _ = Tcgen05SmemDesc.build(&b, tid.mul(16), .{ .layout = layout }); - const idesc = b.constant(.i32, Tcgen05InstrDesc.build(.{ .a_dtype = .bf16, .b_dtype = .bf16, .m_dim = 64, .n_dim = 128 })); - b.tcgen05MmaWs(.f16, tid, .{ .tmem = tid }, desc, idesc, tid.ne(0), .{ .collector_b_buffer = .b2, .collector_op = .use, .col_b_zero_mask = state }); - b.cpAsyncBulkPrefetch(a.out, 128, .{ .l2_cache_hint = state }); - const ir = try b.finish(.{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try expectContains(ir, "nvvm.mbarrier.try_wait.parity.timelimit"); - try std.testing.expectEqual(@as(usize, 0), std.mem.count(u8, ir, "scf.while")); - try expectContains(ir, "nvvm.store.ext"); - try expectContains(ir, "vector<2xi32>"); - try expectContains(ir, "#nvvm.fp_rnd_mode"); - try expectContains(ir, "cute_nvgpu.make_umma_smem_desc"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "gather4 uses CuTe TMA atom construction, partitioning and copy options" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.openProgram(std.testing.allocator, ctx, "gather4_copy"); - defer b.deinit(); - const cfg: TmaConfig = .{ - .dtype = .f32, - .copy_bits = 2048, - .global_basis = b.layoutSpec(.{ 16, 1 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_layout = b.layoutSpec(.{ 128, 64 }, .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }), - .coordinate_rank = 2, - .mode = .gather4, - }; - var pair = cfg; - pair.cta_group = .two; - const args = try b.declareArgs(.{ - .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, - .atom = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(cfg), .grid_constant = true } }, - .pair = .{ .mlir_type = .{ .type_ = b.tmaLoadAtomType(pair), .grid_constant = true } }, - }); - const smem_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); - const smem = b.makeTensorView(b.intToPtr(0, .f32, .smem, 128), smem_layout); - const coords = b.makeCoordTensor(.{ 0, 0 }, b.staticLayout(cfg.coordinate_layout)); - const atom: Atom = .{ .inner = args.atom.inner, .kernel = &b }; - const partitioned = b.tmaPartition(atom, 0, b.makeLayout(1, .{}), smem, .{ coords, args.indices.view() }, smem.type_(), &.{ coords.type_(), args.indices.inner.type_() }); - const mbar = b.allocSmem(.i64, 1, 8); - b.copy(atom, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ - .tma_bar_ptr = mbar, - .cache_policy = b.constant(.i64, CacheHintSm100.evict_last), - }); - b.copy(.{ .inner = args.pair.inner, .kernel = &b }, .{ partitioned.targets[0], partitioned.targets[1] }, .{partitioned.smem}, .{ .tma_bar_ptr = mbar }); - b.endFunction(.{ 32, 1, 1 }); - - b.beginFunction("gather4_copy", .host); - const host = try b.declareArgs(.{ - .data = .{ .tensor = .{ .dtype = .f32, .shape = &.{ 128, 64 }, .stride = &.{ 1, 128 } } }, - .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ 128, 64 }, .stride = &.{ 0, 1 } } }, - }); - const host_layout = b.makeLayout(.{ 16, 4 }, .{ .stride = .{ 1, 16 } }); - const tiler = b.makeTile(&.{ 16, 4 }); - _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{}, host.data, host_layout, tiler, cfg, .{ .gmem_coord_tensor = host.indices.view() }); - _ = b.makeTiledTmaAtom(CopyBulkTensor2DGather4G2SOp{ .cta_group = .two }, host.data, host_layout, tiler, pair, .{ .gmem_coord_tensor = host.indices.view() }); - b.returnHostStatus(b.constant(.i32, 0)); - b.endFunction(null); - const ir = try b.finishProgram(); - defer std.testing.allocator.free(ir); - try expectContains(ir, "atom.make_non_exec_2d_gather4_tma_load"); - try expectContains(ir, "atom.tma_partition"); - try expectContains(ir, "mode = gather4, num_cta = 1"); - try expectContains(ir, "mode = gather4, num_cta = 2"); - try expectContains(ir, "non_exec_2d_gather4_tma_load", .{structFields(b, n, elem)}) catch @panic("OOM")) catch @panic("invalid inline assembly result type")); - const packed_ = b.emit(llvm.inline_asm(b.ctx, text, inner, .{ .result_type = struct_type, .constraints = constraints, .has_side_effects = side_effects }, b.loc())); - const out = b.arena.allocator().alloc(Value, n) catch @panic("OOM"); - for (out, 0..) |*o, i| { - o.* = b.emit(mlir.Operation.make(b.ctx, "llvm.extractvalue", .{ - .operands = .{ .flat = &.{packed_.inner} }, - .results = .{ .flat = &.{elem} }, - .attributes = &.{.named(b.ctx, "position", .denseArray(b.ctx, .i64, &[_]i64{@intCast(i)}))}, - .location = b.loc(), - })); - } - return out; -} - -pub fn structFields(b: *Builder, n: usize, elem: *const mlir.Type) []const u8 { - var w: std.Io.Writer.Allocating = .init(b.arena.allocator()); - for (0..n) |i| w.writer.print("{s}{f}", .{ if (i == 0) "" else ", ", elem }) catch @panic("OOM"); - return w.written(); -} - -pub fn ldAcq(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.gpu.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); -} - -pub fn ldAcqSysU64(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.sys.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); -} - -pub fn ldAcqSysS32(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.sys.global.s32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); -} - -pub fn ldAcqSysU32(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.sys.global.u32 $0, [$1];", &.{ptr}, .i32, "=r,l,~{memory}", true); -} - -pub fn ldAcqGpuU64(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.acquire.gpu.global.u64 $0, [$1];", &.{ptr}, .i64, "=l,l,~{memory}", true); -} - -pub fn atomicAddU32(b: *Builder, ptr: Value, value: anytype) Value { - return asmValue(b, "atom.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r", true); -} - -pub fn atomicAddU64(b: *Builder, ptr: Value, value: Value) Value { - return asmValue(b, "atom.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); -} - -pub fn atomicAddSys(b: *Builder, ptr: Value, value: Value) Value { - return asmValue(b, "atom.sys.global.add.u64 $0, [$1], $2;", &.{ ptr, value }, .i64, "=l,l,l", true); -} - -pub fn atomicAddRel(b: *Builder, ptr: Value, value: anytype) Value { - return asmValue(b, "atom.release.gpu.global.add.u32 $0, [$1], $2;", &.{ ptr, liftAs(b, value, .i32) }, .i32, "=r,l,r,~{memory}", true); -} - -pub fn stRelSys(b: *Builder, ptr: Value, value: Value) void { - asmVoid(b, "st.release.sys.global.u64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); -} - -pub fn redAddS32(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.gpu.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); -} - -pub fn redAddU32(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r"); -} - -pub fn redXorRel(b: *Builder, ptr: Value, value: Value) void { - asmVoid(b, "red.release.gpu.global.xor.b64 [$0], $1;", &.{ ptr, value }, "l,l,~{memory}"); -} - -pub fn redAddRel(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.release.gpu.global.add.u32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); -} - -pub fn redAddRelSys(b: *Builder, ptr: Value, value: anytype) void { - asmVoid(b, "red.release.sys.global.add.s32 [$0], $1;", &.{ ptr, liftAs(b, value, .i32) }, "l,r,~{memory}"); -} - -pub fn atomicAddBlockShared(b: *Builder, smem: Value, value: anytype) Value { - return asmValue(b, "atom.shared.cta.add.u32 $0, [$1], $2;", &.{ smem, liftAs(b, value, .i32) }, .i32, "=r,r,r,~{memory}", true); -} - -pub fn fenceAcqRelCta(b: *Builder) void { - asmVoid(b, "fence.acq_rel.cta;", &.{}, "~{memory}"); -} - -pub fn liftAs(b: *Builder, value: anytype, dt: DType) Value { - if (@TypeOf(value) == Value) return value; - return b.constant(dt, value); -} - -/// `cvta`-style widening of an `i32` shared-memory address to the 64-bit operand -/// `__cvta_generic_to_shared` gives DeepGEMM's `"l"` constraints. -pub fn zext64(b: *Builder, v: Value) Value { - if (v.dtype() == .i64) return v; - return b.emit(arith.extui(b.ctx, v.inner, DType.i64.toMlir(b.ctx), b.loc())); -} - -pub fn i16Of(b: *Builder, v: Value) Value { - return if (v.dtype() == .i16) v else b.emit(arith.trunci(b.ctx, v.inner, DType.i16.toMlir(b.ctx), b.loc())); -} - -pub fn cst32(b: *Builder, v: anytype) Value { - return b.constant(.i32, @as(i32, @bitCast(@as(u32, @truncate(@as(u64, @bitCast(@as(i64, v)))))))); -} - -pub fn cst64(b: *Builder, v: u64) Value { - return b.constant(.i64, @as(i64, @bitCast(v))); -} - -pub fn getSmIdx(b: *Builder) Value { - return asmValue(b, "mov.u32 $0, %smid;", &.{}, .i32, "=r", false); -} - -pub fn getLaneIdx(b: *Builder) Value { - return asmValue(b, "mov.u32 $0, %laneid;", &.{}, .i32, "=r", false); -} - -/// `ptx::get_grid_idx`: the 64-bit `%gridid`, unique per launch. -pub fn getGridIdx(b: *Builder) Value { - return asmValue(b, "mov.u64 $0, %gridid;", &.{}, .i64, "=l", true); -} - -pub fn blockRankInCluster(b: *Builder) Value { - return asmValue(b, "mov.u32 $0, %cluster_ctarank;\n", &.{}, .i32, "=r", true); -} - -/// `clock64()` (`mov.u64 %rd, %clock64;`). -pub fn clock64(b: *Builder) Value { - return asmValue(b, "mov.u64 $0, %clock64;", &.{}, .i64, "=l", true); -} - -/// Keep the original election sequence until typed election matches its small-batch performance. -pub fn electOneSync(b: *Builder) Value { - const pred = asmValue(b, - \\{ - \\.reg .b32 %rx; - \\.reg .pred %px; - \\ mov.u32 $0, 0; - \\ elect.sync %rx|%px, $1; - \\@%px mov.s32 $0, 1; - \\} - , &.{cst32(b, 0xFFFFFFFF)}, .i32, "=r,r", true); - return pred.ne(0); -} - -/// `asm volatile("trap;")` (`DG_TRAP_ONLY_DEVICE_ASSERT`, and this port's timeout). -pub fn trap(b: *Builder) void { - asmVoid(b, "trap;", &.{}, "~{memory}"); -} - -/// `ptx::sync_unaligned(num_threads, barrier_idx)`: `barrier.sync id, n` (threads of a warp may -/// reach it at different program points). -pub fn syncUnaligned(b: *Builder, num_threads: anytype, barrier_idx: anytype) void { - asmVoid(b, "barrier.sync $0, $1;", &.{ liftAs(b, barrier_idx, .i32), liftAs(b, num_threads, .i32) }, "r,r,~{memory}"); -} - -/// `cute::cluster_arrive`. -pub fn clusterArrive(b: *Builder) void { - asmVoid(b, "barrier.cluster.arrive.aligned;\n", &.{}, "~{memory}"); -} - -/// `cudaGridDependencySynchronize()` (`griddepcontrol.wait;`, PDL). -pub fn gridDependencySynchronize(b: *Builder) void { - asmVoid(b, "griddepcontrol.wait;", &.{}, "~{memory}"); -} - -/// `cutlass::arch::warpgroup_reg_alloc`. -pub fn warpgroupRegAlloc(b: *Builder, reg_count: u32) void { - asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.inc.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); -} - -/// `cutlass::arch::warpgroup_reg_dealloc`. -pub fn warpgroupRegDealloc(b: *Builder, reg_count: u32) void { - asmVoid(b, std.fmt.allocPrint(b.arena.allocator(), "setmaxnreg.dec.sync.aligned.u32 {d};\n", .{reg_count}) catch @panic("OOM"), &.{}, ""); -} - -/// `ptx::ld_volatile(const uint64_t*)`. -pub fn ldVolatileU64(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.volatile.global.b64 $0, [$1];", &.{ptr}, .i64, "=l,l", true); -} - -/// `ptx::ld_volatile(const uint32_t*)`. -pub fn ldVolatileU32(b: *Builder, ptr: Value) Value { - return asmValue(b, "ld.volatile.global.b32 $0, [$1];", &.{ptr}, .i32, "=r,l", true); -} - -/// `ptx::ld_shared(const uint32_t*)`. -pub fn ldSharedU32(b: *Builder, smem: Value) Value { - return asmValue(b, "ld.shared.u32 $0, [$1];", &.{zext64(b, smem)}, .i32, "=r,l", true); -} - -/// `ptx::ld_shared(const float2*)`. -pub fn ldSharedF32x2(b: *Builder, smem: Value) [2]Value { - return asmValues(b, 2, "ld.shared.v2.f32 {$0, $1}, [$2];", &.{zext64(b, smem)}, .f32, "=f,=f,l", true); -} - -/// `ptx::ld_shared(const float4*)`. -pub fn ldSharedF32x4(b: *Builder, smem: Value) [4]Value { - return asmValues(b, 4, "ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .f32, "=f,=f,=f,=f,l", true); -} - -/// `ptx::ld_shared(const uint4*)`. -pub fn ldSharedU32x4(b: *Builder, smem: Value) [4]Value { - return asmValues(b, 4, "ld.shared.v4.u32 {$0, $1, $2, $3}, [$4];", &.{zext64(b, smem)}, .i32, "=r,=r,=r,=r,l", true); -} - -/// `ptx::st_shared(const void*, x, y, z, w)`. -pub fn stSharedU32x4(b: *Builder, smem: Value, w: [4]Value) void { - asmVoid(b, "st.shared.v4.u32 [$0], {$1, $2, $3, $4};", &.{ zext64(b, smem), w[0], w[1], w[2], w[3] }, "l,r,r,r,r"); -} - -/// `ptx::st_shared(const uint32_t*, uint32_t)`. -pub fn stSharedU32(b: *Builder, smem: Value, value: Value) void { - asmVoid(b, "st.shared.u32 [$0], $1;", &.{ zext64(b, smem), value }, "l,r"); -} - -/// `ptx::st_shared(const float2*, float2)`. -pub fn stSharedF32x2(b: *Builder, smem: Value, v: [2]Value) void { - asmVoid(b, "st.shared.v2.f32 [$0], {$1, $2};", &.{ zext64(b, smem), v[0], v[1] }, "l,f,f"); -} - -/// `ptx::st_shared_bulk(smem, num_bytes)`: zero `num_bytes` (a multiple of 8) of shared memory. -pub fn stSharedBulk(b: *Builder, smem: Value, num_bytes: u32) void { - std.debug.assert(num_bytes % 8 == 0); - asmVoid(b, "st.bulk.weak.shared::cta [$0], $1, 0;", &.{ zext64(b, smem), b.constant(.i64, num_bytes) }, "l,l,~{memory}"); -} - -/// `ptx::mapa_shared(ptr, dst_cta_idx)`: this CTA's shared address in cluster CTA `cta`. -pub fn mapaShared(b: *Builder, smem: Value, dst_cta_idx: Value) Value { - return asmValue(b, "mapa.shared::cluster.u32 $0, $1, $2;", &.{ smem, dst_cta_idx }, .i32, "=r,r,r", true); -} - -/// `ptx::st_async_cluster(dst, src, dst_cta_idx, barrier)` for `words.len / 4` 16-byte -/// chunks: `st.async` into CTA `dst_cta_idx`'s shared memory, completing the bytes on its -/// mbarrier at `barrier` (both local addresses, mapped with `mapa`). -pub fn stAsyncCluster(b: *Builder, dst: Value, words: []const Value, dst_cta_idx: Value, barrier: Value) void { - std.debug.assert(words.len % 4 == 0); - const mapped_barrier = mapaShared(b, barrier, dst_cta_idx); - const mapped_dst = mapaShared(b, dst, dst_cta_idx); - var i: usize = 0; - while (i < words.len) : (i += 4) { - asmVoid( - b, - "st.async.shared::cluster.mbarrier::complete_tx::bytes.u32.v4 [$0], {$1, $2, $3, $4}, [$5];", - &.{ mapped_dst.add(@as(i32, @intCast(i * 4))), words[i], words[i + 1], words[i + 2], words[i + 3], mapped_barrier }, - "r,r,r,r,r,r,~{memory}", - ); - } -} - -/// `ptx::SM100_U8x4_STSM_T::copy(src, smem)`: `stmatrix.m16n8.x1.trans.b8`. -pub fn stmatrixU8x4T(b: *Builder, src: Value, smem: Value) void { - asmVoid(b, "stmatrix.sync.aligned.m16n8.x1.trans.shared.b8 [$0], {$1};\n", &.{ zext64(b, smem), src }, "l,r"); -} - -/// `ptx::SM90_U32x4_STSM_T::copy(s0, s1, s2, s3, smem)`: `stmatrix.x4.m8n8.trans.b16`. -pub fn stmatrixU32x4T(b: *Builder, src: [4]Value, smem: Value) void { - asmVoid(b, "stmatrix.sync.aligned.x4.m8n8.shared.b16.trans [$0], {$1, $2, $3, $4};\n", &.{ zext64(b, smem), src[0], src[1], src[2], src[3] }, "l,r,r,r,r"); -} - -/// `ClusterBarrier::arrive()` on this CTA's barrier. -pub fn mbarrierArrive(b: *Builder, bar: Value) void { - asmVoid(b, "{\n\tmbarrier.arrive.shared::cta.b64 _, [$0];\n\t}", &.{bar}, "r,~{memory}"); -} - -/// `ClusterBarrier::arrive(cta_id, pred = true)`: arrive on the barrier at the same offset of -/// cluster CTA `cta_id`. -pub fn mbarrierArriveCluster(b: *Builder, bar: Value, cta_id: Value) void { - asmVoid( - b, - "{\n\t.reg .b32 remAddr32;\n\tmapa.shared::cluster.u32 remAddr32, $0, $1;\n\tmbarrier.arrive.shared::cluster.b64 _, [remAddr32];\n\t}", - &.{ bar, cta_id }, - "r,r,~{memory}", - ); -} - -/// `ClusterTransactionBarrier::arrive_and_expect_tx(bytes, cta_id, pred)`: on cluster CTA -/// `cta_id`'s barrier, only when `pred == 1`. -pub fn mbarrierArriveAndExpectTxCluster(b: *Builder, bar: Value, transaction_bytes: anytype, cta_id: Value, pred: Value) void { - asmVoid( - b, - "{\n\t.reg .pred p;\n\t.reg .b32 remAddr32;\n\tsetp.eq.u32 p, $2, 1;\n\t@p mapa.shared::cluster.u32 remAddr32, $0, $1;\n\t@p mbarrier.arrive.expect_tx.shared::cluster.b64 _, [remAddr32], $3;\n\t}", - &.{ bar, cta_id, pred, liftAs(b, transaction_bytes, .i32) }, - "r,r,r,r,~{memory}", - ); -} - -/// `cute::prefetch_tma_descriptor(desc)` (generic descriptor address). -pub fn prefetchTmaDescriptor(b: *Builder, desc: Value) void { - asmVoid(b, "prefetch.tensormap [$0];", &.{desc}, "l,~{memory}"); -} - -/// `SM100_TMA_2SM_LOAD_2D::copy(desc, mbar, cache_hint, smem, c0, c1)`: issued by both CTAs of -/// the pair; the barrier address is masked so the bytes complete on the leader's barrier. -pub fn tma2SmLoad2d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c0: Value, c1: Value) void { - asmVoid( - b, - "cp.async.bulk.tensor.2d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4}], [$2], $5;", - &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c0, c1, cst64(b, cache_hint) }, - "r,l,r,r,r,l,~{memory}", - ); -} - -/// `SM100_TMA_2SM_LOAD_4D::copy(desc, mbar, cache_hint, smem, c0, c1, c2, c3)`. -pub fn tma2SmLoad4d(b: *Builder, desc: Value, mbar: Value, cache_hint: u64, smem: Value, c: [4]Value) void { - asmVoid( - b, - "cp.async.bulk.tensor.4d.cta_group::2.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1, {$3, $4, $5, $6}], [$2], $7;", - &.{ smem, desc, mbar.bitAnd(cst32(b, sm100_mma_peer_bit_mask)), c[0], c[1], c[2], c[3], cst64(b, cache_hint) }, - "r,l,r,r,r,r,r,l,~{memory}", - ); -} - -/// `ptx::tma_load_1d(dst, src, mbar, num_bytes, hint = EVICT_FIRST)`: non-tensor bulk copy of -/// `num_bytes` from a global (possibly peer) address into this CTA's shared memory. -pub fn tmaLoad1d(b: *Builder, dst: Value, src: Value, mbar: Value, num_bytes: anytype, cache_hint: u64) void { - asmVoid( - b, - "cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes.L2::cache_hint [$0], [$1], $2, [$3], $4;\n", - &.{ dst, src, liftAs(b, num_bytes, .i32), mbar, cst64(b, cache_hint) }, - "r,l,r,r,l,~{memory}", - ); -} - -/// `ptx::tma_store_1d(dst, src, num_bytes, hint = EVICT_NORMAL)`. -pub fn tmaStore1d(b: *Builder, dst: Value, src: Value, num_bytes: anytype, cache_hint: u64) void { - asmVoid( - b, - "cp.async.bulk.global.shared::cta.bulk_group.L2::cache_hint [$0], [$1], $2, $3;\n", - &.{ dst, src, liftAs(b, num_bytes, .i32), cst64(b, cache_hint) }, - "l,r,r,l,~{memory}", - ); -} - -/// `cute::SM90_TMA_STORE_2D::copy(desc, smem, c0, c1)`. -pub fn tmaStore2d(b: *Builder, desc: Value, smem: Value, c0: Value, c1: Value) void { - asmVoid(b, "cp.async.bulk.tensor.2d.global.shared::cta.bulk_group [$0, {$2, $3}], [$1];", &.{ desc, smem, c0, c1 }, "l,r,r,r,~{memory}"); -} - -/// `cute::TMEM::Allocator2Sm::allocate(num_columns, dst)`: write the base column to shared `dst`. -pub fn tmemAllocate2Sm(b: *Builder, num_columns: u32, dst: Value) void { - asmVoid(b, "tcgen05.alloc.cta_group::2.sync.aligned.shared::cta.b32 [$0], $1;", &.{ dst, cst32(b, num_columns) }, "r,r,~{memory}"); -} - -/// `cute::TMEM::Allocator2Sm::free(tmem_ptr, num_columns)`. -pub fn tmemFree2Sm(b: *Builder, tmem_ptr: Value, num_columns: u32) void { - asmVoid(b, "{\n\ttcgen05.dealloc.cta_group::2.sync.aligned.b32 $0, $1; \n\t}", &.{ tmem_ptr, cst32(b, num_columns) }, "r,r,~{memory}"); -} - -/// `ptx::tcgen05_before_thread_sync`. -pub fn tcgen05BeforeThreadSync(b: *Builder) void { - asmVoid(b, "tcgen05.fence::before_thread_sync;", &.{}, "~{memory}"); -} - -/// `ptx::tcgen05_after_thread_sync`. -pub fn tcgen05AfterThreadSync(b: *Builder) void { - asmVoid(b, "tcgen05.fence::after_thread_sync;", &.{}, "~{memory}"); -} - -/// `cutlass::arch::fence_view_async_tmem_load` (`tcgen05.wait::ld`). -pub fn fenceViewAsyncTmemLoad(b: *Builder) void { - asmVoid(b, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &.{}, "~{memory}"); -} - -/// `fence_view_async_tmem_load` with the values of earlier TMEM loads threaded through it, so -/// LLVM cannot move their uses above the wait (the PTX is the same single instruction). -pub fn fenceViewAsyncTmemLoadValues(b: *Builder, comptime n: usize, values: [n]Value) [n]Value { - comptime var constraints: []const u8 = ""; - inline for (0..n) |i| constraints = constraints ++ (if (i == 0) "" else ",") ++ "=r"; - inline for (0..n) |i| constraints = constraints ++ std.fmt.comptimePrint(",{d}", .{i}); - constraints = constraints ++ ",~{memory}"; - return asmValues(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", &values, .i32, constraints, true); -} - -/// `fenceViewAsyncTmemLoadValues` for a run-time number of values (the epilogue's store blocks -/// hold `STORE_BLOCK_M / ATOM_M * 8` words): the same single `tcgen05.wait::ld`, every value -/// tied through it. -pub fn fenceViewAsyncTmemLoadSlice(b: *Builder, values: []const Value) []Value { - const n = values.len; - const a = b.arena.allocator(); - var w: std.Io.Writer.Allocating = .init(a); - for (0..n) |i| w.writer.print("{s}=r", .{if (i == 0) "" else ","}) catch @panic("OOM"); - for (0..n) |i| w.writer.print(",{d}", .{i}) catch @panic("OOM"); - w.writer.writeAll(",~{memory}") catch @panic("OOM"); - return asmValuesSlice(b, n, "{\n\ttcgen05.wait::ld.sync.aligned; \n}", values, .i32, w.written(), true); -} - -/// `cute::SM100_TMEM_LOAD_16dp256b1x::copy(src, d0, d1, d2, d3)`: lane t of the warp gets -/// rows `base + t/4` (d0, d1) and `base + 8 + t/4` (d2, d3), columns `2(t%4)`, `2(t%4)+1`. -pub fn tmemLoad16dp256b1x(b: *Builder, taddr: Value) [4]Value { - return asmValues(b, 4, "tcgen05.ld.sync.aligned.16x256b.x1.b32{$0, $1, $2, $3},[$4];\n", &.{taddr}, .i32, "=r,=r,=r,=r,r", true); -} - -/// `cute::SM100_UTCCP_4x32dp128bit_2cta::copy(src_desc, dst_taddr)`. -pub fn utccp4x32dp128bit2cta(b: *Builder, src_desc: Value, dst_taddr: Value) void { - asmVoid(b, "tcgen05.cp.cta_group::2.32x128b.warpx4 [$0], $1;", &.{ dst_taddr, src_desc }, "r,l"); -} - -/// `ptx::SM100_MMA_MXF8F6F4_2x1SM_SS::fma(desc_a, desc_b, tmem_c, scale_c, desc, tmem_sfa, -/// tmem_sfb)`: `desc` is the 64-bit runtime descriptor (`instr << 32`), `scale_c` (i32) -/// nonzero accumulates. -pub fn mmaMxf8f6f42x1SmSs(b: *Builder, desc_a: Value, desc_b: Value, tmem_c: Value, scale_c: Value, desc: Value, tmem_sfa: Value, tmem_sfb: Value) void { - const idesc = b.emit(arith.trunci(b.ctx, desc.shrLogical(32).inner, DType.i32.toMlir(b.ctx), b.loc())); - asmVoid( - b, - "{\n\t.reg .pred p;\n\tsetp.ne.b32 p, $4, 0;\n\ttcgen05.mma.cta_group::2.kind::mxf8f6f4.block_scale [$0], $1, $2, $3, [$5], [$6], p; \n\t}\n", - &.{ tmem_c, desc_a, desc_b, idesc, scale_c, tmem_sfa, tmem_sfb }, - "r,l,l,r,r,r,r", - ); -} - -/// `cutlass::arch::umma_arrive_multicast_2x1SM(barrier, cta_mask)`: inside `elect_one_sync`, -/// `tcgen05.commit` arriving on `barrier` in every CTA of `cta_mask`. -pub fn ummaArriveMulticast2x1Sm(b: *Builder, bar: Value, cta_mask: u16) void { - var elected = b.openIf(electOneSync(b)); - { - asmVoid( - b, - "{\n\ttcgen05.commit.cta_group::2.mbarrier::arrive::one.shared::cluster.multicast::cluster.b64 [$0], $1; \n\t}", - &.{ bar, b.constant(.i16, @as(i16, @bitCast(cta_mask))) }, - "r,h,~{memory}", - ); - elected.close(); - } -} - -pub fn bits32(v: Value) Value { - return if (v.dtype() == .i32) v else v.bitCast(.i32); -} - -pub fn fromBits32(v: Value, dt: DType) Value { - return if (dt == .i32) v else v.bitCast(dt); -} - -/// `__shfl_sync(0xffffffff, v, src_lane)` / `ptx::exchange` (32-bit). -pub fn shflSync(b: *Builder, v: Value, src_lane: anytype) Value { - const r = asmValue(b, "shfl.sync.idx.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, src_lane, .i32) }, .i32, "=r,r,r", true); - return fromBits32(r, v.dtype()); -} - -/// `__shfl_xor_sync(0xffffffff, v, lane_mask)`. -pub fn shflXorSync(b: *Builder, v: Value, lane_mask: anytype) Value { - const r = asmValue(b, "shfl.sync.bfly.b32 $0, $1, $2, 31, -1;", &.{ bits32(v), liftAs(b, lane_mask, .i32) }, .i32, "=r,r,r", true); - return fromBits32(r, v.dtype()); -} - -/// `__shfl_up_sync(0xffffffff, v, delta)`. -pub fn shflUpSync(b: *Builder, v: Value, delta: anytype) Value { - const r = asmValue(b, "shfl.sync.up.b32 $0, $1, $2, 0, -1;", &.{ bits32(v), liftAs(b, delta, .i32) }, .i32, "=r,r,r", true); - return fromBits32(r, v.dtype()); -} - -/// `__reduce_add_sync(0xffffffff, v)` (u32). -pub fn reduceAddSync(b: *Builder, v: Value) Value { - return asmValue(b, "redux.sync.add.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); -} - -/// `__reduce_min_sync(0xffffffff, v)` (u32). -pub fn reduceMinSync(b: *Builder, v: Value) Value { - return asmValue(b, "redux.sync.min.u32 $0, $1, $2;", &.{ v, cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); -} - -/// `__all_sync(0xffffffff, pred)` (`pred` i1; returns i1). -pub fn allSync(b: *Builder, pred: Value) Value { - const r = asmValue(b, "{ .reg .pred p, q; setp.ne.u32 p, $1, 0; vote.sync.all.pred q, p, $2; selp.u32 $0, 1, 0, q; }", &.{ pred.to(.i32), cst32(b, 0xFFFFFFFF) }, .i32, "=r,r,r", true); - return r.ne(0); -} - -pub fn popc(b: *Builder, v: Value) Value { - return b.callIntrinsic("llvm.ctpop.i32", &.{v}, DType.i32.toMlir(b.ctx)).?; -} - -/// `__ffs(v)`: 1-based index of the lowest set bit, 0 for 0 (nvcc: `brev` + `bfind.shiftamt`). -pub fn ffs(b: *Builder, v: Value) Value { - return asmValue(b, "{ .reg .b32 r; brev.b32 r, $1; bfind.shiftamt.u32 $0, r; }", &.{v}, .i32, "=r,r", false).add(1); -} - -/// `__fns(mask, base, offset)`: position of the `offset`-th set bit of `mask` from `base`. -pub fn fns(b: *Builder, mask: Value, base: anytype, offset: Value) Value { - return asmValue(b, "fns.b32 $0, $1, $2, $3;", &.{ mask, liftAs(b, base, .i32), offset }, .i32, "=r,r,r,r", false); -} - -/// `fminf(a, b)` (`min.f32`, NaN-ignoring; `kClampAfterSilu`). -pub fn fminf(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "min.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); -} - -/// `fabsf(x)` (`abs.f32`). -pub fn fabsf(b: *Builder, x: Value) Value { - return asmValue(b, "abs.f32 $0, $1;", &.{x}, .f32, "=f,f", false); -} - -/// `math::fast_rcp(x)`. -pub fn fastRcp(b: *Builder, x: Value) Value { - return asmValue(b, "rcp.approx.ftz.f32 $0, $1;", &.{x}, .f32, "=f,f", true); -} - -/// `__expf(x)` without `--use_fast_math`: `ex2.approx.f32(x * log2(e))` (non-FTZ). -pub fn expfFast(b: *Builder, x: Value) Value { - return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0f3FB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); -} - -/// `__expf(-x)`: nvcc folds the negation into the constant (`mul.f32 t, x, -log2(e)`). -pub fn expfFastNeg(b: *Builder, x: Value) Value { - return asmValue(b, "{ .reg .f32 t; mul.f32 t, $1, 0fBFB8AA3B; ex2.approx.f32 $0, t; }", &.{x}, .f32, "=f,f", false); -} - -/// `__float22bfloat162_rn({lo, hi})`: bf16x2 word, `lo` in the low half. -pub fn float22bfloat162Rn(b: *Builder, lo: Value, hi: Value) Value { - return asmValue(b, "{ cvt.rn.bf16x2.f32 $0, $2, $1;}\n", &.{ lo, hi }, .i32, "=r,f,f", false); -} - -/// `__hmin2(a, b)` on bf16x2 words. -pub fn hmin2Bf16(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "{ min.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); -} - -/// `__hmax2(a, b)` on bf16x2 words. -pub fn hmax2Bf16(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "{ max.bf16x2 $0,$1,$2;\n}", &.{ x, y }, .i32, "=r,r,r", false); -} - -/// `__bfloat1622float2(word)`: `{low, high}` f32 (exact). -pub fn bfloat1622float2(b: *Builder, word: Value) [2]Value { - return asmValues(b, 2, "{ .reg .b16 l, h; mov.b32 {l, h}, $2; cvt.f32.bf16 $0, l; cvt.f32.bf16 $1, h; }", &.{word}, .f32, "=f,=f,r", false); -} - -/// `ptx::accumulate(float2& a, nv_bfloat162 b)`: `a.x += bf16(b.x)`, `a.y += bf16(b.y)` with -/// `add.rn.f32.bf16` (one rounding, the bf16 widened exactly). -pub fn accumulateBf16x2(b: *Builder, acc: [2]Value, bf16x2: Value) [2]Value { - const lo = i16Of(b, bf16x2); - const hi = i16Of(b, bf16x2.shrLogical(16)); - return .{ - asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ lo, acc[0] }, .f32, "=f,h,0", false), - asmValue(b, "add.rn.f32.bf16 $0, $1, $0;\n", &.{ hi, acc[1] }, .f32, "=f,h,0", false), - }; -} -/// IEEE operations used when staging quantized inputs; no FTZ or contraction. -pub fn divRn(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "div.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); -} -pub fn mulRn(b: *Builder, x: Value, y: Value) Value { - return asmValue(b, "mul.rn.f32 $0, $1, $2;", &.{ x, y }, .f32, "=f,f,f", false); -} - -test "canonical primitives preserve aligned barriers and runtime transaction counts" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "compatibility_primitives"); - defer b.deinit(); - _ = try b.declareArgs(.{}); - const bar = b.constant(.i32, 0); - const phase = b.constant(.i32, 0); - const bytes = b.threadIdx().x.add(16); - b.barrierCtaSyncAligned(1, 32); - b.barrierClusterArriveRelaxedAligned(); - b.barrierClusterWaitAligned(); - b.mbarrierInit(bar, 1); - b.fenceMbarrierInit(); - b.mbarrierArriveExpectTx(bar, bytes); - b.mbarrierArriveExpectTx(bar, 16); - b.mbarrierWait(bar, phase, .{ .test_first = false }); - _ = phase.bitXor(1); - b.fenceProxyAsyncShared(); - b.fenceProxyAsyncShared(); - b.cpAsyncBulkCommitGroup(); - b.cpAsyncBulkWaitGroup(0, false); - _ = b.voteBallotSync(bytes.ne(0), null); - _ = b.fmax(b.constant(.f32, 1), b.constant(.f32, 2), .{}); - const ir = try b.finish(.{ 32, 1, 1 }); - defer std.testing.allocator.free(ir); - try expectContains(ir, "llvm.nvvm.barrier.cta.sync.aligned.count"); - try expectContains(ir, "barrier.cluster.arrive.relaxed.aligned"); - try expectContains(ir, "nvvm.mbarrier.arrive.expect_tx"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} diff --git a/dsl/cute/dtype.zig b/dsl/cute/dtype.zig deleted file mode 100644 index ae98cb08..00000000 --- a/dsl/cute/dtype.zig +++ /dev/null @@ -1,67 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -/// Element types a `!cute.ptr` can point at. -pub const DType = enum { - i1, - i8, - i16, - i32, - i64, - f16, - bf16, - f32, - f64, - f4e2m1fn, - f8e4m3fn, - f8e5m2, - f8e8m0fnu, - - pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { - return switch (self) { - .i1 => .int(ctx, .i1), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .f16 => .float(ctx, .f16), - .bf16 => .float(ctx, .bf16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .f4e2m1fn => .float(ctx, .f4e2m1fn), - .f8e4m3fn => .float(ctx, .f8e4m3fn), - .f8e5m2 => .float(ctx, .f8e5m2), - .f8e8m0fnu => .float(ctx, .f8e8m0fnu), - }; - } -}; - -pub fn isFloatDtype(dt: DType) bool { - return switch (dt) { - .f16, .bf16, .f32, .f64, .f4e2m1fn, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => true, - else => false, - }; -} - -pub fn dtypeBitwidth(dt: DType) usize { - return switch (dt) { - .i1 => 1, - .f4e2m1fn => 4, - .i8, .f8e4m3fn, .f8e5m2, .f8e8m0fnu => 8, - .i16, .f16, .bf16 => 16, - .i32, .f32 => 32, - .i64, .f64 => 64, - }; -} - -pub fn mlirToDType(ctx: *mlir.Context, t: *const mlir.Type) DType { - inline for (@typeInfo(DType).@"enum".field_names) |f| { - const dt = @field(DType, f); - if (t.eql(dt.toMlir(ctx))) return dt; - } - std.debug.panic("type {f} is not a CuTe DType", .{t}); -} - -test { - std.testing.refAllDecls(@This()); -} diff --git a/dsl/fly/BUILD.bazel b/dsl/fly/BUILD.bazel deleted file mode 100644 index c8bdf854..00000000 --- a/dsl/fly/BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "builder", - srcs = [ - "dtype.zig", - "layout.zig", - ], - import_name = "dsl/fly/builder", - main = "builder.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//dsl/common:builder", - "//mlir", - "//mlir/dialects/arith", - "//mlir/dialects/fly", - "//mlir/dialects/fly_rocdl", - "//mlir/dialects/func", - "//mlir/dialects/gpu", - "//mlir/dialects/llvm", - "//mlir/dialects/math", - "//mlir/dialects/memref", - "//mlir/dialects/rocdl", - "//mlir/dialects/scf", - "//mlir/dialects/ub", - "//mlir/dialects/vector", - "//stdx", - ], -) - -zig_test( - name = "test", - deps = [":builder"], -) diff --git a/dsl/fly/PORTING.md b/dsl/fly/PORTING.md deleted file mode 100644 index f7cb19f3..00000000 --- a/dsl/fly/PORTING.md +++ /dev/null @@ -1,104 +0,0 @@ -# Porting a FlyDSL Python kernel to Zig - -The Zig DSL builds Fly IR directly. Work from the revision and patches in -[the FlyDSL repository rule](../../third_party/flydsl/repo.bzl), reading -Python sources with `git -C show :`. Check the -[ROCm plugin pin](../../platforms/rocm/rocm.bzl) as well: the emitter and -runtime must agree on the dialect version. FlyDSL HEAD may differ from both. - -## Cheat sheet - -| Python (`flydsl.expr`) | Zig | -|---|---| -| `@flyc.kernel def k(A: Tensor, ...)` | `fly.Kernel(Cfg, .{ .inputs = &.{"a"}, .outputs = &.{"c"}, .run = run })`; `const a = K.args(b)` | -| kernel arg `tiled_copy: fx.TiledCopy` | build it inside `run` (static values cannot be arguments) | -| `fx.thread_idx.x`, `fx.block_idx.x` | `b.threadId(.x)`, `b.blockId(.x)` (`i32`) | -| `A.shape.unpack()` | `a.a.shapeStatic().at(i).leaf.s` | -| `fx.make_layout((8,16),(16,1))` | `fly.L(.{ 8, 16 }, .{ 16, 1 })` (comptime) or `b.static(layout)` | -| `fx.make_ordered_layout((8,16),(1,0))` | `fly.ordered(.{ 8, 16 }, .{ 1, 0 })` | -| `fx.make_tile(64, 8)` / tuple divisor `(64, 8)` | `fly.tile(.{ 64, 8 })` / `.{ 64, 8 }` | -| `fx.make_int_tuple((m, tid))`, coords | `b.intTuple(.{ m, tid })` (`Value`s become `?` leaves) | -| `t[None, None, bx, by]` | `t.slice(.{ null, null, bx, by })` | -| `t[(0, None), None, None]` | `t.slice(.{ .{ 0, null }, null, null })` | -| `t[i]` (no `None`) / `t[i] = v` | `t.at(i)` / `t.set(i, v)` | -| `fx.flat_divide(t, tile)` etc. | `t.flatDivide(tile)`, `.logicalDivide`, `.zippedDivide`, `.tiledDivide` | -| `fx.raked_product(a, b)` etc. | `a.rakedProduct(b)`, `.logicalProduct`, ... | -| `fx.right_inverse(l)`, `fx.composition(l, r)` | `l.rightInverse()`, `l.composition(r)` | -| `fx.get_shape(l)`, `fx.size(l)` (emitted) | `l.emitShape()`, `l.emitSize()` | -| `fx.size(t.shape).unpack()` (host int) | `t.sizeStatic()` | -| `fx.select(tup, [0])`, `fx.get(tup, [1])` | `tup.selectModes(&.{0})`, `tup.get(&.{1})` | -| `fx.make_view((0,0), fx.make_identity_layout((M,N)))` | `b.identity(.{ m, n })` | -| `fx.make_view(ptr, layout)` | `ptr.view(layout)` | -| `fx.make_copy_atom(fx.UniversalCopy128b(), fx.Float32)` | `b.copyAtom(.{ .universal = 128 }, .f32)` | -| `fx.rocdl.BufferCopy32b()` / `BufferCopyLDS128b()` | `.{ .buffer_copy = 32 }` / `.{ .buffer_copy_lds = 128 }` | -| `fx.make_mma_atom(fx.rocdl.MFMA(16,16,4,fx.Float32))` | `b.mmaAtom((try fly.fly_rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ .m = 16, .n = 16, .k = 4, ... })).type_())` | -| `fx.make_tiled_mma(atom, layout)` | `b.tiledMma(atom, layout, null)` | -| `fx.make_tiled_copy_tv(atom, thr, val)` | `b.tiledCopyTV(atom, thr, val)` | -| `fx.make_tiled_copy_A(atom, tiled_mma)` | `b.tiledCopyA(atom, tiled_mma)` | -| `tiled_copy.tile_mn` | `tc.tileMN()` | -| `tiled_mma.tile_size_mnk` etc. | `@import("mlir/dialects/fly").expect(tm.value.type_(), .tiled_mma).tileSizeMNK()` | -| `tiled_copy.get_slice(tid).partition_S(t)` | `tc.getSlice(tid).partitionS(t)` | -| `thr_copy.retile(t)` | `thr.retile(t)` | -| `tiled_mma.make_fragment_A(t)` | `tm.makeFragmentA(t)` | -| `fx.make_fragment_like(t[, dtype=fx.Boolean])` | `t.makeFragmentLike(null)` / `t.makeFragmentLike(.i1)` | -| `fx.make_rmem_tensor(8, fx.Float16)` | `b.rmemTensor(fly.L(8, 1), .f16)` | -| `fx.copy(atom, src, dst, pred=p)` | `b.copy(atom, src, dst, .{ .pred = p })` | -| `fx.copy_atom_call(atom, src, dst)` | `b.copyAtomCall(atom, src, dst)` | -| `fx.gemm(atom, d, a, b, c)` | `b.gemm(atom, d, a, bb, c)` | -| `t.load()`, `t.store(v)`, `t.fill(0)` | same names | -| `fx.elem_less(coord, (M, N))` | `coord.elemLess(.{ m, n })` | -| `fx.rocdl.make_buffer_tensor(t)` | `b.bufferTensor(t)` | -| `fx.SharedAllocator().allocate(fx.Array[f32, n, 16])` | `b.sharedArray(.f32, n, 16)` (returns an `n:1` tensor) | -| `gpu.barrier()` | `b.barrier()` | -| `x.shuffle_xor(off, 64)` | `x.shuffleXor(off, 64)` | -| `v.reduce(ReductionOp.ADD)` | `v.reduce(.add)` | -| `v.to(fx.Float32)` | `v.to(.f32)` | -| `x * rrms` (vector × scalar) | `x.mul(rrms)` | -| `fmath.rsqrt(x, fastmath=fast)` | `b.setFastMath(true); b.rsqrt(x)` | -| `lane < RED_SLOTS`, `cond.select(a, b)` | `lane.lt(n)`, `cond.select(a, b)` | -| `if tid == 0: ...` | `var s = b.openIf(tid.eq(0)); ...; s.close();` | -| `for i in range_constexpr(n)` | a plain Zig `for (0..n)` (unrolled at emit time) | -| `for i in range(lo, hi)` with carried values | `b.openFor(lo, hi, 1, .{ init })` + `.yield(.{ ... })` | -| `launch(grid=..., block=(256,1,1))` | `K.call(inputs, outputs, .{ .grid = ..., .threads = 256 })` | - -## Method - -1. Read the kernel and its helpers at the pin; note every `fx.*` primitive. -2. Write the Zig `run` top to bottom, preserving operation order and numeric - conversions. For matrix loops, use SSA copy and MMA calls as described - under [control flow and register tensors](README.md#control-flow-and-register-tensors): - `b.copyAtomLoad` returns vectors, `b.mmaAtomCall` returns the accumulator, - and `b.copyAtomStore` writes vectors. Carry accumulator values through - `openFor`; do not put mutable register tensors across control-flow regions. -3. `bazel test //dsl/fly:test` (or a `K.emit` test): the module must - verify and re-parse. A panic from `fly.: cannot create operation` - lists the operand types; compare them with the Python trace. -4. Run on the ROCm box against a host reference - (`bazel test //zml:test --@zml//platforms:rocm=true ...`). - -## Things that differ - -- Python config (`N`, `dtype_str`, `const_expr(...)` branches) becomes the - kernel `Config`; branch on it with plain `if` in `run`. -- Python lists of vectors across an unrolled loop are Zig arrays of `Value`. - `v.insert(i, x)` returns a new vector; `v.extract(i)` reads an element. - `v.bitCast(dtype)` preserves total bits and adjusts vector length. -- Keep unsigned semantics explicit for packed bytes and offsets with - `toUnsigned`, `divUnsigned`, `remUnsigned` and `cmpUnsigned`. -- `v.shuffleVector(rhs, indices)` rearranges vector elements within a thread; - `v.shuffle(.up, offset, width)` and `shuffleXor` exchange values across lanes. -- Runtime `while` loops use `b.openWhile(inits, after_types)`, `yieldBefore` - and `yieldAfter`; use a plain Zig `while` only for loops unrolled at emit time. -- Python `Int32(x)` / `Float32(x)` constants are `b.constant(.i32, x)`; most - arithmetic methods also take a literal directly (`tid.add(256)`). -- Cached scalars from LDS are read with `t.at(i)`; there is no implicit load. -- bf16 outputs use `arith.truncf`; the Python's manual round-to-nearest bit - trick for pre-gfx950 parts is not reproduced. - -## Model kernels - -The [MoE kernels](../../zml/kernels/moe/fly/) include native routing and -MXFP4 GEMV/MFMA implementations. llmd's sparse MLA kernel -(`llmd/kernels/mla/fly/sparse_mla.zig`, downstream) uses SSA -matrix accumulators through attention loops. These are examples of the -builder APIs with model tensor layouts and launch wrappers. diff --git a/dsl/fly/README.md b/dsl/fly/README.md deleted file mode 100644 index 687c99dc..00000000 --- a/dsl/fly/README.md +++ /dev/null @@ -1,255 +0,0 @@ -# ZML FlyDSL DSL — Author's Guide - -A Zig builder for AMD's FlyDSL (`fly` / `fly_rocdl` MLIR dialects) that reads -like `flydsl.expr` Python while staying in pure Zig. Kernels are compiled and -launched by ZML's ROCm PJRT plugin through the `__gpu$xla.gpu.fly` custom -call; ZML never links a GPU compiler. - -Two layers define the surface: - -- **Low layer** — `dsl/fly/builder.zig`: `Builder`, `Value`, `TiledCopy`, - `TiledMma`, the comptime layout literals in `layout.zig`, and `DType`. - `Builder.open` takes an `*mlir.Context` from `zml.kernel.fly.newContext()`. -- **High layer** — `zml.kernel.fly.Kernel(Config, spec)`: names the kernel's - inputs and outputs, hands `run` its arguments by name (`K.args(b)`), and - `call`s it from a ZML model. - -The dialect bindings construct typed MLIR operations. Layout-transforming -operations use FlyDSL's `inferReturnTypes`; static layout literals use typed -attribute constructors and accessors. - -## Skeleton of a kernel - -```zig -const zml = @import("zml"); -const fly = zml.kernel.fly; - -pub const Cfg = struct { - thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), // 128 threads - val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), // f32x4 along N -}; - -pub const VectorAdd = fly.Kernel(Cfg, .{ - .name = "vector_add", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, -}); - -fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = VectorAdd.args(b); // .a/.b/.c: row-major !fly.memref views - const shape = a.a.shapeStatic(); - const m = shape.at(0).leaf.s; - const n = shape.at(1).leaf.s; - - const atom = b.copyAtom(.{ .universal = 128 }, .f32); - const tc = b.tiledCopyTV(atom, cfg.thr, cfg.val); // fx.make_tiled_copy_tv - const tid = b.threadId(.x); - const bx = b.blockId(.x); - const by = b.blockId(.y); - - const tile = tc.tileMN(); - const gA = a.a.flatDivide(tile).slice(.{ null, null, bx, by }); // A[None, None, bx, by] - const gB = a.b.flatDivide(tile).slice(.{ null, null, bx, by }); - const gC = a.c.flatDivide(tile).slice(.{ null, null, bx, by }); - const cC = b.identity(.{ m, n }).flatDivide(tile).slice(.{ null, null, bx, by }); - - const thr = tc.getSlice(tid); - const tgA = thr.partitionS(gA); - const tgB = thr.partitionS(gB); - const tgC = thr.partitionD(gC); - const tcC = thr.partitionS(cC).slice(.{ .{ 0, null }, null, null }); - - const rA = tgA.makeFragmentLike(null); - const rB = tgB.makeFragmentLike(null); - const rC = tgC.makeFragmentLike(null); - const pC = tcC.makeFragmentLike(.i1); - for (0..@intCast(pC.sizeStatic())) |i| pC.set(i, tcC.at(i).elemLess(.{ m, n })); - - b.copy(atom, tgA, rA, .{ .pred = pC }); - b.copy(atom, tgB, rB, .{ .pred = pC }); - rC.store(rA.load().add(rB.load())); - b.copy(atom, rC, tgC, .{ .pred = pC }); -} - -// In a model: -const c = VectorAdd.call(.{ .a = a, .b = bt }, .{ .c = a.shape() }, .{ - .cfg = .{}, - .grid = .{ grid_m, grid_n, 1 }, - .threads = 128, -}).c; -``` - -Complete vector-add and tiled-MMA kernels are the tests in `builder.zig` -and `zml/kernel/fly/test.zig`. - -Model kernels live beside their consumers: - -- [MoE routing and MXFP4 projections](../../zml/kernels/moe/fly/moe.zig). -- Sparse MLA, downstream in llmd (`llmd/kernels/mla/fly/sparse_mla.zig`). - -Fly is a separate backend. The [MoE selector](../../zml/kernels/moe/moe.zig) and -llmd's sparse-MLA selector select by platform, -architecture and dtype. The current native kernels target gfx942; their -callers check shape support before routing or launching kernels and use -Triton for unsupported contracts. - -## The ABI - -`Kernel.emit(allocator, cfg, shapes)` takes shapes in input-then-output order -and produces - -```mlir -module attributes {gpu.container_module} { - gpu.module @zml_fly_kernels { - gpu.func @name(%a: !fly.ptr, ..., %c: !fly.ptr) kernel { ... gpu.return } - } -} -``` - -with one `!fly.ptr` per input, then per output. The plugin checks only the -argument **count**, so the pointer element type is on you: `DType.storageElem` -maps predicates to `i8` because that is how XLA stores them. Static DSL values -(layouts, tiles, atoms) must be built inside the body, never passed as -arguments. `Kernel.call` bridges each non-scalar argument to a row-major -`!fly.memref` view of its `zml.Shape`. Rank-zero arguments remain pointers; -use `ptrLoad` and `ptrStore` for their scalar values. - -Launch geometry lives in `CallOpts`: `grid`, `threads` (threads per block, -converted to wavefronts using the device's wave size — 64 on CDNA, 32 on -RDNA — so it must be a multiple of that), optional `waves_per_eu` (sets the -LLVM `amdgpu-waves-per-eu` attribute to `N, N` when positive), -`shared_mem_bytes` (only for `fly.get_dyn_shared`; static LDS from -`sharedArray` needs none) and `zeroed_args` (indices into inputs-then-outputs; -ignored inside HIP graphs, so zero what you need yourself). - -`output_operand_aliases` maps output names to input names for buffers that -the kernel reads and writes. Aliasing preserves the input-then-output ABI; -it does not initialize output memory. - -## Layouts, tiles, coordinates - -`layout.zig` holds comptime literals. `toAttr` turns one into the matching -fly attribute through the dialect's own constructors, and reading a type back -goes through its accessors — neither direction goes through text: - -| Zig | MLIR | -|---|---| -| `fly.it(.{ .{ 2, 4 }, 8 })` | `!fly.int_tuple<((2,4),8)>` | -| `fly.L(.{ 4, 8 }, .{ 1, 4 })` | `!fly.layout<(4,8):(1,4)>` | -| `fly.rowMajor(.{ 4, 8 })` / `fly.colMajor` | `(4,8):(8,1)` / `(4,8):(1,4)` | -| `fly.ordered(.{ 8, 16 }, .{ 1, 0 })` | `fx.make_ordered_layout((8,16),(1,0))`, folded | -| `fly.tile(.{ 128, 64 })`, `fly.tile(.{ null, 8 })` | `!fly.tile<[128\|64]>`, `[*\|8]` | - -`b.static(literal)` materializes one as `fly.static`. Wherever the Python -takes a tuple, the Zig takes a tuple literal: `t.flatDivide(.{ 64, 8 })` is a -tile, `t.slice(.{ null, bid })` is `t[None, bid]`, `b.intTuple(.{ m, tid })` -mixes static ints, `null` (`*`) and dynamic `Value`s (`?` leaves, passed as -operands). `v.shapeStatic()` / `v.sizeStatic()` / `v.elemDType()` read the -inferred type back on the host; `v.emitShape()` / `v.emitSize()` emit ops. - -## Atoms and tiled ops - -- `b.copyAtom(.{ .universal = 128 }, .f32)`, `.{ .buffer_copy = 32 }`, - `.{ .buffer_copy_lds = 128 }`. -- `b.mmaAtom(mma_op)` takes an MLIR type. Generated MMA type constructors - return a typed wrapper; handle their error and pass the wrapper's - `.type_()` result. -- `b.tiledCopyTV(atom, thr, val)`, `b.tiledCopy(atom, layout_tv, tile)`, - `b.tiledCopyA/B/C(copy_atom, tiled_mma)`; `tc.getSlice(tid).partitionS/D`, - `.retile`. -- `b.tiledMma(atom, fly.L(.{ 2, 2, 1 }, .{ 1, 2, 0 }), null)`; - `tm.makeFragmentA/B/C(t)`, `tm.getSlice(tid).partitionA/B/C`. -- `b.copy(atom, src, dst, .{ .pred = p })`, `b.copyAtomCall(atom, src, dst)`, - `b.gemm(atom, d, a, b, c)`. -- `b.mmaAtomCall(atom, a, b, c)` takes SSA operands and returns the updated - accumulator. Operand vectors must match the atom's per-lane fragment - types and layout; this call performs no tiled partitioning. -- `b.bufferTensor(t)` is `fx.rocdl.make_buffer_tensor`: the same tensor over - a CDNA raw-buffer descriptor, for bounds-checked `buffer_copy` atoms. -- Derived layouts come off the type: `@import("mlir/dialects/fly").expect(tm.value.type_(), .tiled_mma)` - then `.tileSizeMNK()`, `.tiledThrValLayoutA()`, ... - -## Memory and scalars - -`t.load()` / `t.store(vec)` move a whole register tensor as a `vector`; -`t.at(coord)` / `t.set(coord, v)` are element accesses (an int tuple for a -coordinate tensor); `t.fill(0)` splats. `b.rmemTensor(layout, dtype)` is -`fx.make_rmem_tensor`; `b.sharedArray(.f32, n, 16)` is a static LDS array -(`fx.SharedAllocator().allocate(fx.Array[...])`). - -Scalars and vectors: `add/sub/mul/div/rem` and `lt(x)` (`cmp` for integers, -`cmpf` for floats; `cmpUnsigned`) broadcast a scalar against a vector; -`to(dtype)`, `cond.select(a, b)`, `v.reduce(.add)`, `v.shuffleXor(off, 64)`, -`b.rsqrt(x)`, `b.constant(.f32, 1.0)`, `b.threadId(.x)` / `b.blockId(.x)` -(already `i32`), `b.barrier()`. - -Integer types are signless in MLIR. `div`, `rem`, `cmp` and `to` use signed -integer semantics, except that `to` treats predicates as unsigned. -`divUnsigned`, `remUnsigned`, `cmpUnsigned` and `toUnsigned` make unsigned -operations explicit. `bitAnd`, `bitOr`, `bitXor`, `shl` and `shrU` operate -on integer bits. - -`splat` creates a vector, including when its length is one. Numeric casts -preserve vector shape. `bitCast` preserves total bits and adjusts vector -length to the destination element width; scalar bitcasts require equal -widths. Use `extract` to obtain a scalar, `insert` to replace an element and -`shuffleVector` to select elements from two vectors. Lane `shuffle` -supports `xor`, `up`, `down` and `idx` modes with a literal or an `i32` -`Value` offset. - -`maximum` propagates NaNs; `maxNum` selects the numeric operand when only -one operand is NaN. `neg`, `Builder.log` and `Builder.fma` provide floating -operations. Fast math is off by default; `setFastMath(true)` tags subsequent -float operations `fastmath` and permits relaxed numerical semantics. - -`Builder.perm` emits AMD byte permutation. `Builder.cvtPkF32Fp8` decodes -two FP8 bytes from the low or high half of an `i32` word using the CDNA3 -packed conversion instruction. Architecture selection belongs in the -backend before these operations are emitted. - -`Builder.ptrAtomicAdd` adds an `i32` counter and returns its old value. -Use `workgroup` scope for LDS pointers and `agent` scope for global -pointers. The ordering is monotonic; callers provide barriers or kernel -dependencies before consuming completed counters. - -Control flow uses `dsl/common`: `b.openFor(lo, hi, step, .{ inits })` -with `.yield(.{ ... })` (`.close()` without inits), `b.openIf(cond)` with -`.close()`, `b.openIfElse(cond, .{ types })` with `.yieldThen` / `.yieldElse` -(`.elseBranch()` / `.close()` without types). Types are dtypes, Values for -their types, or MLIR types; tuples can also be arrays. -`openWhile` has separate condition and body regions: `before_carried` -feeds `yieldBefore`, `after_carried` feeds `yieldAfter`, and `results` -contains the values returned when the condition becomes false. - -## Control flow and register tensors - -The published ROCm plugin lowers `scf` to `cf` **before** running the Fly -passes on a native `gpu.func` entry, and Fly's register-alloca promotion -only rewrites a single-block body. Kernels with runtime control flow must -keep their register state in SSA values: - -- `copyAtomLoad` returns a vector and `copyAtomStore` stores a vector. -- `mmaAtomCall` returns an accumulator without allocating register memory. -- Pass accumulators through `openFor` initial values, `carried`, `yield` - and `results`. Branches and while loops also return SSA values through - their scope objects. - -Put each scope's body in a Zig `{ ... }` block right after `openFor`, -`openIf` or `openWhile`, ending with its `yield` (or `close()` for a scope -without values). `openIfElse` and `openWhile` get one block per region, ending -with `yieldThen`/`yieldElse` (`elseBranch()`/`close()` without results) and -`yieldBefore`/`yieldAfter`. The scope variable stays outside, so its `results` -stay readable after the blocks. - -The MoE and sparse-MLA kernels linked above use this path with the published -runtime. Straight-line kernels can still use `rmemTensor`, -`makeFragmentLike`, `fill` and `gemm`. - -## Debugging - -- A failing `inferReturnTypes` panics with the op name and every operand type - (`fly.make_tiled_copy: cannot create operation ... operand 1: !fly.layout<...>`). -- `Kernel.emit` returns the module text; `bazel test //dsl/fly:test` - builds the example kernels on any host, no GPU needed. -- On the device side, `--xla_dump_to=` dumps the module the plugin saw. diff --git a/dsl/fly/builder.zig b/dsl/fly/builder.zig deleted file mode 100644 index a083d943..00000000 --- a/dsl/fly/builder.zig +++ /dev/null @@ -1,1978 +0,0 @@ -const std = @import("std"); - -const arith = @import("mlir/dialects/arith"); -const common = @import("dsl/common/builder"); -const fly = @import("mlir/dialects/fly"); -const func = @import("mlir/dialects/func"); -const gpu = @import("mlir/dialects/gpu"); -const llvm = @import("mlir/dialects/llvm"); -const math = @import("mlir/dialects/math"); -const memref = @import("mlir/dialects/memref"); -const rocdl = @import("mlir/dialects/rocdl"); -const scf = @import("mlir/dialects/scf"); -const ub = @import("mlir/dialects/ub"); -const vector = @import("mlir/dialects/vector"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -pub const layout = @import("layout.zig"); -pub const dtypes = @import("dtype.zig"); - -pub const DType = dtypes.DType; -pub const fly_rocdl = @import("mlir/dialects/fly_rocdl"); -pub const IntTuple = layout.IntTuple; -pub const Layout = layout.Layout; -pub const Tile = layout.Tile; -pub const it = layout.it; -pub const L = layout.L; -pub const rowMajor = layout.rowMajor; -pub const colMajor = layout.colMajor; -pub const ordered = layout.ordered; -pub const tile = layout.tile; - -pub fn registerDialects(registry: *mlir.DialectRegistry) void { - inline for (.{ func, arith, scf, math, vector, memref, llvm, gpu, rocdl, ub, fly, fly_rocdl }) |dialect| dialect.registerDialect(registry); -} - -pub const FinishError = error{ InvalidMlir, OutOfMemory } || std.Io.Writer.Error; - -pub const Arch = enum { gfx942, gfx950 }; - -pub const Dim = gpu.Dim; - -pub const gpu_module_name = "zml_fly_kernels"; - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Builder); - std.testing.refAllDecls(Value); -} - -// ============================================================================= -// Value -// ============================================================================= - -pub const Value = struct { - inner: *const mlir.Value, - kernel: ?*Builder = null, - - pub const Kind = fly.TypeKind; - - pub fn type_(self: Value) *const mlir.Type { - return self.inner.type_(); - } - - fn kern(self: Value) *Builder { - return self.kernel orelse @panic("fly.Value has no owning Builder; use Builder.* helpers instead"); - } - - /// Derived, never stored: control_flow.zig builds Values with bare - /// `.{ .inner, .kernel }` literals. - pub fn kind(self: Value) Kind { - return fly.typeKind(self.type_()); - } - - pub fn shapeStatic(self: Value) IntTuple { - return self.kern().readIntTuple(fly.layoutLikeShape(self.type_())); - } - - /// Host value, not an op; see `emitSize`. - pub fn sizeStatic(self: Value) i64 { - return self.shapeStatic().product() orelse std.debug.panic("fly: `{f}` has a dynamic shape", .{self.type_()}); - } - - pub fn intTupleStatic(self: Value) IntTuple { - return self.kern().readIntTuple(fly.expect(self.type_(), .int_tuple)); - } - - pub fn elemDType(self: Value) DType { - const t = fly.elemType(self.type_()); - return dtypes.fromMlir(self.kern().ctx, t) orelse std.debug.panic("fly: unsupported element type `{f}`", .{t}); - } - - fn isI64(self: Value) bool { - const int_ty = self.type_().isA(mlir.IntegerType) orelse return false; - return int_ty.width() == 64; - } - - fn isFloatElem(self: Value) bool { - return self.scalarElemType().isFloat(); - } - - /// The element type of a vector, or the type itself. - fn scalarElemType(self: Value) *const mlir.Type { - if (self.type_().isA(mlir.VectorType)) |vec| return vec.shaped().elementType(); - return self.type_(); - } - - fn scalarDType(self: Value) DType { - const t = self.scalarElemType(); - return dtypes.fromMlir(self.kern().ctx, t) orelse std.debug.panic("fly: unsupported scalar type `{f}`", .{t}); - } - - // ------------------------------------------------------- layout algebra - // Every op below takes a Value or a comptime Layout/Tile/tuple literal, - // and FlyDSL infers the result type. - - fn unary(self: Value, comptime build: anytype) Value { - const k = self.kern(); - return k.emit(build(k.ctx, self.inner, k.loc())); - } - - fn binary(self: Value, comptime build: anytype, rhs: anytype) Value { - const k = self.kern(); - const r = k.toStatic(rhs); - return k.emit(build(k.ctx, self.inner, r.inner, k.loc())); - } - - pub fn flatDivide(self: Value, tiler: anytype) Value { - return self.binary(fly.ops.flat_divide, tiler); - } - pub fn logicalDivide(self: Value, tiler: anytype) Value { - return self.binary(fly.ops.logical_divide, tiler); - } - pub fn zippedDivide(self: Value, tiler: anytype) Value { - return self.binary(fly.ops.zipped_divide, tiler); - } - pub fn tiledDivide(self: Value, tiler: anytype) Value { - return self.binary(fly.ops.tiled_divide, tiler); - } - pub fn logicalProduct(self: Value, tiler: anytype) Value { - return self.binary(fly.ops.logical_product, tiler); - } - pub fn zippedProduct(self: Value, tiler: anytype) Value { - return self.binary(fly.ops.zipped_product, tiler); - } - pub fn tiledProduct(self: Value, tiler: anytype) Value { - return self.binary(fly.ops.tiled_product, tiler); - } - pub fn flatProduct(self: Value, tiler: anytype) Value { - return self.binary(fly.ops.flat_product, tiler); - } - pub fn blockedProduct(self: Value, tiler: anytype) Value { - return self.binary(fly.ops.blocked_product, tiler); - } - pub fn rakedProduct(self: Value, tiler: anytype) Value { - return self.binary(fly.ops.raked_product, tiler); - } - pub fn composition(self: Value, inner: anytype) Value { - return self.binary(fly.ops.composition, inner); - } - pub fn rightInverse(self: Value) Value { - return self.unary(fly.ops.right_inverse); - } - pub fn leftInverse(self: Value) Value { - return self.unary(fly.ops.left_inverse); - } - pub fn coalesce(self: Value) Value { - const k = self.kern(); - return k.emit(fly.ops.coalesce(k.ctx, self.inner, .{}, k.loc())); - } - pub fn complement(self: Value) Value { - const k = self.kern(); - return k.emit(fly.ops.complement(k.ctx, self.inner, .{}, k.loc())); - } - - /// `t[coord]` with a `null` somewhere; `coord` is a tuple literal - /// (`.{ null, null, bx, by }`, nesting allowed) or an int-tuple Value. - pub fn slice(self: Value, coord: anytype) Value { - const k = self.kern(); - const c = k.intTuple(coord); - return k.emit(fly.ops.slice(k.ctx, self.inner, c.inner, k.loc())); - } - pub fn dice(self: Value, coord: anytype) Value { - const k = self.kern(); - const c = k.intTuple(coord); - return k.emit(fly.ops.dice(k.ctx, self.inner, c.inner, k.loc())); - } - - /// The sub-tuple / sub-layout at a mode path. - pub fn get(self: Value, mode: []const i32) Value { - const k = self.kern(); - return k.emit(fly.ops.get(k.ctx, self.inner, .denseArray(k.ctx, .i32, mode), k.loc())); - } - pub fn selectModes(self: Value, indices: []const i32) Value { - const k = self.kern(); - return k.emit(fly.ops.select(k.ctx, self.inner, .denseArray(k.ctx, .i32, indices), k.loc())); - } - pub fn take(self: Value, begin: i32, end: i32) Value { - const k = self.kern(); - return k.emit(fly.ops.take(k.ctx, self.inner, .int(k.ctx, .i32, begin), .int(k.ctx, .i32, end), k.loc())); - } - pub fn group(self: Value, begin: i32, end: i32) Value { - const k = self.kern(); - return k.emit(fly.ops.group(k.ctx, self.inner, .int(k.ctx, .i32, begin), .int(k.ctx, .i32, end), k.loc())); - } - - /// Emits an op; `shapeStatic` reads the type instead. - pub fn emitShape(self: Value) Value { - return self.unary(fly.ops.get_shape); - } - pub fn emitStride(self: Value) Value { - return self.unary(fly.ops.get_stride); - } - pub fn emitLayout(self: Value) Value { - return self.unary(fly.ops.get_layout); - } - pub fn emitIter(self: Value) Value { - return self.unary(fly.ops.get_iter); - } - pub fn emitSize(self: Value) Value { - return self.unary(fly.ops.size); - } - pub fn cosize(self: Value) Value { - return self.unary(fly.ops.cosize); - } - pub fn productEach(self: Value) Value { - return self.unary(fly.ops.int_tuple_product_each); - } - - /// A register tensor shaped like `self`. - pub fn makeFragmentLike(self: Value, dtype: ?DType) Value { - const k = self.kern(); - const dtype_attr: ?*const mlir.Attribute = if (dtype) |d| .typeAttr(d.toMlir(k.ctx)) else null; - return k.emit(fly.ops.make_fragment_like(k.ctx, self.inner, .{ .dtype = dtype_attr }, k.loc())); - } - - pub fn elemLess(self: Value, rhs: anytype) Value { - const k = self.kern(); - const r = k.intTuple(rhs); - return k.emit(fly.ops.elem_less(k.ctx, self.inner, r.inner, .int(k.ctx, .i1), k.loc())); - } - - // ---------------------------------------------------------------- tensor access - - /// `t[coord]` with no `null` (a coord tensor yields an int tuple). - pub fn at(self: Value, coord: anytype) Value { - const k = self.kern(); - const c = k.intTuple(coord); - return k.emit(fly.ops.memref_load(k.ctx, self.inner, c.inner, k.loc())); - } - - pub fn set(self: Value, coord: anytype, value: Value) void { - const k = self.kern(); - const c = k.intTuple(coord); - k.emitNone(fly.ops.memref_store(k.ctx, value.inner, self.inner, c.inner, k.loc())); - } - - /// The whole register tensor as a `vector`. - pub fn load(self: Value) Value { - return self.unary(fly.ops.memref_load_vec); - } - - pub fn store(self: Value, vector_: Value) void { - const k = self.kern(); - k.emitNone(fly.ops.memref_store_vec(k.ctx, vector_.inner, self.inner, k.loc())); - } - - /// Store a splat vector of the tensor's static size. - pub fn fill(self: Value, value: anytype) void { - const k = self.kern(); - const n = self.sizeStatic(); - const dt = self.elemDType(); - const cst = k.constant(dt, value); - const vec_ty = mlir.Type.vector(&.{n}, dt.toMlir(k.ctx)); - const vec = k.emit(vector.broadcast(k.ctx, cst.inner, vec_ty, k.loc())); - self.store(vec); - } - - // ---------------------------------------------------------------- pointers - - pub fn addOffset(self: Value, offset: anytype) Value { - const k = self.kern(); - const o = k.intTuple(offset); - return k.emit(fly.ops.add_offset(k.ctx, self.inner, o.inner, k.loc())); - } - - /// Scalar load. - pub fn ptrLoad(self: Value) Value { - const k = self.kern(); - const elem = self.elemDType().toMlir(k.ctx); - return k.emit(fly.ops.ptr_load(k.ctx, self.inner, elem, k.loc())); - } - - pub fn ptrStore(self: Value, value: Value) void { - const k = self.kern(); - k.emitNone(fly.ops.ptr_store(k.ctx, value.inner, self.inner, k.loc())); - } - - /// A tensor over this pointer. - pub fn view(self: Value, lay: anytype) Value { - return self.binary(fly.ops.make_view, lay); - } - - // ---------------------------------------------------------------- arithmetic - - /// Broadcast a scalar against a vector operand. - fn coerceArith(self: Value, rhs: anytype) struct { Value, Value } { - const k = self.kern(); - const r: Value = if (@TypeOf(rhs) == Value) rhs else k.constantLike(rhs, self); - if (self.type_().eql(r.type_())) return .{ self, r }; - if (self.type_().isA(mlir.VectorType) != null and r.type_().isA(mlir.VectorType) == null) - return .{ self, k.emit(vector.broadcast(k.ctx, r.inner, self.type_(), k.loc())) }; - if (self.type_().isA(mlir.VectorType) == null and r.type_().isA(mlir.VectorType) != null) - return .{ k.emit(vector.broadcast(k.ctx, self.inner, r.type_(), k.loc())), r }; - std.debug.panic("fly: cannot combine {f} with {f}", .{ self.type_(), r.type_() }); - } - - pub fn add(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emitFast(if (l.isFloatElem()) arith.addf(k.ctx, l.inner, r.inner, k.loc()) else arith.addi(k.ctx, l.inner, r.inner, k.loc())); - } - pub fn sub(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emitFast(if (l.isFloatElem()) arith.subf(k.ctx, l.inner, r.inner, k.loc()) else arith.subi(k.ctx, l.inner, r.inner, k.loc())); - } - pub fn mul(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emitFast(if (l.isFloatElem()) arith.mulf(k.ctx, l.inner, r.inner, k.loc()) else arith.muli(k.ctx, l.inner, r.inner, k.loc())); - } - pub fn div(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emitFast(if (l.isFloatElem()) arith.divf(k.ctx, l.inner, r.inner, k.loc()) else arith.divsi(k.ctx, l.inner, r.inner, k.loc())); - } - /// Signed. - pub fn rem(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emit(arith.remsi(k.ctx, l.inner, r.inner, k.loc())); - } - - pub fn bitAnd(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emit(arith.andi(k.ctx, l.inner, r.inner, k.loc())); - } - pub fn bitOr(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emit(arith.ori(k.ctx, l.inner, r.inner, k.loc())); - } - pub fn bitXor(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emit(arith.xori(k.ctx, l.inner, r.inner, k.loc())); - } - pub fn shl(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emit(arith.shli(k.ctx, l.inner, r.inner, k.loc())); - } - pub fn shrU(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emit(arith.shrui(k.ctx, l.inner, r.inner, k.loc())); - } - pub fn divUnsigned(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emit(arith.divui(k.ctx, l.inner, r.inner, k.loc())); - } - pub fn remUnsigned(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emit(arith.remui(k.ctx, l.inner, r.inner, k.loc())); - } - pub fn cmpUnsigned(self: Value, pred: Cmp, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - const p: arith.CmpIPredicate = switch (pred) { - .eq => .eq, - .ne => .ne, - .lt => .ult, - .le => .ule, - .gt => .ugt, - .ge => .uge, - }; - return k.emit(arith.cmpi(k.ctx, p, l.inner, r.inner, k.loc())); - } - - pub fn neg(self: Value) Value { - const k = self.kern(); - return k.emitFast(arith.negf(k.ctx, self.inner, k.loc())); - } - /// Propagates NaNs; use `maxNum` to select the numeric operand instead. - pub fn maximum(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emitFast(arith.maximumf(k.ctx, l.inner, r.inner, k.loc())); - } - pub fn maxNum(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - return k.emitFast(arith.maxnumf(k.ctx, l.inner, r.inner, k.loc())); - } - - /// Interpret integer operands and results as unsigned. - pub fn toUnsigned(self: Value, dtype: DType) Value { - const k = self.kern(); - const src = self.scalarDType(); - if (src == dtype) return self; - if (src.isFloat() and dtype.isFloat()) return self.to(dtype); - const result = self.withElemType(dtype.toMlir(k.ctx)); - const op = if (src.isFloat()) - arith.fptoui(k.ctx, self.inner, result, k.loc()) - else if (dtype.isFloat()) - arith.uitofp(k.ctx, self.inner, result, k.loc()) - else if (dtype.bitWidth() > src.bitWidth()) - arith.extui(k.ctx, self.inner, result, k.loc()) - else - arith.trunci(k.ctx, self.inner, result, k.loc()); - return k.emit(op); - } - - pub fn splat(self: Value, n: i64) Value { - const k = self.kern(); - std.debug.assert(self.type_().isA(mlir.VectorType) == null and n > 0); - return k.emit(vector.broadcast(k.ctx, self.inner, .vector(&.{n}, self.type_()), k.loc())); - } - - /// Preserve total bits, adjusting vector length to the destination element. - pub fn bitCast(self: Value, dtype: DType) Value { - const k = self.kern(); - const src = self.scalarDType(); - if (self.type_().isA(mlir.VectorType)) |vec| { - const n = @divExact(vec.dimension(0) * @as(i64, src.bitWidth()), dtype.bitWidth()); - return k.emit(vector.bitcast(k.ctx, self.inner, .vector(&.{n}, dtype.toMlir(k.ctx)), k.loc())); - } - std.debug.assert(src.bitWidth() == dtype.bitWidth()); - return k.emit(arith.bitcast(k.ctx, self.inner, dtype.toMlir(k.ctx), k.loc())); - } - - pub fn insert(self: Value, index: i64, value: Value) Value { - const k = self.kern(); - return k.emit(vector.insert(k.ctx, value.inner, self.inner, &.{index}, &.{}, k.loc())); - } - - pub fn shuffleVector(self: Value, rhs: Value, indices: []const i64) Value { - const k = self.kern(); - return k.emit(vector.shuffle(k.ctx, self.inner, rhs.inner, indices, .vector(&.{@intCast(indices.len)}, self.scalarElemType()), k.loc())); - } - - fn withElemType(self: Value, elem: *const mlir.Type) *const mlir.Type { - if (self.type_().isA(mlir.VectorType)) |vec| return .vector(&.{vec.dimension(0)}, elem); - return elem; - } - - pub fn toI32(self: Value) Value { - const k = self.kern(); - return k.emit(arith.index_cast(k.ctx, self.inner, .int(k.ctx, .i32), k.loc())); - } - - /// 1 for a scalar. - pub fn vectorLen(self: Value) i64 { - const vec = self.type_().isA(mlir.VectorType) orelse return 1; - return vec.dimension(0); - } - - /// Numeric conversion, scalar or vector. `i1` is treated as unsigned, so a - /// predicate converts to 1, not -1; equal-width float pairs (f16 <-> bf16, - /// the fp8 spellings) go through `arith.convertf`, which is the only op - /// that accepts them. - pub fn to(self: Value, dtype: DType) Value { - const k = self.kern(); - const src = self.scalarDType(); - if (src == dtype) return self; - const result = self.withElemType(dtype.toMlir(k.ctx)); - const unsigned_src = src == .i1; - const op = if (src.isFloat() and dtype.isFloat()) - (if (dtype.bitWidth() > src.bitWidth()) - arith.extf(k.ctx, self.inner, result, k.loc()) - else if (dtype.bitWidth() < src.bitWidth()) - arith.truncf(k.ctx, self.inner, result, k.loc()) - else - arith.convertf(k.ctx, self.inner, result, .{}, k.loc())) - else if (src.isFloat()) - arith.fptosi(k.ctx, self.inner, result, k.loc()) - else if (dtype.isFloat()) - (if (unsigned_src) arith.uitofp(k.ctx, self.inner, result, k.loc()) else arith.sitofp(k.ctx, self.inner, result, k.loc())) - else if (dtype.bitWidth() > src.bitWidth()) - (if (unsigned_src) arith.extui(k.ctx, self.inner, result, k.loc()) else arith.extsi(k.ctx, self.inner, result, k.loc())) - else - arith.trunci(k.ctx, self.inner, result, k.loc()); - return k.emit(op); - } - - pub const Cmp = enum { eq, ne, lt, le, gt, ge }; - - /// Signed integer comparison against a Value or an integer literal; a - /// scalar broadcasts against a vector. - pub fn cmp(self: Value, pred: Cmp, rhs: anytype) Value { - const k = self.kern(); - const l, const r = self.coerceArith(rhs); - const p: arith.CmpIPredicate = switch (pred) { - .eq => .eq, - .ne => .ne, - .lt => .slt, - .le => .sle, - .gt => .sgt, - .ge => .sge, - }; - return k.emit(arith.cmpi(k.ctx, p, l.inner, r.inner, k.loc())); - } - - /// Ordered. - pub fn cmpf(self: Value, pred: Cmp, rhs: Value) Value { - const k = self.kern(); - const p: arith.CmpFPredicate = switch (pred) { - .eq => .oeq, - .ne => .one, - .lt => .olt, - .le => .ole, - .gt => .ogt, - .ge => .oge, - }; - return k.emit(arith.cmpf(k.ctx, p, self.inner, rhs.inner, k.loc())); - } - - pub fn lt(self: Value, rhs: anytype) Value { - return self.compare(.lt, rhs); - } - - pub fn le(self: Value, rhs: anytype) Value { - return self.compare(.le, rhs); - } - - pub fn gt(self: Value, rhs: anytype) Value { - return self.compare(.gt, rhs); - } - - pub fn ge(self: Value, rhs: anytype) Value { - return self.compare(.ge, rhs); - } - - pub fn eq(self: Value, rhs: anytype) Value { - return self.compare(.eq, rhs); - } - - pub fn ne(self: Value, rhs: anytype) Value { - return self.compare(.ne, rhs); - } - - /// `cmpf` for floats, `cmp` (signed) for integers, `index` included. - fn compare(self: Value, pred: Cmp, rhs: anytype) Value { - if (!self.isFloatElem()) return self.cmp(pred, rhs); - const l, const r = self.coerceArith(rhs); - return l.cmpf(pred, r); - } - - pub fn select(self: Value, a: Value, b: Value) Value { - const k = self.kern(); - return k.emit(arith.select(k.ctx, self.inner, a.inner, b.inner, k.loc())); - } - - /// Lane `k` reads lane `k ^ offset`, within `width` lanes. - pub fn shuffleXor(self: Value, offset: i32, width: i32) Value { - return self.shuffle(.xor, offset, width); - } - - pub const ShuffleMode = gpu.ShuffleMode; - - pub fn shuffle(self: Value, mode: ShuffleMode, offset: anytype, width: i32) Value { - const k = self.kern(); - const off: Value = if (@TypeOf(offset) == Value) offset else k.constant(.i32, offset); - const wid = k.constant(.i32, width); - return k.emit(gpu.shuffle(k.ctx, self.inner, off.inner, wid.inner, mode, k.loc())); - } - - pub const ReduceKind = enum { add, mul, minf, maxf, minsi, maxsi, and_, or_, xor }; - - /// Reduce a vector to a scalar. - pub fn reduce(self: Value, reduce_kind: ReduceKind) Value { - const k = self.kern(); - const result = self.scalarElemType(); - const ck: vector.CombiningKind = switch (reduce_kind) { - .add => .add, - .mul => .mul, - .minf => .minnumf, - .maxf => .maxnumf, - .minsi => .minsi, - .maxsi => .maxsi, - .and_ => .@"and", - .or_ => .@"or", - .xor => .xor, - }; - const op = vector.reduction(k.ctx, ck, self.inner, result, .{}, k.loc()); - return k.emitFast(op); - } - - /// Static position. - pub fn extract(self: Value, i: i64) Value { - const k = self.kern(); - return k.emit(vector.extract(k.ctx, self.inner, &.{i}, &.{}, self.scalarElemType(), k.loc())); - } -}; - -// ============================================================================= -// Tiled copy / MMA handles -// ============================================================================= - -/// A `!fly.tiled_copy` plus the tile it was built with. -pub const TiledCopy = struct { - value: Value, - tile: Value, - - pub fn getSlice(self: TiledCopy, thr_idx: Value) ThrCopy { - return .{ .tiled_copy = self.value, .thr = self.value.kern().intTuple(thr_idx) }; - } - - pub fn tileMN(self: TiledCopy) Value { - return self.tile; - } -}; - -/// Per-thread slice of a TiledCopy. -pub const ThrCopy = struct { - tiled_copy: Value, - thr: Value, - - pub fn partitionS(self: ThrCopy, src: Value) Value { - const k = self.tiled_copy.kern(); - return k.emit(fly.ops.tiled_copy_partition_src(k.ctx, self.tiled_copy.inner, src.inner, self.thr.inner, k.loc())); - } - - pub fn partitionD(self: ThrCopy, dst: Value) Value { - const k = self.tiled_copy.kern(); - return k.emit(fly.ops.tiled_copy_partition_dst(k.ctx, self.tiled_copy.inner, dst.inner, self.thr.inner, k.loc())); - } - - pub fn retile(self: ThrCopy, t: Value) Value { - const k = self.tiled_copy.kern(); - return k.emit(fly.ops.tiled_copy_retile(k.ctx, self.tiled_copy.inner, t.inner, k.loc())); - } -}; - -pub const TiledMma = struct { - value: Value, - - pub fn getSlice(self: TiledMma, thr_idx: Value) ThrMma { - return .{ .tiled_mma = self.value, .thr = self.value.kern().intTuple(thr_idx) }; - } - - pub const FragmentOpts = struct { - stages: ?i32 = null, - }; - - fn makeFragment(self: TiledMma, operand: fly.MmaOperand, input: Value, opts: FragmentOpts) Value { - const k = self.value.kern(); - const operand_id = Builder.must(fly.attributes.mmaOperandAttr(k.ctx, operand)); - const stages: ?*const mlir.Attribute = if (opts.stages) |st| .int(k.ctx, .i32, st) else null; - return k.emit(fly.ops.mma_make_fragment(k.ctx, self.value.inner, input.inner, operand_id, .{ .stages = stages }, k.loc())); - } - - /// Per-thread register fragment for the A operand. - pub fn makeFragmentA(self: TiledMma, a: Value) Value { - return self.makeFragment(.a, a, .{}); - } - pub fn makeFragmentB(self: TiledMma, b: Value) Value { - return self.makeFragment(.b, b, .{}); - } - pub fn makeFragmentC(self: TiledMma, c: Value) Value { - return self.makeFragment(.c, c, .{}); - } - pub fn makeFragmentAOpts(self: TiledMma, a: Value, opts: FragmentOpts) Value { - return self.makeFragment(.a, a, opts); - } - pub fn makeFragmentBOpts(self: TiledMma, b: Value, opts: FragmentOpts) Value { - return self.makeFragment(.b, b, opts); - } -}; - -pub const ThrMma = struct { - tiled_mma: Value, - thr: Value, - - fn partition(self: ThrMma, operand: fly.MmaOperand, t: Value) Value { - const k = self.tiled_mma.kern(); - const operand_id = Builder.must(fly.attributes.mmaOperandAttr(k.ctx, operand)); - return k.emit(fly.ops.tiled_mma_partition(k.ctx, self.tiled_mma.inner, t.inner, self.thr.inner, operand_id, k.loc())); - } - - pub fn partitionA(self: ThrMma, a: Value) Value { - return self.partition(.a, a); - } - pub fn partitionB(self: ThrMma, b: Value) Value { - return self.partition(.b, b); - } - pub fn partitionC(self: ThrMma, c: Value) Value { - return self.partition(.c, c); - } -}; - -// ============================================================================= -// Builder -// ============================================================================= - -pub const Builder = struct { - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - ctx: *mlir.Context, - /// `builtin.module attributes {gpu.container_module}`. - module: *mlir.Module, - gpu_module: *mlir.Operation, - gpu_body: *mlir.Block, - name: []const u8, - func_op: ?*mlir.Operation = null, - entry_block: ?*mlir.Block = null, - block_stack: common.BlockStack = .{}, - arg_specs: []const ArgSpec = &.{}, - /// The `!fly.memref` view for tensor arguments, the raw `!fly.ptr` otherwise. - arg_values: []Value = &.{}, - target: Arch = .gfx942, - fast_math: bool = false, - - pub const ArgSpec = struct { - name: []const u8, - dtype: DType, - space: fly.AddressSpace = .global, - /// Row-major tensor argument; null for a bare pointer. - dims: ?[]const i64 = null, - }; - - pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { - const loc_: *const mlir.Location = .unknown(ctx); - const module: *mlir.Module = .init(loc_); - errdefer module.deinit(); - module.operation().setAttribute(gpu.containerModule(ctx)); - - const gpu_body = mlir.Block.init(&.{}, &.{}); - const gpu_module = gpu.module(ctx, gpu_module_name, gpu_body, loc_); - _ = gpu_module.appendTo(module.body()); - - var arena: std.heap.ArenaAllocator = .init(allocator); - errdefer arena.deinit(); - - return .{ - .allocator = allocator, - .arena = arena, - .ctx = ctx, - .module = module, - .gpu_module = gpu_module, - .gpu_body = gpu_body, - .name = name, - }; - } - - pub fn deinit(self: *Builder) void { - self.module.deinit(); - self.arena.deinit(); - } - - /// Arguments by name. Each field is `.{ .ptr = dtype }` or - /// `.{ .tensor = .{ .dtype = ..., .dims = &.{ ... } } }`. - pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { - const Spec = @TypeOf(spec); - const fields = @typeInfo(Spec).@"struct".field_names; - var specs: [fields.len]ArgSpec = undefined; - inline for (fields, 0..) |f, i| { - const raw = @field(spec, f); - const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; - const inner = @field(raw, variant); - specs[i] = if (comptime std.mem.eql(u8, variant, "ptr")) - .{ .name = f, .dtype = inner } - else if (comptime std.mem.eql(u8, variant, "tensor")) - .{ .name = f, .dtype = inner.dtype, .dims = inner.dims, .space = if (@hasField(@TypeOf(inner), "space")) inner.space else .global } - else - @compileError("fly.declareArgs: argument `" ++ f ++ "` must be `.{ .ptr = dtype }` or `.{ .tensor = .{ .dtype, .dims } }`"); - } - try self.declareArgsLow(&specs); - var named: common.NamedArgs(Spec, Value) = undefined; - inline for (fields, 0..) |f, i| @field(named, f) = self.arg(i); - return named; - } - - /// One `!fly.ptr` per spec, in order; tensor - /// arguments are bridged to `!fly.memref` views. - pub fn declareArgsLow(self: *Builder, specs: []const ArgSpec) !void { - std.debug.assert(self.entry_block == null); - const ctx = self.ctx; - const scratch = self.arena.allocator(); - self.arg_specs = try scratch.dupe(ArgSpec, specs); - - const arg_types = try scratch.alloc(*const mlir.Type, specs.len); - const arg_locs = try scratch.alloc(*const mlir.Location, specs.len); - for (arg_types, arg_locs, specs) |*ty_, *a_loc, a| { - ty_.* = self.ptrType(a.dtype.storageElem(), self.addressSpaceAttr(a.space)); - a_loc.* = self.loc().named(ctx, a.name); - } - - const entry = mlir.Block.init(arg_types, arg_locs); - const func_op = gpu.func(ctx, .{ - .name = self.name, - .args = arg_types, - .block = entry, - .location = self.loc(), - }); - _ = func_op.appendTo(self.gpu_body); - self.func_op = func_op; - self.entry_block = entry; - self.block_stack.push(self.arena.allocator(), entry); - - // The view keeps the storage element type: recasting a predicate - // argument back to `!fly.ptr` would make a copy atom's element - // count `bits / 1` while it still moves `bits / 8` bytes. - self.arg_values = try scratch.alloc(Value, specs.len); - for (specs, 0..) |a, i| { - const raw = self.rawArg(i); - self.arg_values[i] = if (a.dims) |dims| raw.view(self.staticRowMajor(dims)) else raw; - } - } - - pub fn rawArg(self: *Builder, i: usize) Value { - const eb = self.entry_block orelse @panic("Builder.rawArg called before declareArgs"); - return .{ .inner = eb.argument(i), .kernel = self }; - } - - /// A tensor view when the spec has dims, else the raw pointer. - pub fn arg(self: *Builder, i: usize) Value { - return self.arg_values[i]; - } - - pub fn pushBlock(self: *Builder, b: *mlir.Block) void { - self.block_stack.push(self.arena.allocator(), b); - } - - pub fn popBlock(self: *Builder) void { - self.block_stack.pop(); - } - - pub fn currentBlock(self: *Builder) *mlir.Block { - return self.block_stack.current(); - } - - pub fn loc(self: *const Builder) *const mlir.Location { - return .unknown(self.ctx); - } - - pub fn emit(self: *Builder, op: *mlir.Operation) Value { - _ = op.appendTo(self.currentBlock()); - return .{ .inner = op.result(0), .kernel = self }; - } - - pub fn emitNone(self: *Builder, op: *mlir.Operation) void { - _ = op.appendTo(self.currentBlock()); - } - - // ---------------------------------------------------------------- types - - /// Arena-allocated. - pub fn readIntTuple(self: *Builder, tuple_ty: *const fly.types.IntTupleType) IntTuple { - if (tuple_ty.isLeaf()) { - return switch (tuple_ty.leaf()) { - .static => |v| IntTuple.static(v), - .dynamic, .basis => IntTuple.dyn, - .none => IntTuple.star, - }; - } - const n = tuple_ty.numElements(); - const elems = self.alloc(IntTuple, n); - for (0..n) |i| elems[i] = self.readIntTuple(tuple_ty.element(i)); - return .{ .tup = elems }; - } - - fn alloc(self: *Builder, comptime T: type, n: usize) []T { - return self.arena.allocator().alloc(T, n) catch @panic("fly: OOM"); - } - - /// The shape and stride of a static `!fly.layout`, as host literals. - pub fn readLayout(self: *Builder, layout_ty: *const mlir.Type) Layout { - const lay = fly.expect(layout_ty, .layout); - const shape = lay.shape(); - const stride = lay.stride(); - if (!shape.isStatic() or !stride.isStatic()) { - std.debug.panic("fly: `{f}` is not static", .{layout_ty}); - } - return .{ .shape = self.readIntTuple(shape), .stride = self.readIntTuple(stride) }; - } - - /// A fly type or attribute is rejected only on malformed input, which is - /// a bug in the kernel, not a runtime condition. - pub fn must(result: anytype) @typeInfo(@TypeOf(result)).error_union.payload { - return result catch |err| std.debug.panic("fly: cannot build type ({})", .{err}); - } - - pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { - return dtype.toMlir(self.ctx); - } - - pub fn intTupleType(self: *Builder, t: IntTuple) *const mlir.Type { - return must(fly.types.IntTupleType.get(self.ctx, .{ .attr = must(t.toAttr(self.ctx)) })).type_(); - } - - pub fn layoutType(self: *Builder, lay: Layout) *const mlir.Type { - return must(fly.types.LayoutType.get(self.ctx, .{ .attr = must(lay.toAttr(self.ctx)) })).type_(); - } - - pub fn tileType(self: *Builder, t: Tile) *const mlir.Type { - return must(fly.types.TileType.get(self.ctx, .{ .attr = must(t.toAttr(self.ctx)) })).type_(); - } - - pub fn ptrType(self: *Builder, dtype: DType, space: *const mlir.Attribute) *const mlir.Type { - return must(fly.types.PointerType.get(self.ctx, .{ .elemTy = dtype.toMlir(self.ctx), .addressSpace = space })).type_(); - } - - fn addressSpaceAttr(self: *Builder, space: fly.AddressSpace) *const mlir.Attribute { - return must(fly.attributes.AddressSpaceAttr.get(self.ctx, .{ .addressSpace = space })).attribute(); - } - - /// `!fly.memref`. - pub fn memRefType(self: *Builder, dtype: DType, space: fly.AddressSpace, lay: Layout) *const mlir.Type { - return must(fly.types.MemRefType.get(self.ctx, .{ - .elemTy = dtype.toMlir(self.ctx), - .addressSpace = self.addressSpaceAttr(space), - .layout = must(lay.toAttr(self.ctx)).attribute(), - })).type_(); - } - - // ---------------------------------------------------------------- statics - - /// `fly.static` of an already-built type. - pub fn staticTy(self: *Builder, t: *const mlir.Type) Value { - return self.emit(fly.ops.static(self.ctx, t, self.loc())); - } - - pub fn static(self: *Builder, v: anytype) Value { - const T = @TypeOf(v); - return self.staticTy(if (T == Layout) - self.layoutType(v) - else if (T == Tile) - self.tileType(v) - else if (T == IntTuple) - self.intTupleType(v) - else - @compileError("fly.static: expected a Layout, Tile or IntTuple, got " ++ @typeName(T))); - } - - pub fn staticRowMajor(self: *Builder, dims: []const i64) Value { - if (dims.len == 1) return self.static(Layout{ .shape = .static(dims[0]), .stride = .static(1) }); - const shape = self.alloc(IntTuple, dims.len); - const stride = self.alloc(IntTuple, dims.len); - var acc: i64 = 1; - var i: usize = dims.len; - while (i > 0) { - i -= 1; - shape[i] = .static(dims[i]); - stride[i] = .static(acc); - acc *= dims[i]; - } - return self.static(Layout{ .shape = .{ .tup = shape }, .stride = .{ .tup = stride } }); - } - - /// A Value passes through; a layout literal becomes a `fly.static`, and a - /// bare tuple literal (`.{ 64, 8 }`) is a tile. - pub fn toStatic(self: *Builder, v: anytype) Value { - const T = @TypeOf(v); - if (T == Value) return v; - if (T == Layout or T == Tile or T == IntTuple) return self.static(v); - if (@typeInfo(T) == .@"struct" and @typeInfo(T).@"struct".is_tuple) return self.static(tile(v)); - @compileError("fly: expected a Value, Layout, Tile, IntTuple or tuple literal, got " ++ @typeName(T)); - } - - /// A (possibly nested) int tuple from integers (static leaves), `null` - /// (`*`), `IntTuple` literals and `Value`s (dynamic leaves, passed as - /// operands). An `!fly.int_tuple` Value passes through. - pub fn intTuple(self: *Builder, elems: anytype) Value { - if (@TypeOf(elems) == Value and elems.kind() == .int_tuple) return elems; - var operands: stdx.BoundedArray(*const mlir.Value, 32) = .empty; - const spec = self.intTupleSpec(&operands, elems); - const t = self.intTupleType(spec); - return self.emit(fly.ops.make_int_tuple(self.ctx, operands.constSlice(), t, self.loc())); - } - - /// `intTuple`, spelled for coordinates. - pub const coord = intTuple; - - /// The literal for `intTuple`, collecting each dynamic leaf as an operand. - fn intTupleSpec(self: *Builder, operands: *stdx.BoundedArray(*const mlir.Value, 32), v: anytype) IntTuple { - const T = @TypeOf(v); - if (T == Value) { - if (v.kind() == .int_tuple) { - std.debug.panic("fly.intTuple: nested !fly.int_tuple values are not supported; pass scalars", .{}); - } - operands.appendAssumeCapacity(v.inner); - return if (v.isI64()) .dynamic(64, 1) else .dyn; - } - if (T == IntTuple) return v; - if (T == @TypeOf(null)) return .star; - switch (@typeInfo(T)) { - .comptime_int, .int => return .static(@intCast(v)), - .@"struct" => |s| { - if (!s.is_tuple) @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)); - const elems = self.alloc(IntTuple, s.field_names.len); - inline for (s.field_names, 0..) |f, i| elems[i] = self.intTupleSpec(operands, @field(v, f)); - return .{ .tup = elems }; - }, - else => @compileError("fly.intTuple: expected an integer, null, Value, IntTuple or tuple, got " ++ @typeName(T)), - } - } - - // ---------------------------------------------------------------- scalars & ids - - /// `value` as a constant of `dtype`; a Value passes through. An integer is - /// signed or an unsigned bit pattern of `dtype`'s width; anything else - /// panics. Only the representation `dtype` needs is computed: a float - /// constant outside i64 range (inf, floatMax) must not be forced through - /// `@intFromFloat`, which traps. - pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { - if (@TypeOf(value) == Value) return value; - const is_float_lit = switch (@typeInfo(@TypeOf(value))) { - .float, .comptime_float => true, - else => false, - }; - return switch (dtype) { - .f16, .bf16, .f32, .f64 => blk: { - const f: f64 = if (is_float_lit) @floatCast(value) else @floatFromInt(value); - break :blk self.emit(switch (dtype) { - .f16 => arith.constant_float(self.ctx, f, .f16, self.loc()), - .bf16 => arith.constant_float(self.ctx, f, .bf16, self.loc()), - .f32 => arith.constant_float(self.ctx, f, .f32, self.loc()), - else => arith.constant_float(self.ctx, f, .f64, self.loc()), - }); - }, - .i1, .i8, .i16, .i32, .i64 => blk: { - const i: i128 = if (is_float_lit) @intFromFloat(value) else value; - break :blk self.emit(arith.constant_int(self.ctx, intBits(i, @intCast(dtype.bitWidth()), @tagName(dtype)), dtype.toMlir(self.ctx), self.loc())); - }, - else => std.debug.panic("fly.constant: no constant for {s}", .{@tagName(dtype)}), - }; - } - - /// `value` as the `i64` an integer constant of `bits` bits is built from. - /// It may be signed or an unsigned bit pattern, so it must lie in - /// [-2^(bits-1), 2^bits). - fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { - if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) - std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); - return @truncate(value); - } - - /// A Zig scalar as a constant: a comptime int is `i32`, or `i64` when it - /// doesn't fit; a runtime int is `i64` past 32 bits, else `i32`; a float - /// keeps its width, and a comptime float is `f32`. A Value passes through. - pub fn lift(self: *Builder, value: anytype) Value { - const T = @TypeOf(value); - if (T == Value) return value; - return switch (@typeInfo(T)) { - .comptime_int => self.constant(if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) .i32 else .i64, value), - .int => |info| self.constant(if (info.bits > 32) .i64 else .i32, value), - .comptime_float => self.constant(.f32, value), - .float => |info| switch (info.bits) { - 16 => self.constant(.f16, value), - 32 => self.constant(.f32, value), - 64 => self.constant(.f64, value), - else => @compileError("Builder.lift: unsupported float bitwidth"), - }, - else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), - }; - } - - pub fn constIndex(self: *Builder, value: i64) Value { - return self.emit(arith.constant_index(self.ctx, value, self.loc())); - } - - fn idI32(self: *Builder, op: *mlir.Operation) Value { - return self.emit(op).toI32(); - } - - /// Already cast to `i32`: `fly.make_int_tuple` takes only i32/i64. - pub fn threadId(self: *Builder, dim: Dim) Value { - return self.idI32(gpu.thread_id(self.ctx, dim, self.loc())); - } - pub fn blockId(self: *Builder, dim: Dim) Value { - return self.idI32(gpu.block_id(self.ctx, dim, self.loc())); - } - pub fn blockDim(self: *Builder, dim: Dim) Value { - return self.idI32(gpu.block_dim(self.ctx, dim, self.loc())); - } - pub fn gridDim(self: *Builder, dim: Dim) Value { - return self.idI32(gpu.grid_dim(self.ctx, dim, self.loc())); - } - - pub fn barrier(self: *Builder) void { - self.emitNone(gpu.barrier(self.ctx, self.loc())); - } - - /// Tag float arithmetic emitted from now on with `fastmath`. - pub fn setFastMath(self: *Builder, on: bool) void { - self.fast_math = on; - } - - pub fn emitFast(self: *Builder, op: *mlir.Operation) Value { - if (self.fast_math and op.numResults() > 0 and (Value{ .inner = op.result(0), .kernel = self }).isFloatElem()) { - op.setAttribute(arith.fastmath(self.ctx, .fast) catch unreachable); - } - return self.emit(op); - } - - pub fn rsqrt(self: *Builder, x: Value) Value { - return self.emitFast(math.rsqrt(self.ctx, x.inner, self.loc())); - } - pub fn sqrt(self: *Builder, x: Value) Value { - return self.emitFast(math.sqrt(self.ctx, x.inner, self.loc())); - } - pub fn exp(self: *Builder, x: Value) Value { - return self.emitFast(math.exp(self.ctx, x.inner, self.loc())); - } - - pub fn log(self: *Builder, x: Value) Value { - return self.emitFast(math.log(self.ctx, x.inner, self.loc())); - } - pub fn fma(self: *Builder, a: Value, b: Value, c: Value) Value { - return self.emitFast(math.fma(self.ctx, a.inner, b.inner, c.inner, self.loc())); - } - - /// A matrix atom with SSA operands; unlike `gemm`, this needs no register - /// allocas and can carry its accumulator through control flow. - pub fn mmaAtomCall(self: *Builder, atom: Value, a: Value, b: Value, c: Value) Value { - return self.emit(fly.ops.mma_atom_call_ssa(self.ctx, atom.inner, &.{a.inner}, &.{b.inner}, c.inner, &.{c.type_()}, .{}, self.loc())); - } - - /// AMD byte permutation. ROCDL has no corresponding operation at our pin. - pub fn perm(self: *Builder, hi: Value, lo: Value, selector: Value) Value { - return self.emit(llvm.inline_asm( - self.ctx, - "v_perm_b32 $0, $1, $2, $3", - &.{ hi.inner, lo.inner, selector.inner }, - .{ .constraints = "=v,v,v,v", .result_type = .int(self.ctx, .i32) }, - self.loc(), - )); - } - - /// CDNA3 packed FP8 conversion, selecting two bytes from the low/high word. - pub fn cvtPkF32Fp8(self: *Builder, bits: Value, high: bool) Value { - return self.emit(rocdl.cvtPkF32Fp8(self.ctx, bits.inner, high, self.loc())); - } - - pub const AtomicScope = enum { workgroup, agent }; - - /// Monotonic i32 addition to a global or shared pointer. Callers provide - /// barriers or kernel dependencies before reading the completed counters. - pub fn ptrAtomicAdd(self: *Builder, ptr: Value, value: Value, scope: AtomicScope) Value { - std.debug.assert(ptr.elemDType() == .i32 and value.type_().eql(DType.i32.toMlir(self.ctx))); - const space: i32 = switch (scope) { - .workgroup => 3, - .agent => 1, - }; - const llvm_ptr = self.emit(fly.ops.to_llvm_ptr(self.ctx, ptr.inner, .int(self.ctx, .i32, space), self.loc())); - return self.emit(llvm.atomicrmw(self.ctx, llvm_ptr.inner, value.inner, .add, .monotonic, .{ .syncscope = @tagName(scope), .alignment = 4 }, self.loc())); - } - - /// Static LDS, as an `n:1` tensor in shared memory. Each call is its own - /// `fly.make_ptr {allocBytes, allocAlign}` leaf, which the plugin lowers - /// to one LLVM global, so `shared_mem_bytes` may stay 0. - pub fn sharedArray(self: *Builder, dtype: DType, n: i64, alignment: i32) Value { - const bytes: i64 = @divExact(@as(i64, dtype.bitWidth()) * n, 8); - const raw_ty = must(fly.types.PointerType.get(self.ctx, .{ - .elemTy = .int(self.ctx, .i8), - .addressSpace = self.addressSpaceAttr(.shared), - .alignment = must(fly.attributes.AlignAttr.get(self.ctx, .{ .alignment = alignment })), - })).type_(); - const dict: *const mlir.Attribute = .dict(self.ctx, &.{ - .named(self.ctx, "allocBytes", .int(self.ctx, .i64, bytes)), - .named(self.ctx, "allocAlign", .int(self.ctx, .i64, alignment)), - }); - const raw = self.emit(fly.ops.make_ptr(self.ctx, &.{}, raw_ty, .{ .dict_attrs = dict }, self.loc())); - const typed = self.recast(raw, dtype); - return typed.view(self.static(Layout{ .shape = .static(n), .stride = .static(1) })); - } - - /// A single atom-sized copy. - pub fn copyAtomCall(self: *Builder, atom: Value, src: Value, dst: Value) void { - self.emitNone(fly.ops.copy_atom_call(self.ctx, atom.inner, src.inner, dst.inner, .{}, self.loc())); - } - - /// One atom-sized load straight into a `vector`. - /// - /// Required over an rmem tensor + `copyAtomCall` + `load` in any kernel - /// with control flow: the plugin lowers `scf` to `cf` before the Fly - /// passes, and Fly's register-alloca promotion only handles one block. - pub fn copyAtomLoad(self: *Builder, atom: Value, src: Value) Value { - const n = src.sizeStatic(); - const dt = src.elemDType(); - const vec_ty = mlir.Type.vector(&.{n}, dt.toMlir(self.ctx)); - return self.emit(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, src.inner, &.{vec_ty}, .{}, self.loc())); - } - - /// One atom-sized store of a `vector`; see `copyAtomLoad`. - pub fn copyAtomStore(self: *Builder, atom: Value, vec: Value, dst: Value) void { - self.emitNone(fly.ops.copy_atom_call_ssa(self.ctx, atom.inner, vec.inner, &.{}, .{ .dst = dst.inner }, self.loc())); - } - - // ---------------------------------------------------------------- tensors - - pub fn makeView(self: *Builder, iter: Value, lay: anytype) Value { - _ = self; - return iter.view(lay); - } - - /// A coordinate tensor: value == logical coord. - pub fn identity(self: *Builder, shape: anytype) Value { - const shp = self.intTuple(shape); - const lay = self.emit(fly.ops.make_identity_layout(self.ctx, shp.inner, self.loc())); - const rank_ = shp.intTupleStatic().rank(); - const zero: IntTuple = if (rank_ == 1) .static(0) else blk: { - const zeros = self.alloc(IntTuple, rank_); - for (zeros) |*z| z.* = .static(0); - break :blk .{ .tup = zeros }; - }; - const base_ty = self.intTupleType(zero); - const base = self.emit(fly.ops.make_int_tuple(self.ctx, &.{}, base_ty, self.loc())); - return base.view(lay); - } - - /// A register tensor; see `copyAtomLoad` before using one in a kernel - /// with control flow. - pub fn rmemTensor(self: *Builder, comptime lay: Layout, dtype: DType) Value { - const lay_v = self.static(lay); - const t = self.memRefType(dtype, .register, lay); - return self.emit(fly.ops.memref_alloca(self.ctx, lay_v.inner, t, self.loc())); - } - - /// `rmemTensor` with a runtime `Layout`. - pub fn rmemTensorRuntime(self: *Builder, lay: Layout, dtype: DType) Value { - const lay_v = self.static(lay); - const t = self.memRefType(dtype, .register, lay); - return self.emit(fly.ops.memref_alloca(self.ctx, lay_v.inner, t, self.loc())); - } - - /// Reinterpret a pointer as another element type, keeping the address - /// space, alignment and swizzle. - pub fn recast(self: *Builder, ptr: Value, dtype: DType) Value { - const t = must(fly.ptrWithElem(self.ctx, ptr.type_(), dtype.toMlir(self.ctx))).type_(); - return self.emit(fly.ops.recast_iter(self.ctx, ptr.inner, t, self.loc())); - } - - // ---------------------------------------------------------------- atoms - - /// The copy-op type an atom wraps, by bit size. - pub const CopyOp = union(enum) { - universal: i32, - buffer_copy: i32, - buffer_copy_lds: i32, - - fn type_(self: CopyOp, ctx: *mlir.Context) *const mlir.Type { - return switch (self) { - .universal => |b| must(fly.types.CopyOpUniversalCopyType.get(ctx, .{ .bitSize = b })).type_(), - .buffer_copy => |b| must(fly_rocdl.CopyOpCDNA3BufferCopyType.get(ctx, .{ .bitSize = b })).type_(), - .buffer_copy_lds => |b| must(fly_rocdl.CopyOpCDNA3BufferCopyLDSType.get(ctx, .{ .bitSize = b })).type_(), - }; - } - }; - - pub fn copyAtom(self: *Builder, op: CopyOp, dtype: DType) Value { - const bits: i32 = @intCast(dtype.bitWidth()); - const t = must(fly.types.CopyAtomType.get(self.ctx, .{ .copyOp = op.type_(self.ctx), .valBits = bits })).type_(); - return self.emit(fly.ops.make_copy_atom(self.ctx, &.{}, t, .int(self.ctx, .i32, bits), self.loc())); - } - - /// `op` is an MMA op type, such as a `fly_rocdl.MmaOpCDNA3MFMAType`. - pub fn mmaAtom(self: *Builder, op: *const mlir.Type) Value { - const t = must(fly.types.MmaAtomType.get(self.ctx, .{ .mmaOp = op })).type_(); - return self.emit(fly.ops.make_mma_atom(self.ctx, t, self.loc())); - } - - pub fn tiledCopy(self: *Builder, atom: Value, layout_tv: anytype, tile_mn: anytype) TiledCopy { - const tv = self.toStatic(layout_tv); - const tl = self.toStatic(tile_mn); - const v = self.emit(fly.ops.make_tiled_copy(self.ctx, atom.inner, tv.inner, tl.inner, self.loc())); - return .{ .value = v, .tile = tl }; - } - - /// raked product -> right inverse -> composition -> make_tiled_copy, - /// with the tiler read off the inferred product shape. - pub fn tiledCopyTV(self: *Builder, atom: Value, thr: Layout, val: Layout) TiledCopy { - const thr_size = thr.size() orelse @panic("fly.tiledCopyTV: thr_layout must be static"); - const val_size = val.size() orelse @panic("fly.tiledCopyTV: val_layout must be static"); - const layout_mn = self.static(thr).rakedProduct(self.static(val)); - const tmp = self.static(Layout{ - .shape = .{ .tup = &.{ .static(thr_size), .static(val_size) } }, - .stride = .{ .tup = &.{ .static(1), .static(thr_size) } }, - }); - const layout_tv = layout_mn.rightInverse().composition(tmp); - const tiler = layout_mn.emitShape().productEach(); - // The tiler as a tile literal: one leaf per mode, `[a|b]`. - const shape = tiler.intTupleStatic(); - const tile_modes = self.alloc(Tile, shape.rank()); - for (tile_modes, 0..) |*m, i| m.* = .{ .leaf = switch (shape.at(i)) { - .leaf => |l| l, - else => std.debug.panic("fly.tiledCopyTV: tiler mode {d} is not a leaf", .{i}), - } }; - return self.tiledCopy(atom, layout_tv, Tile{ .modes = tile_modes }); - } - - /// A TiledCopy matched to an MMA operand: the TV layout and tile size - /// come off the `!fly.tiled_mma` type. - pub fn tiledCopyFor(self: *Builder, operand: fly.MmaOperand, copy_atom: Value, tm: TiledMma) TiledCopy { - const tm_ty = fly.expect(tm.value.type_(), .tiled_mma); - const layout_tv = self.staticTy((switch (operand) { - .a => tm_ty.tiledThrValLayoutA(), - .b => tm_ty.tiledThrValLayoutB(), - .c => tm_ty.tiledThrValLayoutC(), - .d => @panic("fly.tiledCopyFor: no D operand copy"), - }).type_()); - const tile_size = self.staticTy(tm_ty.tileSizeMNK().type_()); - const modes: [2]i32 = switch (operand) { - .a => .{ 0, 2 }, - .b => .{ 1, 2 }, - .c => .{ 0, 1 }, - .d => unreachable, - }; - const one = self.intTuple(1); - const tile_modes = self.alloc(Tile, modes.len); - for (modes, 0..) |m, i| { - const lay = self.emit(fly.ops.make_layout(self.ctx, tile_size.selectModes(&.{m}).inner, one.inner, self.loc())); - tile_modes[i] = .{ .layout = self.readLayout(lay.type_()) }; - } - return self.tiledCopy(copy_atom, layout_tv, Tile{ .modes = tile_modes }); - } - - pub fn tiledCopyA(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { - return self.tiledCopyFor(.a, copy_atom, tm); - } - pub fn tiledCopyB(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { - return self.tiledCopyFor(.b, copy_atom, tm); - } - pub fn tiledCopyC(self: *Builder, copy_atom: Value, tm: TiledMma) TiledCopy { - return self.tiledCopyFor(.c, copy_atom, tm); - } - - /// The same tensor over a CDNA buffer resource - /// (`!fly.ptr`), for hardware bounds-checked - /// accesses and `buffer_copy` atoms. Descriptor size is the maximum. - pub fn bufferTensor(self: *Builder, t: Value) Value { - const ptr = t.emitIter(); - const lay = t.emitLayout(); - const c0 = self.constant(.i16, 0); - const nrec = self.constant(.i64, 0xFFFFFFFF); - // (7 << 12) | (4 << 15): the CDNA descriptor flags FlyDSL and XLA use. - const flags = self.constant(.i32, 0x27000); - const elem = t.elemDType(); - const buf_ty = self.ptrType(elem, (must(fly_rocdl.BufferDescAddressAttr.get(self.ctx))).attribute()); - const buf_ptr = self.emit(fly.ops.make_ptr(self.ctx, &.{ ptr.inner, c0.inner, nrec.inner, flags.inner }, buf_ty, .{}, self.loc())); - return buf_ptr.view(lay); - } - - pub fn tiledMma(self: *Builder, atom: Value, atom_layout: anytype, permutation: ?Value) TiledMma { - const al = self.toStatic(atom_layout); - const v = self.emit(fly.ops.make_tiled_mma(self.ctx, atom.inner, al.inner, .{ .permutation = if (permutation) |p| p.inner else null }, self.loc())); - return .{ .value = v }; - } - - pub const CopyOpts = struct { - pred: ?Value = null, - }; - - pub fn copy(self: *Builder, atom: Value, src: Value, dst: Value, opts: CopyOpts) void { - self.emitNone(fly.ops.copy(self.ctx, atom.inner, src.inner, dst.inner, .{ .pred = if (opts.pred) |p| p.inner else null }, self.loc())); - } - - pub fn gemm(self: *Builder, atom: Value, d: Value, a: Value, b: Value, c: Value) void { - self.emitNone(fly.ops.gemm(self.ctx, atom.inner, d.inner, &.{a.inner}, &.{b.inner}, c.inner, .{}, self.loc())); - } - - // ---------------------------------------------------------------- control flow - - pub fn ForScope(comptime N: usize) type { - return common.ForScope(Builder, Value, N); - } - pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); - pub fn IfScope(comptime N: usize) type { - return common.IfScope(Builder, Value, N); - } - pub fn WhileScope(comptime N: usize, comptime M: usize) type { - return common.WhileScope(Builder, Value, N, M); - } - - /// Loop-carried `inits`. Literal bounds take the type of a Value bound, - /// else they are `index`. - pub fn openFor( - self: *Builder, - lower: anytype, - upper: anytype, - step: anytype, - inits: anytype, - ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { - return self.openForOpts(lower, upper, step, inits, .{}); - } - - /// `openFor` with `scf.for` options. - pub fn openForOpts( - self: *Builder, - lower: anytype, - upper: anytype, - step: anytype, - inits: anytype, - opts: common.ForOpts, - ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { - return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.constIndex), inits, opts); - } - - /// `value` as a scalar constant of `like`'s element type (`index` for an - /// `index` Value); a Value passes through. - pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { - if (@TypeOf(value) == Value) return value; - if (like.type_().isA(mlir.IndexType) != null) { - const i: i128 = switch (@typeInfo(@TypeOf(value))) { - .float, .comptime_float => @intFromFloat(value), - else => value, - }; - return self.constIndex(intBits(i, 64, "index")); - } - return self.constant(like.scalarDType(), value); - } - - pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { - return common.openIf(self, Value, cond); - } - - pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { - return common.openIfElse(self, Value, cond, result_types); - } - - pub fn openWhile( - self: *Builder, - inits: anytype, - after_types: anytype, - ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { - return common.openWhile(self, Value, inits, after_types); - } - - // ---------------------------------------------------------------- finalization - - /// Append `gpu.return`, verify, and print the module (no debug info: XLA - /// scrubs locations). Owned by the builder's allocator. - pub fn finish(self: *Builder) FinishError![:0]const u8 { - _ = gpu.return_(self.ctx, &.{}, self.loc()).appendTo(self.currentBlock()); - - try common.verify(self.module, "dsl/fly"); - - var al: std.Io.Writer.Allocating = .init(self.allocator); - defer al.deinit(); - try al.writer.print("{f}", .{self.module.operation().fmt(.{ .debug_info = false })}); - return try self.allocator.dupeSentinel(u8, al.written(), 0); - } -}; - -// ============================================================================= -// Tests -// ============================================================================= - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - registerDialects(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the -/// ragged edge. -pub fn emitVectorAdd(b: *Builder, thr_layout: Layout, val_layout: Layout, a_in: Value, b_in: Value, c_out: Value) void { - const shape = a_in.shapeStatic(); - const m = shape.at(0).leaf.s; - const n = shape.at(1).leaf.s; - - const atom = b.copyAtom(.{ .universal = 128 }, .f32); - const tc = b.tiledCopyTV(atom, thr_layout, val_layout); - - const tid = b.threadId(.x); - const bx = b.blockId(.x); - const by = b.blockId(.y); - - const idC = b.identity(.{ m, n }); - const tile_mn = tc.tileMN(); - - const gA = a_in.flatDivide(tile_mn).slice(.{ null, null, bx, by }); - const gB = b_in.flatDivide(tile_mn).slice(.{ null, null, bx, by }); - const gC = c_out.flatDivide(tile_mn).slice(.{ null, null, bx, by }); - const cC = idC.flatDivide(tile_mn).slice(.{ null, null, bx, by }); - - const thr = tc.getSlice(tid); - const tgA = thr.partitionS(gA); - const tgB = thr.partitionS(gB); - const tgC = thr.partitionD(gC); - const tcC = thr.partitionS(cC).slice(.{ .{ 0, null }, null, null }); - - const rA = tgA.makeFragmentLike(null); - const rB = tgB.makeFragmentLike(null); - const rC = tgC.makeFragmentLike(null); - const pC = tcC.makeFragmentLike(.i1); - - for (0..@intCast(pC.sizeStatic())) |i| { - pC.set(i, tcC.at(i).elemLess(.{ m, n })); - } - - b.copy(atom, tgA, rA, .{ .pred = pC }); - b.copy(atom, tgB, rB, .{ .pred = pC }); - rC.store(rA.load().add(rB.load())); - b.copy(atom, rC, tgC, .{ .pred = pC }); -} - -test "vectorAdd builds, verifies and re-parses" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "vector_add"); - defer b.deinit(); - const a = try b.declareArgs(.{ - .a = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, - .b = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, - .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 100, 1000 } } }, - }); - emitVectorAdd(&b, ordered(.{ 8, 16 }, .{ 1, 0 }), ordered(.{ 1, 4 }, .{ 0, 1 }), a.a, a.b, a.c); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - - const expected = [_][]const u8{ - "module attributes {gpu.container_module}", - "gpu.module @zml_fly_kernels", - "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel", - "fly.static : !fly.layout<(100,1000):(1000,1)>", - "fly.make_copy_atom {valBits = 32 : i32} : !fly.copy_atom, 32>", - "fly.raked_product", - "fly.static : !fly.tile<[8|64]>", - "fly.make_tiled_copy", - "gpu.thread_id x", - "fly.tiled_copy.partition_src", - "fly.tiled_copy.partition_dst", - "fly.make_fragment_like", - "fly.elem_less", - "fly.copy(", - "fly.memref.load_vec", - "arith.addf", - "fly.memref.store_vec", - "gpu.return", - }; - for (expected) |needle| { - if (std.mem.indexOf(u8, ir, needle) == null) { - std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); - return error.TestUnexpectedResult; - } - } - - const reparsed = try mlir.Module.parse(ctx, ir); - defer reparsed.deinit(); - try std.testing.expect(reparsed.operation().verify()); -} - -/// The matrix atom a device family provides. CDNA has MFMA over f32; RDNA has -/// WMMA, whose verifier requires M=N=K=16 and rejects f32 operands, so the -/// operand type and the K tile change with the family, not just the mnemonic. -pub const MmaFlavor = enum { - cdna3_mfma, - gfx11_wmma, - gfx120x_wmma, - - /// MFMA takes f32 operands at 16x16x4; both WMMAs take f16 at 16x16x16. - pub fn atomType(self: MmaFlavor, ctx: *mlir.Context) *const mlir.Type { - const operand = self.operandDType().toMlir(ctx); - const acc: *const mlir.Type = .float(ctx, .f32); - return switch (self) { - .cdna3_mfma => (fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ - .m = 16, - .n = 16, - .k = 4, - .elemTyA = operand, - .elemTyB = operand, - .elemTyAcc = acc, - }) catch unreachable).type_(), - inline .gfx11_wmma, .gfx120x_wmma => |f| blk: { - const T = if (f == .gfx11_wmma) fly_rocdl.MmaOpGFX11WMMAType else fly_rocdl.MmaOpGFX120XWMMAType; - break :blk (T.get(ctx, .{ - .m = 16, - .n = 16, - .k = 16, - .elemTyA = operand, - .elemTyB = operand, - .elemTyAcc = acc, - }) catch unreachable).type_(); - }, - }; - } - - /// A and B operand type. The accumulator is f32 either way. - pub fn operandDType(self: MmaFlavor) DType { - return switch (self) { - .cdna3_mfma => .f32, - .gfx11_wmma, .gfx120x_wmma => .f16, - }; - } - - /// Threads one atom occupies: MFMA runs on a wave64, WMMA on a wave32. - /// Checked against the dialect's own `mma_atom.thr_layout` below. - pub fn atomThreads(self: MmaFlavor) i32 { - return switch (self) { - .cdna3_mfma => 64, - .gfx11_wmma, .gfx120x_wmma => 32, - }; - } - - /// `emitTiledMma` lays the atom out (2,2,1), so a block is four of them. - pub fn blockThreads(self: MmaFlavor) i32 { - return 4 * self.atomThreads(); - } - - /// The block tile's K, a multiple of the atom's K. - pub fn blockK(self: MmaFlavor) i64 { - return switch (self) { - .cdna3_mfma => 8, - .gfx11_wmma, .gfx120x_wmma => 16, - }; - } - - /// Buffer copies are CDNA-only: FlyROCDL declares no RDNA copy atom, so - /// RDNA moves through the core universal copy over plain tensors. - fn hasBufferCopy(self: MmaFlavor) bool { - return self == .cdna3_mfma; - } -}; - -/// A two-mode tile from extents known only at emit time. -fn tile2(b: *Builder, x: i64, y: i64) Tile { - const modes = b.alloc(Tile, 2); - modes[0] = .{ .leaf = .{ .s = x } }; - modes[1] = .{ .leaf = .{ .s = y } }; - return .{ .modes = modes }; -} - -/// C = A @ B^T in one block of 256 threads, tiled (2,2,1) over the family's -/// matrix atom. -pub fn emitTiledMma(b: *Builder, flavor: MmaFlavor, a_in: Value, b_in: Value, c_out: Value) void { - const block_m = 64; - const block_n = 64; - const block_k = flavor.blockK(); - - const tid = b.threadId(.x); - const bid = b.blockId(.x); - - const A = if (flavor.hasBufferCopy()) b.bufferTensor(a_in) else a_in; - const B = if (flavor.hasBufferCopy()) b.bufferTensor(b_in) else b_in; - const C = if (flavor.hasBufferCopy()) b.bufferTensor(c_out) else c_out; - - const bA = A.zippedDivide(tile2(b, block_m, block_k)).slice(.{ null, bid }); - const bB = B.zippedDivide(tile2(b, block_n, block_k)).slice(.{ null, bid }); - const bC = C.zippedDivide(tile2(b, block_m, block_n)).slice(.{ null, bid }); - - const mma_atom = b.mmaAtom(flavor.atomType(b.ctx)); - const tiled_mma = b.tiledMma(mma_atom, L(.{ 2, 2, 1 }, .{ 1, 2, 0 }), null); - - // A and B move at the operand type, C at the f32 accumulator type. One - // element per instruction: a WMMA fragment's values are strided in memory, - // so a wider copy would move the wrong neighbours. - const copy_ab: Builder.CopyOp = if (flavor.hasBufferCopy()) .{ .buffer_copy = 32 } else .{ .universal = @intCast(flavor.operandDType().bitWidth()) }; - const copy_c: Builder.CopyOp = if (flavor.hasBufferCopy()) .{ .buffer_copy = 32 } else .{ .universal = @intCast(c_out.elemDType().bitWidth()) }; - const atom_ab = b.copyAtom(copy_ab, flavor.operandDType()); - const atom_c = b.copyAtom(copy_c, c_out.elemDType()); - - const tiled_copy_a = b.tiledCopyA(atom_ab, tiled_mma); - const tiled_copy_b = b.tiledCopyB(atom_ab, tiled_mma); - const tiled_copy_c = b.tiledCopyC(atom_c, tiled_mma); - - const thr_copy_a = tiled_copy_a.getSlice(tid); - const thr_copy_b = tiled_copy_b.getSlice(tid); - const thr_copy_c = tiled_copy_c.getSlice(tid); - - const copy_src_a = thr_copy_a.partitionS(bA); - const copy_src_b = thr_copy_b.partitionS(bB); - const copy_dst_c = thr_copy_c.partitionS(bC); - - const frag_a = tiled_mma.makeFragmentA(bA); - const frag_b = tiled_mma.makeFragmentB(bB); - const frag_c = tiled_mma.makeFragmentC(bC); - - const copy_frag_a = thr_copy_a.retile(frag_a); - const copy_frag_b = thr_copy_b.retile(frag_b); - const copy_frag_c = thr_copy_c.retile(frag_c); - - b.copy(atom_ab, copy_src_a, copy_frag_a, .{}); - b.copy(atom_ab, copy_src_b, copy_frag_b, .{}); - - frag_c.fill(0); - b.gemm(mma_atom, frag_c, frag_a, frag_b, frag_c); - - b.copy(atom_c, copy_frag_c, copy_dst_c, .{}); -} - -test "tiledMma builds, verifies and re-parses" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "tiled_mma"); - defer b.deinit(); - const a = try b.declareArgs(.{ - .a = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 8 } } }, - .b = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 8 } } }, - .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 64 } } }, - }); - emitTiledMma(&b, .cdna3_mfma, a.a, a.b, a.c); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - - const expected = [_][]const u8{ - "!fly.ptr", - "fly.make_mma_atom : !fly.mma_atom f32>>", - "fly.make_tiled_mma", - "fly.static : !fly.int_tuple<(32,32,4)>", - "fly.mma.make_fragment", - "fly.tiled_copy.retile", - "vector.broadcast", - "fly.gemm(", - }; - for (expected) |needle| { - if (std.mem.indexOf(u8, ir, needle) == null) { - std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); - return error.TestUnexpectedResult; - } - } - const reparsed = try mlir.Module.parse(ctx, ir); - defer reparsed.deinit(); - try std.testing.expect(reparsed.operation().verify()); -} - -test "MmaFlavor.atomThreads matches the dialect" { - const ctx = try testContext(); - defer ctx.deinit(); - inline for (comptime std.enums.values(MmaFlavor)) |flavor| { - const atom = try fly.types.MmaAtomType.get(ctx, .{ .mmaOp = flavor.atomType(ctx) }); - // `mma_atom.thr_layout` is the wavefront an atom occupies; the launch - // geometry is derived from it, so a wrong table silently corrupts C. - const threads = switch (atom.thrLayout().shape().leaf()) { - .static => |v| v, - else => return error.TestUnexpectedResult, - }; - try std.testing.expectEqual(@as(i64, flavor.atomThreads()), threads); - } -} - -test "tiledMma builds for the RDNA WMMA atom" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "tiled_mma_wmma"); - defer b.deinit(); - const k = MmaFlavor.gfx11_wmma.blockK(); - const a = try b.declareArgs(.{ - .a = .{ .tensor = .{ .dtype = .f16, .dims = &.{ 64, k } } }, - .b = .{ .tensor = .{ .dtype = .f16, .dims = &.{ 64, k } } }, - .c = .{ .tensor = .{ .dtype = .f32, .dims = &.{ 64, 64 } } }, - }); - emitTiledMma(&b, .gfx11_wmma, a.a, a.b, a.c); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - - // No buffer descriptor: FlyROCDL declares no RDNA copy atom, so this path - // moves through the core universal copy over plain tensors. - if (std.mem.indexOf(u8, ir, "buffer_desc") != null) { - std.debug.print("unexpected buffer descriptor on the WMMA path:\n{s}\n", .{ir}); - return error.TestUnexpectedResult; - } - for ([_][]const u8{ - "!fly_rocdl.gfx11.wmma<16x16x16, (f16, f16) -> f32", - "!fly.copy_atom, 16>", - "fly.gemm(", - }) |needle| { - if (std.mem.indexOf(u8, ir, needle) == null) { - std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); - return error.TestUnexpectedResult; - } - } - - const reparsed = try mlir.Module.parse(ctx, ir); - defer reparsed.deinit(); - try std.testing.expect(reparsed.operation().verify()); -} - -test "scalar conversions and constants" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "conv"); - defer b.deinit(); - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .f16, .dims = &.{16} } }, - .p = .{ .tensor = .{ .dtype = .i1, .dims = &.{16} } }, - }); - - // A predicate is unsigned: `true` must convert to 1, never -1. - const pred = b.constant(.i32, 3).lt(7); - _ = pred.to(.f32); - _ = pred.to(.i32); - // Equal-width float pairs are neither ext nor trunc. - _ = b.constant(.f16, 1.0).to(.bf16); - // A float constant outside i64 range must not go through @intFromFloat. - _ = b.constant(.f32, -std.math.inf(f32)); - _ = b.constant(.f32, std.math.floatMax(f32)); - // A `.bool` argument keeps its storage element type end to end. - try std.testing.expectEqual(DType.i8, a.p.elemDType()); - try std.testing.expectEqual(DType.f16, a.x.elemDType()); - // Literal inits are lifted, and an integer constant may be a bit pattern. - var loop = b.openFor(0, 4, 1, .{ 0, 0.5 }); - { - loop.yield(.{ loop.carried[0].bitXor(b.constant(.i32, 0xd4d0ccc8)), loop.carried[1].mul(2.0) }); - } - - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - for ([_][]const u8{ - "arith.uitofp", - "arith.extui", - "arith.convertf", - "arith.constant 0xFF800000 : f32", - "!fly.memref", - "-> (i32, f32)", - "arith.constant -724513592 : i32", - }) |needle| { - if (std.mem.indexOf(u8, ir, needle) == null) { - std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); - return error.TestUnexpectedResult; - } - } - if (std.mem.indexOf(u8, ir, "arith.sitofp") != null or std.mem.indexOf(u8, ir, "arith.extsi") != null) { - std.debug.print("i1 was sign-extended:\n{s}\n", .{ir}); - return error.TestUnexpectedResult; - } -} - -test "shuffles and fast math carry their attributes" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "shuffle"); - defer b.deinit(); - _ = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .f32, .dims = &.{16} } }, - }); - - b.setFastMath(true); - const x = b.constant(.f32, 1.0); - _ = x.shuffleXor(1, 64).add(x.shuffle(.idx, 3, 64)); - - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - for ([_][]const u8{ - "gpu.shuffle xor", - "gpu.shuffle idx", - "arith.addf %shuffleResult, %shuffleResult_1 fastmath", - }) |needle| { - if (std.mem.indexOf(u8, ir, needle) == null) { - std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); - return error.TestUnexpectedResult; - } - } -} - -test "static queries read inferred types" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "q"); - defer b.deinit(); - const a = try b.declareArgs(.{ .x = .{ .tensor = .{ .dtype = .bf16, .dims = &.{ 64, 32 } } } }); - try std.testing.expectEqual(@as(i64, 64 * 32), a.x.sizeStatic()); - try std.testing.expectEqual(DType.bf16, a.x.elemDType()); - try std.testing.expectEqual(Value.Kind.memref, a.x.kind()); - const tiled = a.x.flatDivide(tile(.{ 16, 8 })); - try std.testing.expectEqual(@as(i64, 64 * 32), tiled.sizeStatic()); - try std.testing.expectEqual(@as(usize, 4), tiled.shapeStatic().rank()); -} - -test "atomic addition builds, verifies and re-parses" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "atomic_add"); - defer b.deinit(); - _ = try b.declareArgs(.{}); - const counter = b.sharedArray(.i32, 1, 4); - _ = b.ptrAtomicAdd(counter.emitIter(), b.constant(.i32, 1), .workgroup); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - - // LLVM's current custom assembly stores alignment in the property dictionary. - for ([_][]const u8{ - "llvm.atomicrmw add", - "syncscope(\"workgroup\") monotonic ", - ": !llvm.ptr<3>, i32", - }) |needle| { - if (std.mem.indexOf(u8, ir, needle) == null) { - std.debug.print("IR missing `{s}`:\n{s}\n", .{ needle, ir }); - return error.TestUnexpectedResult; - } - } - const reparsed = try mlir.Module.parse(ctx, ir); - defer reparsed.deinit(); - try std.testing.expect(reparsed.operation().verify()); -} - -test "SSA matrix atom builds, verifies and re-parses" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "mma_atom_ssa"); - defer b.deinit(); - _ = try b.declareArgs(.{}); - const atom_type = try fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ - .m = 16, - .n = 16, - .k = 16, - .elemTyA = .float(ctx, .bf16), - .elemTyB = .float(ctx, .bf16), - .elemTyAcc = .float(ctx, .f32), - }); - const atom = b.mmaAtom(atom_type.type_()); - const lhs = b.constant(.bf16, 0).splat(4); - const rhs = b.constant(.bf16, 0).splat(4); - const acc = b.constant(.f32, 0).splat(4); - const result = b.mmaAtomCall(atom, lhs, rhs, acc); - try std.testing.expect(result.type_().eql(acc.type_())); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - - const reparsed = try mlir.Module.parse(ctx, ir); - defer reparsed.deinit(); - try std.testing.expect(reparsed.operation().verify()); -} - -test "comparisons take index ivs and vectors" { - const ctx = try testContext(); - defer ctx.deinit(); - - var b = try Builder.open(std.testing.allocator, ctx, "compare"); - defer b.deinit(); - _ = try b.declareArgs(.{}); - - // Literal bounds make an `index` loop. - var loop = b.openFor(0, 4, 1, .{}); - { - _ = loop.iv.lt(2); - loop.close(); - } - // A literal or a scalar broadcasts against a vector. - const v = b.constant(.f32, 1.0).splat(4); - _ = v.lt(0.5); - _ = v.ge(b.constant(.f32, 2.0)); - _ = b.constant(.i32, 3).splat(4).eq(3); - - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - for ([_][2][]const u8{ - .{ "arith.cmpi slt", ": index" }, - .{ "arith.cmpf olt", ": vector<4xf32>" }, - .{ "arith.cmpf oge", ": vector<4xf32>" }, - .{ "arith.cmpi eq", ": vector<4xi32>" }, - }) |needles| { - var lines = std.mem.splitScalar(u8, ir, '\n'); - const found = while (lines.next()) |line| { - if (std.mem.indexOf(u8, line, needles[0]) != null and std.mem.endsWith(u8, line, needles[1])) break true; - } else false; - if (!found) { - std.debug.print("no `{s} ... {s}` in:\n{s}\n", .{ needles[0], needles[1], ir }); - return error.TestUnexpectedResult; - } - } -} diff --git a/dsl/fly/dtype.zig b/dsl/fly/dtype.zig deleted file mode 100644 index ecc9f070..00000000 --- a/dsl/fly/dtype.zig +++ /dev/null @@ -1,84 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -pub const DType = enum { - i1, - i8, - i16, - i32, - i64, - f16, - bf16, - f32, - f64, - f8e4m3fn, - f8e4m3fnuz, - f8e5m2, - f8e5m2fnuz, - - pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { - return switch (self) { - .i1 => .int(ctx, .i1), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .f16 => .float(ctx, .f16), - .bf16 => .float(ctx, .bf16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .f8e4m3fn => .float(ctx, .f8e4m3fn), - .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), - .f8e5m2 => .float(ctx, .f8e5m2), - .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), - }; - } - - pub fn bitWidth(self: DType) u32 { - return switch (self) { - .i1 => 1, - .i8, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz => 8, - .i16, .f16, .bf16 => 16, - .i32, .f32 => 32, - .i64, .f64 => 64, - }; - } - - pub fn isFloat(self: DType) bool { - return switch (self) { - .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz => true, - else => false, - }; - } - - /// The element type a kernel argument is stored with on device: - /// predicates live in one byte. A copy atom over `!fly.ptr` would - /// claim `bits / 1` elements while moving `bits / 8` bytes, so an - /// argument keeps its storage type and the body converts explicitly. - pub fn storageElem(self: DType) DType { - return switch (self) { - .i1 => .i8, - else => self, - }; - } -}; - -/// Inverse of `toMlir`. -pub fn fromMlir(ctx: *mlir.Context, ty: *const mlir.Type) ?DType { - inline for (comptime std.enums.values(DType)) |dt| { - if (ty.eql(dt.toMlir(ctx))) return dt; - } - return null; -} - -test "dtype types round-trip" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - inline for (comptime std.enums.values(DType)) |dt| { - try std.testing.expectEqual(dt, fromMlir(ctx, dt.toMlir(ctx)).?); - } - try std.testing.expectEqual(DType.i8, DType.i1.storageElem()); - try std.testing.expectEqual(DType.f8e4m3fnuz, DType.f8e4m3fnuz.storageElem()); -} diff --git a/dsl/fly/layout.zig b/dsl/fly/layout.zig deleted file mode 100644 index 06dd64c6..00000000 --- a/dsl/fly/layout.zig +++ /dev/null @@ -1,544 +0,0 @@ -const std = @import("std"); - -const fly = @import("mlir/dialects/fly"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -const attributes = fly.attributes; - -/// Enough for any tuple or tile a kernel writes by hand. -const max_modes = 32; - -/// A dynamic integer leaf: `?`, `?{i64}`, `?{div=8}`, `?{i64 div=8}`. -pub const Dyn = struct { - width: u8 = 32, - div: u32 = 1, -}; - -/// Static, dynamic, or the `*` wildcard. -pub const Leaf = union(enum) { - s: i64, - d: Dyn, - none, - - pub fn print(self: Leaf, w: *std.Io.Writer) std.Io.Writer.Error!void { - switch (self) { - .s => |v| try w.print("{d}", .{v}), - .none => try w.writeAll("*"), - .d => |dy| { - try w.writeAll("?"); - if (dy.width != 32 or dy.div != 1) { - try w.writeAll("{"); - if (dy.width != 32) try w.print("i{d}", .{dy.width}); - if (dy.div != 1) { - if (dy.width != 32) try w.writeAll(" "); - try w.print("div={d}", .{dy.div}); - } - try w.writeAll("}"); - } - }, - } - } -}; - -fn leafAttr(leaf: Leaf, ctx: *mlir.Context) mlir.Error!*const attributes.IntAttr { - return switch (leaf) { - .s => |v| .getStatic(ctx, @intCast(v)), - .d => |dy| .getDynamic(ctx, dy.width, @intCast(dy.div)), - .none => .getNone(ctx), - }; -} - -/// A scaled basis element: `vE0`, `2E1E2`. -pub const Basis = struct { - value: Leaf, - modes: []const i32, -}; - -pub const IntTuple = union(enum) { - leaf: Leaf, - basis: Basis, - tup: []const IntTuple, - - pub const dyn: IntTuple = .{ .leaf = .{ .d = .{} } }; - pub const star: IntTuple = .{ .leaf = .none }; - - pub fn static(v: i64) IntTuple { - return .{ .leaf = .{ .s = v } }; - } - - pub fn dynamic(width: u8, div: u32) IntTuple { - return .{ .leaf = .{ .d = .{ .width = width, .div = div } } }; - } - - /// 1 for a leaf. - pub fn rank(self: IntTuple) usize { - return switch (self) { - .tup => |t| t.len, - else => 1, - }; - } - - pub fn isLeaf(self: IntTuple) bool { - return self != .tup; - } - - pub fn at(self: IntTuple, i: usize) IntTuple { - return switch (self) { - .tup => |t| t[i], - else => if (i == 0) self else @panic("fly.IntTuple.at: index out of range on a leaf"), - }; - } - - /// Null if any leaf is dynamic or `*`. - pub fn product(self: IntTuple) ?i64 { - switch (self) { - .leaf => |l| return switch (l) { - .s => |v| v, - else => null, - }, - .basis => return null, - .tup => |t| { - var p: i64 = 1; - for (t) |e| p *= e.product() orelse return null; - return p; - }, - } - } - - pub fn isStatic(self: IntTuple) bool { - return switch (self) { - .leaf => |l| l == .s, - .basis => |b| b.value == .s, - .tup => |t| for (t) |e| { - if (!e.isStatic()) break false; - } else true, - }; - } - - /// At any depth. - pub fn leafCount(self: IntTuple) usize { - return switch (self) { - .tup => |t| blk: { - var n: usize = 0; - for (t) |e| n += e.leafCount(); - break :blk n; - }, - else => 1, - }; - } - - pub fn print(self: IntTuple, w: *std.Io.Writer) std.Io.Writer.Error!void { - switch (self) { - .leaf => |l| try l.print(w), - .basis => |b| { - try b.value.print(w); - for (b.modes) |m| try w.print("E{d}", .{m}); - }, - .tup => |t| { - try w.writeAll("("); - for (t, 0..) |e, i| { - if (i > 0) try w.writeAll(","); - try e.print(w); - } - try w.writeAll(")"); - }, - } - } - - pub fn format(self: IntTuple, w: *std.Io.Writer) std.Io.Writer.Error!void { - try self.print(w); - } - - pub fn toAttr(self: IntTuple, ctx: *mlir.Context) mlir.Error!*const attributes.IntTupleAttr { - return switch (self) { - .leaf => |l| .get(ctx, .{ .value = (try leafAttr(l, ctx)).attribute() }), - .basis => |b| .getBasis(ctx, try leafAttr(b.value, ctx), b.modes), - .tup => |elems| blk: { - var buf: stdx.BoundedArray(*const attributes.IntTupleAttr, max_modes) = .empty; - for (elems) |e| buf.appendAssumeCapacity(try e.toAttr(ctx)); - break :blk .getTuple(ctx, buf.constSlice()); - }, - }; - } - - pub fn eql(a: IntTuple, b: IntTuple) bool { - if (std.meta.activeTag(a) != std.meta.activeTag(b)) return false; - return switch (a) { - .leaf => |l| std.meta.eql(l, b.leaf), - .basis => |x| std.meta.eql(x.value, b.basis.value) and std.mem.eql(i32, x.modes, b.basis.modes), - .tup => |t| t.len == b.tup.len and for (t, b.tup) |x, y| { - if (!x.eql(y)) break false; - } else true, - }; - } -}; - -/// Integers become static leaves, tuples nest, `null` is `*`, an `IntTuple` -/// passes through: `it(.{ .{ 2, 4 }, 8 })` is `((2,4),8)`. -pub fn it(comptime v: anytype) IntTuple { - return comptime itImpl(v); -} - -fn itImpl(comptime v: anytype) IntTuple { - const T = @TypeOf(v); - if (T == IntTuple) return v; - if (T == Leaf) return .{ .leaf = v }; - if (T == @TypeOf(null)) return IntTuple.star; - switch (@typeInfo(T)) { - .comptime_int, .int => return IntTuple.static(v), - .@"struct" => |s| { - if (!s.is_tuple) @compileError("fly.it: expected an integer, null, IntTuple or tuple, got " ++ @typeName(T)); - var elems: [s.field_names.len]IntTuple = undefined; - for (s.field_names, 0..) |f, i| elems[i] = itImpl(@field(v, f)); - const frozen = elems; - return .{ .tup = &frozen }; - }, - else => @compileError("fly.it: expected an integer, null, IntTuple or tuple, got " ++ @typeName(T)), - } -} - -/// `!fly.layout`. -pub const Layout = struct { - shape: IntTuple, - stride: IntTuple, - - pub fn rank(self: Layout) usize { - return self.shape.rank(); - } - - /// Null if dynamic. - pub fn size(self: Layout) ?i64 { - return self.shape.product(); - } - - pub fn print(self: Layout, w: *std.Io.Writer) std.Io.Writer.Error!void { - try self.shape.print(w); - try w.writeAll(":"); - try self.stride.print(w); - } - - pub fn format(self: Layout, w: *std.Io.Writer) std.Io.Writer.Error!void { - try self.print(w); - } - - pub fn toAttr(self: Layout, ctx: *mlir.Context) mlir.Error!*const attributes.LayoutAttr { - return .get(ctx, .{ .shape = try self.shape.toAttr(ctx), .stride = try self.stride.toAttr(ctx) }); - } - - pub fn eql(a: Layout, b: Layout) bool { - return a.shape.eql(b.shape) and a.stride.eql(b.stride); - } -}; - -/// Two congruent tuples: `L(.{ 4, 8 }, .{ 1, 4 })` is `(4,8):(1,4)`. -pub fn L(comptime shape: anytype, comptime stride: anytype) Layout { - return comptime blk: { - const s = it(shape); - const d = it(stride); - checkCongruent(s, d); - break :blk .{ .shape = s, .stride = d }; - }; -} - -fn checkCongruent(comptime a: IntTuple, comptime b: IntTuple) void { - if (a.isLeaf() != b.isLeaf() or a.rank() != b.rank()) { - @compileError("fly.L: shape and stride are not congruent"); - } - if (a == .tup) { - for (a.tup, b.tup) |x, y| checkCongruent(x, y); - } -} - -/// Last mode contiguous: `rowMajor(.{ 4, 8 })` is `(4,8):(8,1)`. -pub fn rowMajor(comptime shape: anytype) Layout { - return comptime blk: { - const s = it(shape); - const n = s.leafCount(); - var order: [n]i32 = undefined; - for (0..n) |i| order[i] = @intCast(n - 1 - i); - break :blk orderedFlat(s, &order); - }; -} - -/// First mode contiguous. -pub fn colMajor(comptime shape: anytype) Layout { - return comptime blk: { - const s = it(shape); - const n = s.leafCount(); - var order: [n]i32 = undefined; - for (0..n) |i| order[i] = @intCast(i); - break :blk orderedFlat(s, &order); - }; -} - -/// `make_ordered_layout` folded at comptime: the smallest order value gets -/// stride 1, the next the product of the previous extents, and so on. -/// `ordered(.{ 8, 16 }, .{ 1, 0 })` is `(8,16):(16,1)`. -pub fn ordered(comptime shape: anytype, comptime order: anytype) Layout { - return comptime blk: { - const s = it(shape); - const o = it(order); - checkCongruent(s, o); - const n = s.leafCount(); - var flat_order: [n]i32 = undefined; - var idx: usize = 0; - flattenOrder(o, &flat_order, &idx); - break :blk orderedFlat(s, &flat_order); - }; -} - -fn flattenOrder(comptime o: IntTuple, out: []i32, idx: *usize) void { - switch (o) { - .leaf => |l| { - out[idx.*] = @intCast(l.s); - idx.* += 1; - }, - .tup => |t| for (t) |e| flattenOrder(e, out, idx), - .basis => @compileError("fly.ordered: basis elements are not allowed in an order"), - } -} - -fn orderedFlat(comptime shape: IntTuple, comptime order: []const i32) Layout { - comptime { - const n = shape.leafCount(); - var extents: [n]i64 = undefined; - var idx: usize = 0; - flattenStatic(shape, &extents, &idx); - var strides: [n]i64 = undefined; - var assigned: [n]bool = @splat(false); - var acc: i64 = 1; - for (0..n) |_| { - var best: ?usize = null; - for (0..n) |i| { - if (assigned[i]) continue; - if (best == null or order[i] < order[best.?]) best = i; - } - strides[best.?] = acc; - acc *= extents[best.?]; - assigned[best.?] = true; - } - var pos: usize = 0; - return .{ .shape = shape, .stride = unflatten(shape, &strides, &pos) }; - } -} - -fn flattenStatic(comptime t: IntTuple, out: []i64, idx: *usize) void { - switch (t) { - .leaf => |l| switch (l) { - .s => |v| { - out[idx.*] = v; - idx.* += 1; - }, - else => @compileError("fly: a folded layout needs a fully static shape"), - }, - .basis => @compileError("fly: a folded layout needs a plain integer shape"), - .tup => |elems| for (elems) |e| flattenStatic(e, out, idx), - } -} - -fn unflatten(comptime like: IntTuple, values: []const i64, pos: *usize) IntTuple { - switch (like) { - .tup => |elems| { - var out: [elems.len]IntTuple = undefined; - for (elems, 0..) |e, i| out[i] = unflatten(e, values, pos); - const frozen = out; - return .{ .tup = &frozen }; - }, - else => { - const v = values[pos.*]; - pos.* += 1; - return IntTuple.static(v); - }, - } -} - -/// A leaf (`8` or `*`), a layout, or a nested tile. -pub const Tile = union(enum) { - leaf: Leaf, - layout: Layout, - modes: []const Tile, - - pub fn print(self: Tile, w: *std.Io.Writer) std.Io.Writer.Error!void { - switch (self) { - .leaf => |l| try l.print(w), - .layout => |l| try l.print(w), - .modes => |m| { - try w.writeAll("["); - for (m, 0..) |e, i| { - if (i > 0) try w.writeAll("|"); - try e.print(w); - } - try w.writeAll("]"); - }, - } - } - - pub fn format(self: Tile, w: *std.Io.Writer) std.Io.Writer.Error!void { - try self.print(w); - } - - /// One mode: an int, a layout, or a nested tile. - fn modeAttr(self: Tile, ctx: *mlir.Context) mlir.Error!*const mlir.Attribute { - return switch (self) { - .leaf => |l| (try leafAttr(l, ctx)).attribute(), - .layout => |l| (try l.toAttr(ctx)).attribute(), - .modes => (try self.toAttr(ctx)).attribute(), - }; - } - - pub fn toAttr(self: Tile, ctx: *mlir.Context) mlir.Error!*const attributes.TileAttr { - const modes = switch (self) { - .modes => |m| m, - else => return .get(ctx, .{ .value = try self.modeAttr(ctx) }), - }; - var buf: stdx.BoundedArray(*const mlir.Attribute, max_modes) = .empty; - for (modes) |m| buf.appendAssumeCapacity(try m.modeAttr(ctx)); - return .getModes(ctx, buf.constSlice()); - } -}; - -/// `tile(.{ 128, 64 })` is `[128|64]`; `tile(.{ null, 8 })` is `[*|8]`. -pub fn tile(comptime v: anytype) Tile { - return comptime tileImpl(v); -} - -fn tileImpl(comptime v: anytype) Tile { - const T = @TypeOf(v); - if (T == Tile) return v; - if (T == Layout) return .{ .layout = v }; - if (T == Leaf) return .{ .leaf = v }; - if (T == @TypeOf(null)) return .{ .leaf = .none }; - switch (@typeInfo(T)) { - .comptime_int, .int => return .{ .leaf = .{ .s = v } }, - .@"struct" => |s| { - if (!s.is_tuple) @compileError("fly.tile: expected an integer, null, Layout, Tile or tuple, got " ++ @typeName(T)); - var elems: [s.field_names.len]Tile = undefined; - for (s.field_names, 0..) |f, i| elems[i] = tileImpl(@field(v, f)); - const frozen = elems; - return .{ .modes = &frozen }; - }, - else => @compileError("fly.tile: expected an integer, null, Layout, Tile or tuple, got " ++ @typeName(T)), - } -} - -/// `S`. -pub const Swizzle = struct { - mask: i32, - base: i32, - shift: i32, - - pub const trivial: Swizzle = .{ .mask = 0, .base = 0, .shift = 0 }; - - pub fn isTrivial(self: Swizzle) bool { - return self.mask == 0 and self.base == 0 and self.shift == 0; - } - - pub fn print(self: Swizzle, w: *std.Io.Writer) std.Io.Writer.Error!void { - try w.print("S<{d},{d},{d}>", .{ self.mask, self.base, self.shift }); - } - - pub fn format(self: Swizzle, w: *std.Io.Writer) std.Io.Writer.Error!void { - try self.print(w); - } - - pub fn toAttr(self: Swizzle, ctx: *mlir.Context) mlir.Error!*const attributes.SwizzleAttr { - return .get(ctx, .{ .mask = self.mask, .base = self.base, .shift = self.shift }); - } -}; - -// ----------------------------------------------------------------------------- -// Tests -// ----------------------------------------------------------------------------- - -fn expectPrints(expected: []const u8, v: anytype) !void { - var buf: [1024]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{v}); - try std.testing.expectEqualStrings(expected, w.buffered()); -} - -test "int tuple literals print in fly syntax" { - try expectPrints("42", it(42)); - try expectPrints("((2,4),8)", it(.{ .{ 2, 4 }, 8 })); - try expectPrints("(?,8)", it(.{ IntTuple.dyn, 8 })); - try expectPrints("?{i64 div=8}", IntTuple.dynamic(64, 8)); - try expectPrints("?{div=4}", IntTuple.dynamic(32, 4)); - try expectPrints("?{i64}", IntTuple.dynamic(64, 1)); - try expectPrints("(*,8)", it(.{ null, 8 })); - try expectPrints("(1,(2,3))", it(.{ 1, .{ 2, 3 } })); -} - -test "layout literals" { - try expectPrints("(4,8):(1,4)", L(.{ 4, 8 }, .{ 1, 4 })); - try expectPrints("((2,4),8):((1,2),8)", L(.{ .{ 2, 4 }, 8 }, .{ .{ 1, 2 }, 8 })); - try expectPrints("(4,8):(8,1)", rowMajor(.{ 4, 8 })); - try expectPrints("(4,8):(1,4)", colMajor(.{ 4, 8 })); - try expectPrints("((2,4),8):((32,8),1)", rowMajor(.{ .{ 2, 4 }, 8 })); - try expectPrints("(8,16):(16,1)", ordered(.{ 8, 16 }, .{ 1, 0 })); - try expectPrints("(1,4):(1,1)", ordered(.{ 1, 4 }, .{ 0, 1 })); - try expectPrints("(4,2,8):(2,1,8)", ordered(.{ 4, 2, 8 }, .{ 1, 0, 2 })); -} - -test "tiles and swizzles" { - try expectPrints("[128|64]", tile(.{ 128, 64 })); - try expectPrints("[*|8]", tile(.{ null, 8 })); - try expectPrints("[(4,8):(1,4)|16]", tile(.{ L(.{ 4, 8 }, .{ 1, 4 }), 16 })); - try expectPrints("[[2|3]|4]", tile(.{ .{ 2, 3 }, 4 })); - try expectPrints("S<3,3,3>", Swizzle{ .mask = 3, .base = 3, .shift = 3 }); -} - -test "static queries" { - try std.testing.expectEqual(@as(?i64, 64), it(.{ .{ 2, 4 }, 8 }).product()); - try std.testing.expectEqual(@as(?i64, null), it(.{ IntTuple.dyn, 8 }).product()); - try std.testing.expectEqual(@as(usize, 2), it(.{ .{ 2, 4 }, 8 }).rank()); - try std.testing.expectEqual(@as(usize, 3), it(.{ .{ 2, 4 }, 8 }).leafCount()); - try std.testing.expect(it(.{ 2, 4 }).eql(it(.{ 2, 4 }))); - try std.testing.expect(!it(.{ 2, 4 }).eql(it(.{ 4, 2 }))); -} - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - @import("builder.zig").registerDialects(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -/// The type built from the host model must be the type the printed literal -/// parses to: the two paths may never drift. -fn expectBuilt(ctx: *mlir.Context, comptime T: type, comptime fmt: []const u8, value: anytype) !void { - var buf: [512]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print(fmt, .{value}); - const parsed = try mlir.Type.parse(ctx, w.buffered()); - const built = try T.get(ctx, .{ .attr = try value.toAttr(ctx) }); - if (!parsed.eql(built.type_())) { - std.debug.print("parsed `{f}` but built `{f}`\n", .{ parsed, built }); - return error.TestUnexpectedResult; - } -} - -test "the host model builds the types its literals print" { - const ctx = try testContext(); - defer ctx.deinit(); - const IntTupleType = fly.types.IntTupleType; - const LayoutType = fly.types.LayoutType; - const TileType = fly.types.TileType; - - try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(42)); - try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ .{ 2, 4 }, 8 })); - try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ IntTuple.dyn, 8 })); - try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", IntTuple.dynamic(64, 8)); - try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", it(.{ null, 8 })); - try expectBuilt(ctx, IntTupleType, "!fly.int_tuple<{f}>", IntTuple{ .basis = .{ .value = .{ .s = 2 }, .modes = &.{ 1, 2 } } }); - - try expectBuilt(ctx, LayoutType, "!fly.layout<{f}>", L(.{ 4, 8 }, .{ 1, 4 })); - try expectBuilt(ctx, LayoutType, "!fly.layout<{f}>", rowMajor(.{ .{ 2, 4 }, 8 })); - - try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ 128, 64 })); - try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ null, 8 })); - try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ L(.{ 4, 8 }, .{ 1, 4 }), 16 })); - try expectBuilt(ctx, TileType, "!fly.tile<{f}>", tile(.{ .{ 2, 3 }, 4 })); -} diff --git a/dsl/metal/BUILD.bazel b/dsl/metal/BUILD.bazel deleted file mode 100644 index d68355c0..00000000 --- a/dsl/metal/BUILD.bazel +++ /dev/null @@ -1,25 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "builder", - srcs = ["types.zig"], - import_name = "dsl/metal/builder", - main = "builder.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//dsl/common:builder", - "//mlir", - "//mlir/dialects/air", - "//mlir/dialects/arith", - "//mlir/dialects/cf", - "//mlir/dialects/func", - "//mlir/dialects/llvm", - "//mlir/dialects/vector", - ], -) - -zig_test( - name = "test", - deps = [":builder"], -) diff --git a/dsl/metal/README.md b/dsl/metal/README.md deleted file mode 100644 index 1cbc40ce..00000000 --- a/dsl/metal/README.md +++ /dev/null @@ -1,150 +0,0 @@ -# ZML Metal DSL - -A Zig builder for Metal kernels that reads like MSL. It emits the `air` -dialect (`mlir/dialects/air`) and the parts of `arith`, `cf`, `vector` and -`llvm` that MSL code compiles to: values, locals in memory, blocks and -branches. `zml.kernel.metal.Kernel` wraps a kernel into -a `__gpu$xla.gpu.air` custom call, and XLA does the rest: it lowers the module to -Apple's AIR, runs the Metal frontend's optimization pipeline -(`air-opt -O2 -disable-loop-unrolling`) and builds a metallib. Apple's -backend compiles that metallib the way it compiles an MSL kernel. - -## Skeleton of a kernel - -```zig -const metal = @import("zml").kernel.metal; - -pub const Axpy = struct { - pub const Cfg = struct {}; - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "axpy", - .inputs = &.{ "a", "b" }, - .outputs = &.{"out"}, - .constants = &.{ "scale", "n" }, - .run = run, - }); - - fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .a = .{ .buffer = .{ .type = .float, .const_ = true } }, - .b = .{ .buffer = .{ .type = .float, .const_ = true } }, - .out = .{ .buffer = .{ .type = .float } }, - .scale = .{ .constant = .float }, - .n = .{ .constant = .uint }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - b.returnIf(a.tid.ge(a.n)); - a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); - } -}; - -// In a model: -const out = Axpy.Kernel.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ - .cfg = .{}, - .grid = .{ 4, 1, 1 }, - .threadgroup = .{ 256, 1, 1 }, - .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, 1000)) }, -}).out; -``` - -`declareArgs` declares the kernel parameters in order, which is the `[[buffer(n)]]` -order. `inputs` and `outputs` bind buffers to tensors by name. An output named -like an input updates that input's buffer in place. `constants` bind -`constant T &` arguments by value. `Constant.of` takes a scalar, or a host -`extern struct` laid out like the MSL struct. `CallOpts.threadgroup_memory` -sizes `[[threadgroup(0)]]`. - -Function constants have no runtime form. A kernel takes them through its `Cfg`, -so each configuration emits its own specialized kernel. - -## Arguments - -| MSL | `ArgSpec` | -|---|---| -| `device T *p`, `const device T *p` | `.buffer = .{ .type = T, .const_ = true }` | -| `constant T &x` | `.constant = T`, read wherever it is used, as C++ reads a reference | -| `constant S &s` | `.constant_struct = S`, read with `s.field("name")` | -| `threadgroup T *p [[threadgroup(0)]]` | `.threadgroup = T` | -| `uint3 tid [[thread_position_in_grid]]` and the other builtins | `.thread_position_in_grid = .uint3`, ... | - -## Values - -A `Value` is a scalar or vector (`Type`, spelled as in MSL: `.float4`, -`.ushort`), a pointer, a private array, a `simdgroup_matrix` or a constant -struct. Operations follow C++: integer promotion and the usual arithmetic -conversions pick the result type, signed arithmetic is `nsw`, and float math -carries the kernel's fast-math flags. A Zig number is the MSL literal it -spells, an `int` (a `long` when it does not fit) or a `float`, and a runtime -Zig integer keeps its width and signedness; the usual conversions follow, so -`h.mul(0.5)` on a `half` is a `float` multiply, as `h * 0.5` is in MSL. A -scalar next to a vector takes its element type. Stores, assignments and the -functions of one type (`max`, `min`, `clamp`, `fma`) convert a number to their -type. `0.5h` is `b.constant(.half, 0.5)`. - -| MSL | DSL | -|---|---| -| `a + b`, `a * b`, `a < b`, `a & b` | `a.add(b)`, `a.mul(b)`, `a.lt(b)`, `a.bitAnd(b)` | -| `(float) x`, `float4(v)` | `x.to(.float)`, `v.to(.float4)` | -| `as_type(x)` | `x.asType(.uint)` | -| `v.x`, `v[i]`, `v[i] = e`, `v.y = e` | `v.x()`, `v.at(i)`, `v = v.insert(i, e)`, `v = v.insert(1, e)`: a subscript takes the index's type (a literal is an `int`) | -| `p[i]`, `p[i] = v`, `p + i` | `p.load(i)`, `p.store(i, v)`, `p.add(i)` | -| `(threadgroup float4 *) p` | `p.to(.float4)` | -| `T x;`, `T x = e;`, `x = e` | `const x = b.local(.T, null)`, `b.local(.T, e)`, `x.assign(e)` | -| `device T *p = e;` that the code moves | `const p = b.localPointer(e)`, `p.assign(p.add(i))` | -| `float s[8];`, `float s[8] = {};` | `const s = b.array(.float, 8)`, `b.zeroedArray(.float, 8)`, then `s.load(i)` and `s.store(i, v)` | -| `&x` of a local | `x.address()` | -| `float4(a, b, c, d)`, `1.0f` | `b.vec(.float4, .{ a, b, c, d })`, `b.constant(.float, 1)` | -| `c ? x : y` | `b.ternary(c, x, y)`, a select of evaluated operands; `var t = b.openIfElse(c, .{.T})`, `x`, `t.yieldThen(.{x})`, `y`, `t.yieldElse(.{y})`, `t.results[0]` | -| `exp(x)`, `fast::exp(x)`, `precise::tanh(x)` | `b.exp(x)`, `b.fast().exp(x)`, `b.precise().tanh(x)` | -| `fma`, `dot`, `max`, `clamp` | `b.fma`, `b.dot`, `b.max`, `b.clamp` | -| `simd_sum(x)`, `simd_shuffle_xor(x, m)` | `b.simdSum(x)`, `b.simdShuffleXor(x, m)` | -| `threadgroup_barrier(mem_flags::mem_threadgroup)` | `b.threadgroupBarrier(.mem_threadgroup)` | -| `simdgroup_load(m, p, ld, origin, transpose)`, `simdgroup_store(m, p, ld)` | `const m = b.simdgroupLoad(p, ld, .{ .origin = ..., .transpose = true })`, `b.simdgroupStore(m, p, ld, .{})` | -| `simdgroup_multiply_accumulate(d, a, b, c)` | `const d = b.simdgroupMultiplyAccumulate(a, b, c)` | -| `m.thread_elements()` read, assigned | `m.threadElements()`; `const m = b.matrix(.T, elements)` | - -## Control flow - -| MSL | DSL | -|---|---| -| `for (T i = lower; i < upper; i += s)`, `++i` | `var l = b.openFor(lower, upper, s)`, `l.iv`, `l.close()`; a step of 1 is `++i` | -| `#pragma unroll`, `#pragma clang loop unroll(full)` | `b.openForOpts(..., .{ .unroll = .enable })`, `.{ .unroll = .full }` | -| `while (c) { ... }`, `for (init; c; inc)`, `for (;;)` | `var l = b.openLoop(.{})`, `l.body(c)`, the body, `l.next()`, the increment, `l.close()` | -| `break;` | `l.breakLoop()` | -| `if (c) { ... }` | `var s = b.openIf(c)`, `s.close()` | -| `if (c) { ... } else { ... }` | `var s = b.openIfElse(c, .{})`, `s.elseBranch()`, `s.close()` | -| `if (a \|\| (b && !x))` | `var c = b.condition()`, `c.or_(a)`, `c.and_(b)`, `var s = c.openIfNot(x)` | -| `x = a && b && c` | `var l = b.openAnd()`, `l.term(a)`, `l.term(b)`, `l.close(c)` | -| `return;` | `b.return_()`, or `b.returnIf(c)` | - -`condition`, `openAnd`, `openOr` and a ternary `openIfElse` skip a term or an -operand, so use them only when one must not run, such as a load past the end -of a buffer. Combine comparisons with `bitAnd` and `bitOr`, and pick between -values that are safe to compute with `b.ternary`. - -Put each `if` or loop body in a Zig `{ ... }` block right after the opener, -one block per region, each ending with the call that ends the region: -`elseBranch()` or `yieldThen`, then `close()` or `yieldElse` for an `if`; -`body(c)`, `next()`, then `close()` for `openLoop`. The scope variable stays -outside, so its `results` stay readable after the blocks. - -A value is evaluated where it is written; a Zig `const` holds it, and an MSL -function is a Zig function that emits its body. A value a loop or branch -changes is a local, which lives in memory until the pipeline promotes it. -The terms of `&&` and `||` run only when the ones before leave the result -open, and an `openIfElse(c, .{T})` operand only when `c` selects it, each in -its own block. Loops follow C++: every -loop is `mustprogress`, and its condition and increment follow C++'s -conversions, so a `short` loop variable compares as an `int` and `++i` steps -it as a `short`. - -A `simdgroup_matrix` is a value of its 64 elements, of which a lane holds the -first two; the load, store and multiply-accumulate call the -`air.simdgroup_matrix_8x8_*` builtins. - -## Checking a port - -Fast math lets Apple's backend reassociate and contract float math, so a port -computes what its MSL kernel computes up to rounding, not always to the bit. -Compare its outputs with the MSL kernel's on the same inputs, and time both -with the same grid, threadgroup size and threadgroup memory. diff --git a/dsl/metal/builder.zig b/dsl/metal/builder.zig deleted file mode 100644 index dea3d48a..00000000 --- a/dsl/metal/builder.zig +++ /dev/null @@ -1,2320 +0,0 @@ -const std = @import("std"); - -const air = @import("mlir/dialects/air"); -const arith = @import("mlir/dialects/arith"); -const cf = @import("mlir/dialects/cf"); -const common = @import("dsl/common/builder"); -const func = @import("mlir/dialects/func"); -const llvm = @import("mlir/dialects/llvm"); -const mlir = @import("mlir"); -const vector = @import("mlir/dialects/vector"); - -pub const types = @import("types.zig"); - -pub const Scalar = types.Scalar; -pub const Type = types.Type; -pub const Struct = types.Struct; -pub const StructField = types.StructField; -pub const Builtin = air.Builtin; - -pub fn registerDialects(registry: *mlir.DialectRegistry) void { - inline for (.{ air, arith, cf, func, llvm, vector }) |dialect| dialect.registerDialect(registry); -} - -pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Builder); - std.testing.refAllDecls(Value); -} - -pub const AddressSpace = air.AddressSpace; - -/// `mem_flags`: `.mem_threadgroup`, or `.{ .device = true, .threadgroup = true }` -/// for `mem_device | mem_threadgroup`. -pub const MemFlags = packed struct(u32) { - device: bool = false, - threadgroup: bool = false, - texture: bool = false, - _padding: u29 = 0, - - pub const mem_none: MemFlags = .{}; - pub const mem_device: MemFlags = .{ .device = true }; - pub const mem_threadgroup: MemFlags = .{ .threadgroup = true }; - pub const mem_texture: MemFlags = .{ .texture = true }; -}; - -/// A kernel argument, as `Builder.declareArgs` takes it. Device and constant -/// arguments take the next `[[buffer(n)]]`, threadgroup arguments the next -/// `[[threadgroup(n)]]`, in declaration order. -pub const ArgSpec = union(enum) { - /// `device T *p`, or `const device T *p` with `.const_`. - buffer: Buffer, - /// `constant T &x`; the argument is its value. - constant: Type, - /// `constant S &s`; `Value.field` reads a field. - constant_struct: Struct, - /// `threadgroup T *p`. - threadgroup: Type, - thread_position_in_grid: Type, - threadgroup_position_in_grid: Type, - thread_position_in_threadgroup: Type, - threads_per_threadgroup: Type, - threadgroups_per_grid: Type, - threads_per_grid: Type, - thread_index_in_simdgroup: Type, - simdgroup_index_in_threadgroup: Type, - threads_per_simdgroup: Type, - simdgroups_per_threadgroup: Type, - thread_index_in_threadgroup: Type, - - pub const Buffer = struct { - type: Type, - const_: bool = false, - /// `__restrict__`. - restrict: bool = false, - }; -}; - -/// `raw` as an `ArgSpec`: an `ArgSpec`, or a literal of one of its variants -/// such as `.{ .buffer = .{ .type = .float } }`. -fn argSpec(raw: anytype) ArgSpec { - if (@TypeOf(raw) == ArgSpec) return raw; - const variant = comptime literalVariant(@TypeOf(raw), "ArgSpec"); - const inner = @field(raw, variant); - return switch (@field(std.meta.Tag(ArgSpec), variant)) { - .buffer => blk: { - comptime knownFields(@TypeOf(inner), ArgSpec.Buffer, "ArgSpec.buffer"); - break :blk .{ .buffer = .{ - .type = typeOf(inner.type), - .const_ = if (@hasField(@TypeOf(inner), "const_")) inner.const_ else false, - .restrict = if (@hasField(@TypeOf(inner), "restrict")) inner.restrict else false, - } }; - }, - .constant_struct => .{ .constant_struct = inner }, - inline else => |tag| @unionInit(ArgSpec, @tagName(tag), typeOf(inner)), - }; -} - -/// The variant a spec literal names: one, as a union literal does. -fn literalVariant(Literal: type, what: []const u8) []const u8 { - const names = @typeInfo(Literal).@"struct".field_names; - if (names.len != 1) @compileError("dsl/metal: " ++ what ++ " literals name one variant, not " ++ std.fmt.comptimePrint("{d}", .{names.len})); - return names[0]; -} - -/// Rejects a field of a spec literal that `Options` does not have. -fn knownFields(Literal: type, Options: type, what: []const u8) void { - for (@typeInfo(Literal).@"struct".field_names) |name| { - if (!@hasField(Options, name)) @compileError("dsl/metal: " ++ what ++ " has no field " ++ name); - } -} - -/// A `Type`, or the name of one (`.float`). -fn typeOf(t: anytype) Type { - return if (@TypeOf(t) == @EnumLiteral()) @field(Type, @tagName(t)) else t; -} - -/// A declared device, constant or threadgroup argument, for binding. -pub const Argument = struct { - name: []const u8, - kind: enum { buffer, constant, threadgroup }, - /// The `[[buffer(n)]]` or `[[threadgroup(n)]]` index. - index: u32, - /// The size of a constant argument. - size: u64 = 0, -}; - -/// A pointer: its address space and pointee type. -pub const Pointer = struct { - space: AddressSpace, - pointee: Type, -}; - -/// An SSA value with its MSL type. Operators follow C++: operands take the -/// usual arithmetic conversions, integers narrower than `int` are promoted, -/// and signedness picks the operation (`uint / int` divides unsigned). -pub const Value = struct { - inner: *const mlir.Value, - kernel: ?*Builder = null, - kind: Kind = undefined, - - pub const Kind = union(enum) { - value: Type, - pointer: Pointer, - /// A `constant S &` argument. - constant_struct: Struct, - /// A `constant T &` argument or a local: an lvalue, read where it is - /// used as C++ reads it. - reference: Pointer, - /// A `simdgroup_matrix`: its 64 elements, of which a lane - /// holds the first two. - matrix: Scalar, - /// A pointer local. - pointer_reference: Pointer, - }; - - /// The value itself, an lvalue read. - pub fn rvalue(self: Value) Value { - return switch (self.kind) { - .reference => |p| (Value{ .inner = self.inner, .kernel = self.kernel, .kind = .{ .pointer = p } }).load(0), - .pointer_reference => |p| blk: { - const k = self.kern(); - break :blk k.emit(llvm.load(k.ctx, self.inner, llvm.pointerType(k.ctx, p.space), .{ .alignment = 8 }, k.loc()), .{ .pointer = p }); - }, - else => self, - }; - } - - /// `&x` of a local. - pub fn address(reference: Value) Value { - return switch (reference.kind) { - .reference => |p| .{ .inner = reference.inner, .kernel = reference.kernel, .kind = .{ .pointer = p } }, - else => std.debug.panic("dsl/metal: the address of a {t}", .{reference.kind}), - }; - } - - /// `x = value` of a local, converting `value` to its type. - pub fn assign(self: Value, value: anytype) void { - const k = self.kern(); - switch (self.kind) { - .reference => |p| { - if (p.space != .thread) std.debug.panic("dsl/metal: assigning to a {t} reference", .{p.space}); - (Value{ .inner = self.inner, .kernel = k, .kind = .{ .pointer = p } }).store(0, value); - }, - .pointer_reference => |p| { - const v = k.lift(value); - if (v.kind != .pointer or v.kind.pointer.space != p.space) std.debug.panic("dsl/metal: assigning a {t} to a {t} pointer", .{ v.kind, p.space }); - _ = llvm.store(k.ctx, v.inner, self.inner, .{ .alignment = 8 }, k.loc()).appendTo(k.currentBlock()); - }, - else => std.debug.panic("dsl/metal: assigning to a {t}", .{self.kind}), - } - } - - fn kern(self: Value) *Builder { - return self.kernel orelse @panic("dsl/metal: value has no builder"); - } - - /// The MSL type of a scalar or vector value. - pub fn type_(self: Value) Type { - return switch (self.kind) { - .value => |t| t, - .reference => |p| p.pointee, - else => std.debug.panic("dsl/metal: expected a scalar or vector, got a {t}", .{self.kind}), - }; - } - - pub fn pointer(self: Value) Pointer { - return switch (self.kind) { - .pointer => |p| p, - else => std.debug.panic("dsl/metal: expected a pointer, got a {t}", .{self.kind}), - }; - } - - pub fn add(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - if (self.kind == .pointer) return self.kern().offset(self, self.kern().promoted(rhs), false); - return self.kern().arithmetic(.add, self, rhs); - } - pub fn sub(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - if (self.kind == .pointer) return self.kern().offset(self, self.kern().promoted(rhs), true); - return self.kern().arithmetic(.sub, self, rhs); - } - pub fn mul(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().arithmetic(.mul, self, rhs); - } - pub fn div(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().arithmetic(.div, self, rhs); - } - pub fn rem(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().arithmetic(.rem, self, rhs); - } - pub fn bitAnd(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().arithmetic(.bit_and, self, rhs); - } - pub fn bitOr(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().arithmetic(.bit_or, self, rhs); - } - pub fn bitXor(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().arithmetic(.bit_xor, self, rhs); - } - pub fn shl(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().shift(.left, self, rhs); - } - pub fn shr(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().shift(.right, self, rhs); - } - pub fn neg(reference: Value) Value { - const self = reference.rvalue(); - return self.kern().negate(self); - } - /// `~x`. - pub fn bitNot(reference: Value) Value { - const self = reference.rvalue(); - const k = self.kern(); - const v = k.convert(self, self.type_().withScalar(self.type_().scalar.promoted())); - return v.bitXor(k.constant(v.type_(), -1)); - } - - pub fn lt(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().compare(.lt, self, rhs); - } - pub fn le(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().compare(.le, self, rhs); - } - pub fn gt(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().compare(.gt, self, rhs); - } - pub fn ge(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().compare(.ge, self, rhs); - } - pub fn eq(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().compare(.eq, self, rhs); - } - pub fn ne(reference: Value, rhs: anytype) Value { - const self = reference.rvalue(); - return self.kern().compare(.ne, self, rhs); - } - - /// `T(x)`, `(T) x` or `static_cast(x)` on a scalar or vector; on a - /// pointer, `(space T *) p`. - pub fn to(reference: Value, t: Type) Value { - const self = reference.rvalue(); - const k = self.kern(); - return switch (self.kind) { - .pointer => |p| .{ .inner = self.inner, .kernel = k, .kind = .{ .pointer = .{ .space = p.space, .pointee = t } } }, - else => k.convert(self, t), - }; - } - - /// `as_type(x)`: the same bits as another type of the same size. - pub fn asType(reference: Value, t: Type) Value { - const self = reference.rvalue(); - const k = self.kern(); - const from = self.type_(); - if (from.size() != t.size() or from.len == 3 or t.len == 3) - std.debug.panic("dsl/metal: as_type<{s}> of a {s}", .{ t.name(), from.name() }); - if (from.eql(t)) return self; - return k.emit(llvm.bitcast(k.ctx, self.inner, t.mlirType(k.ctx), k.loc()), .{ .value = t }); - } - - /// `v.x`. - pub fn x(reference: Value) Value { - const self = reference.rvalue(); - return self.at(0); - } - pub fn y(reference: Value) Value { - const self = reference.rvalue(); - return self.at(1); - } - pub fn z(reference: Value) Value { - const self = reference.rvalue(); - return self.at(2); - } - pub fn w(reference: Value) Value { - const self = reference.rvalue(); - return self.at(3); - } - - /// `v[i]` of a vector; the index keeps its type, a literal is an `int`. - pub fn at(reference: Value, i: anytype) Value { - const self = reference.rvalue(); - const k = self.kern(); - const t = self.type_(); - if (!t.isVector()) std.debug.panic("dsl/metal: indexing a {s}", .{t.name()}); - const element = t.element(); - return k.emit(llvm.extractelement(k.ctx, self.inner, k.subscript(i).inner, element.mlirType(k.ctx), k.loc()), .{ .value = element }); - } - - /// The vector with `v[i] = element`, `i` typed as `at` types it; `v.y = - /// element` is `insert(@as(i64, 1), element)`. - pub fn insert(reference: Value, i: anytype, element: anytype) Value { - const self = reference.rvalue(); - const k = self.kern(); - const t = self.type_(); - const e = k.convert(k.lift(element), t.element()); - return k.emit(llvm.insertelement(k.ctx, self.inner, e.inner, k.subscript(i).inner, k.loc()), .{ .value = t }); - } - - /// `p[i]`. - pub fn load(reference: Value, i: anytype) Value { - const self = reference.rvalue(); - const k = self.kern(); - const p = self.pointer(); - const at_ = k.offset(self, i, false); - return k.loadMemory(at_.inner, p.pointee); - } - - /// `p[i] = value`, converting `value` to the pointee type. - pub fn store(reference: Value, i: anytype, value: anytype) void { - const self = reference.rvalue(); - const k = self.kern(); - const p = self.pointer(); - var v = k.convert(k.lift(value), p.pointee); - if (p.pointee.scalar == .bool) v = k.emit(arith.extui(k.ctx, v.inner, p.pointee.memory().mlirType(k.ctx), k.loc()), .{ .value = p.pointee.memory() }); - const at_ = k.offset(self, i, false); - _ = llvm.store(k.ctx, v.inner, at_.inner, .{ .alignment = v.type_().alignment() }, k.loc()).appendTo(k.currentBlock()); - } - - /// `s.name` of a constant struct argument. - pub fn field(reference: Value, comptime name: []const u8) Value { - const self = reference.rvalue(); - const k = self.kern(); - const s = switch (self.kind) { - .constant_struct => |s| s, - else => std.debug.panic("dsl/metal: .{s} of a {t}", .{ name, self.kind }), - }; - const index = s.fieldIndex(name); - const f = s.fields[index]; - if (f.count != 0) std.debug.panic("dsl/metal: struct array field {s}", .{name}); - const gep = llvm.getelementptr(k.ctx, self.inner, &.{ .{ .constant = 0 }, .{ .constant = @intCast(index) } }, k.structType(s), k.loc()); - _ = gep.appendTo(k.currentBlock()); - return k.loadMemory(gep.result(0), f.type); - } - - /// `m.thread_elements()` of a `simdgroup_matrix`: the lane's two - /// elements. - pub fn threadElements(m: Value) Value { - const k = m.kern(); - const t: Type = .vec(Builder.matrixScalar(m), 2); - return k.emit(llvm.shufflevector(k.ctx, m.inner, m.inner, t.mlirType(k.ctx), &.{ 0, 1 }, k.loc()), .{ .value = t }); - } -}; - -/// A loop: `while (c)` and `for (init; c; inc)` branch to the body with -/// `body(c)`, `for (;;)` has its body right away; `next()` ends the body -/// and starts the increment, and `close()` ends the loop. -/// -/// A `break` and the end of the body meet in a latch that reads whether to -/// leave from a local, as clang's cleanup does: Apple's compiler then keeps -/// the loop's test at the top instead of rotating the loop, which is faster -/// (paged attention's decode loop by 2%). -pub const LoopScope = struct { - kernel: *Builder, - cond_block: *mlir.Block, - end_block: *mlir.Block, - loop_annotation: *const llvm.LoopAnnotationAttr, - /// Where a `break` goes after setting `leave`; null until the first one. - latch: ?*mlir.Block = null, - leave: Value = undefined, - exits: bool = false, - - /// Leaves the loop when `cond` is false. - pub fn body(self: *LoopScope, cond: Value) void { - const k = self.kernel; - const body_block = mlir.Block.init(&.{}, &.{}); - k.condBranch(cond, body_block, self.end_block); - self.exits = true; - k.startBlock(body_block); - } - - /// `break;`. - pub fn breakLoop(self: *LoopScope) void { - if (self.kernel.insert == null) return; - self.toLatch(true); - self.exits = true; - } - - fn toLatch(self: *LoopScope, leave: bool) void { - const k = self.kernel; - if (self.latch == null) { - self.latch = mlir.Block.init(&.{}, &.{}); - self.leave = k.local(.bool, null); - } - self.leave.assign(leave); - _ = k.branch(self.latch.?); - } - - /// Ends the body: the increment of `for (...; ...; inc)` follows. - pub fn next(self: *LoopScope) void { - const k = self.kernel; - const inc = mlir.Block.init(&.{}, &.{}); - if (self.latch) |latch| { - if (k.insert != null) self.toLatch(false); - k.startBlock(latch); - k.condBranch(self.leave, self.end_block, inc); - self.latch = null; - } else _ = k.branch(inc); - k.startBlock(inc); - } - - pub fn close(self: *LoopScope) void { - const k = self.kernel; - if (self.latch != null) self.next(); - k.branchLoop(self.cond_block, self.loop_annotation); - if (self.exits) k.startBlock(self.end_block) else self.end_block.deinit(); - } -}; - -/// `for (T i = lower; i < upper; i += step)`: `i` is the local `iv`, the -/// condition and the increment follow C++'s conversions, and a step of 1 is -/// `++i`. -pub const ForScope = struct { - loop: LoopScope, - iv: Value, - step: Step, - - const Step = union(enum) { one, value: Value, int: i64 }; - - pub fn breakLoop(self: *ForScope) void { - self.loop.breakLoop(); - } - - pub fn close(self: *ForScope) void { - const k = self.loop.kernel; - self.loop.next(); - if (k.insert != null) switch (self.step) { - .one => self.iv.assign(k.increment(self.iv)), - .value => |v| { - const s = v.rvalue(); - self.iv.assign(self.iv.add(s)); - }, - .int => |n| self.iv.assign(self.iv.add(k.constant(if (std.math.cast(i32, n) != null) .int else .long, n))), - }; - self.loop.close(); - } -}; - -/// `if (c) { ... }`, or `if (c) { ... } else { ... }` with `elseBranch()` -/// between the two. -pub const IfScope = struct { - kernel: *Builder, - else_block: ?*mlir.Block, - end_block: *mlir.Block, - joins: bool = false, - - pub fn elseBranch(self: *IfScope) void { - const k = self.kernel; - const else_block = self.else_block orelse @panic("dsl/metal: elseBranch of an if without else (open it with openIfElse(c, .{})), or a second one"); - if (k.branch(self.end_block)) self.joins = true; - k.startBlock(else_block); - self.else_block = null; - } - - pub fn close(self: *IfScope) void { - const k = self.kernel; - if (k.branch(self.end_block)) self.joins = true; - if (self.joins) k.startBlock(self.end_block) else self.end_block.deinit(); - } -}; - -/// A condition of `&&`, `||` and `!` terms, short-circuited: a term's code -/// runs only when the terms before it leave the outcome open. `and_(x)` is -/// `x && ...`, `or_(x)` is `x || ...`; the last term opens what the condition -/// guards. A group of `&&` terms ends a `||` chain and the -/// other way around, as in `a || (b && c)`. -pub const Condition = struct { - kernel: *Builder, - true_block: *mlir.Block, - false_block: *mlir.Block, - - pub fn and_(self: *Condition, x: Value) void { - self.term(x, false, .@"and"); - } - pub fn andNot(self: *Condition, x: Value) void { - self.term(x, true, .@"and"); - } - pub fn or_(self: *Condition, x: Value) void { - self.term(x, false, .@"or"); - } - pub fn orNot(self: *Condition, x: Value) void { - self.term(x, true, .@"or"); - } - - fn term(self: *Condition, x: Value, negate: bool, comptime op: enum { @"and", @"or" }) void { - const k = self.kernel; - const next = mlir.Block.init(&.{}, &.{}); - const on_true, const on_false = switch (op) { - .@"and" => .{ next, self.false_block }, - .@"or" => .{ self.true_block, next }, - }; - if (negate) k.condBranch(x, on_false, on_true) else k.condBranch(x, on_true, on_false); - k.startBlock(next); - } - - fn last(self: *Condition, x: Value, negate: bool) void { - if (negate) self.kernel.condBranch(x, self.false_block, self.true_block) else self.kernel.condBranch(x, self.true_block, self.false_block); - } - - /// `if (... x) { ... }`. - pub fn openIf(self: *Condition, x: Value) IfScope { - return self.opened(x, false, false); - } - /// `if (... !x) { ... }`. - pub fn openIfNot(self: *Condition, x: Value) IfScope { - return self.opened(x, true, false); - } - /// `if (... x) { ... } else { ... }`; a chain has no result types. - pub fn openIfElse(self: *Condition, x: Value, result_types: anytype) IfScope { - noResults(result_types); - return self.opened(x, false, true); - } - /// `if (... !x) { ... } else { ... }`. - pub fn openIfElseNot(self: *Condition, x: Value, result_types: anytype) IfScope { - noResults(result_types); - return self.opened(x, true, true); - } - - fn noResults(result_types: anytype) void { - if (common.arity(@TypeOf(result_types), "Condition.openIfElse: result_types") != 0) - @compileError("dsl/metal: a condition chain opens no `c ? a : b`"); - } - - fn opened(self: *Condition, x: Value, negate: bool, with_else: bool) IfScope { - const k = self.kernel; - self.last(x, negate); - k.startBlock(self.true_block); - if (!with_else) return .{ .kernel = k, .else_block = null, .end_block = self.false_block, .joins = true }; - return .{ .kernel = k, .else_block = self.false_block, .end_block = mlir.Block.init(&.{}, &.{}) }; - } -}; - -/// `a && b && ...` or `a || b || ...` as a bool, short-circuited: a -/// term's code runs only when the terms before it leave the result open. -/// `term(x)` adds a term followed by more, `close(x)` the last one. -pub const Logical = struct { - kernel: *Builder, - op: enum { @"and", @"or" }, - end_block: *mlir.Block, - - pub fn term(self: *Logical, x: Value) void { - const k = self.kernel; - const next = mlir.Block.init(&.{}, &.{}); - const c = k.convert(x, .bool); - const decided: []const *const mlir.Value = &.{k.constant(.bool, self.op == .@"or").inner}; - _ = switch (self.op) { - .@"and" => cf.cond_br(k.ctx, c.inner, next, &.{}, self.end_block, decided, .{}, k.loc()), - .@"or" => cf.cond_br(k.ctx, c.inner, self.end_block, decided, next, &.{}, .{}, k.loc()), - }.appendTo(k.currentBlock()); - k.insert = null; - k.startBlock(next); - } - - pub fn close(self: *Logical, x: Value) Value { - const k = self.kernel; - const c = k.convert(x, .bool); - _ = cf.br(k.ctx, self.end_block, &.{c.inner}, k.loc()).appendTo(k.currentBlock()); - k.insert = null; - k.startBlock(self.end_block); - return .{ .inner = self.end_block.argument(0), .kernel = k, .kind = .{ .value = .bool } }; - } -}; - -/// `c ? a : b` evaluating one operand: the true operand's code, then -/// `yieldThen(.{a})`, the false operand's code, then `yieldElse(.{b})`; -/// `results[0]` is the value. -pub const TernaryScope = struct { - kernel: *Builder, - type: Type, - else_block: *mlir.Block, - end_block: *mlir.Block, - results: [1]Value = undefined, - - pub fn yieldThen(self: *TernaryScope, values: anytype) void { - const k = self.kernel; - self.branchEnd(values, "yieldThen"); - k.startBlock(self.else_block); - } - - pub fn yieldElse(self: *TernaryScope, values: anytype) void { - const k = self.kernel; - self.branchEnd(values, "yieldElse"); - k.startBlock(self.end_block); - self.results = .{.{ .inner = self.end_block.argument(0), .kernel = k, .kind = .{ .value = self.type } }}; - } - - fn branchEnd(self: *TernaryScope, values: anytype, comptime what: []const u8) void { - if (common.arity(@TypeOf(values), "TernaryScope." ++ what) != 1) @compileError("TernaryScope." ++ what ++ " takes one value"); - const k = self.kernel; - const v = k.convert(k.lift(values[0]), self.type); - _ = cf.br(k.ctx, self.end_block, &.{v.inner}, k.loc()).appendTo(k.currentBlock()); - k.insert = null; - } -}; - -/// What `Builder.openIfElse` opens for `n` result types. -fn IfElseScope(comptime n: usize) type { - return switch (n) { - 0 => IfScope, - 1 => TernaryScope, - else => @compileError("dsl/metal: openIfElse has one result at most, as `c ? a : b`"), - }; -} - -/// A loop pragma: `full` is `#pragma clang loop unroll(full)`, `enable` -/// `#pragma unroll`. -pub const Unroll = enum { full, enable }; - -pub const LoopOpts = struct { - unroll: ?Unroll = null, -}; - -/// MSL's `metal::fast` and `metal::precise` functions. -pub const MathMode = enum { default, fast, precise }; - -/// `metal::fast::` or `metal::precise::`. -pub const Namespace = struct { - b: *Builder, - mode: MathMode, - - pub fn exp(self: Namespace, x: Value) Value { - return self.b.floatMath("exp", self.mode, .{x}); - } - pub fn exp2(self: Namespace, x: Value) Value { - return self.b.floatMath("exp2", self.mode, .{x}); - } - pub fn log(self: Namespace, x: Value) Value { - return self.b.floatMath("log", self.mode, .{x}); - } - pub fn log2(self: Namespace, x: Value) Value { - return self.b.floatMath("log2", self.mode, .{x}); - } - pub fn tanh(self: Namespace, x: Value) Value { - return self.b.floatMath("tanh", self.mode, .{x}); - } - pub fn sqrt(self: Namespace, x: Value) Value { - return self.b.floatMath("sqrt", self.mode, .{x}); - } - pub fn rsqrt(self: Namespace, x: Value) Value { - return self.b.floatMath("rsqrt", self.mode, .{x}); - } - pub fn sin(self: Namespace, x: Value) Value { - return self.b.floatMath("sin", self.mode, .{x}); - } - pub fn cos(self: Namespace, x: Value) Value { - return self.b.floatMath("cos", self.mode, .{x}); - } - pub fn max(self: Namespace, a: Value, b: anytype) Value { - return self.b.floatMath("fmax", self.mode, .{ a, b }); - } - pub fn min(self: Namespace, a: Value, b: anytype) Value { - return self.b.floatMath("fmin", self.mode, .{ a, b }); - } -}; - -pub const Builder = struct { - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - ctx: *mlir.Context, - module: *mlir.Module, - name: []const u8, - fast_math: bool, - func_op: ?*mlir.Operation = null, - entry_block: ?*mlir.Block = null, - /// The kernel's body; blocks join it in the order they start. - body: ?*mlir.Region = null, - /// The block code goes to; null after a `break` or a `return`. - insert: ?*mlir.Block = null, - arguments: std.ArrayList(Argument) = .empty, - - pub const Opts = struct { - /// MSL's default; `-fno-fast-math` otherwise. - fast_math: bool = true, - }; - - pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8, opts: Opts) Builder { - const module: *mlir.Module = .init(.unknown(ctx)); - return .{ - .allocator = allocator, - .arena = .init(allocator), - .ctx = ctx, - .module = module, - .name = name, - .fast_math = opts.fast_math, - }; - } - - pub fn deinit(self: *Builder) void { - self.module.deinit(); - self.arena.deinit(); - } - - /// Declares the kernel's arguments, in order, from a struct literal of - /// `ArgSpec`s, and returns them by name. - pub fn declareArgs(self: *Builder, spec: anytype) FinishError!common.NamedArgs(@TypeOf(spec), Value) { - const ctx = self.ctx; - const location = self.loc(); - const fields = @typeInfo(@TypeOf(spec)).@"struct".field_names; - const arena = self.arena.allocator(); - var arg_types: [fields.len]*const mlir.Type = undefined; - var arg_attrs: [fields.len]*const mlir.Attribute = undefined; - var kinds: [fields.len]Value.Kind = undefined; - var buffers: u32 = 0; - var threadgroups: u32 = 0; - inline for (fields, 0..) |name, i| { - const arg = argSpec(@field(spec, name)); - const opts: air.ArgumentOpts = .{ .name = name }; - switch (arg) { - .buffer => |buffer| { - arg_types[i] = llvm.pointerType(ctx, AddressSpace.device); - arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ - .locationIndex = buffers, - .addressSpace = .device, - .access = if (buffer.const_) .read else .read_write, - .elementType = buffer.type.memory().mlirType(ctx), - .typeName = buffer.type.name(), - .typeSize = buffer.type.size(), - .typeAlignment = buffer.type.alignment(), - .noalias_ = buffer.restrict, - }, opts) catch return error.InvalidMlir; - kinds[i] = .{ .pointer = .{ .space = .device, .pointee = buffer.type } }; - try self.arguments.append(arena, .{ .name = name, .kind = .buffer, .index = buffers }); - buffers += 1; - }, - .constant => |t| { - arg_types[i] = llvm.pointerType(ctx, AddressSpace.constant); - arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ - .locationIndex = buffers, - .addressSpace = .constant, - .access = .read, - .elementType = t.memory().mlirType(ctx), - .typeName = t.name(), - .typeSize = t.size(), - .typeAlignment = t.alignment(), - .bufferSize = t.size(), - }, opts) catch return error.InvalidMlir; - kinds[i] = .{ .reference = .{ .space = .constant, .pointee = t } }; - try self.arguments.append(arena, .{ .name = name, .kind = .constant, .index = buffers, .size = t.size() }); - buffers += 1; - }, - .constant_struct => |s| { - var struct_fields = try arena.alloc(*const air.StructFieldAttr, s.fields.len); - for (s.fields, 0..) |f, j| struct_fields[j] = air.StructFieldAttr.get(ctx, .{ - .offset = s.offset(j), - .size = f.type.size(), - .count = f.count, - .typeName = f.type.name(), - .name = f.name, - }) catch return error.InvalidMlir; - arg_types[i] = llvm.pointerType(ctx, AddressSpace.constant); - arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ - .locationIndex = buffers, - .addressSpace = .constant, - .access = .read, - .elementType = self.structType(s), - .typeName = s.name, - .typeSize = s.size(), - .typeAlignment = s.alignment(), - .bufferSize = s.size(), - .fields = struct_fields, - }, opts) catch return error.InvalidMlir; - kinds[i] = .{ .constant_struct = s }; - try self.arguments.append(arena, .{ .name = name, .kind = .constant, .index = buffers, .size = s.size() }); - buffers += 1; - }, - .threadgroup => |t| { - arg_types[i] = llvm.pointerType(ctx, AddressSpace.threadgroup); - arg_attrs[i] = air.bufferArgumentAttr(ctx, .{ - .locationIndex = threadgroups, - .addressSpace = .threadgroup, - .access = .read_write, - .elementType = t.memory().mlirType(ctx), - .typeName = t.name(), - .typeSize = t.size(), - .typeAlignment = t.alignment(), - }, opts) catch return error.InvalidMlir; - kinds[i] = .{ .pointer = .{ .space = .threadgroup, .pointee = t } }; - try self.arguments.append(arena, .{ .name = name, .kind = .threadgroup, .index = threadgroups }); - threadgroups += 1; - }, - inline else => |t, builtin| { - arg_types[i] = t.mlirType(ctx); - arg_attrs[i] = air.builtinArgumentAttr(ctx, @field(air.Builtin, @tagName(builtin)), opts) catch return error.InvalidMlir; - kinds[i] = .{ .value = t }; - }, - } - } - var locs: [fields.len]*const mlir.Location = @splat(location); - const block = mlir.Block.init(&arg_types, &locs); - self.entry_block = block; - self.func_op = air.kernel(ctx, .{ - .name = self.name, - .block = block, - .args_attributes = &arg_attrs, - .location = location, - .verify = false, - }).appendTo(self.module.body()); - self.body = block.parentRegion(); - self.insert = block; - - var out: common.NamedArgs(@TypeOf(spec), Value) = undefined; - inline for (fields, 0..) |name, i| @field(out, name) = .{ .inner = block.argument(i), .kernel = self, .kind = kinds[i] }; - return out; - } - - pub fn loc(self: *const Builder) *const mlir.Location { - return .unknown(self.ctx); - } - - /// The block code goes to: a fresh, unreachable one after a `break` or - /// a `return`. - pub fn currentBlock(self: *Builder) *mlir.Block { - if (self.insert == null) self.startBlock(mlir.Block.init(&.{}, &.{})); - return self.insert.?; - } - - fn startBlock(self: *Builder, block: *mlir.Block) void { - self.body.?.appendOwnedBlock(block); - self.insert = block; - } - - /// Branches to `dest` unless the code here is unreachable; true if it did. - fn branch(self: *Builder, dest: *mlir.Block) bool { - const block = self.insert orelse return false; - _ = cf.br(self.ctx, dest, &.{}, self.loc()).appendTo(block); - self.insert = null; - return true; - } - - fn condBranch(self: *Builder, cond: Value, on_true: *mlir.Block, on_false: *mlir.Block) void { - const c = self.convert(cond, .bool); - _ = cf.cond_br(self.ctx, c.inner, on_true, &.{}, on_false, &.{}, .{}, self.loc()).appendTo(self.currentBlock()); - self.insert = null; - } - - /// The back edge of a loop, which carries its `llvm.loop` metadata. - fn branchLoop(self: *Builder, dest: *mlir.Block, annotation: *const llvm.LoopAnnotationAttr) void { - const block = self.insert orelse return; - _ = llvm.br(self.ctx, dest, &.{}, .{ .loop_annotation = annotation }, self.loc()).appendTo(block); - self.insert = null; - } - - /// A thread-memory slot of `t`, at the top of the entry block. - fn allocate(self: *Builder, t: *const mlir.Type, alignment: u64) *const mlir.Value { - const ctx = self.ctx; - const entry = self.entry_block.?; - const size = arith.constant_int(ctx, 1, .int(ctx, .i32), self.loc()); - const alloca = llvm.alloca(ctx, size.result(0), llvm.pointerType(ctx, AddressSpace.thread), t, .{ .alignment = alignment }, self.loc()); - if (entry.firstOperation()) |first| entry.insertOwnedOperationBefore(first, alloca) else _ = alloca.appendTo(entry); - entry.insertOwnedOperationBefore(alloca, size); - return alloca.result(0); - } - - /// `T x;`, or `T x = init;`: a local, read where it is used and written by - /// `assign`. - pub fn local(self: *Builder, t: Type, init: anytype) Value { - const memory = t.memory(); - const x: Value = .{ .inner = self.allocate(memory.mlirType(self.ctx), memory.alignment()), .kernel = self, .kind = .{ .reference = .{ .space = .thread, .pointee = t } } }; - if (@TypeOf(init) != @TypeOf(null)) x.assign(init); - return x; - } - - /// `T *p = init;`: a pointer local. - pub fn localPointer(self: *Builder, init: Value) Value { - const p: Pointer = switch (init.kind) { - .pointer, .pointer_reference => |q| q, - else => std.debug.panic("dsl/metal: a pointer local of a {t}", .{init.kind}), - }; - const x: Value = .{ .inner = self.allocate(llvm.pointerType(self.ctx, p.space), 8), .kernel = self, .kind = .{ .pointer_reference = p } }; - x.assign(init); - return x; - } - - /// A `simdgroup_matrix` whose lane holds `elements`, two values - /// converted to `T`: what assigning `m.thread_elements()` makes. - pub fn matrix(self: *Builder, scalar: Scalar, elements: anytype) Value { - const v = self.convert(self.lift(elements), .vec(scalar, 2)); - var mask: [64]i32 = @splat(-1); - mask[0..2].* = .{ 0, 1 }; - return self.emit(llvm.shufflevector(self.ctx, v.inner, v.inner, self.matrixType(scalar), &mask, self.loc()), .{ .matrix = scalar }); - } - - /// `++i`: a narrow integer steps in its own type, as C++ steps it. - fn increment(self: *Builder, i: Value) Value { - const v = i.rvalue(); - const t = v.type_(); - if (t.scalar.isFloat() or t.scalar.bits() >= 32) return v.add(1); - return self.emit(arith.addi(self.ctx, v.inner, self.constant(t, 1).inner, self.loc()), .{ .value = t }); - } - - fn emit(self: *Builder, op: *mlir.Operation, kind: Value.Kind) Value { - _ = op.appendTo(self.currentBlock()); - return .{ .inner = op.result(0), .kernel = self, .kind = kind }; - } - - fn fastmath(self: *const Builder) air.FloatOpts { - return .{ .fastmath = if (self.fast_math) .fast else .none }; - } - - fn setFastmath(self: *Builder, op: *mlir.Operation) void { - if (!self.fast_math) return; - op.setAttribute(arith.fastmath(self.ctx, .fast) catch unreachable); - } - - fn structType(self: *Builder, s: Struct) *const mlir.Type { - var fields: [32]*const mlir.Type = undefined; - for (s.fields, 0..) |f, i| { - const t = f.type.memory().mlirType(self.ctx); - fields[i] = if (f.count != 0) llvm.arrayType(t, f.count) else t; - } - var name: std.ArrayList(u8) = .empty; - name.print(self.arena.allocator(), "struct.{s}", .{s.name}) catch @panic("dsl/metal: out of memory"); - return llvm.identifiedStructType(self.ctx, name.items, fields[0..s.fields.len]) orelse - std.debug.panic("dsl/metal: two structs named {s}", .{s.name}); - } - - /// The MSL type of a Zig number, as `lift` gives it. - fn literalType(value: anytype) Type { - return switch (@typeInfo(@TypeOf(value))) { - .comptime_int => if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) .int else .long, - .comptime_float => .float, - .int => |info| switch (info.bits) { - 8 => if (info.signedness == .signed) .char else .uchar, - 16 => if (info.signedness == .signed) .short else .ushort, - 32 => if (info.signedness == .signed) .int else .uint, - 64 => if (info.signedness == .signed) .long else .ulong, - else => @compileError("dsl/metal: no MSL type for " ++ @typeName(@TypeOf(value))), - }, - else => @compileError("dsl/metal: expected a literal, got " ++ @typeName(@TypeOf(value))), - }; - } - - /// A Zig value as an MSL literal: integers are `int` (`long` when they do - /// not fit), floats are `float`. - pub fn lift(self: *Builder, value: anytype) Value { - const T = @TypeOf(value); - if (T == Value) return value.rvalue(); - return switch (@typeInfo(T)) { - .comptime_int, .comptime_float, .int => self.constant(literalType(value), value), - .bool => self.constant(.bool, value), - .float => |info| self.constant(switch (info.bits) { - 16 => Type.half, - 32 => Type.float, - else => @compileError("dsl/metal: no MSL type for " ++ @typeName(T)), - }, value), - else => @compileError("dsl/metal: expected a Value or a number, got " ++ @typeName(T)), - }; - } - - /// A constant of type `t`, splat across a vector. - pub fn constant(self: *Builder, t: Type, value: anytype) Value { - const ctx = self.ctx; - const ty = t.mlirType(ctx); - const op = if (t.scalar.isFloat()) blk: { - const v: f64 = switch (@typeInfo(@TypeOf(value))) { - .int, .comptime_int => @floatFromInt(value), - .bool => @floatFromInt(@intFromBool(value)), - else => @floatCast(value), - }; - if (t.isVector()) break :blk arith.constant_dense_splat_f(ctx, v, ty, self.loc()); - break :blk switch (t.scalar) { - .half => arith.constant_float(ctx, v, .f16, self.loc()), - .bfloat => arith.constant_float(ctx, v, .bf16, self.loc()), - else => arith.constant_float(ctx, v, .f32, self.loc()), - }; - } else blk: { - const v: i64 = switch (@typeInfo(@TypeOf(value))) { - .bool => @intFromBool(value), - .int, .comptime_int => if (@TypeOf(value) == u64) @bitCast(value) else value, - else => @intFromFloat(value), - }; - if (t.isVector()) break :blk arith.constant_dense_splat_i(ctx, v, ty, self.loc()); - break :blk arith.constant_int(ctx, v, ty, self.loc()); - }; - return self.emit(op, .{ .value = t }); - } - - /// `T(a, b, ...)` from scalars, or `T(x)` splat from one scalar. - pub fn vec(self: *Builder, t: Type, elements: anytype) Value { - const fields = @typeInfo(@TypeOf(elements)).@"struct".field_names; - if (fields.len == 1) return self.convert(self.lift(@field(elements, fields[0])), t); - if (fields.len != t.len) std.debug.panic("dsl/metal: {d} elements of a {s}", .{ fields.len, t.name() }); - var inner: [fields.len]*const mlir.Value = undefined; - inline for (fields, 0..) |name, i| inner[i] = self.convert(self.lift(@field(elements, name)), t.element()).inner; - return self.emit(vector.from_elements(self.ctx, &inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); - } - - fn splat(self: *Builder, scalar: Value, t: Type) Value { - return self.emit(vector.broadcast(self.ctx, scalar.inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); - } - - /// The conversion of `v` to `to`, as C++ converts: integer widths extend - /// by the source's signedness or truncate, integers and floats convert - /// through `air.convert`, and a scalar converts to a vector by splatting. - pub fn convert(self: *Builder, reference: Value, to: Type) Value { - const v = reference.rvalue(); - const ctx = self.ctx; - const from = v.type_(); - if (from.eql(to)) return v; - if (from.len != to.len) { - if (!from.isVector()) return self.splat(self.convert(v, to.element()), to); - std.debug.panic("dsl/metal: cannot convert a {s} to a {s}", .{ from.name(), to.name() }); - } - const fs = from.scalar; - const ts = to.scalar; - const ty = to.mlirType(ctx); - const kind: Value.Kind = .{ .value = to }; - if (ts == .bool) return v.ne(self.constant(from, 0)); - if (fs == .bool) { - if (ts.isFloat()) return self.convert(self.convert(v, to.withScalar(.int)), to); - return self.emit(arith.extui(ctx, v.inner, ty, self.loc()), kind); - } - if (fs.isFloat() and ts.isFloat()) { - if (!from.isVector() and (fs == .float or ts == .float)) { - const op = if (ts == .float) arith.extf(ctx, v.inner, ty, self.loc()) else arith.truncf(ctx, v.inner, ty, self.loc()); - return self.emit(op, kind); - } - return self.emit(air.convert(ctx, v.inner, ty, self.fastmath(), self.loc()), kind); - } - if (fs.isFloat()) { - const op = if (ts.isUnsigned()) air.convert_u(ctx, v.inner, ty, .{}, self.loc()) else air.convert_s(ctx, v.inner, ty, .{}, self.loc()); - return self.emit(op, kind); - } - if (ts.isFloat()) { - const op = if (fs.isUnsigned()) air.convert_u(ctx, v.inner, ty, self.fastmath(), self.loc()) else air.convert_s(ctx, v.inner, ty, self.fastmath(), self.loc()); - return self.emit(op, kind); - } - if (fs.bits() == ts.bits()) return .{ .inner = v.inner, .kernel = self, .kind = kind }; - if (fs.bits() > ts.bits()) return self.emit(arith.trunci(ctx, v.inner, ty, self.loc()), kind); - const op = if (fs.isUnsigned()) arith.extui(ctx, v.inner, ty, self.loc()) else arith.extsi(ctx, v.inner, ty, self.loc()); - return self.emit(op, kind); - } - - /// The operands of a binary operator after the usual arithmetic - /// conversions; a scalar operand of a vector takes its element type. - fn usualConversions(self: *Builder, a: Value, b_: anytype) [2]Value { - const b = self.lift(b_); - const ta = a.type_(); - const tb = b.type_(); - const converted: Type = if (ta.isVector() and tb.isVector()) blk: { - if (!ta.eql(tb)) std.debug.panic("dsl/metal: {s} and {s} operands", .{ ta.name(), tb.name() }); - break :blk ta; - } else if (ta.isVector()) ta else if (tb.isVector()) tb else .{ .scalar = types.commonScalar(ta.scalar, tb.scalar) }; - return .{ self.convert(a, converted), self.convert(b, converted) }; - } - - const ArithmeticOp = enum { add, sub, mul, div, rem, bit_and, bit_or, bit_xor }; - - fn arithmetic(self: *Builder, comptime op: ArithmeticOp, a: Value, b: anytype) Value { - const ctx = self.ctx; - const l, const r = self.usualConversions(a, b); - const t = l.type_(); - const loc_ = self.loc(); - if (t.scalar.isFloat()) { - const result = switch (op) { - .add => arith.addf(ctx, l.inner, r.inner, loc_), - .sub => arith.subf(ctx, l.inner, r.inner, loc_), - .mul => arith.mulf(ctx, l.inner, r.inner, loc_), - .div => arith.divf(ctx, l.inner, r.inner, loc_), - else => std.debug.panic("dsl/metal: {t} of floats", .{op}), - }; - self.setFastmath(result); - return self.emit(result, .{ .value = t }); - } - const unsigned = t.scalar.isUnsigned(); - const result = switch (op) { - .add => arith.addi(ctx, l.inner, r.inner, loc_), - .sub => arith.subi(ctx, l.inner, r.inner, loc_), - .mul => arith.muli(ctx, l.inner, r.inner, loc_), - .div => if (unsigned) arith.divui(ctx, l.inner, r.inner, loc_) else arith.divsi(ctx, l.inner, r.inner, loc_), - .rem => if (unsigned) arith.remui(ctx, l.inner, r.inner, loc_) else arith.remsi(ctx, l.inner, r.inner, loc_), - .bit_and => arith.andi(ctx, l.inner, r.inner, loc_), - .bit_or => arith.ori(ctx, l.inner, r.inner, loc_), - .bit_xor => arith.xori(ctx, l.inner, r.inner, loc_), - }; - // Signed overflow is undefined in C++. - if (!unsigned and (op == .add or op == .sub or op == .mul)) { - result.setAttribute(arith.overflowFlags(ctx, .{ .nsw = true }) catch unreachable); - } - return self.emit(result, .{ .value = t }); - } - - /// `a << b` or `a >> b`. MSL shifts by the amount modulo the width: - /// clang casts the promoted amount to `a`'s promoted type as unsigned, - /// then masks it, which folds away for a number within the width. - fn shift(self: *Builder, comptime direction: enum { left, right }, a: Value, b: anytype) Value { - const t = a.type_().withScalar(a.type_().scalar.promoted()); - const l = self.convert(a, t); - const mask = t.scalar.bits() - 1; - const r = if (@TypeOf(b) == Value) blk: { - const rb = b.rvalue(); - const amount = self.shiftAmount(self.convert(rb, rb.type_().withScalar(rb.type_().scalar.promoted())), t); - break :blk self.emit(arith.andi(self.ctx, amount.inner, self.constant(t, mask).inner, self.loc()), .{ .value = t }); - } else blk: { - if (b < 0 or b > mask) std.debug.panic("dsl/metal: a shift of a {s} by {d}, past its width", .{ t.name(), b }); - const rb = self.lift(b); - break :blk self.convert(self.convert(rb, rb.type_().withScalar(rb.type_().scalar.promoted())), t); - }; - const loc_ = self.loc(); - const op = switch (direction) { - .left => arith.shli(self.ctx, l.inner, r.inner, loc_), - .right => if (t.scalar.isUnsigned()) arith.shrui(self.ctx, l.inner, r.inner, loc_) else arith.shrsi(self.ctx, l.inner, r.inner, loc_), - }; - return self.emit(op, .{ .value = t }); - } - - /// A promoted shift amount `v` as the shifted type `t`: a scalar of a - /// vector converts and splats, as C++ converts it; otherwise the cast is - /// unsigned, as clang casts it. - fn shiftAmount(self: *Builder, v: Value, t: Type) Value { - const from = v.type_(); - if (from.len != t.len) return self.convert(v, t); - if (from.scalar.bits() == t.scalar.bits()) return .{ .inner = v.inner, .kernel = self, .kind = .{ .value = t } }; - const ty = t.mlirType(self.ctx); - const op = if (from.scalar.bits() > t.scalar.bits()) arith.trunci(self.ctx, v.inner, ty, self.loc()) else arith.extui(self.ctx, v.inner, ty, self.loc()); - return self.emit(op, .{ .value = t }); - } - - fn negate(self: *Builder, a: Value) Value { - const t = a.type_(); - if (t.scalar.isFloat()) { - const op = arith.negf(self.ctx, a.inner, self.loc()); - self.setFastmath(op); - return self.emit(op, .{ .value = t }); - } - const x = self.convert(a, t.withScalar(t.scalar.promoted())); - return self.constant(x.type_(), 0).sub(x); - } - - const Comparison = enum { lt, le, gt, ge, eq, ne }; - - fn compare(self: *Builder, comptime op: Comparison, a: Value, b: anytype) Value { - const l, const r = self.usualConversions(a, b); - const t = l.type_(); - const result_type = t.withScalar(.bool); - if (t.scalar.isFloat()) { - const predicate: arith.CmpFPredicate = switch (op) { - .lt => .olt, - .le => .ole, - .gt => .ogt, - .ge => .oge, - .eq => .oeq, - .ne => .une, - }; - const result = arith.cmpf(self.ctx, predicate, l.inner, r.inner, self.loc()); - self.setFastmath(result); - return self.emit(result, .{ .value = result_type }); - } - const unsigned = t.scalar.isUnsigned(); - const predicate: arith.CmpIPredicate = switch (op) { - .lt => if (unsigned) .ult else .slt, - .le => if (unsigned) .ule else .sle, - .gt => if (unsigned) .ugt else .sgt, - .ge => if (unsigned) .uge else .sge, - .eq => .eq, - .ne => .ne, - }; - return self.emit(arith.cmpi(self.ctx, predicate, l.inner, r.inner, self.loc()), .{ .value = result_type }); - } - - /// `i` of `p + i` and `p - i`, promoted as C++ promotes an integer - /// operand; `p[i]` and a vector subscript take `i` as it is. - fn promoted(self: *Builder, i: anytype) Value { - const v = self.subscript(i); - const t = v.type_(); - return self.convert(v, t.withScalar(t.scalar.promoted())); - } - - /// A vector subscript, of its own type as clang keeps it: a literal is an - /// `int`, a `ushort` an `i16`, a `bool` an `i1`. - fn subscript(self: *Builder, i: anytype) Value { - const v = self.lift(i); - const t = v.type_(); - if (t.isVector() or t.scalar.isFloat()) std.debug.panic("dsl/metal: an index of a {s}", .{t.name()}); - return v; - } - - /// A `t` read from `p`, where it is a `t.memory()`. - fn loadMemory(self: *Builder, p: *const mlir.Value, t: Type) Value { - const memory = t.memory(); - const x = self.emit(llvm.load(self.ctx, p, memory.mlirType(self.ctx), .{ .alignment = memory.alignment() }, self.loc()), .{ .value = memory }); - if (t.scalar != .bool) return x; - return self.emit(arith.trunci(self.ctx, x.inner, t.mlirType(self.ctx), self.loc()), .{ .value = t }); - } - - /// `p + i` (`p - i` when `negative`) in elements of the pointee type. - fn offset(self: *Builder, p: Value, i: anytype, negative: bool) Value { - if (@TypeOf(i) == comptime_int and i == 0) return p; - const pointee = switch (p.kind) { - .pointer => |ptr| ptr.pointee, - else => std.debug.panic("dsl/metal: pointer arithmetic on a {t}", .{p.kind}), - }; - var index = self.lift(i); - if (index.type_().scalar.isFloat() or index.type_().isVector()) std.debug.panic("dsl/metal: pointer offset by a {s}", .{index.type_().name()}); - index = self.convert(index, if (index.type_().scalar.isUnsigned()) .ulong else .long); - if (negative) index = self.constant(.long, 0).sub(index); - const gep = llvm.getelementptr(self.ctx, p.inner, &.{.{ .value = index.inner }}, pointee.memory().mlirType(self.ctx), self.loc()); - return self.emit(gep, p.kind); - } - - /// `cond ? a : b` of evaluated operands, a select; - /// `openIfElse(cond, .{T})` evaluates one operand. - pub fn ternary(self: *Builder, cond: Value, a: anytype, b: anytype) Value { - const l, const r = self.usualConversions(self.lift(a), b); - const c = self.convert(cond, if (l.type_().isVector() and cond.type_().isVector()) l.type_().withScalar(.bool) else .bool); - return self.emit(arith.select(self.ctx, c.inner, l.inner, r.inner, self.loc()), l.kind); - } - - fn openTernary(self: *Builder, cond: Value, t: Type) TernaryScope { - const then_block = mlir.Block.init(&.{}, &.{}); - const else_block = mlir.Block.init(&.{}, &.{}); - var loc_ = [1]*const mlir.Location{self.loc()}; - const end_block = mlir.Block.init(&.{t.mlirType(self.ctx)}, &loc_); - self.condBranch(cond, then_block, else_block); - self.startBlock(then_block); - return .{ .kernel = self, .type = t, .else_block = else_block, .end_block = end_block }; - } - - /// `T a[n] = {};`, zeroed as clang zeroes it. - pub fn zeroedArray(self: *Builder, t: Type, n: u32) Value { - const a = self.array(t, n); - const len = self.constant(.long, @as(i64, n) * @as(i64, @intCast(t.size()))); - const operands = [_]*const mlir.Value{ a.inner, self.constant(.char, 0).inner, len.inner, self.constant(.bool, false).inner }; - _ = llvm.call_intrinsic(self.ctx, "llvm.memset.p0.i64", &operands, .{}, self.loc()).appendTo(self.currentBlock()); - return a; - } - - /// `T a[n];` in thread memory, as a pointer to its first element. - pub fn array(self: *Builder, t: Type, n: u32) Value { - return .{ .inner = self.allocate(llvm.arrayType(t.memory().mlirType(self.ctx), n), t.alignment()), .kernel = self, .kind = .{ .pointer = .{ .space = .thread, .pointee = t } } }; - } - - fn floatMath(self: *Builder, comptime name: []const u8, mode: MathMode, args: anytype) Value { - const x = self.lift(args[0]); - const t = x.type_(); - if (!t.scalar.isFloat() or t.scalar == .bfloat) - std.debug.panic("dsl/metal: no {s} of a {s}", .{ name, t.name() }); - const use_fast = switch (mode) { - .default => self.fast_math and t.scalar == .float, - .fast => true, - .precise => false, - }; - if (use_fast and t.scalar != .float) std.debug.panic("dsl/metal: fast::{s} of a {s}", .{ name, t.name() }); - const opts = self.fastmath(); - const loc_ = self.loc(); - const op = switch (args.len) { - 1 => if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, opts, loc_), - 2 => blk: { - const y = self.convert(self.lift(args[1]), t); - break :blk if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, y.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, y.inner, opts, loc_); - }, - 3 => blk: { - const y = self.convert(self.lift(args[1]), t); - const z = self.convert(self.lift(args[2]), t); - break :blk if (use_fast) @field(air, "fast_" ++ name)(self.ctx, x.inner, y.inner, z.inner, opts, loc_) else @field(air, name)(self.ctx, x.inner, y.inner, z.inner, opts, loc_); - }, - else => unreachable, - }; - return self.emit(op, .{ .value = t }); - } - - /// `metal::fast::f`: `b.fast().exp2(x)`. - pub fn fast(self: *Builder) Namespace { - return .{ .b = self, .mode = .fast }; - } - - /// `metal::precise::f`: `b.precise().tanh(x)`. - pub fn precise(self: *Builder) Namespace { - return .{ .b = self, .mode = .precise }; - } - - pub fn exp(self: *Builder, x: Value) Value { - return self.floatMath("exp", .default, .{x}); - } - pub fn exp2(self: *Builder, x: Value) Value { - return self.floatMath("exp2", .default, .{x}); - } - pub fn log(self: *Builder, x: Value) Value { - return self.floatMath("log", .default, .{x}); - } - pub fn log2(self: *Builder, x: Value) Value { - return self.floatMath("log2", .default, .{x}); - } - pub fn tanh(self: *Builder, x: Value) Value { - return self.floatMath("tanh", .default, .{x}); - } - pub fn sqrt(self: *Builder, x: Value) Value { - return self.floatMath("sqrt", .default, .{x}); - } - pub fn rsqrt(self: *Builder, x: Value) Value { - return self.floatMath("rsqrt", .default, .{x}); - } - pub fn ceil(self: *Builder, x: Value) Value { - return self.floatMath("ceil", .default, .{x}); - } - pub fn floor(self: *Builder, x: Value) Value { - return self.floatMath("floor", .default, .{x}); - } - pub fn round(self: *Builder, x: Value) Value { - return self.floatMath("round", .default, .{x}); - } - pub fn trunc(self: *Builder, x: Value) Value { - return self.floatMath("trunc", .default, .{x}); - } - pub fn sin(self: *Builder, x: Value) Value { - return self.floatMath("sin", .default, .{x}); - } - pub fn cos(self: *Builder, x: Value) Value { - return self.floatMath("cos", .default, .{x}); - } - pub fn pow(self: *Builder, x: Value, y: anytype) Value { - return self.floatMath("pow", .default, .{ x, y }); - } - pub fn fmod(self: *Builder, x: Value, y: anytype) Value { - return self.floatMath("fmod", .default, .{ x, y }); - } - - /// `fma(a, b, c)` of floats. - pub fn fma(self: *Builder, a_: Value, b: anytype, c: anytype) Value { - const a = a_.rvalue(); - const t = a.type_(); - if (!t.scalar.isFloat() or t.scalar == .bfloat) std.debug.panic("dsl/metal: fma of a {s}", .{t.name()}); - const y = self.convert(self.lift(b), t); - const z = self.convert(self.lift(c), t); - return self.emit(air.fma(self.ctx, a.inner, y.inner, z.inner, self.fastmath(), self.loc()), .{ .value = t }); - } - - /// `dot(a, b)` of float vectors. - pub fn dot(self: *Builder, a_: Value, b_: Value) Value { - const a = a_.rvalue(); - const b = b_.rvalue(); - const t = a.type_(); - if (!t.eql(b.type_()) or !t.isVector()) std.debug.panic("dsl/metal: dot of {s} and {s}", .{ t.name(), b.type_().name() }); - return self.emit(air.dot(self.ctx, a.inner, b.inner, self.fastmath(), self.loc()), .{ .value = t.element() }); - } - - /// `abs(x)`. - pub fn abs(self: *Builder, x_: Value) Value { - const x = x_.rvalue(); - const t = x.type_(); - if (t.scalar.isFloat()) return self.floatMath("fabs", .default, .{x}); - if (t.scalar.isUnsigned()) return x; - return self.emit(air.abs_s(self.ctx, x.inner, self.loc()), .{ .value = t }); - } - - /// The operands of an MSL function taking several values of one type, - /// read, with literals converted to the other operand's type. - fn sameType(self: *Builder, a: anytype, b: anytype) [2]Value { - if (@TypeOf(a) != Value) { - const r = self.lift(b); - return .{ self.convert(self.lift(a), r.type_()), r }; - } - const l = a.rvalue(); - const r = if (@TypeOf(b) == Value) b.rvalue() else self.convert(self.lift(b), l.type_()); - if (!l.type_().eql(r.type_())) std.debug.panic("dsl/metal: {s} and {s} arguments", .{ l.type_().name(), r.type_().name() }); - return .{ l, r }; - } - - /// `max(a, b)`. - pub fn max(self: *Builder, a: anytype, b: anytype) Value { - const l, const r = self.sameType(a, b); - const t = l.type_(); - if (t.scalar.isFloat()) return self.floatMath("fmax", .default, .{ l, r }); - const op = if (t.scalar.isUnsigned()) air.max_u(self.ctx, l.inner, r.inner, self.loc()) else air.max_s(self.ctx, l.inner, r.inner, self.loc()); - return self.emit(op, .{ .value = t }); - } - - /// `min(a, b)`. - pub fn min(self: *Builder, a: anytype, b: anytype) Value { - const l, const r = self.sameType(a, b); - const t = l.type_(); - if (t.scalar.isFloat()) return self.floatMath("fmin", .default, .{ l, r }); - const op = if (t.scalar.isUnsigned()) air.min_u(self.ctx, l.inner, r.inner, self.loc()) else air.min_s(self.ctx, l.inner, r.inner, self.loc()); - return self.emit(op, .{ .value = t }); - } - - /// `clamp(x, lo, hi)`. - pub fn clamp(self: *Builder, x_: Value, lo: anytype, hi: anytype) Value { - const x = x_.rvalue(); - const t = x.type_(); - const l = self.convert(self.lift(lo), t); - const h = self.convert(self.lift(hi), t); - if (t.scalar.isFloat()) return self.floatMath("clamp", .default, .{ x, l, h }); - const op = if (t.scalar.isUnsigned()) air.clamp_u(self.ctx, x.inner, l.inner, h.inner, self.loc()) else air.clamp_s(self.ctx, x.inner, l.inner, h.inner, self.loc()); - return self.emit(op, .{ .value = t }); - } - - fn simdType(x: Value, comptime name: []const u8) Type { - const t = x.type_(); - switch (t.scalar) { - .half, .float, .char, .uchar, .short, .ushort, .int, .uint => {}, - else => std.debug.panic("dsl/metal: no {s} of a {s}", .{ name, t.name() }), - } - return t; - } - - fn simdReduce(self: *Builder, comptime name: []const u8, x_: Value) Value { - const x = x_.rvalue(); - const t = simdType(x, name); - const op = if (t.scalar.isFloat()) - @field(air, name)(self.ctx, x.inner, self.fastmath(), self.loc()) - else if (t.scalar.isUnsigned()) - @field(air, name ++ "_u")(self.ctx, x.inner, self.loc()) - else - @field(air, name ++ "_s")(self.ctx, x.inner, self.loc()); - return self.emit(op, .{ .value = t }); - } - - /// A function moving `x` across lanes; `x` is frozen, so that a poison - /// lane does not poison the others. - fn shuffle(self: *Builder, comptime name: []const u8, x_: Value, lane: anytype) Value { - const read = x_.rvalue(); - const t = simdType(read, name); - const x = self.emit(llvm.freeze(self.ctx, read.inner, self.loc()), .{ .value = t }); - const l = self.convert(self.lift(lane), .ushort); - const op = if (t.scalar.isFloat()) - @field(air, name)(self.ctx, x.inner, l.inner, self.fastmath(), self.loc()) - else if (t.scalar.isUnsigned()) - @field(air, name ++ "_u")(self.ctx, x.inner, l.inner, self.loc()) - else - @field(air, name ++ "_s")(self.ctx, x.inner, l.inner, self.loc()); - return self.emit(op, .{ .value = t }); - } - - pub fn simdSum(self: *Builder, x: Value) Value { - return self.simdReduce("simd_sum", x); - } - pub fn simdMax(self: *Builder, x: Value) Value { - return self.simdReduce("simd_max", x); - } - pub fn simdMin(self: *Builder, x: Value) Value { - return self.simdReduce("simd_min", x); - } - pub fn simdPrefixExclusiveSum(self: *Builder, x_: Value) Value { - const x = x_.rvalue(); - const t = x.type_(); - if (!t.scalar.isFloat() or t.scalar == .bfloat) std.debug.panic("dsl/metal: simd_prefix_exclusive_sum of a {s}", .{t.name()}); - return self.emit(air.simd_prefix_exclusive_sum(self.ctx, x.inner, self.fastmath(), self.loc()), .{ .value = t }); - } - /// `simd_shuffle(x, lane)`. - pub fn simdShuffle(self: *Builder, x: Value, lane: anytype) Value { - return self.shuffle("simd_shuffle", x, lane); - } - /// `simd_shuffle_xor(x, mask)`. - pub fn simdShuffleXor(self: *Builder, x: Value, mask: anytype) Value { - return self.shuffle("simd_shuffle_xor", x, mask); - } - /// `simd_shuffle_down(x, delta)`. - pub fn simdShuffleDown(self: *Builder, x: Value, delta: anytype) Value { - return self.shuffle("simd_shuffle_down", x, delta); - } - /// `simd_broadcast(x, lane)`. - pub fn simdBroadcast(self: *Builder, x: Value, lane: anytype) Value { - return self.shuffle("simd_broadcast", x, lane); - } - pub fn simdAny(self: *Builder, x: Value) Value { - return self.emit(air.simd_any(self.ctx, self.convert(x, .bool).inner, self.loc()), .{ .value = .bool }); - } - pub fn simdAll(self: *Builder, x: Value) Value { - return self.emit(air.simd_all(self.ctx, self.convert(x, .bool).inner, self.loc()), .{ .value = .bool }); - } - - /// `threadgroup_barrier(flags)`. - pub fn threadgroupBarrier(self: *Builder, flags: MemFlags) void { - _ = air.wg_barrier(self.ctx, @bitCast(flags), self.loc()).appendTo(self.currentBlock()); - } - - /// `simdgroup_barrier(flags)`. - pub fn simdgroupBarrier(self: *Builder, flags: MemFlags) void { - _ = air.simdgroup_barrier(self.ctx, @bitCast(flags), self.loc()).appendTo(self.currentBlock()); - } - - pub const MatrixOpts = struct { - /// `matrix_origin`. - origin: [2]i64 = .{ 0, 0 }, - /// `transpose_matrix`. - transpose: bool = false, - }; - - fn matrixType(self: *Builder, scalar: Scalar) *const mlir.Type { - return .vector(&.{64}, scalar.mlirType(self.ctx)); - } - - fn matrixScalar(m: Value) Scalar { - return switch (m.kind) { - .matrix => |s| s, - else => std.debug.panic("dsl/metal: expected a simdgroup_matrix, got a {t}", .{m.kind}), - }; - } - - /// The extents, strides and origin a matrix load or store passes the - /// builtin, from its `elements_per_row`, `matrix_origin` and - /// `transpose_matrix`. - fn accessLayout(self: *Builder, elements_per_row: anytype, opts: MatrixOpts) [3]*const mlir.Value { - const ulong2: Type = .vec(.ulong, 2); - const epr = self.convert(self.lift(elements_per_row), .ulong); - const eight = self.constant(.ulong, 8); - const one = self.constant(.ulong, 1); - const o0 = self.constant(.ulong, opts.origin[0]); - const o1 = self.constant(.ulong, opts.origin[1]); - const layout: [3]Value = if (opts.transpose) - .{ self.vec(ulong2, .{ eight, epr }), self.vec(ulong2, .{ epr, one }), self.vec(ulong2, .{ o1, o0 }) } - else - .{ self.vec(ulong2, .{ epr, eight }), self.vec(ulong2, .{ one, epr }), self.vec(ulong2, .{ o0, o1 }) }; - return .{ layout[0].inner, layout[1].inner, layout[2].inner }; - } - - /// `simdgroup_load(m, src, elements_per_row, matrix_origin, - /// transpose_matrix)`: the matrix `m` it loads. - pub fn simdgroupLoad(self: *Builder, src_: Value, elements_per_row: anytype, opts: MatrixOpts) Value { - const src = src_.rvalue(); - const t = src.pointer().pointee.scalar; - const layout = self.accessLayout(elements_per_row, opts); - return self.emit(air.simdgroup_matrix_8x8_load(self.ctx, src.inner, layout[0], layout[1], layout[2], self.matrixType(t), self.fastmath(), self.loc()), .{ .matrix = t }); - } - - /// `simdgroup_store(m, dst, elements_per_row, matrix_origin, - /// transpose_matrix)`. - pub fn simdgroupStore(self: *Builder, m: Value, dst_: Value, elements_per_row: anytype, opts: MatrixOpts) void { - const t = matrixScalar(m); - const dst = dst_.rvalue(); - const p = dst.pointer(); - if (p.pointee.scalar != t) std.debug.panic("dsl/metal: simdgroup_store to a {s} pointer", .{p.pointee.name()}); - const layout = self.accessLayout(elements_per_row, opts); - _ = air.simdgroup_matrix_8x8_store(self.ctx, m.inner, dst.inner, layout[0], layout[1], layout[2], self.loc()).appendTo(self.currentBlock()); - } - - /// `simdgroup_multiply_accumulate(d, a, b, c)`: the matrix `d = a * b + c`, - /// of `c`'s type. - pub fn simdgroupMultiplyAccumulate(self: *Builder, a: Value, b: Value, c: Value) Value { - _ = .{ matrixScalar(a), matrixScalar(b) }; - const t = matrixScalar(c); - return self.emit(air.simdgroup_matrix_8x8_multiply_accumulate(self.ctx, a.inner, b.inner, c.inner, self.matrixType(t), self.fastmath(), self.loc()), .{ .matrix = t }); - } - - /// The `llvm.loop` metadata of an MSL loop: C++ loops must make progress. - fn loopAnnotation(self: *Builder, opts: LoopOpts) *const llvm.LoopAnnotationAttr { - const unroll: ?*const llvm.LoopUnrollAttr = if (opts.unroll) |u| switch (u) { - .full => llvm.LoopUnrollAttr.get(self.ctx, .{ .full = true }), - .enable => llvm.LoopUnrollAttr.get(self.ctx, .{ .disable = false }), - } else null; - return llvm.LoopAnnotationAttr.get(self.ctx, .{ .unroll = unroll, .mustProgress = true }); - } - - /// The type `i < upper` compares in, for a loop variable `i` of type `t`: the - /// loop runs in `t`, which a short or char variable may be narrower than. - fn loopComparison(t: Type, upper: Type) Scalar { - const compared = types.commonScalar(t.scalar, upper.scalar); - const narrow = t.scalar.bits() < 32 and compared == .int; - if (t.scalar.isFloat() or t.isVector() or (compared.bits() != t.scalar.bits() and !narrow)) - std.debug.panic("dsl/metal: a {s} loop variable compared to a {s}", .{ t.name(), upper.name() }); - return compared; - } - - /// `for (T i = lower; i < upper; i += step)`, with `T` the type of - /// `lower`; `close` ends the body. - pub fn openFor(self: *Builder, lower: anytype, upper: anytype, step: anytype) ForScope { - return self.openForOpts(lower, upper, step, .{}); - } - - /// `openFor` under a loop pragma. - pub fn openForOpts(self: *Builder, lower: anytype, upper: anytype, step: anytype, opts: LoopOpts) ForScope { - const i = self.local(if (@TypeOf(lower) == Value) lower.type_() else literalType(lower), lower); - var loop = self.openLoop(opts); - _ = loopComparison(i.type_(), if (@TypeOf(upper) == Value) upper.type_() else literalType(upper)); - loop.body(i.lt(upper)); - const step_: ForScope.Step = switch (@typeInfo(@TypeOf(step))) { - .comptime_int, .int => if (step == 1) .one else .{ .int = step }, - else => .{ .value = step }, - }; - return .{ .loop = loop, .iv = i, .step = step_ }; - } - - /// A loop: `while (c)`, `for (init; c; inc)` or `for (;;)`. - pub fn openLoop(self: *Builder, opts: LoopOpts) LoopScope { - const cond_block = mlir.Block.init(&.{}, &.{}); - _ = self.branch(cond_block); - self.startBlock(cond_block); - return .{ .kernel = self, .cond_block = cond_block, .end_block = mlir.Block.init(&.{}, &.{}), .loop_annotation = self.loopAnnotation(opts) }; - } - - /// `if (cond) { ... }`. - pub fn openIf(self: *Builder, cond: Value) IfScope { - var c = self.condition(); - return c.openIf(cond); - } - - /// `if (cond) { ... } else { ... }` with no result types; with one, `T`, - /// `cond ? a : b` of type `T` evaluating one operand. - pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfElseScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { - if (comptime common.arity(@TypeOf(result_types), "openIfElse: result_types") == 0) { - var c = self.condition(); - return c.openIfElse(cond, .{}); - } - return self.openTernary(cond, typeOf(result_types[0])); - } - - /// A condition of `&&`, `||` and `!` terms. - pub fn condition(self: *Builder) Condition { - return .{ .kernel = self, .true_block = mlir.Block.init(&.{}, &.{}), .false_block = mlir.Block.init(&.{}, &.{}) }; - } - - /// `a && b && ...` as a bool. - pub fn openAnd(self: *Builder) Logical { - return self.logicalChain(.@"and"); - } - - /// `a || b || ...` as a bool. - pub fn openOr(self: *Builder) Logical { - return self.logicalChain(.@"or"); - } - - fn logicalChain(self: *Builder, op: @FieldType(Logical, "op")) Logical { - var loc_ = [1]*const mlir.Location{self.loc()}; - return .{ .kernel = self, .op = op, .end_block = mlir.Block.init(&.{Type.bool.mlirType(self.ctx)}, &loc_) }; - } - - /// `return;`. - pub fn return_(self: *Builder) void { - const block = self.insert orelse return; - _ = func.returns(self.ctx, &.{}, self.loc()).appendTo(block); - self.insert = null; - } - - /// `if (cond) return;`. - pub fn returnIf(self: *Builder, cond: Value) void { - var scope = self.openIf(cond); - { - self.return_(); - scope.close(); - } - } - - /// Terminates the kernel and returns its module, verified. - pub fn finish(self: *Builder) FinishError![:0]const u8 { - self.return_(); - if (self.fast_math) air.enableFastMath(self.ctx, self.module.operation()); - try common.verify(self.module, "dsl/metal"); - var writer: std.Io.Writer.Allocating = .init(self.allocator); - defer writer.deinit(); - try writer.writer.print("{f}", .{self.module.operation()}); - return try self.allocator.dupeSentinel(u8, writer.written(), 0); - } -}; - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - registerDialects(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -fn expectContains(ir: []const u8, needles: []const []const u8) !void { - for (needles) |needle| { - if (std.mem.find(u8, ir, needle) == null) { - std.debug.print("missing `{s}` in:\n{s}\n", .{ needle, ir }); - return error.TestExpectedEqual; - } - } -} - -test "arguments and C++ arithmetic" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .x = .{ .buffer = .{ .type = .bfloat, .const_ = true, .restrict = true } }, - .y = .{ .buffer = .{ .type = .float4 } }, - .n = .{ .constant = .int }, - .shmem = .{ .threadgroup = .half }, - .gid = .{ .threadgroup_position_in_grid = .uint3 }, - .lane = .{ .thread_index_in_simdgroup = .ushort }, - }); - // `gid.z / n` divides unsigned, `lane * 2` promotes to int. - const q = a.gid.z().div(a.n); - try std.testing.expect(q.type_().eql(.uint)); - const twice = a.lane.mul(2); - try std.testing.expect(twice.type_().eql(.int)); - const v = a.x.load(q).to(.float).mul(0.5); - try std.testing.expect(v.type_().eql(.float)); - a.y.store(twice, b.vec(.float4, .{v})); - a.shmem.store(a.lane, v); - try std.testing.expectEqual(4, b.arguments.items.len); - try std.testing.expectEqual(1, b.arguments.items[1].index); - try std.testing.expectEqual(4, b.arguments.items[2].size); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ "air.fast_math_enable", "air.kernel", "noalias = true" }); - try expectContains(ir, &.{ "arith.divui", "overflow", "arith.extf", "arith.mulf", "fastmath", "arith.truncf", "llvm.getelementptr", "#air.builtin", "air.arg_name = \"lane\"" }); -} - -test "conversions follow Apple's compiler" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .h = .{ .buffer = .{ .type = .half4, .const_ = true } }, - .o = .{ .buffer = .{ .type = .bfloat4 } }, - .i = .{ .buffer = .{ .type = .int } }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - const f4 = a.h.load(a.tid).to(.float4); - a.o.store(a.tid, f4.to(.bfloat4)); - const n = a.i.load(a.tid); - a.i.store(0, n.to(.float).add(n.to(.half).to(.float)).to(.uint).to(.short)); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ "air.convert", ": vector<4xf16> to vector<4xf32>", ": vector<4xf32> to vector<4xbf16>", "air.convert.s", ": i32 to f32", ": i32 to f16", "arith.extf", "air.convert.u", "arith.trunci" }); -} - -test "math, SIMD-group and barrier functions" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .x = .{ .buffer = .{ .type = .float } }, - .v = .{ .buffer = .{ .type = .float4 } }, - .lane = .{ .thread_index_in_simdgroup = .ushort }, - }); - const x = a.x.load(a.lane); - const v = a.v.load(a.lane); - const m = b.simdMax(b.max(x, b.fast().exp2(x))); - const s = b.simdSum(b.exp(x.sub(m))); - const t = b.precise().tanh(b.fma(s, 2.0, b.dot(v, v))); - const r = b.simdShuffleXor(t, 8); - b.simdgroupBarrier(.mem_threadgroup); - b.threadgroupBarrier(.{ .device = true, .threadgroup = true }); - a.x.store(a.lane, b.ternary(r.gt(0), b.min(a.lane.to(.int), 3).to(.float), r)); - a.x.store(1, b.ternary(r.gt(1), 1.0, 2.0)); - a.v.store(a.lane, b.simdSum(v)); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ "air.fast_exp2", "air.fast_fmax", "air.simd_max", "air.fast_exp", "air.simd_sum", "air.dot", "air.fma", "air.tanh", "air.simd_shuffle_xor", "air.simdgroup.barrier ", "air.wg.barrier ", "air.min.s", "arith.select", ": vector<4xf32>" }); - try std.testing.expect(std.mem.find(u8, ir, "cf.cond_br") == null); -} - -test "loops, ifs and early returns" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .x = .{ .buffer = .{ .type = .float } }, - .n = .{ .constant = .int }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - var ret = b.openIf(a.tid.ge(1024)); - { - a.x.store(0, 0.0); - b.return_(); - ret.close(); - } - b.returnIf(a.n.le(0)); - - // float acc = 0; for (int i = 0; i < n; ++i) { if (x[i] < 0) break; acc += x[i]; } - const acc = b.local(.float, 0.0); - var loop = b.openFor(0, a.n, 1); - { - var neg = b.openIf(a.x.load(loop.iv).lt(0.0)); - { - loop.breakLoop(); - neg.close(); - } - acc.assign(acc.add(a.x.load(loop.iv))); - loop.close(); - } - - // int lo = 0, hi = n; while (lo < hi) { int mid = (lo + hi + 1) / 2; ... } - const lo = b.local(.int, 0); - const hi = b.local(.int, a.n); - var search = b.openLoop(.{}); - { - search.body(lo.lt(hi)); - } - { - const mid = b.local(.int, lo.add(hi).add(1).div(2)); - var up = b.openIfElse(a.x.load(mid).le(acc), .{}); - { - lo.assign(mid); - up.elseBranch(); - } - { - hi.assign(mid.sub(1)); - up.close(); - } - search.close(); - } - - // bool m = lo > 0 || (n > 3 && acc < 1); - var m = b.openOr(); - m.term(lo.gt(0)); - var c = b.condition(); - c.and_(a.n.gt(3)); - var only = c.openIfNot(m.close(acc.lt(1.0))); - { - a.x.store(1, b.ternary(lo.gt(0), acc, 2.0)); - only.close(); - } - - const scratch = b.array(.float, 8); - scratch.store(a.tid.rem(8), acc); - a.x.store(2, scratch.load(3)); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ "cf.cond_br", "llvm.br", "loop_annotation", "mustProgress = true", "llvm.alloca", "!llvm.array<8 x f32>", "arith.remui" }); - try std.testing.expect(std.mem.find(u8, ir, "scf.") == null); - // The two early returns and the last one, each where it is. - try std.testing.expectEqual(3, std.mem.count(u8, ir, "\n return\n")); - // Every local is allocated in the entry block, before any branch. - try std.testing.expect(std.mem.findLast(u8, ir, "llvm.alloca").? < std.mem.find(u8, ir, "cf.cond_br").?); -} - -test "a return inside a loop and in both branches of an if" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .x = .{ .buffer = .{ .type = .float } }, - .n = .{ .constant = .int }, - }); - // for (;;) { if (n <= 0) break; if (x[0] < 0) return; x[0] -= 1; } - var forever = b.openLoop(.{}); - { - var stop = b.openIf(a.n.le(0)); - { - forever.breakLoop(); - stop.close(); - } - b.returnIf(a.x.load(0).lt(0.0)); - a.x.store(0, a.x.load(0).sub(1.0)); - forever.close(); - } - // if (n > 1) { x[1] = 1; return; } else { x[1] = 2; return; } - var both = b.openIfElse(a.n.gt(1), .{}); - { - a.x.store(1, 1.0); - b.return_(); - both.elseBranch(); - } - { - a.x.store(1, 2.0); - b.return_(); - both.close(); - } - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - // The loop's return and one per branch; `finish` adds none after them. - try std.testing.expectEqual(3, std.mem.count(u8, ir, "\n return\n")); - try expectContains(ir, &.{"loop_annotation"}); -} - -test "a break and the end of the body meet in the latch" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .x = .{ .buffer = .{ .type = .float } }, - .n = .{ .constant = .int }, - }); - // for (int i = 0; i < n; ++i) { if (x[i] < 0) break; x[i] = 1; } - var loop = b.openFor(0, a.n, 1); - { - var neg = b.openIf(a.x.load(loop.iv).lt(0.0)); - { - loop.breakLoop(); - neg.close(); - } - a.x.store(loop.iv, 1.0); - loop.close(); - } - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - // `i`, and the flag the break and the end of the body set for the latch. - try std.testing.expectEqual(2, std.mem.count(u8, ir, "llvm.alloca")); - // `i < n`, `x[i] < 0`, and the latch's choice between leaving and `++i`. - try std.testing.expectEqual(3, std.mem.count(u8, ir, "cf.cond_br")); -} - -test "simdgroup matrices" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .o = .{ .buffer = .{ .type = .float2 } }, - .s = .{ .threadgroup = .bfloat }, - .f = .{ .constant = .int }, - .lane = .{ .thread_index_in_simdgroup = .uint }, - }); - const k = b.simdgroupLoad(a.s.add(8), 40, .{ .transpose = true }); - const q = b.simdgroupLoad(a.s, a.f, .{}); - const acc = b.simdgroupMultiplyAccumulate(q, k, b.matrix(.float, a.o.load(a.lane))); - a.o.store(a.lane, acc.threadElements()); - b.simdgroupStore(acc, a.s.to(.float), 8, .{}); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ "air.simdgroup_matrix_8x8_load", "-> vector<64xbf16>", "air.simdgroup_matrix_8x8_multiply_accumulate", "vector<64xbf16>, vector<64xbf16>, vector<64xf32> -> vector<64xf32>", "air.simdgroup_matrix_8x8_store" }); - // `b.matrix` widens two elements to a matrix, `threadElements` takes them back. - try std.testing.expectEqual(2, std.mem.count(u8, ir, "llvm.shufflevector")); - try expectContains(ir, &.{"[0, 1] : vector<64xf32>"}); - try std.testing.expect(std.mem.find(u8, ir, "llvm.alloca") == null); -} - -test "constant struct arguments" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{ .fast_math = false }); - defer b.deinit(); - const Args: Struct = .{ .name = "args_t", .fields = &.{ - .{ .name = "num_heads", .type = .int }, - .{ .name = "scale", .type = .float }, - } }; - const a = try b.declareArgs(.{ - .args = .{ .constant_struct = Args }, - .o = .{ .buffer = .{ .type = .float } }, - }); - a.o.store(0, a.args.field("scale").mul(a.args.field("num_heads"))); - try std.testing.expectEqual(8, b.arguments.items[0].size); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ "", "typeName = \"args_t\"", "!llvm.struct<\"struct.args_t\", (i32, f32)>" }); - try std.testing.expect(std.mem.find(u8, ir, "fastmath") == null); -} - -fn expectCount(ir: []const u8, needle: []const u8, expected: usize) !void { - const n = std.mem.count(u8, ir, needle); - if (n != expected) { - std.debug.print("`{s}` {d} times, not {d}, in:\n{s}\n", .{ needle, n, expected, ir }); - return error.TestExpectedEqual; - } -} - -test "subscripts index with the index's type, swizzles with an int" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .v = .{ .buffer = .{ .type = .float4 } }, - .o = .{ .buffer = .{ .type = .float } }, - .n = .{ .constant = .long }, - .gid = .{ .threadgroup_position_in_grid = .uint3 }, - .lane = .{ .thread_index_in_simdgroup = .ushort }, - }); - // float4 x = v[0]; - // o[gid.z] = x.x + x.y + x.z + x.w + x[2] + x[lane] + x[(char) lane] + x[n]; - const x = b.local(.float4, a.v.load(0)); - const sum = x.x().add(x.y()).add(x.z()).add(x.w()).add(x.at(2)).add(x.at(a.lane)).add(x.at(a.lane.to(.char))).add(x.at(a.n)); - a.o.store(a.gid.z(), sum); - // x[lane] = 1.0f; x[2] = 2.0f; x.y = 3.0f; v[0] = x; - x.assign(x.insert(a.lane, 1.0)); - x.assign(x.insert(2, 2.0)); - x.assign(x.insert(1, 3.0)); - a.v.store(0, x); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - // `i32` for the swizzles and literals, `i16 %lane`, `i8`, `i64 %n`. - try expectCount(ir, ": i32] : vector<4xf32>", 7); - try expectCount(ir, ": i16] : vector<4xf32>", 2); - try expectCount(ir, ": i8] : vector<4xf32>", 1); - try expectCount(ir, ": i64] : vector<4xf32>", 1); - try expectCount(ir, ": i32] : vector<3xi32>", 1); -} - -test "a pointer local" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .o = .{ .buffer = .{ .type = .float } }, - .lane = .{ .thread_index_in_simdgroup = .ushort }, - }); - // device float *p = o + 4; p += lane; *p = 1.0f; - const p = b.localPointer(a.o.add(4)); - p.assign(p.add(a.lane)); - p.store(0, 1.0); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectCount(ir, "llvm.alloca", 1); - try expectCount(ir, "llvm.getelementptr", 2); -} - -test "integer max, min, abs and clamp read locals and constant arguments" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .o = .{ .buffer = .{ .type = .int } }, - .w = .{ .buffer = .{ .type = .uint } }, - .f = .{ .buffer = .{ .type = .float } }, - .n = .{ .constant = .int }, - .u = .{ .constant = .uint }, - }); - // int m = n; o[0] = max(m, n) + min(n, m) + min(m, 7) + abs(m) + clamp(m, 0, 7); - const m = b.local(.int, a.n); - a.o.store(0, b.max(m, a.n).add(b.min(a.n, m)).add(b.min(m, 7)).add(b.abs(m)).add(b.clamp(m, 0, 7))); - // w[0] = max(u, 3u) + min(3u, u) + abs(u) + clamp(u, 1u, 2u); - a.w.store(0, b.max(a.u, 3).add(b.min(3, a.u)).add(b.abs(a.u)).add(b.clamp(a.u, 1, 2))); - // const float x = f[0]; f[1] = clamp(x, 0.0f, 1.0f) + fabs(x) + max(x, x); - const x = a.f.load(0); - a.f.store(1, b.clamp(x, 0.0, 1.0).add(b.abs(x)).add(b.max(x, x))); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ "air.max.s", "air.min.s", "air.abs.s", "air.clamp.s", "air.max.u", "air.min.u", "air.clamp.u", "air.fast_clamp", "air.fast_fabs", "air.fast_fmax" }); - // abs of a uint is the value itself. - try std.testing.expect(std.mem.find(u8, ir, "air.abs.u") == null); -} - -test "char and uchar take the same functions as wider integers" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .c = .{ .buffer = .{ .type = .char } }, - .u = .{ .buffer = .{ .type = .uchar } }, - .v = .{ .buffer = .{ .type = .char2 } }, - .f = .{ .buffer = .{ .type = .float } }, - .w = .{ .buffer = .{ .type = .float2 } }, - .lane = .{ .thread_index_in_simdgroup = .ushort }, - }); - const c = a.c.load(a.lane); - const u = a.u.load(a.lane); - const v = a.v.load(a.lane); - // f[0] = float(c) + float(u); c[0] = char(f[1]); u[0] = uchar(f[2]); - a.f.store(0, c.to(.float).add(u.to(.float))); - a.c.store(0, a.f.load(1).to(.char)); - a.u.store(0, a.f.load(2).to(.uchar)); - // w[0] = float2(v); v[0] = char2(w[1]); - a.w.store(0, v.to(.float2)); - a.v.store(0, a.w.load(1).to(.char2)); - // c[1] = max(c, c); u[1] = min(u, u); c[2] = abs(c); c[3] = clamp(c, 1, 5); v[1] = max(v, v); - a.c.store(1, b.max(c, c)); - a.u.store(1, b.min(u, u)); - a.c.store(2, b.abs(c)); - a.c.store(3, b.clamp(c, 1, 5)); - a.v.store(1, b.max(v, v)); - // c[4] = simd_shuffle(c, lane); c[5] = simd_sum(c); u[2] = simd_max(u); - a.c.store(4, b.simdShuffle(c, a.lane)); - a.c.store(5, b.simdSum(c)); - a.u.store(2, b.simdMax(u)); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ - "air.convert.s", ": i8 to f32", "air.convert.u", ": f32 to i8", - ": vector<2xi8> to vector<2xf32>", ": vector<2xf32> to vector<2xi8>", "air.max.s", "air.min.u", - "air.abs.s", "air.clamp.s", "air.simd_shuffle.s", "air.simd_sum.s", - "air.simd_max.u", - }); -} - -test "bitwise operators and shifts" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .o = .{ .buffer = .{ .type = .int } }, - .w = .{ .buffer = .{ .type = .uint } }, - .l = .{ .buffer = .{ .type = .long } }, - .n = .{ .constant = .int }, - .lane = .{ .thread_index_in_simdgroup = .ushort }, - }); - // int s = n; o[0] = (n << s) ^ (~(n | 3) & -s); w[0] = w[1] >> s; o[1] = lane >> 1; - // l[0] = l[1] << n; - const s = b.local(.int, a.n); - a.o.store(0, a.n.shl(s).bitXor(a.n.bitOr(3).bitNot().bitAnd(s.neg()))); - a.w.store(0, a.w.load(1).shr(s)); - a.o.store(1, a.lane.shr(1)); - a.l.store(0, a.l.load(1).shl(a.n)); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ "arith.shli", "arith.shrui", "arith.shrsi", "arith.xori", "arith.ori", "arith.subi" }); - // clang masks an amount to the width (`and i32 %s, 31`), after casting it - // as unsigned (`zext i32 %n to i64`, `and i64 ..., 63`); a number's mask - // folds away. - try expectCount(ir, "arith.andi", 4); - try expectContains(ir, &.{ "arith.constant 31 : i32", "arith.constant 63 : i64" }); - var lines = std.mem.splitScalar(u8, ir, '\n'); - const unsigned_cast = while (lines.next()) |line| { - if (std.mem.find(u8, line, "arith.extui") != null and std.mem.find(u8, line, "i32 to i64") != null) break true; - } else false; - try std.testing.expect(unsigned_cast); -} - -test "a shift reads its amount once" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .o = .{ .buffer = .{ .type = .int } }, - .n = .{ .constant = .int }, - }); - // int s = n; *o = s << s; - const s = b.local(.int, a.n); - a.o.store(0, s.shl(s)); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - // n, and s twice. - try expectCount(ir, "llvm.load", 3); -} - -test "math functions" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .x = .{ .buffer = .{ .type = .float } }, - .h = .{ .buffer = .{ .type = .half } }, - }); - const v = a.x.load(0); - var sum = b.local(.float, 0.0); - for ([_]Value{ - b.exp2(v), b.log(v), b.log2(v), b.tanh(v), b.sqrt(v), - b.rsqrt(v), b.ceil(v), b.floor(v), b.round(v), b.trunc(v), - b.sin(v), b.cos(v), b.pow(v, 2.0), b.fmod(v, 3.0), - }) |term| sum.assign(sum.add(term)); - const fast_ = b.fast(); - for ([_]Value{ - fast_.exp(v), fast_.log(v), fast_.log2(v), fast_.sqrt(v), fast_.rsqrt(v), - fast_.sin(v), fast_.cos(v), fast_.tanh(v), fast_.max(v, 1.0), fast_.min(v, 0.5), - }) |term| sum.assign(sum.add(term)); - const precise_ = b.precise(); - for ([_]Value{ - precise_.exp(v), precise_.exp2(v), precise_.log(v), precise_.log2(v), precise_.sqrt(v), - precise_.rsqrt(v), precise_.sin(v), precise_.cos(v), precise_.max(v, 1.0), precise_.min(v, 0.5), - }) |term| sum.assign(sum.add(term)); - a.x.store(1, sum); - // half has no fast:: functions: exp of a half is precise. - a.h.store(0, b.exp(a.h.load(1))); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ - "air.fast_exp2", "air.fast_log", "air.fast_log2", "air.fast_tanh", "air.fast_sqrt", "air.fast_rsqrt", - "air.fast_ceil", "air.fast_floor", "air.fast_round", "air.fast_trunc", "air.fast_sin", "air.fast_cos", - "air.fast_pow", "air.fast_fmod", "air.fast_exp", "air.fast_fmax", "air.fast_fmin", - }); - try expectContains(ir, &.{ "air.exp ", "air.exp2 ", "air.log ", "air.log2 ", "air.sqrt ", "air.rsqrt ", "air.sin ", "air.cos ", "air.fmax ", "air.fmin " }); - try expectContains(ir, &.{"f16"}); -} - -test "SIMD-group functions" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .x = .{ .buffer = .{ .type = .float } }, - .i = .{ .buffer = .{ .type = .int } }, - .u = .{ .buffer = .{ .type = .uint } }, - .lane = .{ .thread_index_in_simdgroup = .ushort }, - }); - const f = a.x.load(a.lane); - const n = a.i.load(a.lane); - const m = a.u.load(a.lane); - a.x.store(0, b.simdMin(f).add(b.simdPrefixExclusiveSum(f)).add(b.simdShuffle(f, a.lane)).add(b.simdShuffleDown(f, 1)).add(b.simdBroadcast(f, 0))); - a.i.store(0, b.simdMin(n).add(b.simdShuffle(n, 3)).add(b.simdShuffleDown(n, a.lane)).add(b.simdBroadcast(n, 0))); - a.u.store(0, b.simdMin(m).add(b.simdShuffleXor(m, 1))); - a.i.store(1, b.ternary(b.simdAny(f.gt(0.0)), 1, 0).add(b.ternary(b.simdAll(n.ne(0)), 2, 0))); - b.threadgroupBarrier(.mem_device); - b.simdgroupBarrier(.mem_texture); - b.threadgroupBarrier(.mem_none); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ - "air.simd_min ", "air.simd_min.s", "air.simd_min.u", "air.simd_prefix_exclusive_sum", - "air.simd_shuffle ", "air.simd_shuffle.s", "air.simd_shuffle_down ", "air.simd_shuffle_down.s", - "air.simd_broadcast ", "air.simd_broadcast.s", "air.simd_shuffle_xor.u", "air.simd_any", - "air.simd_all", "llvm.freeze", - }); - try expectContains(ir, &.{ "air.wg.barrier ", "air.simdgroup.barrier ", "air.wg.barrier " }); -} - -test "pragmas, conditions and casts" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = Builder.open(std.testing.allocator, ctx, "k", .{}); - defer b.deinit(); - const a = try b.declareArgs(.{ - .o = .{ .buffer = .{ .type = .float } }, - .n = .{ .constant = .int }, - .lane = .{ .thread_index_in_simdgroup = .ushort }, - }); - - const r = a.o.load(a.lane).mul(2.0); - // float acc[4] = {}; - const acc = b.zeroedArray(.float, 4); - // #pragma clang loop unroll(full) - var full = b.openForOpts(0, 4, 1, .{ .unroll = .full }); - { - acc.store(full.iv, r); - full.close(); - } - // #pragma unroll - var enabled = b.openForOpts(0, a.n, 2, .{ .unroll = .enable }); - { - acc.store(0, acc.load(0).add(enabled.iv.to(.float))); - enabled.close(); - } - // for (int i = 0; i < 4; i += 1) {} - const i = b.local(.int, 0); - var counted = b.openLoop(.{}); - { - counted.body(i.lt(4)); - } - { - counted.next(); - i.assign(i.add(1)); - counted.close(); - } - // const uint bits = as_type(acc[0]); - const bits = acc.load(0).asType(.uint); - // float t = r; *&t = 2.0f; - const t = b.local(.float, r); - t.address().store(0, 2.0); - // bool ok = n > 0 && bits != 0; - var both = b.openAnd(); - both.term(a.n.gt(0)); - const ok = both.close(bits.ne(0)); - // if (!(n > 1) || ok) { ... } else { ... } - var c1 = b.condition(); - c1.orNot(a.n.gt(1)); - var s1 = c1.openIfElse(ok, .{}); - { - a.o.store(1, 1.0); - s1.elseBranch(); - } - { - a.o.store(1, 2.0); - s1.close(); - } - // if (!ok && !(n == 3)) { ... } else { ... } - var c2 = b.condition(); - c2.andNot(ok); - var s2 = c2.openIfElseNot(a.n.eq(3), .{}); - { - a.o.store(2, 1.0); - s2.elseBranch(); - } - { - s2.close(); - } - // o[lane] = n > 2 ? fast::exp(t) : t; - var choice = b.openIfElse(a.n.gt(2), .{.float}); - { - choice.yieldThen(.{b.fast().exp(t)}); - } - { - choice.yieldElse(.{t}); - } - a.o.store(a.lane, choice.results[0]); - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, &.{ "llvm.memset.p0.i64", "full = true", "disable = false", "llvm.bitcast" }); -} diff --git a/dsl/metal/types.zig b/dsl/metal/types.zig deleted file mode 100644 index 9c3072bb..00000000 --- a/dsl/metal/types.zig +++ /dev/null @@ -1,291 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); - -/// An MSL scalar type. -pub const Scalar = enum { - bool, - char, - uchar, - short, - ushort, - int, - uint, - long, - ulong, - half, - bfloat, - float, - - pub fn isFloat(self: Scalar) bool { - return switch (self) { - .half, .bfloat, .float => true, - else => false, - }; - } - - pub fn isUnsigned(self: Scalar) bool { - return switch (self) { - .bool, .uchar, .ushort, .uint, .ulong => true, - else => false, - }; - } - - pub fn bits(self: Scalar) u16 { - return switch (self) { - .bool => 1, - .char, .uchar => 8, - .short, .ushort, .half, .bfloat => 16, - .int, .uint, .float => 32, - .long, .ulong => 64, - }; - } - - /// The size in memory: a bool takes a byte. - pub fn size(self: Scalar) u64 { - return if (self == .bool) 1 else self.bits() / 8; - } - - /// The C++ integer conversion rank. - fn rank(self: Scalar) u8 { - return switch (self) { - .bool => 0, - .char, .uchar => 1, - .short, .ushort => 2, - .int, .uint => 3, - .long, .ulong => 4, - else => unreachable, - }; - } - - fn unsignedOf(self: Scalar) Scalar { - return switch (self) { - .char => .uchar, - .short => .ushort, - .int => .uint, - .long => .ulong, - else => self, - }; - } - - /// C++ integral promotion: types narrower than int become int. - pub fn promoted(self: Scalar) Scalar { - return switch (self) { - .bool, .char, .uchar, .short, .ushort => .int, - else => self, - }; - } - - pub fn mlirType(self: Scalar, ctx: *mlir.Context) *const mlir.Type { - return switch (self) { - .bool => .int(ctx, .i1), - .char, .uchar => .int(ctx, .i8), - .short, .ushort => .int(ctx, .i16), - .int, .uint => .int(ctx, .i32), - .long, .ulong => .int(ctx, .i64), - .half => .float(ctx, .f16), - .bfloat => .float(ctx, .bf16), - .float => .float(ctx, .f32), - }; - } -}; - -/// The C++ usual arithmetic conversions of two scalar operand types. -pub fn commonScalar(a: Scalar, b: Scalar) Scalar { - if (a.isFloat() or b.isFloat()) { - if (!b.isFloat()) return a; - if (!a.isFloat()) return b; - if (a == b) return a; - if (a == .float or b == .float) return .float; - std.debug.panic("dsl/metal: no common type of {t} and {t}", .{ a, b }); - } - const x = a.promoted(); - const y = b.promoted(); - if (x == y) return x; - if (x.isUnsigned() == y.isUnsigned()) return if (x.rank() >= y.rank()) x else y; - const u = if (x.isUnsigned()) x else y; - const s = if (x.isUnsigned()) y else x; - if (u.rank() >= s.rank()) return u; - if (s.bits() > u.bits()) return s; - return s.unsignedOf(); -} - -/// An MSL scalar or vector type: `float`, `uint3`, `bfloat4`. -pub const Type = struct { - scalar: Scalar, - /// 1 for a scalar, else the vector length (2, 3 or 4). - len: u8 = 1, - - pub const @"bool": Type = .{ .scalar = .bool }; - pub const char: Type = .{ .scalar = .char }; - pub const uchar: Type = .{ .scalar = .uchar }; - pub const short: Type = .{ .scalar = .short }; - pub const ushort: Type = .{ .scalar = .ushort }; - pub const int: Type = .{ .scalar = .int }; - pub const uint: Type = .{ .scalar = .uint }; - pub const long: Type = .{ .scalar = .long }; - pub const ulong: Type = .{ .scalar = .ulong }; - pub const half: Type = .{ .scalar = .half }; - pub const bfloat: Type = .{ .scalar = .bfloat }; - pub const float: Type = .{ .scalar = .float }; - pub const bool2: Type = .vec(.bool, 2); - pub const bool3: Type = .vec(.bool, 3); - pub const bool4: Type = .vec(.bool, 4); - pub const char2: Type = .vec(.char, 2); - pub const char3: Type = .vec(.char, 3); - pub const char4: Type = .vec(.char, 4); - pub const uchar2: Type = .vec(.uchar, 2); - pub const uchar3: Type = .vec(.uchar, 3); - pub const uchar4: Type = .vec(.uchar, 4); - pub const short2: Type = .vec(.short, 2); - pub const short3: Type = .vec(.short, 3); - pub const short4: Type = .vec(.short, 4); - pub const ushort2: Type = .vec(.ushort, 2); - pub const ushort3: Type = .vec(.ushort, 3); - pub const ushort4: Type = .vec(.ushort, 4); - pub const int2: Type = .vec(.int, 2); - pub const int3: Type = .vec(.int, 3); - pub const int4: Type = .vec(.int, 4); - pub const uint2: Type = .vec(.uint, 2); - pub const uint3: Type = .vec(.uint, 3); - pub const uint4: Type = .vec(.uint, 4); - pub const long2: Type = .vec(.long, 2); - pub const long3: Type = .vec(.long, 3); - pub const long4: Type = .vec(.long, 4); - pub const ulong2: Type = .vec(.ulong, 2); - pub const ulong3: Type = .vec(.ulong, 3); - pub const ulong4: Type = .vec(.ulong, 4); - pub const half2: Type = .vec(.half, 2); - pub const half3: Type = .vec(.half, 3); - pub const half4: Type = .vec(.half, 4); - pub const bfloat2: Type = .vec(.bfloat, 2); - pub const bfloat3: Type = .vec(.bfloat, 3); - pub const bfloat4: Type = .vec(.bfloat, 4); - pub const float2: Type = .vec(.float, 2); - pub const float3: Type = .vec(.float, 3); - pub const float4: Type = .vec(.float, 4); - - /// `vec`. - pub fn vec(scalar: Scalar, len: u8) Type { - std.debug.assert(len >= 2 and len <= 4); - return .{ .scalar = scalar, .len = len }; - } - - pub fn isVector(self: Type) bool { - return self.len > 1; - } - - pub fn element(self: Type) Type { - return .{ .scalar = self.scalar }; - } - - /// The same shape with another element type. - pub fn withScalar(self: Type, scalar: Scalar) Type { - return .{ .scalar = scalar, .len = self.len }; - } - - /// The type in memory, where a bool is a byte, as clang stores it. - pub fn memory(self: Type) Type { - return if (self.scalar == .bool) self.withScalar(.uchar) else self; - } - - /// `sizeof`: a 3-element vector takes the space of 4. - pub fn size(self: Type) u64 { - const len: u64 = if (self.len == 3) 4 else self.len; - return len * self.scalar.size(); - } - - /// `alignof`, which is the size for MSL scalars and vectors. - pub fn alignment(self: Type) u64 { - return self.size(); - } - - pub fn eql(self: Type, other: Type) bool { - return self.scalar == other.scalar and self.len == other.len; - } - - /// The MSL spelling, `uint3`. - pub fn name(self: Type) []const u8 { - const names = comptime blk: { - const scalars = @typeInfo(Scalar).@"enum".field_names; - var table: [scalars.len][4][]const u8 = undefined; - for (scalars, 0..) |scalar, i| { - table[i] = .{ scalar, scalar ++ "2", scalar ++ "3", scalar ++ "4" }; - } - break :blk table; - }; - return names[@backingInt(self.scalar)][self.len - 1]; - } - - pub fn mlirType(self: Type, ctx: *mlir.Context) *const mlir.Type { - const scalar = self.scalar.mlirType(ctx); - return if (self.isVector()) .vector(&.{self.len}, scalar) else scalar; - } -}; - -/// A field of a constant buffer struct: `float s[3]` is -/// `.{ .name = "s", .type = .float, .count = 3 }`. -pub const StructField = struct { - name: []const u8, - type: Type, - /// The array length, zero for a non-array field. - count: u32 = 0, -}; - -/// A struct passed by constant reference, laid out as MSL lays it out. -pub const Struct = struct { - name: []const u8, - fields: []const StructField, - - pub fn offset(self: Struct, index: usize) u64 { - var end: u64 = 0; - for (self.fields[0..index]) |field| { - end = std.mem.alignForward(u64, end, field.type.alignment()); - end += field.type.size() * @max(field.count, 1); - } - return std.mem.alignForward(u64, end, self.fields[index].type.alignment()); - } - - pub fn alignment(self: Struct) u64 { - var a: u64 = 1; - for (self.fields) |field| a = @max(a, field.type.alignment()); - return a; - } - - pub fn size(self: Struct) u64 { - const last = self.fields.len - 1; - const end = self.offset(last) + self.fields[last].type.size() * @max(self.fields[last].count, 1); - return std.mem.alignForward(u64, end, self.alignment()); - } - - pub fn fieldIndex(self: Struct, field_name: []const u8) usize { - for (self.fields, 0..) |field, i| if (std.mem.eql(u8, field.name, field_name)) return i; - std.debug.panic("dsl/metal: struct {s} has no field {s}", .{ self.name, field_name }); - } -}; - -test commonScalar { - try std.testing.expectEqual(Scalar.uint, commonScalar(.uint, .int)); - try std.testing.expectEqual(Scalar.int, commonScalar(.ushort, .short)); - try std.testing.expectEqual(Scalar.long, commonScalar(.long, .uint)); - try std.testing.expectEqual(Scalar.ulong, commonScalar(.ulong, .int)); - try std.testing.expectEqual(Scalar.float, commonScalar(.int, .float)); - try std.testing.expectEqual(Scalar.half, commonScalar(.half, .uint)); - try std.testing.expectEqual(Scalar.float, commonScalar(.half, .float)); -} - -test Struct { - const args: Struct = .{ .name = "A", .fields = &.{ - .{ .name = "n", .type = .int }, - .{ .name = "s", .type = .float, .count = 3 }, - .{ .name = "v", .type = .uint2 }, - .{ .name = "t", .type = .short }, - } }; - try std.testing.expectEqual(0, args.offset(0)); - try std.testing.expectEqual(4, args.offset(1)); - try std.testing.expectEqual(16, args.offset(2)); - try std.testing.expectEqual(24, args.offset(3)); - try std.testing.expectEqual(32, args.size()); - try std.testing.expectEqualStrings("bfloat4", Type.bfloat4.name()); - try std.testing.expectEqual(16, Type.float3.size()); -} diff --git a/dsl/mosaic_tpu/BUILD.bazel b/dsl/mosaic_tpu/BUILD.bazel deleted file mode 100644 index 15510258..00000000 --- a/dsl/mosaic_tpu/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "builder", - srcs = [ - "dtype.zig", - "pipeline.zig", - ], - import_name = "dsl/mosaic_tpu/builder", - main = "builder.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//dsl/common:builder", - "//mlir", - "//mlir/dialects/affine", - "//mlir/dialects/arith", - "//mlir/dialects/cf", - "//mlir/dialects/func", - "//mlir/dialects/math", - "//mlir/dialects/memref", - "//mlir/dialects/mosaic_tpu", - "//mlir/dialects/scf", - "//mlir/dialects/vector", - "//stdx", - ], -) - -zig_test( - name = "test", - deps = [":builder"], -) diff --git a/dsl/mosaic_tpu/README.md b/dsl/mosaic_tpu/README.md deleted file mode 100644 index cb1e0bab..00000000 --- a/dsl/mosaic_tpu/README.md +++ /dev/null @@ -1,846 +0,0 @@ -# ZML Mosaic TPU DSL — Author's Guide - -A Zig builder for Mosaic TPU IR (the `tpu` dialect in XLA) that tries -to read like `pallas.tpu` Python while staying in pure Zig. Used by -`zml/kernels/moe/mosaic_tpu/`, `zml/kernels/paged_attention/mosaic_tpu/` -and `platforms/tpu/`, and by anything else that wants to emit a TPU kernel -op-for-op against what `pallas_call` produces. - -The DSL builds `tpu` operations and attributes through the bindings in -`mlir/dialects/mosaic_tpu`. -A Mosaic kernel is a pure IR generator: it builds its MLIR in a throwaway -context, canonicalizes it, serializes it to a string, and hands that to a -`stablehlo.custom_call` targeting `tpu_custom_call` — it never touches ZML's -compilation context (which carries only `func`/`stablehlo`/`sdy`). The dialect and -its C API come from the pinned XLA repository. - -Two layers define the surface: - -- **Low layer** — `dsl/mosaic_tpu/builder.zig`: the IR builder primitives. - `Builder.open(allocator, ctx, name)` → `b.declareArgsOpts(spec, results, opts)` - → body via `b.*` helpers → `b.finishOpts(results, .{ .canonicalize = true })` - for the final IR string. Pass `Builder.open` / `Builder.init` a - `zml.kernel.mosaic_tpu.newContext()` (a throwaway context with the dialects - the DSL emits), `defer ctx.deinit()` once you have the string — not a - `Compilation`'s `mlir_ctx`. -- **High layer** — `zml.kernel.mosaic_tpu.Kernel(Config, spec)`: the - declarative kernel form, mirroring the Triton DSL. Bundles a config - type, a typed spec literal (name + named `inputs` / `outputs` + a typed - `run` function pointer), and produces a kernel type. Exposes - `.emit(allocator, cfg)` (IR string; manages its own context) and - `.call(inputs, outputs, opts)` (IR string + `stablehlo.custom_call` - targeting `tpu_custom_call` in one shot — the preferred form for - production model code). - -## Table of contents - -1. [Skeleton of a kernel](#skeleton-of-a-kernel) -2. [Argument specs](#argument-specs) -3. [Pallas `pallas_call` metadata](#pallas-pallas_call-metadata) -4. [The `Value` type](#the-value-type) -5. [Fluent methods on `Value`](#fluent-methods-on-value) -6. [Polymorphic scalars](#polymorphic-scalars) -7. [Constants and constant hoisting](#constants-and-constant-hoisting) -8. [Loads and stores](#loads-and-stores) -9. [Vector and shape helpers](#vector-and-shape-helpers) -10. [Reductions, scans, sorts](#reductions-scans-sorts) -11. [Compute primitives](#compute-primitives) -12. [Memref ops](#memref-ops) -13. [DMAs, semaphores, and barriers](#dmas-semaphores-and-barriers) -14. [Control flow](#control-flow) -15. [Escape hatches](#escape-hatches) -16. [Pallas Python ↔ Zig DSL cheat-sheet](#pallas-python--zig-dsl-cheat-sheet) -17. [Pitfalls](#pitfalls) - ---- - -## Skeleton of a kernel - -```zig -const mlir = @import("mlir"); -const mtt = @import("zml").kernel.mosaic_tpu; - -pub const MyKernel = struct { - pub const Cfg = struct { /* shape/dtype/grid knobs */ }; - pub const Kernel = mtt.Kernel(Cfg, .{ - .name = "my_kernel", - .inputs = &.{ "x", "cu_seqlens" }, - .outputs = &.{"y"}, - .run = run, - }); - fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { - _ = cfg; - const a = try b.declareArgsOpts(.{ - // Grid program-ids — i32 scalars. `pl.program_id(0)`, `pl.program_id(1)`. - .i_b = .{ .scalar = .i32 }, - - // Inputs: VMEM memrefs. - .x = .{ .ref = .{ .shape = &.{ 128, 256 }, .dtype = .bf16 } }, - - // SMEM scalar-prefetch. - .cu_seqlens = .{ .ref = .{ .shape = &.{4}, .dtype = .i32, .memory_space = .smem } }, - - // Outputs. - .y = .{ .ref = .{ .shape = &.{ 128, 256 }, .dtype = .bf16, .role = .output } }, - }, &.{}, .{ - .dimension_semantics = &.{.arbitrary}, - .iteration_bounds = &.{ /* grid */ 4 }, - .pallas_window_params = true, - }); - _ = a; - - // ... emit body via b.* helpers; reach args through `a.`. - } -}; -``` - -A kernel is a `struct {…}` wrapper bundling three decls: - -1. **`pub const Cfg`** — config type, an inline `struct { ... }` of - comptime/runtime parameters. -2. **`pub const Kernel = mtt.Kernel(Cfg, .{ .name, .inputs, .outputs, .run })`** — - the spec literal that turns into a kernel type. `inputs` / `outputs` are - slices of `[:0]const u8` field names; the factory generates `Inputs` - (struct of `Tensor`), `Outputs` (struct of `Shape`), and `Results` - (struct of `Tensor`) types from those names. The `run` field is a typed - function pointer — wrong signature is a compile error here. -3. **`fn run(b, cfg)`** — the kernel body, mirroring the Triton form. First - call `try b.declareArgsOpts(.{…}, &.{}, opts)` to create the `func.func` - op and bind a `NamedArgs(Spec)` struct of `Value`s. The factory has - already opened the builder; it'll terminate, verify, run - `canonicalize,cse,canonicalize`, and serialize after `run` returns — - no manual `b.finish(...)` call. - -The names in `spec.inputs` / `spec.outputs` are the **call-site** names -(used to build the `Inputs` / `Outputs` structs) and don't have to match -the names in `b.declareArgsOpts(...)` (which are the **MLIR-level** arg -names — what shows up in the `func.func` signature). By convention they -often do — pick whatever reads best at the boundary. - -The arg spec inside `b.declareArgsOpts(...)` is a struct literal — its -variant *tags* (`.scalar`, `.iter`, `.ref`, `.sem`) are comptime-known per -call site, but the *payload values* (shapes, dtypes, memory spaces, -windows) can be runtime values from a config. - -> **Why the wrapper struct.** `Cfg` has to be a *named* type so the typed -> `run: *const fn (*Builder, Cfg) FinishError!void` field in the spec can -> reference it. Wrapping the kernel in `pub const MyKernel = struct {...}` -> keeps `Cfg`, `Kernel`, and `run` scoped together and exposes them all to -> callers (`MyKernel.Cfg`, `MyKernel.Kernel`, plus everything the inner -> Kernel re-exports — `MyKernel.Kernel.Config`, `.Inputs`, `.Outputs`, -> `.Results`, `.CallOpts`). -> -> Single-kernel files (e.g. `zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig`) -> can skip the wrapper and put `pub const Cfg` / `pub const Kernel` at -> module scope directly — the wrapper is just convenient when several -> kernels share a file. - -### Two ways to use a kernel - -**Offline / tooling** — get the MLIR string and print or diff it (no MLIR -context needed; `emit` spins up and tears down its own): - -```zig -const ir = try MyKernel.Kernel.emit(allocator, .{ /* cfg */ }); -defer allocator.free(ir); -try stdout.print("{s}\n", .{ir}); -``` - -**Inside a ZML model** — emit IR and drop a `stablehlo.custom_call` -(targeting `tpu_custom_call`) into the model in one shot: - -```zig -const out = MyKernel.Kernel.call( - .{ .x = x_tensor, .cu_seqlens = cu }, // Inputs (by name) - .{ .y = x_tensor.shape() }, // Outputs (Shapes) - .{ - .cfg = .{ /* cfg */ }, - .extras = .{ .vmem_limit_bytes = 1 << 20 }, - }, -).y; -``` - -`Kernel.call` takes a `mosaic_tpu.CallExtras` via `opts.extras` for -TPU-specific knobs (`vmem_limit_bytes`, `disable_bounds_checks`, -`disable_semaphore_checks`, `has_communication`, plus -`additional_attributes` and `output_operand_aliases` for in-place -updates). - ---- - -## Argument specs - -Each field passed to `b.declareArgs(...)` / `b.declareArgsOpts(...)` is an -`ArgSpec.Kind` tagged-union literal. The four kinds mirror what a Mosaic -`func.func` can carry: - -| Kind literal | MLIR type | Use case | -|--------------------------------------------------------|--------------------------------------|-----------------------------------------| -| `.{ .scalar = .i32 }` | plain `i32` | `program_id`, `num_programs`, sizes | -| `.{ .iter = .{ .semantics = .parallel } }` | `index` | Loop / grid iter index with optional `tpu.dimension_semantics` | -| `.{ .ref = .{ .shape = &.{...}, .dtype = .bf16 } }` | `memref<… x T, #tpu.memory_space<...>>` | Pallas-style operand | -| `.{ .sem_array = .{ .shape = &.{2}, .kind = .dma } }` | `memref<…x!tpu.(dma_)semaphore, semaphore_mem>` | Scratch sem pool (e.g. DMA double-buffer) | -| `.{ .sem = .regular }` / `.{ .sem = .dma }` | `!tpu.semaphore` / `!tpu.dma_semaphore` | Single semaphore handle | - -### Roles - -`RefSpec.role` mirrors what Pallas `pallas_call` calls each operand: - -| Role | Where it appears | `window_params` / `transform_N` | -|--------------------|-----------------------------------------------------|---------------------------------| -| `.input` (default) | Inputs declared via `in_specs` | one entry per input | -| `.output` | Outputs declared via `out_specs` | one entry per output | -| `.scalar_prefetch` | Leading SMEM refs (Pallas's `scalar_prefetch`) | none — but appended to every `transform_N` body's args | -| `.scratch` | Trailing scratch refs (kv_bufs, l/m/acc, sem_array) | none | - -The DSL filters `window_params` to only `.input`/`.output` entries (matching -Pallas's `len(grid_mapping.block_mappings)`), and re-emits the -`scalar_prefetch` refs as trailing args on every emitted `transform_N` stub. - -The default `memory_space` for a `.ref` is `.vmem`. Override with `.smem`, -`.cmem`, `.hbm`, `.semaphore_mem`, `.vmem_shared`, or `.any` per the -`MemorySpace` enum. Iteration semantics (`.parallel`, `.arbitrary`, -`.core_parallel`, `.subcore_parallel`) attach `tpu.dimension_semantics` to -the iter arg. - -**Runtime payloads:** the variant tags are fixed per kernel, but shape / -dtype / memory_space values can be runtime — typical for kernels -parameterized by a `Config` struct. - ---- - -## Pallas `pallas_call` metadata - -The wrapper attributes Pallas attaches to the `func.func` are exposed through -`Builder.Opts`: - -| Field | Type | Meaning | -|--------------------------|---------------------------|--------------------------------------------------------------| -| `.@"noinline"` | `bool` (default false) | `no_inline` attribute on the function. | -| `.core_type` | `?CoreType` (default `.tc`) | `tpu.core_type = #tpu.core_type<…>`. | -| `.dimension_semantics` | `[]const DimensionSemantics` | `dimension_semantics = [...]` — one entry per grid dim. | -| `.scalar_prefetch` | `i64` (default 0) | `scalar_prefetch = N : i64`. | -| `.scratch_operands` | `i64` (default 0) | `scratch_operands = N : i64`. | -| `.iteration_bounds` | `?[]const i64` | `iteration_bounds = array` — the grid shape. | -| `.pallas_window_params` | `bool` (default false) | Auto-emit `window_params = [...]` and trailing `transform_N` stub funcs. | - -Per-arg `window_params` behavior follows -`jax/_src/pallas/mosaic/lowering.py:1014–1131` exactly. With -`pallas_window_params = true`: - -- **`.hbm` / `.any` / `.semaphore_mem`** memory spaces ⇒ empty `{}` window - entry, **no `transform_N`**. Anything else gets a populated dict. -- **VMEM trivial-window** (no `RefSpec.window`, or `window.block_shape == ref.shape`) - ⇒ auto-force `pipeline_mode = synchronous` (matching Pallas's - `Buffered(1)` injection). -- **Otherwise** ⇒ emit only attributes the user explicitly set on - `RefSpec.window`. -- **`window_bounds`** = `block_shape` (defaults to operand shape). -- **`transform_N` body** returns `block_rank` i32 zeros, mirroring a trivial - index map. - -Override per ref-arg via `RefSpec.window`: - -```zig -.x = .{ .ref = .{ - .shape = &.{ 256, 512 }, - .dtype = .bf16, - .window = mtt.ArgSpec.WindowSpec{ - .block_shape = &.{ 128, 512 }, - .pipeline_mode = .double_buffered, - .revisit_mode = .any, - .element_window = .{ .pad_low = &.{ 0, 0 }, .pad_high = &.{ 0, 0 } }, - // Custom transform_N return — by default the body returns - // `block_rank` zeros (a trivial index_map). Override per slot. - .transform_returns = &.{ - .{ .program_id = 1 }, // q_blk_idx - .{ .program_id = 0 }, // heads_blk_idx - .zero, - }, - }, -} }, -``` - -The DSL treats the window as **non-trivial** (so it does *not* auto-force -`pipeline_mode = synchronous` for VMEM) when `transform_returns` is non-null -and contains anything other than all `.zero`s. This matches Pallas's -`bm.has_trivial_window()` check. - -`PipelineMode` is `{ synchronous, double_buffered }` (matching Pallas's -`Buffered(1)` / `Buffered(2)`). `RevisitMode` is `{ immediate, any }`. - -> **Anonymous-struct gotcha.** Anonymous-struct literals (`.{ ... }`) don't -> coerce through the optional-pointer wrapping that Zig 0.16 expects for -> `RefSpec.window`. Always write the type explicitly: -> `.window = mtt.ArgSpec.WindowSpec{ ... }`. - ---- - -## The `Value` type - -```zig -pub const Value = struct { - inner: *const mlir.Value, - kernel: ?*Builder = null, - ... -}; -``` - -- `inner` is the underlying MLIR SSA handle; reach for it when calling raw - `tpu.*` / `arith.*` / `vector.*` constructors. -- `kernel` is the owning `Builder` pointer; all fluent methods route through - it. The DSL populates `kernel` automatically for every value it creates - (`emit`, `arg`, loop-region args, etc.). If you construct a `Value` by hand - (`.{ .inner = raw }`), the fluent methods will panic — use `Builder.*` - static helpers instead. - -Introspection methods always available: - -| Method | Returns | Notes | -|-------------------|----------------------------------|------------------------------------------------| -| `type_()` | `*const mlir.Type` | Full MLIR type (scalar, vector, or memref). | -| `elemType()` | `*const mlir.Type` | Element type; same as `type_()` for scalars. | -| `isVector()` | `bool` | `true` for `vector<…>`. | -| `isMemRef()` | `bool` | `true` for `memref<…>`. | -| `isShaped()` | `bool` | Vector or memref. | -| `rank()` | `usize` | `0` for scalars. | -| `dim(i)` | `i64` | Traps on scalars. | -| `shape()` | `Shape` (BoundedArray, MAX_RANK) | Use `.constSlice()` to feed `[]const i64`. | -| `isFloatElem()` | `bool` | True for any float element type. | -| `isIntElem()` | `bool` | True for any integer element type. | - ---- - -## Fluent methods on `Value` - -Every fluent method returns a new `Value`. They dispatch to -`arith.addi` / `arith.addf` / … based on the element type. The RHS is -`anytype` — see [Polymorphic scalars](#polymorphic-scalars). - -### Arithmetic - -| Method | Int op | Float op | -|----------------|--------------|------------------| -| `.add(rhs)` | `arith.addi` | `arith.addf` | -| `.sub(rhs)` | `arith.subi` | `arith.subf` | -| `.mul(rhs)` | `arith.muli` | `arith.mulf` | -| `.div(rhs)` | `arith.divsi`| `arith.divf` | -| `.rem(rhs)` | `arith.remsi`| `arith.remf` | - -For Python-style floor-division (`a // b` matches `jax.numpy.floor_divide`'s -sign-correcting expansion — `divsi` plus a `(sign(a) != sign(b)) && (a % b != -0)` correction), use `b.divFloor(lhs, rhs)`. `divsi` alone truncates toward -zero, which differs for mixed-sign operands. -| `.bitAnd(rhs)` | `arith.andi` | — | -| `.bitOr(rhs)` | `arith.ori` | — | -| `.minimum(rhs)`| `arith.minsi`| `arith.minnumf` | -| `.maximum(rhs)`| `arith.maxsi`| `arith.maxnumf` | - -### Comparisons - -All return i1 (scalar) or `vector<… x i1>`. Signed integer predicates by -default; ordered float predicates for floats. - -| Method | Int predicate | Float predicate | -|------------|---------------|-----------------| -| `.lt(rhs)` | `.slt` | `.olt` | -| `.le(rhs)` | `.sle` | `.ole` | -| `.gt(rhs)` | `.sgt` | `.ogt` | -| `.ge(rhs)` | `.sge` | `.oge` | -| `.eq(rhs)` | `.eq` | `.oeq` | -| `.ne(rhs)` | `.ne` | `.one` | - -### Casts - -| Method | Effect | -|-----------------------|-----------------------------------------------------| -| `.to(dtype)` | Numeric cast preserving shape; auto-dispatches int↔float. | -| `.bitCast(dtype)` | `tpu.bitcast` — same-bitwidth element-type swap. | -| `.abs()` | `math.absf` (float) / `math.absi` (int). | - -`.to(...)` chooses the right MLIR op based on the source vs target kind: - -| Source kind | Target kind | MLIR op | -|-------------|----------------|--------------------| -| int | wider int | `arith.extsi` | -| int | narrower int | `arith.trunci` | -| int | float | `arith.sitofp` | -| float | int | `arith.fptosi` | -| float | wider float | `arith.extf` | -| float | narrower float | `arith.truncf` | -| same type | same type | no-op | - -For `index`-typed values use `b.indexCast(v, target_ty)` or `b.toIndex(v)` -explicitly — `.to(...)` is for numeric types. - ---- - -## Polymorphic scalars - -Most `Builder.*` helpers and every fluent method's RHS take `anytype`. The -lift rule: - -- `Value` → pass through. -- `comptime_int` / `comptime_float` → a constant whose element type matches - the LHS operand. `v_i64.mul(16)` lifts `16` to i64, `v_f32.add(1.0)` lifts - `1.0` to f32. -- Runtime `i8/i16/i32`, `i64`, `f16/f32/f64`, etc. → a constant that - preserves the source Zig width. - -If you need to force a specific width for a runtime variable, cast at the -call site: `pid.mul(@as(i32, @intCast(block_size)))`. - -### When do you need `b.lift` / `b.constant` / `b.constIndex`? - -| Helper | Effect | -|--------------------------------|----------------------------------------------------------| -| `b.lift(value)` | Wrap a Zig scalar; dtype inferred from source. | -| `b.constant(dtype, value)` | Wrap a Zig scalar as a specific `DType`. | -| `b.constantLike(value, like)` | Wrap a Zig scalar as `like`'s element type. | -| `b.constIndex(value)` | `arith.constant N : index` — for memref offsets. | - -Reach for these when the DSL surface demands an existing `Value` and there's -no binop sibling to match against — typically loop seeds for -`openWhile(.{inits}, ...)`, `openFor(0, N, 1, .{init})`, or `var x: Value = -b.constant(.i32, 0);` that a later branch may reassign. - -If the constant flows through any `.add` / `.mul` / `splat` etc., skip the -lift and let auto-lift do it. - ---- - -## Constants and constant hoisting - -The DSL provides splat-style constructors for the dense-splat fast-path that -Pallas/Mosaic uses: - -| Helper | Emits | -|----------------------------------------|---------------------------------------------| -| `b.zeros(shape, dtype)` | `arith.constant dense<0.0> : vector<…>` | -| `b.ones(shape, dtype)` | Ones vector via dense splat. | -| `b.full(shape, value, dtype)` | `arith.constant dense : vector<…>` | -| `b.splat(value, shape, dtype)` | Same as `full` for comptime values; `vector.broadcast` for runtime Values. | - -These emit a single `arith.constant` (instead of a scalar-const + -`vector.broadcast` pair) so the IR diff vs Pallas stays empty for splat-only -constants. - ---- - -## Loads and stores - -Mosaic separates **vector** loads/stores from **scalar** memref loads/stores. -Helpers infer everything from the memref's type — there is no `result_type` -argument. - -### Vector - -| Helper | Pallas / Mosaic equivalent | -|---------------------------------------------------------|-------------------------------------------------------| -| `b.refLoad(ref)` | `vector.load ref[0, …, 0] : memref<…>, vector<…>` | -| `b.vectorLoadAt(ref, indices)` | `vector.load ref[indices...]` with full ref shape. | -| `b.vectorLoadShape(ref, indices, shape)` | Sliced `vector.load` — `ref[i, j, :, k:k+BLOCK]`. | -| `b.tpuVectorLoad(ref, indices, .{ .strides, .mask })` | `tpu.vector_load` — sublane stride + mask form. | -| `b.refStore(ref, value)` | Full-shape `tpu.vector_store ref[0, …]`. | -| `b.vectorStoreAt(ref, value, indices)` | `tpu.vector_store ref[indices…]`. | -| `b.refStoreOpts(ref, value, indices, .{ .strides, .mask, .add })` | Strided / masked / atomic-add store. | -| `b.stridedLoad(ref, indices, strides)` | `tpu.strided_load` — result shape = `ref.shape / strides` (per-dim ceil). | -| `b.stridedLoadShape(ref, indices, strides, result_shape)` | `tpu.strided_load` with explicit result shape — for cropped loads (e.g. half-rows). | -| `b.stridedStore(ref, value, indices, strides)` | `tpu.strided_store`. | - -> **`refLoad` vs `tpuVectorLoad`.** Pallas's default emit is -> `vector.load`, which is what `refLoad` / `vectorLoadAt` produce. Reach for -> `tpuVectorLoad` only when you need sublane stride or a per-element mask -> (the `tpu.vector_load` form). Same on the store side. - -`indices` must be `index`-typed Values (use `b.toIndex(v)` to convert from -i32). If you pass `&.{}`, the helper pads with a cached `arith.constant 0 : -index` to match the ref's rank — the same SSA value is reused per block, so -no `%c0` duplication shows up in the IR. - -### Scalar (SMEM) - -| Helper | MLIR op | -|-------------------------------------------------|------------------------------------------| -| `b.scalarLoad(ref, indices)` | `memref.load` | -| `b.scalarStore(ref, value, indices)` | `memref.store` | - -Use these for SMEM args (e.g. `cu_seqlens`) where Pallas emits scalar -`memref.load`. - ---- - -## Vector and shape helpers - -| Helper | Effect | -|-----------------------------------------------------|----------------------------------------------------------| -| `b.broadcastTo(src, shape)` | `vector.broadcast` — leading-dim broadcast or rank-prepend. | -| `b.shapeCast(src, new_shape)` | `vector.shape_cast` — reshape preserving total lanes. | -| `b.reshape(src, new_shape)` | `tpu.reshape` (TPU-flavored reshape; differs from `shapeCast`). | -| `b.repeat(src, dimension, times)` | `tpu.concatenate` of `times` copies, as `pltpu.repeat`. | -| `b.concatenate(sources, dimension)` | `tpu.concatenate`. | -| `b.transpose(src, permutation)` | `tpu.transpose`. | -| `b.broadcastInSublanes(src, lane)` | `tpu.broadcast_in_sublanes`. | -| `b.rotate(src, .{ .amount, .dimension, .stride, .stride_dimension })` | `tpu.dynamic_rotate` with a constant i32 amount. | -| `b.vectorExtract(src, position)` | `vector.extract` — scalar / lower-rank slice. | -| `b.iota(shape, dtype, dimensions)` | `tpu.iota` over a chosen dim. | -| `b.arange(n, dtype)` | Pallas-style 1-D iota: `<1xN>` `tpu.iota` + `vector.shape_cast` to ``. Mirrors `_iota_lowering_rule`. | - -> **Use `arange`, not `iota(&.{N})`.** The TPU layout pass requires vectors -> to be 2-D with the lane dim trailing; a bare 1-D `tpu.iota` is rejected. -> `arange` emits the `<1xN>` + `shape_cast` pair Pallas's lowering produces. - -> **`shapeCast` vs `reshape`.** `shapeCast` emits the upstream -> `vector.shape_cast` Pallas uses everywhere; `reshape` emits TPU's -> `tpu.reshape`. Stick to `shapeCast` when matching Pallas IR. - -### Auto-broadcast - -The fluent binary ops and comparisons (`add` … `maximum`, `lt` … `ne`), -`b.where`, `b.maximum` and `b.minimum` splat a scalar operand against a -vector one (`vector.broadcast`) and widen the narrower integer side. Nothing -else broadcasts: Mosaic kernels run inside the TPU's tile semantics and the -layout pass is strict about `vector<…>` ranks/shapes, so build rank changes -and size-1 expansions explicitly with `shapeCast` + `broadcastTo`: - -```zig -// b_k[:, None] broadcast to -const bk_2d = k.shapeCast(b_k, &.{ BK, 1 }); -const bk_bc = k.broadcastTo(bk_2d, &.{ BK, BV }); -``` - -For a vector made from a scalar outside a binary op, use -`b.broadcastTo(scalar, shape)` (it routes through `vector.broadcast`) or -`b.full / b.splat` for comptime constants. - ---- - -## Reductions, scans, sorts - -### Vector-dialect reductions - -| Helper | Effect | -|-----------------------------------------------------|----------------------------------------------------------| -| `b.multiReduction(kind, src, acc, axes)` | `vector.multi_reduction` — multi-D reduce along `axes`. | -| `b.reduceToScalar(kind, src, acc)` | 1-D vec → scalar via the Pallas idiom: `shape_cast→<1xN>`, `multi_reduction[1]`, `extract[0]`. | -| `b.vectorReductionFlat(kind, src)` | `vector.reduction` — bare 1-D reduce. Layout pass may reject; prefer `reduceToScalar`. | - -`kind` is a `CombiningKind` (`.add`, `.mul`, `.minsi`, `.maxsi`, `.minnumf`, -`.maxnumf`, `.and_`, `.or_`, `.xor`). - -> **Pallas's reduce-to-scalar idiom.** Pallas never emits a bare -> `vector.reduction` for the L2-norm / sum-to-scalar pattern — it always -> goes through `multi_reduction` after a leading-dim `shape_cast`. The -> `reduceToScalar` wrapper produces exactly that shape, so kernels stay -> byte-equivalent to `pallas_call`'s output. See -> `jax/_src/pallas/mosaic/lowering.py:reduce_lowering_rule`. - -### TPU-dialect reductions / scan / sort - -| Helper | MLIR op | -|-----------------------------------------------------|----------------------------------------------------------| -| `b.allReduce(input, kind, dim)` | `tpu.all_reduce` — keeps `dim` as size 1. | -| `b.reduceIndex(input, kind, axis)` | `tpu.reduce_index` — drop axis, replace with `index`. | -| `b.scan(input, kind, mask)` | `tpu.scan` along the last dimension; optional rank-one mask. | -| `b.sort(keys, values, mask, .{ .descending })` | `tpu.sort` — returns `[3]Value` (sorted_keys, indices, sorted_values). | - ---- - -## Compute primitives - -### Math (scalar / elementwise) - -| Helper | MLIR op | -|------------------------------------------------|--------------------| -| `b.exp(x)` | `math.exp` | -| `b.exp2(x)` | `exp(x * ln2)` — matches Pallas's `_exp2_lowering_rule` (forward-compat). | -| `b.mathExp2(x)` | Raw `math.exp2` — only legal on cloud TPUs after 2025-07-26. | -| `b.log(x)` / `b.log2(x)` | `math.log` / `math.log2` | -| `b.sqrt(x)` / `b.rsqrt(x)` | `math.sqrt` / `math.rsqrt` | -| `b.sin(x)` / `b.cos(x)` / `b.tanh(x)` / `b.erf(x)` | `math.*` | -| `b.absf(x)` / `b.absi(x)` / `b.abs(x)` | `math.absf` / `math.absi` (auto) | -| `b.floor(x)` / `b.ceil(x)` | `math.floor` / `math.ceil` | -| `b.powf(x, y)` | `math.powf` | -| `b.fma(a, b, c)` | `math.fma` | -| `b.reciprocal(x)` / `b.reciprocalOpts(x, .{ .approximate })` | `tpu.reciprocal` | - -### Matmul - -```zig -// y = lhs @ rhs + acc -const y = k.matmul(lhs, rhs, acc); -const y2 = k.matmulOpts(lhs, rhs, acc, .{ - .transpose_lhs = false, - .transpose_rhs = true, - .precision = .bf16, -}); -``` - -`matmul` lowers to `tpu.matmul`; result type is taken from `acc`. Use -`matmulOpts` to set transposes, `tpu.contract_precision`, or a custom -`#tpu.dot_dimension_numbers<…>` attribute. - -### Selects / where - -| Helper | MLIR op | -|---------------------------------|--------------------------| -| `b.select(cond, t, f)` | `arith.select` | -| `b.where(cond, x, y)` | Alias of `select`. | - -### Masks - -| Helper | Effect | -|----------------------------------------------------------|-----------------------------------------| -| `b.createMask(low_bounds, high_bounds, shape)` | `vector.create_mask`. | -| `b.rangeMask(limits, shape)` | `vector.create_mask` from `0..limit`. | - ---- - -## Memref ops - -For carving sub-references out of a larger memref or reinterpreting an -existing one: - -| Helper | MLIR op | -|-----------------------------------------------------------------|--------------------------| -| `b.memRefSlice(mem_ref, offsets, sizes, strides, result_shape)` | `memref.subview`. | -| `b.memRefSqueeze(mem_ref, result_shape)` | `memref.collapse_shape` style. | -| `b.memRefReshape(mem_ref, result_shape)` | `tpu.memref_reshape`. | -| `b.memRefBitcast(mem_ref, dtype)` | `tpu.memref_bitcast` — same shape, new element dtype. | -| `b.memRefBitcastShape(mem_ref, result_shape, dtype)` | `tpu.memref_bitcast` with explicit result shape — for the bf16↔i32 packing trick (`<64x128xbf16>` → `<32x128xi32>`). | -| `b.reinterpretCast(mem_ref, result_shape, dtype)` | `memref.reinterpret_cast`. | -| `b.assumeLayout(src)` | `tpu.assume_layout`. | -| `b.eraseMemRefLayout(mem_ref)` | `memref.cast` to no-layout. | - ---- - -## DMAs, semaphores, and barriers - -For multi-core / async DMA orchestration: - -| Helper | MLIR op | -|------------------------------------------------------------------------------|------------------------| -| `b.semAlloc(.regular)` / `b.semAlloc(.dma)` | `tpu.sem_alloc`. | -| `b.semBarrier()` | `tpu.sem_barrier`. | -| `b.semRead(sem)` | `tpu.sem_read`. | -| `b.semWait(sem, amount)` | `tpu.sem_wait`. | -| `b.semSignal(sem, amount)` / `b.semSignalOpts(sem, amount, .{ .device_id, .core_id, .subcore_id })` | `tpu.sem_signal`. | -| `b.barrier(barrier_id)` | `tpu.barrier`. | -| `b.enqueueDma(src, dst, target_sem, .{ .source_semaphore, .device_id, .core_id, .subcore_id, .priority, .strict_ordering })` | `tpu.enqueue_dma`. | -| `b.waitDma2(sem, src, dst, .{ .device_id, .core_id, .strict_ordering })` | `tpu.wait_dma2`. | -| `b.deviceId()` | `tpu.device_id`. | -| `b.delay(cycles)` | `tpu.delay`. | -| `b.traceStart(level, message)` / `b.traceStop()` | `tpu.trace_start` / `tpu.trace_stop` — Pallas's einsum boundaries. | -| `b.dotDimensionNumbers(lhs_c, rhs_c, lhs_nc, rhs_nc, output_order, lhs_b, rhs_b)` | Build a `#tpu.dot_dimension_numbers<...>` attr for `matmulOpts`. | - ---- - -## Control flow - -SCF regions are built with **scope-based** builders — `b.openFor(...)`, -`b.openIf(...)` / `b.openIfElse(...)`, `b.openWhile(...)` return a typed -scope value. You emit body ops into a bare `{}` block, terminate with -`yield` / `yieldThen` / `yieldAfter` (or `close()` for a scope without -values), and read results from `scope.results` afterward. No `ctx` struct, -no function literal — lexical scope carries captures. - -### `openFor` — scf.for with iter_args - -```zig -var loop = k.openFor(c0_i32, T, c1_i32, .{ b_h_init }); -{ - const t = loop.iv; - const b_h = loop.carried[0]; - - // ... body ops ... - - loop.yield(.{ b_h_updated }); -} -const final_b_h = loop.results[0]; -``` - -- `lower` / `upper` / `step` are `anytype`: pass Values, comptime ints, or - runtime Zig ints. A bound that isn't a Value takes the type of the first one - that is; with none, all three are `index`. Pass `b.constant(.i32, 0)` as - `lower` for an `i32` counter (matching Pallas's `scf.for ... : i32`). -- `inits` is a tuple `.{v1, v2, ...}` or an array. Arity is comptime — - `carried`, `results`, and the `yield(...)` values (also a tuple or an - array) all have the same length. -- Empty loops: `b.openFor(0, N, 1, .{})` + `loop.close()`. -- `b.openForOpts(lower, upper, step, inits, .{ .unsigned_cmp = true })` sets - the loop's options. - -### `openIf` — scf.if without else (side-effects only) - -```zig -var i = k.openIf(cond); -{ - // then body — store, increment, etc. - i.close(); -} -``` - -Use for conditional side-effects (no results). The else block is auto-built -empty. - -### `openIfElse` — scf.if with else and optional results - -```zig -var i = k.openIfElse(cond, .{.f32}); -{ - i.yieldThen(.{ x }); -} -{ - i.yieldElse(.{ y }); -} -const r = i.results[0]; -``` - -- `result_types` is a tuple or array of dtypes (`.f32`), Values (for their - types) or `*const mlir.Type`s such as `k.vectorTy(shape, dtype)` / - `k.memRefTy(shape, dtype, mem_space)`. Pass `.{}` for an if/else with no - results, then end its branches with `elseBranch()` and `close()`. - -### `openWhile` — scf.while - -```zig -var w = k.openWhile(.{ i0 }, .{.i32}); -{ - const s = w.before_carried[0]; - w.yieldBefore(s.lt(10), .{ s }); -} -{ - const s = w.after_carried[0]; - w.yieldAfter(.{ s.add(1) }); -} -const r = w.results[0]; -``` - -- `inits` — a tuple or array of `Value`s; before-region arg types = their types. -- `after_types` — a tuple or array of dtypes (`.f32`), Values (for their - types) or `*const mlir.Type`s; after-region arg types + the scf.while's - result types. -- `yieldBefore(cond, forwarded)` — `forwarded` arity must match `after_types`. -- `yieldAfter(values)` — arity must match `inits`. - -### General tips - -- Scopes stack naturally: `openFor { openIf { openFor { ... } } }` all just - push/pop the kernel's current-block stack. -- Scopes without values end with `close()`: `loop.close()`, `i.close()`, and - `elseBranch()` then `close()` for an if/else. -- After any `yield*`, the scope's `.results` holds the scf op's results as - a `[N]Value`. - ---- - -## Escape hatches - -Everything the DSL doesn't expose is still reachable — `Builder` is a -book-keeping layer, not a walled garden: - -- `v.inner` — the raw `*const mlir.Value`; feed it to any `tpu.*` / - `arith.*` / `vector.*` / `memref.*` / `scf.*` / `math.*` constructor - directly. -- `b.ctx` — the owning `*mlir.Context`. -- `b.currentBlock()` — the active `mlir.Block` (respects `pushBlock` / - `popBlock`). -- `b.emit(op)` / `b.emitMulti(op, n)` — if you've built an `mlir.Operation` - by hand, use these so the result comes back as a `Value` with its - `kernel` field populated. -- `b.scalarTy(dtype)` / `b.vectorTy(shape, dtype)` / - `b.memRefTy(shape, dtype, mem_space)` — type constructors used by raw - ops and by non-scalar `openIfElse.result_types` / `openWhile.after_types`. - -When you drop to raw ops, remember to still **append them to the current -block** — `b.emit(...)` does that for you; `_ = op.appendTo(b.currentBlock())` -is the manual form. - ---- - -## Pallas Python ↔ Zig DSL cheat-sheet - -| Python | Zig | -|-------------------------------------------------------|--------------------------------------------------------------| -| `pl.program_id(0)` | `a.i_v` (declared as `.{ .scalar = .i32 }`) | -| `cu_seqlens[i]` | `k.scalarLoad(a.cu_seqlens, &.{ k.toIndex(i) })` | -| `lax.div(a, b)` | `k.divsi(a, b)` | -| `lax.rem(a, b)` | `k.remsi(a, b)` | -| `jnp.arange(N, dtype=jnp.int32)` | `k.arange(N, .i32)` | -| `o_k < KEY_DIM` | `k.cmpi(.slt, o_k, dim_splat)` / `o_k.lt(dim_splat)` | -| `mask_k[:, None] & mask_v[None, :]` | `k.andi(k.broadcastTo(k.shapeCast(mask_k, &.{BK,1}), &.{BK,BV}), k.broadcastTo(mask_v, &.{BK,BV}))` | -| `jnp.where(mask, x, 0.0)` | `k.select(mask, x, k.zeros(shape, dtype))` | -| `q_ref[0, t, h, :].astype(jnp.float32)` | `k.vectorLoadShape(a.q, &.{c0, t_idx, h_idx, c0}, &.{1,1,1,BK})` then `k.shapeCast(v, &.{BK})` then `.to(.f32)` | -| `q_ref[0, t, h, :] = b_o.astype(o_ref.dtype)` | `k.vectorStoreAt(a.q, k.shapeCast(b_o, &.{1,1,1,BK}), &.{c0, t_idx, h_idx, c0})` | -| `jnp.sqrt(jnp.sum(b_q * b_q) + 1e-6)` | `k.sqrt(k.addf(k.reduceToScalar(.add, k.mulf(b_q, b_q), cst_zero_1), cst_eps))` | -| `jnp.exp(b_g)` | `k.exp(b_g)` | -| `jnp.sum(b_h * b_k[:, None], axis=0)` | `k.multiReduction(.add, k.mulf(b_h, bk_bc), cst_zero_v, &.{0})` | -| `b_h * jnp.exp(b_g)` (broadcast to ) | `k.mulf(b_h, k.broadcastTo(exp_g, &.{BK, BV}))` | -| `b_beta * (b_v - predicted_v)` | `k.mulf(k.broadcastTo(b_beta, &.{BV}), k.subf(b_v, predicted_v))` | -| `lax.fori_loop(0, T, body, init)` | `var loop = k.openFor(c0_i32, T, c1_i32, .{init}); { …; loop.yield(.{…}); }` | -| `if cond: …` (side-effects only) | `var i = k.openIf(cond); { …; i.close(); }` | -| `if cond: x = a else: x = b` | `var i = k.openIfElse(cond, .{ty}); { …; i.yieldThen(.{a}); }; { …; i.yieldElse(.{b}); }` | -| `while cond:` | `var w = k.openWhile(.{inits}, .{after_types}); { …; w.yieldBefore(cond, .{…}); }; { …; w.yieldAfter(.{…}); }` | -| `pltpu.MemorySpace.SMEM` | `.memory_space = .smem` on the `RefSpec` | -| `pl.BlockSpec(memory_space=…)` (trivial window) | Default `RefSpec` — auto-`synchronous` on VMEM | -| `pl.BlockSpec(block_shape=(BM, BK), …)` | `.window = .{ .block_shape = &.{BM, BK} }` | -| `pl.BlockSpec(..., pipeline_mode=Buffered(2))` | `.window = .{ .pipeline_mode = .double_buffered }` | - ---- - -## Pitfalls - -1. **TPU vectors are 2-D minimum.** A bare `tpu.iota : vector` or - `vector.reduction` on `vector` is rejected by the layout pass. - Use `b.arange(N, dtype)` (emits the `<1xN>` iota + `shape_cast`) and - `b.reduceToScalar(...)` (emits the `shape_cast→<1xN>` + - `multi_reduction[1]` + `extract[0]` triple). Match Pallas's emit pattern - verbatim and the layout pass is happy. - -2. **No implicit auto-broadcast.** Unlike the Triton DSL, Mosaic won't - silently `expand_dims` / `broadcast` a rank-mismatched RHS. Build the - broadcasts explicitly (`shapeCast` + `broadcastTo`). The Pallas Python - DSL emits the same explicit broadcast pair — mirroring it keeps the IR - diff empty. - -3. **`vector.load` indices must be `index`-typed.** Convert from i32 with - `k.toIndex(v)` (emits `arith.index_cast`). JAX does no CSE inside - `pallas_call` lowering, so re-emitting the same `index_cast` per use is - the byte-equivalent thing to do. - -4. **Re-emit derived offsets per use.** Pallas re-runs its index_map - computation per operand load/store rather than CSE-ing intermediate - `arith.muli` / `arith.addi`. To match its IR exactly, do the same: emit - `iv_off = i_v * BV` again at every site that needs it. The MLIR - canonicalizer collapses duplicates downstream. - -5. **`refLoad` vs `tpuVectorLoad`.** Pallas's default `ref[...]` lowers to - `vector.load` (not `tpu.vector_load`). Use `refLoad` / `vectorLoadAt` / - `vectorLoadShape` for byte-equivalence; reach for `tpuVectorLoad` only - when you need `tpu.vector_load`-specific features (sublane stride or - per-element mask). - -6. **`shapeCast` vs `reshape`.** `shapeCast` emits the upstream - `vector.shape_cast` Pallas uses everywhere; `reshape` emits TPU's - `tpu.reshape` (different op, different layout semantics). Stick with - `shapeCast` when matching Pallas IR. - -7. **SMEM args take scalar `memref.load`, not `vector.load`.** Use - `k.scalarLoad(a.cu_seqlens, ...)` for SMEM operands (typically the - scalar-prefetch path). VMEM args go through the vector-load helpers. - -8. **`pallas_window_params = true` only adds metadata.** It does not change - the kernel body — it just appends `iteration_bounds`, `window_params`, - and the `transform_N` stub funcs to the `func.func` so downstream passes - that consume `window_params` (DMA scheduling, double-buffering) work. - For non-trivial windows or non-VMEM operands, attach a `RefSpec.window` - to override the auto-`synchronous` default. - -9. **`finish` verifies before serializing.** A verifier failure here almost - always means an operand-type mismatch or a missing block terminator — - `scf.yield` operand count must match the loop's result types, and - `func.return` arity must match the declared result types. - -The removed low-level `tpu.relayout` helper has no replacement: upstream deleted -the unused operation and its vector-layout attribute. `Builder.rotate` retains -its static-amount API and emits `tpu.dynamic_rotate` with an i32 constant. diff --git a/dsl/mosaic_tpu/builder.zig b/dsl/mosaic_tpu/builder.zig deleted file mode 100644 index cffcd658..00000000 --- a/dsl/mosaic_tpu/builder.zig +++ /dev/null @@ -1,2248 +0,0 @@ -//! ZML Mosaic TPU DSL — a Zig builder for Mosaic TPU IR (the `tpu` dialect). -//! -//! Args are memrefs (`!memref`) plus optional scalars, -//! iteration-index, and semaphore args. The dialect must be registered into -//! the MLIR context — see `mlir/dialects/mosaic_tpu`. The entire Mosaic stack is -//! backed by the Mosaic dialect in the pinned XLA repository. - -const std = @import("std"); - -const affine = @import("mlir/dialects/affine"); -const arith = @import("mlir/dialects/arith"); -const cf = @import("mlir/dialects/cf"); -const func = @import("mlir/dialects/func"); -const math = @import("mlir/dialects/math"); -const memref = @import("mlir/dialects/memref"); -const scf = @import("mlir/dialects/scf"); -const vector = @import("mlir/dialects/vector"); -/// `vector`-dialect enums re-exported for convenience. -/// Used by `vectorReduction` / `multiReduction`. -pub const CombiningKind = vector.CombiningKind; -const common = @import("dsl/common/builder"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); -const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); -/// `tpu`-dialect enums re-exported for convenience. -pub const MemorySpace = mosaic_tpu.MemorySpace; -pub const ReductionKind = mosaic_tpu.ReductionKind; -pub const ContractPrecision = mosaic_tpu.ContractPrecision; -pub const RoundingMode = mosaic_tpu.RoundingMode; -pub const DimensionSemantics = mosaic_tpu.DimensionSemantics; -pub const CoreType = mosaic_tpu.CoreType; -pub const PipelineMode = mosaic_tpu.PipelineMode; -pub const RevisitMode = mosaic_tpu.RevisitMode; - -const dtypes = @import("dtype.zig"); -pub const DType = dtypes.DType; -const isFloatDtype = dtypes.isFloatDtype; -const dtypeBitwidth = dtypes.dtypeBitwidth; -const intBitwidth = dtypes.intBitwidth; - -pub const pipeline = @import("pipeline.zig"); - -// ============================================================================= -// Value — a handle to an MLIR value inside the kernel body. -// ============================================================================= - -/// MLIR SSA value with an optional back-pointer to its `Builder` for fluent ops. -pub const Value = struct { - inner: *const mlir.Value, - kernel: ?*Builder = null, - - pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); - - pub fn type_(self: Value) *const mlir.Type { - return self.inner.type_(); - } - - fn kern(self: Value) *Builder { - return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); - } - - /// Element type of a vector / memref value, or the value's own type if - /// scalar. - pub fn elemType(self: Value) *const mlir.Type { - if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.elementType(); - return self.type_(); - } - - /// True when this value is a vector (rank ≥ 1). - pub fn isVector(self: Value) bool { - return self.type_().isA(mlir.VectorType) != null; - } - - /// True when this value is a memref. - pub fn isMemRef(self: Value) bool { - return self.type_().isA(mlir.MemRefType) != null; - } - - /// True when the value is shaped (vector / memref / tensor). - pub fn isShaped(self: Value) bool { - return self.type_().isA(mlir.ShapedType) != null; - } - - /// Rank of this value. Scalars have rank 0. - pub fn rank(self: Value) usize { - if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.rank(); - return 0; - } - - /// `i`-th dim size of a shaped value; traps on scalar. - pub fn dim(self: Value, i: usize) i64 { - return self.type_().isA(mlir.ShapedType).?.dimension(i); - } - - /// Shape as a stack-allocated BoundedArray. Use `.constSlice()` for `[]const i64`. - pub fn shape(self: Value) Shape { - var out: Shape = .empty; - if (self.type_().isA(mlir.ShapedType)) |shaped| { - const r = shaped.rank(); - for (0..r) |i| out.appendAssumeCapacity(shaped.dimension(i)); - } - return out; - } - - /// True if the element type is a floating-point type (f16/bf16/f32/f64/fp8/…). - pub fn isFloatElem(self: Value) bool { - const et = self.elemType(); - inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; - } - return false; - } - - /// True if the element type is an integer type. - pub fn isIntElem(self: Value) bool { - return self.elemType().isA(mlir.IntegerType) != null; - } - - // -------- fluent arithmetic (int/float auto-dispatch from elem type) -------- - - pub fn add(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); - } - - pub fn sub(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); - } - - pub fn mul(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); - } - - pub fn div(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.divf(l, r) else k.divsi(l, r); - } - - pub fn rem(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.remf(l, r) else k.remsi(l, r); - } - - pub fn bitAnd(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.andi(l, r); - } - - pub fn bitOr(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return k.ori(l, r); - } - - pub fn minimum(self: Value, rhs: anytype) Value { - return self.kern().minimum(self, rhs); - } - pub fn maximum(self: Value, rhs: anytype) Value { - return self.kern().maximum(self, rhs); - } - - // -------- fluent comparisons (int predicates default; float uses cmpf) -------- - - pub fn lt(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.cmpf(.olt, l, r) else k.cmpi(.slt, l, r); - } - pub fn le(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.cmpf(.ole, l, r) else k.cmpi(.sle, l, r); - } - pub fn gt(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.cmpf(.ogt, l, r) else k.cmpi(.sgt, l, r); - } - pub fn ge(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.cmpf(.oge, l, r) else k.cmpi(.sge, l, r); - } - pub fn eq(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.cmpf(.oeq, l, r) else k.cmpi(.eq, l, r); - } - pub fn ne(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.coerce(self, rhs); - return if (l.isFloatElem()) k.cmpf(.one, l, r) else k.cmpi(.ne, l, r); - } - - // -------- casts -------- - - /// Numeric cast preserving shape (vector or scalar). Auto-dispatches on - /// source / destination float-vs-int. - pub fn to(self: Value, dtype: DType) Value { - return self.kern().cast(self, dtype); - } - - /// `tpu.bitcast` — same-bitwidth element-type swap. - pub fn bitCast(self: Value, dtype: DType) Value { - return self.kern().bitCast(self, dtype); - } - - /// Absolute value (float→absf, int→absi). - pub fn abs(self: Value) Value { - return self.kern().abs(self); - } - - /// Best-effort fold to an `i64` for `arith.constant` (and `addi`/`muli` of folded - /// operands). Returns `null` for anything else; the caller decides what to do. - /// TODO(raph): check if there is no better way - pub fn asConstantInt(self: Value) ?i64 { - if (self.inner.isA(mlir.OpResult) == null) return null; - - const op = self.inner.owner(); - const op_name = op.name(); - - if (std.mem.eql(u8, op_name, "arith.constant")) { - const attr = op.attributeByName("value") orelse return null; - const iattr = attr.isA(mlir.IntegerAttribute) orelse return null; - return iattr.value(i64); - } - - if (std.mem.eql(u8, op_name, "arith.muli")) { - const lhs: Value = .{ .inner = op.operand(0), .kernel = self.kernel }; - const rhs: Value = .{ .inner = op.operand(1), .kernel = self.kernel }; - const lv = lhs.asConstantInt(); - const rv = rhs.asConstantInt(); - if (lv) |l| if (rv) |r| return l *% r; - // One side null, the other known: still folds if the known side is 0. - if (lv) |l| if (l == 0) return 0; - if (rv) |r| if (r == 0) return 0; - return null; - } - - if (std.mem.eql(u8, op_name, "arith.addi")) { - const lhs: Value = .{ .inner = op.operand(0), .kernel = self.kernel }; - const rhs: Value = .{ .inner = op.operand(1), .kernel = self.kernel }; - const lv = lhs.asConstantInt() orelse return null; - const rv = rhs.asConstantInt() orelse return null; - return lv +% rv; - } - - return null; - } -}; - -// ============================================================================= -// Arg spec — describes one function parameter. -// ============================================================================= - -/// Per-arg specification for building `func.func`'s signature. -pub const ArgSpec = struct { - name: []const u8, - kind: Kind, - - pub const Kind = union(enum) { - /// Plain scalar — typically used for grid sizes / dynamic dims passed - /// in by the runtime. `.{ .scalar = .i32 }`. - scalar: DType, - /// Loop / grid iteration index — `index` type with optional - /// `tpu.dimension_semantics` annotation. - /// `.{ .iter = .{ .semantics = .parallel } }`. - iter: IterOpts, - /// Memref (the bread-and-butter kernel arg). The arg is typed as - /// `memref>`. - /// `.{ .ref = .{ .shape = &.{128, 256}, .dtype = .bf16, .memory_space = .vmem } }`. - ref: RefSpec, - /// `memref` — the - /// scratch-array form Pallas uses for semaphore pools. Pallas - /// classifies it as `.scratch`, so it never appears in - /// `window_params` / `transform_N`. - sem_array: SemArraySpec, - /// `!tpu.semaphore` or `!tpu.dma_semaphore`. - /// `.{ .sem = .regular }` / `.{ .sem = .dma }`. - sem: SemKind, - }; - - pub const IterOpts = struct { - semantics: ?DimensionSemantics = null, - }; - - pub const RefSpec = struct { - shape: []const i64, - dtype: DType, - memory_space: MemorySpace = .vmem, - /// Emit `{sc.persistent}` on this memref argument. Sparse-core Pallas - /// kernels mark persistent HBM operands this way. - persistent: bool = false, - /// Block window. `null` ⇒ trivial window (block == operand shape); - /// VMEM trivial windows auto-get `pipeline_mode = synchronous`. - window: ?WindowSpec = null, - /// Pallas role of this operand. Only `.input`/`.output` get a - /// `window_params` entry and a `transform_N` stub. - /// `.scalar_prefetch` refs become trailing args of every - /// `transform_N` body (Pallas's index_map sees them). - /// `.scratch` refs are not visible to transforms or window_params. - role: Role = .input, - }; - - pub const Role = enum { input, output, scratch, scalar_prefetch }; - - pub const WindowSpec = struct { - /// `null` ⇒ use the operand's full shape (trivial window). - /// Determines `window_bounds` and the `transform_N` arity. - block_shape: ?[]const i64 = null, - /// `#tpu.pipeline_mode<…>`. `null` ⇒ auto-`synchronous` for VMEM trivial windows, omitted otherwise. - pipeline_mode: ?PipelineMode = null, - /// `#tpu.revisit_mode<…>`. `null` ⇒ omit. - revisit_mode: ?RevisitMode = null, - /// `#tpu.element_window<[pad_low], [pad_high]>`. `null` ⇒ omit. - element_window: ?ElementWindowSpec = null, - /// What `transform_N` returns. `null` ⇒ all-zero `block_rank` i32s - /// (matching a trivial index_map). One entry per result; length - /// must match `block_shape.len` (or operand rank if `block_shape` - /// is `null`). - transform_returns: ?[]const TransformReturn = null, - }; - - /// One return slot of a `transform_N` body. Mirrors what - /// `lower_jaxpr_to_transform_func` produces — typically a select of - /// program-ids, scalar-prefetch loads, or constants. We only encode - /// the most common forms; drop to a custom transform body for anything - /// fancier. - pub const TransformReturn = union(enum) { - /// Return `arith.constant 0 : i32`. - zero, - /// Return the Nth program-id (`%argN`). - program_id: usize, - }; - - pub const ElementWindowSpec = struct { - pad_low: []const i64, - pad_high: []const i64, - }; - - pub const SemKind = enum { regular, dma }; - - pub const SemArraySpec = struct { - shape: []const i64, - kind: SemKind = .dma, - }; -}; - -pub const FinishError = error{ InvalidMlir, MlirUnexpected } || std.mem.Allocator.Error || std.Io.Writer.Error; - -// ============================================================================= -// Named-param structs. -// ============================================================================= - -pub const VectorLoadOpts = struct { - /// Per-dim load stride. Pass `&.{}` for unit-strided. - strides: []const i32 = &.{}, - /// Optional element-wise mask. Must be a `vector<...xi1>` matching the - /// load result shape. - mask: ?Value = null, -}; - -pub const VectorStoreOpts = struct { - strides: []const i32 = &.{}, - mask: ?Value = null, - /// If true emits `tpu.vector_store ... add = true` (atomic-add semantics - /// where supported). - add: bool = false, -}; - -pub const MatmulOpts = struct { - transpose_lhs: bool = false, - transpose_rhs: bool = false, - transpose_lhs_hint: bool = false, - precision: ?ContractPrecision = null, - /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the - /// canonicalizer derives one. Build with `mosaic_tpu.dotDimensionNumbersAttr`. - dimension_numbers: ?*const mlir.Attribute = null, -}; - -pub const ReduceOpts = struct { - /// Reduction dim. Defaults to 0. - dim: i64 = 0, -}; - -pub const SortOpts = struct { - descending: bool = false, -}; - -pub const SemSignalOpts = struct { - device_id: ?Value = null, - core_id: ?Value = null, - subcore_id: ?Value = null, -}; - -pub const EnqueueDmaOpts = struct { - source_semaphore: ?Value = null, - device_id: ?Value = null, - core_id: ?Value = null, - subcore_id: ?Value = null, - priority: i32 = 0, - strict_ordering: bool = false, -}; - -pub const WaitDma2Opts = struct { - device_id: ?Value = null, - core_id: ?Value = null, - strict_ordering: bool = false, -}; - -pub const ReciprocalOpts = struct { - approx: bool = false, - full_range: bool = false, -}; - -pub const RotateOpts = struct { - amount: i32, - dimension: i32, - stride: ?i32 = null, - stride_dimension: ?i32 = null, -}; - -pub const CastOpts = struct { - /// Forwarded to `tpu.fptosi` / `tpu.fptoui` / etc. - rounding: ?RoundingMode = null, - /// If true, perform a same-bitwidth `tpu.bitcast` instead of a numeric cast. - bitcast: bool = false, -}; - -pub const MinMaxOpts = struct { - /// IEEE NaN-propagation toggle. Default is non-propagating (maxnumf/minnumf). - propagate_nan: bool = false, -}; - -// ============================================================================= -// Builder — the main DSL builder. -// ============================================================================= - -pub fn registerDialects(registry: *mlir.DialectRegistry) void { - inline for (.{ func, mosaic_tpu, arith, scf, math, cf, memref, vector, affine }) |dialect| dialect.registerDialect(registry); -} - -/// The main DSL builder. Create with `init`, populate the body with helpers, -/// then call `finish` to get the IR string. -pub const Builder = struct { - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - ctx: *mlir.Context, - module: *mlir.Module, - /// Name of the kernel — used when `declareArgs` creates the `func.func`. - name: []const u8, - /// `func.func` op for this kernel. `null` between `open` and the first - /// `declareArgs` call; populated thereafter. - func_op: ?*mlir.Operation, - /// Entry block for the `func.func`. `null` between `open` and the first - /// `declareArgs` call; populated thereafter. - entry_block: ?*mlir.Block, - block_stack: common.BlockStack = .{}, - /// Cached `arith.constant 0 : index` Value (and the block it lives - /// in) so repeat no-index loads / stores share an SSA value. See - /// `zeroIndex` for the lookup logic. - zero_index: ?Value = null, - zero_index_block: ?*mlir.Block = null, - /// Snapshot of the kernel's ref-args saved at `declareArgsLowOpts` - /// time. `finish` walks this list to emit `transform_N` stubs when - /// `pallas_window_params=true`. Allocated in `arena`. - ref_args: []RefArgSnapshot = &.{}, - /// SMEM scalar-prefetch ref types — appended to every `transform_N` - /// body's argument list. Empty when no `.scalar_prefetch` refs were - /// declared. - prefetch_args: []const *const mlir.Type = &.{}, - /// Number of grid program-ids (i32) at the head of every `transform_N` - /// body. Equals `Opts.dimension_semantics.len`. - grid_dim_count: usize = 0, - /// Whether to emit Pallas-style `transform_N` stubs at `finish` time. - /// Set by `declareArgsLowOpts` from `Opts.pallas_window_params`. - emit_transform_stubs: bool = false, - /// Optional prefix for generated transform function names. - transform_name_prefix: ?[]const u8 = null, - /// Whether generated transform functions carry `{sc.is_transform_indices}`. - sc_transform_indices: bool = false, - - /// Per-ref-arg snapshot for `finish`-time `transform_N` stub emission. - /// `skipped` = ANY/HBM/SEMAPHORE memory space (empty window, no transform). - pub const RefArgSnapshot = struct { - block_rank: i64, - skipped: bool, - role: ArgSpec.Role, - transform_returns: ?[]const ArgSpec.TransformReturn = null, - }; - - /// Snapshot of the SMEM scalar-prefetch ref types (after the program-ids) - /// — repeated as trailing args of every `transform_N` body to mirror - /// Pallas's `lower_jaxpr_to_transform_func`. - pub const PrefetchSnapshot = struct { - /// Pre-built MLIR memref types (cached pointers from declareArgs time). - types: []const *const mlir.Type, - }; - - /// Builder-level options. `noinline` is a Zig keyword — write `.{ .@"noinline" = true }`. - pub const Opts = struct { - @"noinline": bool = false, - /// `tpu.core_type` on the `func.func`. `null` skips the attr. - core_type: ?CoreType = .tc, - /// One entry per grid dim. Empty list emits an empty array attr - /// (matching Pallas's `dimension_semantics = []`). - dimension_semantics: []const DimensionSemantics = &.{}, - /// Number of leading scalar-prefetch operands. - scalar_prefetch: i64 = 0, - /// Number of trailing scratch refs (memrefs holding kernel-local - /// scratch state). - scratch_operands: i64 = 0, - /// Pallas's `iteration_bounds = array` — the grid shape - /// (one entry per `dimension_semantics`). `null` omits the attr. - iteration_bounds: ?[]const i64 = null, - /// Emit `scalar_prefetch = N : i64`. Sparse-core kernels with no - /// scalar prefetches omit the zero-valued attr. - emit_scalar_prefetch_attr: bool = true, - /// When `true`, auto-emit `window_params = [...]` and trailing - /// `transform_N` stub funcs at `finish` time. Per-arg behavior - /// matches `lowering.py:1014–1131`: ANY/HBM/SEMAPHORE refs get - /// an empty `{}` and no transform; VMEM trivial windows - /// auto-force `pipeline_mode = synchronous`. Override per-arg - /// via `RefSpec.window`. - pallas_window_params: bool = false, - /// Include HBM refs in Pallas-style `window_params`. Tensor-core - /// Pallas lowering skips HBM; sparse-core lowering includes it. - window_hbm_refs: bool = false, - /// Optional prefix for generated transform function names. - transform_name_prefix: ?[]const u8 = null, - /// Add `{sc.is_transform_indices}` to generated transform stubs. - sc_transform_indices: bool = false, - }; - - /// Create a sub-module containing a single public `func.func` of the - /// given name, signature, and result types. - pub fn init( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - name: []const u8, - args: []const ArgSpec, - result_types: []const *const mlir.Type, - ) !Builder { - return initOpts(allocator, ctx, name, args, result_types, .{}); - } - - pub fn initOpts( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - name: []const u8, - args: []const ArgSpec, - result_types: []const *const mlir.Type, - opts: Opts, - ) !Builder { - var b = try open(allocator, ctx, name); - errdefer b.deinit(); - try b.declareArgsLowOpts(args, result_types, opts); - return b; - } - - /// Create an empty `Builder` with no `func.func` yet. - pub fn open( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - name: []const u8, - ) !Builder { - const unknown_loc: *const mlir.Location = .unknown(ctx); - const module: *mlir.Module = .init(unknown_loc); - errdefer module.deinit(); - - var arena: std.heap.ArenaAllocator = .init(allocator); - errdefer arena.deinit(); - - return .{ - .allocator = allocator, - .arena = arena, - .ctx = ctx, - .module = module, - .name = name, - .func_op = null, - .entry_block = null, - .block_stack = .{}, - }; - } - - /// Declare the kernel's args from a named-field struct literal and - /// return a `NamedArgs(Spec)`. - pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { - return self.declareArgsOpts(spec, &.{}, .{}); - } - - pub fn declareArgsOpts( - self: *Builder, - spec: anytype, - result_types: []const *const mlir.Type, - opts: Opts, - ) !common.NamedArgs(@TypeOf(spec), Value) { - const Spec = @TypeOf(spec); - const field_names = @typeInfo(Spec).@"struct".field_names; - - var arg_specs: [field_names.len]ArgSpec = undefined; - inline for (field_names, 0..) |field_name, i| { - const raw = @field(spec, field_name); - const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; - const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); - const inner = @field(raw, variant); - break :blk switch (tag) { - .scalar => .{ .scalar = inner }, - .iter => .{ .iter = .{ - .semantics = if (@hasField(@TypeOf(inner), "semantics")) inner.semantics else null, - } }, - .ref => .{ .ref = .{ - .shape = inner.shape, - .dtype = inner.dtype, - .memory_space = if (@hasField(@TypeOf(inner), "memory_space")) inner.memory_space else .vmem, - .persistent = if (@hasField(@TypeOf(inner), "persistent")) inner.persistent else false, - .window = if (@hasField(@TypeOf(inner), "window")) inner.window else null, - .role = if (@hasField(@TypeOf(inner), "role")) inner.role else .input, - } }, - .sem_array => .{ .sem_array = .{ - .shape = inner.shape, - .kind = if (@hasField(@TypeOf(inner), "kind")) inner.kind else .dma, - } }, - .sem => .{ .sem = inner }, - }; - }; - arg_specs[i] = .{ .name = field_name, .kind = kind }; - } - - try self.declareArgsLowOpts(&arg_specs, result_types, opts); - - var named: common.NamedArgs(Spec, Value) = undefined; - inline for (field_names, 0..) |field_name, i| { - @field(named, field_name) = self.arg(i); - } - return named; - } - - fn declareArgsLowOpts( - self: *Builder, - args: []const ArgSpec, - result_types: []const *const mlir.Type, - opts: Opts, - ) !void { - std.debug.assert(self.entry_block == null); - const ctx = self.ctx; - const unknown_loc: *const mlir.Location = .unknown(ctx); - const scratch = self.arena.allocator(); - - const arg_types = try scratch.alloc(*const mlir.Type, args.len); - const arg_locs = try scratch.alloc(*const mlir.Location, args.len); - for (args, 0..) |a, i| { - arg_types[i] = switch (a.kind) { - .scalar => |dt| dt.toMlir(ctx), - .iter => .index(ctx), - .ref => |r| .memRef( - r.dtype.toMlir(ctx), - r.shape, - null, - r.memory_space.attribute(ctx), - ), - .sem => |s| switch (s) { - .regular => mosaic_tpu.semaphoreType(ctx), - .dma => mosaic_tpu.dmaSemaphoreType(ctx), - }, - .sem_array => |sa| .memRef( - switch (sa.kind) { - .regular => mosaic_tpu.semaphoreType(ctx), - .dma => mosaic_tpu.dmaSemaphoreType(ctx), - }, - sa.shape, - null, - MemorySpace.semaphore_mem.attribute(ctx), - ), - }; - arg_locs[i] = unknown_loc.named(ctx, a.name); - } - - const entry = mlir.Block.init(arg_types, arg_locs); - - // Build per-arg attribute dicts. - const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); - var any_arg_attr = false; - for (args, 0..) |a, i| { - const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); - switch (a.kind) { - .iter => |io| { - if (io.semantics) |sem| { - arg_attrs[i] = .dict(ctx, &.{ - mosaic_tpu.dimensionSemanticsNamedAttr(ctx, sem), - }); - any_arg_attr = true; - } else { - arg_attrs[i] = empty_dict; - } - }, - .ref => |r| { - if (r.persistent) { - arg_attrs[i] = .dict(ctx, &.{ - .named(ctx, "sc.persistent", .unit(ctx)), - }); - any_arg_attr = true; - } else { - arg_attrs[i] = empty_dict; - } - }, - else => arg_attrs[i] = empty_dict, - } - } - - // Kernel-level func.func attributes. MLIR sorts attrs alphabetically on print. - var extra_attrs: stdx.BoundedArray(mlir.NamedAttribute, 12) = .empty; - const dim_sem_buf = try scratch.alloc(*const mlir.Attribute, opts.dimension_semantics.len); - for (opts.dimension_semantics, 0..) |s, i| dim_sem_buf[i] = s.attribute(ctx); - extra_attrs.appendAssumeCapacity(.named(ctx, "dimension_semantics", .array(ctx, dim_sem_buf))); - if (opts.iteration_bounds) |bounds| { - extra_attrs.appendAssumeCapacity(.named(ctx, "iteration_bounds", .denseArray(ctx, .i64, bounds))); - } - if (opts.emit_scalar_prefetch_attr) { - extra_attrs.appendAssumeCapacity(.named(ctx, "scalar_prefetch", .int(ctx, .i64, opts.scalar_prefetch))); - } - extra_attrs.appendAssumeCapacity(.named(ctx, "scratch_operands", .int(ctx, .i64, opts.scratch_operands))); - if (opts.core_type) |ct| { - extra_attrs.appendAssumeCapacity(mosaic_tpu.coreTypeNamedAttr(ctx, ct)); - } - - // Snapshot the ref args for `finish`-time stub emission. Only - // `.input` / `.output` refs participate in `window_params` and - // `transform_N` — `.scalar_prefetch` and `.scratch` are ignored - // (they're tracked separately for transform-arg expansion). - const arena_alloc = self.arena.allocator(); - var ref_args_buf: std.ArrayList(RefArgSnapshot) = .empty; - var prefetch_types_buf: std.ArrayList(*const mlir.Type) = .empty; - for (args, 0..) |a, i| switch (a.kind) { - .ref => |r| { - if (r.role == .input or r.role == .output) { - const block_shape = if (r.window) |w| - (w.block_shape orelse r.shape) - else - r.shape; - const skipped = isSkippedWindowSpace(r.memory_space) and !(opts.window_hbm_refs and r.memory_space == .hbm); - try ref_args_buf.append(arena_alloc, .{ - .block_rank = @intCast(block_shape.len), - .skipped = skipped, - .role = r.role, - .transform_returns = if (r.window) |w| w.transform_returns else null, - }); - } else if (r.role == .scalar_prefetch) { - try prefetch_types_buf.append(arena_alloc, arg_types[i]); - } - }, - else => {}, - }; - self.ref_args = try arena_alloc.dupe(RefArgSnapshot, ref_args_buf.items); - self.prefetch_args = try arena_alloc.dupe(*const mlir.Type, prefetch_types_buf.items); - self.grid_dim_count = opts.dimension_semantics.len; - self.emit_transform_stubs = opts.pallas_window_params; - self.transform_name_prefix = opts.transform_name_prefix; - self.sc_transform_indices = opts.sc_transform_indices; - - // window_params: one entry per `.input` / `.output` ref. - // ANY/HBM/SEMAPHORE → empty dict, no transform_N. - // Otherwise → {transform_indices, window_bounds, [pipeline_mode], [revisit_mode], [window_kind]}. - // VMEM trivial-window → auto-force pipeline_mode = synchronous. - if (opts.pallas_window_params) { - var idx: usize = 0; - const wp_buf = try scratch.alloc(*const mlir.Attribute, self.ref_args.len); - for (args) |a| switch (a.kind) { - .ref => |r| { - if (r.role != .input and r.role != .output) continue; - if (isSkippedWindowSpace(r.memory_space) and !(opts.window_hbm_refs and r.memory_space == .hbm)) { - wp_buf[idx] = .dict(ctx, &.{}); - idx += 1; - continue; - } - - const w = r.window orelse ArgSpec.WindowSpec{}; - const block_shape = w.block_shape orelse r.shape; - // `bm.has_trivial_window()` in pallas — block_shape == - // operand_shape AND the index_map is the identity - // (returns all zeros). - const has_trivial_index_map = if (w.transform_returns) |trs| blk: { - for (trs) |tr| { - if (tr != .zero) break :blk false; - } - break :blk true; - } else true; - const trivial = std.mem.eql(i64, block_shape, r.shape) and has_trivial_index_map; - - const resolved_pipeline: ?PipelineMode = - w.pipeline_mode orelse blk: { - // VMEM + trivial window ⇒ Buffered(1) ⇒ synchronous. - if (r.memory_space == .vmem and trivial) break :blk .synchronous; - break :blk null; - }; - - const sym_name = if (opts.transform_name_prefix) |prefix| - try std.fmt.allocPrint(scratch, "{s}_transform_{d}", .{ prefix, idx }) - else - try std.fmt.allocPrint(scratch, "transform_{d}", .{idx}); - - var inner: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (resolved_pipeline) |pm| { - inner.appendAssumeCapacity(.named(ctx, "pipeline_mode", pm.attribute(ctx))); - } - if (w.revisit_mode) |rm| { - inner.appendAssumeCapacity(.named(ctx, "revisit_mode", rm.attribute(ctx))); - } - inner.appendAssumeCapacity(.named(ctx, "transform_indices", .flatSymbolRef(ctx, sym_name))); - inner.appendAssumeCapacity(.named(ctx, "window_bounds", .denseArray(ctx, .i64, block_shape))); - if (w.element_window) |ew| { - inner.appendAssumeCapacity(.named(ctx, "window_kind", mosaic_tpu.elementWindowAttr(ctx, ew.pad_low, ew.pad_high))); - } - wp_buf[idx] = .dict(ctx, inner.constSlice()); - idx += 1; - }, - else => {}, - }; - extra_attrs.appendAssumeCapacity(.named(ctx, "window_params", .array(ctx, wp_buf))); - } - - const func_op = func.func(ctx, .{ - .name = self.name, - .args = arg_types, - .args_attributes = if (any_arg_attr) arg_attrs else null, - .results = result_types, - .block = entry, - .location = unknown_loc, - .no_inline = opts.@"noinline", - // Body is empty at declare time — verify only at `finish`. - .verify = false, - .extra_attributes = extra_attrs.constSlice(), - }); - _ = func_op.appendTo(self.module.body()); - - self.func_op = func_op; - self.entry_block = entry; - self.block_stack.push(self.arena.allocator(), entry); - } - - pub fn deinit(self: *Builder) void { - self.module.deinit(); - self.arena.deinit(); - } - - /// The i-th entry block argument. - pub fn arg(self: *Builder, i: usize) Value { - const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); - return .{ .inner = eb.argument(i), .kernel = self }; - } - - pub fn pushBlock(self: *Builder, b: *mlir.Block) void { - self.block_stack.push(self.arena.allocator(), b); - } - - pub fn popBlock(self: *Builder) void { - self.block_stack.pop(); - } - - pub fn currentBlock(self: *Builder) *mlir.Block { - return self.block_stack.current(); - } - - /// Unpack a slice of `Value`s into an arena-allocated slice of raw MLIR - /// value pointers. - fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { - const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); - for (values, 0..) |v, i| out[i] = v.inner; - return out; - } - - /// Emit `op` into the current block and return its first result as a Value. - pub fn emit(self: *Builder, op: *mlir.Operation) Value { - _ = op.appendTo(self.currentBlock()); - return .{ .inner = op.result(0), .kernel = self }; - } - - pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { - _ = op.appendTo(self.currentBlock()); - const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); - for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; - return out; - } - - pub fn yield(self: *Builder, values: anytype) []const Value { - const T = @TypeOf(values); - const info = @typeInfo(T); - if (info != .@"struct" or !info.@"struct".is_tuple) - @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".field_names.len; - const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { - if (Field != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); - out[i] = @field(values, field_name); - } - return out; - } - - pub fn loc(self: *const Builder) *const mlir.Location { - return .unknown(self.ctx); - } - - // ==================== type helpers ==================== - - pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { - return dtype.toMlir(self.ctx); - } - - pub fn vectorTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { - return .vector(shape, dtype.toMlir(self.ctx)); - } - - pub fn memRefTy(self: *const Builder, shape: []const i64, dtype: DType, memory_space: MemorySpace) *const mlir.Type { - return .memRef(dtype.toMlir(self.ctx), shape, null, memory_space.attribute(self.ctx)); - } - - fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { - return dtypes.mlirElemToDType(self.ctx, elem); - } - - /// Replace `src`'s element type with `out_dtype`, preserving rank/shape. - fn swapElem(self: *const Builder, src: Value, out_dtype: DType) *const mlir.Type { - const out_elem = out_dtype.toMlir(self.ctx); - if (src.isVector()) return .vector(src.shape().constSlice(), out_elem); - if (src.isMemRef()) { - const mr = src.type_().isA(mlir.MemRefType).?; - return .memRef(out_elem, src.shape().constSlice(), null, mr.memorySpace()); - } - return out_elem; - } - - // ==================== constants ==================== - - fn emitInt(self: *Builder, dtype: DType, value: i64) Value { - return self.emit(arith.constant_int(self.ctx, value, dtype.toMlir(self.ctx), self.loc())); - } - - fn emitFloat(self: *Builder, dtype: DType, value: f64) Value { - return switch (dtype) { - .f16 => self.emit(arith.constant_float(self.ctx, value, .f16, self.loc())), - .bf16 => self.emit(arith.constant_float(self.ctx, value, .bf16, self.loc())), - .f32 => self.emit(arith.constant_float(self.ctx, value, .f32, self.loc())), - .f64 => self.emit(arith.constant_float(self.ctx, value, .f64, self.loc())), - // fp8 has no direct arith.constant; emit f32 then truncf. - else => self.arithTruncf(self.emitFloat(.f32, value), dtype), - }; - } - - /// Constant `index`-typed value (for memref / loop bounds). - pub fn constIndex(self: *Builder, value: i64) Value { - return self.emit(arith.constant_index(self.ctx, value, self.loc())); - } - - /// `openFor`'s default lower bound, an `index` constant. - fn indexBound(self: *Builder, value: anytype) Value { - return self.constIndex(intBits(value, 64, "index")); - } - - /// Lift a Zig scalar (or pass-through Value) to a DSL constant. - pub fn lift(self: *Builder, value: anytype) Value { - const T = @TypeOf(value); - if (T == Value) return value; - return switch (@typeInfo(T)) { - .comptime_int => blk: { - if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) - break :blk self.emitInt(.i32, value); - if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) - break :blk self.emitInt(.i64, value); - if (value >= 0 and value <= std.math.maxInt(u64)) - break :blk self.emitInt(.i64, @bitCast(@as(u64, value))); - @compileError("Builder.lift: integer literal out of 64-bit range"); - }, - .int => |info| blk: { - const signed = info.signedness == .signed; - if (info.bits > 32) { - const v: i64 = if (signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))); - break :blk self.emitInt(.i64, v); - } - const v32: i32 = if (signed) @intCast(value) else @bitCast(@as(u32, @intCast(value))); - break :blk self.emitInt(.i32, v32); - }, - .comptime_float => self.emitFloat(.f32, @floatCast(value)), - .float => |info| switch (info.bits) { - 16 => self.emitFloat(.f16, @floatCast(value)), - 32 => self.emitFloat(.f32, @floatCast(value)), - 64 => self.emitFloat(.f64, value), - else => @compileError("Builder.lift: unsupported float bitwidth"), - }, - else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), - }; - } - - /// `value` as a constant of `dtype`; a Value passes through. An integer is - /// signed or an unsigned bit pattern of `dtype`'s width; anything else - /// panics. - pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { - const T = @TypeOf(value); - if (T == Value) return value; - if (isFloatDtype(dtype)) { - const v_f64: f64 = switch (@typeInfo(T)) { - .comptime_int, .int => @floatFromInt(value), - .comptime_float, .float => @floatCast(value), - else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), - }; - return self.emitFloat(dtype, v_f64); - } - const v: i128 = switch (@typeInfo(T)) { - .comptime_int, .int => value, - .comptime_float, .float => @intFromFloat(value), - else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), - }; - return self.emitInt(dtype, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); - } - - /// `value` as the `i64` an integer constant of `bits` bits is built from. - /// It may be signed or an unsigned bit pattern, so it must lie in - /// [-2^(bits-1), 2^bits). - fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { - if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) - std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); - return @truncate(value); - } - - /// `value` as a constant of `like`'s element type; a Value passes through. - pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { - if (@TypeOf(value) == Value) return value; - return self.constMatching(value, like.elemType()); - } - - /// Build an arith.constant of the given MLIR element type from a Zig - /// numeric. Used for matching the type of an existing reference value. - fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { - if (elem.eql(.index(self.ctx))) return self.constIndex(intBits(value, 64, "index")); - return self.constant(self.mlirElemToDType(elem), value); - } - - fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { - const T = @TypeOf(value); - if (T == Value) return value; - return switch (@typeInfo(T)) { - .comptime_int, .comptime_float => self.constMatching(value, ref_elem), - else => self.lift(value), - }; - } - - /// Splat a scalar to a vector. Zig scalars take the dense-splat fast path - /// (`arith.constant dense`); a `Value` falls back to `vector.broadcast`. - pub fn splat(self: *Builder, value: anytype, shape: []const i64, dtype: DType) Value { - const T = @TypeOf(value); - const ty = mlir.Type.vector(shape, dtype.toMlir(self.ctx)); - if (T == Value) return self.emit(vector.broadcast(self.ctx, value.inner, ty, self.loc())); - if (isFloatDtype(dtype)) { - const v_f64: f64 = switch (@typeInfo(T)) { - .comptime_int, .int => @floatFromInt(value), - .comptime_float, .float => @floatCast(value), - else => @compileError("Builder.splat: unsupported scalar " ++ @typeName(T)), - }; - return self.emit(arith.constant_dense_splat_f(self.ctx, v_f64, ty, self.loc())); - } - const v_i64: i64 = switch (@typeInfo(T)) { - .comptime_int => @intCast(value), - .int => |info| if (info.signedness == .signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))), - .comptime_float, .float => @intFromFloat(value), - else => @compileError("Builder.splat: unsupported scalar " ++ @typeName(T)), - }; - return self.emit(arith.constant_dense_splat_i(self.ctx, v_i64, ty, self.loc())); - } - - /// Zero-valued vector of `shape x dtype`. - pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { - return self.splat(@as(i64, 0), shape, dtype); - } - - pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { - return self.splat(@as(i64, 1), shape, dtype); - } - - pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { - return self.splat(value, shape, dtype); - } - - /// Lift scalars and broadcast scalar → vector to match the other operand. - pub fn coerce(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { - const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; - const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; - var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); - var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); - - // integer-width promotion (int family only). - if (av.isIntElem() and bv.isIntElem()) { - const a_dt = self.mlirElemToDType(av.elemType()); - const b_dt = self.mlirElemToDType(bv.elemType()); - const aw = intBitwidth(a_dt); - const bw = intBitwidth(b_dt); - if (aw < bw) av = av.to(b_dt); - if (bw < aw) bv = bv.to(a_dt); - } - - // scalar↔vector splat. - if (av.isVector() and !bv.isVector()) { - const dt = self.mlirElemToDType(av.elemType()); - bv = self.splat(bv, av.shape().constSlice(), dt); - } - if (!av.isVector() and bv.isVector()) { - const dt = self.mlirElemToDType(bv.elemType()); - av = self.splat(av, bv.shape().constSlice(), dt); - } - - return .{ av, bv }; - } - - // ==================== arith: binary ==================== - - pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.addi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.subi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.muli(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.addf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.subf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.mulf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.divf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn divsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.divsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn divui(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.divui(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - /// Mirrors `jax.numpy.floor_divide` (`jax/_src/numpy/ufuncs.py:2487`): - /// truncating `divsi` plus the sign-correcting expansion - /// `q - 1 if (sign(a) != sign(b) and rem != 0) else q`. - /// `lhs` must be a vector of i32. `rhs` is `anytype` — passing a - /// scalar comptime int produces the dense<...> splat Pallas emits. - /// JAX's MLIR canonicalizer does not fold the expansion away even - /// when one operand is a constant, so this helper is needed for - /// IR-equivalent floor-div with `pallas_call`. - pub fn divFloor(self: *Builder, lhs: Value, rhs: anytype) Value { - const shape = lhs.shape().constSlice(); - const rhs_v: Value = if (@TypeOf(rhs) == Value) rhs else self.splat(rhs, shape, .i32); - const zero_v = self.zeros(shape, .i32); - const one_v = self.ones(shape, .i32); - const q = self.divsi(lhs, rhs_v); - const sign_a = self.subi( - self.extui(self.cmpi(.sgt, lhs, zero_v), .i32), - self.extui(self.cmpi(.slt, lhs, zero_v), .i32), - ); - const sign_b = self.subi( - self.extui(self.cmpi(.sgt, rhs_v, zero_v), .i32), - self.extui(self.cmpi(.slt, rhs_v, zero_v), .i32), - ); - const sign_diff = self.cmpi(.ne, sign_a, sign_b); - const rem_nz = self.cmpi(.ne, self.remsi(lhs, rhs_v), zero_v); - const cond = self.andi(sign_diff, rem_nz); - return self.select(cond, self.subi(q, one_v), q); - } - - pub fn remsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.remsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn remui(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.remui(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.shli(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn shrsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.shrsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn shrui(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.shrui(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn maxsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.maxsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn minsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.minsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn maximumf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.maximumf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn minimumf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.minimumf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn maxnumf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.maxnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn minnumf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.minnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn negf(self: *Builder, src: Value) Value { - return self.emit(arith.negf(self.ctx, src.inner, self.loc())); - } - pub fn cmpi(self: *Builder, predicate: arith.CmpIPredicate, lhs: Value, rhs: Value) Value { - return self.emit(arith.cmpi(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); - } - pub fn cmpf(self: *Builder, predicate: arith.CmpFPredicate, lhs: Value, rhs: Value) Value { - return self.emit(arith.cmpf(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); - } - pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { - return self.emit(arith.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); - } - pub fn where(self: *Builder, condition: Value, x: Value, y: Value) Value { - const xv, const yv = self.coerce(x, y); - const c = if (condition.isVector() == xv.isVector()) condition else blk: { - // Splat a scalar predicate to match a vector x/y. - if (xv.isVector()) { - break :blk self.splat(condition, xv.shape().constSlice(), .i1); - } - break :blk condition; - }; - return self.select(c, xv, yv); - } - - /// Elementwise max with auto-broadcast scalar→vector. Use `maxOpts` - /// for NaN-propagation control. - pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { - return self.maximumOpts(a, b, .{}); - } - pub fn maximumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { - const l, const r = self.coerce(a, b); - if (l.isFloatElem()) { - return if (opts.propagate_nan) self.maximumf(l, r) else self.maxnumf(l, r); - } - return self.maxsi(l, r); - } - pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { - return self.minimumOpts(a, b, .{}); - } - pub fn minimumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { - const l, const r = self.coerce(a, b); - if (l.isFloatElem()) { - return if (opts.propagate_nan) self.minimumf(l, r) else self.minnumf(l, r); - } - return self.minsi(l, r); - } - - // ==================== arith casts ==================== - - pub fn extsi(self: *Builder, src: Value, dt: DType) Value { - return self.emit(arith.extsi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn extui(self: *Builder, src: Value, dt: DType) Value { - return self.emit(arith.extui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn arithExtf(self: *Builder, src: Value, dt: DType) Value { - return self.emit(arith.extf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn trunci(self: *Builder, src: Value, dt: DType) Value { - return self.emit(arith.trunci(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn arithTruncf(self: *Builder, src: Value, dt: DType) Value { - return self.emit(arith.truncf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn arithSitofp(self: *Builder, src: Value, dt: DType) Value { - return self.emit(arith.sitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn arithUitofp(self: *Builder, src: Value, dt: DType) Value { - return self.emit(arith.uitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn arithFptosi(self: *Builder, src: Value, dt: DType) Value { - return self.emit(arith.fptosi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn arithFptoui(self: *Builder, src: Value, dt: DType) Value { - return self.emit(arith.fptoui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn arithBitcast(self: *Builder, src: Value, dt: DType) Value { - return self.emit(arith.bitcast(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - - /// `arith.index_cast` — cast between integer and `index` types. - pub fn indexCast(self: *Builder, src: Value, target: *const mlir.Type) Value { - return self.emit(arith.index_cast(self.ctx, src.inner, target, self.loc())); - } - - /// Cast a scalar integer (i32 / i64 / …) to the `index` type. Common - /// pattern for using i32 program-ids and scalar-prefetch values as - /// memref offsets. - pub fn toIndex(self: *Builder, src: Value) Value { - return self.indexCast(src, .index(self.ctx)); - } - - // ==================== TPU casts ==================== - - /// `tpu.bitcast` — same-bitwidth element-type swap. Shape preserved. - pub fn bitCast(self: *Builder, src: Value, dt: DType) Value { - return self.emit(mosaic_tpu.bitcast(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn extf(self: *Builder, src: Value, dt: DType) Value { - return self.emit(mosaic_tpu.extf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn truncf(self: *Builder, src: Value, dt: DType) Value { - return self.emit(mosaic_tpu.truncf(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn sitofp(self: *Builder, src: Value, dt: DType) Value { - return self.emit(mosaic_tpu.sitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn uitofp(self: *Builder, src: Value, dt: DType) Value { - return self.emit(mosaic_tpu.uitofp(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn fptosi(self: *Builder, src: Value, dt: DType) Value { - return self.emit(mosaic_tpu.fptosi(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - pub fn fptoui(self: *Builder, src: Value, dt: DType) Value { - return self.emit(mosaic_tpu.fptoui(self.ctx, src.inner, self.swapElem(src, dt), self.loc())); - } - - /// Numeric cast with auto-dispatch. Routes through `arith.*` ops for the - /// common float / int conversions (which don't carry a rounding-mode - /// attribute). The `tpu.*` cast variants (`tpuTruncf`, `tpuFptosi`, …) - /// are exposed separately for callers that need rounding-mode control — - /// pass `opts.rounding` to opt in. - pub fn cast(self: *Builder, src: Value, dt: DType) Value { - return self.castOpts(src, dt, .{}); - } - pub fn castOpts(self: *Builder, src: Value, dt: DType, opts: CastOpts) Value { - if (opts.bitcast) return self.bitCast(src, dt); - const cur_elem = src.elemType(); - const tgt_elem = dt.toMlir(self.ctx); - if (cur_elem.eql(tgt_elem)) return src; - const cur_dtype = self.mlirElemToDType(cur_elem); - const cur_is_float = src.isFloatElem(); - const tgt_is_float = isFloatDtype(dt); - if (cur_is_float and tgt_is_float) { - const cur_bw = dtypeBitwidth(cur_dtype); - const tgt_bw = dtypeBitwidth(dt); - if (tgt_bw > cur_bw) return self.arithExtf(src, dt); - return self.arithTruncf(src, dt); - } - if (cur_is_float and !tgt_is_float) return self.arithFptosi(src, dt); - if (!cur_is_float and tgt_is_float) { - if (cur_dtype == .i1) return self.arithUitofp(src, dt); - return self.arithSitofp(src, dt); - } - const cur_bw = dtypeBitwidth(cur_dtype); - const tgt_bw = dtypeBitwidth(dt); - if (tgt_bw > cur_bw) return self.extsi(src, dt); - if (tgt_bw < cur_bw) return self.trunci(src, dt); - return self.arithBitcast(src, dt); - } - - // ==================== math dialect ==================== - - pub fn exp(self: *Builder, x: Value) Value { - return self.emit(math.exp(self.ctx, x.inner, self.loc())); - } - /// `exp2(x)` → `exp(x * ln(2))`. Matches Pallas's TPU lowering rule - /// (`jax/_src/pallas/mosaic/lowering.py:_exp2_lowering_rule`), which - /// expands `lax.exp2_p` into the same `x * ln2` pattern in forward-compat - /// mode. Materializes as an `arith.constant` (matching x's element type) - /// + `arith.mulf x, ln2` + `math.exp`. For the raw `math.exp2` op - /// (post-2025-07-26 cloud TPUs), use `mathExp2`. - pub fn exp2(self: *Builder, x: Value) Value { - const ln2 = self.constant(self.mlirElemToDType(x.elemType()), @as(f64, 0.6931471805599453)); - const ln2_bc = if (x.isShaped()) self.broadcastTo(ln2, x.shape().constSlice()) else ln2; - return self.exp(self.mulf(x, ln2_bc)); - } - pub fn mathExp2(self: *Builder, x: Value) Value { - return self.emit(math.exp2(self.ctx, x.inner, self.loc())); - } - pub fn log(self: *Builder, x: Value) Value { - return self.emit(math.log(self.ctx, x.inner, self.loc())); - } - pub fn log1p(self: *Builder, x: Value) Value { - return self.emit(math.log1p(self.ctx, x.inner, self.loc())); - } - pub fn log2(self: *Builder, x: Value) Value { - return self.emit(math.log2(self.ctx, x.inner, self.loc())); - } - pub fn sqrt(self: *Builder, x: Value) Value { - return self.emit(math.sqrt(self.ctx, x.inner, self.loc())); - } - pub fn rsqrt(self: *Builder, x: Value) Value { - return self.emit(math.rsqrt(self.ctx, x.inner, self.loc())); - } - pub fn sin(self: *Builder, x: Value) Value { - return self.emit(math.sin(self.ctx, x.inner, self.loc())); - } - pub fn cos(self: *Builder, x: Value) Value { - return self.emit(math.cos(self.ctx, x.inner, self.loc())); - } - pub fn tanh(self: *Builder, x: Value) Value { - return self.emit(math.tanh(self.ctx, x.inner, self.loc())); - } - pub fn erf(self: *Builder, x: Value) Value { - return self.emit(math.erf(self.ctx, x.inner, self.loc())); - } - pub fn absf(self: *Builder, x: Value) Value { - return self.emit(math.absf(self.ctx, x.inner, self.loc())); - } - pub fn absi(self: *Builder, x: Value) Value { - return self.emit(math.absi(self.ctx, x.inner, self.loc())); - } - pub fn abs(self: *Builder, x: Value) Value { - if (x.isFloatElem()) return self.absf(x); - return self.absi(x); - } - pub fn floor(self: *Builder, x: Value) Value { - return self.emit(math.floor(self.ctx, x.inner, self.loc())); - } - pub fn ceil(self: *Builder, x: Value) Value { - return self.emit(math.ceil(self.ctx, x.inner, self.loc())); - } - pub fn powf(self: *Builder, x: Value, y: Value) Value { - return self.emit(math.powf(self.ctx, x.inner, y.inner, self.loc())); - } - pub fn fma(self: *Builder, a: Value, b: Value, c_: Value) Value { - return self.emit(math.fma(self.ctx, a.inner, b.inner, c_.inner, self.loc())); - } - - // ==================== TPU memory: vector_load / vector_store / scalar load/store ==================== - - /// Build the result vector type for a full-shape `tpu.vector_load` from - /// the input ref's shape + element type. - fn refVectorType(self: *Builder, ref: Value) *const mlir.Type { - _ = self; - const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("Builder.refVectorType: not a memref"); - var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); - return .vector(s.constSlice(), mr.elementType()); - } - - /// Full-shape `vector.load`. Use `tpuVectorLoad` for sublane stride / masking. - pub fn refLoad(self: *Builder, ref: Value) Value { - const result_ty = self.refVectorType(ref); - const idx_inner = self.indicesOrZero(ref, &.{}); - return self.emit(vector.load(self.ctx, ref.inner, idx_inner, result_ty, .{}, self.loc())); - } - - /// `vector.load(ref, indices)` — explicit `index`-typed offsets. - pub fn vectorLoadAt(self: *Builder, ref: Value, indices: []const Value) Value { - const result_ty = self.refVectorType(ref); - const idx_inner = self.indicesOrZero(ref, indices); - return self.emit(vector.load(self.ctx, ref.inner, idx_inner, result_ty, .{}, self.loc())); - } - - /// `tpu.vector_load` — sublane stride + optional element-wise mask. - pub fn tpuVectorLoad(self: *Builder, ref: Value, indices: []const Value, opts: VectorLoadOpts) Value { - const ctx = self.ctx; - const result_ty = self.refVectorType(ref); - const idx_inner = self.indicesOrZero(ref, indices); - const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; - return self.emit(mosaic_tpu.vector_load(ctx, ref.inner, idx_inner, result_ty, .{ .strides = opts.strides, .mask = mask_inner }, self.loc())); - } - - /// `tpu.vector_load` with an explicit result vector shape. - pub fn tpuVectorLoadShape(self: *Builder, ref: Value, indices: []const Value, shape: []const i64, opts: VectorLoadOpts) Value { - const ctx = self.ctx; - const result_ty = mlir.Type.vector(shape, ref.elemType()); - const idx_inner = self.indicesOrZero(ref, indices); - const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; - return self.emit(mosaic_tpu.vector_load(ctx, ref.inner, idx_inner, result_ty, .{ .strides = opts.strides, .mask = mask_inner }, self.loc())); - } - - /// `tpu.vector_store(value, ref)` — full-shape vector store. - pub fn refStore(self: *Builder, ref: Value, value: Value) void { - self.refStoreOpts(ref, value, &.{}, .{}); - } - - pub fn vectorStoreAt(self: *Builder, ref: Value, value: Value, indices: []const Value) void { - self.refStoreOpts(ref, value, indices, .{}); - } - - pub fn refStoreOpts(self: *Builder, ref: Value, value: Value, indices: []const Value, opts: VectorStoreOpts) void { - const idx_inner = self.indicesOrZero(ref, indices); - const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; - _ = mosaic_tpu.vector_store(self.ctx, value.inner, ref.inner, idx_inner, .{ .strides = opts.strides, .add = opts.add, .mask = mask_inner }, self.loc()).appendTo(self.currentBlock()); - } - - /// `memref.load` — scalar load. Indices must be `index`-typed. - pub fn scalarLoad(self: *Builder, ref: Value, indices: []const Value) Value { - const idx_inner = self.indicesOrZero(ref, indices); - return self.emit(memref.load(self.ctx, ref.inner, idx_inner, .{}, self.loc())); - } - - /// `memref.store` — scalar store. - pub fn scalarStore(self: *Builder, ref: Value, value: Value, indices: []const Value) void { - const idx_inner = self.indicesOrZero(ref, indices); - _ = memref.store(self.ctx, value.inner, ref.inner, idx_inner, .{}, self.loc()).appendTo(self.currentBlock()); - } - - /// Returns a `*const mlir.Value` slice of length `ref.rank()`, padding with - /// a cached `arith.constant 0 : index` when `indices` is empty. - fn indicesOrZero(self: *Builder, ref: Value, indices: []const Value) []const *const mlir.Value { - const r = ref.rank(); - if (indices.len == r) return self.innerSlice(indices); - std.debug.assert(indices.len == 0); - const zero = self.zeroIndex(); - const out = self.arena.allocator().alloc(*const mlir.Value, r) catch @panic("Builder.indicesOrZero OOM"); - for (out) |*p| p.* = zero.inner; - return out; - } - - /// Return a cached `arith.constant 0 : index` Value, lazily emitting - /// it the first time it's needed. The cache is invalidated whenever - /// the insertion block changes — `index` constants are scoped to the - /// block they're emitted in. - fn zeroIndex(self: *Builder) Value { - const block = self.currentBlock(); - if (self.zero_index) |z| { - if (self.zero_index_block == block) return z; - } - const z = self.constIndex(0); - self.zero_index = z; - self.zero_index_block = block; - return z; - } - - /// `tpu.strided_load` — multi-dim strided load. Result shape is derived - /// from `ref.shape / strides` (per-dim ceil divide). - pub fn stridedLoad(self: *Builder, ref: Value, indices: []const Value, strides: []const i32) Value { - const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("stridedLoad: not a memref"); - var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..mr.rank()) |i| { - const stride: i64 = if (i < strides.len) strides[i] else 1; - const dim = mr.dimension(i); - s.appendAssumeCapacity(@divTrunc(dim + stride - 1, stride)); - } - const result_ty = mlir.Type.vector(s.constSlice(), mr.elementType()); - return self.emit(mosaic_tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); - } - - /// `tpu.strided_load` with an explicit result shape — when you know the - /// load shape doesn't match the simple `ref.shape / strides` form (e.g. - /// inner-dim crop). - pub fn stridedLoadShape(self: *Builder, ref: Value, indices: []const Value, strides: []const i32, result_shape: []const i64) Value { - const mr = ref.type_().isA(mlir.MemRefType) orelse @panic("stridedLoadShape: not a memref"); - const result_ty = mlir.Type.vector(result_shape, mr.elementType()); - return self.emit(mosaic_tpu.strided_load(self.ctx, ref.inner, self.innerSlice(indices), strides, result_ty, self.loc())); - } - - /// `tpu.strided_store` — multi-dim strided store. - pub fn stridedStore(self: *Builder, ref: Value, value: Value, indices: []const Value, strides: []const i32) void { - _ = mosaic_tpu.strided_store(self.ctx, value.inner, ref.inner, self.innerSlice(indices), strides, self.loc()).appendTo(self.currentBlock()); - } - - // ==================== TPU compute: matmul / iota / reciprocal ==================== - - /// `tpu.matmul(lhs, rhs, acc)`. Result type is taken from `acc`. Use - /// `matmulOpts` to set transposes / precision / dimension_numbers. - pub fn matmul(self: *Builder, lhs: Value, rhs: Value, acc: Value) Value { - return self.matmulOpts(lhs, rhs, acc, .{}); - } - - pub fn matmulOpts(self: *Builder, lhs: Value, rhs: Value, acc: Value, opts: MatmulOpts) Value { - return self.emit(mosaic_tpu.matmul( - self.ctx, - lhs.inner, - rhs.inner, - acc.inner, - acc.type_(), - .{ - .transpose_lhs = opts.transpose_lhs, - .transpose_rhs = opts.transpose_rhs, - .transpose_lhs_hint = opts.transpose_lhs_hint, - .precision = opts.precision, - .dimension_numbers = opts.dimension_numbers, - }, - self.loc(), - )); - } - - /// `tpu.iota` over the given shape / dimensions. Pass `null` for - /// `dimensions` to use the default. - pub fn iota(self: *Builder, shape: []const i64, dtype: DType, dimensions: ?[]const i32) Value { - const ty = mlir.Type.vector(shape, dtype.toMlir(self.ctx)); - return self.emit(mosaic_tpu.iota(self.ctx, ty, .{ .dimensions = dimensions }, self.loc())); - } - - /// 1-D `arange(0, n)`. TPU layout pass requires rank ≥ 2 with the lane dim - /// trailing, so this emits a `<1xN>` `tpu.iota` then `shape_cast` to ``. - pub fn arange(self: *Builder, n: i64, dtype: DType) Value { - const elem = dtype.toMlir(self.ctx); - const ty_2d = .vector(&.{ 1, n }, elem); - const iota_2d = self.emit(mosaic_tpu.iota(self.ctx, ty_2d, .{ .dimensions = &.{1} }, self.loc())); - return self.shapeCast(iota_2d, &.{n}); - } - - /// `tpu.reciprocal(x)` with default options. - pub fn reciprocal(self: *Builder, x: Value) Value { - return self.reciprocalOpts(x, .{}); - } - - pub fn reciprocalOpts(self: *Builder, x: Value, opts: ReciprocalOpts) Value { - return self.emit(mosaic_tpu.reciprocal(self.ctx, x.inner, .{ - .approx = opts.approx, - .full_range = opts.full_range, - }, self.loc())); - } - - // ==================== Vector dialect helpers ==================== - // - // The `tpu.*` shape ops (`reshape`, `transpose`, - // `broadcastInSublanes`) are Mosaic-specific layout-aware variants. - - /// `vector.broadcast` to `shape`. For Zig scalar literals prefer `splat` (dense-splat fast path). - pub fn broadcastTo(self: *Builder, src: Value, shape: []const i64) Value { - const ty = mlir.Type.vector(shape, src.elemType()); - return self.emit(vector.broadcast(self.ctx, src.inner, ty, self.loc())); - } - - /// `vector.shape_cast` — reshape preserving total lanes. `reshape` emits `tpu.reshape` instead. - pub fn shapeCast(self: *Builder, src: Value, new_shape: []const i64) Value { - const ty = mlir.Type.vector(new_shape, src.elemType()); - return self.emit(vector.shape_cast(self.ctx, src.inner, ty, self.loc())); - } - - /// `vector.extract` — extract a scalar (or lower-rank vector) at a - /// static position. Result shape = `src.shape[position.len..]`. - pub fn vectorExtract(self: *Builder, src: Value, position: []const i64) Value { - const sh = src.shape(); - var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (position.len..sh.len) |i| out.appendAssumeCapacity(sh.get(i)); - const ty: *const mlir.Type = if (out.len == 0) - src.elemType() - else - .vector(out.constSlice(), src.elemType()); - return self.emit(vector.extract(self.ctx, src.inner, position, &.{}, ty, self.loc())); - } - - /// `vector.extract_strided_slice` — statically slice a vector. - pub fn vectorExtractStridedSlice(self: *Builder, src: Value, offsets: []const i64, sizes: []const i64, strides: []const i64) Value { - const ty = mlir.Type.vector(sizes, src.elemType()); - return self.emit(vector.extract_strided_slice(self.ctx, src.inner, .{ - .offsets = offsets, - .sizes = sizes, - .strides = strides, - }, ty, self.loc())); - } - - /// `vector.reduction ` — reduce a 1-D vector to a scalar. - /// Emits the bare upstream op; on real TPU the layout pass may reject - /// 1-D vectors, so prefer `reduceToScalar` for kernel-level reductions. - pub fn vectorReductionFlat(self: *Builder, kind: CombiningKind, src: Value) Value { - return self.emit(vector.reduction(self.ctx, kind, src.inner, src.elemType(), .{}, self.loc())); - } - - /// 1-D vector → scalar. TPU vectors are 2-D, so this adds a leading - /// singleton before reducing: shape_cast→<1xN>, multi_reduction[1]→<1>, extract[0]. - /// `acc` is the reduction identity (e.g. `k.zeros(&.{1}, .f32)` for sum). - pub fn reduceToScalar(self: *Builder, kind: CombiningKind, src: Value, acc: Value) Value { - std.debug.assert(src.rank() == 1); - const n = src.shape().get(0); - const src_2d = self.shapeCast(src, &.{ 1, n }); - const reduced = self.multiReduction(kind, src_2d, acc, &.{1}); - return self.vectorExtract(reduced, &.{0}); - } - - /// `vector.multi_reduction ` — reduce a multi-D vector along - /// the given axes with an explicit accumulator. Result rank = - /// `src.rank - axes.len`. - pub fn multiReduction( - self: *Builder, - kind: CombiningKind, - src: Value, - acc: Value, - axes: []const i64, - ) Value { - const sh = src.shape(); - var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - outer: for (0..sh.len) |i| { - for (axes) |a| if (a == @as(i64, @intCast(i))) continue :outer; - out.appendAssumeCapacity(sh.get(i)); - } - const ty: *const mlir.Type = if (out.len == 0) - src.elemType() - else - .vector(out.constSlice(), src.elemType()); - return self.emit(vector.multi_reduction(self.ctx, kind, src.inner, acc.inner, axes, ty, self.loc())); - } - - /// `vector.load` with an explicit result shape — for `ref[i, j, :, k:k+BLOCK]`-style sliced loads. - pub fn vectorLoadShape(self: *Builder, ref: Value, indices: []const Value, shape: []const i64) Value { - const idx_inner = self.indicesOrZero(ref, indices); - const ty = mlir.Type.vector(shape, ref.elemType()); - return self.emit(vector.load(self.ctx, ref.inner, idx_inner, ty, .{}, self.loc())); - } - - // ==================== TPU shape ==================== - - pub fn reshape(self: *Builder, src: Value, new_shape: []const i64) Value { - const ty = mlir.Type.vector(new_shape, src.elemType()); - return self.emit(mosaic_tpu.reshape(self.ctx, src.inner, ty, self.loc())); - } - - pub fn repeat(self: *Builder, src: Value, dimension: i32, times: i32) Value { - const in_shape = src.shape(); - var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..in_shape.len) |i| { - const d = in_shape.get(i); - out.appendAssumeCapacity(if (i == @as(usize, @intCast(dimension))) d * times else d); - } - if (times == 1) return src; - const ty = mlir.Type.vector(out.constSlice(), src.elemType()); - const sources = self.arena.allocator().alloc(*const mlir.Value, @intCast(times)) catch @panic("OOM"); - @memset(sources, src.inner); - return self.emit(mosaic_tpu.concatenate(self.ctx, sources, dimension, ty, self.loc())); - } - - pub fn concatenate(self: *Builder, sources: []const Value, dimension: i32) Value { - std.debug.assert(sources.len > 0); - const ax: usize = @intCast(dimension); - var sum: i64 = 0; - for (sources) |s| sum += s.dim(ax); - const head = sources[0].shape(); - var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..head.len) |i| out.appendAssumeCapacity(if (i == ax) sum else head.get(i)); - const ty = mlir.Type.vector(out.constSlice(), sources[0].elemType()); - return self.emit(mosaic_tpu.concatenate(self.ctx, self.innerSlice(sources), dimension, ty, self.loc())); - } - - pub fn transpose(self: *Builder, src: Value, permutation: []const i64) Value { - const in_shape = src.shape(); - std.debug.assert(permutation.len == in_shape.len); - var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (permutation) |p| out.appendAssumeCapacity(in_shape.get(@intCast(p))); - const ty = mlir.Type.vector(out.constSlice(), src.elemType()); - return self.emit(mosaic_tpu.transpose(self.ctx, src.inner, permutation, ty, self.loc())); - } - - pub fn broadcastInSublanes(self: *Builder, src: Value, lane: i32) Value { - // Result shape = src shape (the op replicates within sublanes). - return self.emit(mosaic_tpu.broadcast_in_sublanes(self.ctx, src.inner, lane, src.type_(), self.loc())); - } - - pub fn rotate(self: *Builder, value: Value, opts: RotateOpts) Value { - const amount = self.constant(.i32, opts.amount); - return self.emit(mosaic_tpu.dynamic_rotate(self.ctx, value.inner, amount.inner, .{ - .dimension = opts.dimension, - .stride = opts.stride, - .stride_dimension = opts.stride_dimension, - }, self.loc())); - } - - // ==================== TPU reductions / scan / sort ==================== - - /// `tpu.all_reduce(input, dim, kind)` over a single dim. - pub fn allReduce(self: *Builder, input: Value, kind: ReductionKind, dim: i64) Value { - const in_shape = input.shape(); - var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..in_shape.len) |i| out.appendAssumeCapacity(if (i == @as(usize, @intCast(dim))) 1 else in_shape.get(i)); - const ty = mlir.Type.vector(out.constSlice(), input.elemType()); - return self.emit(mosaic_tpu.all_reduce(self.ctx, input.inner, dim, kind, ty, self.loc())); - } - - /// `tpu.reduce_index(input, kind, axis)` — drop axis, replace with - /// `index`-valued indices (integer). - pub fn reduceIndex(self: *Builder, input: Value, kind: ReductionKind, axis: i32) Value { - const in_shape = input.shape(); - var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (in_shape.constSlice(), 0..) |d, i| { - if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); - } - const ty = mlir.Type.vector(out.constSlice(), .int(self.ctx, .i32)); - return self.emit(mosaic_tpu.reduce_index(self.ctx, input.inner, axis, kind, ty, self.loc())); - } - - /// Scan the last dimension; a mask must be rank one with that dimension - /// as its length. The result has the same shape as the input. - pub fn scan(self: *Builder, input: Value, kind: ReductionKind, mask: ?Value) Value { - const m_inner: ?*const mlir.Value = if (mask) |m| m.inner else null; - return self.emit(mosaic_tpu.scan(self.ctx, input.inner, kind, input.type_(), .{ .mask = m_inner }, self.loc())); - } - - /// `tpu.sort(keys, values, mask?)` — returns (output_mask, sorted_keys, sorted_values). - pub fn sort(self: *Builder, keys: Value, values: Value, mask: ?Value, opts: SortOpts) [3]Value { - const m_inner: ?*const mlir.Value = if (mask) |m| m.inner else null; - const out_mask_ty = if (mask) |m| m.type_() else blk: { - // Default: vector. - break :blk .vector(keys.shape().constSlice(), .integer(self.ctx, .i1)); - }; - const op = mosaic_tpu.sort( - self.ctx, - keys.inner, - values.inner, - out_mask_ty, - keys.type_(), - values.type_(), - .{ .descending = opts.descending, .mask = m_inner }, - self.loc(), - ); - _ = op.appendTo(self.currentBlock()); - return .{ - .{ .inner = op.result(0), .kernel = self }, - .{ .inner = op.result(1), .kernel = self }, - .{ .inner = op.result(2), .kernel = self }, - }; - } - - // ==================== TPU memref-shape ops ==================== - - pub fn memRefSlice( - self: *Builder, - mem_ref: Value, - base_idx: []const Value, - result_shape: []const i64, - dynamic_sizes: []const Value, - ) Value { - const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSlice: not a memref"); - const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); - return self.emit(mosaic_tpu.memref_slice(self.ctx, mem_ref.inner, .{ - .base_idx = self.innerSlice(base_idx), - .dynamic_sizes = self.innerSlice(dynamic_sizes), - }, result_ty, self.loc())); - } - - pub fn memRefSqueeze(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { - const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefSqueeze: not a memref"); - const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); - return self.emit(mosaic_tpu.memref_squeeze(self.ctx, mem_ref.inner, result_ty, self.loc())); - } - - pub fn memRefReshape(self: *Builder, mem_ref: Value, result_shape: []const i64) Value { - const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefReshape: not a memref"); - const result_ty = mlir.Type.memRef(mr.elementType(), result_shape, null, mr.memorySpace()); - return self.emit(mosaic_tpu.memref_reshape(self.ctx, mem_ref.inner, result_ty, self.loc())); - } - - pub fn memRefAlloca(self: *Builder, shape: []const i64, dtype: DType, memory_space: MemorySpace) Value { - return self.emit(memref.alloca(self.ctx, self.memRefTy(shape, dtype, memory_space), &.{}, &.{}, .{}, self.loc())); - } - - pub fn memRefBitcast(self: *Builder, mem_ref: Value, dt: DType) Value { - const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcast: not a memref"); - var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); - const result_ty = .memRef(dt.toMlir(self.ctx), s.constSlice(), null, mr.memorySpace()); - return self.emit(mosaic_tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); - } - - /// `tpu.memref_bitcast` with an explicit result shape — needed for the - /// `bf16 → i32` packing trick in ragged-paged-attention's - /// `strided_load_kv` path (`<64x128xbf16>` → `<32x128xi32>`). - pub fn memRefBitcastShape(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { - const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("memRefBitcastShape: not a memref"); - const result_ty = mlir.Type.memRef(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); - return self.emit(mosaic_tpu.memref_bitcast(self.ctx, mem_ref.inner, result_ty, self.loc())); - } - - pub fn reinterpretCast(self: *Builder, mem_ref: Value, result_shape: []const i64, dt: DType) Value { - const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("reinterpretCast: not a memref"); - const result_ty = mlir.Type.memRef(dt.toMlir(self.ctx), result_shape, null, mr.memorySpace()); - return self.emit(mosaic_tpu.reinterpret_cast(self.ctx, mem_ref.inner, result_ty, self.loc())); - } - - pub fn assumeLayout(self: *Builder, src: Value) Value { - return self.emit(mosaic_tpu.assume_layout(self.ctx, src.inner, src.type_(), self.loc())); - } - - pub fn eraseMemRefLayout(self: *Builder, mem_ref: Value) Value { - const mr = mem_ref.type_().isA(mlir.MemRefType) orelse @panic("eraseMemRefLayout: not a memref"); - var s: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..mr.rank()) |i| s.appendAssumeCapacity(mr.dimension(i)); - // Result drops the layout but keeps shape, element type, memory space. - const result_ty = .memRef(mr.elementType(), s.constSlice(), null, mr.memorySpace()); - return self.emit(mosaic_tpu.erase_memref_layout(self.ctx, mem_ref.inner, result_ty, self.loc())); - } - - // ==================== Mask building ==================== - - /// `tpu.create_mask` — 2-sided mask `[low, high)` per dim. - pub fn createMask(self: *Builder, low_bounds: []const Value, high_bounds: []const Value, shape: []const i64) Value { - const ty = mlir.Type.vector(shape, .integer(self.ctx, .i1)); - return self.emit(mosaic_tpu.create_mask(self.ctx, self.innerSlice(low_bounds), self.innerSlice(high_bounds), ty, self.loc())); - } - - /// Convenience: 1-sided range mask `(0 <= idx < limit)` — emits two - /// `index` constants and a `tpu.create_mask`. - pub fn rangeMask(self: *Builder, limits: []const Value, shape: []const i64) Value { - const zero = self.constIndex(0); - const lows = self.arena.allocator().alloc(Value, limits.len) catch @panic("rangeMask OOM"); - for (lows) |*l| l.* = zero; - return self.createMask(lows, limits, shape); - } - - // ==================== Semaphores / DMA / barriers ==================== - - pub fn semAlloc(self: *Builder, kind: ArgSpec.SemKind) Value { - const elem = switch (kind) { - .regular => mosaic_tpu.semaphoreType(self.ctx), - .dma => mosaic_tpu.dmaSemaphoreType(self.ctx), - }; - const ty = mlir.Type.memRef(elem, &.{}, null, MemorySpace.semaphore_mem.attribute(self.ctx)); - return self.emit(mosaic_tpu.sem_alloc(self.ctx, ty, self.loc())); - } - - pub fn semBarrier(self: *Builder) Value { - const ty = mlir.Type.memRef(mosaic_tpu.semaphoreType(self.ctx), &.{}, null, MemorySpace.semaphore_mem.attribute(self.ctx)); - return self.emit(mosaic_tpu.sem_barrier(self.ctx, ty, self.loc())); - } - - pub fn semRead(self: *Builder, semaphore: Value) Value { - return self.emit(mosaic_tpu.sem_read(self.ctx, semaphore.inner, self.loc())); - } - - pub fn semWait(self: *Builder, semaphore: Value, amount: anytype) void { - const a = if (@TypeOf(amount) == Value) amount else self.lift(amount); - _ = mosaic_tpu.sem_wait(self.ctx, semaphore.inner, a.inner, self.loc()).appendTo(self.currentBlock()); - } - - pub fn semSignal(self: *Builder, semaphore: Value, amount: anytype) void { - self.semSignalOpts(semaphore, amount, .{}); - } - - pub fn semSignalOpts(self: *Builder, semaphore: Value, amount: anytype, opts: SemSignalOpts) void { - const a = if (@TypeOf(amount) == Value) amount else self.lift(amount); - const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; - const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; - const subcore: ?*const mlir.Value = if (opts.subcore_id) |sc| sc.inner else null; - _ = mosaic_tpu.sem_signal(self.ctx, semaphore.inner, a.inner, .{ - .device_id = dev, - .core_id = core, - .subcore_id = subcore, - }, self.loc()).appendTo(self.currentBlock()); - } - - pub fn barrier(self: *Builder, barrier_id: Value) void { - _ = mosaic_tpu.barrier(self.ctx, barrier_id.inner, self.loc()).appendTo(self.currentBlock()); - } - - pub fn enqueueDma(self: *Builder, source: Value, target: Value, target_semaphore: Value, opts: EnqueueDmaOpts) void { - const src_sem: ?*const mlir.Value = if (opts.source_semaphore) |s| s.inner else null; - const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; - const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; - const subcore: ?*const mlir.Value = if (opts.subcore_id) |sc| sc.inner else null; - _ = mosaic_tpu.enqueue_dma(self.ctx, source.inner, target.inner, target_semaphore.inner, .{ - .source_semaphore = src_sem, - .device_id = dev, - .core_id = core, - .subcore_id = subcore, - .priority = opts.priority, - .strict_ordering = opts.strict_ordering, - }, self.loc()).appendTo(self.currentBlock()); - } - - pub fn waitDma2(self: *Builder, semaphore: Value, src: Value, dst: Value, opts: WaitDma2Opts) void { - const dev: ?*const mlir.Value = if (opts.device_id) |d| d.inner else null; - const core: ?*const mlir.Value = if (opts.core_id) |c_| c_.inner else null; - _ = mosaic_tpu.wait_dma2(self.ctx, semaphore.inner, src.inner, dst.inner, .{ - .device_id = dev, - .core_id = core, - .strict_ordering = opts.strict_ordering, - }, self.loc()).appendTo(self.currentBlock()); - } - - pub fn deviceId(self: *Builder) Value { - return self.emit(mosaic_tpu.device_id(self.ctx, self.loc())); - } - - /// `tpu.trace_start` — pair with `traceStop`. Used by Pallas to mark - /// einsum boundaries (e.g. `"nd,md->nm"`). - pub fn traceStart(self: *Builder, level: i32, message: []const u8) void { - _ = mosaic_tpu.trace_start(self.ctx, level, message, self.loc()).appendTo(self.currentBlock()); - } - - pub fn traceStop(self: *Builder) void { - _ = mosaic_tpu.trace_stop(self.ctx, self.loc()).appendTo(self.currentBlock()); - } - - /// Build a `#tpu.dot_dimension_numbers<...>` attribute for `matmulOpts`. - pub fn dotDimensionNumbers( - self: *Builder, - lhs_contracting: []const i64, - rhs_contracting: []const i64, - lhs_non_contracting: []const i64, - rhs_non_contracting: []const i64, - output_dim_order: []const i64, - lhs_batch: []const i64, - rhs_batch: []const i64, - ) *const mlir.Attribute { - return mosaic_tpu.dotDimensionNumbersAttr( - self.ctx, - lhs_contracting, - rhs_contracting, - lhs_non_contracting, - rhs_non_contracting, - output_dim_order, - lhs_batch, - rhs_batch, - ); - } - - pub fn delay(self: *Builder, cycles: anytype) void { - const c_ = if (@TypeOf(cycles) == Value) cycles else self.lift(cycles); - _ = mosaic_tpu.delay(self.ctx, c_.inner, self.loc()).appendTo(self.currentBlock()); - } - - pub fn assumeMultiple(self: *Builder, src: Value, multiple: i32) Value { - return self.emit(mosaic_tpu.assume_multiple(self.ctx, src.inner, multiple, self.loc())); - } - - pub fn openRegion(self: *Builder) *mlir.Block { - const block = mlir.Block.init(&.{}, &.{}); - self.pushBlock(block); - return block; - } - - pub fn closeRegion(self: *Builder, block: *mlir.Block) void { - _ = mosaic_tpu.yield(self.ctx, &.{}, self.loc()).appendTo(block); - self.popBlock(); - _ = mosaic_tpu.region(self.ctx, block, &.{}, self.loc()).appendTo(self.currentBlock()); - } - - // ==================== SCF regions ==================== - - /// Open a scoped `scf.for`. Literal bounds take the type of a Value bound, - /// else they are `index`. `inits` is `.{v1, v2, ...}` or an array. - pub fn openFor( - self: *Builder, - lower: anytype, - upper: anytype, - step: anytype, - inits: anytype, - ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { - return self.openForOpts(lower, upper, step, inits, .{}); - } - - /// `openFor` with `scf.for` options. - pub fn openForOpts( - self: *Builder, - lower: anytype, - upper: anytype, - step: anytype, - inits: anytype, - opts: common.ForOpts, - ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { - return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, indexBound), inits, opts); - } - - pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { - return common.openIf(self, Value, cond); - } - - pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { - return common.openIfElse(self, Value, cond, result_types); - } - - pub fn openWhile( - self: *Builder, - inits: anytype, - after_types: anytype, - ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { - return common.openWhile(self, Value, inits, after_types); - } - - // ==================== finalization ==================== - - /// Append `func.return` with the given results, verify the module, and - /// serialize to a NUL-terminated MLIR string owned by the kernel's - /// allocator. Caller must `defer allocator.free(ir)`. - pub fn finish(self: *Builder, results: []const Value) FinishError![:0]const u8 { - return self.finishOpts(results, .{}); - } - - pub const FinishOpts = struct { - /// Run `canonicalize,cse` on the module before serializing — lets - /// the user code stay one-line-per-Pallas-line and lean on MLIR - /// to dedupe duplicated constants / identical subexpressions, the - /// same way Pallas does (its lowering doesn't dedupe either; the - /// IR you read in `debug=True` is post-canonicalize). - canonicalize: bool = false, - }; - - pub fn finishOpts(self: *Builder, results: []const Value, opts: FinishOpts) FinishError![:0]const u8 { - const current = self.currentBlock(); - _ = func.returns(self.ctx, self.innerSlice(results), self.loc()).appendTo(current); - - if (self.emit_transform_stubs) self.appendTransformStubs(); - - try common.verify(self.module, "dsl/mosaic_tpu"); - - if (opts.canonicalize) { - const pm = mlir.PassManager.init(self.ctx); - defer pm.deinit(); - // `canonicalize` folds `x * 1`, `x + 0`, etc.; `cse` then - // dedupes identical constants and subexpressions. Same - // pipeline `zml/Compiler.zig` runs on every compiled module. - const opm = pm.asOpPassManager(); - inline for (.{ "canonicalize", "cse", "canonicalize" }) |pass| { - try opm.addPipeline(pass); - } - try pm.runOnOp(self.module.operation()); - } - - var al: std.Io.Writer.Allocating = .init(self.allocator); - defer al.deinit(); - try al.writer.print("{f}", .{self.module.operation().fmt(.{ - .debug_info = false, - })}); - - return try self.allocator.dupeSentinel(u8, al.written(), 0); - } - - /// Emit `transform_N` stub funcs — one per non-skipped i/o ref-arg. - /// Each takes `(i32 ×grid_dim_count, prefetch_refs...)` and returns - /// `block_rank` i32 values per `RefArgSnapshot.transform_returns` - /// (default: all zeros, matching a trivial index_map). - fn appendTransformStubs(self: *Builder) void { - const ctx = self.ctx; - const unknown_loc: *const mlir.Location = .unknown(ctx); - const i32_ty: *const mlir.Type = .int(ctx, .i32); - - const arena_alloc = self.arena.allocator(); - const total_args = self.grid_dim_count + self.prefetch_args.len; - const arg_types = arena_alloc.alloc(*const mlir.Type, total_args) catch @panic("OOM"); - const arg_locs = arena_alloc.alloc(*const mlir.Location, total_args) catch @panic("OOM"); - for (0..self.grid_dim_count) |i| { - arg_types[i] = i32_ty; - arg_locs[i] = unknown_loc; - } - for (self.prefetch_args, 0..) |ty, i| { - arg_types[self.grid_dim_count + i] = ty; - arg_locs[self.grid_dim_count + i] = unknown_loc; - } - - for (self.ref_args, 0..) |ra, idx| { - if (ra.skipped) continue; - - const rank: usize = @intCast(ra.block_rank); - const result_types = arena_alloc.alloc(*const mlir.Type, rank) catch @panic("OOM"); - for (result_types) |*r| r.* = i32_ty; - - const block = mlir.Block.init(arg_types, arg_locs); - - const c0 = arith.constant_int(ctx, 0, i32_ty, unknown_loc); - _ = c0.appendTo(block); - - const ret_vals = arena_alloc.alloc(*const mlir.Value, rank) catch @panic("OOM"); - const returns = ra.transform_returns; - for (ret_vals, 0..) |*v, i| { - if (returns) |r_list| { - if (i < r_list.len) { - v.* = switch (r_list[i]) { - .zero => c0.result(0), - .program_id => |pid| block.argument(pid), - }; - continue; - } - } - v.* = c0.result(0); - } - _ = func.returns(ctx, ret_vals, unknown_loc).appendTo(block); - - const name = if (self.transform_name_prefix) |prefix| - std.fmt.allocPrint(arena_alloc, "{s}_transform_{d}", .{ prefix, idx }) catch @panic("OOM") - else blk: { - var name_buf: [32]u8 = undefined; - break :blk arena_alloc.dupe(u8, std.fmt.bufPrint(&name_buf, "transform_{d}", .{idx}) catch unreachable) catch @panic("OOM"); - }; - - var extra_attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (self.sc_transform_indices) { - extra_attrs.appendAssumeCapacity(.named(ctx, "sc.is_transform_indices", .unit(ctx))); - } - - const stub_op = func.func(ctx, .{ - .name = name, - .args = arg_types, - .results = result_types, - .block = block, - .location = unknown_loc, - .verify = false, - .extra_attributes = extra_attrs.constSlice(), - }); - _ = stub_op.appendTo(self.module.body()); - } - } -}; - -/// Memory spaces that don't support windowing (empty window_params, no transform_N). -fn isSkippedWindowSpace(ms: MemorySpace) bool { - return switch (ms) { - .hbm, .any, .semaphore_mem => true, - else => false, - }; -} - -// ==================== scope-type aliases pinned to Mosaic's Builder ==================== - -pub fn ForScope(comptime N: usize) type { - return common.ForScope(Builder, Value, N); -} - -pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); - -pub fn IfScope(comptime N: usize) type { - return common.IfScope(Builder, Value, N); -} - -pub fn WhileScope(comptime N: usize, comptime M: usize) type { - return common.WhileScope(Builder, Value, N, M); -} - -test { - std.testing.refAllDecls(@This()); -} - -test "static rotate uses the current dynamic rotate operation" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - registerDialects(registry); - func.registerExtensions(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - var b = try Builder.init(std.testing.allocator, ctx, "rotate", &.{}, &.{}); - defer b.deinit(); - const input = b.zeros(&.{ 8, 128 }, .f32); - const rotated = b.rotate(input, .{ .amount = 3, .dimension = 1 }); - try std.testing.expectEqualStrings("tpu.dynamic_rotate", rotated.inner.owner().name()); - const amount: Value = .{ .inner = rotated.inner.owner().operand(1), .kernel = &b }; - try std.testing.expectEqual(@as(?i64, 3), amount.asConstantInt()); - const ir = try b.finish(&.{}); - defer std.testing.allocator.free(ir); -} diff --git a/dsl/mosaic_tpu/dtype.zig b/dsl/mosaic_tpu/dtype.zig deleted file mode 100644 index 549c69da..00000000 --- a/dsl/mosaic_tpu/dtype.zig +++ /dev/null @@ -1,72 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -pub const DType = enum { - i1, - i8, - i16, - i32, - i64, - f16, - bf16, - f32, - f64, - f8e4m3fn, - f8e5m2, - - pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { - return switch (self) { - .i1 => .int(ctx, .i1), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .f16 => .float(ctx, .f16), - .bf16 => .float(ctx, .bf16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .f8e4m3fn => .float(ctx, .f8e4m3fn), - .f8e5m2 => .float(ctx, .f8e5m2), - }; - } -}; - -pub fn isFloatDtype(dt: DType) bool { - return switch (dt) { - .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e5m2 => true, - else => false, - }; -} - -pub fn dtypeBitwidth(dt: DType) usize { - return switch (dt) { - .i1 => 1, - .i8, .f8e4m3fn, .f8e5m2 => 8, - .i16, .f16, .bf16 => 16, - .i32, .f32 => 32, - .i64, .f64 => 64, - }; -} - -pub fn intBitwidth(dt: DType) u32 { - return switch (dt) { - .i1 => 1, - .i8 => 8, - .i16 => 16, - .i32 => 32, - .i64 => 64, - else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), - }; -} - -pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (comptime std.meta.fieldNames(DType)) |field_name| { - const dt = @field(DType, field_name); - if (elem.eql(dt.toMlir(ctx))) return dt; - } - @panic("element type not a recognized Mosaic-TPU DType"); -} - -test { - std.testing.refAllDecls(@This()); -} diff --git a/dsl/mosaic_tpu/pipeline.zig b/dsl/mosaic_tpu/pipeline.zig deleted file mode 100644 index 01264373..00000000 --- a/dsl/mosaic_tpu/pipeline.zig +++ /dev/null @@ -1,1147 +0,0 @@ -//! Zig port of `pltpu.emit_pipeline` (jax/_src/pallas/mosaic/pipeline.py) — -//! emits lowered `tpu`-dialect IR byte-equivalent to `pallas_call`. -//! -//! `.blocked`/`.squeezed`/one `.bounded_slice` block dim; trivial-windowing -//! inputs peeled as sync copy; `trace_scopes`. Unsupported features panic. - -const std = @import("std"); -const func = @import("mlir/dialects/func"); -const memref = @import("mlir/dialects/memref"); -const mlir = @import("mlir"); -const scf = @import("mlir/dialects/scf"); -const builder = @import("builder.zig"); - -const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); - -const Builder = builder.Builder; -const Value = builder.Value; -const DType = builder.DType; -const MemorySpace = builder.MemorySpace; -const FinishError = builder.FinishError; - -/// `ShapedType::kDynamic` — sentinel for a `?` dim in a memref shape. -const kDyn: i64 = std.math.minInt(i64); - -/// One entry of `pl.BlockSpec.block_shape`. -pub const BlockDim = union(enum) { - /// `pl.Blocked(n)` — windowed dim of size `n`. - blocked: i64, - /// `pl.Squeezed()` / `None` — dim dropped in the window. - squeezed, - /// `pl.BoundedSlice(n)` — index_map returns `.slice`; `n` is the max. - bounded_slice: i64, -}; - -/// One entry returned by `pl.BlockSpec.index_map`. -pub const BlockIndex = union(enum) { - /// `pl.ds(idx*block, block)` — for `.blocked`/`.squeezed` dims. - scalar: Value, - /// `pl.ds(start, size)` — for `.bounded_slice` dims. - slice: struct { start: Value, size: Value }, -}; - -/// `pl.BlockSpec.index_map`. `ctx` is opaque user data (Zig has no closures). -pub const IndexMapFn = *const fn (b: *Builder, indices: []const Value, ctx: ?*anyopaque) []const BlockIndex; - -/// Working window for one pipeline operand. Body must re-emit -/// `memref_slice`/`squeeze` at each use site. `slot == null` ⇒ trivial-windowing; -/// `buf == null` ⇒ no operand. -pub const RefWindow = struct { buf: ?Value = null, slot: ?Value = null }; - -/// The pipeline body — `pallas_call`'s inner kernel. -pub const BodyFn = *const fn (b: *Builder, grid_indices: []const Value, refs: []const RefWindow, scratches: []const Value, ctx: ?*anyopaque) FinishError!void; - -// Optional overrides for a buffered reference's DMA operations. -// Unspecified operations retain the standard pipeline behavior. -pub const BufferedRefStrategy = struct { - ctx: ?*anyopaque = null, - copy_in: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool, ctx: ?*anyopaque) void = null, - wait_in: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, - copy_out: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, - wait_out: ?*const fn (br: *BufferedRef, b: *Builder, indices: []const Value, ctx: ?*anyopaque) void = null, -}; - -/// `pl.BlockSpec` for one pipeline operand. `null` slot ↔ `None` spec. -pub const PipeSpec = struct { - full_shape: []const i64, - dtype: DType, - /// Optional HBM source/result view to materialize at each DMA use site. - source_reshape_shape: ?[]const i64 = null, - /// `null` ⇒ trivial windowing (the whole array). - block_shape: ?[]const BlockDim = null, - /// `null` only allowed with trivial windowing. - index_map: ?IndexMapFn = null, - index_map_ctx: ?*anyopaque = null, - /// Only `.vmem` supported. - memory_space: MemorySpace = .vmem, - - buffer_count: ?usize = null, - /// Tile that `_create_bounded_slice` rounds the DMA size up to. - bounded_slice_tiling: i32 = 1, - /// Pallas's `disable_bounds_checks`: skip OOB clamp, emit only `tpu.assume_multiple`. - bounded_slice_skip_clamp: bool = false, - - /// Optional custom DMA behavior for this operand. - transfer_strategy: ?BufferedRefStrategy = null, - /// Override the role infered from whether this spec appears - /// in `in_specs` or `out_specs` - buffer_type: ?BufferType = null, -}; - -pub const PipelineOpts = struct { - body: BodyFn, - body_ctx: ?*anyopaque = null, - /// Dynamic (`i32`) grid bounds. Comptime: pass `b.constant(.i32, N)`. - grid: []const Value, - /// One per HBM input ref (slot `null` <-> `None` spec). - in_specs: []const ?PipeSpec, - out_specs: []const ?PipeSpec, - trace_scopes: bool = true, -}; - -/// Emits `pltpu.emit_pipeline(body, grid=..., in_specs=..., out_specs=...)(*refs, scratches=...)`. -/// `refs` = `in_refs ++ out_refs`. Emits into `b.currentBlock()`. -pub fn emitPipeline( - b: *Builder, - refs: []const ?Value, - scratches: []const Value, - opts: PipelineOpts, -) FinishError!void { - std.debug.assert(opts.grid.len >= 1); - if (opts.grid.len > 4) @panic("emitPipeline: grids with rank > 4 unsupported"); - const n_in = opts.in_specs.len; - std.debug.assert(refs.len == n_in + opts.out_specs.len); - - const a = b.arena.allocator(); - - var brefs: std.ArrayList(BufferedRef) = try .initCapacity(a, opts.in_specs.len + opts.out_specs.len); - for (opts.in_specs, 0..) |maybe_spec, i| { - if (maybe_spec) |spec| { - std.debug.assert(refs[i] != null); - brefs.appendAssumeCapacity(BufferedRef.create(b, spec, .input, refs[i].?, i)); - } - } - for (opts.out_specs, 0..) |maybe_spec, j| { - if (maybe_spec) |spec| { - std.debug.assert(refs[n_in + j] != null); - brefs.appendAssumeCapacity(BufferedRef.create(b, spec, .output, refs[n_in + j].?, n_in + j)); - } - } - - // Whole pipeline lives in a `tpu.region`. - const region_block = mlir.Block.init(&.{}, &.{}); - b.pushBlock(region_block); - - for (brefs.items) |*br| br.allocate(b); - - const num_steps = gridSize(b, opts.grid); - const c0_i32 = b.constant(.i32, 0); - const c1_i32 = b.constant(.i32, 1); - { - var when = b.openIf(b.cmpi(.sgt, num_steps, c0_i32)); - { - for (brefs.items) |*br| if (br.is_trivial and (br.buffer_type == .input or br.buffer_type == .input_output)) br.syncCopyIn(b); - - const num_stages = blk: { - var m: usize = 2; - for (brefs.items) |br| if (br.buffer_count > m) { - m = br.buffer_count; - }; - break :blk m; - }; - var sched0 = Scheduler.init(b, c0_i32, opts.grid, num_steps, num_stages, opts.trace_scopes); - for (0..num_stages - 1) |step| { - if (opts.trace_scopes) { - var buf: [32]u8 = undefined; - b.traceStart(10, std.fmt.bufPrint(&buf, "ep_initialize_{d}", .{step}) catch unreachable); - } - for (brefs.items) |*br| sched0.initializeStep(b, br, step); - if (opts.trace_scopes) b.traceStop(); - } - - // iter_args: each bref's slots in spec order (see appendSlotInits), then grid indices. - var inits: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 4 + opts.grid.len); - for (brefs.items) |*br| br.appendSlotInits(b, &inits); - for (0..opts.grid.len) |_| inits.appendAssumeCapacity(c0_i32.inner); - const n_slots = inits.items.len - opts.grid.len; - - const i32_ty = b.scalarTy(.i32); - const block_types = a.alloc(*const mlir.Type, inits.items.len + 1) catch unreachable; - const block_locs = a.alloc(*const mlir.Location, inits.items.len + 1) catch unreachable; - for (block_types) |*t| t.* = i32_ty; - for (block_locs) |*l| l.* = b.loc(); - const for_body = mlir.Block.init(block_types, block_locs); - b.pushBlock(for_body); - - const iv: Value = .{ .inner = for_body.argument(0), .kernel = b }; - { - var cur: usize = 1; - for (brefs.items) |*br| cur = br.bindSlotsAt(b, for_body, cur); - std.debug.assert(cur == n_slots + 1); - } - var grid_indices_buf: [4]Value = undefined; - for (0..opts.grid.len) |i| { - grid_indices_buf[i] = .{ .inner = for_body.argument(@intCast(n_slots + 1 + i)), .kernel = b }; - } - - var sched = Scheduler.init(b, iv, opts.grid, num_steps, num_stages, opts.trace_scopes); - for (0..opts.grid.len) |i| sched.indices_storage[i] = grid_indices_buf[i]; - sched.recomputeDerivedIndices(b); - - for (brefs.items) |*br| { - if (br.buffer_type == .input or br.buffer_type == .input_output) - sched.copyIn(b, br); - } - for (brefs.items) |*br| { - if (br.buffer_type == .input or br.buffer_type == .input_output) - sched.waitIn(b, br); - } - // Emit slot `rem` ops BEFORE the body; body re-emits slice/squeeze per use. - var current_refs = a.alloc(RefWindow, refs.len) catch unreachable; - for (current_refs) |*r| r.* = .{}; - for (brefs.items) |*br| { - if (br.is_trivial or !br.isBuffered()) { - current_refs[br.spec_index] = .{ .buf = br.window_ref }; - } else { - const active = switch (br.buffer_type) { - .input => br.wait_in_slot.?, - .output, .input_output => br.copy_out_slot.?, - }; - current_refs[br.spec_index] = .{ .buf = br.window_ref, .slot = b.remui(active, b.constant(.i32, br.buffer_count)) }; - } - } - - if (opts.trace_scopes) b.traceStart(10, "ep_run_kernel"); - try opts.body(b, sched.indices_storage[0..opts.grid.len], current_refs, scratches, opts.body_ctx); - if (opts.trace_scopes) b.traceStop(); - - for (brefs.items) |*br| { - if (br.buffer_type == .output or br.buffer_type == .input_output) - sched.copyOut(b, br); - } - for (brefs.items) |*br| { - if (br.buffer_type == .output or br.buffer_type == .input_output) - sched.waitOut(b, br); - } - - for (brefs.items) |*br| { - if (br.buffer_type == .input or br.buffer_type == .input_output) - sched.advanceSlots(b, br); - } - var yields: std.ArrayList(*const mlir.Value) = try .initCapacity(a, brefs.items.len * 4 + opts.grid.len); - for (brefs.items) |*br| br.appendSlotYields(b, &yields); - const next_indices = nextIndexND(b, sched.indices(), opts.grid); - for (next_indices) |idx| yields.appendAssumeCapacity(idx.inner); - _ = scf.yield(b.ctx, yields.items, b.loc()).appendTo(for_body); - b.popBlock(); - - const for_op = scf.for_(b.ctx, c0_i32.inner, num_steps.inner, c1_i32.inner, inits.items, for_body, .{}, b.loc()); - _ = for_op.appendTo(b.currentBlock()); - var k: usize = 0; - for (brefs.items) |*br| k = br.readbackSlots(b, for_op, k); - var final_grid_indices_buf: [4]Value = undefined; - for (0..opts.grid.len) |i| { - final_grid_indices_buf[i] = .{ .inner = for_op.result(@intCast(n_slots + i)), .kernel = b }; - } - - // prev_index BEFORE Scheduler.init so common arithmetic CSEs with `step`. - const final_idx = prevIndexND(b, final_grid_indices_buf[0..opts.grid.len], opts.grid); - var sched_fin = Scheduler.init(b, b.subi(num_steps, c1_i32), opts.grid, num_steps, num_stages, opts.trace_scopes); - for (0..opts.grid.len) |i| sched_fin.indices_storage[i] = final_idx[i]; - sched_fin.recomputeDerivedIndices(b); - for (brefs.items) |*br| sched_fin.finalize(b, br); - for (brefs.items) |*br| if (br.is_trivial and (br.buffer_type == .output or br.buffer_type == .input_output)) br.syncCopyOut(b); - - when.close(); - } - } - - _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(region_block); - b.popBlock(); - _ = mosaic_tpu.region(b.ctx, region_block, &.{}, b.loc()).appendTo(b.currentBlock()); -} - -pub const BufferType = enum { input, output, input_output }; - -/// VMEM double-buffer + DMA semaphores for one pipeline operand. -pub const BufferedRef = struct { - spec: PipeSpec, - buffer_type: BufferType, - buffer_count: usize, - is_trivial: bool, - /// Index into `refs` / `current_refs`. - spec_index: usize, - src: Value, - /// Block shape with `.squeezed` dims removed. - block_compact: []const i64, - /// Indices into `spec.block_shape` parallel to `block_compact`. - block_keep: []const usize, - - /// `memref` (or full shape if trivial). - window_ref: Value = undefined, - sem_recvs: ?Value = null, - sem_sends: ?Value = null, - - // Slot iter_arg state — see appendSlotInits for init values. - copy_in_slot: ?Value = null, - wait_in_slot: ?Value = null, - copy_out_slot: ?Value = null, - wait_out_slot: ?Value = null, - slot: ?Value = null, - - fn create(b: *Builder, spec: PipeSpec, bt: BufferType, src: Value, spec_index: usize) BufferedRef { - const a = b.arena.allocator(); - const buffer_type = spec.buffer_type orelse bt; - if (spec.memory_space != .vmem) @panic("emitPipeline: only VMEM operands supported"); - const trivial = specHasTrivialWindowing(spec); - var bc: usize = if (spec.buffer_count) |c| c else 2; - if (spec.buffer_count == null and trivial) bc = 1; - if (buffer_type != .input and bc > 2) @panic("emitPipeline: output and input_output buffer_count > 2 unsupported"); - - const block_len = if (spec.block_shape) |bs| bs.len else 0; - var keep = std.ArrayList(usize).initCapacity(a, block_len) catch unreachable; - var compact = std.ArrayList(i64).initCapacity(a, block_len) catch unreachable; - if (spec.block_shape) |bs| { - for (bs, 0..) |bd, di| switch (bd) { - .blocked => |n| { - keep.appendAssumeCapacity(di); - compact.appendAssumeCapacity(n); - }, - .bounded_slice => |n| { - keep.appendAssumeCapacity(di); - compact.appendAssumeCapacity(n); - }, - .squeezed => {}, - }; - } - return .{ - .spec = spec, - .buffer_type = buffer_type, - .buffer_count = bc, - .is_trivial = trivial, - .spec_index = spec_index, - .src = src, - .block_compact = compact.items, - .block_keep = keep.items, - }; - } - - fn isBuffered(self: *const BufferedRef) bool { - return self.buffer_count > 0; - } - - fn allocate(self: *BufferedRef, b: *Builder) void { - const a = b.arena.allocator(); - const vmem = MemorySpace.vmem.attribute(b.ctx); - const elem = self.spec.dtype.toMlir(b.ctx); - if (self.is_trivial) { - const ty = mlir.Type.memRef(elem, self.spec.full_shape, null, vmem); - self.window_ref = memRefAlloca(b, ty); - } else { - if (self.block_compact.len == 1) @panic("emitPipeline: 1-D flat block buffers unsupported (TODO)"); - var shape = a.alloc(i64, self.block_compact.len + 1) catch unreachable; - shape[0] = @intCast(self.buffer_count); - @memcpy(shape[1..], self.block_compact); - const ty = mlir.Type.memRef(elem, shape, null, vmem); - self.window_ref = memRefAlloca(b, ty); - } - if (!self.is_trivial) { - const sem_arr_ty = dmaSemRefTy(b, self.buffer_count); - if (self.buffer_type == .input or self.buffer_type == .input_output) self.sem_recvs = semAllocOp(b, sem_arr_ty); - if (self.buffer_type == .output or self.buffer_type == .input_output) self.sem_sends = semAllocOp(b, sem_arr_ty); - } - } - - fn appendSlotInits(self: *BufferedRef, b: *Builder, out: *std.ArrayList(*const mlir.Value)) void { - const c0 = b.constant(.i32, 0).inner; - switch (self.buffer_type) { - .input => { - if (self.is_trivial) { - out.appendAssumeCapacity(c0); - } else { - out.appendAssumeCapacity((self.copy_in_slot orelse b.constant(.i32, 0)).inner); - out.appendAssumeCapacity(c0); - } - }, - .output => { - out.appendAssumeCapacity(c0); - out.appendAssumeCapacity(c0); - }, - .input_output => { - out.appendAssumeCapacity( - (self.copy_in_slot orelse b.constant(.i32, 0)).inner, - ); - out.appendAssumeCapacity(c0); - out.appendAssumeCapacity(c0); - out.appendAssumeCapacity(c0); - }, - } - } - - fn nSlots(self: *const BufferedRef) usize { - return switch (self.buffer_type) { - .input => if (self.is_trivial) 1 else 2, - .output => 2, - .input_output => 4, - }; - } - - fn bindSlotsAt(self: *BufferedRef, b: *Builder, body: *mlir.Block, start: usize) usize { - var k = start; - const arg = struct { - fn f(b_: *Builder, body_: *mlir.Block, i: *usize) Value { - const v: Value = .{ .inner = body_.argument(@intCast(i.*)), .kernel = b_ }; - i.* += 1; - return v; - } - }.f; - switch (self.buffer_type) { - .input => { - if (self.is_trivial) { - self.slot = arg(b, body, &k); - } else { - self.copy_in_slot = arg(b, body, &k); - self.wait_in_slot = arg(b, body, &k); - } - }, - .output => { - self.copy_out_slot = arg(b, body, &k); - self.wait_out_slot = arg(b, body, &k); - }, - .input_output => { - self.copy_in_slot = arg(b, body, &k); - self.wait_in_slot = arg(b, body, &k); - self.copy_out_slot = arg(b, body, &k); - self.wait_out_slot = arg(b, body, &k); - }, - } - return k; - } - - fn appendSlotYields(self: *BufferedRef, b: *Builder, out: *std.ArrayList(*const mlir.Value)) void { - _ = b; - switch (self.buffer_type) { - .input => { - if (self.is_trivial) { - out.appendAssumeCapacity(self.slot.?.inner); - } else { - out.appendAssumeCapacity(self.copy_in_slot.?.inner); - out.appendAssumeCapacity(self.wait_in_slot.?.inner); - } - }, - .output => { - out.appendAssumeCapacity(self.copy_out_slot.?.inner); - out.appendAssumeCapacity(self.wait_out_slot.?.inner); - }, - .input_output => { - out.appendAssumeCapacity(self.copy_in_slot.?.inner); - out.appendAssumeCapacity(self.wait_in_slot.?.inner); - out.appendAssumeCapacity(self.copy_out_slot.?.inner); - out.appendAssumeCapacity(self.wait_out_slot.?.inner); - }, - } - } - - fn readbackSlots(self: *BufferedRef, b: *Builder, for_op: *mlir.Operation, start: usize) usize { - var k = start; - const take = struct { - fn f(b_: *Builder, op: *mlir.Operation, i: *usize) Value { - const v: Value = .{ .inner = op.result(@intCast(i.*)), .kernel = b_ }; - i.* += 1; - return v; - } - }.f; - switch (self.buffer_type) { - .input => { - if (self.is_trivial) { - self.slot = take(b, for_op, &k); - } else { - self.copy_in_slot = take(b, for_op, &k); - self.wait_in_slot = take(b, for_op, &k); - } - }, - .output => { - self.copy_out_slot = take(b, for_op, &k); - self.wait_out_slot = take(b, for_op, &k); - }, - .input_output => { - self.copy_in_slot = take(b, for_op, &k); - self.wait_in_slot = take(b, for_op, &k); - self.copy_out_slot = take(b, for_op, &k); - self.wait_out_slot = take(b, for_op, &k); - }, - } - return k; - } - - fn computeIndex(self: *const BufferedRef, b: *Builder, indices: []const Value) []const BlockIndex { - if (self.spec.index_map) |im| return im(b, indices, self.spec.index_map_ctx); - @panic("emitPipeline: index_map required for non-trivial windowing"); - } - - /// Per `block_shape` dim: `{base, size?}`. `size == null` ⇒ static-full. - /// `static_zero_start` enables prologue shortcut (Python-int 0 start folds). - const DmaSlice = struct { base: []Value, size: []?Value }; - fn getDmaSlice(self: *const BufferedRef, b: *Builder, indices: []const Value, static_zero_start: bool) DmaSlice { - const a = b.arena.allocator(); - const bs = self.spec.block_shape.?; - const bidx = self.computeIndex(b, indices); - std.debug.assert(bidx.len == bs.len); - var base = a.alloc(Value, bs.len) catch unreachable; - var size = a.alloc(?Value, bs.len) catch unreachable; - for (bs, 0..) |bd, d| { - switch (bd) { - .squeezed => { - base[d] = switch (bidx[d]) { - .scalar => |v| v, - .slice => @panic("emitPipeline: squeezed dim must get a scalar index"), - }; - size[d] = null; - }, - .blocked => |n| { - const idx = switch (bidx[d]) { - .scalar => |v| v, - .slice => @panic("emitPipeline: blocked dim must get a scalar index"), - }; - base[d] = if (n == 1) idx else idx.mul(@as(i32, @intCast(n))); - const dim_size = self.spec.full_shape[d]; - if (@mod(dim_size, n) == 0) { - size[d] = null; - } else { - const tiling: i64 = if (d == bs.len - 1) 128 else if (bs.len >= 2 and d == bs.len - 2) 8 else 1; - const clamped = @min(n, dim_size); - const sz_const = @divFloor(clamped + tiling - 1, tiling) * tiling; - size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, b.constant(.i32, sz_const).inner, @as(i32, @intCast(tiling)), b.loc())); - } - }, - .bounded_slice => { - const sl = switch (bidx[d]) { - .scalar => @panic("emitPipeline: bounded_slice dim must get a `.slice` index from index_map"), - .slice => |s| s, - }; - const tiling: i32 = self.spec.bounded_slice_tiling; - const dim_size = self.spec.full_shape[d]; - base[d] = sl.start; - if (self.spec.bounded_slice_skip_clamp) { - size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sl.size.inner, tiling, b.loc())); - continue; - } - // Prologue shortcut: Python-int 0 start ⇒ `_round_up_to_nearest_multiple` folds. - if (static_zero_start and sl.start.asConstantInt() == @as(?i64, 0)) { - const is_oob = b.cmpi(.sgt, sl.size, b.constant(.i32, dim_size)); - const sz = b.select(is_oob, b.constant(.i32, dim_size), sl.size); - size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); - continue; - } - const is_oob = b.cmpi(.sgt, b.addi(sl.start, sl.size), b.constant(.i32, dim_size)); - const rounded = blk: { - if (tiling == 1) break :blk b.subi(b.constant(.i32, dim_size + 1), sl.start); - // lax sign-fixup for `jnp.remainder`. - const m = b.lift(tiling); - const c0 = b.constant(.i32, 0); - const rem_arg = b.subi(b.constant(.i32, dim_size), sl.start); - const r = b.remsi(rem_arg, m); - const r_nz = b.cmpi(.ne, r, c0); - const r_neg = b.cmpi(.slt, r, c0); - const fixup = b.andi(r_neg, r_nz); - const r_plus = b.addi(r, m); - const floormod = b.select(fixup, r_plus, r); - const round_down = b.subi(rem_arg, floormod); - break :blk b.addi(round_down, m); - }; - const sz = b.select(is_oob, rounded, sl.size); - size[d] = b.emit(mosaic_tpu.assume_multiple(b.ctx, sz.inner, tiling, b.loc())); - }, - } - } - return .{ .base = base, .size = size }; - } - - fn sliceSrc(self: *const BufferedRef, b: *Builder, dma: DmaSlice) Value { - const a = b.arena.allocator(); - const bs = self.spec.block_shape.?; - const src = if (self.spec.source_reshape_shape) |shape| b.memRefReshape(self.src, shape) else self.src; - var base = a.alloc(Value, bs.len) catch unreachable; - var dyn = std.ArrayList(Value).initCapacity(a, bs.len) catch unreachable; - var result_shape = a.alloc(i64, bs.len) catch unreachable; - for (bs, 0..) |bd, d| { - base[d] = dma.base[d]; - if (dma.size[d]) |sz| { - result_shape[d] = kDyn; - dyn.appendAssumeCapacity(sz); - } else { - result_shape[d] = switch (bd) { - .squeezed => 1, - .blocked => |n| n, - .bounded_slice => unreachable, - }; - } - } - const sliced = b.memRefSlice(src, base, result_shape, dyn.items); - if (self.block_keep.len == bs.len) return sliced; - - const squeezed_shape = a.alloc(i64, self.block_compact.len) catch unreachable; - for (self.block_keep, 0..) |sd, ci| squeezed_shape[ci] = result_shape[sd]; - return b.memRefSqueeze(sliced, squeezed_shape); - } - - /// Slices `alloca[slot, 0…]` (non-squeezed dims) then squeezes the slot dim. - fn sliceWindowAt(self: *const BufferedRef, b: *Builder, slot: Value, dma: DmaSlice) Value { - const a = b.arena.allocator(); - const bs = self.spec.block_shape.?; - const c0 = b.constant(.i32, 0); - const nd = 1 + self.block_compact.len; - var base = a.alloc(Value, nd) catch unreachable; - var result_shape = a.alloc(i64, nd) catch unreachable; - var dyn = std.ArrayList(Value).initCapacity(a, self.block_keep.len) catch unreachable; - base[0] = slot; - result_shape[0] = 1; - for (self.block_keep, 0..) |sd, ci| { - base[ci + 1] = c0; - if (dma.size[sd]) |sz| { - result_shape[ci + 1] = kDyn; - dyn.appendAssumeCapacity(sz); - } else { - result_shape[ci + 1] = switch (bs[sd]) { - .blocked => |n| n, - else => unreachable, - }; - } - } - const sliced = b.memRefSlice(self.window_ref, base, result_shape, dyn.items); - const squeezed_shape = a.alloc(i64, self.block_compact.len) catch unreachable; - @memcpy(squeezed_shape, result_shape[1..]); - return b.memRefSqueeze(sliced, squeezed_shape); - } - - fn semAt(b: *Builder, sem_array: Value, i: Value) Value { - const sliced = b.memRefSlice(sem_array, &.{i}, &.{1}, &.{}); - return b.memRefSqueeze(sliced, &.{}); - } - - fn currentRef(self: *BufferedRef, b: *Builder) Value { - if (self.is_trivial or !self.isBuffered()) return self.window_ref; - const slot = switch (self.buffer_type) { - .output => b.remui(self.copy_out_slot.?, b.constant(.i32, self.buffer_count)), - .input => b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)), - }; - const a = b.arena.allocator(); - const nd = 1 + self.block_compact.len; - var base = a.alloc(Value, nd) catch unreachable; - var result_shape = a.alloc(i64, nd) catch unreachable; - const c0 = b.constant(.i32, 0); - base[0] = slot; - result_shape[0] = 1; - for (self.block_compact, 0..) |n, ci| { - base[ci + 1] = c0; - result_shape[ci + 1] = n; - } - const sliced = b.memRefSlice(self.window_ref, base, result_shape, &.{}); - return b.memRefSqueeze(sliced, self.block_compact); - } - - fn syncCopyIn(self: *BufferedRef, b: *Builder) void { - const blk = mlir.Block.init(&.{}, &.{}); - b.pushBlock(blk); - const sem = semAllocOp(b, dmaSemRefTy(b, null)); - b.enqueueDma(self.src, self.window_ref, sem, .{}); - b.waitDma2(sem, self.src, self.window_ref, .{}); - _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); - b.popBlock(); - _ = mosaic_tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); - } - - fn syncCopyOut(self: *BufferedRef, b: *Builder) void { - const blk = mlir.Block.init(&.{}, &.{}); - b.pushBlock(blk); - const sem = semAllocOp(b, dmaSemRefTy(b, null)); - b.enqueueDma(self.window_ref, self.src, sem, .{}); - b.waitDma2(sem, self.window_ref, self.src, .{}); - _ = mosaic_tpu.yield(b.ctx, &.{}, b.loc()).appendTo(blk); - b.popBlock(); - _ = mosaic_tpu.region(b.ctx, blk, &.{}, b.loc()).appendTo(b.currentBlock()); - } - - // Op order mirrors pipeline.py: copy_in/copy_out/wait_out take slot rem - // before get_dma_slice; wait_in does it after. - - fn defaultCopyIn(self: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool) void { - const slot = b.remui(slot_cumulative, b.constant(.i32, self.buffer_count)); - const dma = self.getDmaSlice(b, indices, static_zero_start); - const src_sl = self.sliceSrc(b, dma); - const dst_sl = self.sliceWindowAt(b, slot, dma); - const sem = semAt(b, self.sem_recvs.?, slot); - b.enqueueDma(src_sl, dst_sl, sem, .{}); - } - - fn defaultWaitIn(self: *BufferedRef, b: *Builder, indices: []const Value) void { - const dma = self.getDmaSlice(b, indices, false); - const slot = b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)); - const src_sl = self.sliceSrc(b, dma); - const dst_sl = self.sliceWindowAt(b, slot, dma); - const sem = semAt(b, self.sem_recvs.?, slot); - b.waitDma2(sem, src_sl, dst_sl, .{}); - } - - fn defaultCopyOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { - if (self.buffer_count == 1) @panic("emitPipeline: single-buffered output sync_copy_out unsupported (TODO)"); - const slot = b.remui(self.copy_out_slot.?, b.constant(.i32, self.buffer_count)); - const dma = self.getDmaSlice(b, indices, false); - // Pallas evaluates `src` (VMEM window) before `dst` (HBM). - const src_sl = self.sliceWindowAt(b, slot, dma); - const dst_sl = self.sliceSrc(b, dma); - const sem = semAt(b, self.sem_sends.?, slot); - b.enqueueDma(src_sl, dst_sl, sem, .{}); - } - - fn defaultWaitOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { - if (self.buffer_count <= 1) return; - const slot = b.remui(self.wait_out_slot.?, b.constant(.i32, self.buffer_count)); - const dma = self.getDmaSlice(b, indices, false); - const src_sl = self.sliceWindowAt(b, slot, dma); - const dst_sl = self.sliceSrc(b, dma); - const sem = semAt(b, self.sem_sends.?, slot); - b.waitDma2(sem, src_sl, dst_sl, .{}); - } - - fn copyIn(self: *BufferedRef, b: *Builder, indices: []const Value, slot_cumulative: Value, static_zero_start: bool) void { - if (self.is_trivial) return; - - if (self.spec.transfer_strategy) |strategy| { - if (strategy.copy_in) |customCopyIn| { - customCopyIn(self, b, indices, slot_cumulative, static_zero_start, strategy.ctx); - return; - } - } - - self.defaultCopyIn(b, indices, slot_cumulative, static_zero_start); - } - - fn waitIn(self: *BufferedRef, b: *Builder, indices: []const Value) void { - if (self.spec.transfer_strategy) |strategy| { - if (strategy.wait_in) |customWaitIn| { - customWaitIn(self, b, indices, strategy.ctx); - return; - } - } - self.defaultWaitIn(b, indices); - } - - fn copyOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { - if (self.spec.transfer_strategy) |strategy| { - if (strategy.copy_out) |customCopyOut| { - customCopyOut(self, b, indices, strategy.ctx); - return; - } - } - self.defaultCopyOut(b, indices); - } - - fn waitOut(self: *BufferedRef, b: *Builder, indices: []const Value) void { - if (self.spec.transfer_strategy) |strategy| { - if (strategy.wait_out) |customWaitOut| { - customWaitOut(self, b, indices, strategy.ctx); - return; - } - } - self.defaultWaitOut(b, indices); - } - - pub fn sourceRef(self: *const BufferedRef) Value { - return self.src; - } - - pub fn windowRef(self: *const BufferedRef) Value { - return self.window_ref; - } - - pub fn copyInSlot(self: *const BufferedRef, b: *Builder, slot_cumulative: Value) Value { - return b.remui(slot_cumulative, b.constant(.i32, self.buffer_count)); - } - - pub fn waitInSlot(self: *const BufferedRef, b: *Builder) Value { - return b.remui(self.wait_in_slot.?, b.constant(.i32, self.buffer_count)); - } - - pub fn receiveSemaphoreAt(self: *const BufferedRef, b: *Builder, slot: Value) Value { - return semAt(b, self.sem_recvs.?, slot); - } - - pub fn copyOutSlot(self: *const BufferedRef, b: *Builder) Value { - const buffer_count: i32 = @intCast(self.buffer_count); - return b.remui(self.copy_out_slot.?, b.lift(buffer_count)); - } - - pub fn waitOutSlot(self: *const BufferedRef, b: *Builder) Value { - const buffer_count: i32 = @intCast(self.buffer_count); - return b.remui(self.wait_out_slot.?, b.lift(buffer_count)); - } - - pub fn sendSemaphoreAt( - self: *const BufferedRef, - b: *Builder, - slot: Value, - ) Value { - return semAt(b, self.sem_sends.?, slot); - } -}; - -/// Prologue/loop/epilogue copy+wait schedule. -const Scheduler = struct { - step: Value, - grid: []const Value, - num_steps: Value, - num_stages: usize, - trace_scopes: bool, - first_step: Value, - last_step: Value, - indices_storage: [4]Value = undefined, - prev_storage: [4]Value = undefined, - next_storage: [4]Value = undefined, - fetch_storage: [8][4]Value = undefined, - fetch_len: usize, - indices_len: usize, - - fn init(b: *Builder, step: Value, grid: []const Value, num_steps: Value, num_stages: usize, trace_scopes: bool) Scheduler { - if (num_stages + 1 > 8) @panic("emitPipeline: buffer_count > 7 unsupported"); - const c0 = b.constant(.i32, 0); - const c1 = b.constant(.i32, 1); - var s: Scheduler = .{ - .step = step, - .grid = grid, - .num_steps = num_steps, - .num_stages = num_stages, - .trace_scopes = trace_scopes, - .first_step = b.cmpi(.eq, step, c0), - .last_step = b.cmpi(.eq, step, b.subi(num_steps, c1)), - .fetch_len = num_stages + 1, - .indices_len = grid.len, - }; - for (0..grid.len) |i| s.indices_storage[i] = c0; - s.recomputeDerivedIndices(b); - return s; - } - - fn indices(self: *Scheduler) []const Value { - return self.indices_storage[0..self.indices_len]; - } - fn prevIndices(self: *Scheduler) []const Value { - return self.prev_storage[0..self.indices_len]; - } - fn nextIndices(self: *Scheduler) []const Value { - return self.next_storage[0..self.indices_len]; - } - fn fetchIndices(self: *Scheduler, lookahead: usize) []const Value { - std.debug.assert(lookahead < self.fetch_len); - return self.fetch_storage[lookahead][0..self.indices_len]; - } - - fn recomputeDerivedIndices(self: *Scheduler, b: *Builder) void { - const prev = prevIndexND(b, self.indices(), self.grid); - const next = nextIndexND(b, self.indices(), self.grid); - for (0..self.indices_len) |i| { - self.prev_storage[i] = prev[i]; - self.next_storage[i] = next[i]; - self.fetch_storage[0][i] = self.indices_storage[i]; - self.fetch_storage[1][i] = next[i]; - } - var fetch = next; - var lookahead: usize = 2; - while (lookahead < self.fetch_len) : (lookahead += 1) { - fetch = nextIndexND(b, fetch, self.grid); - for (0..self.indices_len) |i| self.fetch_storage[lookahead][i] = fetch[i]; - } - } - - fn namedScope(self: *Scheduler, b: *Builder, name: []const u8) void { - if (self.trace_scopes) b.traceStart(10, name); - } - fn endScope(self: *Scheduler, b: *Builder) void { - if (self.trace_scopes) b.traceStop(); - } - - /// `~out_of_fetch` as `step < num_steps + (1 - buffer_count)`. - fn notOutOfFetch(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { - std.debug.assert(br.isBuffered()); - const off: i32 = 1 - @as(i32, @intCast(br.buffer_count)); - return b.cmpi(.slt, self.step, b.addi(self.num_steps, b.lift(off))); - } - - fn notFirstStep(self: *Scheduler, b: *Builder) Value { - return b.cmpi(.ne, self.step, b.constant(.i32, 0)); - } - - fn indicesDiffer(b: *Builder, br: *const BufferedRef, ai: []const Value, ci: []const Value) Value { - const xa = br.computeIndex(b, ai); - const xc = br.computeIndex(b, ci); - std.debug.assert(xa.len == xc.len and xa.len > 0); - var acc: ?Value = null; - for (xa, xc) |x, y| { - const d: Value = switch (x) { - .scalar => |xv| b.cmpi(.ne, xv, y.scalar), - .slice => |xs| b.ori(b.cmpi(.ne, xs.start, y.slice.start), b.cmpi(.ne, xs.size, y.slice.size)), - }; - acc = if (acc) |p| b.ori(p, d) else d; - } - return acc.?; - } - - fn hasChanged(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { - std.debug.assert(br.isBuffered() and !br.is_trivial); - return indicesDiffer(b, br, self.indices(), self.prevIndices()); - } - fn willChangeCurrent(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { - std.debug.assert(br.isBuffered() and !br.is_trivial); - return indicesDiffer(b, br, self.indices(), self.nextIndices()); - } - fn willChangeFetch(self: *Scheduler, b: *Builder, br: *const BufferedRef) Value { - std.debug.assert(br.isBuffered() and !br.is_trivial); - if (br.buffer_count < 2) return self.hasChanged(b, br); - return indicesDiffer(b, br, self.fetchIndices(br.buffer_count - 2), self.fetchIndices(br.buffer_count - 1)); - } - - fn advance(b: *Builder, slot: Value, pred: Value) Value { - return b.select(pred, b.addi(slot, b.constant(.i32, 1)), slot); - } - - fn initializeStep(self: *Scheduler, b: *Builder, br: *BufferedRef, step: usize) void { - if ((br.buffer_type != .input and br.buffer_type != .input_output) or !br.isBuffered() or br.is_trivial) return; - if (step + 1 >= br.buffer_count) return; - if (br.copy_in_slot == null) br.copy_in_slot = b.constant(.i32, 0); - - const pred = if (step == 0) self.first_step else blk: { - const block_changed = indicesDiffer(b, br, self.fetchIndices(step), self.fetchIndices(step - 1)); - break :blk b.andi(self.first_step, block_changed); - }; - const fetch_indices = self.fetchIndices(step); - if (step == 0) { - // static_zero_start=true: Python-int 0 grid indices enable folds in getDmaSlice. - br.copyIn(b, fetch_indices, br.copy_in_slot.?, true); - } else { - var when = b.openIf(pred); - { - br.copyIn(b, fetch_indices, br.copy_in_slot.?, false); - when.close(); - } - } - br.copy_in_slot = advance(b, br.copy_in_slot.?, pred); - } - - fn copyIn(self: *Scheduler, b: *Builder, br: *BufferedRef) void { - if (br.is_trivial) return; - var pred = b.andi(self.willChangeFetch(b, br), self.notOutOfFetch(b, br)); - if (br.isBuffered() and br.buffer_count < 2) pred = b.ori(pred, self.first_step); - if (br.buffer_type != .input and br.buffer_type != .input_output) return; - { - var when = b.openIf(pred); - { - self.namedScope(b, "ep_copy_in"); - br.copyIn(b, self.fetchIndices(br.buffer_count - 1), br.copy_in_slot.?, false); - self.endScope(b); - when.close(); - } - } - br.copy_in_slot = advance(b, br.copy_in_slot.?, pred); - } - - fn waitIn(self: *Scheduler, b: *Builder, br: *BufferedRef) void { - if (br.is_trivial) return; - const pred = b.ori(self.hasChanged(b, br), self.first_step); - { - var when = b.openIf(pred); - { - self.namedScope(b, "ep_wait_in"); - if (br.buffer_type == .input or br.buffer_type == .input_output) br.waitIn(b, self.indices()); - self.endScope(b); - when.close(); - } - } - } - - fn copyOut(self: *Scheduler, b: *Builder, br: *BufferedRef) void { - if (br.is_trivial) return; - const pred = b.ori(self.willChangeCurrent(b, br), self.last_step); - { - var when = b.openIf(pred); - { - self.namedScope(b, "ep_copy_out"); - if (br.buffer_type == .output or br.buffer_type == .input_output) br.copyOut(b, self.indices()); - self.endScope(b); - when.close(); - } - } - br.copy_out_slot = advance(b, br.copy_out_slot.?, pred); - } - - fn waitOut(self: *Scheduler, b: *Builder, br: *BufferedRef) void { - if (br.is_trivial) return; - const pred = b.andi(self.hasChanged(b, br), self.notFirstStep(b)); - { - var when = b.openIf(pred); - { - self.namedScope(b, "ep_wait_out"); - // copy_out was issued the previous iteration. - if (br.buffer_type == .output or br.buffer_type == .input_output) br.waitOut(b, self.prevIndices()); - self.endScope(b); - when.close(); - } - } - br.wait_out_slot = advance(b, br.wait_out_slot.?, pred); - } - - fn finalize(self: *Scheduler, b: *Builder, br: *BufferedRef) void { - if (br.is_trivial or (br.buffer_type != .output and br.buffer_type != .input_output)) return; - var when = b.openIf(self.last_step); - { - self.namedScope(b, "ep_finalize"); - br.waitOut(b, self.indices()); - self.endScope(b); - when.close(); - } - } - - fn advanceSlots(self: *Scheduler, b: *Builder, br: *BufferedRef) void { - if (br.buffer_type != .input and br.buffer_type != .input_output) return; - if (br.is_trivial) { - switch (br.buffer_type) { - .input => br.slot = advance(b, br.slot.?, self.last_step), - .input_output => br.wait_in_slot = advance(b, br.wait_in_slot.?, self.last_step), - .output => unreachable, - } - } else { - const pred = b.ori(self.willChangeCurrent(b, br), self.last_step); - br.wait_in_slot = advance(b, br.wait_in_slot.?, pred); - } - } -}; - -pub fn memRefAlloca(b: *Builder, ty: *const mlir.Type) Value { - return b.emit(memref.alloca(b.ctx, ty, &.{}, &.{}, .{}, b.loc())); -} - -pub fn semAllocOp(b: *Builder, ty: *const mlir.Type) Value { - return b.emit(mosaic_tpu.sem_alloc(b.ctx, ty, b.loc())); -} - -/// `pl.SemaphoreType.DMA((n,))`, or rank-0 for `n == null`. -pub fn dmaSemRefTy(b: *Builder, n: ?usize) *const mlir.Type { - const shape: []const i64 = if (n) |m| &.{@intCast(m)} else &.{}; - return mlir.Type.memRef(mosaic_tpu.dmaSemaphoreType(b.ctx), shape, null, MemorySpace.semaphore_mem.attribute(b.ctx)); -} - -fn gridSize(b: *Builder, grid: []const Value) Value { - var acc = grid[0]; - for (grid[1..]) |g| acc = b.muli(acc, g); - return acc; -} - -fn filterIndices(b: *Builder, indices: []const Value, grid: []const Value) [4]Value { - var out: [4]Value = undefined; - const c0 = b.constant(.i32, 0); - for (indices, grid, 0..) |i, g, pos| { - out[pos] = if (g.asConstantInt() == @as(?i64, 1)) c0 else i; - } - return out; -} - -fn nextIndexND(b: *Builder, indices: []const Value, grid: []const Value) []const Value { - const a = b.arena.allocator(); - const out = a.alloc(Value, indices.len) catch @panic("nextIndexND OOM"); - const c0 = b.constant(.i32, 0); - var carry: Value = b.cmpi(.eq, c0, c0); - var rev: usize = 0; - while (rev < indices.len) : (rev += 1) { - const pos = indices.len - 1 - rev; - const inc = b.select(carry, b.addi(indices[pos], b.constant(.i32, 1)), indices[pos]); - carry = b.cmpi(.eq, inc, grid[pos]); - out[pos] = b.select(carry, c0, inc); - } - const filtered = filterIndices(b, out, grid); - for (0..indices.len) |i| out[i] = filtered[i]; - return out; -} - -fn prevIndexND(b: *Builder, indices: []const Value, grid: []const Value) []const Value { - const a = b.arena.allocator(); - const out = a.alloc(Value, indices.len) catch @panic("prevIndexND OOM"); - const c0 = b.constant(.i32, 0); - var borrow: Value = b.cmpi(.eq, c0, c0); - const cm1 = b.constant(.i32, -1); - var rev: usize = 0; - while (rev < indices.len) : (rev += 1) { - const pos = indices.len - 1 - rev; - const dec = b.select(borrow, b.subi(indices[pos], b.constant(.i32, 1)), indices[pos]); - borrow = b.cmpi(.eq, dec, cm1); - out[pos] = b.select(borrow, b.subi(grid[pos], b.constant(.i32, 1)), dec); - } - const filtered = filterIndices(b, out, grid); - for (0..indices.len) |i| out[i] = filtered[i]; - return out; -} - -/// `null` block_shape, or every dim is `.blocked(full_dim)`. -fn specHasTrivialWindowing(spec: PipeSpec) bool { - const bs = spec.block_shape orelse return true; - if (bs.len != spec.full_shape.len) return false; - for (bs, spec.full_shape) |bd, fs| switch (bd) { - .blocked => |n| if (n != fs) return false, - else => return false, - }; - return true; -} - -test { - std.testing.refAllDecls(@This()); -} - -test "trivial input-output ref is copied in and out" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - builder.registerDialects(registry); - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - - var b = try Builder.init(std.testing.allocator, ctx, "trivial_input_output", &.{ - .{ - .name = "state", - .kind = .{ .ref = .{ - .shape = &.{ 8, 128 }, - .dtype = .bf16, - .memory_space = .hbm, - } }, - }, - }, &.{}); - defer b.deinit(); - - const noOpBody = struct { - fn call( - _: *Builder, - _: []const Value, - _: []const RefWindow, - _: []const Value, - _: ?*anyopaque, - ) FinishError!void {} - }.call; - - try emitPipeline(&b, &.{b.arg(0)}, &.{}, .{ - .body = noOpBody, - .grid = &.{b.constant(.i32, 1)}, - .in_specs = &.{.{ - .full_shape = &.{ 8, 128 }, - .dtype = .bf16, - .buffer_type = .input_output, - }}, - .out_specs = &.{}, - .trace_scopes = false, - }); - - const ir = try b.finish(&.{}); - defer std.testing.allocator.free(ir); - - // One enqueue/wait pair for copy_in, one for copy_out. - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "tpu.enqueue_dma")); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "tpu.wait_dma")); -} diff --git a/dsl/tcl/BUILD.bazel b/dsl/tcl/BUILD.bazel deleted file mode 100644 index 096a3a67..00000000 --- a/dsl/tcl/BUILD.bazel +++ /dev/null @@ -1,24 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "builder", - srcs = [ - "dtype.zig", - ], - import_name = "dsl/tcl/builder", - main = "builder.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//dsl/common:builder", - "//mlir", - "//mlir/dialects/arith", - "//mlir/dialects/func", - "//mlir/dialects/tcl", - ], -) - -zig_test( - name = "test", - deps = [":builder"], -) diff --git a/dsl/tcl/README.md b/dsl/tcl/README.md deleted file mode 100644 index b01b7189..00000000 --- a/dsl/tcl/README.md +++ /dev/null @@ -1,170 +0,0 @@ -# TCL kernels in Zig - -`dsl/tcl` is a Zig front end for Furiosa's TCL (Tensor Contraction -Language), shaped after the Python `furiosa.tcl` DSL: a kernel declares named -axes and typed tensors, opens tensor operations that `fetch`, `contract`, run -VE instructions and `commit`, and returns tensors. The `Builder` emits the -native `tcl` MLIR dialect (`mlir/dialects/tcl`), and -`zml.kernel.tcl.Kernel` turns it into a `furiosa.tcl` custom call that the -Furiosa plugin exports to TCL source and compiles with `furiosa-tcc` as one -native unit. - -## A kernel - -`furiosa.tcl.examples.softmax`, in Zig: - -```zig -const zml = @import("zml"); -const tcl = zml.kernel.tcl; - -const Cfg = struct { b: i64, s: i64, d: i64 }; - -pub const Softmax = tcl.Kernel(Cfg, .{ - .name = "softmax_kernel", - .inputs = &.{"input"}, - .outputs = &.{"out"}, - .run = run, -}); - -fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - const B = b.axis("B", cfg.b); - const S = b.axis("S", cfg.s); - const D = b.axis("D", cfg.d); - const t = try b.declareArgs(.{ .input = .{ .dtype = .bf16, .axes = &.{ B, S, D } } }); - - const op = b.tensorOperation(.{}); - const x = op.fetch(t.input, .{ .typecast_to = .f32 }); - const exped = x.subf(x.reduce(&.{D}, .maxf)).exp(); - const out = try op.commit(exped.divf(exped.reduce(&.{D}, .addf)), .{ .dtype = .bf16 }); - b.ret(&.{out}); -} - -// In a model: -const y = Softmax.call(.{ .input = x }, .{ .out = x.shape() }, .{ - .cfg = .{ .b = 2, .s = 16, .d = 1024 }, -}).out; -``` - -The plugin compiles it to the same TCL the Python DSL prints: - -``` -def softmax_kernel(B: Axis = 2, D: Axis = 1024, S: Axis = 16, v0: [B, S, D]/bf16) -> [B, S, D]/bf16: - v1: [B, S, D]/bf16 = tk.ReduceByVe( - %0 = read(v0, typecast_to=f32) - %1 = ve.exec(max_f t[D] %0) - %2 = ve.exec(%0 - %1) - %3 = ve.exec(exp %2) - %4 = ve.exec(+ t[D] %3) - %5 = ve.exec(%3 / %4) - write.to([B, S, D]/bf16) - ) - return v1 -``` - -## Python to Zig - -| `furiosa.tcl` | `dsl/tcl` | -|------------------------------------------------|-------------------------------------------------------| -| `x: tcl.Tensor[tcl.bf16, [B, S, D]]` | `.x = .{ .dtype = .bf16, .axes = &.{ B, S, D } }` | -| `x: tcl.Dram[tcl.bf16, [tcl.Broadcast], [M, N]]` | `.x = .{ .dtype = .bf16, .axes = &.{ M, N }, .dram = .{} }` | -| composite axis `C: Axis = (A * B)` | `b.axis("C", b.expr(.mul, A, B))` | -| `@tcl.kernel(config={...}, auto_config={...})` | `b.compilerConfig(.{ ... })`, `b.autoConfig(.{ ... })` | -| `@tcl.tensor_operation` | `const op = b.tensorOperation(.{})` | -| `fetch.typecast(x, tcl.f32)` | `op.fetch(x, .{ .typecast_to = .f32 })` | -| `fetch.broadcast(x, ...)`, `fetch.table_lookup` | `.broadcast_to = &.{...}`, `.table_lookup = table` | -| `fetch.pad(x, axis=L, left=1, right=1)` | `.pad = &.{.{ .axis = L, .left = 1, .right = 1 }}` | -| `fetch.slide(x, axis=L, frame_axis=Lf, window_axis=Lw)` | `.slide = &.{.{ .axis = L, .frame_axis = Lf, .window_axis = Lw }}` | -| `operation.contract(lhs, rhs, to=[B, M, N])` | `op.contract(lhs, rhs, &.{ B, M, N })` | -| `operation.subf(a, b)`, `operation.exp(a)` | `a.subf(b)`, `a.exp()`, or `op.ve(.subf, .{ a, b })` | -| `operation.reduce(x, axes=[D], op=tcl.Maxf)` | `x.reduce(&.{D}, .maxf)` | -| `operation.where(x > 0, a, b)` | `op.where(x, .gt, 0, a, b)` | -| `operation.to_fp(x, 15)`, `to_fxp(x, 15)` | `x.toFp(15)`, `x.toFxp(15)` | -| `compiler_hints=tcl.context(layout={Chip: ...})` | `.context = .{ .layout = .{ .chip = .broadcast } }` | -| `commit.typecast(v, tcl.bf16)` / `commit.permute` | `op.commit(v, .{ .dtype = .bf16, .axes = ... })` | -| `tcl.reshape`, `transmute`, `concat`, `slice` | `b.reshape`, `b.transmute`, `b.concat`, `b.slice` | -| `tcl.gather`, `scatter`, `arange`, `vector`, `all_gather` | `b.gather`, `b.scatter`, `b.arange`, `b.vector`, `b.allGather` | -| `for i in tcl.range(0, n):` (carried `acc`) | `var loop = b.openFor(n, .{acc})` ... `try loop.yield(.{new_acc})`, then `loop.results` | -| `x[i]`, `out[i] = v`, `tcl.scratchpad()` | `b.indexRead(x, loop.iv)`, `b.indexWrite(out, loop.iv, v)`, `b.scratchpad` | -| `core.reduce_max_i32(t)`, `core.eval_expr(...)` | `b.reduceMaxI32(t)`, `b.symExpr(.div, n, 64)` | -| `return out` | `b.ret(&.{out})` | - -* **Axes are static.** `b.axis(name, size)` declares `name: Axis = size`, - with an integer or an `Expr` over other axes; axis names start with an - uppercase letter. `furiosa-tcc` needs every axis bound. Kernel inputs and - outputs must match the XLA shapes axis by axis (row-major), which `call` - checks. -* **Loops.** Put the body in a Zig `{ ... }` block right after `openFor`, - ending with `try loop.yield(...)`. `loop` stays outside, so `loop.results` - stays readable after the block. -* **DRAM tensors.** A `.dram` argument comes back as its `as_logical` view; - `b.asDram(t, .{...})` returns a result in a DRAM mapping. -* **Multi-chip kernels.** A kernel that places data or work on chips (a - chip-split DRAM tensor, a `.context`, `allGather`) is compiled by the plugin - for all the chips of its SPMD program (2 or 4), and runs as one launch - across the partitions; other kernels stay per chip. Its arguments and - results are DRAM tensors: `Dram{ .chip = &.{.{ .axis = C }} }` splits axis - `C`, sized to the chip count, across chips (each partition holds its share), - the default `.broadcast` keeps a copy per chip. -* **Convolutions** pad and slide the input in its read: a slide replaces its - axis by `window_axis, frame_axis`, then `contract` with the filter. On its - own (without a slide) the padding is dropped. -* **The mainstream rule.** As in the Python DSL, instructions are pipelined: - `commit` must take the last instruction, there is at most one contraction - and it comes before VE instructions. Reads are hoisted in front. -* **VE registers are f32 or i32**, decided by the instruction. Scalar - operands become `f32`/`i32` constants (`x.mulf(0.5)`). -* **Tactics are guessed** like `@tcl.tensor_operation` (`EinsumByDpe`, - `ReduceByVe`, `Interleaving`, ...); `b.tensorOperation(.{ .tactic = ... })` - overrides it. -* **Tensors passed directly** to an instruction are read without modifiers. -* **Constant outputs.** `furiosa-tcc` 2026.3 fails ("No producer") on a - function returning two input-independent tensors, such as `arange` and - `vector`. -* **Loops** run on the device. `x[i]` and `acc[i] = v` address the outermost - axis, and the body must index something with `i`: that bounds the trip - count. A dynamic trip count must be computed in the kernel (`reduceMaxI32`, - `symExpr`); one read straight from an argument adds a CPU node the plugin - rejects. - -## Good practices on RNGD - -The TCL CPU interpreter (`furiosa.torch.TclModule`) accepts several programs -that compute garbage on the hardware: check kernels on RNGD. - -* **One VE stream per tensor operation.** Each instruction takes the previous - result and one side operand (a read or a constant). A select whose branches - are both constants must test the operation's first read. -* **Outer broadcasts** (`[B, H, V] * [B, H, K]` to `[B, H, V, K]`, tactic - `EinsumByVe`) need a shared leading axis, and the read with more axes first. -* **Contraction epilogues** read their extra tensors before the contraction - and combine the contraction result first: `contract(q, k).addf(bias)`. -* **Conversions.** `typecast_to` converts floats only; `toFp(n)` of an i32 - yields `x / 2^31` whatever `n`. -* **Update caches in place.** `scatter(rows, idx, axis, .{ .init = cache })` - on a kernel argument shares its buffer with the result (a whole-cache copy - otherwise), so return it as an aliased output. -* **Few axes, few operations.** tcc's tactic search grows with the number and - extents of axes, and more than linearly with the number of operations in a - function: merge axes before contractions (`[N, H, G*Q, C]`, not - `[N, H, G, Q, P, C]`) and set `.lowering_mode = "Heuristic"` above a few - tokens. -* **Prefer one-shot graphs to hand-written online softmax.** Furiosa's - production attention gathers the context, contracts, masks with a select, - runs the softmax as one `ReduceByVe` and contracts again; gathers take - `valid_length` to skip padding, and padding indices are spread - (`idx | (pos % n)`) rather than all pointing at one row. -* **Furiosa's kernels** in the SDK (`furiosa/kernels/`) are the reference for - layouts, tile sizes and compiler configs: `common/attention.py` (attention - and decode/prefill configs), `common/topk.py` and - `common/blockwise_moe_index.py` (routing on the device), - `qwen3_moe/optimized/` (blockwise MoE for prefill, activation-stationary MoE - for decode), `exaone_moe/k_exaone_w4fa16kv16/flash_attention.py` (chunked - softmax over a dynamic trip count). - -## The custom call - -`ops.tcl` emits `stablehlo.custom_call @furiosa.tcl` with the printed module -as its backend config. The plugin parses it, checks the function signature -against the operand and result shapes, exports TCL source and compiles it -with the rest of the program's units. With `--xla_dump_to=`, -`*.furiosa-unit-sources.txt` holds each unit's TCL. diff --git a/dsl/tcl/builder.zig b/dsl/tcl/builder.zig deleted file mode 100644 index f86d338b..00000000 --- a/dsl/tcl/builder.zig +++ /dev/null @@ -1,1429 +0,0 @@ -const std = @import("std"); - -const arith = @import("mlir/dialects/arith"); -const common = @import("dsl/common/builder"); -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const tcl = @import("mlir/dialects/tcl"); - -pub const dtypes = @import("dtype.zig"); - -pub const DType = dtypes.DType; -pub const Tactic = tcl.Tactic; -pub const ReduceMode = tcl.ReduceMode; -pub const Predicate = tcl.Predicate; - -pub fn registerDialects(registry: *mlir.DialectRegistry) void { - inline for (.{ func, arith, tcl }) |dialect| dialect.registerDialect(registry); -} - -pub const FinishError = error{ InvalidMlir, OutOfMemory } || std.Io.Writer.Error; - -test { - std.testing.refAllDecls(@This()); -} - -/// A named TCL axis with its extent, from `Builder.axis`. -pub const Axis = struct { - name: []const u8, - size: i64, -}; - -/// A TCL expression over axes and integers, for composite axes -/// (`C: Axis = (A * B)`) and DRAM mappings (`D @ Dp`, `F / 2`, `Broadcast`). -/// `Builder.expr` makes the nodes. -pub const Expr = union(enum) { - int: i64, - axis: Axis, - broadcast, - node: *const Node, - - pub const Kind = enum { add, sub, mul, div, rem, exact_div, stride, modulo, padding, resize, pair }; - pub const Node = struct { kind: Kind, lhs: Expr, rhs: Expr }; - - /// The value of integer arithmetic over axes; null for mappings. - pub fn value(self: Expr) ?i64 { - return switch (self) { - .int => |n| n, - .axis => |a| a.size, - .broadcast => null, - .node => |n| { - const l = n.lhs.value() orelse return null; - const r = n.rhs.value() orelse return null; - return switch (n.kind) { - .add => l + r, - .sub => l - r, - .mul => l * r, - .div, .exact_div => if (r == 0) null else @divFloor(l, r), - .rem => if (r == 0) null else @mod(l, r), - else => null, - }; - }, - }; - } -}; - -/// `dram(chip | inner)/T`: a tensor in an explicit DRAM mapping, Python's -/// `tcl.Dram[T, chip, inner, tcl.original[...]]`. `chip` is `.broadcast` -/// (a copy per chip) or the axes split across chips; `inner` defaults to the -/// tensor's other axes. -pub const Dram = struct { - chip: []const Expr = &.{.broadcast}, - inner: ?[]const Expr = null, - original: []const Axis = &.{}, - - fn shards(self: Dram) i64 { - var n: i64 = 1; - for (self.chip) |e| if (e == .axis) { - n *= e.axis.size; - }; - return n; - } - - fn isChipAxis(self: Dram, a: Axis) bool { - for (self.chip) |e| if (e == .axis and std.mem.eql(u8, e.axis.name, a.name)) return true; - return false; - } -}; - -/// `@ context(operator = {...}, heuristic_hint = {...})`, Python's -/// `tcl.context(layout=..., heuristic_hint=...)`. -pub const Context = struct { - layout: ?Layout = null, - heuristic_hint: []const Hint = &.{}, - - /// `{Chip: ..., Cluster: ..., Split: [...]}`: `chip` and `cluster` are an - /// `.axis` or `.broadcast`. - pub const Layout = struct { - chip: Expr, - cluster: ?Expr = null, - split: []const Axis = &.{}, - }; - - pub const Hint = struct { - name: []const u8, - value: union(enum) { int: i64, float: f64, boolean: bool, axis: Axis }, - }; -}; - -/// The VE (vector engine) instructions, named like `furiosa.tcl.operation`. -pub const VeOp = enum { - exp, - neg_exp, - sqrt, - tanh, - sigmoid, - erf, - log, - sin, - cos, - negf, - absf, - addi, - addi_sat, - subi, - subi_sat, - mul_q31, - muli, - shl, - shl_sat, - shr_logical, - shr_arith, - shr_arith_round, - bit_andi, - bit_ori, - bit_xori, - mini, - maxi, - absmini, - absmaxi, - addf, - subf, - mulf, - divf, - bit_andf, - bit_orf, - bit_xorf, - minf, - maxf, - absminf, - absmaxf, - reinterpret_as_float, - reinterpret_as_int, - to_float, - to_int, - - pub fn arity(self: VeOp) usize { - return switch (self) { - .exp, .neg_exp, .sqrt, .tanh, .sigmoid, .erf, .log, .sin, .cos, .negf, .absf, .reinterpret_as_float, .reinterpret_as_int, .to_float, .to_int => 1, - else => 2, - }; - } - - /// VE registers are f32 or i32, according to the instruction. - pub fn result(self: VeOp) DType { - return switch (self) { - .exp, .neg_exp, .sqrt, .tanh, .sigmoid, .erf, .log, .sin, .cos, .negf, .absf, .addf, .subf, .mulf, .divf, .bit_andf, .bit_orf, .bit_xorf, .minf, .maxf, .absminf, .absmaxf, .reinterpret_as_float, .to_float => .f32, - else => .i32, - }; - } -}; - -/// A graph-level tensor: a kernel argument, or the result of a tensor or -/// graph operation (`x: [A, B]/bf16` in TCL). -pub const Tensor = struct { - inner: *const mlir.Value, - dtype: DType, - axes: []const Axis, - /// Chips a DRAM tensor is split across: each SPMD partition holds - /// 1/shards of it. - shards: i64 = 1, -}; - -/// A value inside a tensor operation (`%N` in TCL): the result of a read, -/// the contraction or a VE instruction. -pub const Value = struct { - op: *TensorOperation, - inner: *const mlir.Value, - dtype: DType, - axes: []const Axis, - - pub fn unary(self: Value, comptime opcode: VeOp) Value { - return self.op.ve(opcode, .{self}); - } - - /// `rhs` is a `Value`, a `Tensor` (read on use) or a scalar literal. - pub fn binary(self: Value, comptime opcode: VeOp, rhs: anytype) Value { - return self.op.ve(opcode, .{ self, rhs }); - } - - pub fn reduce(self: Value, axes: []const Axis, mode: ReduceMode) Value { - return self.op.reduce(self, axes, mode); - } - - pub fn cumsum(self: Value, axes: []const Axis) Value { - return self.op.cumsum(self, axes); - } - - pub fn addf(self: Value, rhs: anytype) Value { - return self.binary(.addf, rhs); - } - pub fn subf(self: Value, rhs: anytype) Value { - return self.binary(.subf, rhs); - } - pub fn mulf(self: Value, rhs: anytype) Value { - return self.binary(.mulf, rhs); - } - pub fn divf(self: Value, rhs: anytype) Value { - return self.binary(.divf, rhs); - } - pub fn maxf(self: Value, rhs: anytype) Value { - return self.binary(.maxf, rhs); - } - pub fn minf(self: Value, rhs: anytype) Value { - return self.binary(.minf, rhs); - } - pub fn addi(self: Value, rhs: anytype) Value { - return self.binary(.addi, rhs); - } - pub fn subi(self: Value, rhs: anytype) Value { - return self.binary(.subi, rhs); - } - pub fn muli(self: Value, rhs: anytype) Value { - return self.binary(.muli, rhs); - } - pub fn exp(self: Value) Value { - return self.unary(.exp); - } - pub fn sqrt(self: Value) Value { - return self.unary(.sqrt); - } - pub fn tanh(self: Value) Value { - return self.unary(.tanh); - } - pub fn sigmoid(self: Value) Value { - return self.unary(.sigmoid); - } - pub fn log(self: Value) Value { - return self.unary(.log); - } - pub fn negf(self: Value) Value { - return self.unary(.negf); - } - pub fn toFloat(self: Value) Value { - return self.unary(.to_float); - } - pub fn toInt(self: Value) Value { - return self.unary(.to_int); - } - pub fn toFp(self: Value, int_width: u5) Value { - return self.op.toFp(self, int_width); - } - pub fn toFxp(self: Value, int_width: u5) Value { - return self.op.toFxp(self, int_width); - } -}; - -/// One TCL tensor operation, `tk.(...)`: reads, at most one -/// contraction, VE instructions, then `commit` writes its result tensor. -/// Like `@tcl.tensor_operation`, every instruction after the reads is -/// pipelined: `commit` takes the last one. -pub const TensorOperation = struct { - pub const Options = struct { - /// Guessed from the instructions like the Python DSL when null. - tactic: ?Tactic = null, - context: ?Context = null, - }; - - pub const FetchOptions = struct { - typecast_to: ?DType = null, - broadcast_to: ?[]const Axis = null, - subtraction: ?Tensor = null, - table_lookup: ?Tensor = null, - /// Padding along an axis, applied by a `slide` over it. - pad: []const Pad = &.{}, - /// Sliding windows: each replaces its axis by `window_axis, frame_axis`. - slide: []const Slide = &.{}, - }; - - pub const Slide = struct { - axis: Axis, - frame_axis: Axis, - window_axis: Axis, - /// The window before dilation; `window_axis.size` by default. - undilated_window: ?i64 = null, - stride: i64 = 1, - dilation: i64 = 1, - }; - - pub const Pad = struct { - axis: Axis, - left: i64 = 0, - right: i64 = 0, - fill: f64 = 0, - }; - - pub const CommitOptions = struct { - dtype: ?DType = null, - /// A permutation of the committed value's axes. - axes: ?[]const Axis = null, - }; - - b: *Builder, - tactic: ?Tactic, - context: ?Context, - body: *mlir.Block, - inputs: std.ArrayList(Tensor) = .empty, - plain_reads: std.ArrayList(struct { tensor: *const mlir.Value, value: Value }) = .empty, - first_compute: ?*mlir.Operation = null, - last: ?*const mlir.Value = null, - has_dpe: bool = false, - has_ve: bool = false, - has_reduce: bool = false, - has_select: bool = false, - - /// `read(t, typecast_to=..., broadcast_to=..., ...)`. - pub fn fetch(self: *TensorOperation, t: Tensor, opts: FetchOptions) Value { - const b = self.b; - const ctx = b.ctx; - var operands: [3]*const mlir.Value = .{ self.input(t), undefined, undefined }; - var n: usize = 1; - var fields: [6]mlir.NamedAttribute = undefined; - var nfields: usize = 0; - var dtype = t.dtype; - var axes = t.axes; - if (opts.subtraction) |s| { - operands[n] = self.input(s); - fields[nfields] = .named(ctx, "subtraction", .int(ctx, .i64, n)); - n += 1; - nfields += 1; - } - if (opts.table_lookup) |table| { - operands[n] = self.input(table); - fields[nfields] = .named(ctx, "table_lookup", .int(ctx, .i64, n)); - dtype = table.dtype; - n += 1; - nfields += 1; - } - if (opts.typecast_to) |to| { - fields[nfields] = .named(ctx, "typecast_to", .typeAttr(to.toMlir(ctx))); - dtype = to; - nfields += 1; - } - if (opts.pad.len > 0) { - const pads = b.arena.allocator().alloc(mlir.NamedAttribute, opts.pad.len) catch @panic("OOM"); - for (pads, opts.pad) |*dst, p| dst.* = .named(ctx, p.axis.name, .array(ctx, &.{ - .int(ctx, .i64, p.left), - .int(ctx, .i64, p.right), - .float(ctx, .f64, p.fill), - })); - fields[nfields] = .named(ctx, "pad", .dict(ctx, pads)); - nfields += 1; - } - if (opts.slide.len > 0) { - const slides = b.arena.allocator().alloc(mlir.NamedAttribute, opts.slide.len) catch @panic("OOM"); - var slid: std.ArrayList(Axis) = .empty; - slid.appendSlice(b.arena.allocator(), axes) catch @panic("OOM"); - for (slides, opts.slide) |*dst, sl| { - dst.* = .named(ctx, sl.axis.name, .dict(ctx, &.{ - .named(ctx, "undilated_window", .int(ctx, .i64, sl.undilated_window orelse sl.window_axis.size)), - .named(ctx, "frame_axis", b.axisAttr(sl.frame_axis)), - .named(ctx, "window_axis", b.axisAttr(sl.window_axis)), - .named(ctx, "stride", .int(ctx, .i64, sl.stride)), - .named(ctx, "dilation", .int(ctx, .i64, sl.dilation)), - })); - const at = for (slid.items, 0..) |a, i| { - if (std.mem.eql(u8, a.name, sl.axis.name)) break i; - } else std.debug.panic("tcl: {s}: slide axis {s} is not read", .{ b.name, sl.axis.name }); - slid.items[at] = sl.window_axis; - slid.insert(b.arena.allocator(), at + 1, sl.frame_axis) catch @panic("OOM"); - } - fields[nfields] = .named(ctx, "slide", .dict(ctx, slides)); - nfields += 1; - axes = slid.items; - } - if (opts.broadcast_to) |to| { - fields[nfields] = .named(ctx, "broadcast_to", b.axesAttr(to)); - axes = b.dupeAxes(to); - nfields += 1; - } - const options = if (nfields == 0) null else tcl.readOptionsAttr(ctx, fields[0..nfields]) catch - std.debug.panic("tcl: invalid read options for {s}", .{b.name}); - const op = tcl.read(ctx, operands[0..n], b.logicalType(dtype, axes), .{ .options = options }, b.loc()); - return self.emit(op, dtype, axes, true); - } - - /// `dpe.exec(lhs, rhs -> [to])`: shared axes missing from `to` are - /// contracted. Accumulates in f32 (float inputs) or i32. - pub fn contract(self: *TensorOperation, lhs: anytype, rhs: anytype, to: []const Axis) Value { - if (self.has_dpe or self.has_ve) std.debug.panic("tcl: one contraction per tensor operation, before VE instructions", .{}); - const l = self.lift(lhs, .f32); - const r = self.lift(rhs, .f32); - const dtype: DType = if (l.dtype.isFloat()) .f32 else .i32; - self.has_dpe = true; - const op = tcl.dpe(self.b.ctx, l.inner, r.inner, self.b.logicalType(dtype, to), self.b.loc()); - return self.emit(op, dtype, self.b.dupeAxes(to), false); - } - - /// `ve.exec(...)` of a unary or binary instruction. Operands are - /// `Value`s, `Tensor`s (read on use) or scalar literals. - pub fn ve(self: *TensorOperation, comptime opcode: VeOp, operands: anytype) Value { - if (comptime operands.len != opcode.arity()) @compileError("tcl: wrong operand count for " ++ @tagName(opcode)); - return self.veNamed(@tagName(opcode), opcode.result(), operands); - } - - /// `ve.exec(x to_f)`: Q-format fixed point to f32, Python's - /// `operation.to_fp`. - pub fn toFp(self: *TensorOperation, x: anytype, int_width: u5) Value { - var buf: [8]u8 = undefined; - return self.veNamed(std.fmt.bufPrint(&buf, "to_f{d}", .{int_width}) catch unreachable, .f32, .{x}); - } - - /// `ve.exec(x to_i)`: f32 to Q-format fixed point, Python's - /// `operation.to_fxp`. - pub fn toFxp(self: *TensorOperation, x: anytype, int_width: u5) Value { - var buf: [8]u8 = undefined; - return self.veNamed(std.fmt.bufPrint(&buf, "to_i{d}", .{int_width}) catch unreachable, .i32, .{x}); - } - - fn veNamed(self: *TensorOperation, opcode: []const u8, scalar: DType, operands: anytype) Value { - var values: [2]*const mlir.Value = undefined; - var axes: []const Axis = &.{}; - inline for (operands, 0..) |o, i| { - const v = self.lift(o, scalar); - values[i] = v.inner; - axes = self.b.unionAxes(axes, v.axes); - } - self.has_ve = true; - const ctx = self.b.ctx; - const code = tcl.veOpcodeAttr(ctx, opcode) catch std.debug.panic("tcl: unknown VE instruction {s}", .{opcode}); - const op = tcl.ve(ctx, values[0..operands.len], self.b.logicalType(scalar, axes), code, self.b.loc()); - return self.emit(op, scalar, axes, false); - } - - /// `ve.exec( t[axes] x)`: removes `axes`. - pub fn reduce(self: *TensorOperation, x: anytype, axes: []const Axis, mode: ReduceMode) Value { - if (mode == .cumsum) return self.cumsum(x, axes); - const v = self.lift(x, .f32); - var kept: std.ArrayList(Axis) = .empty; - for (v.axes) |a| { - if (!containsAxis(axes, a)) kept.append(self.b.arena.allocator(), a) catch @panic("OOM"); - } - self.has_ve = true; - self.has_reduce = true; - const op = tcl.ve_reduce(self.b.ctx, v.inner, self.b.logicalType(v.dtype, kept.items), mode, self.b.axisAttrs(axes), self.b.loc()); - return self.emit(op, v.dtype, kept.items, false); - } - - /// `cumsum t[axes] x`, like `operation.cumsum`: the running sum of an i32 - /// value along `axes`, flattened in that order. - pub fn cumsum(self: *TensorOperation, x: anytype, axes: []const Axis) Value { - const v = self.lift(x, .i32); - self.has_ve = true; - self.has_reduce = true; - const op = tcl.ve_reduce(self.b.ctx, v.inner, self.b.logicalType(v.dtype, v.axes), .cumsum, self.b.axisAttrs(axes), self.b.loc()); - return self.emit(op, v.dtype, v.axes, false); - } - - /// `ve.exec(if cond threshold { yes } else { no })`, like - /// `operation.where(cond threshold, yes, no)`. - pub fn where(self: *TensorOperation, cond: anytype, pred: Predicate, threshold: anytype, yes: anytype, no: anytype) Value { - const ctx = self.b.ctx; - const c = self.lift(cond, .f32); - const float = isFloatOperand(yes) or isFloatOperand(no); - const scalar: DType = if (float) .f32 else .i32; - const t = self.lift(yes, scalar); - const f = self.lift(no, scalar); - const threshold_attr: *const mlir.Attribute = if (c.dtype.isFloat()) - .float(ctx, .f32, threshold) - else - .int(ctx, .i32, threshold); - const axes = self.b.unionAxes(self.b.unionAxes(c.axes, t.axes), f.axes); - self.has_ve = true; - self.has_select = true; - const op = tcl.ve_select(ctx, c.inner, t.inner, f.inner, self.b.logicalType(scalar, axes), pred, threshold_attr, self.b.loc()); - return self.emit(op, scalar, axes, false); - } - - /// `write.to(...)`: closes the tensor operation and returns its result. - pub fn commit(self: *TensorOperation, v: Value, opts: CommitOptions) FinishError!Tensor { - const b = self.b; - if (self.last != v.inner) std.debug.panic("tcl: {s}: commit must take the last instruction", .{b.name}); - const axes = if (opts.axes) |a| b.dupeAxes(a) else v.axes; - if (!sameAxisSet(axes, v.axes)) std.debug.panic("tcl: {s}: commit can only permute axes", .{b.name}); - const dtype = opts.dtype orelse v.dtype; - _ = tcl.write(b.ctx, v.inner, b.loc()).appendTo(self.body); - - const inputs = try b.arena.allocator().alloc(*const mlir.Value, self.inputs.items.len); - for (inputs, self.inputs.items) |*dst, t| dst.* = t.inner; - const context = if (self.context) |c| b.contextAttr(c) else null; - const kernel = tcl.kernel(b.ctx, inputs, b.logicalType(dtype, axes), self.tactic orelse self.guessTactic(axes), self.body, .{ .context = context }, b.loc()); - try b.append(kernel); - return .{ .inner = kernel.result(0), .dtype = dtype, .axes = axes }; - } - - /// Python's `BuilderHistory.guess_tactic_kind`. - fn guessTactic(self: *const TensorOperation, output: []const Axis) Tactic { - if (self.has_dpe) return .einsum_by_dpe; - if (!self.has_ve) return .tensor_operation; - if (self.has_reduce) return .reduce_by_ve; - const in = self.inputs.items; - if (in.len > 0 and in[0].axes.len < output.len) return .einsum_by_ve; - if (!self.has_select and in.len == 2 and sameAxisSet(in[0].axes, in[1].axes)) return .interleaving; - return .elementwise; - } - - const Operand = struct { inner: *const mlir.Value, dtype: DType, axes: []const Axis }; - - fn lift(self: *TensorOperation, x: anytype, scalar: DType) Operand { - const T = @TypeOf(x); - if (T == Value) return .{ .inner = x.inner, .dtype = x.dtype, .axes = x.axes }; - if (T == Tensor) { - const r = self.plainRead(x); - return .{ .inner = r.inner, .dtype = r.dtype, .axes = r.axes }; - } - const ctx = self.b.ctx; - const op = switch (@typeInfo(T)) { - .comptime_int, .int => if (scalar == .f32) - arith.constant_float(ctx, @floatFromInt(x), .f32, self.b.loc()) - else - arith.constant_int(ctx, x, .int(ctx, .i32), self.b.loc()), - .comptime_float, .float => blk: { - if (scalar != .f32) std.debug.panic("tcl: float literal for an integer VE instruction", .{}); - break :blk arith.constant_float(ctx, x, .f32, self.b.loc()); - }, - else => @compileError("tcl: expected a Value, a Tensor or a scalar, got " ++ @typeName(T)), - }; - _ = op.appendTo(self.body); - return .{ .inner = op.result(0), .dtype = scalar, .axes = &.{} }; - } - - /// Registers `t` as a kernel operand. - fn input(self: *TensorOperation, t: Tensor) *const mlir.Value { - if (t.inner.type_().isA(tcl.LogicalType) == null) - std.debug.panic("tcl: {s}: tensor operations read logical tensors, not DRAM ones", .{self.b.name}); - for (self.inputs.items) |known| { - if (known.inner == t.inner) return t.inner; - } - self.inputs.append(self.b.arena.allocator(), t) catch @panic("OOM"); - return t.inner; - } - - fn plainRead(self: *TensorOperation, t: Tensor) Value { - for (self.plain_reads.items) |r| if (r.tensor == t.inner) return r.value; - const r = self.fetch(t, .{}); - self.plain_reads.append(self.b.arena.allocator(), .{ .tensor = t.inner, .value = r }) catch @panic("OOM"); - return r; - } - - /// Reads go before every other instruction, as TCL lists them. - fn emit(self: *TensorOperation, op: *mlir.Operation, dtype: DType, axes: []const Axis, is_read: bool) Value { - if (is_read and self.first_compute != null) { - self.body.insertOwnedOperationBefore(self.first_compute.?, op); - } else { - _ = op.appendTo(self.body); - if (!is_read and self.first_compute == null) self.first_compute = op; - self.last = op.result(0); - } - return .{ .op = self, .inner = op.result(0), .dtype = dtype, .axes = axes }; - } -}; - -fn isFloatOperand(x: anytype) bool { - return switch (@typeInfo(@TypeOf(x))) { - .comptime_float, .float => true, - .comptime_int, .int => false, - else => x.dtype.isFloat(), - }; -} - -fn containsAxis(axes: []const Axis, a: Axis) bool { - for (axes) |x| if (std.mem.eql(u8, x.name, a.name)) return true; - return false; -} - -fn sameAxisSet(a: []const Axis, b: []const Axis) bool { - if (a.len != b.len) return false; - for (a) |x| if (!containsAxis(b, x)) return false; - return true; -} - -/// Builds one TCL function, `def name(axes..., args...) -> results:`, as a -/// `func.func` of the native `tcl` dialect. -pub const Builder = struct { - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - ctx: *mlir.Context, - module: *mlir.Module, - name: []const u8, - axes: std.ArrayList(Declared) = .empty, - compiler_config: ?*const mlir.Attribute = null, - auto_config: ?*const mlir.Attribute = null, - fusible: bool = false, - func_op: ?*mlir.Operation = null, - entry: ?*mlir.Block = null, - args: []const Tensor = &.{}, - results: ?[]const Tensor = null, - /// Arguments `update` writes back into, by argument number. - updated: std.ArrayList(usize) = .empty, - /// Blocks being built inside the entry block (loop bodies), innermost last. - block_stack: common.BlockStack = .{}, - - pub fn open(allocator: std.mem.Allocator, ctx: *mlir.Context, name: []const u8) !Builder { - _ = tcl.symbolAttr(ctx, name) catch std.debug.panic("tcl: `{s}` is not a TCL function name", .{name}); - return .{ - .allocator = allocator, - .arena = .init(allocator), - .ctx = ctx, - .module = .init(.unknown(ctx)), - .name = name, - }; - } - - pub fn deinit(self: *Builder) void { - self.module.deinit(); - self.arena.deinit(); - } - - pub fn loc(self: *const Builder) *const mlir.Location { - return .unknown(self.ctx); - } - - const Declared = struct { axis: Axis, def: ?Expr }; - - /// `A: Axis = size`, where `size` is an integer or an `Expr` over other - /// axes (`b.axis("C", b.expr(.mul, A, B))`). Axis names start with an - /// uppercase letter. - pub fn axis(self: *Builder, name: []const u8, size: anytype) Axis { - const def: ?Expr, const n: i64 = if (comptime @TypeOf(size) == Expr) - .{ size, size.value() orelse std.debug.panic("tcl: axis {s} needs an integer size", .{name}) } - else - .{ null, size }; - if (name.len == 0 or !std.ascii.isUpper(name[0]) or n <= 0) - std.debug.panic("tcl: invalid axis {s} = {d}", .{ name, n }); - _ = tcl.symbolAttr(self.ctx, name) catch std.debug.panic("tcl: invalid axis name {s}", .{name}); - for (self.axes.items) |d| if (std.mem.eql(u8, d.axis.name, name)) { - if (d.axis.size != n) std.debug.panic("tcl: axis {s} redeclared as {d} (was {d})", .{ name, n, d.axis.size }); - return d.axis; - }; - const a: Axis = .{ .name = self.arena.allocator().dupe(u8, name) catch @panic("OOM"), .size = n }; - self.axes.append(self.arena.allocator(), .{ .axis = a, .def = def }) catch @panic("OOM"); - return a; - } - - /// An `Expr` node: `lhs` and `rhs` are axes, integers or expressions. - pub fn expr(self: *Builder, kind: Expr.Kind, lhs: anytype, rhs: anytype) Expr { - const node = self.arena.allocator().create(Expr.Node) catch @panic("OOM"); - node.* = .{ .kind = kind, .lhs = toExpr(lhs), .rhs = toExpr(rhs) }; - return .{ .node = node }; - } - - fn toExpr(x: anytype) Expr { - return switch (@TypeOf(x)) { - Expr => x, - Axis => .{ .axis = x }, - else => .{ .int = x }, - }; - } - - /// `@compiler_config({...})`, Python's `@tcl.kernel(config=...)`: an - /// anonymous struct written out as JSON, `.{ .lowering_mode = "Heuristic" }`. - pub fn compilerConfig(self: *Builder, config: anytype) void { - self.compiler_config = tcl.configAttr(self.ctx, self.jsonFields(config)) catch - std.debug.panic("tcl: {s}: invalid compiler config", .{self.name}); - } - - /// Lets the plugin compile the function into the unit of the work around - /// it instead of a unit of its own; the unit's compiler config applies. - pub fn fuse(self: *Builder) void { - self.fusible = true; - } - - /// `@auto_config({...})`, Python's `@tcl.kernel(auto_config=...)`. - pub fn autoConfig(self: *Builder, config: anytype) void { - self.auto_config = tcl.configAttr(self.ctx, self.jsonFields(config)) catch - std.debug.panic("tcl: {s}: invalid auto config", .{self.name}); - } - - /// One field per argument: `.{ .x = .{ .dtype = .bf16, .axes = &.{ M, K } } }`. - /// With `.dram = Dram{...}` (or `.{}`) the argument is DRAM-mapped and the - /// field is its `as_logical` view. - pub fn declareArgs(self: *Builder, spec: anytype) FinishError!ArgsOf(@TypeOf(spec)) { - std.debug.assert(self.entry == null); - const fields = @typeInfo(@TypeOf(spec)).@"struct".field_names; - const scratch = self.arena.allocator(); - var types: [fields.len]*const mlir.Type = undefined; - var locs: [fields.len]*const mlir.Location = undefined; - const args = try scratch.alloc(Tensor, fields.len); - inline for (fields, 0..) |f, i| { - const s = @field(spec, f); - const axes = self.dupeAxes(s.axes); - const dram: ?Dram = if (@hasField(@TypeOf(s), "dram")) toDram(s.dram) else null; - types[i] = if (dram) |d| self.mappedType(s.dtype, axes, d) else self.logicalType(s.dtype, axes); - locs[i] = self.loc(); - args[i] = .{ .inner = undefined, .dtype = s.dtype, .axes = axes, .shards = if (dram) |d| d.shards() else 1 }; - } - const entry = mlir.Block.init(&types, &locs); - // In place from the start: the kernel verifier walks to its inputs' - // regions. `finish` sets the function type. - const func_op = func.func(self.ctx, .{ .name = self.name, .block = entry, .results = &.{}, .location = self.loc(), .visibility = null, .verify = false }); - _ = func_op.appendTo(self.module.body()); - self.func_op = func_op; - self.entry = entry; - self.block_stack.push(self.arena.allocator(), entry); - var named: ArgsOf(@TypeOf(spec)) = undefined; - inline for (fields, 0..) |f, i| { - args[i].inner = entry.argument(i); - if (@hasField(@TypeOf(@field(spec, f)), "dram")) - args[i].inner = try self.graphOp(.as_logical, &.{entry.argument(i)}, self.logicalType(args[i].dtype, args[i].axes), &.{}, null); - @field(named, f) = args[i]; - } - self.args = args; - return named; - } - - fn toDram(d: anytype) Dram { - const D = @TypeOf(d); - if (D == Dram) return d; - return .{ - .chip = if (@hasField(D, "chip")) d.chip else &.{.broadcast}, - .inner = if (@hasField(D, "inner")) d.inner else null, - .original = if (@hasField(D, "original")) d.original else &.{}, - }; - } - - fn ArgsOf(comptime Spec: type) type { - const in = @typeInfo(Spec).@"struct".field_names; - comptime var names: [in.len][]const u8 = undefined; - inline for (in, 0..) |f, i| names[i] = f; - return @Struct(.auto, null, &names, &@splat(Tensor), &@splat(.{})); - } - - /// Opens a `tk.(...)` tensor operation; `commit` closes it. - pub fn tensorOperation(self: *Builder, opts: TensorOperation.Options) *TensorOperation { - _ = self.entryBlock(); - const op = self.arena.allocator().create(TensorOperation) catch @panic("OOM"); - op.* = .{ .b = self, .tactic = opts.tactic, .context = opts.context, .body = mlir.Block.init(&.{}, &.{}) }; - return op; - } - - pub const GraphOptions = struct { context: ?Context = null }; - - /// `reshape(t)` to `to`, same element count. - pub fn reshape(self: *Builder, t: Tensor, to: []const Axis, opts: GraphOptions) FinishError!Tensor { - if (elements(t.axes) != elements(to)) std.debug.panic("tcl: reshape changes the element count", .{}); - return self.graph(.reshape, &.{t}, t.dtype, to, &.{}, opts.context); - } - - /// `transmute(t)`: the same bits as `dtype` over `to`. - pub fn transmute(self: *Builder, t: Tensor, dtype: DType, to: []const Axis, opts: GraphOptions) FinishError!Tensor { - return self.graph(.transmute, &.{t}, dtype, to, &.{}, opts.context); - } - - /// `concat([inputs])` along the one axis of `to` the inputs lack. - pub fn concat(self: *Builder, inputs: []const Tensor, to: []const Axis, opts: GraphOptions) FinishError!Tensor { - return self.graph(.concat, inputs, inputs[0].dtype, to, &.{}, opts.context); - } - - /// `slice(t, axis, offset)`: the window of `to` starting at `offset` along `axis`. - pub fn slice(self: *Builder, t: Tensor, along: Axis, offset: i64, to: []const Axis, opts: GraphOptions) FinishError!Tensor { - return self.graph(.slice, &.{t}, t.dtype, to, &.{ - .named(self.ctx, "axis", self.axisAttr(along)), - .named(self.ctx, "offset", .int(self.ctx, .i64, offset)), - }, opts.context); - } - - pub const GatherOptions = struct { - batch_axis: ?Axis = null, - valid_length: ?Tensor = null, - context: ?Context = null, - }; - - /// `gather(table, indices, axis)`: rows of `table` along `axis`. - pub fn gather(self: *Builder, table: Tensor, indices: Tensor, along: Axis, to: []const Axis, opts: GatherOptions) FinishError!Tensor { - var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "axis", self.axisAttr(along)), undefined }; - if (opts.batch_axis) |a| options[1] = .named(self.ctx, "batch_axis", self.axisAttr(a)); - var inputs: [3]Tensor = .{ table, indices, opts.valid_length orelse undefined }; - const n_inputs = @as(usize, 2) + @intFromBool(opts.valid_length != null); - return self.graph(.gather, inputs[0..n_inputs], table.dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.batch_axis != null)], opts.context); - } - - pub const ScatterOptions = struct { - /// The tensor scattered into; zeros otherwise. - init: ?Tensor = null, - batch_axis: ?Axis = null, - context: ?Context = null, - }; - - /// `scatter(updates, indices, axis, init=...)`: `updates` written at - /// `indices` along `axis` of the result. - pub fn scatter(self: *Builder, updates: Tensor, indices: Tensor, along: Axis, to: []const Axis, opts: ScatterOptions) FinishError!Tensor { - var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "axis", self.axisAttr(along)), undefined }; - if (opts.batch_axis) |a| options[1] = .named(self.ctx, "batch_axis", self.axisAttr(a)); - var inputs: [3]Tensor = .{ updates, indices, opts.init orelse undefined }; - const n_inputs = @as(usize, 2) + @intFromBool(opts.init != null); - return self.graph(.scatter, inputs[0..n_inputs], updates.dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.batch_axis != null)], opts.context); - } - - pub const ArangeOptions = struct { - start: ?i64 = null, - step: ?i64 = null, - context: ?Context = null, - }; - - /// `arange(start=, end, step=)` over the one axis of `to`. - pub fn arange(self: *Builder, end: i64, dtype: DType, to: []const Axis, opts: ArangeOptions) FinishError!Tensor { - var options: [3]mlir.NamedAttribute = undefined; - var n: usize = 0; - if (opts.start) |v| { - options[n] = .named(self.ctx, "start", .int(self.ctx, .i64, v)); - n += 1; - } - options[n] = .named(self.ctx, "end", .int(self.ctx, .i64, end)); - n += 1; - if (opts.step) |v| { - options[n] = .named(self.ctx, "step", .int(self.ctx, .i64, v)); - n += 1; - } - return self.graph(.arange, &.{}, dtype, to, options[0..n], opts.context); - } - - pub const VectorOptions = struct { - /// Repeats `values` cyclically along this axis. - repeat_to: ?Axis = null, - context: ?Context = null, - }; - - /// `vector([values], repeat_to=)`: a constant tensor. - pub fn vector(self: *Builder, values: []const f64, dtype: DType, to: []const Axis, opts: VectorOptions) FinishError!Tensor { - const attrs = try self.arena.allocator().alloc(*const mlir.Attribute, values.len); - for (attrs, values) |*dst, v| dst.* = if (dtype.isFloat()) .float(self.ctx, .f64, v) else .int(self.ctx, .i64, @as(i64, @intFromFloat(v))); - var options: [2]mlir.NamedAttribute = .{ .named(self.ctx, "values", .array(self.ctx, attrs)), undefined }; - if (opts.repeat_to) |a| options[1] = .named(self.ctx, "repeat_to", self.axisAttr(a)); - return self.graph(.vector, &.{}, dtype, to, options[0 .. @as(usize, 1) + @intFromBool(opts.repeat_to != null)], opts.context); - } - - /// `all_gather(t, axis)` across chips. - pub fn allGather(self: *Builder, t: Tensor, along: Axis, opts: GraphOptions) FinishError!Tensor { - return self.graph(.all_gather, &.{t}, t.dtype, t.axes, &.{ - .named(self.ctx, "axis", self.axisAttr(along)), - }, opts.context); - } - - /// `as_dram(t)`: `t` in the DRAM mapping `dram`, to return it as such. - pub fn asDram(self: *Builder, t: Tensor, dram: Dram) FinishError!Tensor { - const out = try self.graphOp(.as_dram, &.{t.inner}, self.mappedType(t.dtype, t.axes, dram), &.{}, null); - return .{ .inner = out, .dtype = t.dtype, .axes = t.axes, .shards = dram.shards() }; - } - - /// Any `tcl.graph.*` operation; `options` are its TCL keyword fields. - pub fn graph(self: *Builder, comptime op: tcl.GraphOp, inputs: []const Tensor, dtype: DType, to: []const Axis, options: []const mlir.NamedAttribute, context: ?Context) FinishError!Tensor { - const values = try self.arena.allocator().alloc(*const mlir.Value, inputs.len); - for (values, inputs) |*v, t| v.* = t.inner; - const axes = self.dupeAxes(to); - const out = try self.graphOp(op, values, self.logicalType(dtype, axes), options, context); - return .{ .inner = out, .dtype = dtype, .axes = axes }; - } - - fn graphOp(self: *Builder, comptime op: tcl.GraphOp, inputs: []const *const mlir.Value, output: *const mlir.Type, options: []const mlir.NamedAttribute, context: ?Context) FinishError!*const mlir.Value { - const g = tcl.graph(self.ctx, op, inputs, output, .{ .options = options, .context = if (context) |c| self.contextAttr(c) else null }, self.loc()); - try self.append(g); - return g.result(0); - } - - /// `for iv in 0..limit (acc = init, ...):`, closed by `yield`. - pub fn ForScope(comptime N: usize) type { - return struct { - b: *Builder, - op: *mlir.Operation, - body: *mlir.Block, - /// The loop index, a scalar i32 tensor for `indexRead`, - /// `indexWrite` and `symExpr`. - iv: Tensor, - carried: [N]Tensor, - results: [N]Tensor = undefined, - - const Self = @This(); - - /// `yield values;` closes the loop and sets `results`. - pub fn yield(self: *Self, values: [N]Tensor) FinishError!void { - const b = self.b; - var inner: [N]*const mlir.Value = undefined; - for (&inner, values) |*v, t| v.* = t.inner; - _ = tcl.yield(b.ctx, &inner, b.loc()).appendTo(self.body); - b.popBlock(); - if (!self.op.verify()) { - std.log.err("tcl: {s}: invalid loop:\n{f}", .{ b.name, self.op.fmt(.{ .print_generic_op_form = true }) }); - return error.InvalidMlir; - } - for (&self.results, self.carried, 0..) |*r, c, i| r.* = .{ .inner = self.op.result(i), .dtype = c.dtype, .axes = c.axes, .shards = c.shards }; - } - }; - } - - /// Opens a device loop from 0 to `limit`: an `Axis`, an integer, or a - /// scalar i32 `Tensor` computed in the function (`reduceMaxI32`, - /// `symExpr`). The body must index a tensor with `iv`, which bounds the - /// trip count. `inits` is a tuple of the accumulators' initial tensors. - pub fn openFor(self: *Builder, limit: anytype, inits: anytype) ForScope(@typeInfo(@TypeOf(inits)).@"struct".field_names.len) { - const N = @typeInfo(@TypeOf(inits)).@"struct".field_names.len; - var types: [N + 1]*const mlir.Type = undefined; - var locs: [N + 1]*const mlir.Location = @splat(self.loc()); - var values: [N]*const mlir.Value = undefined; - var carried: [N]Tensor = undefined; - types[0] = self.logicalType(.i32, &.{}); - inline for (0..N) |i| { - carried[i] = inits[i]; - values[i] = inits[i].inner; - types[i + 1] = inits[i].inner.type_(); - } - const body = mlir.Block.init(&types, &locs); - const tensor_limit: ?*const mlir.Value, const bound: ?*const mlir.Attribute = switch (@TypeOf(limit)) { - Tensor => .{ limit.inner, null }, - Axis => .{ null, self.axisAttr(limit) }, - else => .{ null, .int(self.ctx, .i64, limit) }, - }; - // Attached before its body is built: the index operations' verifiers - // look for their loop. - const op = tcl.loop(self.ctx, &values, types[1..], body, .{ .limit = tensor_limit, .bound = bound }, self.loc()); - _ = op.appendTo(self.currentBlock()); - self.pushBlock(body); - for (&carried, 0..) |*c, i| c.inner = body.argument(i + 1); - return .{ .b = self, .op = op, .body = body, .iv = .{ .inner = body.argument(0), .dtype = .i32, .axes = &.{} }, .carried = carried }; - } - - /// `t[i]`: the slice of `t`'s outermost axis at loop index `i`. - pub fn indexRead(self: *Builder, t: Tensor, i: Tensor) FinishError!Tensor { - return self.graph(.index_read, &.{ t, i }, t.dtype, t.axes[1..], &.{}, null); - } - - /// `acc[i] = v`, in place: `acc` must have no other use; use the result. - pub fn indexWrite(self: *Builder, acc: Tensor, i: Tensor, v: Tensor) FinishError!Tensor { - return self.graph(.index_write, &.{ acc, i, v }, acc.dtype, acc.axes, &.{}, null); - } - - /// `scratchpad()`: an uninitialized tensor, typically a loop's output that - /// `indexWrite` fills. - pub fn scratchpad(self: *Builder, dtype: DType, to: []const Axis) FinishError!Tensor { - return self.graph(.scratchpad, &.{}, dtype, to, &.{}, null); - } - - /// `full(value)`: a constant tensor, like `tcl.full`. - pub fn full(self: *Builder, value: f64, dtype: DType, to: []const Axis) FinishError!Tensor { - const v: *const mlir.Attribute = if (dtype.isFloat()) .float(self.ctx, .f64, value) else .int(self.ctx, .i64, @as(i64, @intFromFloat(value))); - return self.graph(.full, &.{}, dtype, to, &.{.named(self.ctx, "value", v)}, null); - } - - /// `reduce_max_i32(t)`: the maximum of an i32 tensor, as a scalar. - pub fn reduceMaxI32(self: *Builder, t: Tensor) FinishError!Tensor { - return self.graph(.reduce_max_i32, &.{t}, .i32, &.{}, &.{}, null); - } - - /// `sym_expr(lhs rhs)`: scalar i32 arithmetic on the device, over - /// scalar tensors (a loop index, `reduceMaxI32`) and integers. - pub fn symExpr(self: *Builder, kind: Expr.Kind, lhs: anytype, rhs: anytype) FinishError!Tensor { - var inputs: [2]Tensor = undefined; - var n: usize = 0; - const l = self.symOperand(lhs, &inputs, &n); - const r = self.symOperand(rhs, &inputs, &n); - const e = tcl.exprAttr(self.ctx, @tagName(kind), &.{ l, r }) catch - std.debug.panic("tcl: {s}: invalid {s} expression", .{ self.name, @tagName(kind) }); - return self.graph(.sym_expr, inputs[0..n], .i32, &.{}, &.{.named(self.ctx, "expr", e)}, null); - } - - fn symOperand(self: *Builder, x: anytype, inputs: *[2]Tensor, n: *usize) *const mlir.Attribute { - if (@TypeOf(x) != Tensor) return .int(self.ctx, .i64, x); - inputs[n.*] = x; - n.* += 1; - return tcl.exprAttr(self.ctx, "arg", &.{.int(self.ctx, .i64, @as(i64, @intCast(n.* - 1)))}) catch unreachable; - } - - /// `copy(src=value, dst=argument)`: `value` written back into the - /// DRAM-mapped argument `index` in place. Return the result like any - /// other: the caller sees it as an output aliasing that argument. - pub fn update(self: *Builder, index: usize, value: Tensor) FinishError!Tensor { - const arg = self.entryBlock().argument(index); - const back = try self.graphOp(.as_dram, &.{value.inner}, arg.type_(), &.{}, null); - try self.append(tcl.copy(self.ctx, back, arg, self.loc())); - self.updated.append(self.arena.allocator(), index) catch @panic("OOM"); - return .{ .inner = arg, .dtype = value.dtype, .axes = value.axes, .shards = self.args[index].shards }; - } - - /// The argument each result updates in place, or null for results the - /// function returns. - pub fn updatedArgument(self: *const Builder, result: Tensor) ?usize { - for (self.updated.items) |i| if (self.entryBlock().argument(i) == result.inner) return i; - return null; - } - - /// `return results...`. - pub fn ret(self: *Builder, results: []const Tensor) void { - self.results = self.arena.allocator().dupe(Tensor, results) catch @panic("OOM"); - } - - /// The verified module: one `func.func` carrying its axes in `tcl.axes`. - pub fn finish(self: *Builder) FinishError![:0]const u8 { - const ctx = self.ctx; - const f = self.func_op orelse return error.InvalidMlir; - const results = self.results orelse return error.InvalidMlir; - const scratch = self.arena.allocator(); - var values: std.ArrayList(*const mlir.Value) = .empty; - var result_types: std.ArrayList(*const mlir.Type) = .empty; - const updated_inputs = try scratch.alloc(i64, results.len); - for (updated_inputs, results) |*u, r| { - if (self.updatedArgument(r)) |i| { - u.* = @intCast(i); - continue; - } - u.* = -1; - try values.append(scratch, r.inner); - try result_types.append(scratch, r.inner.type_()); - } - _ = func.returns(ctx, values.items, self.loc()).appendTo(self.entry.?); - - const entry = self.entry.?; - const arg_types = try scratch.alloc(*const mlir.Type, entry.numArguments()); - for (arg_types, 0..) |*t, i| t.* = entry.argument(i).type_(); - f.setAttributeByName("function_type", .typeAttr(.function(ctx, arg_types, result_types.items))); - - const axes = try scratch.alloc(mlir.NamedAttribute, self.axes.items.len); - for (axes, self.axes.items) |*dst, d| { - dst.* = .named(ctx, d.axis.name, if (d.def) |e| self.exprAttr(e) else .int(ctx, .i64, d.axis.size)); - } - tcl.annotateFunction(ctx, f, axes, .{ - .updated_inputs = if (self.updated.items.len > 0) updated_inputs else null, - .compiler_config = self.compiler_config, - .auto_config = self.auto_config, - .fusible = self.fusible, - }); - try common.verify(self.module, "dsl/tcl"); - - var printed: std.Io.Writer.Allocating = .init(self.allocator); - defer printed.deinit(); - try printed.writer.print("{f}", .{self.module.operation()}); - return try self.allocator.dupeSentinel(u8, printed.written(), 0); - } - - fn append(self: *Builder, op: *mlir.Operation) FinishError!void { - _ = op.appendTo(self.currentBlock()); - if (!op.verify()) { - std.log.err("tcl: {s}: invalid operation:\n{f}", .{ self.name, op.fmt(.{ .print_generic_op_form = true }) }); - return error.InvalidMlir; - } - } - - fn entryBlock(self: *const Builder) *mlir.Block { - return self.entry orelse std.debug.panic("tcl: {s}: declareArgs first", .{self.name}); - } - - pub fn pushBlock(self: *Builder, block: *mlir.Block) void { - self.block_stack.push(self.arena.allocator(), block); - } - - pub fn popBlock(self: *Builder) void { - self.block_stack.pop(); - } - - pub fn currentBlock(self: *const Builder) *mlir.Block { - return self.block_stack.current(); - } - - pub fn axisAttr(self: *const Builder, a: Axis) *const mlir.Attribute { - return tcl.symbolAttr(self.ctx, a.name) catch unreachable; - } - - fn axisAttrs(self: *Builder, axes: []const Axis) []const *const mlir.Attribute { - const attrs = self.arena.allocator().alloc(*const mlir.Attribute, axes.len) catch @panic("OOM"); - for (attrs, axes) |*dst, a| dst.* = self.axisAttr(a); - return attrs; - } - - fn axesAttr(self: *Builder, axes: []const Axis) *const mlir.Attribute { - return .array(self.ctx, self.axisAttrs(axes)); - } - - pub fn exprAttr(self: *Builder, e: Expr) *const mlir.Attribute { - return switch (e) { - .int => |n| .int(self.ctx, .i64, n), - .axis => |a| self.axisAttr(a), - .broadcast => tcl.exprAttr(self.ctx, "broadcast", &.{}) catch unreachable, - .node => |n| tcl.exprAttr(self.ctx, @tagName(n.kind), &.{ self.exprAttr(n.lhs), self.exprAttr(n.rhs) }) catch - std.debug.panic("tcl: {s}: invalid {s} expression", .{ self.name, @tagName(n.kind) }), - }; - } - - fn exprAttrs(self: *Builder, exprs: []const Expr) []const *const mlir.Attribute { - const attrs = self.arena.allocator().alloc(*const mlir.Attribute, exprs.len) catch @panic("OOM"); - for (attrs, exprs) |*dst, e| dst.* = self.exprAttr(e); - return attrs; - } - - pub fn contextAttr(self: *Builder, c: Context) *const mlir.Attribute { - const ctx = self.ctx; - var fields: [2]mlir.NamedAttribute = undefined; - var n: usize = 0; - if (c.layout) |l| { - var layout: [3]mlir.NamedAttribute = undefined; - var m: usize = 0; - layout[m] = .named(ctx, "Chip", self.exprAttr(l.chip)); - m += 1; - if (l.cluster) |cl| { - layout[m] = .named(ctx, "Cluster", self.exprAttr(cl)); - m += 1; - } - if (l.split.len > 0) { - layout[m] = .named(ctx, "Split", self.axesAttr(l.split)); - m += 1; - } - fields[n] = .named(ctx, "operator", .dict(ctx, layout[0..m])); - n += 1; - } - if (c.heuristic_hint.len > 0) { - const hints = self.arena.allocator().alloc(mlir.NamedAttribute, c.heuristic_hint.len) catch @panic("OOM"); - for (hints, c.heuristic_hint) |*dst, h| dst.* = .named(ctx, h.name, switch (h.value) { - .int => |v| .int(ctx, .i64, v), - .float => |v| .float(ctx, .f64, v), - .boolean => |v| .boolean(ctx, v), - .axis => |a| self.axisAttr(a), - }); - fields[n] = .named(ctx, "heuristic_hint", .dict(ctx, hints)); - n += 1; - } - return tcl.contextAttr(ctx, fields[0..n]) catch std.debug.panic("tcl: {s}: invalid context", .{self.name}); - } - - fn jsonFields(self: *Builder, config: anytype) []const mlir.NamedAttribute { - const fields = @typeInfo(@TypeOf(config)).@"struct".field_names; - const out = self.arena.allocator().alloc(mlir.NamedAttribute, fields.len) catch @panic("OOM"); - inline for (fields, 0..) |f, i| out[i] = .named(self.ctx, f, self.json(@field(config, f))); - return out; - } - - fn json(self: *Builder, v: anytype) *const mlir.Attribute { - const T = @TypeOf(v); - return switch (@typeInfo(T)) { - .bool => .boolean(self.ctx, v), - .comptime_int, .int => .int(self.ctx, .i64, v), - .comptime_float, .float => .float(self.ctx, .f64, v), - .enum_literal, .@"enum" => .string(self.ctx, @tagName(v)), - .@"struct" => |info| if (info.is_tuple) blk: { - var items: [info.field_names.len]*const mlir.Attribute = undefined; - inline for (info.field_names, 0..) |f, i| items[i] = self.json(@field(v, f)); - break :blk .array(self.ctx, &items); - } else .dict(self.ctx, self.jsonFields(v)), - .pointer => if (std.meta.Elem(T) == u8) .string(self.ctx, v) else blk: { - const items = self.arena.allocator().alloc(*const mlir.Attribute, v.len) catch @panic("OOM"); - for (items, v) |*dst, x| dst.* = self.json(x); - break :blk .array(self.ctx, items); - }, - else => @compileError("tcl: config values are JSON: bool, numbers, strings, structs, tuples"), - }; - } - - fn mappedType(self: *Builder, dtype: DType, axes: []const Axis, dram: Dram) *const mlir.Type { - for (axes) |a| _ = self.axis(a.name, a.size); - const inner = if (dram.inner) |e| self.exprAttrs(e) else blk: { - var rest: std.ArrayList(Axis) = .empty; - for (axes) |a| if (!dram.isChipAxis(a)) rest.append(self.arena.allocator(), a) catch @panic("OOM"); - break :blk self.axisAttrs(rest.items); - }; - const mapping = tcl.dramMappingAttr(self.ctx, self.exprAttrs(dram.chip), inner, self.axisAttrs(dram.original)) catch - std.debug.panic("tcl: {s}: invalid DRAM mapping", .{self.name}); - return tcl.mappedType(self.ctx, dtype.toMlir(self.ctx), mapping) catch unreachable; - } - - pub fn logicalType(self: *Builder, dtype: DType, axes: []const Axis) *const mlir.Type { - for (axes) |a| _ = self.axis(a.name, a.size); - return tcl.logicalType(self.ctx, dtype.toMlir(self.ctx), self.axisAttrs(axes)) catch - std.debug.panic("tcl: {s}: invalid tensor type (repeated axis?)", .{self.name}); - } - - fn dupeAxes(self: *Builder, axes: []const Axis) []const Axis { - return self.arena.allocator().dupe(Axis, axes) catch @panic("OOM"); - } - - fn unionAxes(self: *Builder, a: []const Axis, b: []const Axis) []const Axis { - var out: std.ArrayList(Axis) = .empty; - const long, const short = if (a.len >= b.len) .{ a, b } else .{ b, a }; - out.appendSlice(self.arena.allocator(), long) catch @panic("OOM"); - for (short) |x| if (!containsAxis(out.items, x)) out.append(self.arena.allocator(), x) catch @panic("OOM"); - return out.items; - } -}; - -fn elements(axes: []const Axis) i64 { - var n: i64 = 1; - for (axes) |a| n *= a.size; - return n; -} - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - registerDialects(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -fn expectContains(haystack: []const u8, needle: []const u8) !void { - if (std.mem.indexOf(u8, haystack, needle) == null) { - std.debug.print("missing {s} in:\n{s}\n", .{ needle, haystack }); - return error.TestExpectedContains; - } -} - -test "softmax matches furiosa.tcl.examples.softmax" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "softmax_kernel"); - defer b.deinit(); - - const B = b.axis("B", 2); - const S = b.axis("S", 16); - const D = b.axis("D", 1024); - const t = try b.declareArgs(.{ .input = .{ .dtype = .bf16, .axes = &.{ B, S, D } } }); - - const op = b.tensorOperation(.{}); - const x = op.fetch(t.input, .{ .typecast_to = .f32 }); - const centered = x.subf(x.reduce(&.{D}, .maxf)); - const exped = centered.exp(); - const out = try op.commit(exped.divf(exped.reduce(&.{D}, .addf)), .{ .dtype = .bf16 }); - b.ret(&.{out}); - - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, "tcl.axes = {B = 2 : i64, D = 1024 : i64, S = 16 : i64}"); - try expectContains(ir, "options = #tcl.read_options<{typecast_to = f32}>"); - try expectContains(ir, "mode = #tcl.reduce_mode<\"Maxf\">"); - try expectContains(ir, "opcode = #tcl.ve_opcode<\"divf\">"); - try expectContains(ir, "tactic = #tcl.tactic<\"ReduceByVe\">"); - try expectContains(ir, "-> !tcl.logical, #tcl.symbol<\"S\">, #tcl.symbol<\"D\">]>"); - const parsed = try mlir.Module.parse(ctx, ir); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "batched matmul, scalar operands and graph reshape" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "bmm_scaled"); - defer b.deinit(); - - const B = b.axis("B", 2); - const M = b.axis("M", 128); - const K = b.axis("K", 64); - const N = b.axis("N", 256); - const t = try b.declareArgs(.{ - .lhs = .{ .dtype = .bf16, .axes = &.{ B, M, K } }, - .rhs = .{ .dtype = .bf16, .axes = &.{ B, K, N } }, - }); - - const mm = b.tensorOperation(.{}); - const acc = mm.contract(t.lhs, t.rhs, &.{ B, M, N }); - const prod = try mm.commit(acc.mulf(0.5), .{ .dtype = .bf16 }); - const flat = try b.reshape(prod, &.{ B, b.axis("MN", 128 * 256) }, .{}); - b.ret(&.{ prod, flat }); - - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, "tactic = #tcl.tactic<\"EinsumByDpe\">"); - try expectContains(ir, "\"tcl.dpe\""); - try expectContains(ir, "arith.constant 5.000000e-01 : f32"); - try expectContains(ir, "\"tcl.graph.reshape\""); -} - -test "conv1d matches furiosa.kernels.convolution" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "conv"); - defer b.deinit(); - - const N = b.axis("N", 2); - const C = b.axis("C", 64); - const L = b.axis("L", 128); - const K = b.axis("K", 128); - const Lw = b.axis("Lw", 3); - const Lf = b.axis("Lf", 128); - const t = try b.declareArgs(.{ - .inp = .{ .dtype = .bf16, .axes = &.{ N, C, L } }, - .flt = .{ .dtype = .bf16, .axes = &.{ K, C, Lw } }, - .bias = .{ .dtype = .bf16, .axes = &.{K} }, - }); - - const op = b.tensorOperation(.{}); - const windows = op.fetch(t.inp, .{ - .pad = &.{.{ .axis = L, .left = 1, .right = 1 }}, - .slide = &.{.{ .axis = L, .frame_axis = Lf, .window_axis = Lw }}, - }); - try std.testing.expectEqualStrings("Lw", windows.axes[2].name); - const out = try op.commit(op.contract(windows, t.flt, &.{ N, K, Lf }).addf(t.bias), .{ .dtype = .bf16 }); - b.ret(&.{out}); - - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, "slide = {L = {dilation = 1 : i64, frame_axis = #tcl.symbol<\"Lf\">, stride = 1 : i64, undilated_window = 3 : i64, window_axis = #tcl.symbol<\"Lw\">}}"); - try expectContains(ir, "tactic = #tcl.tactic<\"EinsumByDpe\">"); -} - -test "composite axes, DRAM, fixed point, context, config and graph ops" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "features"); - defer b.deinit(); - - const M = b.axis("M", 64); - const N = b.axis("N", 128); - const G = b.axis("G", 8); - const MN = b.axis("MN", b.expr(.mul, M, N)); - try std.testing.expectEqual(64 * 128, MN.size); - b.compilerConfig(.{ .lowering_mode = "Heuristic", .max_num_partitioning_axes = 1 }); - const t = try b.declareArgs(.{ - .x = .{ .dtype = .f32, .axes = &.{ M, N }, .dram = .{} }, - .idx = .{ .dtype = .i32, .axes = &.{G} }, - }); - - const op = b.tensorOperation(.{ .context = .{ - .layout = .{ .chip = .broadcast, .split = &.{M} }, - .heuristic_hint = &.{.{ .name = "max_num_partitioning_axes", .value = .{ .int = 2 } }}, - } }); - const sum = try op.commit(op.fetch(t.x, .{}).reduce(&.{N}, .addf).toFxp(15).toFp(15), .{}); - - const rows = try b.gather(t.x, t.idx, M, &.{ G, N }, .{}); - const back = try b.scatter(rows, t.idx, M, &.{ M, N }, .{ .init = t.x }); - const iota = try b.arange(N.size, .i32, &.{N}, .{}); - const pattern = try b.vector(&.{ 1, 2 }, .f32, &.{N}, .{ .repeat_to = N }); - const bits = try b.transmute(back, .i32, &.{ M, N }, .{}); - const flat = try b.asDram(try b.reshape(bits, &.{MN}, .{}), .{}); - b.ret(&.{ sum, iota, pattern, flat }); - - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, "MN = #tcl.expr<\"mul\", [#tcl.symbol<\"M\">, #tcl.symbol<\"N\">]>"); - try expectContains(ir, "tcl.compiler_config = #tcl.config<{lowering_mode = \"Heuristic\", max_num_partitioning_axes = 1 : i64}>"); - try expectContains(ir, "%arg0: !tcl.mapped], [#tcl.symbol<\"M\">, #tcl.symbol<\"N\">], []>>"); - try expectContains(ir, "\"tcl.graph.as_logical\"(%arg0)"); - try expectContains(ir, "opcode = #tcl.ve_opcode<\"to_i15\">"); - try expectContains(ir, "operator = {Chip = #tcl.expr<\"broadcast\", []>, Split = [#tcl.symbol<\"M\">]}"); - inline for (.{ "gather", "scatter", "arange", "vector", "transmute", "reshape", "as_dram" }) |g| - try expectContains(ir, "\"tcl.graph." ++ g ++ "\""); -} - -test "scatter written back into its DRAM argument in place" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "write_rows"); - defer b.deinit(); - - const M = b.axis("M", 64); - const N = b.axis("N", 128); - const G = b.axis("G", 8); - const t = try b.declareArgs(.{ - .cache = .{ .dtype = .bf16, .axes = &.{ M, N }, .dram = .{} }, - .rows = .{ .dtype = .bf16, .axes = &.{ G, N } }, - .idx = .{ .dtype = .i32, .axes = &.{G} }, - }); - const written = try b.update(0, try b.scatter(t.rows, t.idx, M, &.{ M, N }, .{ .init = t.cache })); - try std.testing.expectEqual(@as(?usize, 0), b.updatedArgument(written)); - b.ret(&.{written}); - - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, "\"tcl.graph.copy\""); - try expectContains(ir, "tcl.updated_inputs = array"); -} - -test "loop over a device-computed trip count with index reads and writes" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "prefix"); - defer b.deinit(); - - const T = b.axis("T", 8); - const D = b.axis("D", 64); - const B = b.axis("B", 4); - const t = try b.declareArgs(.{ - .x = .{ .dtype = .f32, .axes = &.{ T, D } }, - .s = .{ .dtype = .f32, .axes = &.{D} }, - .lens = .{ .dtype = .i32, .axes = &.{B} }, - }); - // Running sums of the rows up to the longest length. - const n = try b.symExpr(.add, try b.reduceMaxI32(t.lens), 0); - var loop = b.openFor(n, .{ t.s, try b.scratchpad(.f32, &.{ T, D }) }); - { - const op = b.tensorOperation(.{}); - const sum = try op.commit(op.fetch(try b.indexRead(t.x, loop.iv), .{}).addf(op.fetch(loop.carried[0], .{})), .{}); - try loop.yield(.{ sum, try b.indexWrite(loop.carried[1], loop.iv, sum) }); - } - b.ret(&loop.results); - - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, "\"tcl.graph.for\""); - try expectContains(ir, "operandSegmentSizes = array"); - try expectContains(ir, "\"tcl.graph.index_write\""); - try expectContains(ir, "#tcl.expr<\"add\", [#tcl.expr<\"arg\", [0]>, 0]>"); -} - -test "multi-chip kernel over chip-split DRAM tensors" { - const ctx = try testContext(); - defer ctx.deinit(); - var b = try Builder.open(std.testing.allocator, ctx, "split_exp"); - defer b.deinit(); - - const C = b.axis("C", 2); - const A = b.axis("A", 1024); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = Dram{ .chip = &.{.{ .axis = C }} } } }); - try std.testing.expectEqual(2, t.x.shards); - - const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); - const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); - const all = try b.allGather(local, C, .{}); - b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); - - const ir = try b.finish(); - defer std.testing.allocator.free(ir); - try expectContains(ir, "%arg0: !tcl.mapped], [#tcl.symbol<\"A\">], []>>"); - try expectContains(ir, "operator = {Chip = #tcl.symbol<\"C\">}"); - try expectContains(ir, "\"tcl.graph.all_gather\""); -} diff --git a/dsl/tcl/dtype.zig b/dsl/tcl/dtype.zig deleted file mode 100644 index 9faf1e18..00000000 --- a/dsl/tcl/dtype.zig +++ /dev/null @@ -1,57 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); - -/// TCL element types, named as in TCL (`[A]/f8_e4`). -pub const DType = enum { - bool, - i4, - i8, - i16, - i32, - i64, - u8, - u16, - u32, - u64, - f16, - bf16, - f32, - f64, - f4_e2, - f8_e4, - f8_e5, - - pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { - return switch (self) { - .bool => .int(ctx, .i1), - .i4 => .int(ctx, .i4), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .u8 => .int(ctx, .u8), - .u16 => .int(ctx, .u16), - .u32 => .int(ctx, .u32), - .u64 => .int(ctx, .u64), - .f16 => .float(ctx, .f16), - .bf16 => .float(ctx, .bf16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .f4_e2 => .float(ctx, .f4e2m1fn), - .f8_e4 => .float(ctx, .f8e4m3fn), - .f8_e5 => .float(ctx, .f8e5m2), - }; - } - - pub fn isFloat(self: DType) bool { - return switch (self) { - .f16, .bf16, .f32, .f64, .f4_e2, .f8_e4, .f8_e5 => true, - else => false, - }; - } -}; - -test { - std.testing.refAllDecls(@This()); -} diff --git a/dsl/triton/BUILD.bazel b/dsl/triton/BUILD.bazel deleted file mode 100644 index 6699694a..00000000 --- a/dsl/triton/BUILD.bazel +++ /dev/null @@ -1,29 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "builder", - srcs = [ - "dtype.zig", - ], - import_name = "dsl/triton/builder", - main = "builder.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//dsl/common:builder", - "//mlir", - "//mlir/dialects/arith", - "//mlir/dialects/cf", - "//mlir/dialects/func", - "//mlir/dialects/llvm", - "//mlir/dialects/math", - "//mlir/dialects/scf", - "//mlir/dialects/ttir", - "//stdx", - ], -) - -zig_test( - name = "test", - deps = [":builder"], -) diff --git a/dsl/triton/PORTING.md b/dsl/triton/PORTING.md deleted file mode 100644 index 0d328710..00000000 --- a/dsl/triton/PORTING.md +++ /dev/null @@ -1,758 +0,0 @@ -# Porting Triton kernels from Python to the Zig DSL - -This guide covers the day-to-day workflow for **adding a new Triton kernel -to ZML** (port from a Python `@triton.jit` source) and **fixing an existing -Zig kernel that diverges from Python** (using the comparison harness in -`examples/triton_emitter/`). - -> **The harness is not in this repository.** `examples/triton_emitter/` and -> its scripts (`run.sh`, `dump_python_ir.py`, `kernels_zig.zig`, -> `dump_via_xla.zig`, `compare_ir.py`) are not part of ZML; the steps that -> use them describe the comparison it runs. Writing the kernel and its -> caller (Steps 1.2 and 1.3) and the porting bugs of Phase 3 don't need it. - -## Goals (in priority order) - -1. **Lowered code matches.** The Zig version produces **the same LLVM IR + - PTX** as the Python version. TTIR/TTGIR may differ cosmetically (op - ordering, naming) — what matters is what actually runs on the GPU. - -2. **Source code looks alike.** A Python kernel and its Zig port should be - *visually transparent* side by side: same overall structure, same - variable names, same comment placement, same line ordering. Someone - reading both should be able to follow them in lockstep. - - Visual fidelity matters because: - - It makes future ports trivial — patterns transfer one-to-one. - - It makes upstream-Triton patches easy to apply (the Python diff - translates almost mechanically to a Zig diff). - - It catches porting bugs before they reach the comparison harness: - when the Zig code reads differently from Python, that's where the - real-vs-cosmetic divergence usually hides. - - Concretely: if a Python source line is - ```python - y_ptr_offset = (row.to(tl.int64) * y_row_stride) + (row_g_id.to(tl.int64) * group_size) - ``` - the Zig port should be one line, named the same way: - ```zig - const y_ptr_offset = row.to(.i64).mul(y_row_stride).add(row_g_id.to(.i64).mul(group_size)); - ``` - Not split across three temporaries with renamed locals — even if the - resulting LLIR is identical. Goal #1 is a hard requirement; goal #2 - is what makes this codebase maintainable. - -For the DSL surface (`zml.kernel.triton.Kernel`, `Builder`, `Value`, etc.), see -[`README.md`](./README.md). This file is about the *process*, not the API. - ---- - -## Prerequisites - -- CUDA or ROCm GPU (XLA's Triton backend is GPU-only). -- The Python venv at `examples/triton_emitter/.venv` activated, or a - system-level Python with `triton`, `torch`, `numpy` installed. -- Bazel + the standard ZML build deps. - -One-time: - -```sh -cd examples/triton_emitter -python -m venv .venv -source .venv/bin/activate -pip install -r requirements.txt -``` - ---- - -## Phase 1 — Initial port - -Start by copying the Python kernel verbatim into the Zig DSL form. The -goal is a kernel that *compiles*, *verifies*, **and reads as similarly -to the Python source as possible**. - -> **Author hint:** open the Python file in a left pane and the Zig file -> in a right pane. Translate **line by line**, top to bottom. Keep -> variable names, comment placement, and intermediate-binding granularity -> identical wherever the DSL allows. Don't refactor, don't rename, don't -> "improve" — that's a separate task you do after the LLIR matches. - -### Step 1.1 — Drop the Python source next to the Zig file - -For comparison purposes, the Python source has to live under -`examples/triton_emitter/kernels_py/`. The dump script auto-discovers -every `@triton.jit` function in that directory. - -If your kernel's already in `vllm/` or another upstream tree, copy the -relevant `@triton.jit` definition (and any `@triton.jit` helpers it -calls) into a single `.py` file there. Strip away launch wrappers, -autotune decorators, and any code that isn't actually inside the kernel -function. - -### Step 1.2 — Sketch the Zig declaration - -Inside `zml/kernels/moe/triton/` or `zml/kernels/paged_attention/triton/` -(or whichever production file fits), add: - -```zig -const tri = zml.kernel.triton; - -pub const MyKernel = struct { - pub const Cfg = struct { - // One field per `tl.constexpr` parameter, named as in Python. - BLOCK_SIZE: usize, - // ... - }; - pub const Kernel = tri.Kernel(Cfg, .{ - .name = "my_kernel", // EXACTLY matches the Python `def my_kernel` - // Call-site names, in `tt.func` argument order: inputs, then outputs. - .inputs = &.{"x_ptr"}, - .outputs = &.{"out_ptr"}, - .run = run, - }); - - fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { - const a = try b.declareArgs(.{ - // One field per *runtime* parameter (pointers, dims that - // arrive as 0-d tensors). Use the same names + order Python - // uses; the README's "Argument specs" table maps the - // tagged-union variants. - .x_ptr = .{ .ptr = .f32 }, - // ... - }); - - // Body — translate Python line by line. Cheat-sheet in the - // README's "Python Triton ↔ Zig DSL cheat-sheet" section is - // your best friend here. - } -}; -``` - -### Step 1.3 — Wire the production caller - -Inside the same `.zig` directory, find the function that launches your -kernel (e.g. `pagedAttention2d`, `fusedExpertsImpl`). Replace the -TTIR-emit + `ops.triton(...)` two-step with a single -`MyKernel.Kernel.call(inputs, outputs, opts)` — see the -[README](./README.md#calling-kernels-from-a-zml-model--kkernelcall) for the -exact shape. - -If this is a brand-new kernel with no existing caller, write the caller -fresh. - -### Step 1.4 — Wire the comparison harness - -Add the kernel to `examples/triton_emitter/kernels_zig.zig`: - -```zig -withConfig(moe.MyKernel, .{ .BLOCK_SIZE = 64, /* ... */ }), -``` - -If the kernel takes runtime tensor inputs, also add a `forward` struct + -`KERNELS` entry to `examples/triton_emitter/dump_via_xla.zig` so XLA's -codegen pipeline can lower it. The tensor sizes are dummies — XLA -doesn't launch the kernel, it just runs the lowering passes. - -For the Python side, the auto-discovery in `dump_python_ir.py` works for -simple kernels (only `*_ptr` runtime args + `tl.constexpr` params). -**If the signature mixes runtime args and constexpr in the middle of the -parameter list**, write an override function and add it to `_OVERRIDES` -(follow `_per_token_group_quant_fp8` / `_kernel_unified_attention_2d_ptr` -as templates). Use the same constexpr values + tensor shapes you used in -the Zig `withConfig(...)` call. - -### Step 1.5 — Build it - -```sh -bazel build //examples/triton_emitter:dump_zig_ir # Zig side compiles + verifies -bazel build //zml:test # production side compiles -``` - -If the Zig DSL `b.finish()` rejects your IR, it logs the module. Mixed -integer widths and scalar/tensor operands are not the cause: the fluent ops -widen the narrower integer side (`v_i32.mul(v_i64)` is an i64 multiply), and -`addPtr` splats a scalar pointer or offset against a tensor partner. A common -cause is an **`scf.yield` type mismatch** — the values you pass to -`loop.yield(...)` / `i.yieldThen(...)` must have the types of the loop's -`inits` / the if's `result_types` (their arity is checked at compile time). - -### Step 1.6 — First dump - -```sh -bash examples/triton_emitter/run.sh -``` - -This runs the full pipeline (Python TTIR → XLA → per-stage IR; same on -the Zig side; diff each stage). At the end you get a per-kernel -verdict: - -``` -my_kernel - ttir +30/-30 - ttgir +30/-30 - llir +50/-50 - ptx +40/-40 -``` - -Don't worry yet about cosmetic diffs — go through Phase 2 to find -*real* divergences first. - ---- - -## Phase 2 — The comparison pipeline - -The harness has six steps (see `run.sh`), output appears under -`examples/triton_emitter/`: - -| Step | Output | What it is | -|------|------------------------------|-----------------------------------------------| -| 1 | `py_ir/*.ttir`/`.ttgir`/... | Python's per-stage IR (Triton's own pipeline) | -| 2 | `zig_ir/*.ttir` | Just the TTIR your DSL emits. | -| 3 | `xla_dump_py/` | XLA's per-pass dumps lowering Python's TTIR. | -| 4 | `xla_dump_zig/` | XLA's per-pass dumps lowering the Zig TTIR. | -| 5 | `xla_py/*.{ttir,ttgir,llir,ptx}` + `xla_zig/*.{...}` | Per-stage IR extracted by `extract_xla_dump.py`. Both sides go through XLA, not Triton-the-Python-frontend, so the comparison is apples-to-apples. | -| 6 | stdout | `compare_ir.py` diff summary per kernel. | - -### Useful flags - -```sh -# Just one kernel (faster iteration): -# (use ZIG_IR/PY_IR filter — but note dump_zig_ir's CLI expects --kernel=NAME). -bash run.sh # full run -python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel -python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel --stage ttir --show-diffs - -# Skip the slow XLA-lowering steps if only the TTIR changed: -python dump_python_ir.py --out-dir py_ir -bazel run //examples/triton_emitter:dump_zig_ir -- --out-dir=$PWD/zig_ir -diff py_ir/my_kernel.ttir zig_ir/my_kernel.ttir -``` - -### Reading the verdict - -``` -my_kernel - ttir +30/-30 <- TTIR diverges 30 lines - ttgir +30/-30 <- so does TTGIR (same shape) - llir ✓ <- but LLIR converges! cosmetic-only TTIR diff - ptx ✓ <- and PTX matches -``` - -This kernel is **good** — final code is identical, the TTIR diff is just -op-naming/ordering cosmetics that XLA's lowering canonicalizes away. - -``` -my_kernel - ttir +5/-5 - ttgir +5/-5 - llir +60/-58 <- LLIR diverges → real semantic difference - ptx +50/-48 -``` - -This kernel needs work: a real divergence appeared somewhere between -TTIR and LLIR. Drill into the LLIR diff to find which op got lowered -differently. - ---- - -## Phase 3 — Fixing divergences - -Workflow: pick the *smallest* diverging stage's diff and locate the -op-by-op mismatch. Most fixes fall into one of the categories below. - -### Step 3.1 — `--show-diffs` to locate the divergence - -```sh -python compare_ir.py --left xla_py --right xla_zig --kernel my_kernel --stage ttir --show-diffs -``` - -Read the diff carefully. The first diverging op tells you what to look -at; subsequent diffs are usually downstream consequences. - -### Step 3.2 — Common porting bugs - -These are the gotchas we've hit while porting kernels. If your diff -shows any of these patterns, fix the corresponding Zig DSL call. - -#### 3.2.1 — `arith.maxnumf` vs `arith.maximumf` (and `min*`) - -**Symptom:** -``` -- %v43 = arith.maxnumf %v37, %v14 : f32 (Python) -+ %v43 = arith.maximumf %v37, %v13 : f32 (Zig) -``` - -**Fix:** **Use `b.maximum` / `Value.maximum` / `b.minimum` / -`Value.minimum`** — they default to `maxnumf`/`minnumf` and match -Python's `tl.max`/`tl.min`/`tl.maximum`/`tl.minimum`. **Don't reach -for `b.maximumf` / `b.minimumf` directly** — those are the IEEE-754 -NaN-propagating escape hatches (`arith.maximumf`/`minimumf`), exposed -for the rare case where you genuinely want NaN propagation. The naming -is unfortunate (the `f` suffix reads like "the float version") — it's -actually "the NaN-propagating version." See README's "Reductions and -scans" gotcha table for the full mapping. - -#### 3.2.2 — 2-operand vs 3-operand `tt.load` - -**Symptom:** -``` -- %v33 = tt.load %v32, %v30, %v9 : tensor<128x!tt.ptr> (Python — has `other`) -+ %v33 = tt.load %v32, %v30 : tensor<128x!tt.ptr> (Zig — no `other`) -``` - -**Fix:** Match Python's exact form. The DSL has only one masked-load -API (`b.loadOpts`); whether it emits 2-operand or 3-operand `tt.load` -depends entirely on whether you pass `.other`: - -```zig -// Python: tl.load(ptr, mask=mask, other=0.0) → 3-operand -const zero = b.zeros(&.{BLOCK}, dtype); -const x = b.loadOpts(ptr, .{ .mask = mask, .other = zero }); - -// Python: tl.load(ptr, mask=mask) → 2-operand -const x = b.loadOpts(ptr, .{ .mask = mask }); -``` - -The two forms are *semantically* close but not identical (out-of-bounds -lanes are undefined for 2-operand, zero for 3-operand). Always copy -Python's form, even if you suspect the difference is harmless — XLA's -canonicalization may or may not erase it. - -#### 3.2.3 — `i32` vs `i64` width in pointer offsets - -**Symptom:** -``` -- %v26 = tt.addptr %v6, %v16 : !tt.ptr, i32 (Python — uses i32) -+ %v25 = tt.addptr %v6, %v16 : !tt.ptr, i64 (Zig — uses i64) -``` - -**Fix:** Python's pointer arithmetic preserves the original integer -width. `tl.program_id(0)` is i32; if Python writes -`y_s_ptr += tl.program_id(0)` (no cast), the offset stays i32. In Zig, -if you converted `pid` to i64 early to use it elsewhere, pass the raw -`b.programId(.x)` (i32) to that one `addPtr` site: - -```zig -// raw i32 program_id for one site, i64 elsewhere -const pid_i32 = b.programId(.x); -const g_id = pid_i32.to(.i64); -// ... -const y_s_ptr_shifted = a.y_s_ptr.addPtr(pid_i32); // i32, not g_id -``` - -#### 3.2.4 — Mask comparison width - -**Symptom:** -``` -- %v30 = arith.cmpi slt, %v28, %v29 : tensor<128xi64> (Python — i64 vs i64) -+ %v28 = arith.trunci %v10 : i64 to i32 -+ %v29 = tt.splat %v28 : i32 -> tensor<128xi32> -+ %v30 = arith.cmpi slt, %v26, %v29 : tensor<128xi32> (Zig — trunc + splat + i32 cmp) -``` - -**Fix:** Match Python's promotion direction. Python implicitly promotes -the narrower operand to the wider one. If `cols` is i32 and -`group_size` is i64, Python emits `extsi cols i32→i64` then compares -at i64. The fluent ops promote the same way, so compare the two as they -are; a `trunci` like the one above comes from narrowing one side by hand: - -```zig -const cols = b.arange(0, BLOCK, .i32); -const mask = cols.lt(group_size); // extsi cols to i64, no trunc -``` - -#### 3.2.5 — Op emit order - -**Symptom:** TTIR diff shows the same ops in a different order: -``` -- %v18 = arith.divsi %v17, %v15 : i64 -- %v19 = arith.remsi %v17, %v15 : i64 -- %v20 = arith.muli %v18, %v13 : i64 -- %v21 = arith.muli %v19, %v11 : i64 -- %v22 = arith.addi %v20, %v21 : i64 -+ %v17 = arith.divsi %v16, %v14 : i64 -+ %v18 = arith.muli %v17, %v12 : i64 -+ %v19 = arith.remsi %v16, %v14 : i64 -+ %v20 = arith.muli %v19, %v10 : i64 -+ %v21 = arith.addi %v18, %v20 : i64 -``` - -This usually doesn't affect LLIR/PTX (compiler reorders), but if it -does (or if you want a clean TTIR diff), match Python's source order -op-by-op. Bind every intermediate to a named local in the same order -Python evaluates: - -```zig -// Python: y_ptr += (g_id // groups_per_row) * y_row_stride + (g_id % groups_per_row) * group_size -// eval order: divsi, remsi, muli, muli, addi -const row = g_id.div(groups_per_row); // divsi -const row_g_id = g_id.rem(groups_per_row); // remsi -const row_off = row.mul(y_row_stride); // muli -const grp_off = row_g_id.mul(group_size); // muli -const y_ptr_shifted = a.y_ptr.addPtr(row_off.add(grp_off)); // addi + addptr -``` - -#### 3.2.6 — `arith.maxnumf` (and `splat`) vs `tt.splat` of a runtime value - -**Symptom:** -``` -- %splat = tt.splat %scalar : i32 -> tensor<128xi32> (Python — runtime splat) -+ %const = arith.constant dense<...> : tensor<128xi32> (Zig — constant pool) -``` - -**Fix:** `b.splat(comptime_literal, shape)` materializes a constant -dense; `b.splat(runtime_value, shape)` emits `tt.splat`. If Python -uses a runtime value (loaded scalar, function arg) and you accidentally -hardcoded the constant, switch to the runtime path. - -#### 3.2.7 — `tl.constexpr` divisibility hints - -**Symptom:** TTIR header differs in `tt.divisibility = N : i32` attrs. - -**Fix:** Default `.{ .ptr = .dtype }` carries `tt.divisibility = 16`. -For scalars, switch to `.scalar_opts`: - -```zig -.n_elements = .{ .scalar_opts = .{ .dtype = .i32, .divisibility = 16 } }, -``` - -This matches what Python's JIT runtime auto-attaches when the runtime -value is a multiple of 16. - -#### 3.2.8 — Reductions over rank-1 tensors - -**Symptom:** Python emits a `tt.reshape allow_reorder` before the -`tt.reduce`; Zig doesn't. - -**Fix:** Use the `axis = null` (default) form: `b.max(x)`, not -`b.maxOpts(x, .{ .axis = 0 })`. The `axis = null` path inserts the -reshape; explicit-axis paths skip it. - -#### 3.2.9 — `tl.dot` precision and accumulator - -**Symptom:** TTIR shows different `tt.dot` `inputPrecision` attribute or -different `acc` type. - -**Fix:** Match Python's `tl.dot(a, b, acc, input_precision="...")` -explicitly: - -```zig -const new_acc = b.dotOpts(a_val, b_val, acc, .{ - .input_precision = .tf32, // matches `input_precision="tf32"` - .max_num_imprecise_acc = 0, -}); -``` - -Default `input_precision` differs across Triton versions — always be -explicit. - -#### 3.2.10 — `iter_args` order in `scf.for` - -**Symptom:** TTIR diff shows the loop signature reordered: -``` -- %v152:3 = scf.for ... iter_args(%v149 = %v131, %v150 = %v143, %v151 = %v37) -- -> (tensor<...!tt.ptr>, tensor<...!tt.ptr>, tensor<...xf32>) -+ %v152:3 = scf.for ... iter_args(%v149 = %v37, %v150 = %v131, %v151 = %v143) -+ -> (tensor<...xf32>, tensor<...!tt.ptr>, tensor<...!tt.ptr>) -``` - -**Fix:** Python's `scf.for` lowering captures `iter_args` in the order -they're **declared** above the loop, not the order they're mutated inside. -For `fused_moe_kernel`: - -```python -a_ptrs = ... # declared 1st -b_ptrs = ... # declared 2nd -accumulator = tl.zeros(...) # declared 3rd -for k in range(...): - accumulator += tl.dot(a, b) # mutated 1st - a_ptrs += BLOCK_SIZE_K # mutated 2nd - b_ptrs += BLOCK_SIZE_K # mutated 3rd -``` - -Python's `iter_args` order = declaration order = `(a_ptrs, b_ptrs, acc)`, -**not** mutation order. The DSL `openFor` follows the order you pass — so -match Python's declaration order: - -```zig -var loop = b.openFor(0, num_k_iters, 1, .{ a_ptrs_init, b_ptrs_init, acc_init }); -{ - const a_ptrs = loop.carried[0]; - const b_ptrs = loop.carried[1]; - const acc = loop.carried[2]; - // ... - loop.yield(.{ new_a_ptrs, new_b_ptrs, new_acc }); -} -const accumulator = loop.results[2]; -``` - -The wrong order survives all the way to PTX as different register-block -layouts. - -#### 3.2.11 — Operand order in commutative `arith.muli` - -**Symptom:** TTIR diff shows `arith.muli` operands swapped: -``` -- %v137 = arith.muli %v136, %v135 : tensor<1x64xi64> (Python: stride on LHS) -+ %v137 = arith.muli %v135, %v136 : tensor<1x64xi64> (Zig: offs on LHS) -``` - -**Fix:** `arith.muli` is commutative semantically, but the operand order -affects downstream layout decisions in TTGIR. Match Python's source order -exactly. For `stride * offs[None, :]`, Python emits `arith.muli %stride_splat, -%offs`: - -```zig -// Python: stride_cm * offs_token[:, None] — stride on LHS -const cm_col = stride_cm.mul(offs_token_col); -// NOT: offs_token_col.mul(stride_cm) — offs on LHS, wrong order -``` - -Same applies to `arith.addi`, `arith.subi`, `arith.andi`, etc. - -#### 3.2.12 — Two-step `addPtr` for `c_ptr + cm[:, None] + cn[None, :]` - -**Symptom:** Python emits two `tt.addptr` ops; the Zig port collapses -them into one big addptr with a pre-merged 2D offset. -``` -Python: - %ptr_col = tt.addptr %c_ptr_splat_Mx1, %cm_col_Mx1 : Mx1x!tt.ptr, Mx1xi64 - %ptr_2d = tt.broadcast %ptr_col → MxNx!tt.ptr - %final_ptr = tt.addptr %ptr_2d, %cn_off_2d : MxNx!tt.ptr, MxNxi64 -Zig: - %offset = arith.addi %cm_term, %cn_term : MxNxi64 - %final_ptr = tt.addptr %c_ptr_splat, %offset : MxNx!tt.ptr, MxNxi64 -``` - -**Fix:** Python evaluates `c_ptr + cm[:, None] + cn[None, :]` left-to-right, -so it emits two `tt.addptr` ops. Match by splitting: - -```zig -// First addptr: scalar c_ptr + (Mx1) cm offset → Mx1 ptr tensor -const cm_col = stride_cm.mul(b.expandDims(offs_token, 1)); -const c_ptrs_col = b.splat(c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); - -// Second addptr: broadcast to MxN, add cn offset -const cn_row = b.expandDims(offs_cn, 0); -const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); -const c_ptrs = c_ptrs_2d.addPtr(b.broadcastTo(cn_row, &.{ block_size_m, block_size_n })); -``` - -This pattern shows up wherever Python writes `ptr + a[:, None] + b[None, :]` -(both A and B ptrs in fused_moe, the C-output store, etc.). - -#### 3.2.13 — `extsi` placement relative to sibling `expand_dims`/`mul` - -**Symptom:** Same set of ops, different positions: -``` -- %v134 = expand_dims %offs_k_i32 (axis=1) (Python: offs_k expand) -- %v135 = expand_dims %offs_bn_i64 (axis=0) (Python: offs_bn expand) -- %v136 = splat %stride_bn → 1x64xi64 (Python: bn stride) -- %v137 = muli %v135, %v136 (Python: bn_row mul) -- %v138 = extsi %v134 → 32x1xi64 (Python: extsi DEFERRED) -+ %v135 = extsi %v134 → 32x1xi64 (Zig: extsi EAGER) -+ ... (rest in different order) -``` - -**Fix:** When Python writes `offs_k[:, None]` and uses it later in an -`addi` against an `i64`-typed tensor, the implicit `extsi` happens at the -**use site** (where the auto-promotion fires), not at the `expand_dims`. -Mirror this — keep the chain at i32 until just before it joins the i64 -expression: - -```zig -// Wrong: extsi happens immediately after expand_dims -const offs_k_col = b.expandDims(offs_k_i32, 1).to(.i64); // extsi NOW -const offs_bn_row = b.expandDims(offs_bn, 0); -const bn_row = offs_bn_row.mul(stride_bn); - -// Right: extsi deferred to after offs_bn's mul -const offs_k_col_i32 = b.expandDims(offs_k_i32, 1); // 32x1xi32 -const offs_bn_row = b.expandDims(offs_bn, 0); -const bn_row = offs_bn_row.mul(stride_bn); // 1x64xi64 -const offs_k_col = offs_k_col_i32.to(.i64); // extsi HERE -``` - -Same idea for placing a `muli` before an `extsi` on a sibling load: - -```zig -// Python emits muli before the extsi at the cmpi: -const num_tokens_post_padded_i32 = b.load(...); // load -const pid_m_block = pid_m.mul(block_size_m); // muli FIRST -const num_tokens_post_padded = num_tokens_post_padded_i32.to(.i64); // extsi -const out_of_range = pid_m_block.ge(num_tokens_post_padded); -``` - -### Step 3.3 — Iterate - -After each fix: - -```sh -bazel build //examples/triton_emitter:dump_zig_ir -bash examples/triton_emitter/run.sh -``` - -Re-read the diff. The line counts should drop, and ideally the LLIR/PTX -columns flip to `✓`. If not, drill again — there's another mismatch -hiding behind the first. - -### Step 3.4 — Don't be a perfectionist about TTIR (but do read every line) - -You only need **LLIR + PTX** to match. If TTIR/TTGIR have +5 cosmetic -lines but LLIR is byte-identical, ship it. The XLA canonicalization + -CSE passes erase a lot of low-level differences before LLIR. - -A kernel like `count_and_sort_expert_tokens_kernel` (TTIR/TTGIR +8/-8, -LLIR/PTX ✓) is a clean port. Don't keep grinding it. - -**But** — when LLIR is **not** ✓ and the diff is small (tens of lines), -do **not** label the rest "scheduling noise" without a name-normalized -diff. Most lines in a small LLIR diff are downstream SSA renumbering -caused by 1–4 high-leverage TTIR-level fixes (typically the patterns in -3.2.10–3.2.13). Run: - -```sh -diff \ - <(sed -E 's/%[a-zA-Z_.][a-zA-Z0-9_.]*[0-9]+/%V/g; s/%[0-9]+/%V/g; s/^[0-9]+:/V:/g' xla_py/foo.llir) \ - <(sed -E 's/%[a-zA-Z_.][a-zA-Z0-9_.]*[0-9]+/%V/g; s/%[0-9]+/%V/g; s/^[0-9]+:/V:/g' xla_zig/foo.llir) -``` - -If this normalized diff is small (~tens of lines), every remaining line -is pointing at a structural fix. The "+100 LLIR is just XLA scheduling" -intuition has been wrong every time so far — `fused_moe_kernel` went from -LLIR +100 → ✓ via four TTIR-level fixes (iter_args order, muli operand -order, two-step addPtr, extsi placement) that were all visible in the -+100-line diff once read carefully. - -### Step 3.5 — Visual-transparency pass - -Once the LLIR matches, do one pass over the Zig source with the Python -source open beside it. Check (in order): - -- **Variable names.** Python `groups_per_row` → Zig `groups_per_row`, - not `gp_row` or `gpr`. Same for arg names — `y_ptr` not `output_ptr`. -- **Line order.** Each Python line should correspond to roughly one - Zig line, in the same order. If a single Python expression became - three Zig temporaries, ask: does the DSL force the split? If not, - collapse them onto one chained `.add(...).mul(...)` line. -- **Granularity of locals.** If Python writes - `_absmax = tl.maximum(tl.max(tl.abs(y)), eps)` as a one-liner, the - Zig port should also be one line: `const absmax = b.max(b.absf(y)).maximum(eps);`. -- **Comments.** Carry over Python's comments verbatim where they apply. - If Python writes `# Map the program id to the row of X and Y it - should compute.` above its block, paste the same comment above the - Zig block. They serve as anchors for future side-by-side reads. -- **Helper functions.** If Python factors something into a - `@triton.jit` helper (e.g. `find_seq_idx`), the Zig port should have - a matching `fn findSeqIdx(b: *Builder, ...) Value` helper, not inline - the body. Same arguments, same name (snake_case → camelCase). -- **Control flow shape.** Python `for k in range(...)` → `b.openFor`, - Python `if ... return` → `b.returnIf` / `b.openReturnIf`. Don't - flatten an `if` ladder into a `select` chain (or vice versa) just - because the DSL allows it. - -If a transparency fix changes the LLIR diff, **the LLIR-match goal -wins** — back out the visual change and leave a comment explaining the -divergence. Goal #1 always beats goal #2. - -A useful sanity check: open both files side by side in your editor and -scroll both at the same rate. If the cursors stay roughly aligned, the -port reads transparently. If you have to scroll one twice as fast as -the other, there's structural drift to clean up. - ---- - -## Phase 4 — Production wiring (post-port) - -Once the LLIR matches *and* the visual-transparency pass is done: - -1. **Side-by-side review.** Pull up Python and Zig in adjacent panes - one more time. Anything that still reads differently between them - either has a justification (DSL constraint, language constraint — - leave a brief comment explaining why) or needs another pass. -2. **Remove any `// TODO match python` comments** from the kernel body. -3. **Verify the production caller passes the right cfg** — the dump - tool's config and the production caller's config can differ (the - dump tool uses small dimensions; production uses real ones). - Production behavior depends on the runtime cfg, so make sure - `makeFooConfig(...)` in the caller produces sensible values. -4. **Run the model end-to-end** if possible. The comparison harness - only validates IR equivalence — runtime correctness still needs a - real test (mismatched grid, missing strides, etc. won't show up in - the diff but will crash at execution time). - ---- - -## Cheat-sheet: when to look at which file - -| Problem | File | -|---------------------------------------------------|----------------------------------------------| -| "How do I express `tl.X(...)` in Zig?" | `dsl/triton/README.md` (Python ↔ Zig table) | -| "What does `b.declareArgs(...)` accept?" | `dsl/triton/README.md` (Argument specs) | -| "How do I call this kernel from `forward`?" | `dsl/triton/README.md` (`K.call(...)`) | -| "My kernel diverges from Python at LLIR" | This file (Phase 3) | -| "How do I add a kernel to the comparison harness?"| Phase 1 above + `examples/triton_emitter/` | -| "I want to inspect raw TTIR ops" | `mlir/dialects/ttir/ttir.zig` (escape hatch) | - ---- - -## When you're stuck - -If a divergence resists all the standard fixes: - -1. **Print both TTIRs side by side.** The visual structure usually - reveals which control-flow region or loop body is misaligned. -2. **Check if the Python kernel uses any non-`@triton.jit` helpers** — - if it imports something from `triton.language.extra` or has a custom - `@triton.jit` helper, that helper might also need a port (or a - `_SKIP` entry if it's pure-comptime). -3. **Verify the Python override args match the Zig config.** Mismatched - constexpr values (e.g. `BLOCK_SIZE=64` in Zig vs `1024` auto-default - in Python) generate completely different IR. -4. **Compare against the `zml/kernels/moe/triton/` MoE kernels** — they - went through this loop and are reasonable references for the patterns - above. - ---- - -## Before "fixing" the DSL - -Every time a divergence makes you suspect the DSL itself is buggy and you -reach for `dsl/triton/builder.zig` — **verify against the Triton sources -first** in a Triton checkout. The DSL exists to mirror what Python's -frontend emits; the source of truth is the Python trace, not what the rule -"looks like" or what feels symmetric. - -Concretely: - -- If you think `Value.cdiv` is wrong, read - `triton/python/triton/language/standard.py` for the definition of - `tl.cdiv` and compare. Don't assume the desugaring — check. -- If you think a binop dispatches to the wrong arith op - (`maxnumf` vs `maximumf`, `divsi` vs `divui`, …), read - `triton/python/triton/language/semantic.py` and find the cast / - promotion rule that actually fires. -- If you think the type-promotion is wrong (i32 stays i32, doesn't - auto-extend to i64, …), read `semantic.py:cast()` and the binop - promotion helpers. - -A second pitfall: **the local kernel may define its own helper that -shadows the stdlib name.** `unified_attention.py` defines a private -`cdiv_fn(x, y) = (x + y - 1) // y` — left-to-right Python parses this -as `((x + y) - 1)`, so it emits `addi` then `subi`. Triton's *stdlib* -`tl.cdiv` is `(x + (div - 1)) // div` and emits `subi` then `addi`. -These desugar identically when the divisor is comptime (Triton folds -`div - 1` at trace time), but differ when the divisor is a runtime -tensor. If your divergence is just an `add↔sub` reorder, check whether -the Python kernel actually uses `tl.cdiv` or a private `cdiv`-shaped -helper, and either keep the DSL aligned with `tl.cdiv` and write a -local Zig helper for the kernel-private form, or the other way around. - -In short: when a "bug" in the DSL would also affect kernels that -already match Python, the bug is almost certainly elsewhere. - -**And if you do change the DSL**, run `examples/triton_emitter/run.sh` -afterwards and confirm no kernel that previously matched (or was -codegen-equivalent) regressed at LLIR/PTX. A DSL edit that fixes one -kernel by introducing a divergence in another is not a fix — it's a -trade. Always check the full table before committing the change. diff --git a/dsl/triton/README.md b/dsl/triton/README.md deleted file mode 100644 index 939a1b07..00000000 --- a/dsl/triton/README.md +++ /dev/null @@ -1,1116 +0,0 @@ -# ZML Triton DSL — Author's Guide - -A Zig builder for Triton IR (TTIR) that tries to read like `triton.language` -Python while staying in pure Zig. Used by `zml/kernels/moe/triton/`, -`zml/kernels/paged_attention/triton/`, and by anything else that wants to call -Triton kernels from a ZML model. - -Two layers define the surface: - -- **Low layer** — `dsl/triton/builder.zig`: the IR builder primitives. - `Builder`, `Value`, `DType`, `ArgSpec`. Use these directly when you need - full control (custom op insertion, dynamic-arity arg lists, escape hatches). - `Builder.open` / `Builder.init` take an `*mlir.Context` — pass them a - `zml.kernel.triton.newContext()` (a throwaway context with the dialects the - DSL emits), `defer ctx.deinit()` once you have the IR string. Not a - `zml.Compiler`'s `mlir_ctx`: that one carries only `func`/`stablehlo`/`sdy` - (a registered `llvm` dialect makes XLA:TPU choke). -- **High layer** — `zml.kernel.triton.Kernel(Config, spec)`: the declarative - kernel form. Bundles a config type, a typed spec literal (name + named - `inputs` / `outputs` + a typed `run` function pointer), and produces a - kernel type. Exposes `.emit(allocator, cfg)` (TTIR string; manages its own - throwaway context) and `.call(inputs, outputs, opts)` (TTIR string + - `stablehlo.custom_call` in one shot — the preferred form for production - model code). Inputs and outputs are by-name structs generated from the - spec, so missing / typo'd / extra arguments are compile errors at the callsite. - -## Table of contents - -1. [Skeleton of a kernel](#skeleton-of-a-kernel) -2. [`zml.kernel.triton.Kernel(Config, spec)` — the declarative form](#zmlkerneltritonkernelconfig-spec--the-declarative-form) -3. [Calling kernels from a ZML model — `K.Kernel.call(...)`](#calling-kernels-from-a-zml-model--kkernelcall) -4. [Argument specs](#argument-specs) -5. [The `Value` type](#the-value-type) -6. [Fluent methods on `Value`](#fluent-methods-on-value) -7. [Polymorphic scalars](#polymorphic-scalars) -8. [Comptime vs runtime widths](#comptime-vs-runtime-widths) -9. [Shape helpers](#shape-helpers) -10. [Loads and stores](#loads-and-stores) -11. [Reductions and scans](#reductions-and-scans) -12. [Control flow](#control-flow) -13. [Escape hatches](#escape-hatches) -14. [Lower layer — `Builder.open`](#lower-layer--builderopen) -15. [Python Triton ↔ Zig DSL cheat-sheet](#python-triton--zig-dsl-cheat-sheet) -16. [Pitfalls](#pitfalls) - ---- - -## Skeleton of a kernel - -```zig -const tri = @import("zml").kernel.triton; -const zml = @import("zml"); - -pub const MyKernel = struct { - pub const Cfg = struct { BLOCK: i32 = 64 }; - pub const Kernel = tri.Kernel(Cfg, .{ - .name = "my_kernel", - .inputs = &.{ "x_ptr", "y_ptr", "n" }, - .outputs = &.{"out"}, - .run = run, - }); - fn run(b: *tri.Builder, cfg: Cfg) tri.FinishError!void { - const a = try b.declareArgs(.{ - .x_ptr = .{ .ptr = .f32 }, - .y_ptr = .{ .ptr = .f32 }, - .n = .{ .scalar = .i32 }, - .out_ptr = .{ .ptr = .f32 }, - }); - - const pid = b.programId(.x); - const offs = b.arange(0, cfg.BLOCK, .i32).add(pid.mul(cfg.BLOCK)); - const mask = offs.lt(a.n); - - const x = b.loadOpts(a.x_ptr.addPtr(offs), .{ .mask = mask }); - const y = x.mul(2.0); - b.storeOpts(a.out_ptr.addPtr(offs), y, .{ .mask = mask }); - } -}; -``` - -A kernel is a `struct {…}` wrapper bundling three decls: - -1. **`pub const Cfg`** — config type, an inline `struct { ... }` of - comptime/runtime parameters. Use field defaults for sensible defaults. -2. **`pub const Kernel = tri.Kernel(Cfg, .{ .name, .inputs, .outputs, .run })`** — - the spec literal that turns into a kernel type. `inputs` / `outputs` are - slices of `[:0]const u8` field names; the factory generates `Inputs` - (struct of `Tensor`), `Outputs` (struct of `Shape`), and `Results` - (struct of `Tensor`) types from those names. The `run` field is a typed - function pointer — wrong signature is a compile error here. -3. **`fn run(b, cfg)`** — the kernel body. First call `try b.declareArgs(.{…})` - to create the `tt.func` op and bind a struct of `Value`s named after the - spec's fields. The rest of `run` builds the body via `b.*` helpers. No - string return, no allocator, no manual `tt.return` — the factory - terminates, verifies, and serializes for you. - -The names in `spec.inputs` / `spec.outputs` are the **call-site** names -(used to build the `Inputs` / `Outputs` structs) and don't have to match -the names in `b.declareArgs(...)` (which are the **TTIR-level** arg names). -By convention they often do — pick whatever reads best at the boundary. - -The arg spec inside `b.declareArgs(...)` is a struct literal — its variant -*tags* (`.ptr`, `.scalar`, etc.) are comptime-known per call site, but the -*payload values* (`cfg.input_dtype`, `cfg.output_dtype`, …) can be runtime -values. This is what makes the same form work for both comptime-pinned -configs (offline tools, simple kernels) and runtime configs (production -models where dtypes/sizes come from `Tensor.dim()` / `Options`). - -> **Why the wrapper struct.** `Cfg` has to be a *named* type so the typed -> `run: *const fn (*Builder, Cfg) FinishError!void` field in the spec can -> reference it. Wrapping the kernel in `pub const MyKernel = struct {...}` -> keeps `Cfg`, `Kernel`, and `run` scoped together and exposes them all to -> callers (`MyKernel.Cfg`, `MyKernel.Kernel`, plus everything the inner -> Kernel re-exports — `MyKernel.Kernel.Config`, `.Inputs`, `.Outputs`, -> `.Results`, `.CallOpts`). - -### Two ways to use a kernel - -**Offline / tooling** — get the TTIR string (no MLIR context needed; `emit` -spins up and tears down its own): - -```zig -const ttir = try MyKernel.Kernel.emit(allocator, .{ .BLOCK = 1024 }); -defer allocator.free(ttir); -``` - -**Inside a ZML `forward(...)`** — emit TTIR and drop a `stablehlo.custom_call` -into the model in one shot: - -```zig -const out = MyKernel.Kernel.call( - .{ .x_ptr = x_tensor, .y_ptr = y_tensor, .n = n_tensor }, // Inputs - .{ .out = x_tensor.shape() }, // Outputs (Shapes) - .{ - .cfg = .{ .BLOCK = 1024 }, - .grid = .{ ceilDiv(n, 1024), 1, 1 }, - .num_warps = 4, - .num_stages = 2, - }, -).out; -``` - -`Kernel.call` returns a `Results` struct (one `Tensor` per output name); -read fields by name (`out.out`, `align_outs.sorted_token_ids`, …). - ---- - -## `zml.kernel.triton.Kernel(Config, spec)` — the declarative form - -`zml.kernel.triton.Kernel(Config, spec)` is a comptime function returning a -kernel type. `Config` is the per-call config type (used both for the -generated `Inputs`/`Outputs`/`CallOpts` and for the typed `run` signature); -`spec` is a `Spec(Config)` literal: - -| Field | Type | Meaning | -|-------------|---------------------------------------------------|---------------------------------------------------------------------| -| `.name` | `[:0]const u8` | Kernel name (used as the `tt.func` symbol). | -| `.inputs` | `[]const [:0]const u8` | Field names for the call-site `Inputs` struct (fields are `Tensor`). Order must match the `tt.func` arg order for input pointers. | -| `.outputs` | `[]const [:0]const u8` | Field names for the `Outputs` (struct of `Shape`) and `Results` (struct of `Tensor`). Order must match the `tt.func` arg order for output pointers — XLA's lowering passes inputs first, then outputs. | -| `.run` | `*const fn (*Builder, Config) FinishError!void` | Typed function pointer to the kernel body. Wrong signature → compile error here, not at first call. | - -Wrong arity, missing field, typo'd name on the call site → compile error -at the call site (because `Inputs` / `Outputs` are real struct types, not -tuples). - -`run` is responsible for declaring the kernel's args (via -`b.declareArgs(...)`) and emitting the body. It emits ops via `b.*` helpers -and never returns a string; `Kernel.emit(...)` does the -terminate-verify-serialize dance after `run` returns. - ---- - -## Calling kernels from a ZML model — `K.Kernel.call(...)` - -Inside a model's `forward(...)`, drop in a Triton kernel alongside regular -ZML ops with `K.Kernel.call(...)`: - -```zig -pub fn forward(self: Model, x: zml.Tensor, y: zml.Tensor) zml.Tensor { - const x_pre = x.mul(2.0); // stablehlo - - const out = MyKernel.Kernel.call( - .{ .x_ptr = x_pre, .y_ptr = y, .n = n_tensor }, // Inputs (by name) - .{ .out = x_pre.shape() }, // Outputs (Shapes) - .{ - .cfg = .{ .BLOCK = 1024 }, // runtime config - .grid = .{ ceilDiv(x.dim(0), 1024), 1, 1 }, - .num_warps = 4, - .num_stages = 2, - }, - ).out; // .out: Tensor - - return out.relu(); // stablehlo -} -``` - -`call` is a thin wrapper over `Kernel.emit(...)` + `zml.ops.triton(...)`. -It must be called from inside a `Compilation` (i.e., during -`zml.Compiler.compile(...)`) — `emit` builds the TTIR string in its own -throwaway context; `zml.ops.triton(...)` drops the `stablehlo.custom_call` -into the model's context. For offline tools, `Kernel.emit(allocator, cfg)` -gives you just the TTIR string. - -Signature: `Kernel.call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results`. - -| Positional arg | Type | Meaning | -|----------------|------------------|-------------------------------------------------------------------------| -| `inputs` | `Kernel.Inputs` | Struct of `Tensor`, one field per name in `spec.inputs`. | -| `outputs` | `Kernel.Outputs` | Struct of `Shape`, one field per name in `spec.outputs`. | - -`Kernel.CallOpts` (typed struct, all fields named): - -| Field | Type | Meaning | -|---------------------------|-----------------------|------------------------------------------| -| `.cfg` | `Kernel.Config` | Per-call kernel config. | -| `.grid` | `[3]i32` | Triton launch grid `(grid_x, grid_y, grid_z)`. | -| `.num_warps` | `i32` | Triton launch param. | -| `.num_stages` | `i32` | Triton launch param. | -| `.output_operand_aliases` | `?zml.ops.CustomCallOutputOperandAliases(Inputs, Outputs)` (default `null`) | For in-place updates: each output field names the input it aliases — `.{ .out = .x_ptr }`. | -| `.debug` | `bool` (default false)| Emit `tt.debug = true` on the custom call. | - -Returns `Kernel.Results` — a struct of `Tensor` named like `spec.outputs`. -Read by name: `out.out`, `attn_output.segm_max`, etc. - -The `cfg` value can be a comptime literal *or* a runtime expression built -from `Tensor.dim()` / `Options` fields. The kernel TTIR is regenerated per -call (cached by the MLIR canonicalizer / CSE pass at the XLA layer). - -> **Why named `Inputs` / `Outputs`.** The previous form took -> `anytype` tuples and matched by position; misordering 27 args (typical -> in `unified_attention.zig`) was a silent corruption hazard. With -> generated structs (`Inputs = StructOf(spec.inputs, Tensor)`, -> `Outputs = StructOf(spec.outputs, Shape)`), Zig refuses unknown fields, -> missing fields, and wrong-typed values at the callsite. - ---- - -## Argument specs - -Each field passed to `b.declareArgs(...)` is an `ArgSpec.Kind` tagged-union -literal. - -The four arg kinds mirror what TTIR's `tt.func` can carry: - -| Kind literal | MLIR type | Use case | -|---------------------------------------------------------|--------------------------|---------------------------------| -| `.{ .ptr = .f32 }` | `!tt.ptr` | Pointers (default divisibility) | -| `.{ .scalar = .i32 }` | plain `i32` | Strides, sizes, flags | -| `.{ .tensor = .{ &.{64, 128}, .f32 } }` | `tensor<64x128 x f32>` | Pre-built tensor input (rare) | -| `.{ .ptr_opts = .{ .dtype = .f32, .divisibility = 16 } }` | `!tt.ptr` with override | Custom `address_space`/divisibility | -| `.{ .scalar_opts = .{ .dtype = .i32, .divisibility = 16 } }` | plain `i32` with `tt.divisibility = 16 : i32` | Scalar with specialization hint | - -`.ptr` gets a default `tt.divisibility = 16` hint (matching Python Triton's -default). Use `.ptr_opts` to change that (or set `divisibility = null` to -suppress). `.scalar` carries no extra hint; switch to `.scalar_opts` when -you want to declare the same `tt.divisibility` Python's JIT auto-attaches -to int args whose runtime value is a multiple of 16. - -**Runtime dtypes:** the variant *tags* (`.ptr`, `.scalar`, etc.) must be -fixed per kernel — every call to `b.declareArgs(...)` for a given kernel -uses the same struct shape. The variant *payloads* (the dtype enum, the -divisibility int) can be runtime values from `cfg`: -`.a_ptr = .{ .ptr = cfg.a_dtype }` works fine even when `cfg.a_dtype` is a -runtime field. - ---- - -## The `Value` type - -```zig -pub const Value = struct { - inner: *const mlir.Value, - kernel: ?*Builder = null, - ... -}; -``` - -- `inner` is the underlying MLIR SSA handle; reach for it when you need to call - into raw `ttir.*` / `arith.*` constructors. -- `kernel` is the owning `Builder` pointer; all fluent methods route through it. - The DSL populates `kernel` automatically for every value it creates (`emit`, - `arg`, loop-region args, reduction-region args, etc.). If you construct a - `Value` by hand (`.{ .inner = raw }`), the fluent methods will panic — use - `Builder.*` static helpers instead. - -Introspection methods always available: - -| Method | Returns | Notes | -|-------------------|-----------------------------------|-----------------------------------------| -| `type_()` | `*const mlir.Type` | Full MLIR type (scalar or tensor). | -| `elemType()` | `*const mlir.Type` | Element type; same as `type_()` for scalars. | -| `isTensor()` | `bool` | `true` for ranked tensors. | -| `rank()` | `usize` | `0` for scalars. | -| `dim(i)` | `i64` | Traps on scalars. | -| `shape()` | `Shape` (BoundedArray, `MAX_RANK`)| Use `.constSlice()` to feed `[]const i64`. | -| `isFloatElem()` | `bool` | True for any float element type. | -| `isIntElem()` | `bool` | True for any integer element type. | - ---- - -## Fluent methods on `Value` - -Every fluent method returns a new `Value`. They dispatch to -`arith.addi` / `arith.addf` / ... based on the element type. The RHS is -`anytype` — see [Polymorphic scalars](#polymorphic-scalars). - -### Arithmetic - -| Method | Int op | Float op | Notes | -|----------------|--------------|--------------|------------------------------------| -| `.add(rhs)` | `arith.addi` | `arith.addf` | | -| `.sub(rhs)` | `arith.subi` | `arith.subf` | | -| `.mul(rhs)` | `arith.muli` | `arith.mulf` | | -| `.div(rhs)` | `arith.divsi`| `arith.divf` | Signed integer divide. | -| `.rem(rhs)` | `arith.remsi`| `arith.remf` | Signed integer remainder. | -| `.cdiv(rhs)` | `arith.ceildivsi` | — | Int only — for `tl.cdiv`. | -| `.minimum(rhs)`| `arith.minsi`| `arith.minnumf` | Matches `tl.minimum` (non-NaN-prop). | -| `.maximum(rhs)`| `arith.maxsi`| `arith.maxnumf` | Matches `tl.maximum` (non-NaN-prop). | -| `.bitAnd(rhs)` | `arith.andi` | — | Also handles i1 masks. | -| `.bitOr(rhs)` | `arith.ori` | — | | - -### Comparisons - -All return i1 (scalar) or `tensor<... x i1>`. Signed integer predicates by -default; `.olt` / `.oge` / ... for floats. - -| Method | Int predicate | Float predicate | -|--------------|---------------|-----------------| -| `.lt(rhs)` | `.slt` | `.olt` | -| `.le(rhs)` | `.sle` | `.ole` | -| `.gt(rhs)` | `.sgt` | `.ogt` | -| `.ge(rhs)` | `.sge` | `.oge` | -| `.eq(rhs)` | `.eq` | `.oeq` | -| `.ne(rhs)` | `.ne` | `.one` | - -### Casts — `.to(dtype)` - -`v.to(.f32)` chooses the right MLIR op based on the current vs target kind: - -| Source kind | Target kind | MLIR op | -|-------------|------------------|-------------------------------| -| int | wider int | `arith.extsi` | -| int | narrower int | `arith.trunci` | -| int | float | `arith.sitofp` | -| **i1** | **float** | **`arith.uitofp`** (i1 is unsigned bool; sign-extending `1` would land at `-1.0`) | -| float | int | `arith.fptosi` | -| float | wider float | `arith.extf` (no rounding) | -| float | narrower float | fp8 involved or custom rounding → `tt.fp_to_fp`; otherwise `arith.truncf` | -| same type | same type | no-op (returns `self`) | - -Triton's verifier rejects `tt.fp_to_fp` with a rounding attribute when the -target is not strictly narrower than the source (e.g. `bf16 → f32`, or -same-width `bf16 ↔ f16`). The DSL routes strict upcasts through -`arith.extf` to match Python Triton's own lowering; only genuine -downcasts carry the `rtne` rounding attribute. - -For narrowing FP→FP, the dispatch follows -`triton/python/triton/language/semantic.py:cast()`: standard FP narrowing -between fp64 / fp32 / fp16 / bf16 lowers to `arith.truncf` (rank-independent -— scalar and tensor both), while `tt.fp_to_fp` is reserved for cases where -fp8 is on either side or the caller requested a non-default -`fp_downcast_rounding`. - -### Constants - -There is no constant cache and no hoisting: every lifted literal emits its -own `arith.constant` where it is used (see Pitfall 5). `zeros(shape, dtype)`, -`ones(shape, dtype)`, `full(shape, value, dtype)` and `splat` of a literal -emit a scalar `arith.constant` and a `tt.splat`; `splat(Value, shape)` emits -the `tt.splat` alone. - -### Shape / pointer helpers - -| Method | Effect | -|------------------------------|-------------------------------------------------------------| -| `v.expandDims(axis)` / `b.expandDims(v, axis)` | `tt.expand_dims` — insert a size-1 axis at `axis`. Mirrors `v[:, None]` (`axis=1`) / `v[None, :]` (`axis=0`). | -| `b.broadcastTo(v, shape)` | `tt.broadcast` — explicit broadcast (size-1 dims → target). Rarely needed; auto-broadcast handles it. | -| `.splatTo(shape)` | `tt.splat` this scalar to the given shape. | -| `.addPtr(offset)` | `tt.addptr`; `offset` can be a Value or a comptime int. | - -> **Auto-broadcast is everywhere.** Mirrors Triton's -> `semantic.broadcast_impl_value` (`triton/python/triton/language/semantic.py:724`): -> rank-unification (prepend size-1 axes) + size-1 → size-N expansion. -> Applied transparently at: -> - every fluent binop (`.add`, `.mul`, `.lt`, `.bitAnd`, …) -> - `.addPtr(offset)` (either side may be scalar or rank-mismatched) -> - `b.where(cond, x, y)` / `b.select(...)` -> - `mask` and `other` of `b.loadOpts` / `b.storeOpts` -> -> So `v[:, None] * stride + w[None, :] * other_stride` is just: -> -> ```zig -> v.expandDims(1).mul(stride).add(w.expandDims(0).mul(other_stride)) -> ``` -> -> No `broadcastTo`, no full-shape `tt.splat` — the broadcast materializes -> implicitly at the `addi`. This matches Python's emit pattern op-for-op -> (Python's frontend never emits an eager `tt.broadcast` followed by a -> `arith.muli` either; it emits `expand_dims` + `splat + muli`). -> -> Reach for `b.broadcastTo` only when the consumer is a low-level op that -> bypasses the DSL's auto-broadcast — direct `arith.*` / raw `ttir.*` -> escape-hatch paths. - ---- - -## Polymorphic scalars - -Every fluent method's RHS (and every `b.splat` / `b.lift` input) is `anytype`. -The lift rule: - -- `Value` → pass through. -- `comptime_int` / `comptime_float` → a constant whose element type matches - the LHS operand. `v_i64.mul(16)` lifts `16` to i64, `v_f32.add(1.0)` lifts - `1.0` to f32, and so on. -- Runtime `i8/i16/i32`, `i64`, `f16/f32/f64`, etc. → a constant that preserves - the source Zig width. i8/i16/i32 (signed or unsigned) → i32 const; i64/u64 → - i64 const; f16 → f16; f32 → f32; f64 → f64. Unsigned ints are bit-cast - (bit pattern preserved), so `@as(u32, 0xFFFF_FFFF)` becomes an i32 with the - same all-ones bit pattern. - -This is why the common Python idiom `offs < 64` works in Zig as -`offs.lt(64)` without any wrapper — as long as the comptime scalar can be -represented in `offs`'s element type. - -If you need to force a specific width for a runtime variable, cast at the call -site: `pid.mul(@as(i32, @intCast(block_size)))`. - -### When is `@as` actually needed? (Rule of thumb) - -Auto-lift covers most cases. `@as(...)` around a binop rhs is **redundant** -in all of these — drop it: - -| Pattern | Auto-lifts to… | -|------------------------------------------|---------------------------------------| -| `v_i64.eq(0)` / `.div(16)` / `.add(-1)` | comptime_int → match lhs elem type | -| `v_f32.mul(1.0 / config.fp8_max)` | already-f32 expression → f32 const | -| `v_i64.lt(NUM_QUERY_HEADS)` (Zig i64) | runtime i64 → i64 const | -| `v_i32.add(pad_term)` (Zig i32) | runtime i32 → i32 const | -| `b.splat(-1, shape)` / `b.openFor(0, N, 1, ...)` | comptime → matches elem / the first Value bound | -| `b.constant(.i32, 0)` | constant takes anytype — wrap nothing | - -`@as(...)` **is** required exactly when you're narrowing a runtime Zig int -to a smaller width that `lift` won't pick on its own. The idiom is -`@as(i32, @intCast(X_i64))` — you need the `@as` because `@intCast` has no -target type to infer, and without the narrowing `lift` would emit an i64 -constant, which widens the i32 lhs and the result to i64: - -```zig -// BLOCK_Q is a Zig i64 (from config). lhs is i32. Must narrow. -q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx) -``` - -The alternative — `q_start_raw.to(.i64).div(BLOCK_Q).to(.i32)` — mirrors -Python's `.to(tl.int64)` style but costs two extra Value-level casts; prefer -the narrowing `@as` or a pre-declared `const FOO_I32: i32 = @intCast(FOO);` -when the same width cast recurs. - -### When do you need `b.lift` / `b.constant`? - -Almost never. The DSL auto-lifts rhs for every fluent binop, `b.splat`, -`b.full`, `b.openFor` bounds, `.addPtr` offsets, etc. Reach for `b.lift` / -`b.constant` **only** when the DSL surface demands an existing `Value` and -there's no binop sibling to match against — typically: - -- Loop seeds for `openWhile(.{inits}, ...)` where you start with a raw - comptime literal: `const left_init = b.constant(.i32, 0);`. -- Mutable `var x: Value = b.constant(.i32, 0);` that a later branch may - reassign — the var needs an initial Value even if the comptime path would - work on its own. - -If the constant is about to flow through any `.add` / `.mul` / `.splatTo` -etc., skip the lift and let auto-lift do it. And if a stride is hardcoded -to 1, **delete it**: `x.mul(1)` is a no-op, the DSL isn't Python `constexpr` -where such names are required. - -### Scalar ↔ tensor auto-broadcast - -Fluent arithmetic and comparison ops (`.add`, `.sub`, `.mul`, `.div`, `.rem`, -`.cdiv`, `.bitAnd`, `.bitOr`, `.minimum`, `.maximum`, `.lt`, `.le`, `.gt`, -`.ge`, `.eq`, `.ne`) are symmetric: if one operand is a tensor and the other a -scalar, the scalar is splatted to match. Either direction works: - -```zig -const offs = iv.add(cols); // scalar iv + tensor cols → tensor -const offs = cols.add(iv); // tensor cols + scalar iv → tensor -``` - -This mirrors Triton/NumPy's `i + cols`. **Size-1 and rank-unification -broadcasting also work** (mirroring Python's -`semantic.broadcast_impl_value`): a `(M, 1)` and a `(1, N)` operand -broadcast to `(M, N)` automatically, and a `(M,)` lined up against a -`(M, N)` is first `expand_dims`'d to `(1, M)` and then broadcast. So -`offs_token[:, None] + offs_k[None, :]` is just: - -```zig -const offs = b.expandDims(offs_token, 1).add(b.expandDims(offs_k, 0)); -``` - -with no `b.broadcastTo` calls needed. `b.where` / `b.select` and the -`mask` / `other` operands of `b.loadOpts` / `b.storeOpts` also auto-broadcast, -so the only time `b.broadcastTo` is needed is when the consumer bypasses -the DSL (raw `arith.*` / `ttir.*` escape-hatch paths). - -`.addPtr` auto-splats in **both directions**, matching Triton's -`ptr + offset` semantics: a scalar pointer is splatted to match a tensor -offset (`x_ptr.addPtr(tensor_offs)`), and a scalar offset is splatted to -match a tensor pointer (`a_ptrs.addPtr(BLOCK_K)`). Pre-splatting via -`b.splat` is no longer required for the loop-stride case. - -### When is `scalar.splatTo(shape)` redundant? - -Because fluent binops (`.add` / `.mul` / …) already auto-splat scalars -against tensor siblings, and `.addPtr` auto-splats either side (scalar -pointer or scalar offset) against a tensor partner, a pre-emptive -`scalar.splatTo(shape)` is usually noise: - -```zig -// redundant -pid_n.mul(N).splatTo(shape).add(b.arange(0, N, .i64)).rem(n_block.splatTo(shape)) -// equivalent — scalars auto-splat against the tensor sibling -pid_n.mul(N).add(b.arange(0, N, .i64)).rem(n_block) -``` - -Keep the explicit `splatTo` only when the scalar feeds into a raw -`arith.*` / `ttir.*` escape-hatch op that doesn't go through the DSL's -auto-broadcast. - ---- - -## Comptime vs runtime widths - -TTIR has no implicit int-width conversion — `arith.muli(i32, i64)` is a verifier -error. The DSL mirrors Python and **auto-promotes** integer binops when the -two runtime operands have different widths: it `extsi`s the narrower side -to the wider, matching `semantic.integer_promote_impl`. So -`v_i32.mul(v_i64)` works without a manual `.to(.i64)` — it emits exactly -the same `extsi` + `muli` pair Python would. - -For the rare case where you want explicit control, you can still cast -yourself; the auto-promotion is a no-op when widths already agree. - -Recurring example from `zml/kernels/moe/triton/moe.zig`: - -```zig -// pid (i32) vs block (i64 because it came from a usize config) -const block_i32: i32 = @intCast(block); -const token_start_init = pid.mul(block_i32); -// alternative: pid.to(.i64).mul(block) -``` - -For pointer math it is often cleaner to promote everything to i64 up front: - -```zig -const pid = b.programId(.x).to(.i64); -const offs = b.arange(0, BLOCK, .i64); -``` - ---- - -## Shape helpers - -| Helper | Effect | -|----------------------------------------|--------------------------------------------------| -| `b.arange(start, end, dtype)` | `tl.arange(start, end).to(dtype)`. Pass `.i32` for no-op cast. | -| `b.zeros(shape, dtype)` | `tl.zeros(shape, dtype=dtype)`. | -| `b.ones(shape, dtype)` | Ones tensor. | -| `b.splat(value, shape)` | `tl.splat`; accepts Value or comptime scalar. | -| `b.expandDims(v, axis)` / `v.expandDims(axis)` | `tt.expand_dims` — insert a size-1 axis. Mirrors `v[:, None]` (`axis=1`) / `v[None, :]` (`axis=0`). | -| `b.mask2d(cond_m, cond_n, _, _)` | `cond_m[:, None] & cond_n[None, :]` — emits the `(m,1) & (1,n)` form; auto-broadcast at the next consumer. | -| `b.lift(value)` | Wrap a Zig scalar as a DSL Value; dtype inferred from source. | -| `b.constant(dtype, value)` | Wrap a Zig scalar as a DSL Value of a specific `DType`. | -| `b.constantLike(value, like)` | Wrap a Zig scalar as a DSL Value of `like`'s element type. | - ---- - -## Loads and stores - -Load helpers infer everything from the pointer's type — scalar `!tt.ptr` -loads a scalar `T`, `tensor<... x !tt.ptr>` loads a `tensor<... x T>`. -No `result_type` / `shape` / `dtype` arguments required. - -| Helper | Python equivalent | -|-------------------------------------------------|-------------------------------------------| -| `b.load(ptr)` | `tl.load(ptr)` — scalar or tensor. | -| `b.loadOpts(ptr, .{ .mask, .other, ... })` | `tl.load(ptrs, mask=..., other=..., cache_modifier=..., eviction_policy=..., volatile=...)`. | -| `b.store(ptr, value)` | `tl.store(ptr, value)`. | -| `b.storeOpts(ptr, value, .{ .mask, ... })` | `tl.store(ptr, value, mask=..., cache_modifier=..., eviction_policy=...)`. | - -`LoadOpts` / `StoreOpts` fields (`.mask`, `.other`, `.cache_modifier`, -`.eviction_policy`, `.@"volatile"`) take DSL-level `Value`s / enum literals — -no raw MLIR types needed. - -> **Match Python's emit form for masked loads.** `tl.load(ptr, mask=m, -> other=0.0)` lowers to a 3-operand `tt.load` op. To match it byte-for-byte -> in the Zig DSL, pass an explicit `.other`: -> -> ```zig -> const zero = b.zeros(&.{BLOCK}, dtype); -> const x = b.loadOpts(ptr, .{ .mask = mask, .other = zero }); -> ``` -> -> Calling `loadOpts` with `.mask` only (no `.other`) emits a 2-operand -> `tt.load` — semantically equivalent (out-of-bounds lanes are undefined, -> not zero), but the TTIR/TTGIR diff vs Python will not be empty. Triton's -> `tl.load(ptr, mask=...)` without `other=` lowers to the 2-operand form, -> so you only need the `.other` operand when the Python source explicitly -> passes `other=`. - -### Result-type inference - -Most helpers that produce a new Value infer their result type from the inputs, -so user code never constructs `*const mlir.Type` manually: - -| Helper | How the result type is derived | -|-----------------------------------------------------|---------------------------------------------| -| `b.load(ptr)` / `b.loadOpts(ptr, opts)` | From `ptr.type_()` (scalar or tensor of ptrs). | -| `b.dot(a, b, acc)` / `b.dotOpts(a, b, acc, opts)` | `acc.type_()`. | -| `b.dotScaled(...)` / `b.dotScaledOpts(...)` | `acc.type_()`. | -| `b.reshape(src, new_shape)` / `b.reshapeOpts(...)` | `new_shape` + `src` element type. | -| `b.gather(src, indices, axis)` | `indices` shape + `src` element type. | -| `b.histogram(src, num_bins)` / `b.histogramOpts(...)` | `tensor`. | -| `b.fpToFp(src, out_dtype)` / `b.fpToFpOpts(...)` | `src` shape + `out_dtype`. | -| `b.convertf(src, out_dtype)` / `b.convertfOpts(...)`| `src` shape + `out_dtype`. | -| `b.scalingTruncf(src, scale, out_dtype)` / …Opts | `src` shape + `out_dtype`. | -| `b.descriptorLoad(desc, indices, shape, dtype)` | `shape` + `dtype`. | - -Rule of thumb: you only supply things the op needs beyond what inputs carry — -a `new_shape`, a `num_bins`, an `out_dtype`, a TMA tile descriptor. The DSL -never asks you to restate information it can read off a `Value`. - -`b.scalarTy(dtype)` / `b.tensorTy(shape, dtype)` are still available but only -needed for tensor result types in `openIfElse` / `openWhile` (scalar ones are -plain dtypes) and by the raw `externElementwise` / `inlineAsmElementwise` -escape hatches. - -### `fn X / fn XOpts` pattern - -Every Builder method whose Python Triton counterpart has named keyword-only -arguments comes in two flavours: - -- `b.X(non_named_args...)` — defaults applied; matches `tl.X(args...)` in - Python, which in Python would use the zero-arg defaults. -- `b.XOpts(non_named_args..., struct { named... })` — full named-param set - as a typed Zig struct literal with field defaults matching Triton. - -Pairs in place today: `load` / `loadOpts`, `store` / `storeOpts`, -`dot` / `dotOpts`, `reshape` / `reshapeOpts`, -`atomicRmw` / `atomicRmwOpts`, `atomicCas` / `atomicCasOpts`, -`fpToFp` / `fpToFpOpts`, `clampf` / `clampfOpts`, -`cast` / `castOpts`, `cat` / `catOpts`, -`convertf` / `convertfOpts`, `scalingTruncf` / `scalingTruncfOpts`, -`dotScaled` / `dotScaledOpts`, -`inlineAsmElementwise` / `inlineAsmElementwiseOpts`, -`externElementwise` / `externElementwiseOpts`, -`descriptorLoad` / `descriptorLoadOpts`, `histogram` / `histogramOpts`, -`devicePrint` / `devicePrintOpts`, -`deviceAssert` / `deviceAssertOpts`, -`maximum` / `maximumOpts`, `minimum` / `minimumOpts`, -`sum` / `sumOpts`, `max` / `maxOpts`, `min` / `minOpts`, -`cumsum` / `cumsumOpts`, `cumprod` / `cumprodOpts`, -`openFor` / `openForOpts` (`tl.range(..., num_stages=...)`). - -DSL-only pairs (no Python kwarg counterpart — `.inline_return` is about IR -block layout, see the `returnIf` / `returnIfOpts` section under Control flow): -`returnIf` / `returnIfOpts`, `openReturnIf` / `openReturnIfOpts`. - -Ops with no kwargs in Python (pure positional): `gather(src, indices, axis)`, -`trans(src, order)`, `permute(src, order)`, `cat(lhs, rhs)` (dim=0 default), -`split(src)`, `join(lhs, rhs)`, `item(src)`, `broadcast(a, b)` (symmetric), -`broadcastTo(value, shape)`, `where(cond, x, y)`, `zeros(shape, dtype)`, -`full(shape, value, dtype)`. - ---- - -## Reductions and scans - -```zig -const absmax = b.max(abs_y); // tl.max(abs_y) — axis=None (all dims) -const total = b.sum(padded_counts); // tl.sum(padded_counts) -const cumul = b.cumsum(x); // tl.cumsum(x, axis=0) - -// Keyword arguments → the *Opts variant with a typed struct literal. -const axis0_max = b.maxOpts(abs_y, .{ .axis = 0 }); // tl.max(x, axis=0) -const absmax_kd = b.maxOpts(abs_y, .{ .axis = 0, .keep_dims = true }); // tl.max(x, 0, keep_dims=True) -const rev_cum = b.cumsumOpts(x, .{ .reverse = true }); // tl.cumsum(x, 0, reverse=True) -``` - -Under the hood these build the required `tt.reduce` / `tt.scan` region with a -matching `arith.addf` / `arith.addi` / `maxnumf` / `maxsi` combine function — -no more hand-rolled combiner closures for the common cases. - -> **Gotcha — `maxnumf` vs `maximumf` (and `minnumf` vs `minimumf`):** -> Triton's `tl.max`, `tl.min`, `tl.maximum`, `tl.minimum` all default to -> the **non-NaN-propagating** form (`arith.maxnumf` / `arith.minnumf`). -> The DSL fluent forms match Python: -> -> | Python | DSL | MLIR op | -> |-------------------------------------|--------------------------------------|-------------------| -> | `tl.max(x)` | `b.max(x)` / `x.max()` | `arith.maxnumf` | -> | `tl.min(x)` | `b.min(x)` / `x.min()` | `arith.minnumf` | -> | `tl.maximum(a, b)` | `a.maximum(b)` / `b.maximum(a, b)` | `arith.maxnumf` | -> | `tl.minimum(a, b)` | `a.minimum(b)` / `b.minimum(a, b)` | `arith.minnumf` | -> | `tl.maximum(a, b, propagate_nan=NAN_ALL)` | `a.maximumOpts(b, .{ .propagate_nan = .all })` | `arith.maximumf` | -> -> The lower-level builders **`b.maximumf(a, b)` / `b.minimumf(a, b)` skip -> Python's defaults** and emit IEEE-754 NaN-propagating ops directly. Reach -> for them only when you genuinely want NaN propagation; otherwise stick -> with `b.maximum` / `b.minimum` (or the fluent `Value.maximum` / -> `Value.minimum`). Mismatched ops here cause subtle TTIR/LLIR diffs vs -> Python and are a frequent porting bug. - -When `.keep_dims = true`, the DSL inserts a trailing -`tt.expand_dims` / `splat` so the result shape matches the input rank with a -size-1 reduced axis (matching Python Triton). For `axis = null` (the default), -the reduction goes through a `tt.reshape allow_reorder` first — even on rank-1 -input — so the layout-assignment pass picks the same reduction-friendly -layout Python's frontend gets. - -For custom combiners, use the low-level `b.reduce(ctx, .{ .src, .axis, .elem, -.result, .combine })` and `b.scan(ctx, .{ .src, .axis, .elem, .result, -.combine, .reverse })` — they take a `fn(*Builder, Value, Value, CtxT) Value` -closure for `combine` and give you full control. - ---- - -## Control flow - -SCF regions are built with **scope-based** builders — `b.openFor(...)`, -`b.openIf(...)` / `b.openIfElse(...)`, `b.openWhile(...)` return a typed scope -value. You emit body ops into a bare `{}` block, terminate with `yield` / -`yieldThen` / `yieldAfter` (or `close()` for a scope without values), and read -results from `scope.results` afterward. -No `ctx` struct, no function literal, no `kk`/`k` split — lexical scope -carries captures because everything runs in the same function. - -### `openFor` — scf.for with iter_args - -```zig -var loop = b.openFor(0, N, BLOCK, .{acc0}); -{ - const iv = loop.iv; - const acc = loop.carried[0]; - const offs = iv.add(cols); - const tile = b.loadOpts(x_ptr.addPtr(offs), .{ .mask = offs.lt(N) }); - const partial = tile.sum(); - loop.yield(.{ acc.add(partial) }); -} -const total = loop.results[0]; -``` - -- `lower`/`upper`/`step` are `anytype`: pass Values, comptime ints, or runtime - Zig ints. A bound that isn't a Value takes the type of the first one that is; - with none, `lower` is lifted (`i32` when it fits) and the others follow it. - Pass a typed constant (`b.constant(.i64, 0)`) to pick another width. -- `inits` is a tuple `.{v1, v2, ...}` or an array. Arity is comptime — - `carried`, `results`, and the `yield(...)` values (also a tuple or an array) - all have the same length. -- Empty loops: `b.openFor(0, N, S, .{})` + `loop.close()` (no iter_args). -- `b.openForOpts(lower, upper, step, inits, .{ .num_stages = 3 })` sets the - loop's options (`tt.num_stages`, `.unsigned_cmp`). - -### `openIf` — scf.if without else (side-effects only) - -```zig -var i = b.openIf(cond); -{ - // then body - i.close(); // closes scope, builds scf.if with empty else -} -``` - -Use for conditional side-effects (no results). There is no else branch — the -else block is auto-built empty. - -### `openIfElse` — scf.if with else and optional results - -```zig -var i = b.openIfElse(cond, .{.f32}); // result types -{ - // then body - i.yieldThen(.{ x }); -} -{ - // else body - i.yieldElse(.{ y }); -} -const r = i.results[0]; -``` - -- `result_types` is a tuple or array of dtypes (`.f32`), Values (for their - types) or `*const mlir.Type`s such as `b.tensorTy(shape, dtype)`. Pass - `.{}` for an if/else with no results but distinct then/else bodies, then - end them with `elseBranch()` and `close()`. -- `yieldThen` swaps to the else block; `yieldElse` builds the `scf.if` op. -- Both branches must yield arity-matching tuples. - -### `openWhile` — scf.while - -```zig -var w = b.openWhile(.{ i0 }, .{.i32}); -{ - // before region (condition) - const s = w.before_carried[0]; - w.yieldBefore(s.lt(10), .{ s }); // scf.condition: cond + values forwarded to after -} -{ - // after region (body) - const s = w.after_carried[0]; - w.yieldAfter(.{ s.add(1) }); // values forwarded back to before -} -const r = w.results[0]; -``` - -- `inits` — a tuple or array of `Value`s; before-region arg types = their types. -- `after_types` — a tuple or array of dtypes (`.f32`), Values (for their - types) or `*const mlir.Type`s; after-region arg types + the scf.while's - result types. -- `yieldBefore(cond, forwarded)` — `forwarded` arity must match `after_types`. -- `yieldAfter(values)` — arity must match `inits`. - -### `returnIf` / `returnIfOpts` — early `tt.return` via cf.cond_br - -```zig -// Python: if pid_m * BLOCK >= num_tokens_post_padded: return -const out_of_range = pid_m.mul(block).ge(num_tokens); -b.returnIf(out_of_range); -// ... ops here live in a fresh fall-through block ... -``` - -Lowers to: - -```mlir -cf.cond_br %out_of_range, ^ret, ^cont -^ret: - tt.return -^cont: - // subsequent ops -``` - -- Kernels return nothing, so neither form takes return values. -- Works **only at the top-level `tt.func` body** — don't call inside an - `scf.if`/`scf.for`/`scf.while` region (those must stay structured; cf - branches can't escape their parent scf region). -- After `returnIf`, the kernel's current block is swapped to a fresh - fall-through block — any subsequent `b.*` calls emit into that block, and - `b.finish()` appends the final `tt.return` there. -- Use this to match Python Triton's `if cond: return` lowering exactly. - Python emits `cf.cond_br` for early-return idioms while using `scf.for` / - `scf.if` for structured loops and value-carrying branches; our DSL does - the same. - -**`returnIfOpts(cond, .{ .inline_return = true })` — match Python's -per-`return` block layout.** By default `returnIf` routes every return path -(every early `returnIf` / `openReturnIf` plus the natural fall-through from -`b.finish`) through one **shared exit block** holding the single `tt.return`. -Python's frontend does the opposite: each `if cond: return` (and the natural -exit) gets its **own** `tt.return`-only block, and LLVM's SimplifyCFG later -merges them into a single `common.ret`. The two are semantically identical -but differ in the function-exit block's label and position — which renumbers -every `$L__BB` label downstream and blows up the PTX diff. When the Python source contains *any* literal `return` statement (it then -has ≥2 `ret` blocks in LLVM — the explicit one(s) plus the implicit -function-end one — which SimplifyCFG merges into `common.ret`), pass -`.{ .inline_return = true }` on *every* return site, including each -`openReturnIfOpts` (see below). Each one -then emits `tt.return` inline and SimplifyCFG reproduces Python's -`common.ret`. The default shared exit block emits exactly one `tt.return`, -so the merge never fires and the function-exit block lands somewhere else. - -### `openReturnIf` — early `tt.return` with side-effects in the taken branch - -```zig -// Python: if off_experts == -1: write_zeros_to_output(...); return -var scope = b.openReturnIf(is_dead); // or openReturnIfOpts, see `returnIfOpts` above -{ - writeZerosToOutput(k, c_ptr, ...); // ops here emit into ^ret - scope.close(); // closes ^ret with tt.return (or cf.br ^exit) -} -// ... ops here emit into ^cont (fall-through) ... -``` - -Lowers to the same `cf.cond_br ^ret, ^cont` shape as `returnIf`, but the -taken branch can carry arbitrary side-effects between the `cf.cond_br` and -the `tt.return`. Use this instead of the -`openIf(cond){body}close; returnIf(cond)` pair — that pattern -duplicates the predicate (emits both `scf.if` and `cf.cond_br` on the same -boolean) and drifts from Triton's reference IR. - -`openReturnIfOpts(cond, .{ .inline_return = true })` makes `close` end -`^ret` with `tt.return` directly instead of `cf.br ^exit`. Use it on every -`returnIf` / `openReturnIf` of a kernel when matching a Python source that has -literal `return` statements. - -Same restrictions as `returnIf`: top-level `tt.func` body only. - -### When to use `cf` (returnIf / openReturnIf) vs `scf` (openIf / openIfElse / openFor) - -| Python source | Emits | DSL builder | -|----------------------------------------|-------|---------------------------------| -| `if cond: return` | `cf` | `b.returnIf(cond)` | -| `if cond: ; return` | `cf` | `var s = b.openReturnIf(cond); {...; s.close()}` | -| `if cond: ` | `scf` | `b.openIf(cond) {...; close()}` | -| `if cond: x=A else: x=B` (value-carrying) | `scf` | `b.openIfElse(cond, .{ty}) {...; yieldThen(.{a})} {...; yieldElse(.{b})}` | -| `for i in range(...)` | `scf` | `b.openFor(lo, hi, step, .{inits})` | -| `while cond:` | `scf` | `b.openWhile(.{inits}, .{after_types})` | - -Rule of thumb: **`scf` for structured, value-carrying control. `cf` for -early-exits.** Both lower to identical PTX after the -`convert-scf-to-cf` pass — the difference only matters at the TTIR layer -(pattern-matching passes that run before lowering). - -### `reduce` / `scan` — custom combiners (still callback-based) - -```zig -b.reduce(ctx, .{ .src, .axis, .elem, .result, .combine = fn (*Builder, Value, Value, Ctx) Value }); -b.scan (ctx, .{ .src, .axis, .reverse = false, .elem, .result, .combine }); -``` - -These stay on the callback pattern because the combine fn is a tiny pure -function (`(a, b) -> a ⊕ b`) where a `ctx` struct is a cleaner fit than a -scoped builder. Reach for `b.sum` / `b.max` / `b.cumsum` / etc. first — the -built-in reductions cover the common cases. - -### General tips - -- Scopes stack naturally: `openFor { openIf { openFor { ... } } }` all just - push/pop the kernel's current-block stack (same for `openIfElse` / - `openWhile`). -- Scopes without values end with `close()`: `loop.close()`, `i.close()`, and - `elseBranch()` then `close()` for an if/else. -- After any `yield*`, the scope's `.results` field holds the scf op's results - as a `[N]Value`. - ---- - -## Escape hatches - -Everything the DSL doesn't expose is still reachable — `Builder` is a -book-keeping layer, not a walled garden: - -- `v.inner` — the raw `*const mlir.Value`; feed it to any `ttir.*` / - `arith.*` / `math.*` / `scf.*` constructor directly. -- `b.ctx` — the owning `*mlir.Context`. -- `b.currentBlock()` — the active `mlir.Block` (respects `pushBlock` / - `popBlock`). -- `b.emit(op)` / `b.emitMulti(op, n)` — if you've built an `mlir.Operation` - by hand, use these so the result comes back as a `Value` with its `kernel` - field populated (you'll need that for any subsequent fluent calls). - -When you drop to raw ops, remember to still **append them to the current -block** — `b.emit(...)` does that for you; `_ = op.appendTo(b.currentBlock())` -is the manual form. - ---- - -## Lower layer — `Builder.open` - -The declarative `zml.kernel.triton.Kernel(Config, spec)` form covers most -cases. For dynamic-arity arg lists, or escape-hatch scenarios where you want -to drive the IR construction yourself, use the `Builder` directly: - -```zig -const Builder = tri.Builder; - -pub fn fusedMoeKernel( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - config: GenerationConfig, -) ![:0]const u8 { - var b = try Builder.open(allocator, ctx, "fused_moe_kernel"); - defer b.deinit(); - const a = try b.declareArgs(.{ - .a_ptr = .{ .ptr = config.a_dtype }, - .b_ptr = .{ .ptr = config.b_dtype }, - // ... - }); - - // ... emit body via b.* helpers; same DSL as inside `Kernel(...)` ... - - return b.finish(); -} -``` - -- `Builder.open(allocator, ctx, name)` creates an empty builder, and - `b.declareArgs(spec)` creates the `tt.func` and returns its arguments as - the same `NamedArgs(Spec)` struct as inside `Kernel(...)` — exactly what - `zml.kernel.triton.Kernel(...)` does internally. - `b.declareArgsOpts(spec, .{ .@"noinline" = true })` sets the function's - options. -- `b.finish()` terminates with `tt.return`, verifies, and serializes to a - `[:0]const u8` owned by `allocator`. `b.deinit()` frees the module. - -For dynamic-arg-count construction, `Builder.init(allocator, ctx, name, -args)` takes a runtime `[]const ArgSpec` instead of a comptime struct -literal; you grab block args with `b.arg(i)`. - -Pass these constructors a `zml.kernel.triton.newContext()`, not a -`Compilation`'s `mlir_ctx`; `defer ctx.deinit()` once you have the IR -string. `Kernel.emit` does this for you, which is why it takes no context. - -Prefer `zml.kernel.triton.Kernel(Config, spec)` for the common case. - ---- - -## Python Triton ↔ Zig DSL cheat-sheet - -| Python | Zig | -|-----------------------------------------------|--------------------------------------------------------------------| -| `tl.program_id(0)` | `b.programId(.x)` | -| `tl.num_programs(0)` | `b.numPrograms(.x)` | -| `tl.arange(0, N)` | `b.arange(0, N, .i32)` | -| `x.to(tl.int64)` | `x.to(.i64)` | -| `a + b`, `a * b`, `a // b` | `a.add(b)`, `a.mul(b)`, `a.div(b)` | -| `a < b` | `a.lt(b)` | -| `mask & other` | `mask.bitAnd(other)` | -| `tl.where(c, a, b)` | `b.where(c, a, b)` (alias of `b.select`) | -| `vec[:, None]` | `vec.expandDims(1)` | -| `vec[None, :]` | `vec.expandDims(0)` | -| `m[:, None] & (c[None, :] < N)` | `b.mask2d(m, c.lt(N), m_sz, n_sz)` | -| `ptr + offs` | `ptr.addPtr(offs)` (auto-splats scalar↔tensor either way) | -| `tl.load(ptr)` | `b.load(ptrs)` | -| `tl.load(ptr, mask=m)` | `b.loadOpts(ptrs, .{ .mask = m })` | -| `tl.load(ptr, mask=m, other=o)` | `b.loadOpts(ptrs, .{ .mask = m, .other = o })` | -| `tl.store(ptr, v)` | `b.store(ptr, v)` | -| `tl.store(ptr, v, mask=mask)` | `b.storeOpts(ptr, v, .{ .mask = mask })` | -| `tl.zeros((M, N), dtype=tl.float32)` | `b.zeros(&.{ M, N }, .f32)` | -| `tl.full((M, N), 3, tl.int32)` | `b.full(&.{ M, N }, 3, .i32)` | -| `tl.sum(x)` | `b.sum(x)` — axis=None default | -| `tl.sum(x, axis=0)` | `b.sumOpts(x, .{ .axis = 0 })` | -| `tl.sum(x, axis=0, keep_dims=True)` | `b.sumOpts(x, .{ .axis = 0, .keep_dims = true })` | -| `tl.max(x)` / `tl.min(x)` | `b.max(x)` / `b.min(x)` | -| `tl.max(x, axis=0)` | `b.maxOpts(x, .{ .axis = 0 })` | -| `tl.maximum(a, b)` / `tl.minimum(a, b)` | `a.maximum(b)` / `a.minimum(b)` or `b.maximum(a, b)` | -| `tl.cumsum(x)` / `tl.cumprod(x)` | `b.cumsum(x)` / `b.cumprod(x)` | -| `tl.cumsum(x, axis=0, reverse=True)` | `b.cumsumOpts(x, .{ .reverse = true })` | -| `tl.dot(a, b, acc=acc)` | `b.dot(a, b, acc)` — defaults to `input_precision = tf32` (NVIDIA) | -| `tl.dot(a, b, acc, input_precision="ieee")` | `b.dotOpts(a, b, acc, .{ .input_precision = .ieee })` | -| `tl.clamp(x, mn, mx)` | `b.clampf(x, mn, mx)` | -| `tl.clamp(x, mn, mx, propagate_nan=NAN_ALL)` | `b.clampfOpts(x, mn, mx, .{ .propagate_nan = .all })` | -| `tl.atomic_add(p, v)` | `b.atomicRmw(.add, p, v)` | -| `tl.atomic_add(p, v, mask=m, sem="relaxed")` | `b.atomicRmwOpts(.add, p, v, .{ .mask = m, .sem = .relaxed })` | -| `tl.histogram(x, num_bins)` | `b.histogram(x, num_bins)` | -| `tl.histogram(x, num_bins, mask=m)` | `b.histogramOpts(x, num_bins, .{ .mask = m })` | -| `tl.reshape(x, shape)` | `b.reshape(x, &.{ ...shape })` | -| `tl.reshape(x, shape, can_reorder=True)` | `b.reshapeOpts(x, &.{ ...shape }, .{ .can_reorder = true })` | -| `tl.cast(x, tl.bfloat16)` | `b.cast(x, .bf16)` (or fluent `x.to(.bf16)`) | -| `tl.cast(x, tl.bf16, fp_downcast_rounding="rtne")` | `b.castOpts(x, .bf16, .{ .fp_downcast_rounding = .rtne })` | -| `tl.cast(x, tl.int32, bitcast=True)` | `b.castOpts(x, .i32, .{ .bitcast = true })` | -| `src[indices]` (gather) | `b.gather(src, indices, axis)` | -| `tl.trans(x, dims)` / `tl.permute(x, dims)` | `b.trans(x, order)` / `b.permute(x, order)` | -| `tl.cat(a, b)` | `b.cat(a, b)` (dim=0 default) | -| `tl.cat(a, b, can_reorder=True)` | `b.catOpts(a, b, .{ .can_reorder = true })` | -| `tl.join(a, b)` / `tl.split(x)` | `b.join(a, b)` / `b.split(x)` | -| `tl.item(x)` | `b.item(x)` | -| `tl.abs(x)` | `b.abs(x)` (auto-dispatch) or fluent `x.abs()` | -| `tl.broadcast(a, b)` (symmetric) | `b.broadcast(a, b)` → `.{ Value, Value }` | -| `tl.broadcast_to(v, shape)` | `b.broadcastTo(v, &.{ ...shape })` | -| `tl.umulhi(a, b)` | `b.umulhi(a, b)` | -| `tl.sqrt_rn(x)` / `tl.div_rn(x, y)` | `b.sqrtRn(x)` / `b.divRn(x, y)` | -| `tl.device_print("dbg", args...)` | `b.devicePrint("dbg", args, is_signed)` | -| `tl.device_assert(cond, "msg")` | `b.deviceAssert(cond, "msg")` | -| `tl.inline_asm_elementwise(...)` | `b.inlineAsmElementwise(...)` | -| `tl.extern_elementwise(...)` | `b.externElementwise(srcs, shape, dtype, lib, path, sym)` | -| `for i in range(0, N, S): ...` | `var loop = b.openFor(0, N, S, .{inits...}); { ...; loop.yield(.{...}); }` | -| `while cond: ...` | `var w = b.openWhile(.{inits}, .{after_types}); { ...; w.yieldBefore(cond, .{...}); }; { ...; w.yieldAfter(.{...}); }` | -| `if cond: ...` (side-effects, no else) | `var i = b.openIf(cond); { ...; i.close(); }` | -| `if cond: ... else: ...` | `var i = b.openIfElse(cond, .{result_types}); { ...; i.yieldThen(.{...}); }; { ...; i.yieldElse(.{...}); }` | -| `if cond: return` (early exit) | `b.returnIf(cond);` — emits `cf.cond_br` + `tt.return`, continues in fall-through block (`b.returnIfOpts(cond, .{ .inline_return = true })` when matching a Python source that has literal `return`s) | -| `if cond: ; return` (early exit, with side-effects) | `var s = b.openReturnIf(cond); { ...; s.close(); }` — single `cf.cond_br`; body emits into `^ret` before the `tt.return` (`b.openReturnIfOpts(cond, .{ .inline_return = true })` to match, alongside `returnIfOpts`) | - ---- - -## Pitfalls - -1. **Mixed integer widths widen.** `v_i32.mul(v_i64)` is an i64 multiply - with an `arith.extsi` on the i32 side, and so is `v_i32.mul(x)` for a - runtime Zig `i64` `x`. Narrow with `@as(i32, @intCast(x))` or `.to(.i32)` - when the result must stay i32. Comptime literals take the other operand's - type instead. - -2. **Tensor-of-pointers via `.addPtr`.** Auto-splats either side: scalar - `!tt.ptr` + tensor offset (`x_ptr.addPtr(tensor_offs)`) and tensor - `tensor<...x!tt.ptr>` + scalar offset (`a_ptrs.addPtr(BLOCK_K)`) - both work without manual `.splatTo`. - -3. **Scope-based control flow — no closures needed.** `openFor` / `openIf` / - `openIfElse` / `openWhile` push a block onto the kernel's insertion stack; - body ops emitted in the bare `{}` that follows go into that block. Captures - are plain lexical scope, so you can reference outer Values, sizes, etc. - directly — no `ctx` struct, no function literal. - -4. **`Value.kernel == null` panics fluent calls.** Only happens if you build a - `Value` by hand via `.{ .inner = raw }`. Route through `b.emit(...)` to keep - the back-pointer populated — or just stick with the explicit `Builder.*` - helpers for that one op. - -5. **Don't worry about duplicate constants.** Every lifted literal (from - `.add(42)`, `.lt(0)`, `b.splat(1.0, shape)`, an `openFor(0, N, 1, …)` - bound, etc.) emits a fresh `arith.constant`. This matches Python Triton's - own frontend — the MLIR canonicalizer + CSE pass collapses duplicates on the - way to PTX, so the generated code is identical whether you emit one - constant or a hundred. Optimize for readability at the builder level. - -6. **`finish()` verifies before serializing.** A verifier - failure here almost always means an operand-type mismatch or a - missing block terminator — a common one is forgetting that `scf.yield` - operand types must match the loop's result types. - -7. **Yielding from loop / if / while bodies.** Inside a scope (`openFor`, - `openIf`, `openIfElse`, `openWhile`), you *don't* return values — you call - `scope.yield(...)` (or `yieldThen` / `yieldElse` / `yieldBefore` / - `yieldAfter`) with a tuple of Values matching the scope's declared arity. - Scopes with results (`openFor`, `openIfElse`, `openWhile`) store the scf - op's results in `scope.results` as a fixed-size `[N]Value` array. `openIf` - has no results — `close()` closes and builds scf.if with an empty - else block in one step. diff --git a/dsl/triton/builder.zig b/dsl/triton/builder.zig deleted file mode 100644 index 5ccec273..00000000 --- a/dsl/triton/builder.zig +++ /dev/null @@ -1,2132 +0,0 @@ -const std = @import("std"); - -const arith = @import("mlir/dialects/arith"); -const cf = @import("mlir/dialects/cf"); -const common = @import("dsl/common/builder"); -const func = @import("mlir/dialects/func"); -const llvm = @import("mlir/dialects/llvm"); -const math = @import("mlir/dialects/math"); -const scf = @import("mlir/dialects/scf"); -const ttir = @import("mlir/dialects/ttir"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -const dtypes = @import("dtype.zig"); -pub const DType = dtypes.DType; -const isFloatDtype = dtypes.isFloatDtype; -const dtypeBitwidth = dtypes.dtypeBitwidth; -const intBitwidth = dtypes.intBitwidth; - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Builder); - std.testing.refAllDecls(Value); -} - -pub const Value = struct { - inner: *const mlir.Value, - kernel: ?*Builder = null, - - pub const Shape = stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK); - - pub fn type_(self: Value) *const mlir.Type { - return self.inner.type_(); - } - - fn kern(self: Value) *Builder { - return self.kernel orelse @panic("Value has no owning kernel; use Builder.* helpers instead"); - } - - pub fn elemType(self: Value) *const mlir.Type { - if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.elementType(); - return self.type_(); - } - - pub fn isTensor(self: Value) bool { - return self.type_().isA(mlir.ShapedType) != null; - } - - pub fn rank(self: Value) usize { - if (self.type_().isA(mlir.ShapedType)) |shaped| return shaped.rank(); - return 0; - } - - pub fn dim(self: Value, i: usize) i64 { - return self.type_().isA(mlir.ShapedType).?.dimension(i); - } - - pub fn shape(self: Value) Shape { - var out: Shape = .empty; - if (self.type_().isA(mlir.ShapedType)) |shaped| { - const r = shaped.rank(); - for (0..r) |i| out.appendAssumeCapacity(shaped.dimension(i)); - } - return out; - } - - pub fn isFloatElem(self: Value) bool { - const et = self.elemType(); - inline for (comptime std.meta.fieldNames(mlir.FloatTypes)) |field_name| { - if (et.isA(mlir.FloatType(@field(mlir.FloatTypes, field_name))) != null) return true; - } - return false; - } - - pub fn isIntElem(self: Value) bool { - return self.elemType().isA(mlir.IntegerType) != null; - } - - pub fn add(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return if (l.isFloatElem()) k.addf(l, r) else k.addi(l, r); - } - - pub fn sub(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return if (l.isFloatElem()) k.subf(l, r) else k.subi(l, r); - } - - pub fn mul(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return if (l.isFloatElem()) k.mulf(l, r) else k.muli(l, r); - } - - pub fn div(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return if (l.isFloatElem()) k.divf(l, r) else k.divsi(l, r); - } - - pub fn rem(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return if (l.isFloatElem()) k.remf(l, r) else k.remsi(l, r); - } - - /// Ceil-div: `(x + (div - 1)) // div`. Comptime rhs folds `rhs - 1`. - pub fn cdiv(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return switch (@typeInfo(@TypeOf(rhs))) { - .comptime_int, .comptime_float => k.divsi(l.add(rhs - 1), r), - else => k.divsi(k.addi(l, r.sub(1)), r), - }; - } - - pub fn bitAnd(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return k.andi(l, r); - } - - pub fn bitOr(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return k.ori(l, r); - } - - pub fn minimum(self: Value, rhs: anytype) Value { - return self.kern().minimum(self, rhs); - } - pub fn minimumOpts(self: Value, rhs: anytype, opts: MinMaxOpts) Value { - return self.kern().minimumOpts(self, rhs, opts); - } - - pub fn maximum(self: Value, rhs: anytype) Value { - return self.kern().maximum(self, rhs); - } - pub fn maximumOpts(self: Value, rhs: anytype, opts: MinMaxOpts) Value { - return self.kern().maximumOpts(self, rhs, opts); - } - - pub fn lt(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return if (l.isFloatElem()) k.cmpf(.olt, l, r) else k.cmpi(.slt, l, r); - } - pub fn le(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return if (l.isFloatElem()) k.cmpf(.ole, l, r) else k.cmpi(.sle, l, r); - } - pub fn gt(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return if (l.isFloatElem()) k.cmpf(.ogt, l, r) else k.cmpi(.sgt, l, r); - } - pub fn ge(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return if (l.isFloatElem()) k.cmpf(.oge, l, r) else k.cmpi(.sge, l, r); - } - pub fn eq(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return if (l.isFloatElem()) k.cmpf(.oeq, l, r) else k.cmpi(.eq, l, r); - } - pub fn ne(self: Value, rhs: anytype) Value { - const k = self.kern(); - const l, const r = k.broadcast(self, rhs); - return if (l.isFloatElem()) k.cmpf(.one, l, r) else k.cmpi(.ne, l, r); - } - - pub fn to(self: Value, dtype: DType) Value { - return self.kern().cast(self, dtype); - } - pub fn bitCast(self: Value, dtype: DType) Value { - return self.kern().bitCast(self, dtype); - } - pub fn toOpts(self: Value, dtype: DType, opts: CastOpts) Value { - return self.kern().castOpts(self, dtype, opts); - } - - pub fn expandDims(self: Value, axis: i32) Value { - return self.kern().expandDims(self, axis); - } - - pub fn splatTo(self: Value, shape_: []const i64) Value { - return self.kern().splat(self, shape_); - } - - pub fn sum(self: Value) Value { - return self.kern().sum(self); - } - pub fn sumOpts(self: Value, opts: ReduceOpts) Value { - return self.kern().sumOpts(self, opts); - } - - pub fn max(self: Value) Value { - return self.kern().max(self); - } - pub fn maxOpts(self: Value, opts: ReduceOpts) Value { - return self.kern().maxOpts(self, opts); - } - - pub fn min(self: Value) Value { - return self.kern().min(self); - } - pub fn minOpts(self: Value, opts: ReduceOpts) Value { - return self.kern().minOpts(self, opts); - } - - pub fn cumsum(self: Value) Value { - return self.kern().cumsum(self); - } - pub fn cumsumOpts(self: Value, opts: ScanOpts) Value { - return self.kern().cumsumOpts(self, opts); - } - - pub fn cumprod(self: Value) Value { - return self.kern().cumprod(self); - } - pub fn cumprodOpts(self: Value, opts: ScanOpts) Value { - return self.kern().cumprodOpts(self, opts); - } - - pub fn gather(self: Value, indices: Value, axis: i32) Value { - return self.kern().gather(self, indices, axis); - } - - pub fn abs(self: Value) Value { - return self.kern().abs(self); - } - - pub fn addPtr(self: Value, offset: anytype) Value { - const k = self.kern(); - const off: Value = if (@TypeOf(offset) == Value) offset else k.lift(offset); - const ptr = if (!self.isTensor() and off.isTensor()) k.splat(self, off.shape().constSlice()) else self; - const off2 = if (self.isTensor() and !off.isTensor()) k.splat(off, self.shape().constSlice()) else off; - return k.addptr(ptr, off2); - } -}; - -fn computeReducedType(src_shape: []const i64, axis: i32, elem: *const mlir.Type) *const mlir.Type { - if (src_shape.len <= 1) return elem; - var out: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (src_shape, 0..) |d, i| { - if (@as(i32, @intCast(i)) != axis) out.appendAssumeCapacity(d); - } - return .rankedTensor(out.constSlice(), elem); -} - -pub const ArgSpec = struct { - name: []const u8, - kind: Kind, - - pub const Kind = union(enum) { - ptr: DType, - scalar: DType, - tensor: struct { []const i64, DType }, - ptr_opts: PtrOpts, - scalar_opts: ScalarOpts, - }; - - pub const PtrOpts = struct { - dtype: DType, - address_space: i32 = 1, - divisibility: ?u32 = 16, - }; - - pub const ScalarOpts = struct { - dtype: DType, - divisibility: ?u32 = null, - }; -}; - -pub const FinishError = error{InvalidMlir} || std.mem.Allocator.Error || std.Io.Writer.Error; - -pub const ClampOpts = struct { - propagate_nan: ttir.PropagateNan = .none, -}; - -pub const HistogramOpts = struct { - mask: ?Value = null, -}; - -pub const PrintOpts = struct { - hex: bool = false, -}; - -/// `axis=null` reduces all dims. -pub const ReduceOpts = struct { - axis: ?i32 = null, - keep_dims: bool = false, -}; - -pub const ScanOpts = struct { - axis: i32 = 0, - reverse: bool = false, -}; - -pub const LoadOpts = struct { - mask: ?Value = null, - other: ?Value = null, - cache_modifier: ttir.CacheModifier = .none, - eviction_policy: ttir.EvictionPolicy = .normal, - @"volatile": bool = false, -}; - -pub const StoreOpts = struct { - mask: ?Value = null, - cache_modifier: ttir.CacheModifier = .none, - eviction_policy: ttir.EvictionPolicy = .normal, -}; - -pub const ReshapeOpts = struct { - can_reorder: bool = false, - efficient_layout: bool = false, -}; - -pub const DotOpts = struct { - input_precision: ttir.InputPrecision = .tf32, - max_num_imprecise_acc: i32 = 0, -}; - -pub const FpToFpOpts = struct { - rounding: ?ttir.RoundingMode = null, -}; - -pub const AtomicRMWOpts = struct { - mask: ?Value = null, - sem: ttir.MemSemantic = .acq_rel, - scope: ttir.MemSyncScope = .gpu, -}; - -pub const AtomicCasOpts = struct { - sem: ttir.MemSemantic = .acq_rel, - scope: ttir.MemSyncScope = .gpu, -}; - -pub const DotScaledOpts = struct { - fast_math: bool = false, - lhs_k_pack: bool = true, - rhs_k_pack: bool = true, -}; - -pub const ExternElementwiseOpts = struct { - is_pure: bool = true, -}; - -pub const DescriptorLoadOpts = struct { - cache_modifier: ttir.CacheModifier = .none, - eviction_policy: ttir.EvictionPolicy = .normal, -}; - -pub const InlineAsmOpts = struct { - is_pure: bool = true, - pack: i32 = 1, -}; - -pub const CatOpts = struct { - can_reorder: bool = false, - dim: i32 = 0, -}; - -pub const CastOpts = struct { - fp_downcast_rounding: ?ttir.RoundingMode = null, - bitcast: bool = false, -}; - -pub const MinMaxOpts = struct { - propagate_nan: ttir.PropagateNan = .none, -}; - -pub const DeviceAssertOpts = struct { - mask: ?Value = null, -}; - -pub const ReturnIfOpts = struct { - inline_return: bool = false, -}; - -pub fn ForScope(comptime N: usize) type { - return common.ForScope(Builder, Value, N); -} - -/// `if (cond) { ...; return; }`: `close` returns at the end of the body, -/// then the kernel continues after it. -pub const ReturnIfScope = struct { - kernel: *Builder, - ret_block: *mlir.Block, - cont_block: *mlir.Block, - opts: ReturnIfOpts, - - pub fn close(self: *ReturnIfScope) void { - const k = self.kernel; - if (self.opts.inline_return) { - _ = ttir.return_(k.ctx, &.{}, k.loc()).appendTo(self.ret_block); - } else { - _ = cf.br(k.ctx, k.exitBlock(), &.{}, k.loc()).appendTo(self.ret_block); - } - k.popBlock(); - k.continueIn(self.cont_block); - } -}; - -pub const IfOnlyScope = common.IfOnlyScope(Builder, Value); - -pub fn IfScope(comptime N: usize) type { - return common.IfScope(Builder, Value, N); -} - -pub fn WhileScope(comptime N: usize, comptime M: usize) type { - return common.WhileScope(Builder, Value, N, M); -} - -pub fn ReduceArgs(comptime CtxT: type) type { - return struct { - src: Value, - axis: i32, - elem: *const mlir.Type, - result: *const mlir.Type, - combine: *const fn (*Builder, Value, Value, CtxT) Value, - }; -} - -pub fn ScanArgs(comptime CtxT: type) type { - return struct { - src: Value, - axis: i32, - reverse: bool = false, - elem: *const mlir.Type, - result: *const mlir.Type, - combine: *const fn (*Builder, Value, Value, CtxT) Value, - }; -} - -pub fn registerDialects(registry: *mlir.DialectRegistry) void { - inline for (.{ func, ttir, arith, scf, math, cf, llvm }) |dialect| dialect.registerDialect(registry); -} - -pub const Builder = struct { - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - ctx: *mlir.Context, - module: *mlir.Module, - name: []const u8, - func_op: ?*mlir.Operation, - entry_block: ?*mlir.Block, - exit_block: ?*mlir.Block = null, - block_stack: common.BlockStack = .{}, - - /// `noinline` is a Zig keyword — write `.{ .@"noinline" = true }`. - pub const Opts = struct { - @"noinline": bool = false, - }; - - pub fn init( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - name: []const u8, - args: []const ArgSpec, - ) !Builder { - return initOpts(allocator, ctx, name, args, .{}); - } - - pub fn initOpts( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - name: []const u8, - args: []const ArgSpec, - opts: Opts, - ) !Builder { - var b = try open(allocator, ctx, name); - errdefer b.deinit(); - try b.declareArgsLowOpts(args, opts); - return b; - } - - pub fn open( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - name: []const u8, - ) !Builder { - const unknown_loc: *const mlir.Location = .unknown(ctx); - const module: *mlir.Module = .init(unknown_loc); - errdefer module.deinit(); - - var arena: std.heap.ArenaAllocator = .init(allocator); - errdefer arena.deinit(); - - return .{ - .allocator = allocator, - .arena = arena, - .ctx = ctx, - .module = module, - .name = name, - .func_op = null, - .entry_block = null, - .block_stack = .{}, - }; - } - - pub fn declareArgs(self: *Builder, spec: anytype) !common.NamedArgs(@TypeOf(spec), Value) { - return self.declareArgsOpts(spec, .{}); - } - - pub fn declareArgsOpts( - self: *Builder, - spec: anytype, - opts: Opts, - ) !common.NamedArgs(@TypeOf(spec), Value) { - const Spec = @TypeOf(spec); - const field_names = @typeInfo(Spec).@"struct".field_names; - - var arg_specs: [field_names.len]ArgSpec = undefined; - inline for (field_names, 0..) |field_name, i| { - const raw = @field(spec, field_name); - const kind: ArgSpec.Kind = if (@TypeOf(raw) == ArgSpec.Kind) raw else blk: { - const variant = @typeInfo(@TypeOf(raw)).@"struct".field_names[0]; - const tag = @field(std.meta.Tag(ArgSpec.Kind), variant); - const inner = @field(raw, variant); - break :blk switch (tag) { - .ptr => .{ .ptr = inner }, - .scalar => .{ .scalar = inner }, - .tensor => .{ .tensor = inner }, - .ptr_opts => .{ .ptr_opts = .{ - .dtype = inner.dtype, - .address_space = if (@hasField(@TypeOf(inner), "address_space")) inner.address_space else 1, - .divisibility = if (@hasField(@TypeOf(inner), "divisibility")) inner.divisibility else 16, - } }, - .scalar_opts => .{ .scalar_opts = .{ - .dtype = inner.dtype, - .divisibility = if (@hasField(@TypeOf(inner), "divisibility")) inner.divisibility else null, - } }, - }; - }; - arg_specs[i] = .{ .name = field_name, .kind = kind }; - } - - try self.declareArgsLowOpts(&arg_specs, opts); - - var named: common.NamedArgs(Spec, Value) = undefined; - inline for (field_names, 0..) |field_name, i| { - @field(named, field_name) = self.arg(i); - } - return named; - } - - fn declareArgsLowOpts( - self: *Builder, - args: []const ArgSpec, - opts: Opts, - ) !void { - std.debug.assert(self.entry_block == null); - const ctx = self.ctx; - const unknown_loc: *const mlir.Location = .unknown(ctx); - const scratch = self.arena.allocator(); - - const arg_types = try scratch.alloc(*const mlir.Type, args.len); - const arg_locs = try scratch.alloc(*const mlir.Location, args.len); - for (arg_types, arg_locs, args) |*ty, *a_loc, a| { - ty.* = switch (a.kind) { - .ptr => |dt| ttir.pointerType(dt.toMlir(ctx), 1), - .ptr_opts => |p| ttir.pointerType(p.dtype.toMlir(ctx), p.address_space), - .scalar => |dt| dt.toMlir(ctx), - .scalar_opts => |s| s.dtype.toMlir(ctx), - .tensor => |t| .rankedTensor(t[0], t[1].toMlir(ctx)), - }; - a_loc.* = unknown_loc.named(ctx, a.name); - } - - const entry = mlir.Block.init(arg_types, arg_locs); - - const arg_attrs = try scratch.alloc(*const mlir.Attribute, args.len); - const empty_dict: *const mlir.Attribute = .dict(ctx, &.{}); - var any_arg_attr = false; - for (arg_attrs, args) |*attr, a| { - attr.* = switch (a.kind) { - .ptr => .dict(ctx, &.{ttir.divisibility(ctx, 16)}), - .ptr_opts => |p| if (p.divisibility) |v| - .dict(ctx, &.{ttir.divisibility(ctx, v)}) - else - empty_dict, - .scalar_opts => |s| if (s.divisibility) |v| - .dict(ctx, &.{ttir.divisibility(ctx, v)}) - else - empty_dict, - else => empty_dict, - }; - if (attr.* != empty_dict) any_arg_attr = true; - } - - const func_op = ttir.func(ctx, .{ - .name = self.name, - .args = arg_types, - .args_attributes = if (any_arg_attr) arg_attrs else null, - .results = &.{}, - .block = entry, - .location = unknown_loc, - .no_inline = opts.@"noinline", - }); - _ = func_op.appendTo(self.module.body()); - - self.func_op = func_op; - self.entry_block = entry; - self.block_stack.push(self.arena.allocator(), entry); - } - - pub fn deinit(self: *Builder) void { - self.module.deinit(); - self.arena.deinit(); - } - - pub fn arg(self: *Builder, i: usize) Value { - const eb = self.entry_block orelse @panic("Builder.arg called before declareArgs"); - return .{ .inner = eb.argument(i), .kernel = self }; - } - - pub fn pushBlock(self: *Builder, b: *mlir.Block) void { - self.block_stack.push(self.arena.allocator(), b); - } - - pub fn popBlock(self: *Builder) void { - self.block_stack.pop(); - } - - pub fn currentBlock(self: *Builder) *mlir.Block { - return self.block_stack.current(); - } - - fn innerSlice(self: *Builder, values: []const Value) []const *const mlir.Value { - const out = self.arena.allocator().alloc(*const mlir.Value, values.len) catch @panic("Builder.innerSlice OOM"); - for (values, 0..) |v, i| out[i] = v.inner; - return out; - } - - pub fn emit(self: *Builder, op: *mlir.Operation) Value { - _ = op.appendTo(self.currentBlock()); - return .{ .inner = op.result(0), .kernel = self }; - } - - pub fn emitMulti(self: *Builder, op: *mlir.Operation, n: usize) []Value { - _ = op.appendTo(self.currentBlock()); - const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.emitMulti OOM"); - for (0..n) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; - return out; - } - - pub fn yield(self: *Builder, values: anytype) []const Value { - const T = @TypeOf(values); - const info = @typeInfo(T); - if (info != .@"struct" or !info.@"struct".is_tuple) - @compileError("Builder.yield expects a tuple literal like `.{ v1, v2 }`"); - const n = info.@"struct".field_names.len; - const out = self.arena.allocator().alloc(Value, n) catch @panic("Builder.yield OOM"); - inline for (info.@"struct".field_names, info.@"struct".field_types, 0..) |field_name, Field, i| { - if (Field != Value) - @compileError("Builder.yield: every tuple element must be a Value; got " ++ @typeName(Field)); - out[i] = @field(values, field_name); - } - return out; - } - - pub fn loc(self: *const Builder) *const mlir.Location { - return .unknown(self.ctx); - } - - pub fn scalarTy(self: *const Builder, dtype: DType) *const mlir.Type { - return dtype.toMlir(self.ctx); - } - - pub fn tensorTy(self: *const Builder, shape: []const i64, dtype: DType) *const mlir.Type { - return .rankedTensor(shape, dtype.toMlir(self.ctx)); - } - - fn loadResultType(self: *const Builder, ptr: Value) *const mlir.Type { - _ = self; - const pt = ptr.type_(); - if (pt.isA(mlir.ShapedType)) |shaped| { - const elem_ptr_ty = shaped.elementType(); - const pointee = elem_ptr_ty.isA(ttir.PointerType).?.pointee(); - var shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..shaped.rank()) |i| shape.appendAssumeCapacity(shaped.dimension(i)); - return .rankedTensor(shape.constSlice(), pointee); - } - return pt.isA(ttir.PointerType).?.pointee(); - } - - fn swapElem(self: *const Builder, src: Value, out_dtype: DType) *const mlir.Type { - const out_elem = out_dtype.toMlir(self.ctx); - if (src.isTensor()) return .rankedTensor(src.shape().constSlice(), out_elem); - return out_elem; - } - - // ==================== SPMD ==================== - - pub fn programId(self: *Builder, dim: ttir.ProgramIDDim) Value { - return self.emit(ttir.get_program_id(self.ctx, dim, self.loc())); - } - - pub fn numPrograms(self: *Builder, dim: ttir.ProgramIDDim) Value { - return self.emit(ttir.get_num_programs(self.ctx, dim, self.loc())); - } - - pub fn assume(self: *Builder, cond: Value) void { - _ = llvm.intr_assume(self.ctx, cond.inner, self.loc()).appendTo(self.currentBlock()); - } - - // ==================== constants ==================== - - fn emitInt(self: *Builder, dtype: DType, value: i64) Value { - return self.emit(arith.constant_int(self.ctx, value, dtype.toMlir(self.ctx), self.loc())); - } - - fn emitFloat(self: *Builder, dtype: DType, value: f64) Value { - return switch (dtype) { - .f32 => self.emit(arith.constant_float(self.ctx, value, .f32, self.loc())), - .f64 => self.emit(arith.constant_float(self.ctx, value, .f64, self.loc())), - else => self.fpToFp(self.emitFloat(.f32, value), dtype), - }; - } - - // ==================== ranges and shape manipulation ==================== - - /// `[start, end)` as `tensor<(end-start)xi32>`. Both must fit in i32. - pub fn makeRange(self: *Builder, start: anytype, end: anytype) Value { - const s: i32 = @intCast(start); - const e: i32 = @intCast(end); - std.debug.assert(e >= s); - const len: i64 = @intCast(e - s); - const ty = mlir.Type.rankedTensor(&.{len}, .int(self.ctx, .i32)); - return self.emit(ttir.make_range(self.ctx, s, e, ty, self.loc())); - } - - pub fn arange(self: *Builder, start: anytype, end: anytype, dtype: DType) Value { - const r = self.makeRange(start, end); - return switch (dtype) { - .i32 => r, - .i64 => self.extsi(r, dtype), - .i16, .i8 => self.trunci(r, dtype), - .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => self.sitofp(r, dtype), - else => @panic("Builder.arange: unsupported dtype"), - }; - } - - /// Broadcast a scalar Value or literal across `shape`. Literals lift first via `lift`. - pub fn splat(self: *Builder, value: anytype, shape: []const i64) Value { - const v: Value = if (@TypeOf(value) == Value) value else self.lift(value); - const ty: *const mlir.Type = .rankedTensor(shape, v.type_()); - return self.emit(ttir.splat(self.ctx, v.inner, ty, self.loc())); - } - - /// Lift a Zig scalar to a DSL constant. Dtype follows the source type; - /// unsigned ints are bit-cast (signless MLIR). Use `constant` for an explicit dtype. - pub fn lift(self: *Builder, value: anytype) Value { - const T = @TypeOf(value); - if (T == Value) return value; - return switch (@typeInfo(T)) { - .comptime_int => blk: { - if (value >= std.math.minInt(i32) and value <= std.math.maxInt(i32)) - break :blk self.emitInt(.i32, value); - if (value >= std.math.minInt(i64) and value <= std.math.maxInt(i64)) - break :blk self.emitInt(.i64, value); - if (value >= 0 and value <= std.math.maxInt(u64)) - break :blk self.emitInt(.i64, @bitCast(@as(u64, value))); - @compileError("Builder.lift: integer literal out of 64-bit range"); - }, - .int => |info| blk: { - const signed = info.signedness == .signed; - if (info.bits > 32) { - const v: i64 = if (signed) @intCast(value) else @bitCast(@as(u64, @intCast(value))); - break :blk self.emitInt(.i64, v); - } - const v32: i32 = if (signed) @intCast(value) else @bitCast(@as(u32, @intCast(value))); - break :blk self.emitInt(.i32, v32); - }, - .comptime_float => self.emitFloat(.f32, @floatCast(value)), - .float => |info| switch (info.bits) { - 16 => self.emitFloat(.f16, @floatCast(value)), - 32 => self.emitFloat(.f32, @floatCast(value)), - 64 => self.emitFloat(.f64, value), - else => @compileError("Builder.lift: unsupported float bitwidth"), - }, - else => @compileError("Builder.lift: unsupported type " ++ @typeName(T)), - }; - } - - /// `lift` but comptime scalars match `ref_elem`. Runtime Zig scalars keep their source width. - fn liftMatching(self: *Builder, value: anytype, ref_elem: *const mlir.Type) Value { - const T = @TypeOf(value); - if (T == Value) return value; - return switch (@typeInfo(T)) { - .comptime_int, .comptime_float => self.constMatching(value, ref_elem), - else => self.lift(value), - }; - } - - /// Lift `value` and splat to `ref.shape` when `ref` is a tensor. - pub fn broadcastLike(self: *Builder, value: anytype, ref: Value) Value { - const v = self.liftMatching(value, ref.elemType()); - if (ref.isTensor() and !v.isTensor()) return self.splat(v, ref.shape().constSlice()); - return v; - } - - /// Binary-op coercion: lift, int-width promote, scalar↔tensor splat, rank-align, size-1 broadcast. - pub fn broadcast(self: *Builder, a: anytype, b: anytype) struct { Value, Value } { - const a_ref: ?*const mlir.Type = if (@TypeOf(b) == Value) b.elemType() else null; - const b_ref: ?*const mlir.Type = if (@TypeOf(a) == Value) a.elemType() else null; - var av = if (a_ref) |t| self.liftMatching(a, t) else self.lift(a); - var bv = if (b_ref) |t| self.liftMatching(b, t) else self.lift(b); - - if (av.isIntElem() and bv.isIntElem()) { - const a_dt = self.mlirElemToDType(av.elemType()); - const b_dt = self.mlirElemToDType(bv.elemType()); - const aw = intBitwidth(a_dt); - const bw = intBitwidth(b_dt); - if (aw < bw) av = av.to(b_dt); - if (bw < aw) bv = bv.to(a_dt); - } - - if (av.isTensor() and !bv.isTensor()) bv = self.splat(bv, av.shape().constSlice()); - if (!av.isTensor() and bv.isTensor()) av = self.splat(av, bv.shape().constSlice()); - - if (av.isTensor() and bv.isTensor()) { - const a_rank = av.rank(); - const b_rank = bv.rank(); - if (a_rank < b_rank) { - var n = b_rank - a_rank; - while (n > 0) : (n -= 1) av = self.expandDims(av, 0); - } else if (b_rank < a_rank) { - var n = a_rank - b_rank; - while (n > 0) : (n -= 1) bv = self.expandDims(bv, 0); - } - const a_sh = av.shape(); - const b_sh = bv.shape(); - var ret_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - var grow_a = false; - var grow_b = false; - for (0..a_sh.len) |i| { - const l = a_sh.get(i); - const r = b_sh.get(i); - if (l == r) { - ret_shape.appendAssumeCapacity(l); - } else if (l == 1) { - ret_shape.appendAssumeCapacity(r); - grow_a = true; - } else if (r == 1) { - ret_shape.appendAssumeCapacity(l); - grow_b = true; - } else { - std.debug.panic( - "Builder.broadcast: incompatible shapes {any} vs {any} at axis {d}", - .{ a_sh.constSlice(), b_sh.constSlice(), i }, - ); - } - } - if (grow_a) av = self.broadcastTo(av, ret_shape.constSlice()); - if (grow_b) bv = self.broadcastTo(bv, ret_shape.constSlice()); - } - - return .{ av, bv }; - } - - fn mlirElemToDType(self: *const Builder, elem: *const mlir.Type) DType { - inline for (comptime std.meta.fieldNames(DType)) |field_name| { - const dt = @field(DType, field_name); - if (elem.eql(dt.toMlir(self.ctx))) return dt; - } - @panic("element type not a recognized DSL DType"); - } - - /// `value` as a constant of `dtype`; a Value passes through. An integer is - /// signed or an unsigned bit pattern of `dtype`'s width; anything else - /// panics. - pub fn constant(self: *Builder, dtype: DType, value: anytype) Value { - const T = @TypeOf(value); - if (T == Value) return value; - if (isFloatDtype(dtype)) { - const v_f64: f64 = switch (@typeInfo(T)) { - .comptime_int, .int => @floatFromInt(value), - .comptime_float, .float => @floatCast(value), - else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), - }; - return self.emitFloat(dtype, v_f64); - } - const v: i128 = switch (@typeInfo(T)) { - .comptime_int, .int => value, - .comptime_float, .float => @intFromFloat(value), - else => @compileError("Builder.constant: unsupported scalar " ++ @typeName(T)), - }; - return self.emitInt(dtype, intBits(v, @intCast(intBitwidth(dtype)), @tagName(dtype))); - } - - /// `value` as the `i64` an integer constant of `bits` bits is built from. - /// It may be signed or an unsigned bit pattern, so it must lie in - /// [-2^(bits-1), 2^bits). - fn intBits(value: i128, bits: u7, dtype: []const u8) i64 { - if (value < -(@as(i128, 1) << (bits - 1)) or value >= @as(i128, 1) << bits) - std.debug.panic("Builder.constant: {d} does not fit {s}", .{ value, dtype }); - return @truncate(value); - } - - /// `value` as a constant of `like`'s element type; a Value passes through. - pub fn constantLike(self: *Builder, value: anytype, like: Value) Value { - if (@TypeOf(value) == Value) return value; - return self.constMatching(value, like.elemType()); - } - - fn constMatching(self: *Builder, value: anytype, elem: *const mlir.Type) Value { - return self.constant(self.mlirElemToDType(elem), value); - } - - pub fn zeros(self: *Builder, shape: []const i64, dtype: DType) Value { - return self.full(shape, 0, dtype); - } - - pub fn ones(self: *Builder, shape: []const i64, dtype: DType) Value { - return self.full(shape, 1, dtype); - } - - pub fn full(self: *Builder, shape: []const i64, value: anytype, dtype: DType) Value { - return self.splat(self.constant(dtype, value), shape); - } - - /// 2-D mask `(m, 1) & (1, n)` — auto-broadcasts to `(m, n)`. - pub fn mask2d(self: *Builder, cond_m: Value, cond_n: Value, _: i64, _: i64) Value { - const cm2 = self.expandDims(cond_m, 1); - const cn2 = self.expandDims(cond_n, 0); - return cm2.bitAnd(cn2); - } - - pub fn sum(self: *Builder, src: Value) Value { - return self.sumOpts(src, .{}); - } - - pub fn sumOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { - return self.reduceDispatch(src, opts, .sum); - } - - pub fn max(self: *Builder, src: Value) Value { - return self.maxOpts(src, .{}); - } - - pub fn maxOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { - return self.reduceDispatch(src, opts, .max); - } - - pub fn min(self: *Builder, src: Value) Value { - return self.minOpts(src, .{}); - } - - pub fn minOpts(self: *Builder, src: Value, opts: ReduceOpts) Value { - return self.reduceDispatch(src, opts, .min); - } - - fn reduceDispatch(self: *Builder, src: Value, opts: ReduceOpts, comptime kind: enum { sum, max, min }) Value { - const src_shape = src.shape(); - const input: Value, const axis: i32 = if (opts.axis) |a| .{ src, a } else blk: { - var total: i64 = 1; - for (0..src_shape.len) |i| total *= src_shape.get(i); - break :blk .{ self.reshapeOpts(src, &.{total}, .{ .can_reorder = true }), 0 }; - }; - - const elem = input.elemType(); - const in_shape = input.shape(); - const result_ty: *const mlir.Type = computeReducedType(in_shape.constSlice(), axis, elem); - const is_float = input.isFloatElem(); - const Ctx = struct { is_float: bool, kind: @TypeOf(kind) }; - const combine = struct { - fn c(kk: *Builder, lhs: Value, rhs: Value, cc: Ctx) Value { - return switch (cc.kind) { - .sum => if (cc.is_float) kk.addf(lhs, rhs) else kk.addi(lhs, rhs), - .max => if (cc.is_float) kk.maxnumf(lhs, rhs) else kk.maxsi(lhs, rhs), - .min => if (cc.is_float) kk.minnumf(lhs, rhs) else kk.minsi(lhs, rhs), - }; - } - }.c; - const out = self.reduce(Ctx{ .is_float = is_float, .kind = kind }, .{ - .src = input, - .axis = axis, - .elem = elem, - .result = result_ty, - .combine = combine, - }); - - if (!opts.keep_dims) return out; - if (opts.axis == null) { - var ones_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..src_shape.len) |_| ones_shape.appendAssumeCapacity(1); - return self.splat(out, ones_shape.constSlice()); - } - if (in_shape.len <= 1) return self.splat(out, &.{1}); - return self.expandDims(out, axis); - } - - pub fn cumsum(self: *Builder, src: Value) Value { - return self.cumsumOpts(src, .{}); - } - - pub fn cumsumOpts(self: *Builder, src: Value, opts: ScanOpts) Value { - return self.scanDispatch(src, opts, .sum); - } - - pub fn cumprod(self: *Builder, src: Value) Value { - return self.cumprodOpts(src, .{}); - } - - pub fn cumprodOpts(self: *Builder, src: Value, opts: ScanOpts) Value { - return self.scanDispatch(src, opts, .prod); - } - - fn scanDispatch(self: *Builder, src: Value, opts: ScanOpts, comptime kind: enum { sum, prod }) Value { - const elem = src.elemType(); - const is_float = src.isFloatElem(); - const Ctx = struct { is_float: bool, kind: @TypeOf(kind) }; - const combine = struct { - fn c(kk: *Builder, lhs: Value, rhs: Value, cc: Ctx) Value { - return switch (cc.kind) { - .sum => if (cc.is_float) kk.addf(lhs, rhs) else kk.addi(lhs, rhs), - .prod => if (cc.is_float) kk.mulf(lhs, rhs) else kk.muli(lhs, rhs), - }; - } - }.c; - return self.scan(Ctx{ .is_float = is_float, .kind = kind }, .{ - .src = src, - .axis = opts.axis, - .reverse = opts.reverse, - .elem = elem, - .result = src.type_(), - .combine = combine, - }); - } - - pub fn expandDims(self: *Builder, value: Value, axis: i32) Value { - const in_shape = value.shape(); - var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - const ax: usize = @intCast(axis); - for (0..in_shape.len + 1) |i| { - if (i == ax) out_shape.appendAssumeCapacity(1) else out_shape.appendAssumeCapacity(in_shape.get(if (i < ax) i else i - 1)); - } - const ty = mlir.Type.rankedTensor(out_shape.constSlice(), value.elemType()); - return self.emit(ttir.expand_dims(self.ctx, value.inner, axis, ty, self.loc())); - } - - pub fn broadcastTo(self: *Builder, value: Value, result_shape: []const i64) Value { - const elem = mlir.RankedTensorType.fromShaped(value.inner.type_().isA(mlir.ShapedType).?).elementType(); - const ty = mlir.Type.rankedTensor(result_shape, elem); - return self.emit(ttir.broadcast(self.ctx, value.inner, ty, self.loc())); - } - - // ==================== pointer arithmetic ==================== - - pub fn addptr(self: *Builder, ptr: Value, offset: Value) Value { - return self.emit(ttir.addptr(self.ctx, ptr.inner, offset.inner, self.loc())); - } - - pub fn load(self: *Builder, ptr: Value) Value { - return self.loadOpts(ptr, .{}); - } - - pub fn loadOpts(self: *Builder, ptr: Value, opts: LoadOpts) Value { - var mask_v = opts.mask; - var other_v = opts.other; - if (ptr.isTensor()) { - if (mask_v) |m| { - _, mask_v = self.broadcast(ptr, m); - } - if (other_v) |o| { - _, other_v = self.broadcast(ptr, o); - } - } - const mask_inner: ?*const mlir.Value = if (mask_v) |m| m.inner else null; - const other_inner: ?*const mlir.Value = if (other_v) |o| o.inner else null; - return self.emit(ttir.load( - self.ctx, - ptr.inner, - self.loadResultType(ptr), - opts.cache_modifier, - opts.eviction_policy, - opts.@"volatile", - .{ .mask = mask_inner, .other = other_inner }, - self.loc(), - )); - } - - pub fn store(self: *Builder, ptr: Value, value: Value) void { - self.storeOpts(ptr, value, .{}); - } - - pub fn storeOpts(self: *Builder, ptr: Value, value: Value, opts: StoreOpts) void { - var mask_v = opts.mask; - if (ptr.isTensor()) { - if (mask_v) |m| { - _, mask_v = self.broadcast(ptr, m); - } - } - const mask_inner: ?*const mlir.Value = if (mask_v) |m| m.inner else null; - _ = ttir.store(self.ctx, ptr.inner, value.inner, opts.cache_modifier, opts.eviction_policy, .{ .mask = mask_inner }, self.loc()).appendTo(self.currentBlock()); - } - - // ==================== arith: binary ==================== - - pub fn addi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.addi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn subi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.subi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn muli(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.muli(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn addf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.addf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn subf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.subf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn mulf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.mulf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn divf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.divf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - - pub fn cmpi(self: *Builder, predicate: arith.CmpIPredicate, lhs: Value, rhs: Value) Value { - return self.emit(arith.cmpi(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); - } - pub fn cmpf(self: *Builder, predicate: arith.CmpFPredicate, lhs: Value, rhs: Value) Value { - return self.emit(arith.cmpf(self.ctx, predicate, lhs.inner, rhs.inner, self.loc())); - } - - pub fn select(self: *Builder, cond: Value, t: Value, f: Value) Value { - return self.emit(arith.select(self.ctx, cond.inner, t.inner, f.inner, self.loc())); - } - - /// Select `x` where condition is true, `y` elsewhere. Auto-broadcasts. - pub fn where(self: *Builder, condition: Value, x: Value, y: Value) Value { - var c = condition; - var xv, var yv = .{ x, y }; - xv, yv = self.broadcast(xv, yv); - if (c.isTensor()) { - c, xv = self.broadcast(c, xv); - xv, yv = self.broadcast(xv, yv); - } else { - c, _ = self.broadcast(c, xv); - } - // Arithmetic broadcasting can widen i1 when the selected values are - // integers. The select predicate must remain boolean. - return self.select(c.to(.i1), xv, yv); - } - - pub fn divsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.divsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn divui(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.divui(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn remsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.remsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn remui(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.remui(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn ceildivsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.ceildivsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn floordivsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.floordivsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - - pub fn andi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.andi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn ori(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.ori(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn xori(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.xori(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn shli(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.shli(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn shrsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.shrsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn shrui(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.shrui(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - - pub fn maxsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.maxsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn minsi(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.minsi(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - - pub fn remf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.remf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn maximumf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.maximumf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn minimumf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.minimumf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn maxnumf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.maxnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn minnumf(self: *Builder, lhs: Value, rhs: Value) Value { - return self.emit(arith.minnumf(self.ctx, lhs.inner, rhs.inner, self.loc())); - } - pub fn negf(self: *Builder, src: Value) Value { - return self.emit(arith.negf(self.ctx, src.inner, self.loc())); - } - - pub fn maximum(self: *Builder, a: anytype, b: anytype) Value { - return self.maximumOpts(a, b, .{}); - } - - pub fn maximumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { - const l, const r = self.broadcast(a, b); - if (l.isFloatElem()) { - return switch (opts.propagate_nan) { - .none => self.maxnumf(l, r), - .all => self.maximumf(l, r), - }; - } - return self.maxsi(l, r); - } - - pub fn minimum(self: *Builder, a: anytype, b: anytype) Value { - return self.minimumOpts(a, b, .{}); - } - - pub fn minimumOpts(self: *Builder, a: anytype, b: anytype, opts: MinMaxOpts) Value { - const l, const r = self.broadcast(a, b); - if (l.isFloatElem()) { - return switch (opts.propagate_nan) { - .none => self.minnumf(l, r), - .all => self.minimumf(l, r), - }; - } - return self.minsi(l, r); - } - - // Casts — each preserves the src shape and swaps the element type to `out_dtype`. - pub fn extsi(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(arith.extsi(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - pub fn extui(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(arith.extui(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - pub fn extf(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(arith.extf(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - pub fn trunci(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(arith.trunci(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - pub fn truncf(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(arith.truncf(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - pub fn sitofp(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(arith.sitofp(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - pub fn uitofp(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(arith.uitofp(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - pub fn fptosi(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(arith.fptosi(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - pub fn fptoui(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(arith.fptoui(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - pub fn arithBitcast(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(arith.bitcast(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - - // ==================== Triton-specific ops ==================== - - /// tt.bitcast — same-bitwidth cast. Shape preserved from src. - pub fn bitCast(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(ttir.bitcast(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - - /// Numeric cast with auto-dispatch. Use `castOpts` for rounding / bitcast. - pub fn cast(self: *Builder, src: Value, dtype: DType) Value { - return self.castOpts(src, dtype, .{}); - } - - /// Numeric cast with options. Dispatch: - /// - fp↔fp with fp8: tt.fp_to_fp, with rounding only on downcasts. - /// - other fp↔fp narrow: truncf, except custom rounding → tt.fp_to_fp. - /// - other fp↔fp wide: extf. - /// - i1→fp: uitofp. int↔int: extsi/trunci/bitcast by width. - pub fn castOpts(self: *Builder, src: Value, dtype: DType, opts: CastOpts) Value { - if (opts.bitcast) return self.bitCast(src, dtype); - const cur_elem = src.elemType(); - const tgt_elem = dtype.toMlir(self.ctx); - if (cur_elem.eql(tgt_elem)) return src; - const cur_dtype = self.mlirElemToDType(cur_elem); - const cur_is_float = src.isFloatElem(); - const tgt_is_float = isFloatDtype(dtype); - if (cur_is_float and tgt_is_float) { - const cur_bw = dtypeBitwidth(cur_dtype); - const tgt_bw = dtypeBitwidth(dtype); - const fp8_involved = (cur_dtype == .f8e4m3fn or cur_dtype == .f8e4m3fnuz or cur_dtype == .f8e5m2 or - dtype == .f8e4m3fn or dtype == .f8e4m3fnuz or dtype == .f8e5m2); - if (fp8_involved) { - return self.fpToFpOpts(src, dtype, .{ - .rounding = if (tgt_bw < cur_bw) opts.fp_downcast_rounding orelse .rtne else null, - }); - } - if (tgt_bw > cur_bw) return self.extf(src, dtype); - const custom_rounding = opts.fp_downcast_rounding != null and opts.fp_downcast_rounding.? != .rtne; - if (custom_rounding) { - return self.fpToFpOpts(src, dtype, .{ .rounding = opts.fp_downcast_rounding orelse .rtne }); - } - return self.truncf(src, dtype); - } - if (cur_is_float and !tgt_is_float) return self.fptosi(src, dtype); - if (!cur_is_float and tgt_is_float) { - if (cur_dtype == .i1) return self.uitofp(src, dtype); - return self.sitofp(src, dtype); - } - const cur_bw = dtypeBitwidth(cur_dtype); - const tgt_bw = dtypeBitwidth(dtype); - if (tgt_bw > cur_bw) { - if (cur_dtype == .i1) return self.extui(src, dtype); - return self.extsi(src, dtype); - } - if (tgt_bw < cur_bw) return self.trunci(src, dtype); - return self.arithBitcast(src, dtype); - } - /// tt.int_to_ptr — cast int (scalar or tensor) to pointer, preserving shape. - pub fn intToPtr(self: *Builder, src: Value, pointee: DType, address_space: i32) Value { - const ptr_elem = ttir.pointerType(pointee.toMlir(self.ctx), address_space); - const result_type: *const mlir.Type = if (src.isTensor()) - .rankedTensor(src.shape().constSlice(), ptr_elem) - else - ptr_elem; - return self.emit(ttir.int_to_ptr(self.ctx, src.inner, result_type, self.loc())); - } - /// tt.ptr_to_int — cast pointer (scalar or tensor) to integer, preserving shape. - pub fn ptrToInt(self: *Builder, src: Value, out_dtype: DType) Value { - return self.emit(ttir.ptr_to_int(self.ctx, src.inner, self.swapElem(src, out_dtype), self.loc())); - } - /// tt.fp_to_fp with default rtne rounding. Use `fpToFpOpts` for other rounding modes. - pub fn fpToFp(self: *Builder, src: Value, out_dtype: DType) Value { - return self.fpToFpOpts(src, out_dtype, .{ .rounding = .rtne }); - } - /// tt.fp_to_fp with rounding option. - pub fn fpToFpOpts(self: *Builder, src: Value, out_dtype: DType, opts: FpToFpOpts) Value { - return self.emit(ttir.fp_to_fp(self.ctx, src.inner, self.swapElem(src, out_dtype), .{ .rounding = opts.rounding }, self.loc())); - } - /// tt.clampf with propagate_nan=.none. Use `clampfOpts` to override. - pub fn clampf(self: *Builder, x: Value, lo: Value, hi: Value) Value { - return self.clampfOpts(x, lo, hi, .{}); - } - /// tt.clampf with full options. - pub fn clampfOpts(self: *Builder, x: Value, lo: Value, hi: Value, opts: ClampOpts) Value { - return self.emit(ttir.clampf(self.ctx, x.inner, lo.inner, hi.inner, opts.propagate_nan, self.loc())); - } - /// `tl.sqrt_rn(x)` — precise square root (IEEE round-to-nearest). - pub fn sqrtRn(self: *Builder, x: Value) Value { - return self.emit(ttir.precise_sqrt(self.ctx, x.inner, self.loc())); - } - /// `tl.div_rn(x, y)` — precise division (IEEE round-to-nearest). - pub fn divRn(self: *Builder, x: Value, y: Value) Value { - return self.emit(ttir.precise_divf(self.ctx, x.inner, y.inner, self.loc())); - } - /// `tl.umulhi(x, y)` — most-significant N bits of the 2N-bit unsigned product. - pub fn umulhi(self: *Builder, x: Value, y: Value) Value { - return self.emit(ttir.mulhiui(self.ctx, x.inner, y.inner, self.loc())); - } - - /// tt.dot — matmul with accumulator. Result type = c_acc. Use `dotOpts` for precision. - pub fn dot(self: *Builder, a: Value, b: Value, c_acc: Value) Value { - return self.dotOpts(a, b, c_acc, .{}); - } - - /// tt.dot with full options. Result type = `c_acc.type_()`. - pub fn dotOpts(self: *Builder, a: Value, b: Value, c_acc: Value, opts: DotOpts) Value { - return self.emit(ttir.dot(self.ctx, a.inner, b.inner, c_acc.inner, c_acc.type_(), opts.input_precision, opts.max_num_imprecise_acc, self.loc())); - } - - /// tt.reshape, element type preserved. Use `reshapeOpts` for can_reorder / efficient_layout. - pub fn reshape(self: *Builder, src: Value, new_shape: []const i64) Value { - return self.reshapeOpts(src, new_shape, .{}); - } - - /// tt.reshape with full options. - pub fn reshapeOpts(self: *Builder, src: Value, new_shape: []const i64, opts: ReshapeOpts) Value { - const result_ty = mlir.Type.rankedTensor(new_shape, src.elemType()); - return self.emit(ttir.reshape(self.ctx, src.inner, result_ty, opts.can_reorder, opts.efficient_layout, self.loc())); - } - - /// tt.trans — permute dimensions. Result shape is src.shape permuted by order. - pub fn trans(self: *Builder, src: Value, order: []const i32) Value { - const src_shape = src.shape(); - std.debug.assert(order.len == src_shape.len); - var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (order) |i| out_shape.appendAssumeCapacity(src_shape.get(@intCast(i))); - const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), src.elemType()); - return self.emit(ttir.trans(self.ctx, src.inner, order, result_type, self.loc())); - } - - /// `tl.permute(src, *dims)` — alias for `trans`. - pub fn permute(self: *Builder, src: Value, order: []const i32) Value { - return self.trans(src, order); - } - - /// `tl.cat` along dim 0. Use `catOpts` for can_reorder / dim. - pub fn cat(self: *Builder, lhs: Value, rhs: Value) Value { - return self.catOpts(lhs, rhs, .{}); - } - - /// `tl.cat` with full options: `join`, `permute` and `reshape`, as Triton lowers it. - pub fn catOpts(self: *Builder, lhs: Value, rhs: Value, opts: CatOpts) Value { - _ = opts.can_reorder; - const ls = lhs.shape(); - const rs = rhs.shape(); - std.debug.assert(ls.len == rs.len); - for (0..ls.len) |i| std.debug.assert(ls.get(i) == rs.get(i)); - const rank: i32 = @intCast(ls.len); - const dim: usize = @intCast(if (opts.dim < 0) opts.dim + rank else opts.dim); - var order: stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK) = .empty; - for (0..ls.len) |i| order.appendAssumeCapacity(@intCast(i)); - order.insert(dim, rank) catch unreachable; - var new_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..ls.len) |i| new_shape.appendAssumeCapacity(ls.get(i)); - new_shape.slice()[dim] = ls.get(dim) + rs.get(dim); - return self.reshape(self.permute(self.join(lhs, rhs), order.constSlice()), new_shape.constSlice()); - } - - /// `tt.join` — stack two equal-shape tensors along a new minor dim of size 2. - pub fn join(self: *Builder, lhs: Value, rhs: Value) Value { - const ls = lhs.shape(); - var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..ls.len) |i| out_shape.appendAssumeCapacity(ls.get(i)); - out_shape.appendAssumeCapacity(2); - const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), lhs.elemType()); - return self.emit(ttir.join(self.ctx, lhs.inner, rhs.inner, result_type, self.loc())); - } - - /// `tt.split` — split a tensor whose last dim is 2 into two tensors with - /// that last dim dropped. Returns (lhs, rhs). - pub fn split(self: *Builder, src: Value) [2]Value { - const ss = src.shape(); - std.debug.assert(ss.len >= 1 and ss.get(ss.len - 1) == 2); - var out_shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..ss.len - 1) |i| out_shape.appendAssumeCapacity(ss.get(i)); - const result_type = mlir.Type.rankedTensor(out_shape.constSlice(), src.elemType()); - const op = ttir.split(self.ctx, src.inner, result_type, self.loc()); - _ = op.appendTo(self.currentBlock()); - return .{ - .{ .inner = op.result(0), .kernel = self }, - .{ .inner = op.result(1), .kernel = self }, - }; - } - - /// `tl.item(src)` / `tt.unsplat` — 1-element tensor → scalar of the same element type. - pub fn item(self: *Builder, src: Value) Value { - return self.emit(ttir.unsplat(self.ctx, src.inner, src.elemType(), self.loc())); - } - - /// tt.gather — output shape matches indices, element type matches src. - pub fn gather(self: *Builder, src: Value, indices: Value, axis: i32) Value { - const result_ty = mlir.Type.rankedTensor(indices.shape().constSlice(), src.elemType()); - return self.emit(ttir.gather(self.ctx, src.inner, indices.inner, axis, result_ty, false, self.loc())); - } - - /// tt.histogram — output is `tensor`. Use `histogramOpts` for mask. - pub fn histogram(self: *Builder, src: Value, num_bins: i64) Value { - return self.histogramOpts(src, num_bins, .{}); - } - - /// tt.histogram with mask option. - pub fn histogramOpts(self: *Builder, src: Value, num_bins: i64, opts: HistogramOpts) Value { - const result_ty = mlir.Type.rankedTensor(&.{num_bins}, DType.i32.toMlir(self.ctx)); - const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; - return self.emit(ttir.histogram(self.ctx, src.inner, result_ty, .{ .mask = mask_inner }, self.loc())); - } - - /// tt.assert — device-side assert (i1 or tensor<...xi1>). Use `deviceAssertOpts` for mask. - pub fn deviceAssert(self: *Builder, condition: Value, message: []const u8) void { - self.deviceAssertOpts(condition, message, .{}); - } - - /// tt.assert with mask option. - pub fn deviceAssertOpts(self: *Builder, condition: Value, message: []const u8, opts: DeviceAssertOpts) void { - // TODO: threading mask through ttir.assert_ when supported. - _ = opts.mask; - _ = ttir.assert_(self.ctx, condition.inner, message, self.loc()).appendTo(self.currentBlock()); - } - - /// tt.atomic_rmw — returns old value. Use `atomicRmwOpts` for mask / sem / scope. - pub fn atomicRmw(self: *Builder, rmw: ttir.RMWOp, ptr: Value, val: Value) Value { - return self.atomicRmwOpts(rmw, ptr, val, .{}); - } - - /// tt.atomic_rmw with full options. - pub fn atomicRmwOpts(self: *Builder, rmw: ttir.RMWOp, ptr: Value, val: Value, opts: AtomicRMWOpts) Value { - const mask_inner: ?*const mlir.Value = if (opts.mask) |m| m.inner else null; - return self.emit(ttir.atomic_rmw(self.ctx, rmw, ptr.inner, val.inner, opts.sem, opts.scope, .{ .mask = mask_inner }, self.loc())); - } - - /// tt.atomic_cas — returns old value. Use `atomicCasOpts` for sem / scope. - pub fn atomicCas(self: *Builder, ptr: Value, cmp: Value, val: Value) Value { - return self.atomicCasOpts(ptr, cmp, val, .{}); - } - - /// tt.atomic_cas with full options. - pub fn atomicCasOpts(self: *Builder, ptr: Value, cmp: Value, val: Value, opts: AtomicCasOpts) Value { - return self.emit(ttir.atomic_cas(self.ctx, ptr.inner, cmp.inner, val.inner, opts.sem, opts.scope, self.loc())); - } - - /// `tt.call` — call another tt.func in this module by symbol name. - pub fn call(self: *Builder, callee: []const u8, operands: []const Value, result_types: []const *const mlir.Type) []Value { - const op = ttir.call(self.ctx, callee, self.innerSlice(operands), result_types, .{}, self.loc()); - return self.emitMulti(op, result_types.len); - } - - /// tt.dot_scaled — dot with microscaling factors. Use `dotScaledOpts` for options. - pub fn dotScaled( - self: *Builder, - a: Value, - b: Value, - c_acc: Value, - a_scale: ?Value, - b_scale: ?Value, - a_elem_type: ttir.ScaleDotElemType, - b_elem_type: ttir.ScaleDotElemType, - ) Value { - return self.dotScaledOpts(a, b, c_acc, a_scale, b_scale, a_elem_type, b_elem_type, .{}); - } - - /// `tt.dot_scaled` with the full named-param set. Result type = `c_acc.type_()`. - pub fn dotScaledOpts( - self: *Builder, - a: Value, - b: Value, - c_acc: Value, - a_scale: ?Value, - b_scale: ?Value, - a_elem_type: ttir.ScaleDotElemType, - b_elem_type: ttir.ScaleDotElemType, - opts: DotScaledOpts, - ) Value { - const a_s: ?*const mlir.Value = if (a_scale) |s| s.inner else null; - const b_s: ?*const mlir.Value = if (b_scale) |s| s.inner else null; - return self.emit(ttir.dot_scaled( - self.ctx, - a.inner, - b.inner, - c_acc.inner, - a_elem_type, - b_elem_type, - c_acc.type_(), - opts.fast_math, - opts.lhs_k_pack, - opts.rhs_k_pack, - .{ .a_scale = a_s, .b_scale = b_s }, - self.loc(), - )); - } - - /// tt.extern_elementwise — call a library symbol pointwise. - /// Result is `tensor`; pass `&.{}` for scalar. - pub fn externElementwise( - self: *Builder, - srcs: []const Value, - result_shape: []const i64, - result_dtype: DType, - libname: []const u8, - libpath: []const u8, - symbol: []const u8, - ) Value { - return self.externElementwiseOpts(srcs, result_shape, result_dtype, libname, libpath, symbol, .{}); - } - - /// `tt.extern_elementwise` with the full named-param set. - pub fn externElementwiseOpts( - self: *Builder, - srcs: []const Value, - result_shape: []const i64, - result_dtype: DType, - libname: []const u8, - libpath: []const u8, - symbol: []const u8, - opts: ExternElementwiseOpts, - ) Value { - const result_type: *const mlir.Type = if (result_shape.len == 0) - result_dtype.toMlir(self.ctx) - else - .rankedTensor(result_shape, result_dtype.toMlir(self.ctx)); - return self.emit(ttir.extern_elementwise( - self.ctx, - self.innerSlice(srcs), - result_type, - libname, - libpath, - symbol, - opts.is_pure, - self.loc(), - )); - } - - /// tt.print — device-side print. `is_signed` has one entry per arg (1=signed, 0=unsigned). - /// Use `devicePrintOpts` for hex. - pub fn devicePrint( - self: *Builder, - prefix: []const u8, - args: []const Value, - is_signed: []const i32, - ) void { - self.devicePrintOpts(prefix, args, is_signed, .{}); - } - - /// tt.print with full options. - pub fn devicePrintOpts( - self: *Builder, - prefix: []const u8, - args: []const Value, - is_signed: []const i32, - opts: PrintOpts, - ) void { - _ = ttir.print(self.ctx, prefix, opts.hex, self.innerSlice(args), is_signed, self.loc()).appendTo(self.currentBlock()); - } - - /// tt.make_tensor_descriptor — build a TMA descriptor. - /// shape/strides are runtime; block_shape/dtype describe the compile-time tile. - pub fn makeTensorDescriptor( - self: *Builder, - base: Value, - shape: []const Value, - strides: []const Value, - padding: ttir.PaddingOption, - block_shape: []const i64, - dtype: DType, - ) Value { - const result_type = ttir.tensorDescType(block_shape, dtype.toMlir(self.ctx), .{}); - return self.emit(ttir.make_tensor_descriptor( - self.ctx, - base.inner, - self.innerSlice(shape), - self.innerSlice(strides), - padding, - result_type, - self.loc(), - )); - } - - /// tt.descriptor_load — TMA load. Use `descriptorLoadOpts` for cache / eviction. - pub fn descriptorLoad( - self: *Builder, - desc: Value, - indices: []const Value, - shape: []const i64, - dtype: DType, - ) Value { - return self.descriptorLoadOpts(desc, indices, shape, dtype, .{}); - } - - /// `tt.descriptor_load` with the full named-param set. - pub fn descriptorLoadOpts( - self: *Builder, - desc: Value, - indices: []const Value, - shape: []const i64, - dtype: DType, - opts: DescriptorLoadOpts, - ) Value { - const result_ty = mlir.Type.rankedTensor(shape, dtype.toMlir(self.ctx)); - return self.emit(ttir.descriptor_load(self.ctx, desc.inner, self.innerSlice(indices), result_ty, opts.cache_modifier, opts.eviction_policy, self.loc())); - } - - /// `tt.descriptor_store` — TMA store. - pub fn descriptorStore(self: *Builder, desc: Value, src: Value, indices: []const Value) void { - _ = ttir.descriptor_store(self.ctx, desc.inner, src.inner, self.innerSlice(indices), self.loc()).appendTo(self.currentBlock()); - } - - /// `tt.descriptor_reduce` — TMA reducing store. - pub fn descriptorReduce( - self: *Builder, - kind: ttir.DescriptorReduceKind, - desc: Value, - src: Value, - indices: []const Value, - ) void { - _ = ttir.descriptor_reduce(self.ctx, kind, desc.inner, src.inner, self.innerSlice(indices), self.loc()).appendTo(self.currentBlock()); - } - - /// tt.descriptor_gather — TMA gather by x_offsets + y_offset. - pub fn descriptorGather( - self: *Builder, - desc: Value, - x_offsets: Value, - y_offset: Value, - shape: []const i64, - dtype: DType, - ) Value { - const result_ty = mlir.Type.rankedTensor(shape, dtype.toMlir(self.ctx)); - return self.emit(ttir.descriptor_gather(self.ctx, desc.inner, x_offsets.inner, y_offset.inner, result_ty, self.loc())); - } - - /// `tt.descriptor_scatter` — TMA scatter. - pub fn descriptorScatter(self: *Builder, desc: Value, x_offsets: Value, y_offset: Value, src: Value) void { - _ = ttir.descriptor_scatter(self.ctx, desc.inner, x_offsets.inner, y_offset.inner, src.inner, self.loc()).appendTo(self.currentBlock()); - } - - // ==================== math dialect ==================== - - pub fn exp(self: *Builder, x: Value) Value { - return self.emit(math.exp(self.ctx, x.inner, self.loc())); - } - pub fn exp2(self: *Builder, x: Value) Value { - return self.emit(math.exp2(self.ctx, x.inner, self.loc())); - } - pub fn log(self: *Builder, x: Value) Value { - return self.emit(math.log(self.ctx, x.inner, self.loc())); - } - pub fn log2(self: *Builder, x: Value) Value { - return self.emit(math.log2(self.ctx, x.inner, self.loc())); - } - pub fn sqrt(self: *Builder, x: Value) Value { - return self.emit(math.sqrt(self.ctx, x.inner, self.loc())); - } - pub fn rsqrt(self: *Builder, x: Value) Value { - return self.emit(math.rsqrt(self.ctx, x.inner, self.loc())); - } - pub fn sin(self: *Builder, x: Value) Value { - return self.emit(math.sin(self.ctx, x.inner, self.loc())); - } - pub fn cos(self: *Builder, x: Value) Value { - return self.emit(math.cos(self.ctx, x.inner, self.loc())); - } - pub fn tan(self: *Builder, x: Value) Value { - return self.emit(math.tan(self.ctx, x.inner, self.loc())); - } - pub fn tanh(self: *Builder, x: Value) Value { - return self.emit(math.tanh(self.ctx, x.inner, self.loc())); - } - pub fn erf(self: *Builder, x: Value) Value { - return self.emit(math.erf(self.ctx, x.inner, self.loc())); - } - pub fn absf(self: *Builder, x: Value) Value { - return self.emit(math.absf(self.ctx, x.inner, self.loc())); - } - pub fn absi(self: *Builder, x: Value) Value { - return self.emit(math.absi(self.ctx, x.inner, self.loc())); - } - /// `tl.abs(x)` — absolute value, auto-dispatch by element type (float→`absf`, - /// signed int→`absi`, unsigned int→no-op). - pub fn abs(self: *Builder, x: Value) Value { - if (x.isFloatElem()) return self.absf(x); - return self.absi(x); - } - pub fn floor(self: *Builder, x: Value) Value { - return self.emit(math.floor(self.ctx, x.inner, self.loc())); - } - pub fn ceil(self: *Builder, x: Value) Value { - return self.emit(math.ceil(self.ctx, x.inner, self.loc())); - } - pub fn powf(self: *Builder, x: Value, y: Value) Value { - return self.emit(math.powf(self.ctx, x.inner, y.inner, self.loc())); - } - pub fn fma(self: *Builder, a: Value, b: Value, c_: Value) Value { - return self.emit(math.fma(self.ctx, a.inner, b.inner, c_.inner, self.loc())); - } - /// `math.clampf` — distinct from `tt.clampf` (no propagateNan attr). - pub fn mathClampf(self: *Builder, value: Value, lo: Value, hi: Value) Value { - return self.emit(math.clampf(self.ctx, value.inner, lo.inner, hi.inner, self.loc())); - } - /// `math.sincos` — returns (sin, cos). - pub fn sincos(self: *Builder, x: Value) [2]Value { - const op = math.sincos(self.ctx, x.inner, self.loc()); - _ = op.appendTo(self.currentBlock()); - return .{ .{ .inner = op.result(0), .kernel = self }, .{ .inner = op.result(1), .kernel = self } }; - } - /// `math.fpowi` — float base, integer exponent. - pub fn fpowi(self: *Builder, base: Value, power: Value) Value { - return self.emit(math.fpowi(self.ctx, base.inner, power.inner, self.loc())); - } - pub fn ipowi(self: *Builder, base: Value, power: Value) Value { - return self.emit(math.ipowi(self.ctx, base.inner, power.inner, self.loc())); - } - pub fn atan2(self: *Builder, y: Value, x: Value) Value { - return self.emit(math.atan2(self.ctx, y.inner, x.inner, self.loc())); - } - pub fn copysign(self: *Builder, mag: Value, sign: Value) Value { - return self.emit(math.copysign(self.ctx, mag.inner, sign.inner, self.loc())); - } - pub fn mathRound(self: *Builder, x: Value) Value { - return self.emit(math.round(self.ctx, x.inner, self.loc())); - } - pub fn mathRoundEven(self: *Builder, x: Value) Value { - return self.emit(math.roundeven(self.ctx, x.inner, self.loc())); - } - pub fn mathTrunc(self: *Builder, x: Value) Value { - return self.emit(math.trunc(self.ctx, x.inner, self.loc())); - } - - /// `arith.convertf` — same-bitwidth float cast (e.g. f16 ↔ bf16) with - /// default rounding. Shape preserved from `src`. Use `convertfOpts` for - /// `fastMath`. - pub fn convertf(self: *Builder, src: Value, out_dtype: DType) Value { - return self.convertfOpts(src, out_dtype, .{}); - } - /// `arith.convertf` with the full named-param set. - pub fn convertfOpts(self: *Builder, src: Value, out_dtype: DType, opts: arith.ConvertFOpts) Value { - return self.emit(arith.convertf(self.ctx, src.inner, self.swapElem(src, out_dtype), opts, self.loc())); - } - /// `arith.scaling_truncf` — MXFP downcast with per-block scales, default - /// options. Shape preserved from `src`. Use `scalingTruncfOpts` for - /// `fastMath`. - pub fn scalingTruncf(self: *Builder, src: Value, scale: Value, out_dtype: DType) Value { - return self.scalingTruncfOpts(src, scale, out_dtype, .{}); - } - /// `arith.scaling_truncf` with the full named-param set. - pub fn scalingTruncfOpts(self: *Builder, src: Value, scale: Value, out_dtype: DType, opts: arith.ConvertFOpts) Value { - return self.emit(arith.scaling_truncf(self.ctx, src.inner, scale.inner, self.swapElem(src, out_dtype), opts, self.loc())); - } - - // ==================== tt.map_elementwise / elementwise_inline_asm ==================== - - /// `tt.map_elementwise` — body builder receives one scalar Value per - /// `srcs` tensor (repeated `pack` times, flattened), and must return the - /// scalar results to feed back via `tt.map_elementwise.return`. - pub fn mapElementwise( - self: *Builder, - srcs: []const Value, - pack: i32, - scalar_arg_types: []const *const mlir.Type, - result_tensor_types: []const *const mlir.Type, - comptime body_fn: anytype, - body_ctx: anytype, - ) []Value { - const scratch = self.arena.allocator(); - const locs = scratch.alloc(*const mlir.Location, scalar_arg_types.len) catch @panic("Builder.mapElementwise OOM"); - for (locs) |*l| l.* = self.loc(); - const body_block = mlir.Block.init(scalar_arg_types, locs); - - self.pushBlock(body_block); - const block_args = scratch.alloc(Value, scalar_arg_types.len) catch @panic("Builder.mapElementwise OOM"); - for (block_args, 0..) |*v, i| v.* = .{ .inner = body_block.argument(i), .kernel = self }; - const out_vals: []const Value = body_fn(self, block_args, body_ctx); - _ = ttir.map_elementwise_return(self.ctx, self.innerSlice(out_vals), self.loc()).appendTo(body_block); - self.popBlock(); - - const op = ttir.map_elementwise( - self.ctx, - self.innerSlice(srcs), - pack, - body_block, - result_tensor_types, - self.loc(), - ); - return self.emitMulti(op, result_tensor_types.len); - } - - /// `tl.inline_asm_elementwise(asm, constraints, args, dtype)` — inline asm - /// pointwise op with default options. Use `inlineAsmElementwiseOpts` for - /// `is_pure` / `pack`. - pub fn inlineAsmElementwise( - self: *Builder, - asm_string: []const u8, - constraints: []const u8, - args: []const Value, - result_types: []const *const mlir.Type, - ) []Value { - return self.inlineAsmElementwiseOpts(asm_string, constraints, args, result_types, .{}); - } - - /// `tl.inline_asm_elementwise(asm, constraints, args, dtype, is_pure=..., pack=...)` - /// with the full named-param set. - pub fn inlineAsmElementwiseOpts( - self: *Builder, - asm_string: []const u8, - constraints: []const u8, - args: []const Value, - result_types: []const *const mlir.Type, - opts: InlineAsmOpts, - ) []Value { - const op = ttir.elementwise_inline_asm(self.ctx, asm_string, constraints, self.innerSlice(args), result_types, opts.is_pure, opts.pack, self.loc()); - return self.emitMulti(op, result_types.len); - } - - // ==================== scf.parallel / scf.reduce / scf.forall ==================== - - pub fn parallel( - self: *Builder, - lbs: []const Value, - ubs: []const Value, - steps: []const Value, - inits: []const Value, - reduction_result_types: []const *const mlir.Type, - comptime body_fn: anytype, - comptime combiners: anytype, - body_ctx: anytype, - ) []Value { - std.debug.assert(lbs.len == ubs.len); - std.debug.assert(inits.len == reduction_result_types.len); - - const scratch = self.arena.allocator(); - const idx_ty = .index(self.ctx); - const iv_types = scratch.alloc(*const mlir.Type, lbs.len) catch @panic("Builder.parallel OOM"); - const iv_locs = scratch.alloc(*const mlir.Location, lbs.len) catch @panic("Builder.parallel OOM"); - for (iv_types, iv_locs) |*t, *l| { - t.* = idx_ty; - l.* = self.loc(); - } - const body_block = mlir.Block.init(iv_types, iv_locs); - - self.pushBlock(body_block); - const iv_args = scratch.alloc(Value, lbs.len) catch @panic("Builder.parallel OOM"); - for (iv_args, 0..) |*v, i| v.* = .{ .inner = body_block.argument(i), .kernel = self }; - const reduce_operands: []const Value = body_fn(self, iv_args, body_ctx); - std.debug.assert(reduce_operands.len == reduction_result_types.len); - - // Build scf.reduce with one region per reduction. - const red_blocks = scratch.alloc(*mlir.Block, combiners.len) catch @panic("Builder.parallel OOM"); - inline for (combiners, 0..) |comb, i| { - const arg_ty = reduce_operands[i].type_(); - const arg_locs: [2]*const mlir.Location = .{ self.loc(), self.loc() }; - const red_block = mlir.Block.init(&.{ arg_ty, arg_ty }, &arg_locs); - self.pushBlock(red_block); - const lhs_v: Value = .{ .inner = red_block.argument(0), .kernel = self }; - const rhs_v: Value = .{ .inner = red_block.argument(1), .kernel = self }; - const combined: Value = comb(self, lhs_v, rhs_v, body_ctx); - _ = scf.reduce_return(self.ctx, combined.inner, self.loc()).appendTo(red_block); - self.popBlock(); - red_blocks[i] = red_block; - } - - _ = scf.reduce(self.ctx, self.innerSlice(reduce_operands), red_blocks, self.loc()).appendTo(body_block); - self.popBlock(); - - const op = scf.parallel( - self.ctx, - self.innerSlice(lbs), - self.innerSlice(ubs), - self.innerSlice(steps), - self.innerSlice(inits), - body_block, - reduction_result_types, - self.loc(), - ); - _ = op.appendTo(self.currentBlock()); - - const out = self.arena.allocator().alloc(Value, reduction_result_types.len) catch @panic("Builder.parallel OOM"); - for (0..reduction_result_types.len) |i| out[i] = .{ .inner = op.result(i), .kernel = self }; - return out; - } - - // ==================== reduce / scan ==================== - - /// `tt.reduce` along `axis`, single-input. See `ReduceArgs` for fields. - /// `ctx` is separate from `args` so `ReduceArgs(@TypeOf(ctx))` can be - /// inferred. - pub fn reduce(self: *Builder, ctx: anytype, args: ReduceArgs(@TypeOf(ctx))) Value { - const region = mlir.Block.init(&.{ args.elem, args.elem }, &.{ self.loc(), self.loc() }); - - self.pushBlock(region); - const lhs: Value = .{ .inner = region.argument(0), .kernel = self }; - const rhs: Value = .{ .inner = region.argument(1), .kernel = self }; - const combined: Value = args.combine(self, lhs, rhs, ctx); - _ = ttir.reduce_return(self.ctx, &.{combined.inner}, self.loc()).appendTo(region); - self.popBlock(); - - const op = ttir.reduce(self.ctx, &.{args.src.inner}, args.axis, region, &.{args.result}, self.loc()); - _ = op.appendTo(self.currentBlock()); - return .{ .inner = op.result(0), .kernel = self }; - } - - /// `tt.scan` along `axis`, single-input. See `ScanArgs` for fields. - pub fn scan(self: *Builder, ctx: anytype, args: ScanArgs(@TypeOf(ctx))) Value { - const region = mlir.Block.init(&.{ args.elem, args.elem }, &.{ self.loc(), self.loc() }); - - self.pushBlock(region); - const lhs: Value = .{ .inner = region.argument(0), .kernel = self }; - const rhs: Value = .{ .inner = region.argument(1), .kernel = self }; - const combined: Value = args.combine(self, lhs, rhs, ctx); - _ = ttir.scan_return(self.ctx, &.{combined.inner}, self.loc()).appendTo(region); - self.popBlock(); - - const op = ttir.scan(self.ctx, &.{args.src.inner}, args.axis, args.reverse, region, &.{args.result}, self.loc()); - _ = op.appendTo(self.currentBlock()); - return .{ .inner = op.result(0), .kernel = self }; - } - - // ==================== SCF regions ==================== - - /// `for iv in [lower, upper) step` carrying `inits`. Literal bounds take the - /// type of a Value bound, else `lower` is lifted (`i32` when it fits). - pub fn openFor( - self: *Builder, - lower: anytype, - upper: anytype, - step: anytype, - inits: anytype, - ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { - return self.openForOpts(lower, upper, step, inits, .{}); - } - - /// `openFor` with `scf.for` options. - pub fn openForOpts( - self: *Builder, - lower: anytype, - upper: anytype, - step: anytype, - inits: anytype, - opts: common.ForOpts, - ) ForScope(common.arity(@TypeOf(inits), "openFor: inits")) { - return common.openFor(self, Value, common.forBounds(self, Value, lower, upper, step, Builder.lift), inits, opts); - } - - pub fn returnIf(self: *Builder, cond: Value) void { - self.returnIfOpts(cond, .{}); - } - - pub fn returnIfOpts(self: *Builder, cond: Value, opts: ReturnIfOpts) void { - const current = self.currentBlock(); - const region = current.parentRegion(); - const cont_block = mlir.Block.init(&.{}, &.{}); - - const ret_target: *mlir.Block = if (opts.inline_return) blk: { - const rb = mlir.Block.init(&.{}, &.{}); - _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(rb); - region.appendOwnedBlock(rb); - break :blk rb; - } else self.exitBlock(); - - region.appendOwnedBlock(cont_block); - - _ = cf.cond_br(self.ctx, cond.inner, ret_target, &.{}, cont_block, &.{}, .{}, self.loc()).appendTo(current); - self.continueIn(cont_block); - } - - /// The block every non-inline return branches to, which `finish` appends. - fn exitBlock(self: *Builder) *mlir.Block { - if (self.exit_block == null) { - self.exit_block = mlir.Block.init(&.{}, &.{}); - _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(self.exit_block.?); - } - return self.exit_block.?; - } - - /// Continues in `block`, the fall-through of an early return, in place of - /// the current block. - fn continueIn(self: *Builder, block: *mlir.Block) void { - self.block_stack.replaceTop(block); - } - - pub fn openReturnIf(self: *Builder, cond: Value) ReturnIfScope { - return self.openReturnIfOpts(cond, .{}); - } - - pub fn openReturnIfOpts(self: *Builder, cond: Value, opts: ReturnIfOpts) ReturnIfScope { - const current = self.currentBlock(); - const region = current.parentRegion(); - - const ret_block = mlir.Block.init(&.{}, &.{}); - const cont_block = mlir.Block.init(&.{}, &.{}); - region.appendOwnedBlock(ret_block); - region.appendOwnedBlock(cont_block); - - _ = cf.cond_br(self.ctx, cond.inner, ret_block, &.{}, cont_block, &.{}, .{}, self.loc()).appendTo(current); - - self.pushBlock(ret_block); - - return .{ - .kernel = self, - .ret_block = ret_block, - .cont_block = cont_block, - .opts = opts, - }; - } - - pub fn openIf(self: *Builder, cond: Value) IfOnlyScope { - return common.openIf(self, Value, cond); - } - - pub fn openIfElse(self: *Builder, cond: Value, result_types: anytype) IfScope(common.arity(@TypeOf(result_types), "openIfElse: result_types")) { - return common.openIfElse(self, Value, cond, result_types); - } - - pub fn openWhile( - self: *Builder, - inits: anytype, - after_types: anytype, - ) WhileScope(common.arity(@TypeOf(inits), "openWhile: inits"), common.arity(@TypeOf(after_types), "openWhile: after_types")) { - return common.openWhile(self, Value, inits, after_types); - } - - // ==================== finalization ==================== - - /// Append the final `tt.return` (kernels return nothing), verify the - /// module, and serialize to a NUL-terminated TTIR string owned by the - /// kernel's allocator (passed to `init`). Caller must - /// `defer allocator.free(ir)`. - pub fn finish(self: *Builder) FinishError![:0]const u8 { - // After any `returnIf` call, the entry block already has a - // `cf.cond_br` terminator; the fall-through block sits on the stack - // as the new insertion point. `currentBlock()` returns that block - // (the entry block if no cf was emitted), so the final terminator - // goes to the correct "tail" block in either case. - const current = self.currentBlock(); - if (self.exit_block) |exit| { - const region = self.entry_block.?.parentRegion(); - region.appendOwnedBlock(exit); - _ = cf.br(self.ctx, exit, &.{}, self.loc()).appendTo(current); - } else { - _ = ttir.return_(self.ctx, &.{}, self.loc()).appendTo(current); - } - - try common.verify(self.module, "dsl/triton"); - - var al: std.Io.Writer.Allocating = .init(self.allocator); - defer al.deinit(); - - try al.writer.print("{f}", .{self.module.operation().fmt(.{ - .debug_info = true, - .debug_info_pretty_form = false, - .print_name_loc_as_prefix = true, - })}); - - return try self.allocator.dupeSentinel(u8, al.written(), 0); - } -}; diff --git a/dsl/triton/dtype.zig b/dsl/triton/dtype.zig deleted file mode 100644 index 45018e67..00000000 --- a/dsl/triton/dtype.zig +++ /dev/null @@ -1,74 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); - -pub const DType = enum { - i1, - i8, - i16, - i32, - i64, - f16, - bf16, - f32, - f64, - f8e4m3fn, - f8e4m3fnuz, - f8e5m2, - - pub fn toMlir(self: DType, ctx: *mlir.Context) *const mlir.Type { - return switch (self) { - .i1 => .int(ctx, .i1), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .f16 => .float(ctx, .f16), - .bf16 => .float(ctx, .bf16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .f8e4m3fn => .float(ctx, .f8e4m3fn), - .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), - .f8e5m2 => .float(ctx, .f8e5m2), - }; - } -}; - -pub fn isFloatDtype(dt: DType) bool { - return switch (dt) { - .f16, .bf16, .f32, .f64, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => true, - else => false, - }; -} - -pub fn dtypeBitwidth(dt: DType) usize { - return switch (dt) { - .i1 => 1, - .i8, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2 => 8, - .i16, .f16, .bf16 => 16, - .i32, .f32 => 32, - .i64, .f64 => 64, - }; -} - -pub fn intBitwidth(dt: DType) u32 { - return switch (dt) { - .i1 => 1, - .i8 => 8, - .i16 => 16, - .i32 => 32, - .i64 => 64, - else => std.debug.panic("intBitwidth: not an integer DType: {s}", .{@tagName(dt)}), - }; -} - -pub fn mlirElemToDType(ctx: *mlir.Context, elem: *const mlir.Type) DType { - inline for (comptime std.meta.fieldNames(DType)) |field_name| { - const dt = @field(DType, field_name); - if (elem.eql(dt.toMlir(ctx))) return dt; - } - @panic("element type not a recognized Triton DType"); -} - -test { - std.testing.refAllDecls(@This()); -} diff --git a/examples/benchmark/main.zig b/examples/benchmark/main.zig index 7ff70707..5927ad6e 100644 --- a/examples/benchmark/main.zig +++ b/examples/benchmark/main.zig @@ -5,10 +5,10 @@ const zml = @import("zml"); const stdx = zml.stdx; pub fn benchmark(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return a.dot(b, .k).withPartitioning(.benchmark_mesh, .{ .m = .m, .n = .replicated }); + return a.dot(b, .k); } -pub fn main(init: std.process.Init) !void { +pub fn main() !void { const CliArgs = struct { pub const help = \\ benchmark --size=4096 --dtype=f16 @@ -17,31 +17,39 @@ pub fn main(init: std.process.Init) !void { dtype: zml.DataType = .f16, }; - const allocator = init.gpa; - const io = init.io; + const allocator = std.heap.smp_allocator; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const io = threaded.io(); // Auto-select platform const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator, io); + defer platform.deinit(allocator); - log.info("\n{f}", .{platform.fmtVerbose()}); + const cli_args: CliArgs = blk: { + var ret: CliArgs = .{}; + var it = std.process.args(); + defer it.deinit(); + while (it.next()) |arg| { + if (std.mem.startsWith(u8, arg, "--dtype=")) { + ret.dtype = std.meta.stringToEnum(zml.DataType, arg["--dtype=".len..]) orelse unreachable; + } else if (std.mem.startsWith(u8, arg, "--size=")) { + ret.size = try std.fmt.parseUnsigned(usize, arg["--size=".len..], 10); + } + } + break :blk ret; + }; - const mesh: *const zml.Mesh = try platform.registerMesh("mesh", .mesh( - .{ .m = .low_bandwidth, .n = .high_bandwidth }, - )); - - const cli_args: CliArgs = stdx.flags.parse(init.minimal.args, CliArgs); - - const a = zml.Tensor.init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype) - .withPartitioning(mesh, .{ .m = .m, .k = .replicated }); - const b = zml.Tensor.init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype) - .withPartitioning(mesh, .{ .k = .replicated, .n = .n }); + const a: zml.Tensor = .init(.{ .m = cli_args.size, .k = cli_args.size }, cli_args.dtype); + const b: zml.Tensor = .init(.{ .k = cli_args.size, .n = cli_args.size }, cli_args.dtype); var exe = blk: { log.info("⏱️ Compiling benchmark...", .{}); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Compiled benchmark [{f}]", .{now.untilNow(io, .awake)}); - break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }, .{ .meshes = &.{mesh} }); + var timer = try std.time.Timer.start(); + defer log.info("✅ Compiled benchmark [{D}]", .{timer.read()}); + break :blk try platform.compileFn(allocator, io, benchmark, .{ a, b }); }; defer exe.deinit(); @@ -65,30 +73,29 @@ pub fn main(init: std.process.Init) !void { // Ignore first run { - exe.call(io, exe_args, &exe_results, .{}); + exe.call(exe_args, &exe_results); var result = exe_results.get(zml.Buffer); defer result.deinit(); } // call our executable module - const run_start: std.Io.Timestamp = .now(io, .awake); - exe.call(io, exe_args, &exe_results, .{}); + var timer = try std.time.Timer.start(); + exe.call(exe_args, &exe_results); var result = exe_results.get(zml.Buffer); _ = try result.await(io); defer result.deinit(); - const elapsed = run_start.untilNow(io, .awake); - const elapsed_ns = elapsed.toNanoseconds(); + const elapsed_ns = timer.read(); const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / std.time.ns_per_s; log.info("✅ Benchmark done!", .{}); const floating_op_count = 2 * cli_args.size * cli_args.size * cli_args.size; const flops = @as(f64, @floatFromInt(floating_op_count)) / elapsed_s; - log.info("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {f} - {d:.3} GFLOP/s", .{ + log.info("Dot product size: {d}x{d} - Datatype: {s} - Elapsed: {D} - {d:.3} GFLOP/s", .{ cli_args.size, cli_args.size, @tagName(cli_args.dtype), - elapsed, + elapsed_ns, flops / 1_000_000_000, }); } @@ -99,19 +106,18 @@ fn createRandomBuffer(allocator: std.mem.Allocator, io: std.Io, platform: *const switch (shape.dtype()) { inline else => |v| { - const ZigType = v.toPackedZigType(); + const ZigType = v.toZigType(); switch (comptime v.class()) { .bool => unreachable, .integer => { - for (slice.items(ZigType)) |*e| e.* = @bitCast(random.int(@Int(.unsigned, @bitSizeOf(ZigType)))); + for (slice.items(ZigType)) |*e| e.* = random.int(ZigType); }, .float => { const value = random.float(f32); for (slice.items(ZigType)) |*e| e.* = switch (ZigType) { f64, f32 => value, f16 => @floatCast(value), - zml.floats.Float4E2M1.Packed => .fromF32(value, -value), - inline else => |T| if (@hasDecl(T, "fromF32")) .fromF32(value) else unreachable, + inline else => |T| if (@hasDecl(T, "fromF32")) T.fromF32(value) else unreachable, }; }, .complex => unreachable, diff --git a/examples/io/main.zig b/examples/io/main.zig deleted file mode 100644 index c9dc05ea..00000000 --- a/examples/io/main.zig +++ /dev/null @@ -1,362 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); -const stdx = zml.stdx; - -const log = std.log.scoped(.vfs); - -pub const std_options: std.Options = .{ - .log_level = .info, -}; - -const Command = enum { cat, tree, ls, cp, stat, realpath, safetensors, load }; - -// -- ls hf://openai/gpt-oss-20b@6cee5e8 -// -- ls hf://Qwen/Qwen3-235B-A22B-Instruct-2507 -// -- ls hf://meta-llama/Llama-3.1-8B-Instruct@0e9e39f -// -- tree s3://noaa-goes19/ABI-Flood-Day-Shapefiles/2025/08 -// -- cat https://iprs.fly.dev -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - - var it = init.minimal.args.iterate(); - _ = it.next(); // skip program name - const command: Command = std.meta.stringToEnum(Command, it.next() orelse return error.MissingCommand) orelse return error.CommandInvalid; - const path = it.next() orelse return error.MissingPath; - - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - - try http_client.initDefaultProxies(allocator, init.environ_map); - defer http_client.deinit(); - - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - - var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); - defer vfs_https.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); - defer gcs_vfs.deinit(); - - var vfs: zml.io.VFS = try .init(allocator, init.io); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - try vfs.register("https", vfs_https.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); - try vfs.register("gs", gcs_vfs.io()); - - const io = vfs.io(); - - const buffer = try allocator.alignedAlloc(u8, .fromByteUnits(4 * 1024), 16 * 1024 * 1024); - defer allocator.free(buffer); - - var stdout_writer = std.Io.File.stdout().writer(io, buffer); - defer stdout_writer.interface.flush() catch {}; - - switch (command) { - .cat => { - var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); - defer file.close(io); - - var reader: std.Io.File.Reader = .initStreaming(file, io, &.{}); - - const read = try reader.interface.streamRemaining(&stdout_writer.interface); - _ = try stdout_writer.interface.write("\n"); - - try stdout_writer.interface.print("Wrote {B:.2} to stdout from {s}\n", .{ read, path }); - }, - .ls, .tree => { - var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{ .iterate = true }); - defer dir.close(io); - - const dir_stat = try dir.stat(io); - try stdout_writer.interface.print("{s} - {B:.2}\n", .{ path, dir_stat.size }); - - var counts: TreeCounts = .{}; - try printTree(io, &stdout_writer.interface, dir, "", if (command == .tree) 10 else 1, &counts); - try stdout_writer.interface.print("\n{d} directories, {d} files\n", .{ counts.dirs, counts.files }); - }, - .cp => { - const destination_path = it.next() orelse { - try stdout_writer.interface.print("Usage: cp \n", .{}); - return error.InvalidArgument; - }; - - const source = try std.Io.Dir.openFile(.cwd(), io, path, .{}); - defer source.close(io); - - const destination = try std.Io.Dir.createFile(.cwd(), io, destination_path, .{}); - defer destination.close(io); - - var reader: std.Io.File.Reader = .initStreaming(source, io, &.{}); - var writer = destination.writer(io, buffer); - - const read = try reader.interface.streamRemaining(&writer.interface); - try writer.interface.flush(); - - try stdout_writer.interface.print("Copied {B:.2} from {s} to {s}\n", .{ read, path, destination_path }); - }, - .stat => { - const stat = std.Io.Dir.statFile(.cwd(), io, path, .{}) catch |err| blk: { - if (err == error.IsDir) { - var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{}); - defer dir.close(io); - - break :blk try dir.stat(io); - } - return err; - }; - - try stdout_writer.interface.print("{s}: {B:.2} ({s})\n", .{ path, stat.size, @tagName(stat.kind) }); - }, - .realpath => { - var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{}); - defer dir.close(io); - - var real_path_buf: [256]u8 = undefined; - const len = try dir.realPath(io, &real_path_buf); - - try stdout_writer.interface.print("{s}\n", .{real_path_buf[0..len]}); - }, - .safetensors => { - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); - defer registry.deinit(); - - const root = try TensorNode.init(init.arena.allocator(), ""); - - for (registry.tensors.values()) |tensor| { - var current = root; - var parts = std.mem.tokenizeScalar(u8, tensor.name, '.'); - while (parts.next()) |part| { - const gop = try current.children.getOrPut(current.allocator, part); - if (!gop.found_existing) { - gop.value_ptr.* = try TensorNode.init(init.arena.allocator(), part); - } - current = gop.value_ptr.*; - } - - current.tensor = tensor; - } - - try stdout_writer.interface.print("{s}\n", .{path}); - try printTensorTree(&stdout_writer.interface, root, "", true, true); - try stdout_writer.interface.flush(); - }, - .load => { - const MeshType = enum { replicated, sharded }; - - const mesh_type: MeshType = std.meta.stringToEnum(MeshType, it.next() orelse "sharded") orelse return error.InvalidMesheKind; - - const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator, io); - - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, path); - defer registry.deinit(); - - const tensor_parallel: *const zml.Mesh = try platform.registerMesh( - "tp", - .mesh(.{ .model = .high_bandwidth }), - ); - const tp_partitioning: zml.Sharding.Partitioning = .init(&.{.sharded(@intCast(tensor_parallel.resolveLogicalAxis("model").?))}); - - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{tensor_parallel}); - defer store.deinit(); - - const AllTensorsModel = struct { - tensors: []zml.Tensor, - }; - - const tensor_count = registry.tensors.count(); - - const tensors = try allocator.alloc(zml.Tensor, tensor_count); - defer allocator.free(tensors); - - var load_count: usize = 0; - for (registry.tensors.keys(), registry.tensors.values()) |name, tensor| { - tensors[load_count] = switch (mesh_type) { - .replicated => store.view().createReplicatedTensor(name, null), - .sharded => if (tensor.shape.rank() > 0) - store.view().createTensor(name, null, .tp, tp_partitioning) - else - store.view().createReplicatedTensor(name, null), - }; - load_count += 1; - } - - const model: AllTensorsModel = .{ .tensors = tensors }; - - var progress = std.Progress.start(io, .{ .root_name = "zml.examples.load" }); - progress.increaseEstimatedTotalItems(load_count); - defer progress.end(); - - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, AllTensorsModel, &model); - defer { - for (buffers.tensors) |*b| b.deinit(); - allocator.free(buffers.tensors); - } - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .parallelism = 8, - .dma_chunks = 16, - .dma_chunk_size = 256 * zml.MiB, - }); - defer loader.deinit(); - - try loader.load(io, AllTensorsModel, &model, &buffers, &store, .{ .progress = &progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - }, - } -} - -const TreeCounts = struct { - dirs: usize = 0, - files: usize = 0, -}; - -fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, prefix: []const u8, max_depth: usize, counts: *TreeCounts) !void { - if (max_depth == 0) return; - - var entries: stdx.BoundedArray(std.Io.Dir.Entry, 1024) = .empty; - var it = dir.iterate(); - while (try it.next(io)) |entry| { - entries.appendAssumeCapacity(entry); - } - - for (entries.constSlice(), 0..) |entry, idx| { - const is_last = (idx == entries.len - 1); - const connector = if (is_last) "└── " else "├── "; - const extension = if (is_last) " " else "│ "; - - const size: u64 = switch (entry.kind) { - .file, .sym_link => blk: { - const stat = try dir.statFile(io, entry.name, .{}); - break :blk stat.size; - }, - .directory => blk: { - var sub_dir = dir.openDir(io, entry.name, .{}) catch break :blk 0; - defer sub_dir.close(io); - - const stat = try sub_dir.stat(io); - break :blk stat.size; - }, - else => 0, - }; - - try writer.print("{s}{s}{s} - {B:.2}\n", .{ prefix, connector, entry.name, size }); - - if (entry.kind == .directory) { - counts.dirs += 1; - var sub_dir = dir.openDir(io, entry.name, .{ .iterate = true }) catch continue; - defer sub_dir.close(io); - - var new_prefix_buf: [4096]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&new_prefix_buf, "{s}{s}", .{ prefix, extension }) catch continue; - try printTree(io, writer, sub_dir, new_prefix, max_depth - 1, counts); - } else { - counts.files += 1; - } - } -} - -const TensorNode = struct { - name: []const u8, - children: std.StringArrayHashMapUnmanaged(*TensorNode) = .empty, - allocator: std.mem.Allocator, - tensor: ?zml.safetensors.Tensor = null, - - fn init(allocator: std.mem.Allocator, name: []const u8) !*TensorNode { - const node = try allocator.create(TensorNode); - node.* = .{ - .name = name, - .allocator = allocator, - }; - return node; - } -}; - -fn getSortedChildren(allocator: std.mem.Allocator, node: *TensorNode) ![]*TensorNode { - const children_nodes = try allocator.alloc(*TensorNode, node.children.count()); - var it = node.children.iterator(); - var idx: usize = 0; - while (it.next()) |entry| : (idx += 1) { - children_nodes[idx] = entry.value_ptr.*; - } - - std.mem.sort(*TensorNode, children_nodes, {}, struct { - fn lessThan(_: void, a: *TensorNode, b: *TensorNode) bool { - const a_num = std.fmt.parseInt(usize, a.name, 10) catch null; - const b_num = std.fmt.parseInt(usize, b.name, 10) catch null; - if (a_num != null and b_num != null) return a_num.? < b_num.?; - return std.mem.lessThan(u8, a.name, b.name); - } - }.lessThan); - return children_nodes; -} - -fn printTensorTree( - writer: anytype, - node: *TensorNode, - prefix: []const u8, - is_last: bool, - is_root: bool, -) !void { - const allocator = node.allocator; - - if (is_root) { - const children_nodes = try getSortedChildren(allocator, node); - for (children_nodes, 0..) |child, i| { - try printTensorTree(writer, child, "", i == children_nodes.len - 1, false); - } - return; - } - - var compacted_name = node.name; - var walk = node; - while (walk.children.count() == 1 and walk.tensor == null) { - const next = walk.children.values()[0]; - compacted_name = try std.fmt.allocPrint(allocator, "{s}.{s}", .{ compacted_name, next.name }); - walk = next; - } - - const connector = if (is_last) "└── " else "├── "; - try writer.print("{s}{s}{s}", .{ prefix, connector, compacted_name }); - - if (walk.tensor) |t| { - try writer.print(" [shape={f} size={B:.2}]", .{ t.shape, t.byteSize() }); - } - try writer.print("\n", .{}); - - const extension = if (is_last) " " else "│ "; - const child_prefix = try std.fmt.allocPrint(allocator, "{s}{s}", .{ prefix, extension }); - const children_nodes = try getSortedChildren(allocator, walk); - - const show_count = 2; - const skip_threshold = 8; - - for (children_nodes, 0..) |child, i| { - if (children_nodes.len > skip_threshold) { - if (i >= show_count and i < children_nodes.len - show_count) { - if (i == show_count) { - try writer.print("{s}├── ...\n", .{child_prefix}); - } - continue; - } - } - try printTensorTree(writer, child, child_prefix, i == children_nodes.len - 1, false); - } -} diff --git a/examples/llama/BUILD.bazel b/examples/llama/BUILD.bazel new file mode 100644 index 00000000..c7c91eb4 --- /dev/null +++ b/examples/llama/BUILD.bazel @@ -0,0 +1,80 @@ +load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar") +load("@aspect_bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") +load("@bazel_skylib//rules:build_test.bzl", "build_test") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") +load("@rules_zig//zig:defs.bzl", "zig_binary") + +zig_binary( + name = "llama", + srcs = ["llama.zig"], + main = "main.zig", + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "//zml", + ], +) + +zig_binary( + name = "loader", + srcs = ["llama.zig"], + main = "main2.zig", + visibility = ["//visibility:public"], + deps = ["//zml"], +) + +mtree_spec( + name = "mtree", + srcs = [":llama"], +) + +tar( + name = "archive", + srcs = [":llama"], + args = [ + "--options", + ",".join([ + "zstd:compression-level=9", + "zstd:threads=16", + ]), + ], + compress = "zstd", + mtree = ":mtree", +) + +oci_image( + name = "image_", + base = "@distroless_cc_debian12", + entrypoint = ["/{}/llama".format(package_name())], + target_compatible_with = [ + "@platforms//os:linux", + ], + tars = [":archive"], +) + +platform_transition_filegroup( + name = "image", + srcs = [":image_"], + tags = ["manual"], + target_platform = "//platforms:linux_amd64", +) + +oci_load( + name = "load", + image = ":image", + repo_tags = ["zmlai/llama:latest"], + tags = ["manual"], +) + +oci_push( + name = "push", + image = ":image", + remote_tags = ["latest"], + repository = "index.docker.io/zmlai/llama", + tags = ["manual"], +) + +build_test( + name = "test", + targets = [":llama"], +) diff --git a/examples/llama/llama.zig b/examples/llama/llama.zig new file mode 100644 index 00000000..0968b83e --- /dev/null +++ b/examples/llama/llama.zig @@ -0,0 +1,536 @@ +const std = @import("std"); +const testing = std.testing; + +const zml = @import("zml"); +const Buffer = zml.Buffer; +const Tensor = zml.Tensor; +const ShapeOf = zml.ShapeOf; +const stdx = zml.stdx; + +const log = std.log.scoped(.llama); + +pub const TransferCtx = struct { + allocator: std.mem.Allocator, + pool: *zml.mem.DynamicBufferPool, + transferred_bytes: *usize, + progress: *std.Progress.Node, +}; + +/// Llama architecture, using huggingface transformers naming. +/// Dimensions of activations: {.b, .s, .d} +pub const LlamaLM = struct { + pub const Config = struct { + bos_token_id: u32, + eos_token_id: stdx.json.Union(union(enum) { + int: u32, + ints: []u32, + }), + head_dim: ?u32 = null, + hidden_size: u32, + num_hidden_layers: u32, + num_attention_heads: u32, + num_key_value_heads: u32, + rope_theta: f32, + max_position_embeddings: u32, + rms_norm_eps: f32, + hf_rope_impl: bool = true, + tie_word_embeddings: bool = false, + rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, + }; + + pub const Options = struct { + sampling_strategy: ?zml.nn.SamplingStrategy, + max_seq_len: u32, + }; + + lm_head: ?zml.nn.Linear, + model: Llama, + + // Options controlling generation + gen_opts: zml.nn.SamplingStrategy = .{}, + config: Config, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, options: Options) !LlamaLM { + const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensorWithTags("weight", .{ .dout, .d })) |weight| + .init(weight, null, .d) + else + null; + + return .{ + .lm_head = lm_head, + .model = try .init(allocator, store.withPrefix("model"), config), + .gen_opts = options.sampling_strategy orelse .{}, + .config = config, + }; + } + + pub fn deinit(self: LlamaLM, allocator: std.mem.Allocator) void { + self.model.deinit(allocator); + } + + pub fn load( + self: *const LlamaLM, + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + progress: *std.Progress.Node, + ) !zml.Bufferized(LlamaLM) { + progress.increaseEstimatedTotalItems(store.view().count()); + const now: std.Io.Timestamp = .now(io, .awake); + var total_bytes: usize = 0; + defer { + const took = now.untilNow(io, .awake); + log.info("Loaded weights [{Bi:.2}, {D}, {Bi:.2}/s]", .{ + total_bytes, + stdx.fmt.fmtDuration(took), + total_bytes / (@as(usize, @intCast(took.toNanoseconds())) * std.time.ns_per_s), + }); + } + return zml.io.load(LlamaLM, self, allocator, io, platform, .{ + .dma_chunks = 32, + .dma_chunk_size = 128 * zml.MiB, + .progress = progress, + .store = store, + .parallelism = 16, + .total_bytes = &total_bytes, + }); + } + + pub fn unloadBuffers(self: *zml.Bufferized(LlamaLM), allocator: std.mem.Allocator) void { + if (self.lm_head) |*lm_head| lm_head.weight.deinit(); + Llama.unloadBuffers(&self.model, allocator); + } + + /// Predicts the token at `token_index` position. + /// Returns: + /// - updated `tokens`, + /// - updated KV cache + /// - a Rng state to allow for probabilistic generation + pub fn forward( + self: LlamaLM, + tokens_: Tensor, + token_index: Tensor, + kv_cache: KvCache, + rng: Tensor.Rng, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, + ) struct { Tensor, KvCache, Tensor.Rng } { + stdx.debug.assert(tokens_.dtype() == .u32 and tokens_.rank() >= 1 and token_index.dtype() == .u32 and token_index.rank() <= 1, "Can't run Llama ! Expected >=1d tokens and 0d token_index, got: {f} and {f}", .{ tokens_, token_index }); + const tokens = tokens_.withPartialTags(.{.s}); + const out, const updated_kv_cache = self.model.forward(tokens, token_index, kv_cache, attention_metadata, attention_parameters); + const new_tokens, const new_rng = self.sampleTokens(self.lm_head, out, rng, self.gen_opts); + return .{ new_tokens.convert(tokens.dtype()).reuseBuffer(tokens), updated_kv_cache, new_rng }; + } + + pub fn sampleTokens( + self: LlamaLM, + lm_head_: ?zml.nn.Linear, + out_: Tensor, + rng: Tensor.Rng, + opts: zml.nn.SamplingStrategy, + ) struct { Tensor, Tensor.Rng } { + const out = out_.withPartialTags(.{ .s, .d }); + + var logits = blk: { + if (lm_head_) |lm_head| { + break :blk lm_head.forward(out).rename(.{ .dout = .d }); + } else { + break :blk self.model.embed_tokens.weight.withTags(.{ .voc, .d }).dot(out, .d); + } + }; + + if (logits.shape().hasTag(.voc) == null) + logits = logits.rename(.{ .d = .voc }); + + const next_tokens, const new_rng = zml.nn.sampleTokens(logits, opts, rng); + return .{ next_tokens, new_rng }; + } +}; + +pub const Llama = struct { + embed_tokens: zml.nn.TokenEmbedding, + norm: RmsNorm, + layers: []TransformerLayer, + + max_seq_len: u32 = 0, + num_heads: u32 = 32, + num_kv_heads: u32 = 32, + rope_opts: zml.nn.RopeOpts = .{ + .layout = .interleaved, + .freq_base = 10_000, + }, + + pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: LlamaLM.Config) !Llama { + const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); + errdefer allocator.free(layers); + + for (layers, 0..) |*layer, i| { + layer.* = try .init(store.withPrefix("layers").withLayer(i), config); + } + + return .{ + .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight") }, + .norm = .{ .weight = store.withPrefix("norm").createTensorWithTags("weight", .{.d}), .eps = config.rms_norm_eps }, + .layers = layers, + }; + } + + pub fn deinit(self: Llama, allocator: std.mem.Allocator) void { + allocator.free(self.layers); + } + + pub fn loadBuffers( + self: *const Llama, + bufferize_ctx: zml.io.BufferizeContext(TransferCtx), + group: *stdx.Io.LimitedGroup, + store: zml.io.TensorStore.View, + cb: zml.io.CallbackTensorBufferTransfer(TransferCtx), + ) !zml.Bufferized(Llama) { + const bufferized_layers = try bufferize_ctx.allocator.alloc(zml.Bufferized(TransformerLayer), self.layers.len); + + var bufferized: zml.Bufferized(Llama) = .{ .embed_tokens = undefined, .norm = undefined, .layers = bufferized_layers }; + + const transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &self.embed_tokens, &bufferized.embed_tokens, store.withPrefix("embed_tokens")); + for (transfers) |t| { + try group.concurrent(bufferize_ctx.io, cb, .{t}); + } + + const norm_transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &self.norm, &bufferized.norm, store.withPrefix("norm")); + for (norm_transfers) |t| { + try group.concurrent(bufferize_ctx.io, cb, .{t}); + } + + for (self.layers, bufferized_layers, 0..) |layer, *buf_layer, i| { + const layer_transfers = try zml.io.bufferize(TransferCtx, bufferize_ctx, &layer, buf_layer, store.withPrefix("layers").withLayer(i)); + for (layer_transfers) |t| { + try group.concurrent(bufferize_ctx.io, cb, .{t}); + } + } + + return bufferized; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { + self.embed_tokens.weight.deinit(); + RmsNorm.unloadBuffers(&self.norm); + for (self.layers) |*layer| { + TransformerLayer.unloadBuffers(layer); + } + allocator.free(self.layers); + } + + /// Forward one token, using KV cache for previous tokens. + /// Returns result and updated KV cache. + pub fn forward( + self: Llama, + tokens: Tensor, + token_index: Tensor, + kv_cache: KvCache, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, + ) struct { Tensor, KvCache } { + const embeds = embed(self.embed_tokens, tokens); + var hidden = embeds; + + var updated_kv_cache = kv_cache; + for (self.layers, 0..) |layer, i| { + hidden, updated_kv_cache = layer.forward(hidden, token_index, updated_kv_cache.atLayer(i), attention_metadata, attention_parameters); + } + const output = self.norm.forward(hidden); + + return .{ output, updated_kv_cache.reuseBuffer(kv_cache) }; + } + + pub fn embed(embed_tokens_: zml.nn.TokenEmbedding, tokens_: Tensor) Tensor { + return embed_tokens_.forward(tokens_).withPartialTags(.{.d}); + } +}; + +pub const TransformerLayer = struct { + input_layernorm: RmsNorm, + self_attn: SelfAttn, + post_attention_layernorm: RmsNorm, + mlp: Mlp, + + pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config) !TransformerLayer { + return .{ + .input_layernorm = .init(store.withPrefix("input_layernorm"), config.rms_norm_eps), + .self_attn = try .init(store.withPrefix("self_attn"), config), + .post_attention_layernorm = .init(store.withPrefix("post_attention_layernorm"), config.rms_norm_eps), + .mlp = .init(store.withPrefix("mlp")), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { + RmsNorm.unloadBuffers(&self.input_layernorm); + SelfAttn.unloadBuffers(&self.self_attn); + RmsNorm.unloadBuffers(&self.post_attention_layernorm); + Mlp.unloadBuffers(&self.mlp); + } + + pub fn forward( + self: TransformerLayer, + x0: Tensor, + token_index: Tensor, + kv_cache: KvCache, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, + ) struct { Tensor, KvCache } { + // Self Attention + //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); + stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); + + const x0_normalized = self.input_layernorm.forward(x0); + const delta0, const updated_kv_cache = self.self_attn.forward(x0_normalized, token_index, kv_cache, attention_metadata, attention_parameters); + const x1 = x0.add(delta0); + + // Fully Connected + const x1_normalized = self.post_attention_layernorm.forward(x1); + const x2 = self.mlp.forward(x1_normalized).add(x1).rename(.{ .dout = .d }); + + return .{ x2.reuseBuffer(x0), updated_kv_cache }; + } +}; + +const RmsNorm = struct { + weight: Tensor, + eps: f32 = 1e-5, + + pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { + return .{ .weight = store.createTensorWithTags("weight", .{.d}), .eps = eps }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { + self.weight.deinit(); + } + + /// L2 normalization of input tensor along `.d` axis. + pub fn forward(self: RmsNorm, input: Tensor) Tensor { + const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); + const normalized = zml.nn.rmsNorm(x, .d, self.eps); + return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); + } +}; + +const Mlp = struct { + up_proj: zml.nn.Linear, // (dim -> hidden_dim) + gate_proj: zml.nn.Linear, // (dim -> hidden_dim) + down_proj: zml.nn.Linear, // (hidden_dim -> dim) + + fn initLinear(store: zml.io.TensorStore.View) zml.nn.Linear { + return .init(store.createTensorWithTags("weight", .{ .dout, .d }), store.maybeCreateTensorWithTags("bias", .{.dout}), .d); + } + + pub fn init(store: zml.io.TensorStore.View) Mlp { + return .{ + .up_proj = initLinear(store.withPrefix("up_proj")), + .gate_proj = initLinear(store.withPrefix("gate_proj")), + .down_proj = initLinear(store.withPrefix("down_proj")), + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { + self.up_proj.weight.deinit(); + if (self.up_proj.bias) |*bias| bias.deinit(); + self.gate_proj.weight.deinit(); + if (self.gate_proj.bias) |*bias| bias.deinit(); + self.down_proj.weight.deinit(); + if (self.down_proj.bias) |*bias| bias.deinit(); + } + + pub fn forward(self: Mlp, x: Tensor) Tensor { + const proj = self.up_proj.forward(x); + var output = self.gate_proj.forward(x); + output = output.silu().mul(proj).rename(.{ .dout = .d }); + return self.down_proj.forward(output); + } +}; + +pub const SelfAttn = struct { + q_proj: zml.nn.Linear, + k_proj: zml.nn.Linear, + v_proj: zml.nn.Linear, + + q_norm: ?RmsNorm, + k_norm: ?RmsNorm, + + o_proj: zml.nn.Linear, + num_heads: i64 = undefined, + num_kv_heads: i64 = 0, + rope_opts: zml.nn.RopeOpts = undefined, + + fn initProj(store: zml.io.TensorStore.View) zml.nn.Linear { + return .init(store.createTensorWithTags("weight", .{ .dout, .d }), store.maybeCreateTensorWithTags("bias", .{.dout}), .d); + } + + pub fn init(store: zml.io.TensorStore.View, config: LlamaLM.Config) !SelfAttn { + return .{ + .q_proj = initProj(store.withPrefix("q_proj")), + .k_proj = initProj(store.withPrefix("k_proj")), + .v_proj = initProj(store.withPrefix("v_proj")), + .o_proj = initProj(store.withPrefix("o_proj")), + // TODO(Corentin): fix that + .q_norm = null, + .k_norm = null, + .num_heads = @intCast(config.num_attention_heads), + .num_kv_heads = @intCast(config.num_key_value_heads), + .rope_opts = .{ + .layout = if (config.hf_rope_impl) .sequential else .interleaved, + .freq_base = config.rope_theta, + .scaling = config.rope_scaling, + }, + }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { + self.q_proj.weight.deinit(); + if (self.q_proj.bias) |*bias| bias.deinit(); + self.k_proj.weight.deinit(); + if (self.k_proj.bias) |*bias| bias.deinit(); + self.v_proj.weight.deinit(); + if (self.v_proj.bias) |*bias| bias.deinit(); + self.o_proj.weight.deinit(); + if (self.o_proj.bias) |*bias| bias.deinit(); + + if (self.q_norm) |*q_norm| RmsNorm.unloadBuffers(q_norm); + if (self.k_norm) |*k_norm| RmsNorm.unloadBuffers(k_norm); + } + + /// Self Attention. + /// - If token_index is set, x is assumed to be the representation of one new token, + /// and kv_cache will be read for the previous tokens. + /// - If token_index is not set, x is assumed to be the representation of all tokens + /// since the beginning of the sequence, and kv_cache won't be read. + /// In both case, kv_cache will be updated with the computed key and value. + /// x: {.b, .s, .d } -> .{.b, .s, .d} + pub fn forward( + self: SelfAttn, + x: Tensor, + token_index: Tensor, + kv_cache: KvCache, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, + ) struct { Tensor, KvCache } { + const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; + var q = self.q_proj.forward(x).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); + var k = self.k_proj.forward(x).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + var v = self.v_proj.forward(x).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); + + // In self-attention, .s axis is used both for keys and queries. + const pos_index = b: { + const temp = Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); + break :b temp.add(token_index.broad(temp.shape())); + }; + + if (self.q_norm) |norm| q = norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + if (self.k_norm) |norm| k = norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); + q = zml.nn.rope(q, pos_index, self.rope_opts); + k = zml.nn.rope(k, pos_index, self.rope_opts); + q = q.rename(.{ .s = .q }); + k = k.rename(.{ .s = .k }); + v = v.rename(.{ .s = .k }); + + const dtype = q.dtype(); + const new_kv_cache = kv_cache.update(k, v, token_index); + k = new_kv_cache.keys().convert(dtype); + v = new_kv_cache.values().convert(dtype); + + const attn_output = zml.attention.attention.attention(q, k, v, token_index, attention_metadata, attention_parameters); + + const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); + return .{ self.o_proj.forward(attn), new_kv_cache }; + } +}; + +pub const KvCache = struct { + k: Tensor, + v: Tensor, + layer_index: Tensor, + + pub fn init(kv_shape: zml.Shape) KvCache { + return .{ + .k = .fromShape(kv_shape), + .v = .fromShape(kv_shape), + .layer_index = .init(.{}, .u32), + }; + } + + pub fn initShape(kv_shape: zml.Shape) ShapeOf(KvCache) { + return .{ + .k = kv_shape, + .v = kv_shape, + .layer_index = zml.Shape.init(.{}, .u32), + }; + } + + pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { + return .{ + .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), + .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), + .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { + self.k.deinit(); + self.v.deinit(); + self.layer_index.deinit(); + } + + pub fn keys(self: KvCache) Tensor { + return self.k.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn values(self: KvCache) Tensor { + return self.v.dynamicSlice(.{ .layer = Tensor.DynSlice{ .start = self.layer_index, .len = 1 } }).squeeze(.layer); + } + + pub fn update(self: KvCache, new_k: Tensor, new_v: Tensor, token_index: ?Tensor) KvCache { + const k_shape = self.k.shape().drop(.layer); + var layer = self.layer_index; + layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; + + return if (token_index) |idx| .{ + .k = self.k.scatterSlices( + .{ .layer = layer, .k = idx }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer, .k = idx }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + } else .{ + .k = self.k.scatterSlices( + .{ .layer = layer }, + new_k.convert(self.k.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.k), + .v = self.v.scatterSlices( + .{ .layer = layer }, + new_v.convert(self.v.dtype()).transpose(k_shape), + .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, + ).reuseBuffer(self.v), + .layer_index = self.layer_index, + }; + } + + pub fn atLayer(self: KvCache, layer_index: usize) KvCache { + return .{ + .k = self.k, + .v = self.v, + .layer_index = Tensor.scalar(layer_index, .u32), + }; + } + + pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { + return .{ + .k = self.k.reuseBuffer(other.k), + .v = self.v.reuseBuffer(other.v), + .layer_index = self.layer_index.reuseBuffer(other.layer_index), + }; + } +}; diff --git a/examples/llama/main.zig b/examples/llama/main.zig new file mode 100644 index 00000000..aa048103 --- /dev/null +++ b/examples/llama/main.zig @@ -0,0 +1,552 @@ +const std = @import("std"); +const log = std.log; + +const zml = @import("zml"); +const Buffer = zml.Buffer; +const Tensor = zml.Tensor; +const ShapeOf = zml.ShapeOf; +const stdx = zml.stdx; + +const llama = @import("llama.zig"); +const LlamaLM = llama.LlamaLM; +const Llama = llama.Llama; +const KvCache = llama.KvCache; +const TransformerLayer = llama.TransformerLayer; +const SelfAttn = llama.SelfAttn; + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +const CliArgs = struct { + model: []const u8, + prompt: ?[]const u8 = null, + seqlen: u32 = 512, + reader_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, + writer_buffer_size: stdx.flags.ByteSize = .{ .value = 32, .unit = .mib }, + async_limit: ?u32 = null, + backend: ?zml.attention.attention.Backend = null, + + pub const help = + \\Usage: llama --model= [options] + \\ + \\Run LLaMA inference on a model. + \\ + \\Options: + \\ --model= Path to model (local path or HuggingFace repo) + \\ --prompt= Input prompt (reads from stdin if not provided) + \\ --seqlen= Maximum sequence length (default: 512) + \\ --backend= Attention backend to use ([vanilla, cuda_fa2, cuda_fa3], default: auto-selection) + \\ --reader-buffer-size= Reader buffer size (default: 32MiB) + \\ --writer-buffer-size= Writer buffer size (default: 32MiB) + \\ --async-limit= Async I/O concurrency limit + \\ -h, --help Show this help message + \\ + \\Examples: + \\ llama --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="Hello, world!" + \\ llama --model=$(realpath ../Llama-3.1-8B-Instruct) --seqlen=1024 + \\ echo "What is 2+2?" | llama --model=meta-llama/Llama-3.1-8B-Instruct + \\ + ; +}; + +pub fn main(init: std.process.Init) !void { + const allocator = init.gpa; + const arena = init.arena; + + log.info("LLama was compiled with {}", .{@import("builtin").mode}); + + const args = stdx.flags.parse(init.minimal.args, CliArgs); + + // var vfs: zml.io.VFS = try .init(allocator, init.io); + // defer vfs.deinit(); + + // var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); + // defer vfs_file.deinit(); + // try vfs.register("file", vfs_file.io()); + + // var vfs_https: zml.io.VFS.HTTP = try .init(allocator, init.io, &http_client, .https); + // defer vfs_https.deinit(); + // try vfs.register("https", vfs_https.io()); + + // var http_client: std.http.Client = .{ + // .allocator = allocator, + // .io = init.io, + // }; + // try http_client.initDefaultProxies(arena.allocator(), init.environ_map); + // defer http_client.deinit(); + + // var hf_vfs: zml.io.VFS.HF = try .auto( + // allocator, + // init.io, + // &http_client, + // ); + // defer hf_vfs.deinit(); + // try vfs.register("hf", hf_vfs.io()); + + const io = init.io; + + log.info("Resolving model repo", .{}); + const repo = try zml.safetensors.resolveModelRepo(io, args.model); + + const parsed_config = try parseConfig(allocator, io, repo); + defer parsed_config.deinit(); + const config = parsed_config.value; + + var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); + defer registry.deinit(); + + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); + defer store.deinit(); + + // Write metadata from the config file into the LlamaLm struct. + const llama_options: llama.LlamaLM.Options = .{ + .max_seq_len = args.seqlen, + .sampling_strategy = .{ + .topk = 1, + .temperature = 1.0, + }, + }; + + var platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + log.info("\n{f}", .{platform.fmtVerbose()}); + // log.info("Devices:", .{}); + // for (platform.devices) |device| { + // log.info("\t- {f}", .{device}); + // } + + // Initialize the Llama struct and map the content of the .safetensors to the model tensors. + var llama_model: llama.LlamaLM = try .init(allocator, store.view(), config, llama_options); + defer llama_model.deinit(allocator); + + const backend = args.backend orelse b: { + const selected = zml.attention.attention.Backend.auto(platform); + log.info("Selected backend: {}", .{selected}); + break :b selected; + }; + + // Specify shapes of input arguments + const dtype = llama_model.model.embed_tokens.weight.dtype(); + const llama_parameters: LlamaParameters = .{ + .prefill_tokens = .init(.{ .s = llama_options.max_seq_len }, .u32), + .decode_tokens = .init(.{ .s = 1 }, .u32), + .token_index = .init(.{}, .u32), + .kv_cache = .init(.init(.{ + .layer = llama_model.model.layers.len, + .k = args.seqlen, + .h = config.num_key_value_heads, + .hd = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads), + }, dtype)), + .rng = .init(), + .attention_metadata = .init(.fromBackend(backend, @intCast(args.seqlen))), + .attention_parameters = .init(.fromBackend(backend)), + }; + + var progress = std.Progress.start(io, .{ .root_name = args.model }); + + var tokenizer_future = try io.concurrent(loadTokenizer, .{ allocator, io, repo, &progress }); + errdefer blk: { + var v = tokenizer_future.cancel(io) catch break :blk; + v.deinit(); + } + + var compiled_model_result_future = try io.concurrent(compileModel, .{ allocator, io, platform, llama_model, llama_parameters, &progress }); + errdefer if (compiled_model_result_future.cancel(io)) |v| { + defer v.prefill_exe.deinit(); + defer v.decode_exe.deinit(); + } else |_| {}; + + var load_group: stdx.Io.LimitedGroup = .init(16); + defer load_group.cancel(io); + + var llama_buffers_future = try io.concurrent(LlamaLM.load, .{ + &llama_model, + arena.allocator(), + io, + platform, + &store, + &progress, + }); + errdefer b: { + var v = llama_buffers_future.cancel(io) catch break :b; + v = v; // autofix + // LlamaLM.unloadBuffers(&v, allocator); + } + + const compiled_model_result = try compiled_model_result_future.await(io); + defer compiled_model_result.prefill_exe.deinit(); + defer compiled_model_result.decode_exe.deinit(); + + var llama_buffers = try llama_buffers_future.await(io); + llama_buffers = llama_buffers; // autofix + // defer LlamaLM.unloadBuffers(&llama_buffers, allocator); + progress.end(); + + log.info("Creating KvCache", .{}); + var kv_cache_buffers = try llama_parameters.kv_cache.initBuffer(io, platform); + defer llama.KvCache.deinitBuffer(&kv_cache_buffers); + + var attention_metadata_buffers: zml.Bufferized(zml.attention.attention.Metadata) = try llama_parameters.attention_metadata.initBuffer(io, platform); + defer zml.attention.attention.Metadata.deinitBuffer(&attention_metadata_buffers); + + var tokenizer = try tokenizer_future.await(io); + defer tokenizer.deinit(); + + const prompt = if (args.prompt) |p| p else blk: { + var reader = std.Io.File.stdin().reader(io, &.{}); + break :blk try reader.interface.allocRemaining(allocator, .unlimited); + }; + + log.info("✅ Prompt: {s}", .{prompt}); + + // Unbuffered writing of the tokens to stdout. + var stdout = std.Io.File.stdout().writer(io, &.{}); + + try generateText( + allocator, + io, + &llama_buffers, + compiled_model_result.prefill_exe, + compiled_model_result.decode_exe, + &kv_cache_buffers, + &attention_metadata_buffers, + tokenizer, + config, + llama_options, + @intCast(std.Io.Clock.now(.awake, io).toNanoseconds()), + prompt[0..], + false, + &stdout.interface, + platform, + ); +} + +fn parseConfig(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(LlamaLM.Config) { + const now: std.Io.Timestamp = .now(io, .awake); + log.info("Loading model config", .{}); + defer log.info("Loaded model config [{D}]", .{ + stdx.fmt.fmtDuration(now.untilNow(io, .awake)), + }); + + const parsed_config = blk: { + const config_json_file = try dir.openFile(io, "config.json", .{}); + defer config_json_file.close(io); + var config_json_buffer: [256]u8 = undefined; + var config_reader = config_json_file.reader(io, &config_json_buffer); + var reader: std.json.Reader = .init(allocator, &config_reader.interface); + defer reader.deinit(); + break :blk try std.json.parseFromTokenSource(llama.LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); + }; + errdefer parsed_config.deinit(); + + return parsed_config; +} + +fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { + // log.info("Loading tokenizer", .{}); + progress.increaseEstimatedTotalItems(1); + var node = progress.start("Loading tokenizer...", 1); + defer node.end(); + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Loaded tokenizer [{D}]", .{ + stdx.fmt.fmtDuration(now.untilNow(io, .awake)), + }); + const bytes = b: { + const file = try dir.openFile(io, "tokenizer.json", .{}); + defer file.close(io); + var reader = file.reader(io, &.{}); + break :b try reader.interface.readAlloc(allocator, try file.length(io)); + }; + defer allocator.free(bytes); + + return try .fromBytes(allocator, io, bytes); +} + +const LlamaParameters = struct { + prefill_tokens: zml.Tensor, + decode_tokens: zml.Tensor, + token_index: zml.Tensor, + kv_cache: llama.KvCache, + rng: zml.Tensor.Rng, + attention_metadata: zml.attention.attention.Metadata, + attention_parameters: zml.attention.attention.Parameters, +}; + +const CompileModelResult = struct { + prefill_exe: zml.Exe, + decode_exe: zml.Exe, +}; + +fn compileModel( + allocator: std.mem.Allocator, + io: std.Io, + platform: *const zml.Platform, + llama_model: llama.LlamaLM, + parameters: LlamaParameters, + progress: *std.Progress.Node, +) !CompileModelResult { + const now: std.Io.Timestamp = .now(io, .awake); + defer log.info("Compiled model [{D}]", .{stdx.fmt.fmtDuration(now.untilNow(io, .awake))}); + + // Compile the model twice, one for prefill, one for generation. + var prefill_future = try io.concurrent(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling prefill...", 1); + defer node_.end(); + const now_: std.Io.Timestamp = .now(io_, .awake); + defer log.info("Compiled prefill [{D}]", .{stdx.fmt.fmtDuration(now_.untilNow(io_, .awake))}); + return platform_.compile(allocator_, io_, llama_model_, .forward, .{ + parameters_.prefill_tokens, + parameters_.token_index, + parameters_.kv_cache, + parameters_.rng, + parameters_.attention_metadata, + parameters_.attention_parameters, + }); + } + }.call, .{ allocator, io, platform, llama_model, parameters, progress }); + errdefer if (prefill_future.cancel(io)) |v| v.deinit() else |_| {}; + + var decode_future = try io.concurrent(struct { + fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const zml.Platform, llama_model_: llama.LlamaLM, parameters_: LlamaParameters, progress_: *std.Progress.Node) !zml.Exe { + progress_.increaseEstimatedTotalItems(1); + var node_ = progress_.start("Compiling decode...", 1); + defer node_.end(); + const now_: std.Io.Timestamp = .now(io_, .awake); + defer log.info("Compiled decode [{D}]", .{stdx.fmt.fmtDuration(now_.untilNow(io_, .awake))}); + return platform_.compile(allocator_, io_, llama_model_, .forward, .{ + parameters_.decode_tokens, + parameters_.token_index, + parameters_.kv_cache, + parameters_.rng, + parameters_.attention_metadata, + parameters_.attention_parameters, + }); + } + }.call, .{ allocator, io, platform, llama_model, parameters, progress }); + errdefer if (decode_future.cancel(io)) |v| v.deinit() else |_| {}; + + const prefill_exe = try prefill_future.await(io); + const decode_exe = try decode_future.await(io); + + return .{ .prefill_exe = prefill_exe, .decode_exe = decode_exe }; +} + +fn loadModelBuffers( + allocator: std.mem.Allocator, + buffers_allocator: std.mem.Allocator, + io: std.Io, + progress: *std.Progress.Node, + platform: *const zml.Platform, + store: *zml.io.TensorStore, + llama_model: llama.LlamaLM, + pool: *zml.mem.DynamicBufferPool, + async_limit: usize, +) !zml.Bufferized(llama.LlamaLM) { + var transferred_bytes: usize = 0; + + const now: std.Io.Timestamp = .now(io, .awake); + defer { + const duration = now.untilNow(io, .awake); + const seconds = @as(f64, @floatFromInt(duration.ns)) / 1e9; + const gb_per_sec = @as(f64, @floatFromInt(transferred_bytes)) / (1024.0 * 1024.0 * 1024.0) / seconds; + const gbps = gb_per_sec * 8.0; + log.info("Loaded model [{D} {d:.3} GB/s {d:.3} gbps]", .{ stdx.fmt.fmtDuration(duration), gb_per_sec, gbps }); + } + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + var group: stdx.Io.LimitedGroup = .init(async_limit); + defer group.cancel(io); + + const bufferize_ctx: zml.io.BufferizeContext(llama.TransferCtx) = .{ + .allocator = allocator, + .arena = &arena, + .io = io, + .platform = platform, + .cb_ctx = .{ + .allocator = buffers_allocator, + .pool = pool, + .transferred_bytes = &transferred_bytes, + .progress = progress, + }, + }; + + const bufferized = llama_model.loadBuffers(bufferize_ctx, &group, store.view(), transferBuffer); + try group.await(io); + + return bufferized; +} + +fn transferBuffer(ctx: zml.io.TensorBufferTransfer(llama.TransferCtx)) !void { + var transfer_progress = ctx.cb_ctx.progress.start(ctx.tensor.name, ctx.tensor.byteSize()); + + var reader = zml.safetensors.TensorReader.init(ctx.io, ctx.tensor, &.{}) catch unreachable; + defer reader.deinit(); + + var writer = zml.io.DeviceWriter2.init(ctx.cb_ctx.allocator, ctx.io, ctx.platform, ctx.cb_ctx.pool, ctx.tensor.shape, ctx.buffer, .device) catch unreachable; + defer { + writer.interface.flush() catch unreachable; + writer.deinit(); + } + + _ = reader.interface.streamRemaining(&writer.interface) catch unreachable; + writer.interface.flush() catch unreachable; + + transfer_progress.end(); + + ctx.cb_ctx.transferred_bytes.* += ctx.tensor.byteSize(); +} + +pub fn tokenizePrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, config: LlamaLM.Config, prompt: []const u8, skip_llama3_encoding: bool) ![]u32 { + var encoder = try tokenizer.encoder(); + defer encoder.deinit(); + + if (skip_llama3_encoding) { + // Copy so the ownership is the same in both branches. + return try allocator.dupe(u32, try encoder.encode(prompt)); + } + + const start_header = tokenizer.tokenToId("<|start_header_id|>") orelse return error.NoSuchToken; + const end_header = tokenizer.tokenToId("<|end_header_id|>") orelse return error.NoSuchToken; + const user = tokenizer.tokenToId("user") orelse return error.NoSuchToken; + const assistant = tokenizer.tokenToId("assistant") orelse return error.NoSuchToken; + const eot = tokenizer.tokenToId("<|eot_id|>") orelse return error.NoSuchToken; + const newline = (try encoder.encode("\n"))[0]; + + var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); + try tokens.appendSlice(allocator, &.{ config.bos_token_id, start_header, user, end_header, newline }); + + try tokens.appendSlice(allocator, try encoder.encode(prompt)); + try tokens.appendSlice(allocator, &.{ eot, newline }); + + try tokens.appendSlice(allocator, &.{ start_header, assistant, end_header, newline }); + + return tokens.toOwnedSlice(allocator); +} + +pub fn generateText( + allocator: std.mem.Allocator, + io: std.Io, + llama_buffers: *zml.Bufferized(LlamaLM), + prefill_exe: zml.exe.Exe, + decode_exe: zml.exe.Exe, + kv_cache_buffers: *zml.Bufferized(llama.KvCache), + attention_metadata_buffers: *zml.Bufferized(zml.attention.attention.Metadata), + tokenizer: zml.tokenizer.Tokenizer, + config: LlamaLM.Config, + options: LlamaLM.Options, + seed: u128, + prompt: []const u8, + skip_llama3_encoding: bool, + writer: *std.Io.Writer, + platform: *const zml.Platform, +) !void { + const prompt_tok: []const u32 = try tokenizePrompt(allocator, tokenizer, config, prompt, skip_llama3_encoding); + defer allocator.free(prompt_tok); + + var tokenizer_decoder = try tokenizer.decoder(); + defer tokenizer_decoder.deinit(); + + const max_seq_len = options.max_seq_len; + + // init RNG and buffers + var rng_buffers = try zml.Tensor.Rng.initBuffer(platform, seed, io); + defer zml.Tensor.Rng.deinitBuffer(&rng_buffers); + var generated_token_slice: zml.Slice = try .alloc(allocator, zml.Shape.init(.{ .s = 1 }, .u32)); + defer generated_token_slice.free(allocator); + + { + var prefill_args = try prefill_exe.args(allocator); + defer prefill_args.deinit(allocator); + + var prefill_results = try prefill_exe.results(allocator); + defer prefill_results.deinit(allocator); + + // prepare device buffers for the prefill tokens and their positions + const prefill_tokens_slice: zml.Slice = try .alloc(allocator, .init(.{max_seq_len}, .u32)); + defer prefill_tokens_slice.free(allocator); + @memcpy(prefill_tokens_slice.items(u32)[0..prompt_tok.len], prompt_tok); + + var prefill_tokens_buffer: zml.Buffer = try .fromSlice(io, platform, prefill_tokens_slice); + defer prefill_tokens_buffer.deinit(); + var prefill_token_pos_buffer = try zml.Buffer.scalar(io, platform, 0, .u32); + defer prefill_token_pos_buffer.deinit(); + + prefill_args.set(.{ llama_buffers, prefill_tokens_buffer, prefill_token_pos_buffer, kv_cache_buffers, rng_buffers, attention_metadata_buffers }); + + prefill_exe.call(prefill_args, &prefill_results); + + prefill_results.fill(.{ &prefill_tokens_buffer, kv_cache_buffers, &rng_buffers }); + try prefill_tokens_buffer.toSlice(io, prefill_tokens_slice); + generated_token_slice.items(u32)[0] = prefill_tokens_slice.items(u32)[prompt_tok.len - 1]; + + std.log.info("\n\ngenerated tokens: {any}\n\n", .{prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]}); + const out = try tokenizer_decoder.decode(prefill_tokens_slice.constItems(u32)[0..prompt_tok.len]); + std.log.info("\n\ngenerated: {s}\n\n", .{out}); + } + + // Prepare for token-by-token generation, + + var decode_args = try decode_exe.args(allocator); + defer decode_args.deinit(allocator); + + var decode_results = try decode_exe.results(allocator); + defer decode_results.deinit(allocator); + + // start with the token generated based on the full prompt. + var current_token_buffer: zml.Buffer = try .fromSlice(io, platform, generated_token_slice); + defer current_token_buffer.deinit(); + + const output_tokens_len = max_seq_len - prompt_tok.len - 1; + const now: std.Io.Timestamp = .now(io, .awake); + + // One token has alreadyh been generated by the prefill. + var num_tokens_generated: usize = 1; + + generation: for (0..output_tokens_len + 1) |i| { + // collect and print generated sequence + num_tokens_generated += 1; + const generated_token = generated_token_slice.items(u32)[0]; + if (try tokenizer_decoder.next(generated_token)) |chunk| { + try writer.writeAll(chunk); + try writer.flush(); + } + + // check for eos + if (i == output_tokens_len) break :generation; + switch (config.eos_token_id.value) { + .int => |eos| if (generated_token == @as(u32, @intCast(eos))) break :generation, + .ints => |eos_list| { + for (eos_list) |eos| { + if (generated_token == @as(u32, @intCast(eos))) break :generation; + } + }, + } + + // current token pos needs to go into a zml.Buffer + const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(prompt_tok.len + i)})); + var token_pos_buffer: zml.Buffer = try .fromSlice(io, platform, token_pos_slice); + defer token_pos_buffer.deinit(); + + // call to generate the next token + decode_args.set(.{ llama_buffers, current_token_buffer, token_pos_buffer, kv_cache_buffers, rng_buffers, attention_metadata_buffers }); + + decode_exe.call(decode_args, &decode_results); + + decode_results.fill(.{ ¤t_token_buffer, kv_cache_buffers, &rng_buffers }); + + // extract the generated token from the buffer + try current_token_buffer.toSlice(io, generated_token_slice); + } + const duration = now.untilNow(io, .awake); + std.debug.print("\n", .{}); + log.info("✅ Generated {} tokens in {D}: {:.3}tok/s", .{ + num_tokens_generated, + stdx.fmt.fmtDuration(duration), + stdx.Io.Duration.hzFloat(stdx.Io.Duration.div(duration, num_tokens_generated)), + }); +} diff --git a/examples/llama/test.zig b/examples/llama/test.zig new file mode 100644 index 00000000..6c4ebc84 --- /dev/null +++ b/examples/llama/test.zig @@ -0,0 +1,119 @@ +const std = @import("std"); + +const async = @import("async"); +const zml = @import("zml"); +const Tensor = zml.Tensor; +const stdx = zml.stdx; +const flags = stdx.flags; + +const llama_mod = @import("./llama.zig"); +const LlamaLM = llama_mod.LlamaLM; + +pub fn main() !void { + try async.AsyncThread.main(std.heap.c_allocator, asyncMain); +} + +pub fn asyncMain() !void { + const CliArgs = struct { + pub const help = + \\ test-implementation --weights=llama3.8B.safetensors --config=config.json --reference=activation.safetensors + ; + weights: []const u8, + config: []const u8, + reference: []const u8, + num_heads: ?i64 = null, + num_kv_heads: ?i64 = null, + rope_freq_base: ?i64 = null, + }; + var gpa = std.heap.GeneralPurposeAllocator(.{ .thread_safe = true }){}; + defer _ = gpa.deinit(); + const allocator = gpa.allocator(); + + // Create ZML context + var context = try zml.Context.init(); + defer context.deinit(); + + // Select platform + const platform = context.autoPlatform(.{}); + + // Parse program args + var args = std.process.args(); + const cli_args = flags.parse(&args, CliArgs); + const model_file = cli_args.weights; + + // Memory arena dedicated to model shapes and weights + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const model_arena = arena_state.allocator(); + + std.log.info("Model file: {s}", .{model_file}); + + // Read model shapes. + var buffer_store = try zml.aio.detectFormatAndOpen(allocator, model_file); + defer buffer_store.deinit(); + + // Create the model and configure it. + var llama = try zml.aio.populateModel(LlamaLM, model_arena, buffer_store); + const num_heads: i64 = cli_args.num_heads orelse buffer_store.metadata("num_heads", .int) orelse @panic("--num_heads is required for this model"); + const num_kv_heads: i64 = cli_args.num_kv_heads orelse buffer_store.metadata("num_kv_heads", .int) orelse num_heads; + + const config = blk: { + var config_json_file = try async.File.open(cli_args.config, .{ .mode = .read_only }); + defer config_json_file.close() catch unreachable; + var reader = std.json.reader(allocator, config_json_file.reader()); + defer reader.deinit(); + const config_obj = try std.json.parseFromTokenSourceLeaky(LlamaLM.Config, allocator, &reader, .{ .ignore_unknown_fields = true }); + break :blk config_obj; + }; + std.log.info("Parsed llama config: {}", .{config}); + + const llama_config: LlamaLM.Config = .{ + .eos_token_id = config.eos_token_id, + .bos_token_id = config.bos_token_id, + .num_key_value_heads = @intCast(num_kv_heads), + .num_hidden_layers = @intCast(config.num_hidden_layers), + .num_attention_heads = @intCast(num_heads), + .max_position_embeddings = config.max_position_embeddings, + .rope_theta = config.rope_theta, + .rms_norm_eps = @floatCast(buffer_store.metadata("rms_norm_eps", .float) orelse 1e-5), + .hf_rope_impl = true, + }; + + const llama_options: LlamaLM.Options = .{ + .max_seq_len = 256, + .sampling_strategy = .{ + .topk = 1, + .temperature = 1.0, + }, + }; + + std.log.info("Parsed llama config: {}", .{llama_options}); + llama.init(llama_config, llama_options); + + // Load the weights. + var llama_weights = try zml.aio.loadBuffers(LlamaLM, .{ llama_config, llama_options }, buffer_store, model_arena, platform); + defer zml.aio.unloadBuffers(&llama_weights); + + // Load the activations. + var activation_buffer_store = try zml.aio.torch.open(allocator, cli_args.reference); + defer activation_buffer_store.deinit(); + + // Test implementation + try testImplementation(platform, llama, llama_weights, activation_buffer_store); +} + +fn testImplementation( + platform: zml.Platform, + llama: LlamaLM, + llama_weights: zml.Bufferized(LlamaLM), + buffer_store: zml.aio.BufferStore, +) !void { + try zml.testing.testLayer(platform, buffer_store, "embed_tokens", llama.model.embed_tokens, llama_weights.model.embed_tokens, 1e-3); + try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.v_proj", llama.model.layers[0].self_attn.v_proj, llama_weights.model.layers[0].self_attn.v_proj, 1e-2); + try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.q_proj", llama.model.layers[0].self_attn.q_proj, llama_weights.model.layers[0].self_attn.q_proj, 2e-2); + try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.k_proj", llama.model.layers[0].self_attn.k_proj, llama_weights.model.layers[0].self_attn.k_proj, 2e-2); + try zml.testing.testLayer(platform, buffer_store, "layers.0.self_attn.o_proj", llama.model.layers[0].self_attn.o_proj, llama_weights.model.layers[0].self_attn.o_proj, 2e-2); + try zml.testing.testLayer(platform, buffer_store, "layers.0.mlp", llama.model.layers[0].mlp, llama_weights.model.layers[0].mlp, 1e-2); + try zml.testing.testLayer(platform, buffer_store, "layers.0.input_layernorm", llama.model.layers[0].input_layernorm, llama_weights.model.layers[0].input_layernorm, 1e-2); + try zml.testing.testLayer(platform, buffer_store, "layers.0.post_attention_layernorm", llama.model.layers[0].post_attention_layernorm, llama_weights.model.layers[0].post_attention_layernorm, 1e-2); +} diff --git a/examples/llm/BUILD.bazel b/examples/llm/BUILD.bazel deleted file mode 100644 index 6a501aea..00000000 --- a/examples/llm/BUILD.bazel +++ /dev/null @@ -1,117 +0,0 @@ -load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") -load("@rules_zig//zig:defs.bzl", "zig_binary") -load("@tar.bzl//tar:mtree.bzl", "mtree_spec") -load("@tar.bzl//tar:tar.bzl", "tar") - -zig_binary( - name = "llm", - srcs = glob(["**/*.zig"]), - main = "main.zig", - visibility = ["//visibility:public"], - deps = ["//zml", "@linenoise"], -) - -zig_binary( - name = "lfm2_tests", - srcs = glob(["**/*.zig"]), - main = "models/lfm2_tests.zig", - visibility = ["//visibility:public"], - deps = ["//zml"], -) - -zig_binary( - name = "llama_tests", - srcs = glob(["**/*.zig"]), - main = "models/llama_tests.zig", - visibility = ["//visibility:public"], - deps = ["//zml"], -) - -build_test( - name = "test", - targets = [":llm"], -) - -mtree_spec( - name = "mtree", - srcs = [":llm"], - tags = ["manual"] -) - -tar( - name = "archive", - srcs = [":llm"], - args = [ - "--options", - ",".join([ - "zstd:compression-level=9", - "zstd:threads=16", - ]), - ], - compress = "zstd", - mtree = ":mtree", - tags = ["manual"] -) - -oci_image( - name = "image_", - base = "@distroless_cc_debian12", - entrypoint = ["./{}/llm".format(package_name())], - target_compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], - tars = [":archive"], - tags = ["manual"] -) - -platform_transition_filegroup( - name = "image", - srcs = [":image_"], - tags = ["manual"], - target_platform = "//platforms:linux_amd64", -) - -oci_load( - name = "load", - image = ":image", - repo_tags = ["zmlai/llm:latest"], - tags = ["manual"], -) - -oci_push( - name = "push", - image = ":image", - remote_tags = ["latest"], - repository = "index.docker.io/zmlai/llm", - tags = ["manual"], -) - - -oci_image( - name = "image_debug_", - base = "@debian_12", - entrypoint = ["./examples/llm/llm"], - target_compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], - tars = [":archive"], - tags = ["manual"], -) - -platform_transition_filegroup( - name = "image_debug", - srcs = [":image_debug_"], - tags = ["manual"], - target_platform = "//platforms:linux_amd64", -) - -oci_load( - name = "load_debug", - image = ":image_debug", - repo_tags = ["zmlai/llm:debug"], - tags = ["manual"], -) diff --git a/examples/llm/README.md b/examples/llm/README.md deleted file mode 100644 index fcd061ba..00000000 --- a/examples/llm/README.md +++ /dev/null @@ -1,41 +0,0 @@ -# LLM - -`//examples/llm` runs interactive or one-shot text generation from a model repository from HuggingFace. - -We support the following models, automatically detected from the `model_type` in the `config.json`: - -- Llama 3.1 -- Qwen 3.5 -- LFM 2.5 - -## Run - -To load a model from HuggingFace directly: - -```bash -# CPU -bazel run //examples/llm -- --model=hf://meta-llama/Llama-3.1-8B-Instruct -# CUDA -bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct -# ROCm -bazel run //examples/llm --@zml//platforms:rocm=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct -``` - -From a local directory: - -```bash -bazel run //examples/llm --@zml//platforms:cuda=true -- --model=/var/models/meta-llama/Llama-3.1-8B-Instruct/ -``` - -For a single non-interactive prompt: - -```bash -bazel run //examples/llm --@zml//platforms:cuda=true -- --model=hf://meta-llama/Llama-3.1-8B-Instruct --prompt="Write a haiku about Zig" -``` - -## Options - -- `--model=`: Required. Model repository to load. This can be a local path or a huggingface/S3 URI such as `hf://...` or `s3://...`. -- `--prompt=`: Optional. Runs a single prompt instead of opening the interactive chat loop. -- `--seqlen=`: Optional. Maximum sequence length. Defaults to `2048`. -- `--backend=`: Optional. Attention backend. `cute` uses CuTe flash attention on CUDA GPUs with compute capability 8.0 or newer. If omitted, the program auto-selects one for the current platform, including `cute` on supported CUDA GPUs. diff --git a/examples/llm/chat.zig b/examples/llm/chat.zig deleted file mode 100644 index c02efd56..00000000 --- a/examples/llm/chat.zig +++ /dev/null @@ -1,176 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const zml = @import("zml"); - -const models = @import("models.zig"); - -const log = std.log.scoped(.llm); - -pub const prompt_prefix: [:0]const u8 = "> "; -pub const history_file_path: [:0]const u8 = ".llm_chat_history"; - -pub fn initHistory() void { - _ = c.linenoiseHistorySetMaxLen(1_000); - _ = c.linenoiseHistoryLoad(history_file_path.ptr); -} - -pub fn rememberPrompt(line: [*:0]const u8) void { - if (line[0] == 0) return; - if (c.linenoiseHistoryAdd(line) == 1) { - _ = c.linenoiseHistorySave(history_file_path.ptr); - } -} - -pub const Chat = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - session: models.Session, - tokens: std.ArrayList(u32), - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - tokenizer: zml.tokenizer.Tokenizer, - compiled_model: *models.CompiledModel, - model_buffers: *models.Buffers, - ) !Chat { - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, compiled_model.seqlen); - errdefer tokens.deinit(allocator); - - const session = try compiled_model.newSession( - allocator, - io, - platform, - model_buffers, - tokenizer, - ); - - return .{ - .allocator = allocator, - .io = io, - .platform = platform, - .session = session, - .tokens = tokens, - }; - } - - pub fn deinit(self: *Chat) void { - self.tokens.deinit(self.allocator); - self.session.deinit(); - } - - pub const RunOnceOptions = struct { - profile: bool = false, - }; - - pub fn runOnce(self: *Chat, prompt: []const u8, opts: RunOnceOptions) !void { - const prompt_tokens = try self.session.tokenizePrompt(self.allocator, prompt); - defer self.allocator.free(prompt_tokens); - - var stdout = std.Io.File.stdout().writerStreaming(self.io, &.{}); - var profiler: ?zml.Platform.Profiler = null; - defer if (profiler) |*p| p.deinit(); - - if (opts.profile) { - var session_id_buf: [64]u8 = undefined; - const session_id = try std.fmt.bufPrint( - &session_id_buf, - "llm-runonce-{d}", - .{@as(u64, @intCast(std.Io.Clock.now(.real, self.io).toNanoseconds()))}, - ); - profiler = try self.platform.profiler(self.allocator, self.io, .{ - .session_id = session_id, - }); - try profiler.?.start(); - } - - try self.runPrefill(prompt_tokens, &stdout.interface); - try self.runDecodeTurn(&stdout.interface); - _ = if (profiler) |*p| try p.stop() else null; - - try stdout.interface.flush(); - } - - pub fn runInteractive(self: *Chat, initial_prompt: []const u8) !void { - var turn_tokens = try self.session.tokenizePrompt(self.allocator, initial_prompt); - defer self.allocator.free(turn_tokens); - - var stdout = std.Io.File.stdout().writerStreaming(self.io, &.{}); - - while (self.tokens.items.len <= self.session.maxTokens()) { - try self.runPrefill(turn_tokens, &stdout.interface); - try self.runDecodeTurn(&stdout.interface); - - turn_tokens = if (self.tokens.items.len >= self.session.maxTokens()) b: { - self.allocator.free(turn_tokens); - break :b try self.allocator.alloc(u32, 0); - } else try self.readNextTurn(turn_tokens, &stdout.interface) orelse return; - if (self.tokens.items.len + turn_tokens.len >= self.session.maxTokens()) { - log.warn("Not enough tokens remaining ({} available, {} needed). Consider using a higher seqlen, for example: `--seqlen={d}`.", .{ - self.session.maxTokens() -| self.tokens.items.len, - turn_tokens.len, - std.math.ceilPowerOfTwo(u32, self.session.maxTokens() + 1) catch std.math.maxInt(u32), - }); - return; - } - } - } - - fn runPrefill(self: *Chat, prompt_tokens: []const u32, stdout: *std.Io.Writer) !void { - if (prompt_tokens.len + self.tokens.items.len > self.session.maxTokens()) { - return error.PromptTooLong; - } - try stdout.writeAll("\x1b[2mprefill...\x1b[0m"); - try stdout.flush(); - - try self.tokens.appendSlice(self.allocator, prompt_tokens); - try self.session.runPrefill(self.tokens.items); - - try stdout.writeAll("\r \r"); - try stdout.flush(); - } - - fn runDecodeTurn(self: *Chat, stdout: *std.Io.Writer) !void { - const decode_start: std.Io.Timestamp = .now(self.io, .awake); - const decode_pos_before = self.tokens.items.len; - try self.session.runDecode(&self.tokens, stdout); - const decode_duration = decode_start.untilNow(self.io, .awake); - const decode_tokens: u64 = self.tokens.items.len - decode_pos_before; - - try stdout.writeAll("\n\n"); - try stdout.print("\x1b[36mdecode\x1b[0m \x1b[2m{f} · {:.1}tok/s\x1b[0m\n\n\n", .{ - decode_duration, - tokensPerSecond(decode_duration, decode_tokens), - }); - - try stdout.flush(); - } - - fn readNextTurn(self: *Chat, previous_turn_tokens: []const u32, stdout: *std.Io.Writer) !?[]const u32 { - while (true) { - const line = c.linenoise(prompt_prefix.ptr) orelse { - try stdout.print("\x1b[2m{}/{} tokens used\x1b[0m\n\n", .{ self.tokens.items.len, self.session.maxTokens() }); - try stdout.flush(); - return null; - }; - defer c.linenoiseFree(line); - - rememberPrompt(line); - const input = std.mem.sliceTo(line, 0); - if (input.len == 0) continue; - - self.allocator.free(previous_turn_tokens); - return try self.session.tokenizeTurn(self.allocator, input); - } - } -}; - -fn tokensPerSecond(duration: std.Io.Duration, tokens: u64) f64 { - if (tokens == 0) return 0; - const seconds = @as(f64, @floatFromInt(duration.toNanoseconds())) / 1e9; - if (seconds <= 0) return 0; - return @as(f64, @floatFromInt(tokens)) / seconds; -} diff --git a/examples/llm/main.zig b/examples/llm/main.zig deleted file mode 100644 index b24c022a..00000000 --- a/examples/llm/main.zig +++ /dev/null @@ -1,205 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const zml = @import("zml"); -const stdx = zml.stdx; - -const chat = @import("chat.zig"); -const models = @import("models.zig"); - -pub const std_options: std.Options = .{ - .log_level = .info, -}; - -const log = std.log.scoped(.llm); - -const Args = struct { - model: []const u8, - prompt: ?[]const u8 = null, - seqlen: u32 = 2048, - topk: u32 = 4, - backend: ?enum { stablehlo, cute, nki, metal } = null, - profile: bool = false, - - pub const help = - \\ Use llm --model= [options] - \\ - \\ Run text generation with a model selected from `model_type` in the `config.json`. - \\ - \\ Options: - \\ --model= Path to the model repository (required) - \\ --prompt= Prompt to use for generation (default: none) - \\ --seqlen= Sequence length (default: 2048) - \\ --topk= Top-k sampling cutoff (default: 4) - \\ --backend= Attention backend to use ([stablehlo, cute, nki, metal], default: auto-selection) - \\ --profile Capture a PJRT profile for non-interactive runs and write a Perfetto trace - \\ - ; -}; - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - - // `bazel run` executes binaries from Bazel's runfiles tree by default. - // If available, switch back to the shell's original working directory. - if (init.environ_map.get("BUILD_WORKING_DIRECTORY")) |build_working_directory| { - var working_dir = try std.Io.Dir.openDirAbsolute(init.io, build_working_directory, .{}); - defer working_dir.close(init.io); - try std.process.setCurrentDir(init.io, working_dir); - } - - const args = stdx.flags.parse(init.minimal.args, Args); - - // - // Virtual File Systems - // - var vfs_file: zml.io.VFS.File = .init(allocator, init.io, .{}); - defer vfs_file.deinit(); - - var http_client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - defer http_client.deinit(); - - var hf_vfs: zml.io.VFS.HF = try .auto(allocator, init.io, &http_client, init.environ_map); - defer hf_vfs.deinit(); - - var s3_vfs: zml.io.VFS.S3 = try .auto(allocator, init.io, &http_client, init.environ_map); - defer s3_vfs.deinit(); - - var gcs_vfs: zml.io.VFS.GCS = try .auto(allocator, init.io, &http_client, init.environ_map); - defer gcs_vfs.deinit(); - - var vfs: zml.io.VFS = try .init(allocator, init.io); - defer vfs.deinit(); - - try vfs.register("file", vfs_file.io()); - try vfs.register("gs", gcs_vfs.io()); - try vfs.register("hf", hf_vfs.io()); - try vfs.register("s3", s3_vfs.io()); - - const io = vfs.io(); - - // - // Platform and Backend Selection - // - const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator, io); - - log.info("\n{f}", .{platform.fmtVerbose()}); - - const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { - .stablehlo => .{ .stablehlo = {} }, - .cute => .{ .flash_attention = .cute }, - .nki => .{ .flash_attention = .nki }, - .metal => .{ .flash_attention = .metal }, - } else zml.attention.Backend.auto(platform); - log.info("Selected backend: {}", .{backend}); - - // - // Model initialization - // - log.info("Resolving model repository..", .{}); - const repo = try zml.safetensors.resolveModelRepo(io, args.model); - defer repo.close(io); - - log.info("Initializing model..", .{}); - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - - // Defines how the model's tensors are sharded across the available devices. - const meshes: models.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); - defer store.deinit(); - - const generation: models.GenerationOptions = .{ - .sampling_strategy = .{ - .topk = args.topk, - }, - }; - - var model = try models.LoadedModel.load(allocator, io, repo, store.view(), generation); - defer model.deinit(allocator); - - // - // Load the model and compile it - // - var progress = std.Progress.start(io, .{ .root_name = args.model }); - errdefer progress.end(); - - var tokenizer = try loadTokenizer(allocator, io, repo, &progress); - defer tokenizer.deinit(); - - var compiled_model = try allocator.create(models.CompiledModel); - defer allocator.destroy(compiled_model); - compiled_model.* = try models.LoadedModel.compile(&model, allocator, io, platform, backend, meshes, args.seqlen, &progress); - defer compiled_model.deinit(); - - // Load buffers after the model compilation to be sure to give enough room to the autotune. - var model_buffers = try models.LoadedModel.loadBuffers(&model, allocator, io, platform, &store, &progress); - defer model.unloadBuffers(&model_buffers, allocator); - - progress.end(); - - // - // We're ready to go! Let's start a chat... - // - try printZmlLogo(io, init.environ_map); - - const interactive = args.prompt == null; - const prompt = if (args.prompt) |prompt_arg| b: { - break :b try allocator.dupe(u8, prompt_arg); - } else b: { - chat.initHistory(); - const line = c.linenoise(chat.prompt_prefix) orelse return; - defer c.linenoiseFree(line); - chat.rememberPrompt(line); - break :b try allocator.dupe(u8, std.mem.sliceTo(line, 0)); - }; - defer allocator.free(prompt); - - var llm_chat = try chat.Chat.init( - allocator, - io, - platform, - tokenizer, - compiled_model, - &model_buffers, - ); - defer llm_chat.deinit(); - - if (interactive) { - try llm_chat.runInteractive(prompt); - } else { - try llm_chat.runOnce(prompt, .{ - .profile = args.profile, - }); - } -} - -fn loadTokenizer(allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir, progress: *std.Progress.Node) !zml.tokenizer.Tokenizer { - progress.increaseEstimatedTotalItems(1); - var node = progress.start("Loading tokenizer...", 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Loaded tokenizer [{f}]", .{now.untilNow(io, .awake)}); - const bytes = b: { - const file = try dir.openFile(io, "tokenizer.json", .{}); - defer file.close(io); - var reader = file.reader(io, &.{}); - break :b try reader.interface.readAlloc(allocator, try file.length(io)); - }; - defer allocator.free(bytes); - - return try .fromBytes(allocator, bytes); -} - -pub fn printZmlLogo(io: std.Io, environ_map: *const std.process.Environ.Map) !void { - const stdout = std.Io.File.stdout(); - const logo_options: zml.logo.Options = .{ - .color_support = zml.logo.detectColorSupport(io, stdout, environ_map), - }; - var writer = stdout.writerStreaming(io, &.{}); - try writer.interface.writeAll("\n\n"); - try zml.logo.writeZmlArt(&writer.interface, logo_options); - try writer.interface.writeAll("\n\n\n"); - try writer.interface.flush(); -} diff --git a/examples/llm/models.zig b/examples/llm/models.zig deleted file mode 100644 index 20aa60c7..00000000 --- a/examples/llm/models.zig +++ /dev/null @@ -1,201 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const common = @import("models/common.zig"); -pub const Meshes = common.Meshes; -pub const GenerationOptions = common.GenerationOptions; -pub const parseConfig = common.parseConfig; -pub const lfm2 = @import("models/lfm2.zig"); -pub const llama = @import("models/llama.zig"); -pub const qwen3_5 = @import("models/qwen3_5.zig"); -pub const qwen3_5_moe = @import("models/qwen3_5_moe.zig"); - -const log = std.log.scoped(.llm); - -pub const ModelType = enum { - lfm2, - llama, - qwen3_5, - qwen3_5_moe, -}; - -const RawConfig = struct { - model_type: []const u8, -}; - -pub const LoadedModel = union(ModelType) { - lfm2: lfm2.LoadedModel, - llama: llama.LoadedModel, - qwen3_5: qwen3_5.LoadedModel, - qwen3_5_moe: qwen3_5_moe.LoadedModel, - - pub fn load( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - store: zml.io.TensorStore.View, - generation: GenerationOptions, - ) !LoadedModel { - const model_type = try detectModelType(allocator, io, repo); - log.info("Detected model type: {}", .{model_type}); - - return switch (model_type) { - inline else => |t| @unionInit( - LoadedModel, - @tagName(t), - try .init(allocator, io, repo, store, generation), - ), - }; - } - - pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { - switch (self.*) { - inline else => |*m| m.deinit(allocator), - } - } - - pub fn loadBuffers( - self: *LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - return switch (self.*) { - inline else => |*m, t| @unionInit( - Buffers, - @tagName(t), - try m.loadBuffers(allocator, io, platform, store, progress), - ), - }; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - switch (self.*) { - inline else => |*loaded, t| loaded.unloadBuffers(&@field(buffers, @tagName(t)), allocator), - } - } - - pub fn compile( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - backend: zml.attention.Backend, - meshes: Meshes, - seqlen: usize, - progress: *std.Progress.Node, - ) !CompiledModel { - const inner: CompiledModel.Inner = switch (self.*) { - inline else => |*m, t| @unionInit( - CompiledModel.Inner, - @tagName(t), - try m.compile(allocator, io, platform, backend, meshes, seqlen, progress), - ), - }; - return .{ - .inner = inner, - .seqlen = @intCast(seqlen), - }; - } -}; - -pub const CompiledModel = struct { - const Inner = union(ModelType) { - lfm2: lfm2.inference.CompiledModel, - llama: llama.inference.CompiledModel, - qwen3_5: qwen3_5.inference.CompiledModel, - qwen3_5_moe: qwen3_5_moe.inference.CompiledModel, - }; - - inner: Inner, - seqlen: u32, - - pub fn deinit(self: *CompiledModel) void { - switch (self.inner) { - inline else => |*b| b.deinit(), - } - } - - pub fn newSession( - self: *CompiledModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - model_buffers: *Buffers, - tokenizer: zml.tokenizer.Tokenizer, - ) !Session { - return switch (self.inner) { - inline else => |*compiled, t| .{ - .inner = @unionInit( - Session.Inner, - @tagName(t), - try .init(allocator, io, platform, tokenizer, compiled, &@field(model_buffers, @tagName(t))), - ), - .seqlen = self.seqlen, - }, - }; - } -}; - -pub const Buffers = union(ModelType) { - lfm2: lfm2.Buffers, - llama: llama.Buffers, - qwen3_5: qwen3_5.Buffers, - qwen3_5_moe: qwen3_5_moe.Buffers, -}; - -pub const Session = struct { - const Inner = union(ModelType) { - lfm2: lfm2.Session, - llama: llama.Session, - qwen3_5: qwen3_5.Session, - qwen3_5_moe: qwen3_5_moe.Session, - }; - - inner: Inner, - seqlen: u32, - - pub fn deinit(self: *Session) void { - switch (self.inner) { - inline else => |*s| s.deinit(), - } - } - - pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { - try switch (self.inner) { - inline else => |*s| s.runPrefill(all_tokens), - }; - } - - pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), writer: *std.Io.Writer) !void { - try switch (self.inner) { - inline else => |*s| s.runDecode(all_tokens, writer), - }; - } - - pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { - return switch (self.inner) { - inline else => |*s| s.tokenizePrompt(allocator, prompt), - }; - } - - pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { - return switch (self.inner) { - inline else => |*s| s.tokenizeTurn(allocator, prompt), - }; - } - - pub fn maxTokens(self: *const Session) u32 { - return self.seqlen; - } -}; - -pub fn detectModelType(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !ModelType { - const parsed = try parseConfig(RawConfig, allocator, io, repo); - defer parsed.deinit(); - if (std.meta.stringToEnum(ModelType, parsed.value.model_type)) |model_type| return model_type; - return error.UnknownModelType; -} diff --git a/examples/llm/models/common.zig b/examples/llm/models/common.zig deleted file mode 100644 index 8a5c785d..00000000 --- a/examples/llm/models/common.zig +++ /dev/null @@ -1,88 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -pub const SessionOptions = struct { - seqlen: u32, - backend: zml.attention.Backend, -}; - -pub const GenerationOptions = struct { - sampling_strategy: zml.nn.SamplingStrategy = .{}, -}; - -pub const Phase = enum { - prefill, - decode, - - pub fn isPrefill(self: Phase) bool { - return self == .prefill; - } - - pub fn label(self: Phase) []const u8 { - return @tagName(self); - } - - pub fn startMessage(self: Phase, comptime component: []const u8) []const u8 { - return switch (self) { - .prefill => "Compiling prefill " ++ component ++ "...", - .decode => "Compiling decode " ++ component ++ "...", - }; - } - - pub fn programName(self: Phase, comptime model_name: []const u8, comptime component: []const u8) []const u8 { - return switch (self) { - .prefill => "llm_" ++ model_name ++ "_prefill_" ++ component, - .decode => "llm_" ++ model_name ++ "_decode_" ++ component, - }; - } - - pub fn logCompileDone(self: Phase, logger: anytype, comptime component: []const u8, io: std.Io, from: std.Io.Timestamp) void { - logger.info("Compiled {s} " ++ component ++ " [{f}]", .{ self.label(), from.untilNow(io, .awake) }); - } -}; - -pub const Meshes = struct { - model: *const zml.Mesh, - experts: *const zml.Mesh, - - pub fn init(platform: *zml.Platform) !Meshes { - switch (platform.target) { - .tpu => { - const has_link_y = platform.physical_mesh.hasAxis(.link_y); - - var strategy_experts: zml.Sharding.Strategy = .parseBindings(.{ .experts = .link_x }); - var strategy_model: zml.Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - if (has_link_y) { - strategy_experts.addFold(.link_x, &.{ .link_x, .link_y }); - strategy_model.addFold(.link_x, &.{ .link_x, .link_y }); - } - - return .{ - .model = try platform.registerMeshWithStrategy("model", .mesh(.{ .model = .high_bandwidth }), strategy_model), - .experts = try platform.registerMeshWithStrategy("experts", .mesh(.{ .experts = .high_bandwidth }), strategy_experts), - }; - }, - .cuda, .rocm, .oneapi, .neuron, .metal, .cpu, .furiosa => return .{ - .model = try platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })), - .experts = try platform.registerMesh("experts", .mesh(.{ .experts = .high_bandwidth })), - }, - } - } - - pub fn all(self: Meshes) [2]*const zml.Mesh { - return .{ self.model, self.experts }; - } -}; - -pub fn parseConfig(comptime T: type, allocator: std.mem.Allocator, io: std.Io, dir: std.Io.Dir) !std.json.Parsed(T) { - const file = try dir.openFile(io, "config.json", .{}); - defer file.close(io); - - var buffer: [256]u8 = undefined; - var file_reader = file.reader(io, &buffer); - var reader: std.json.Reader = .init(allocator, &file_reader.interface); - defer reader.deinit(); - - return try std.json.parseFromTokenSource(T, allocator, &reader, .{ .ignore_unknown_fields = true }); -} diff --git a/examples/llm/models/lfm2.zig b/examples/llm/models/lfm2.zig deleted file mode 100644 index 4b58c18a..00000000 --- a/examples/llm/models/lfm2.zig +++ /dev/null @@ -1,11 +0,0 @@ -pub const inference = @import("lfm2/inference.zig"); -pub const CompilationParameters = inference.CompilationParameters; -pub const CompilationOptions = inference.CompilationParameters; -pub const CompiledModel = inference.CompiledModel; -pub const model = @import("lfm2/model.zig"); -pub const Config = model.Config; -pub const Buffers = model.Buffers; -pub const Model = model.Model; -pub const LoadedModel = model.LoadedModel; -pub const session = @import("lfm2/session.zig"); -pub const Session = session.Session; diff --git a/examples/llm/models/lfm2/inference.zig b/examples/llm/models/lfm2/inference.zig deleted file mode 100644 index 0e37a999..00000000 --- a/examples/llm/models/lfm2/inference.zig +++ /dev/null @@ -1,321 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); -const stdx = zml.stdx; - -const common = @import("../common.zig"); -const Phase = common.Phase; -const model = @import("model.zig"); - -const log = std.log.scoped(.lfm); - -pub const CompilationParameters = struct { - hidden_dim: usize, - batch_dim: usize, - rng: zml.Tensor.Rng, - cache: model.Cache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - seqlen: u32, - meshes: common.Meshes, - - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { - stdx.debug.assert(seqlen >= config.conv_L_cache, "seqlen ({}) must be at least conv_L_cache ({})", .{ seqlen, config.conv_L_cache }); - const cache: model.Cache = .{ - .kv = .init(.init(.{ - .layer = mdl.num_attention_layers, - .batch = 1, - .k = seqlen, - .h = config.num_key_value_heads, - .hd = config.hidden_size / config.num_attention_heads, - }, mdl.embed_tokens.weight.dtype()), meshes.model), - .conv = .init(.init(.{ - .layer = mdl.num_conv_layers, - .batch = 1, - .seq = config.conv_L_cache, - .d = config.hidden_size, - }, mdl.embed_tokens.weight.dtype())), - }; - - return .{ - .hidden_dim = config.hidden_size, - .batch_dim = 1, - .rng = .init(), - .cache = cache, - .attention_metadata = .init(.fromBackend(backend)), - .attention_parameters = .init(.fromBackend(backend)), - .seqlen = seqlen, - .meshes = meshes, - }; - } -}; - -pub const CompilationOptions = CompilationParameters; - -pub const Args = struct { - io: std.Io, - tokens_buf: *zml.Buffer, - next_token_buf: *zml.Buffer, - tokens_pos_buf: *zml.Buffer, - active_length_buf: *zml.Buffer, - rng_buf: *zml.Bufferized(zml.Tensor.Rng), - cache_buffers: *zml.Bufferized(model.Cache), - attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata), -}; - -pub const CompiledModel = struct { - loaded_model: *const model.LoadedModel, - prefill: KernelExe, - decode: KernelExe, - params: CompilationParameters, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - loaded_model: *const model.LoadedModel, - mdl: model.Model, - opts: CompilationParameters, - progress: *std.Progress.Node, - ) !CompiledModel { - const prefill = try compileKernel(allocator, io, platform, mdl, opts, opts.seqlen, .prefill, progress); - errdefer prefill.deinit(); - const decode = try compileKernel(allocator, io, platform, mdl, opts, 1, .decode, progress); - return .{ - .loaded_model = loaded_model, - .prefill = prefill, - .decode = decode, - .params = opts, - }; - } - - pub fn deinit(self: *CompiledModel) void { - self.prefill.deinit(); - self.decode.deinit(); - } -}; - -pub const Inference = CompiledModel; - -pub const KernelExe = struct { - embed: zml.FnExe(model.TokenEmbedding.forward), - conv: zml.FnExe(model.DecoderLayer.forward), - self_attn: zml.FnExe(model.DecoderLayer.forward), - sample: zml.FnExe(model.LmHead.forward), - - pub fn deinit(self: *const KernelExe) void { - self.embed.deinit(); - self.conv.deinit(); - self.self_attn.deinit(); - self.sample.deinit(); - } -}; - -pub const KernelRunner = struct { - embed: zml.FnExe(model.TokenEmbedding.forward).Runner(.{.embedding}), - layers: []zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}), - sample: zml.FnExe(model.LmHead.forward).Runner(.{ .lm_head, .embed_tokens }), - - pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { - var embed = try zml.FnExe(model.TokenEmbedding.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ .embedding = buffers.embed_tokens }); - errdefer embed.deinit(allocator); - - const layers = try allocator.alloc(zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}), buffers.layers.len); - errdefer allocator.free(layers); - var initialized_layers: usize = 0; - errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); - for (layers, buffers.layers) |*layer, layer_buffers| { - const layer_exe: *const zml.FnExe(model.DecoderLayer.forward) = switch (layer_buffers.operator) { - .conv => &exe.conv, - .self_attn => &exe.self_attn, - }; - layer.* = try zml.FnExe(model.DecoderLayer.forward).Runner(.{.layer}).init(layer_exe, allocator, .{ .layer = layer_buffers }); - initialized_layers += 1; - } - - var sample = try zml.FnExe(model.LmHead.forward).Runner(.{ .lm_head, .embed_tokens }).init(&exe.sample, allocator, .{ - .lm_head = buffers.lm_head, - .embed_tokens = buffers.embed_tokens, - }); - errdefer sample.deinit(allocator); - - return .{ .embed = embed, .layers = layers, .sample = sample }; - } - - pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { - self.embed.deinit(allocator); - for (self.layers) |*layer| layer.deinit(allocator); - allocator.free(self.layers); - self.sample.deinit(allocator); - } -}; - -pub fn run( - runner: *KernelRunner, - args: Args, - conv_cache_index_buffer: *zml.Buffer, - kv_cache_index_buffer: *zml.Buffer, -) void { - var hidden_buffer: zml.Buffer = undefined; - runner.embed.run(args.io, .{ - .inputs = .{ - .tokens = args.tokens_buf.*, - }, - .outputs = .{ .hidden = &hidden_buffer }, - }, .{}); - defer hidden_buffer.deinit(); - - for (runner.layers) |*layer| { - layer.run(args.io, .{ - .inputs = .{ - .hidden = hidden_buffer, - .tokens_position_offset = args.tokens_pos_buf.*, - .active_length = args.active_length_buf.*, - .cache = args.cache_buffers.*, - .conv_cache_index = conv_cache_index_buffer.*, - .kv_cache_index = kv_cache_index_buffer.*, - .attention_metadata = args.attention_metadata_buffers, - }, - .outputs = .{ - .hidden = &hidden_buffer, - .cache = args.cache_buffers, - .conv_cache_index = conv_cache_index_buffer, - .kv_cache_index = kv_cache_index_buffer, - }, - }, .{}); - } - - runner.sample.run(args.io, .{ - .inputs = .{ - .hidden = hidden_buffer, - .tokens = args.tokens_buf.*, - .active_length = args.active_length_buf.*, - .rng = args.rng_buf.*, - }, - .outputs = .{ - .tokens = args.next_token_buf, - .rng = args.rng_buf, - }, - }, .{}); -} - -fn compileKernel( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - phase: Phase, - progress: *std.Progress.Node, -) !KernelExe { - const embed = try compileEmbed(allocator, io, platform, mdl.embed_tokens, opts, seqlen, phase, progress); - errdefer embed.deinit(); - const conv_layer = try compileLayer(allocator, io, platform, mdl, opts, seqlen, .conv, phase, progress); - errdefer conv_layer.deinit(); - const attn_layer = try compileLayer(allocator, io, platform, mdl, opts, seqlen, .full_attention, phase, progress); - errdefer attn_layer.deinit(); - const sample = try compileSample(allocator, io, platform, mdl, opts, seqlen, phase, progress); - errdefer sample.deinit(); - return .{ .embed = embed, .conv = conv_layer, .self_attn = attn_layer, .sample = sample }; -} - -fn compileEmbed( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - embed_tokens: model.TokenEmbedding, - opts: CompilationOptions, - seqlen: u32, - phase: Phase, - progress: *std.Progress.Node, -) !zml.FnExe(model.TokenEmbedding.forward) { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("embed_tokens"), 1); - defer node.end(); - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "embed_tokens", io, from); - - return zml.FnExe(model.TokenEmbedding.forward).compile(allocator, io, platform, .{ - .meshes = &opts.meshes.all(), - .program_name = phase.programName("lfm2", "embed_tokens"), - }, .{.{ - .embedding = embed_tokens, - .tokens = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32), - }}); -} - -fn compileLayer( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - comptime kind: model.OperatorKind, - phase: Phase, - progress: *std.Progress.Node, -) !zml.FnExe(model.DecoderLayer.forward) { - const label = switch (kind) { - .conv => "conv layer", - .full_attention => "attn layer", - }; - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage(label), 1); - defer node.end(); - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, label, io, from); - - const layer = for (mdl.layers) |candidate| { - const candidate_kind: model.OperatorKind = switch (candidate.operator) { - .conv => .conv, - .self_attn => .full_attention, - }; - if (candidate_kind == kind) break candidate; - } else unreachable; - - return zml.FnExe(model.DecoderLayer.forward).compile(allocator, io, platform, .{ - .meshes = &opts.meshes.all(), - .program_name = phase.programName("lfm2", if (kind == .conv) "conv_layer" else "attn_layer"), - }, .{.{ - .layer = layer, - .hidden = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()), - .tokens_position_offset = zml.Tensor.init(.{ .batch = opts.batch_dim }, .u32), - .active_length = .init(.{}, .u32), - .cache = opts.cache, - .conv_cache_index = zml.Tensor.init(.{}, .u32), - .kv_cache_index = zml.Tensor.init(.{}, .u32), - .attention_metadata = opts.attention_metadata, - .attention_parameters = opts.attention_parameters, - .conv_parameters = .{ .is_prefill = phase.isPrefill() }, - }}); -} - -fn compileSample( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - mdl: model.Model, - opts: CompilationOptions, - seqlen: u32, - phase: Phase, - progress: *std.Progress.Node, -) !zml.FnExe(model.LmHead.forward) { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("lm_head"), 1); - defer node.end(); - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "lm_head", io, from); - - return zml.FnExe(model.LmHead.forward).compile(allocator, io, platform, .{ - .meshes = &opts.meshes.all(), - .program_name = phase.programName("lfm2", "lm_head"), - }, .{.{ - .lm_head = mdl.lm_head, - .embed_tokens = mdl.embed_tokens, - .hidden = zml.Tensor.init(.{ .batch = opts.batch_dim, .seq = seqlen, .d = opts.hidden_dim }, mdl.embed_tokens.weight.dtype()), - .tokens = .init(.{ .batch = opts.batch_dim, .seq = seqlen }, .u32), - .active_length = .init(.{}, .u32), - .rng = opts.rng, - }}); -} diff --git a/examples/llm/models/lfm2/model.zig b/examples/llm/models/lfm2/model.zig deleted file mode 100644 index 9f7cb909..00000000 --- a/examples/llm/models/lfm2/model.zig +++ /dev/null @@ -1,761 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); -const stdx = zml.stdx; - -const common = @import("../common.zig"); -const inference = @import("inference.zig"); - -const log = std.log.scoped(.lfm); - -pub const Config = struct { - architectures: []const []const u8, - block_auto_adjust_ff_dim: bool, - block_dim: u32, - block_ff_dim: u32, - block_ffn_dim_multiplier: f32, - block_mlp_init_scale: f32, - block_multiple_of: u32, - block_norm_eps: f32, - block_out_init_scale: f32, - block_use_swiglu: bool, - block_use_xavier_init: bool, - bos_token_id: u32, - conv_L_cache: u32, - conv_bias: bool, - conv_dim: u32, - conv_use_xavier_init: bool, - dtype: []const u8, - eos_token_id: u32, - hidden_size: u32, - initializer_range: f32, - intermediate_size: u32, - layer_types: []const []const u8, - max_position_embeddings: u32, - model_type: []const u8, - norm_eps: f32, - num_attention_heads: u32, - num_heads: u32, - num_hidden_layers: u32, - num_key_value_heads: u32, - pad_token_id: u32, - rope_theta: f32, - tie_embedding: bool, - transformers_version: []const u8, - use_cache: bool, - use_pos_enc: bool, - vocab_size: u32, -}; - -pub const LoadedModel = struct { - inner: Model, - parsed_config: std.json.Parsed(Config), - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - store: zml.io.TensorStore.View, - generation: common.GenerationOptions, - ) !LoadedModel { - const parsed_config = try common.parseConfig(Config, allocator, io, repo); - errdefer parsed_config.deinit(); - - return .{ - .inner = .init(allocator, store, parsed_config.value, generation), - .parsed_config = parsed_config, - }; - } - - pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { - self.inner.deinit(allocator); - self.parsed_config.deinit(); - } - - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - Model.unloadBuffers(buffers, allocator); - } - - pub fn compile( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - backend: zml.attention.Backend, - meshes: common.Meshes, - seqlen: usize, - progress: *std.Progress.Node, - ) !inference.CompiledModel { - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); - return inference.CompiledModel.init(allocator, io, @constCast(platform), self, self.inner, params, progress); - } -}; - -pub const Buffers = zml.Bufferized(Model); - -pub const Model = struct { - embed_tokens: TokenEmbedding, - lm_head: LmHead, - layers: []DecoderLayer, - num_attention_layers: usize, - num_conv_layers: usize, - - pub fn init( - allocator: std.mem.Allocator, - root_store: zml.io.TensorStore.View, - config: Config, - generation: common.GenerationOptions, - ) Model { - const store = root_store.withPrefix("model"); - stdx.debug.assert(config.layer_types.len == config.num_hidden_layers, "Expected layer_types len {} to match num_hidden_layers {}", .{ config.layer_types.len, config.num_hidden_layers }); - - const layers = allocator.alloc(DecoderLayer, config.num_hidden_layers) catch unreachable; - var num_attention_layers: usize = 0; - var num_conv_layers: usize = 0; - for (layers, 0..) |*layer, i| { - const layer_store = store.withPrefix("layers").withLayer(i); - const kind = DecoderLayer.parseOperatorKind(config.layer_types[i]); - switch (kind) { - .conv => num_conv_layers += 1, - .full_attention => num_attention_layers += 1, - } - layer.* = DecoderLayer.init(config, layer_store, kind); - } - - return .{ - .embed_tokens = .init(store.withPrefix("embed_tokens")), - .lm_head = LmHead.init(store, config, generation.sampling_strategy), - .layers = layers, - .num_attention_layers = num_attention_layers, - .num_conv_layers = num_conv_layers, - }; - } - - pub fn loadBuffers( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 8, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - LmHead.unloadBuffers(&self.lm_head); - for (self.layers) |*layer| { - DecoderLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - } - - pub fn forward( - self: Model, - tokens: zml.Tensor, - tokens_position_offset: zml.Tensor, - active_length: zml.Tensor, - rng: zml.Tensor.Rng, - cache_: Cache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - conv_parameters: ConvParameters, - ) struct { zml.Tensor, Cache, zml.Tensor.Rng } { - stdx.debug.assert(tokens.shape().hasTags(.{ .batch, .seq }), "Tokens should have tags {{.batch, .seq}}, got {f}", .{tokens.shape()}); - - const embeds = TokenEmbedding.forward(.{ - .embedding = self.embed_tokens, - .tokens = tokens, - }).hidden; - - var hidden = embeds; - var cache = cache_; - // Incremented by the individual layers. - var conv_cache_index = zml.Tensor.scalar(@as(u32, 0), .u32); - var kv_cache_index = zml.Tensor.scalar(@as(u32, 0), .u32); - for (self.layers) |layer| { - const result = DecoderLayer.forward(.{ - .layer = layer, - .hidden = hidden, - .tokens_position_offset = tokens_position_offset, - .active_length = active_length, - .cache = cache, - .conv_cache_index = conv_cache_index, - .kv_cache_index = kv_cache_index, - .attention_metadata = attention_metadata, - .attention_parameters = attention_parameters, - .conv_parameters = conv_parameters, - }); - hidden = result.hidden; - cache = result.cache; - conv_cache_index = result.conv_cache_index; - kv_cache_index = result.kv_cache_index; - } - const result = LmHead.forward(.{ - .lm_head = self.lm_head, - .embed_tokens = self.embed_tokens, - .hidden = hidden, - .tokens = tokens, - .active_length = active_length, - .rng = rng, - }); - return .{ result.tokens, cache.reuseBuffer(cache_), result.rng }; - } - - pub fn deinit(self: Model, allocator: std.mem.Allocator) void { - allocator.free(self.layers); - } -}; - -pub const TokenEmbedding = struct { - weight: zml.Tensor, - - pub const Input = struct { - embedding: TokenEmbedding, - tokens: zml.Tensor, - }; - - pub const Output = struct { - hidden: zml.Tensor, - }; - - pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { - return .{ .weight = store.createTensor("weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }; - } - - pub fn forward(input: Input) Output { - stdx.debug.assert(input.tokens.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{input.tokens}); - stdx.debug.assert(input.embedding.weight.rank() == 2, "TokenEmbedding expects it's weight zml.Tensor to be a 2D matrix, got {f}", .{input.embedding.weight}); - return .{ .hidden = input.embedding.weight.gather(.{ .voc = input.tokens }, .{}) }; - } - - pub fn unembed(self: TokenEmbedding, embeds: zml.Tensor) zml.Tensor { - stdx.debug.assert(embeds.shape().hasTags(.{.d}), "TokenEmbedding expects the input embeds to have a .d tag, got {f}", .{embeds.shape()}); - return self.weight.dot(embeds, .d); - } -}; - -pub const LmHead = struct { - embedding_norm: RmsNorm, - sampling_strategy: zml.nn.SamplingStrategy, - - pub const Input = struct { - lm_head: LmHead, - embed_tokens: TokenEmbedding, - hidden: zml.Tensor, - tokens: zml.Tensor, - active_length: zml.Tensor, - rng: zml.Tensor.Rng, - }; - - pub const Output = struct { - tokens: zml.Tensor, - rng: zml.Tensor.Rng, - }; - - pub fn init(store: zml.io.TensorStore.View, config: Config, sampling_strategy: zml.nn.SamplingStrategy) LmHead { - return .{ - .embedding_norm = RmsNorm.init(store.withPrefix("embedding_norm"), config.norm_eps, .d), - .sampling_strategy = sampling_strategy, - }; - } - - pub fn forward(input: Input) Output { - const self = input.lm_head; - const last_index = input.active_length.convert(.i32).addConstant(-1); - const hidden = input.hidden.slice(.seq, .dyn(last_index, 1)); - const logits = input.embed_tokens.unembed(self.embedding_norm.forward(hidden)); - const new_tokens, const new_rng = zml.nn.sampleTokens(logits, self.sampling_strategy, input.rng); - const tokens = new_tokens.convert(.u32); - return .{ .tokens = if (input.tokens.dim(.seq) == 1) tokens.reuseBuffer(input.tokens) else tokens, .rng = new_rng }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(LmHead)) void { - RmsNorm.unloadBuffers(&self.embedding_norm); - } -}; - -pub const OperatorKind = enum { conv, full_attention }; -const Operator = union(enum) { conv: ShortConv, self_attn: Attention }; - -pub const DecoderLayer = struct { - operator: Operator, - operator_norm: RmsNorm, - ffn_norm: RmsNorm, - feed_forward: Mlp, - - pub const Input = struct { - layer: DecoderLayer, - hidden: zml.Tensor, - tokens_position_offset: zml.Tensor, - active_length: zml.Tensor, - cache: Cache, - conv_cache_index: zml.Tensor, - kv_cache_index: zml.Tensor, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - conv_parameters: ConvParameters, - }; - - pub const Output = struct { - hidden: zml.Tensor, - cache: Cache, - conv_cache_index: zml.Tensor, - kv_cache_index: zml.Tensor, - }; - - pub fn parseOperatorKind(layer_type: []const u8) OperatorKind { - return std.meta.stringToEnum(OperatorKind, layer_type) orelse { - stdx.debug.assert(false, "Unsupported layer type {s}", .{layer_type}); - unreachable; - }; - } - - pub fn init(config: Config, store: zml.io.TensorStore.View, kind: OperatorKind) DecoderLayer { - const operator_norm = RmsNorm.init(store.withPrefix("operator_norm"), config.norm_eps, .d); - const ffn_norm = RmsNorm.init(store.withPrefix("ffn_norm"), config.norm_eps, .d); - const feed_forward = Mlp.init(store.withPrefix("feed_forward")); - const operator: Operator = switch (kind) { - .conv => .{ .conv = ShortConv.init(config, store.withPrefix("conv")) }, - .full_attention => .{ .self_attn = Attention.init(config, store.withPrefix("self_attn")) }, - }; - - return .{ - .operator = operator, - .operator_norm = operator_norm, - .ffn_norm = ffn_norm, - .feed_forward = feed_forward, - }; - } - - pub fn forward(input: Input) Output { - const self = input.layer; - var cache = input.cache; - var conv_cache_index = input.conv_cache_index; - var kv_cache_index = input.kv_cache_index; - const residual = switch (self.operator) { - .conv => |operator| b: { - const residual, const updated_conv_cache = operator.forward( - self.operator_norm.forward(input.hidden), - input.tokens_position_offset, - input.active_length, - cache.conv, - conv_cache_index, - input.conv_parameters, - ); - cache.conv = updated_conv_cache; - conv_cache_index = conv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); - break :b residual; - }, - .self_attn => |operator| b: { - const residual, const updated_kv_cache = operator.forward( - self.operator_norm.forward(input.hidden), - input.tokens_position_offset, - cache.kv, - kv_cache_index, - input.attention_metadata, - input.attention_parameters, - ); - cache.kv = updated_kv_cache; - kv_cache_index = kv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); - break :b residual; - }, - }; - - const x = input.hidden.add(residual); - - return .{ - .hidden = x.add(self.feed_forward.forward(self.ffn_norm.forward(x))).reuseBuffer(input.hidden), - .cache = cache.reuseBuffer(input.cache), - .conv_cache_index = conv_cache_index.reuseBuffer(input.conv_cache_index), - .kv_cache_index = kv_cache_index.reuseBuffer(input.kv_cache_index), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(DecoderLayer)) void { - switch (self.operator) { - .conv => |*operator| ShortConv.unloadBuffers(operator), - .self_attn => |*operator| Attention.unloadBuffers(operator), - } - RmsNorm.unloadBuffers(&self.operator_norm); - RmsNorm.unloadBuffers(&self.ffn_norm); - Mlp.unloadBuffers(&self.feed_forward); - } -}; - -pub const ConvParameters = struct { is_prefill: bool }; - -pub const ShortConv = struct { - in_proj: Linear, - out_proj: Linear, - kernel: zml.Tensor, - config: Config, - - pub fn init(config: Config, store: zml.io.TensorStore.View) ShortConv { - stdx.debug.assert(!config.conv_bias, "conv_bias is not supported.", .{}); - return .{ - .in_proj = .init(store.withPrefix("in_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), - .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), - .kernel = store.createReplicatedTensor("conv.weight", .{ .out, .in, .kernel_size }), - .config = config, - }; - } - - pub fn forward(self: ShortConv, input: zml.Tensor, tokens_position_offset: zml.Tensor, active_length: zml.Tensor, cache_: ConvCache, cache_index: zml.Tensor, parameters: ConvParameters) struct { zml.Tensor, ConvCache } { - var cache = cache_; - const BCx = self.in_proj.forward(input.withPartitioning(.model, .{ .d = .replicated })); - - const B, const C, const x = BCx.chunkExact(.d, 3); - const Bx = B.mul(x); - - const conv_out = if (parameters.is_prefill) b: { - const active_length_i32 = active_length.convert(.i32); - const start = active_length_i32.sub(.scalar(self.config.conv_L_cache, .i32)).maximum(.scalar(0, .i32)); - const cache_seq_indices = lkp: { - const n = active_length_i32.sub(start); - const left_pad = zml.Tensor.scalar(@as(i32, @intCast(self.config.conv_L_cache)), .i32).sub(n); - const sh = tokens_position_offset.shape().insert(.last, .{ .seq = self.config.conv_L_cache }); - break :lkp zml.Tensor.iota(sh, .seq).convert(.u32).add(left_pad.convert(.u32).broad(sh)).broad(sh); - }; - const scatter_data = Bx.slice(.seq, .dyn(start, self.config.conv_L_cache)); - cache.state = cache.state.scatterSlices(.{ .layer = cache_index, .seq = cache_seq_indices }, scatter_data, .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(cache.state); - const padding = self.config.conv_L_cache - 1; - const conv_out_ = zml.Tensor.conv1d(Bx, self.kernel, .{ - .padding = &.{ padding, 0 }, - .input_batch_dimension = Bx.axis(.batch), - .input_feature_dimension = B.axis(.d), - .input_spatial_dimensions = Bx.axis(.seq), - .kernel_output_feature_dimension = self.kernel.axis(.out), - .kernel_input_feature_dimension = self.kernel.axis(.in), - .kernel_spatial_dimensions = self.kernel.axis(.kernel_size), - .output_batch_dimension = Bx.axis(.batch), - .output_feature_dimension = Bx.axis(.d), - .output_spatial_dimensions = Bx.axis(.seq), - .feature_group_count = Bx.dim(.d), - }); - break :b conv_out_.slice(.seq, .{ .end = Bx.dim(.seq) }); - } else b: { - cache.state = cache.state.rollRight1d(.seq, -1).scatterSlices(.{ .layer = cache_index, .seq = tokens_position_offset.convert(.u32).clamp(.scalar(@as(u32, 0), .u32), .scalar(self.config.conv_L_cache - 1, .u32)) }, Bx, .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(cache.state); - const kernel = self.kernel.squeeze(.in).rename(.{ .out = .d, .kernel_size = .seq }); - const sh = kernel.shape().insert(.last, .{ .batch = tokens_position_offset.dim(.batch) }); - break :b cache.state.slice(.layer, .dynSingle(cache_index)).mul(kernel.broad(sh).transpose(.{ .batch, .seq, .d })).sum(.seq); - }; - - const y = C.mul(conv_out); - - const output = self.out_proj.forward(y); - return .{ output.reuseBuffer(input), cache.reuseBuffer(cache_) }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(ShortConv)) void { - Linear.unloadBuffers(&self.in_proj); - Linear.unloadBuffers(&self.out_proj); - self.kernel.deinit(); - } -}; - -pub const Attention = struct { - q_proj: Linear, - k_proj: Linear, - v_proj: Linear, - out_proj: Linear, - q_layernorm: RmsNorm, - k_layernorm: RmsNorm, - head_dim: usize, - num_key_value_groups: usize, - rope_opts: zml.nn.RopeOpts, - - pub fn init(config: Config, store: zml.io.TensorStore.View) Attention { - const head_dim = config.hidden_size / config.num_attention_heads; - const num_key_value_groups = config.num_attention_heads / config.num_key_value_heads; - return .{ - .q_proj = .init(store.withPrefix("q_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), - .k_proj = .init(store.withPrefix("k_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), - .v_proj = .init(store.withPrefix("v_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), - .out_proj = .init(store.withPrefix("out_proj").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), - .q_layernorm = RmsNorm.init(store.withPrefix("q_layernorm"), config.norm_eps, .hd), - .k_layernorm = RmsNorm.init(store.withPrefix("k_layernorm"), config.norm_eps, .hd), - .head_dim = head_dim, - .num_key_value_groups = num_key_value_groups, - .rope_opts = .{ .layout = .real_im_pass, .scaling = .{ .default = .{ .rope_theta = config.rope_theta } } }, - }; - } - - pub fn forward( - self: Attention, - x: zml.Tensor, - tokens_position_offset: zml.Tensor, - kv_cache: KvCache, - cache_index: zml.Tensor, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - ) struct { zml.Tensor, KvCache } { - const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); - - var q = self.q_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); - var k = self.k_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); - var v = self.v_proj.forward(x_qkv).splitAxis(-1, .{ .h = .auto, .hd = self.head_dim }); - - q = self.q_layernorm.forward(q); - k = self.k_layernorm.forward(k); - - const token_positions = b: { - const sh = tokens_position_offset.shape().insert(.last, .{ .seq = x.dim(.seq) }); - break :b zml.Tensor.iota(sh, .seq).convert(.u32).add(tokens_position_offset.broad(sh)); - }; - - q = zml.nn.rope(q, token_positions, self.rope_opts); - k = zml.nn.rope(k, token_positions, self.rope_opts); - - q = q.rename(.{ .seq = .q }); - k = k.rename(.{ .seq = .k }); - v = v.rename(.{ .seq = .k }); - - const new_kv_cache = kv_cache.update(k, v, tokens_position_offset, cache_index); - k = new_kv_cache.keys(cache_index); - v = new_kv_cache.values(cache_index); - - stdx.debug.assert(q.dim(.batch) == 1, "LFM attention currently expects batch size 1 for flash attention backend, got {}", .{q.dim(.batch)}); - const mesh = zml.Compiler.current().mesh(.model); - const partitioning: zml.attention.Partitioning = .head(.model); - const attn = switch (attention_parameters) { - .stablehlo => zml.attention.attention( - q, - k, - v, - tokens_position_offset, - attention_metadata, - attention_parameters, - mesh, - partitioning, - ).merge(.{ .d = .{ .h, .hd } }), - else => zml.attention.attention( - q.squeeze(.batch), - k.squeeze(.batch), - v.squeeze(.batch), - tokens_position_offset.squeeze(.batch), - attention_metadata, - attention_parameters, - mesh, - partitioning, - ).merge(.{ .d = .{ .h, .hd } }) - .rename(.{ .q = .seq }) - .insertAxes(.seq, .{.batch}), - }; - - return .{ self.out_proj.forward(attn).reuseBuffer(x), new_kv_cache.reuseBuffer(kv_cache) }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Attention)) void { - Linear.unloadBuffers(&self.q_proj); - Linear.unloadBuffers(&self.k_proj); - Linear.unloadBuffers(&self.v_proj); - Linear.unloadBuffers(&self.out_proj); - RmsNorm.unloadBuffers(&self.q_layernorm); - RmsNorm.unloadBuffers(&self.k_layernorm); - } -}; - -pub const Cache = struct { - conv: ConvCache, - kv: KvCache, - - pub fn initBuffers(self: Cache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Cache) { - return .{ .conv = try self.conv.initBuffers(allocator, io, platform), .kv = try self.kv.initBuffers(io, platform) }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Cache)) void { - ConvCache.unloadBuffers(&self.conv); - KvCache.unloadBuffers(&self.kv); - } - - pub fn reuseBuffer(self: Cache, other: Cache) Cache { - return .{ .conv = self.conv.reuseBuffer(other.conv), .kv = self.kv.reuseBuffer(other.kv) }; - } -}; - -pub const ConvCache = struct { - state: zml.Tensor, - - pub fn init(shape: zml.Shape) ConvCache { - return .{ .state = .fromShape(shape) }; - } - - pub fn initBuffers(self: ConvCache, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(ConvCache) { - const sh = self.state.shape(); - const host = try allocator.alloc(u8, sh.byteSize()); - defer allocator.free(host); - @memset(host, 0); - return .{ .state = try zml.Buffer.fromBytes(io, platform, sh, host) }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(ConvCache)) void { - self.state.deinit(); - } - - pub fn reuseBuffer(self: ConvCache, other: ConvCache) ConvCache { - return .{ .state = self.state.reuseBuffer(other.state) }; - } -}; - -pub const KvCache = struct { - k: zml.Tensor, - v: zml.Tensor, - - pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { - const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); - return .{ .k = .fromShape(sharded_shape), .v = .fromShape(sharded_shape) }; - } - - pub fn initBuffers(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { - return .{ .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}) }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(KvCache)) void { - self.k.deinit(); - self.v.deinit(); - } - - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { - return .{ .k = self.k.reuseBuffer(other.k), .v = self.v.reuseBuffer(other.v) }; - } - - pub fn keys(self: KvCache, cache_index: zml.Tensor) zml.Tensor { - return self.k.slice(.layer, .dynSingle(cache_index)); - } - - pub fn values(self: KvCache, cache_index: zml.Tensor) zml.Tensor { - return self.v.slice(.layer, .dynSingle(cache_index)); - } - - pub fn update(self: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_position: zml.Tensor, cache_index: zml.Tensor) KvCache { - const k_shape = self.k.shape().drop(.layer); - const layer = cache_index.broad(token_position.shape()); - return .{ - .k = self.k.scatterSlices(.{ .layer = layer, .k = token_position }, new_k.transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.k), - .v = self.v.scatterSlices(.{ .layer = layer, .k = token_position }, new_v.transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(self.v), - }; - } -}; - -pub const Linear = struct { - weight: zml.Tensor, - bias: ?zml.Tensor = null, - in_tag: zml.Shape.Tag, - out_tag: zml.Shape.Tag, - - pub fn init(weight: zml.Tensor, bias: ?zml.Tensor, tag: anytype) Linear { - stdx.debug.guard(weight.shape().hasTag(tag) != null, @src()); - const axis = weight.shape().axis(tag); - const out_tag = weight.shape().tag(1 - axis); - return .{ .weight = weight, .bias = bias, .in_tag = zml.Shape.toTag(tag), .out_tag = out_tag }; - } - - pub fn unloadBuffers(linear: *zml.Bufferized(Linear)) void { - linear.weight.deinit(); - if (linear.bias) |*bias| bias.deinit(); - } - - pub fn forward(self: Linear, x: zml.Tensor) zml.Tensor { - var y = x.dot(self.weight, self.in_tag).renameTag(self.out_tag, self.in_tag); - return if (self.bias) |bias| y.add(bias.broad(y.shape())).reuseBuffer(x) else y; - } -}; - -const Mlp = struct { - w1: Linear, - w2: Linear, - w3: Linear, - - pub fn init(store: zml.io.TensorStore.View) Mlp { - return .{ - .w1 = .init(store.withPrefix("w1").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), - .w2 = .init(store.withPrefix("w2").createTensor("weight", .{ .out, .d }, .model, .{ .out = .replicated, .d = .model }), null, .d), - .w3 = .init(store.withPrefix("w3").createTensor("weight", .{ .out, .d }, .model, .{ .out = .model, .d = .replicated }), null, .d), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - Linear.unloadBuffers(&self.w1); - Linear.unloadBuffers(&self.w2); - Linear.unloadBuffers(&self.w3); - } - - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - return self.w2.forward(self.w1.forward(x).silu().mul(self.w3.forward(x))).reuseBuffer(x); - } -}; - -const RmsNorm = struct { - weight: zml.Tensor, - eps: f32, - tag: zml.Shape.Tag, - - pub fn init(store: zml.io.TensorStore.View, eps: f32, tag: anytype) RmsNorm { - return .{ .weight = store.createReplicatedTensor("weight", .{tag}), .eps = eps, .tag = zml.Shape.toTag(tag) }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - - pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { - const normalized = zml.nn.rmsNorm(input, self.tag, self.eps); - return normalized.mul(self.weight.broad(input.shape())).reuseBuffer(input); - } -}; diff --git a/examples/llm/models/lfm2/session.zig b/examples/llm/models/lfm2/session.zig deleted file mode 100644 index ecc5abf8..00000000 --- a/examples/llm/models/lfm2/session.zig +++ /dev/null @@ -1,227 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const inference = @import("inference.zig"); -const model = @import("model.zig"); - -pub const Session = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - compiled_model: *inference.CompiledModel, - prefill: inference.KernelRunner, - decode: inference.KernelRunner, - config: *const model.Config, - seqlen: u32, - cache_buffers: zml.Bufferized(model.Cache), - rng_buf: zml.Bufferized(zml.Tensor.Rng), - generated_token_slice: zml.Slice, - think_start: ?u32, - think_end: ?u32, - tokenizer: zml.tokenizer.Tokenizer, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - tokenizer: zml.tokenizer.Tokenizer, - compiled_model: *inference.CompiledModel, - model_buffers: *model.Buffers, - ) !Session { - const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var cache_buffers = try compiled_model.params.cache.initBuffers(allocator, io, platform); - errdefer model.Cache.unloadBuffers(&cache_buffers); - var rng_buf = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); - errdefer zml.Tensor.Rng.deinitBuffer(&rng_buf); - const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .batch = 1, .seq = 1 }, .u32)); - errdefer generated_token_slice.free(allocator); - - var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); - errdefer prefill.deinit(allocator); - const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); - - return .{ - .allocator = allocator, - .io = io, - .platform = platform, - .compiled_model = compiled_model, - .prefill = prefill, - .decode = decode, - .tokenizer = tokenizer, - .config = &compiled_model.loaded_model.parsed_config.value, - .seqlen = compiled_model.params.seqlen, - .cache_buffers = cache_buffers, - .rng_buf = rng_buf, - .generated_token_slice = generated_token_slice, - .think_start = tokenizer.tokenId("") orelse unreachable, - .think_end = tokenizer.tokenId("") orelse unreachable, - }; - } - - pub fn deinit(self: *Session) void { - self.prefill.deinit(self.allocator); - self.decode.deinit(self.allocator); - model.Cache.unloadBuffers(&self.cache_buffers); - zml.Tensor.Rng.deinitBuffer(&self.rng_buf); - self.generated_token_slice.free(self.allocator); - } - - pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { - var encoder = try self.tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; - const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ self.config.bos_token_id, im_start }); - const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); - defer allocator.free(user_tokens); - try tokens.appendSlice(allocator, user_tokens); - const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); - defer allocator.free(prompt_tokens); - try tokens.appendSlice(allocator, prompt_tokens); - try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); - defer allocator.free(assistant_tokens); - try tokens.appendSlice(allocator, assistant_tokens); - return tokens.toOwnedSlice(allocator); - } - - pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { - var encoder = try self.tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = self.tokenizer.tokenId("<|im_start|>") orelse return error.NoSuchToken; - const im_end = self.tokenizer.tokenId("<|im_end|>") orelse return error.NoSuchToken; - const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len); - try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); - defer allocator.free(user_tokens); - try tokens.appendSlice(allocator, user_tokens); - const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); - defer allocator.free(prompt_tokens); - try tokens.appendSlice(allocator, prompt_tokens); - try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); - defer allocator.free(assistant_tokens); - try tokens.appendSlice(allocator, assistant_tokens); - return tokens.toOwnedSlice(allocator); - } - - pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { - const tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{ .batch = 1, .seq = self.seqlen }, .u32)); - defer tokens_slice.free(self.allocator); - const tokens = tokens_slice.items(u32); - @memset(tokens, self.config.pad_token_id); - @memcpy(tokens[0..all_tokens.len], all_tokens); - - var tokens_buf: zml.Buffer = try .fromSlice(self.io, self.platform, tokens_slice); - defer tokens_buf.deinit(); - - const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{0})); - var tokens_pos_buf: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); - defer tokens_pos_buf.deinit(); - - const active_length_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.len)})); - var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); - defer active_length_buf.deinit(); - - const params = self.compiled_model.params; - var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { - .stablehlo => try params.attention_metadata.initBuffer(self.io, self.platform), - .flash_attention => |metadata| .{ .flash_attention = switch (metadata) { - .metal => .{ .metal = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, - else => try metadata.initBuffer(self.io, self.platform), - } }, - }; - defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); - - var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); - defer conv_cache_index_buffer.deinit(); - var kv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); - defer kv_cache_index_buffer.deinit(); - - var next_token_buf: zml.Buffer = undefined; - inference.run(&self.prefill, .{ - .io = self.io, - .tokens_buf = &tokens_buf, - .next_token_buf = &next_token_buf, - .tokens_pos_buf = &tokens_pos_buf, - .active_length_buf = &active_length_buf, - .rng_buf = &self.rng_buf, - .cache_buffers = &self.cache_buffers, - .attention_metadata_buffers = attention_metadata_buffers, - }, &conv_cache_index_buffer, &kv_cache_index_buffer); - - defer next_token_buf.deinit(); - try next_token_buf.toSlice(self.io, self.generated_token_slice); - } - - pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { - var decoder = try self.tokenizer.decoder(); - defer decoder.deinit(); - - var current_token_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, self.generated_token_slice); - defer current_token_buffer.deinit(); - - const active_length_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{1})); - var active_length_buf: zml.Buffer = try .fromSlice(self.io, self.platform, active_length_slice); - defer active_length_buf.deinit(); - - const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); - defer self.allocator.free(out_tokens_buffer); - - const params = self.compiled_model.params; - var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); - defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); - - generation: while (true) { - const token_id = self.generated_token_slice.items(u32)[0]; - - if (token_id == self.config.eos_token_id) break :generation; - - const token = try decoder.feedOne(token_id, out_tokens_buffer); - if (self.think_start) |think_start| if (token_id == think_start) { - try stdout.writeAll("\x1b[2m"); - }; - try stdout.writeAll(token); - if (self.think_end) |think_end| if (token_id == think_end) { - try stdout.writeAll("\x1b[0m"); - }; - try stdout.flush(); - - try all_tokens.append(self.allocator, token_id); - if (all_tokens.items.len >= self.seqlen) break :generation; - - const token_pos_slice: zml.Slice = .init(zml.Shape.init(.{ .batch = 1 }, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(all_tokens.items.len)})); - var token_pos_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, token_pos_slice); - defer token_pos_buffer.deinit(); - - var conv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); - defer conv_cache_index_buffer.deinit(); - var kv_cache_index_buffer: zml.Buffer = try .scalar(self.io, self.platform, 0, .u32); - defer kv_cache_index_buffer.deinit(); - - inference.run(&self.decode, .{ - .io = self.io, - .tokens_buf = ¤t_token_buffer, - .next_token_buf = ¤t_token_buffer, - .tokens_pos_buf = &token_pos_buffer, - .active_length_buf = &active_length_buf, - .rng_buf = &self.rng_buf, - .cache_buffers = &self.cache_buffers, - .attention_metadata_buffers = attention_metadata_buffers, - }, &conv_cache_index_buffer, &kv_cache_index_buffer); - - try current_token_buffer.toSlice(self.io, self.generated_token_slice); - } - - try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); - try stdout.flush(); - } -}; diff --git a/examples/llm/models/lfm2_tests.zig b/examples/llm/models/lfm2_tests.zig deleted file mode 100644 index 19b28df6..00000000 --- a/examples/llm/models/lfm2_tests.zig +++ /dev/null @@ -1,319 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const common = @import("common.zig"); -const lfm2 = @import("lfm2.zig"); -const model = @import("lfm2/model.zig"); - -pub const std_options: std.Options = .{ - .log_level = .info, -}; - -const Args = struct { - model: []const u8, - activations: []const u8, - backend: ?enum { stablehlo, cute, nki, metal } = null, - - pub const help = - \\Use lfm2_tests --model= --activations= [options] - \\ - \\ Validate the LFM2 implementation against activation fixtures. - \\ - \\ Options: - \\ --model= Path to the model repository - \\ --activations= Path to activation safetensors - \\ --backend= Attention backend to use - \\ - ; -}; - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - const io = init.io; - const args = zml.stdx.flags.parse(init.minimal.args, Args); - - const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator, io); - - const repo = try zml.safetensors.resolveModelRepo(io, args.model); - defer repo.close(io); - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - const meshes: common.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); - defer store.deinit(); - - var repo_model = try lfm2.LoadedModel.init(allocator, io, repo, store.view(), .{}); - defer repo_model.deinit(allocator); - - var progress = std.Progress.start(io, .{ .root_name = args.model }); - - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); - defer repo_model.unloadBuffers(&model_buffers, allocator); - - const backend: zml.attention.Backend = if (args.backend) |selected| switch (selected) { - .stablehlo => .{ .stablehlo = {} }, - .cute => .{ .flash_attention = .cute }, - .nki => .{ .flash_attention = .nki }, - .metal => .{ .flash_attention = .metal }, - } else zml.attention.Backend.auto(platform); - const params = lfm2.CompilationParameters.init( - repo_model.inner, - repo_model.parsed_config.value, - repo_model.parsed_config.value.max_position_embeddings, - backend, - meshes, - ); - progress.end(); - - try run(allocator, io, platform, args.activations, repo_model.parsed_config.value, repo_model.inner, &model_buffers, params.attention_metadata, params.attention_parameters, meshes); -} - -pub fn run( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - activations_path: []const u8, - config: lfm2.Config, - mdl: lfm2.Model, - model_buffers: *lfm2.Buffers, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - meshes: common.Meshes, -) !void { - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); - defer registry.deinit(); - - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); - defer activation_store.deinit(); - - var ctx = TestContext{ - .allocator = allocator, - .io = io, - .platform = platform, - .activations_store = &activation_store, - .attention_metadata = attention_metadata, - .attention_parameters = attention_parameters, - .mesh = platform.replicated_mesh, - }; - - try ctx.testLayer("embed_tokens", .{ .batch, .seq }, mdl.embed_tokens, model_buffers.embed_tokens, .{}); - - var num_attn_layers: usize = 0; - var num_conv_layers: usize = 0; - for (0..config.num_hidden_layers) |i| { - const kind = std.meta.stringToEnum(model.OperatorKind, config.layer_types[i]) orelse unreachable; - const layer = mdl.layers[i]; - const layer_buffers = model_buffers.layers[i]; - switch (kind) { - .conv => { - try ctx.testLayerPrint("layers.{d}.conv.in_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.conv.in_proj, layer_buffers.operator.conv.in_proj, .{}); - try ctx.testLayerPrint("layers.{d}.conv.out_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.conv.out_proj, layer_buffers.operator.conv.out_proj, .{}); - try ctx.testLayerPrint("layers.{d}.feed_forward.w1", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w1, layer_buffers.feed_forward.w1, .{}); - try ctx.testLayerPrint("layers.{d}.feed_forward.w2", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w2, layer_buffers.feed_forward.w2, .{}); - try ctx.testLayerPrint("layers.{d}.feed_forward.w3", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w3, layer_buffers.feed_forward.w3, .{}); - try ctx.testLayerPrint("layers.{d}.feed_forward", .{i}, .{ .batch, .seq, .d }, layer.feed_forward, layer_buffers.feed_forward, .{}); - try ctx.testConvLayer(i, num_conv_layers, mdl.layers[i].operator.conv, model_buffers.layers[i].operator.conv, .{}); - num_conv_layers += 1; - }, - .full_attention => { - try ctx.testLayerPrint("layers.{d}.self_attn.k_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.k_proj, layer_buffers.operator.self_attn.k_proj, .{}); - try ctx.testLayerPrint("layers.{d}.self_attn.q_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.q_proj, layer_buffers.operator.self_attn.q_proj, .{}); - try ctx.testLayerPrint("layers.{d}.self_attn.out_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.out_proj, layer_buffers.operator.self_attn.out_proj, .{}); - try ctx.testLayerPrint("layers.{d}.self_attn.v_proj", .{i}, .{ .batch, .seq, .d }, layer.operator.self_attn.v_proj, layer_buffers.operator.self_attn.v_proj, .{}); - try ctx.testLayerPrint("layers.{d}.feed_forward.w1", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w1, layer_buffers.feed_forward.w1, .{}); - try ctx.testLayerPrint("layers.{d}.feed_forward.w2", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w2, layer_buffers.feed_forward.w2, .{}); - try ctx.testLayerPrint("layers.{d}.feed_forward.w3", .{i}, .{ .batch, .seq, .d }, layer.feed_forward.w3, layer_buffers.feed_forward.w3, .{}); - try ctx.testLayerPrint("layers.{d}.feed_forward", .{i}, .{ .batch, .seq, .d }, layer.feed_forward, layer_buffers.feed_forward, .{}); - try ctx.testAttnLayer(i, num_attn_layers, mdl.layers[i].operator.self_attn, model_buffers.layers[i].operator.self_attn, .{}); - num_attn_layers += 1; - }, - } - } -} - -const TestContext = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - activations_store: *zml.io.TensorStore, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - mesh: *const zml.Mesh, - - fn testLayerPrint(self: *TestContext, comptime name_fmt: []const u8, name_args: anytype, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { - const name = try std.fmt.allocPrint(self.allocator, name_fmt, name_args); - defer self.allocator.free(name); - try self.testLayer(name, tagz, layer, layer_buffers, opts); - } - - fn testLayer(self: *TestContext, name: []const u8, tagz: anytype, layer: anytype, layer_buffers: anytype, opts: zml.testing.CompareOpts) !void { - std.log.info("Testing layer: {s}", .{name}); - - const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); - defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); - defer in_buffer.deinit(); - const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(tagz); - - const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); - defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); - defer out_buffer_expected.deinit(); - - const exe = if (comptime @TypeOf(layer) == model.TokenEmbedding) - try self.platform.compileFn(self.allocator, self.io, model.TokenEmbedding.forward, .{.{ .embedding = layer, .tokens = in_tensor }}, .{ .meshes = &.{self.mesh} }) - else - try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor }, .{ .meshes = &.{self.mesh} }); - defer exe.deinit(); - - var args = try exe.args(self.allocator); - defer args.deinit(self.allocator); - args.set(.{ layer_buffers, in_buffer }); - - var res = try exe.results(self.allocator); - defer res.deinit(self.allocator); - - exe.call(self.io, args, &res, .{}); - - var out_result = res.get(zml.Buffer); - defer out_result.deinit(); - - try zml.testing.expectClose(self.io, out_result, out_buffer_expected, opts); - std.log.info("Layer {s} passed!", .{name}); - } - - fn testConvLayer(self: *TestContext, ix: usize, cache_ix: usize, layer: model.ShortConv, layer_buffers: zml.Bufferized(model.ShortConv), opts: zml.testing.CompareOpts) !void { - const name = try std.fmt.allocPrint(self.allocator, "layers.{d}.conv", .{ix}); - defer self.allocator.free(name); - std.log.info("Testing layer: {s}", .{name}); - - const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); - defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); - defer in_buffer.deinit(); - const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); - - const cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache", .{name}); - defer self.allocator.free(cache_key); - var cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_key); - defer cache_buffer.deinit(); - const cache_tensor = zml.Tensor.fromShape(cache_buffer.shape()).withTags(.{ .layer, .batch, .seq, .d }); - - const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); - defer self.allocator.free(cache_pos_key); - var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); - defer cache_pos_buffer.deinit(); - const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); - - const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); - defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); - defer out_buffer_expected.deinit(); - - const actual_seq_len_tensor: zml.Tensor = .init(.{}, .u32); - const actual_seq_len: u32 = @intCast(in_tensor.dim(.seq)); - const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, actual_seq_len_tensor, model.ConvCache{ .state = cache_tensor }, cache_index_tensor, model.ConvParameters{ .is_prefill = false } }, .{ .meshes = &.{self.mesh} }); - defer exe.deinit(); - - const conv_cache: zml.Bufferized(model.ConvCache) = .{ .state = cache_buffer }; - - const actual_seq_len_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{actual_seq_len})); - var actual_seq_len_buf: zml.Buffer = try .fromSlice(self.io, self.platform, actual_seq_len_slice); - defer actual_seq_len_buf.deinit(); - - const cache_index_slice: zml.Slice = .init(zml.Shape.init(.{}, .u32), std.mem.sliceAsBytes(&[_]u32{@intCast(cache_ix)})); - var cache_index_buf: zml.Buffer = try .fromSlice(self.io, self.platform, cache_index_slice); - defer cache_index_buf.deinit(); - - var runner = try exe.runner(self.allocator); - defer runner.deinit(self.allocator); - runner.run( - self.io, - .{ layer_buffers, in_buffer, cache_pos_buffer, actual_seq_len_buf, conv_cache, cache_index_buf }, - .{ &in_buffer, &cache_buffer }, - .{}, - ); - - try zml.testing.expectClose(self.io, in_buffer, out_buffer_expected, opts); - std.log.info("Layer {s} passed!", .{name}); - } - - fn testAttnLayer(self: *TestContext, ix: usize, cache_ix: usize, layer: model.Attention, layer_buffers: zml.Bufferized(model.Attention), opts: zml.testing.CompareOpts) !void { - const name = try std.fmt.allocPrint(self.allocator, "layers.{d}.self_attn", .{ix}); - defer self.allocator.free(name); - std.log.info("Testing layer: {s}", .{name}); - - const in_key = try std.fmt.allocPrint(self.allocator, "{s}.in", .{name}); - defer self.allocator.free(in_key); - var in_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, in_key); - defer in_buffer.deinit(); - const in_tensor = zml.Tensor.fromShape(in_buffer.shape()).withTags(.{ .batch, .seq, .d }); - - const key_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.key", .{name}); - defer self.allocator.free(key_cache_key); - var key_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, key_cache_key); - defer key_cache_buffer.deinit(); - const key_cache_tensor = zml.Tensor.fromShape(key_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); - - const value_cache_key = try std.fmt.allocPrint(self.allocator, "{s}.cache.value", .{name}); - defer self.allocator.free(value_cache_key); - var value_cache_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, value_cache_key); - defer value_cache_buffer.deinit(); - const value_cache_tensor = zml.Tensor.fromShape(value_cache_buffer.shape()).withTags(.{ .layer, .batch, .h, .k, .hd }); - - const cache_pos_key = try std.fmt.allocPrint(self.allocator, "{s}.cache_position", .{name}); - defer self.allocator.free(cache_pos_key); - var cache_pos_buffer = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, cache_pos_key); - defer cache_pos_buffer.deinit(); - const cache_pos_tensor = zml.Tensor.fromShape(cache_pos_buffer.shape()).withTags(.{.batch}); - - const out_key = try std.fmt.allocPrint(self.allocator, "{s}.out", .{name}); - defer self.allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(self.allocator, self.io, self.platform, self.activations_store, out_key); - defer out_buffer_expected.deinit(); - - const cache_index_tensor: zml.Tensor = .init(.{}, .u32); - - const exe = try self.platform.compileFn(self.allocator, self.io, @TypeOf(layer).forward, .{ layer, in_tensor, cache_pos_tensor, model.KvCache{ .k = key_cache_tensor, .v = value_cache_tensor }, cache_index_tensor, self.attention_metadata, self.attention_parameters }, .{ .meshes = &.{self.mesh} }); - defer exe.deinit(); - - const kv_cache: zml.Bufferized(model.KvCache) = .{ .k = key_cache_buffer, .v = value_cache_buffer }; - - var attention_metadata_buffers = try self.attention_metadata.initBuffer(self.io, self.platform); - defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); - - var cache_index_buf: zml.Buffer = try .scalar(self.io, self.platform, cache_ix, .u32); - defer cache_index_buf.deinit(); - - var runner = try exe.runner(self.allocator); - defer runner.deinit(self.allocator); - runner.run( - self.io, - .{ layer_buffers, in_buffer, cache_pos_buffer, kv_cache, cache_index_buf, attention_metadata_buffers }, - .{ &in_buffer, &key_cache_buffer, &value_cache_buffer }, - .{}, - ); - - try zml.testing.expectClose(self.io, in_buffer, out_buffer_expected, opts); - std.log.info("Layer {s} passed!", .{name}); - } -}; - -fn loadBufferFromStore(allocator: std.mem.Allocator, io: anytype, platform: *zml.Platform, store: *zml.io.TensorStore, key: []const u8) !zml.Buffer { - const shape = store.view().getShape(key) orelse return error.NotFound; - - const host_bytes = try allocator.alloc(u8, shape.byteSize()); - defer allocator.free(host_bytes); - - var io_buffer: [8 * 1024]u8 = undefined; - var reader = try store.view().getReader(key, io, &io_buffer); - defer reader.deinit(); - - _ = try reader.interface.readSliceAll(host_bytes); - - return zml.Buffer.fromBytes(io, platform, shape, host_bytes); -} diff --git a/examples/llm/models/llama.zig b/examples/llm/models/llama.zig deleted file mode 100644 index 79f69e36..00000000 --- a/examples/llm/models/llama.zig +++ /dev/null @@ -1,9 +0,0 @@ -pub const inference = @import("llama/inference.zig"); -pub const CompilationParameters = inference.CompilationParameters; -pub const model = @import("llama/model.zig"); -pub const Config = model.Config; -pub const Buffers = model.Buffers; -pub const Model = model.Model; -pub const LoadedModel = model.LoadedModel; -pub const session = @import("llama/session.zig"); -pub const Session = session.Session; diff --git a/examples/llm/models/llama/inference.zig b/examples/llm/models/llama/inference.zig deleted file mode 100644 index 51ea3f9c..00000000 --- a/examples/llm/models/llama/inference.zig +++ /dev/null @@ -1,209 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const common = @import("../common.zig"); -const model = @import("model.zig"); - -const log = std.log.scoped(.llama); -const Phase = common.Phase; - -pub const CompilationParameters = struct { - prefill_tokens: zml.Tensor, - decode_tokens: zml.Tensor, - token_index: zml.Tensor, - kv_cache: model.KvCache, - rng: zml.Tensor.Rng, - attention_metadata: zml.attention.Metadata, - prefill_attention_parameters: zml.attention.Parameters, - decode_attention_parameters: zml.attention.Parameters, - seqlen: usize, - meshes: common.Meshes, - - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, backend: zml.attention.Backend, meshes: common.Meshes) CompilationParameters { - const head_dim = config.head_dim orelse @divExact(config.hidden_size, config.num_attention_heads); - - return .{ - .prefill_tokens = .init(.{ .s = seqlen }, .u32), - .decode_tokens = .init(.{ .s = 1 }, .u32), - .token_index = .init(.{}, .u32), - .kv_cache = .init(.init(.{ - .layer = mdl.model.layers.len, - .k = seqlen, - .h = config.num_key_value_heads, - .hd = head_dim, - }, mdl.model.embed_tokens.weight.dtype()), meshes.model), - .rng = .init(), - .attention_metadata = .init(.fromBackend(backend)), - .prefill_attention_parameters = .init(.fromBackend(backend)), - .decode_attention_parameters = .init(.fromBackend(backend)), - .seqlen = seqlen, - .meshes = meshes, - }; - } -}; - -pub const CompilationOptions = CompilationParameters; - -pub const Args = struct { - io: std.Io, - tokens_buf: *zml.Buffer, - next_token_buf: *zml.Buffer, - token_index_buf: *zml.Buffer, - active_length_buf: *zml.Buffer, - kv_cache_buffers: *zml.Bufferized(model.KvCache), - rng_buffers: *zml.Bufferized(zml.Tensor.Rng), - attention_metadata_buffers: *const zml.Bufferized(zml.attention.Metadata), -}; - -pub const CompiledModel = struct { - loaded_model: *const model.LoadedModel, - prefill: KernelExe, - decode: KernelExe, - params: CompilationParameters, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - loaded_model: *const model.LoadedModel, - llama_model: model.Model, - parameters: CompilationParameters, - progress: *std.Progress.Node, - ) !CompiledModel { - const prefill = try compileKernel(allocator, io, platform, llama_model, parameters, @intCast(parameters.prefill_tokens.dim(.s)), parameters.prefill_attention_parameters, .prefill, progress); - errdefer prefill.deinit(); - const decode = try compileKernel(allocator, io, platform, llama_model, parameters, @intCast(parameters.decode_tokens.dim(.s)), parameters.decode_attention_parameters, .decode, progress); - - return .{ - .loaded_model = loaded_model, - .prefill = prefill, - .decode = decode, - .params = parameters, - }; - } - - pub fn deinit(self: *CompiledModel) void { - self.prefill.deinit(); - self.decode.deinit(); - } -}; - -pub const Inference = CompiledModel; - -const Forward = struct { - pub const Input = struct { - weights: model.Model, - tokens: zml.Tensor, - token_index: zml.Tensor, - active_length: zml.Tensor, - kv_cache: model.KvCache, - rng: zml.Tensor.Rng, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - }; - - pub const Output = struct { - tokens: zml.Tensor, - kv_cache: model.KvCache, - rng: zml.Tensor.Rng, - }; - - pub fn forward(input: Input) Output { - var hidden = model.EmbedTokens.forward(.{ - .embedding = .{ .embed_tokens = input.weights.model.embed_tokens }, - .tokens = input.tokens, - }).hidden; - var kv_cache = input.kv_cache; - for (input.weights.model.layers, 0..) |layer, index| { - const result = model.TransformerLayer.forward(.{ - .layer = layer, - .hidden = hidden, - .token_index = input.token_index, - .kv_cache = kv_cache, - .kv_cache_index = .scalar(@as(u32, @intCast(index)), .u32), - .attention_metadata = input.attention_metadata, - .attention_parameters = input.attention_parameters, - }); - hidden = result.hidden; - kv_cache = result.kv_cache; - } - // LmHead owns final normalization; Llama.forward would apply it twice. - const result = model.LmHead.forward(.{ - .lm_head = .init(input.weights), - .hidden = hidden, - .active_length = input.active_length, - .rng = input.rng, - }); - return .{ - .tokens = if (input.tokens.dim(.s) == 1) result.tokens.reuseBuffer(input.tokens) else result.tokens, - .kv_cache = kv_cache.reuseBuffer(input.kv_cache), - .rng = .{ ._state = result.rng._state.reuseBuffer(input.rng._state) }, - }; - } -}; - -pub const KernelExe = zml.FnExe(Forward.forward); - -pub const KernelRunner = struct { - forward: KernelExe.Runner(.{.weights}), - - pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { - return .{ .forward = try .init(exe, allocator, .{ .weights = buffers.* }) }; - } - - pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { - self.forward.deinit(allocator); - } -}; - -pub fn run(runner: *KernelRunner, args: Args) void { - runner.forward.run(args.io, .{ - .inputs = .{ - .tokens = args.tokens_buf.*, - .token_index = args.token_index_buf.*, - .active_length = args.active_length_buf.*, - .kv_cache = args.kv_cache_buffers.*, - .rng = args.rng_buffers.*, - .attention_metadata = args.attention_metadata_buffers.*, - }, - .outputs = .{ - .tokens = args.next_token_buf, - .kv_cache = args.kv_cache_buffers, - .rng = args.rng_buffers, - }, - }, .{}); -} - -fn compileKernel( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - llama_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - attention_parameters: zml.attention.Parameters, - phase: Phase, - progress: *std.Progress.Node, -) !KernelExe { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("forward"), 1); - defer node.end(); - - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "forward", io, from); - - return KernelExe.compile(allocator, io, platform, .{ - .meshes = ¶meters.meshes.all(), - .program_name = phase.programName("llama", "forward"), - }, .{.{ - .weights = llama_model, - .tokens = zml.Tensor.init(.{ .s = seqlen }, .u32), - .token_index = parameters.token_index, - .active_length = .init(.{}, .u32), - .kv_cache = parameters.kv_cache, - .rng = parameters.rng, - .attention_metadata = parameters.attention_metadata, - .attention_parameters = attention_parameters, - }}); -} diff --git a/examples/llm/models/llama/model.zig b/examples/llm/models/llama/model.zig deleted file mode 100644 index 8a6a09e9..00000000 --- a/examples/llm/models/llama/model.zig +++ /dev/null @@ -1,651 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); -const stdx = zml.stdx; - -const common = @import("../common.zig"); -const inference = @import("inference.zig"); - -const log = std.log.scoped(.llama); - -pub const Config = struct { - bos_token_id: u32, - eos_token_id: EosTokens, - head_dim: ?u32 = null, - hidden_size: u32, - num_hidden_layers: u32, - num_attention_heads: u32, - num_key_value_heads: u32, - rope_theta: f32, - max_position_embeddings: u32, - rms_norm_eps: f32, - hf_rope_impl: bool = true, - tie_word_embeddings: bool = false, - rope_scaling: zml.nn.RopeOpts.Scaling = .{ .default = .{} }, - - pub const EosTokens = union(enum) { - int: u32, - ints: []u32, - - const Helpers = stdx.json.UnionHelpers(@This()); - pub const jsonParse = Helpers.jsonParse; - pub const jsonParseFromValue = Helpers.jsonParseFromValue; - pub const jsonStringify = Helpers.jsonStringify; - }; -}; - -const Options = struct { - sampling_strategy: ?zml.nn.SamplingStrategy, - max_seq_len: u32, -}; - -pub const LoadedModel = struct { - inner: Model, - parsed_config: std.json.Parsed(Config), - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - store: zml.io.TensorStore.View, - generation: common.GenerationOptions, - ) !LoadedModel { - const parsed_config = try common.parseConfig(Config, allocator, io, repo); - errdefer parsed_config.deinit(); - - const options: Options = .{ - .sampling_strategy = generation.sampling_strategy, - .max_seq_len = parsed_config.value.max_position_embeddings, - }; - - return .{ - .inner = try .init(allocator, store, parsed_config.value, options), - .parsed_config = parsed_config, - }; - } - - pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { - self.inner.deinit(allocator); - self.parsed_config.deinit(); - } - - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(_: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - if (buffers.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); - Llama.unloadBuffers(&buffers.model, allocator); - } - - pub fn compile( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - backend: zml.attention.Backend, - meshes: common.Meshes, - seqlen: usize, - progress: *std.Progress.Node, - ) !inference.CompiledModel { - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), backend, meshes); - - return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); - } -}; - -pub const Buffers = zml.Bufferized(Model); - -/// Llama architecture, using huggingface transformers naming. -/// Dimensions of activations: {.b, .s, .d} -pub const Model = struct { - lm_head: ?zml.nn.Linear, - model: Llama, - - gen_opts: zml.nn.SamplingStrategy = .{}, - config: Config, - - pub fn init( - allocator: std.mem.Allocator, - store: zml.io.TensorStore.View, - config: Config, - options: Options, - ) !Model { - const lm_head: ?zml.nn.Linear = if (store.withPrefix("lm_head").maybeCreateTensor( - "weight", - .{ .dout, .d }, - .model, - .{ .dout = .model, .d = .replicated }, - )) |weight| - .init(weight, null, .d) - else - null; - - return .{ - .lm_head = lm_head, - .model = try .init(allocator, store.withPrefix("model"), config), - .gen_opts = options.sampling_strategy orelse .{}, - .config = config, - }; - } - - pub fn deinit(self: Model, allocator: std.mem.Allocator) void { - self.model.deinit(allocator); - } - - pub fn loadBuffers( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - if (self.lm_head) |*lm_head| zml.nn.Linear.unloadBuffers(lm_head); - Llama.unloadBuffers(&self.model, allocator); - } - - fn lmHead(self: Model) LmHead { - return .init(self); - } - - /// Predicts the token at `token_index` position. - /// Returns: - /// - updated `tokens`, - /// - updated KV cache - /// - a Rng state to allow for probabilistic generation - pub fn forward( - self: Model, - tokens_: zml.Tensor, - token_index: zml.Tensor, - active_length: zml.Tensor, - kv_cache: KvCache, - rng: zml.Tensor.Rng, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { - const tokens = tokens_.withPartialTags(.{.s}); - - const out, const updated_kv_cache = self.model.forward( - tokens, - token_index, - kv_cache, - attention_metadata, - attention_parameters, - ); - - const sample = LmHead.forward(.{ - .lm_head = self.lmHead(), - .hidden = out, - .active_length = active_length, - .rng = rng, - }); - - return .{ sample.tokens, updated_kv_cache, sample.rng }; - } -}; - -const Llama = struct { - embed_tokens: zml.nn.TokenEmbedding, - norm: RmsNorm, - layers: []TransformerLayer, - - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) !Llama { - const layers = try allocator.alloc(TransformerLayer, config.num_hidden_layers); - errdefer allocator.free(layers); - - for (layers, 0..) |*layer, i| { - layer.* = try .init(store.withPrefix("layers").withLayer(i), config); - } - - return .{ - .embed_tokens = .{ .weight = store.createTensor( - "embed_tokens.weight", - .{ .voc, .d }, - .model, - .{ .voc = .replicated, .d = .model }, - ) }, - .norm = .{ - .weight = store.withPrefix("norm").createTensor("weight", .{.d}, .model, .{ .d = .replicated }), - .eps = config.rms_norm_eps, - }, - .layers = layers, - }; - } - - pub fn deinit(self: Llama, allocator: std.mem.Allocator) void { - allocator.free(self.layers); - } - - pub fn unloadBuffers(self: *zml.Bufferized(Llama), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - RmsNorm.unloadBuffers(&self.norm); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - } - - /// Forward one token, using KV cache for previous tokens. - /// Returns result and updated KV cache. - pub fn forward( - self: Llama, - tokens: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - ) struct { zml.Tensor, KvCache } { - const embeds = self.embed_tokens.forward(tokens).withPartialTags(.{.d}); - var hidden = embeds; - var updated_kv_cache = kv_cache; - var kv_cache_index = zml.Tensor.scalar(0, .u32); - - for (self.layers) |layer| { - const result = TransformerLayer.forward(.{ - .layer = layer, - .hidden = hidden, - .token_index = token_index, - .kv_cache = updated_kv_cache, - .kv_cache_index = kv_cache_index, - .attention_metadata = attention_metadata, - .attention_parameters = attention_parameters, - }); - hidden = result.hidden; - updated_kv_cache = result.kv_cache; - kv_cache_index = kv_cache_index.add(zml.Tensor.scalar(@as(u32, 1), .u32)); - } - - return .{ self.norm.forward(hidden), updated_kv_cache.reuseBuffer(kv_cache) }; - } -}; - -pub const EmbedTokens = struct { - embed_tokens: zml.nn.TokenEmbedding, - - pub const Input = struct { - embedding: EmbedTokens, - tokens: zml.Tensor, - }; - - pub const Output = struct { - hidden: zml.Tensor, - }; - - pub fn forward(input: Input) Output { - const tokens = input.tokens.withPartialTags(.{.s}); - return .{ .hidden = input.embedding.embed_tokens.forward(tokens) - .withPartialTags(.{.d}) - .replicate() }; - } -}; - -pub const LmHead = struct { - lm_head: ?zml.nn.Linear, - embed_tokens: zml.nn.TokenEmbedding, - norm: RmsNorm, - gen_opts: zml.nn.SamplingStrategy, - - pub fn init(mdl: Model) LmHead { - return .{ - .lm_head = mdl.lm_head, - .embed_tokens = mdl.model.embed_tokens, - .norm = mdl.model.norm, - .gen_opts = mdl.gen_opts, - }; - } - - pub const Input = struct { - lm_head: LmHead, - hidden: zml.Tensor, - active_length: zml.Tensor, - rng: zml.Tensor.Rng, - }; - - pub const Output = struct { - tokens: zml.Tensor, - rng: zml.Tensor.Rng, - }; - - pub fn forward(input: Input) Output { - const self = input.lm_head; - const last_index = input.active_length.convert(.i32).addConstant(-1); - const hidden = self.norm.forward(input.hidden.withPartialTags(.{ .s, .d }).slice(.s, .dyn(last_index, 1))); - - var logits = blk: { - if (self.lm_head) |lm_head| { - break :blk lm_head.forward(hidden, hidden.dtype()).rename(.{ .dout = .d }); - } else { - break :blk self.embed_tokens.weight.withTags(.{ .voc, .d }).dot(hidden, .d); - } - }; - - if (logits.shape().hasTag(.voc) == null) - logits = logits.rename(.{ .d = .voc }); - - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_opts, input.rng); - return .{ .tokens = next_tokens.convert(.u32), .rng = new_rng }; - } -}; - -pub const TransformerLayer = struct { - input_layernorm: RmsNorm, - self_attn: SelfAttn, - post_attention_layernorm: RmsNorm, - mlp: Mlp, - - pub fn init(store: zml.io.TensorStore.View, config: Config) !TransformerLayer { - return .{ - .input_layernorm = .init(store.withPrefix("input_layernorm"), config.rms_norm_eps), - .self_attn = try .init(store.withPrefix("self_attn"), config), - .post_attention_layernorm = .init(store.withPrefix("post_attention_layernorm"), config.rms_norm_eps), - .mlp = .init(store.withPrefix("mlp")), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - SelfAttn.unloadBuffers(&self.self_attn); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - Mlp.unloadBuffers(&self.mlp); - } - - pub const Input = struct { - layer: TransformerLayer, - hidden: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache, - kv_cache_index: zml.Tensor, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - }; - - pub const Output = struct { - hidden: zml.Tensor, - kv_cache: KvCache, - }; - - pub fn forward(input: Input) Output { - const self = input.layer; - const x0 = input.hidden; - // Self Attention - //log.debug("TransformerLayer({f}) -> {f}", .{ x0, self.input_layernorm.forward(x0) }); - stdx.debug.assert(x0.rank() >= 2 and x0.shape().hasTags(.{ .s, .d }), "TransformerLayer expected input shape: {{..., .s, .d}}, received: {f}", .{x0}); - - // Keep the residual stream replicated to avoid repeated gathers before q/k/v. - const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); - const x0_normalized = self.input_layernorm.forward(x0_replicated); - const delta0, const updated_kv_cache = self.self_attn.forward( - x0_normalized, - input.token_index, - input.kv_cache, - input.kv_cache_index, - input.attention_metadata, - input.attention_parameters, - ); - - // Fully Connected - const x1 = x0_replicated.add(delta0).withPartitioning(.model, .{ .d = .replicated }); - const x1_normalized = self.post_attention_layernorm.forward(x1); - const x2 = self.mlp.forward(x1_normalized) - .rename(.{ .dout = .d }) - .withPartitioning(.model, .{ .d = .replicated }) - .add(x1) - .withPartitioning(.model, .{ .d = .replicated }); - - return .{ .hidden = x2.reuseBuffer(x0), .kv_cache = updated_kv_cache }; - } -}; - -const RmsNorm = struct { - weight: zml.Tensor, - eps: f32, - - pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ - .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), - .eps = eps, - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - - /// L2 normalization of input tensor along `.d` axis. - pub fn forward(self: RmsNorm, input: zml.Tensor) zml.Tensor { - const x = if (input.shape().isFullyTagged()) input else input.withPartialTags(.{.d}); - const normalized = zml.nn.rmsNorm(x, .d, self.eps); - return normalized.mul(self.weight.convert(x.dtype()).withTags(.{.d}).broad(x.shape())); - } -}; - -const Mlp = struct { - up_proj: zml.nn.Linear, // (dim -> hidden_dim) - gate_proj: zml.nn.Linear, // (dim -> hidden_dim) - down_proj: zml.nn.Linear, // (hidden_dim -> dim) - - pub fn init(store: zml.io.TensorStore.View) Mlp { - return .{ - .up_proj = .init(store.createTensor("up_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), - .gate_proj = .init(store.createTensor("gate_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), - .down_proj = .init(store.createTensor("down_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const proj = self.up_proj.forward(x, x.dtype()); - var output = self.gate_proj.forward(x, x.dtype()); - output = output.silu().mul(proj).rename(.{ .dout = .d }); - return self.down_proj.forward(output, output.dtype()); - } -}; - -const SelfAttn = struct { - q_proj: zml.nn.Linear, - k_proj: zml.nn.Linear, - v_proj: zml.nn.Linear, - - q_norm: ?RmsNorm, - k_norm: ?RmsNorm, - - o_proj: zml.nn.Linear, - num_heads: i64 = undefined, - num_kv_heads: i64 = 0, - rope_opts: zml.nn.RopeOpts = undefined, - - pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { - var rope_scaling = config.rope_scaling; - rope_scaling.setRopeTheta(config.rope_theta); - return .{ - .q_proj = .init(store.createTensor("q_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), - .k_proj = .init(store.createTensor("k_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), - .v_proj = .init(store.createTensor("v_proj.weight", .{ .dout, .d }, .model, .{ .dout = .model }), null, .d), - .o_proj = .init(store.createTensor("o_proj.weight", .{ .dout, .d }, .model, .{ .d = .model }), null, .d), - // TODO(Corentin): fix that - .q_norm = null, - .k_norm = null, - .num_heads = @intCast(config.num_attention_heads), - .num_kv_heads = @intCast(config.num_key_value_heads), - .rope_opts = .{ - .layout = if (config.hf_rope_impl) .real_im_pass else .interleaved, - .scaling = rope_scaling, - }, - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - - /// Self zml.attention. - /// - If token_index is set, x is assumed to be the representation of one new token, - /// and kv_cache will be read for the previous tokens. - /// - If token_index is not set, x is assumed to be the representation of all tokens - /// since the beginning of the sequence, and kv_cache won't be read. - /// In both case, kv_cache will be updated with the computed key and value. - /// x: {.b, .s, .d } -> .{.b, .s, .d} - pub fn forward( - self: SelfAttn, - x: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache, - kv_cache_index: zml.Tensor, - attention_metadata: zml.attention.Metadata, - attention_parameters: zml.attention.Parameters, - ) struct { zml.Tensor, KvCache } { - const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - - // Make hidden state replicated once and reuse it across q/k/v projections. - // This avoids paying gather-style collectives independently for each projection. - const x_qkv = x.withPartitioning(.model, .{ .d = .replicated }); - - var q = self.q_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = self.num_heads, .hd = .auto }); - var k = self.k_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); - var v = self.v_proj.forward(x_qkv, x_qkv.dtype()).splitAxis(-1, .{ .h = num_kv_heads, .hd = .auto }); - - // In self-attention, .s axis is used both for keys and queries. - const pos_index = b: { - const temp = zml.Tensor.arange(.{ .end = x.dim(.s) }, token_index.dtype()).withTags(.{.s}).broad(zml.Shape.init(.{ .s = x.dim(.s) }, token_index.dtype())); - break :b temp.add(token_index.broad(temp.shape())); - }; - - if (self.q_norm) |norm| q = norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); - if (self.k_norm) |norm| k = norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); - q = zml.nn.rope(q, pos_index, self.rope_opts); - k = zml.nn.rope(k, pos_index, self.rope_opts); - q = q.rename(.{ .s = .q }); - k = k.rename(.{ .s = .k }); - v = v.rename(.{ .s = .k }); - - const dtype = q.dtype(); - const new_kv_cache = kv_cache.updateAt(k, v, token_index, kv_cache_index); - k = new_kv_cache.keysAt(kv_cache_index).convert(dtype); - v = new_kv_cache.valuesAt(kv_cache_index).convert(dtype); - - const attn_output = zml.attention.attention( - q, - k, - v, - token_index, - attention_metadata, - attention_parameters, - zml.Compiler.current().mesh(.model), - .head(.model), - ); - - const attn = attn_output.merge(.{ .d = .{ .h, .hd } }).rename(.{ .q = .s }); - const delta = self.o_proj.forward(attn, attn.dtype()) - .rename(.{ .dout = .d }) - .withPartitioning(.model, .{ .d = .replicated }); - return .{ delta, new_kv_cache }; - } -}; - -pub const KvCache = struct { - k: zml.Tensor, - v: zml.Tensor, - - pub const Buffer = zml.Bufferized(KvCache); - - pub fn init(kv_shape: zml.Shape, mesh: *const zml.Mesh) KvCache { - const sharded_shape = kv_shape.withPartitioning(mesh, .{ .h = .model }); - - return .{ - .k = .fromShape(sharded_shape), - .v = .fromShape(sharded_shape), - }; - } - - pub fn initBuffer(kv: KvCache, io: std.Io, platform: *const zml.Platform) !Buffer { - return .{ - .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), - .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), - }; - } - - pub fn deinitBuffer(kv: *Buffer) void { - kv.k.deinit(); - kv.v.deinit(); - } - - pub fn keysAt(kv: KvCache, layer_index: zml.Tensor) zml.Tensor { - return kv.k.slice(.layer, .dynSingle(layer_index)); - } - - pub fn valuesAt(kv: KvCache, layer_index: zml.Tensor) zml.Tensor { - return kv.v.slice(.layer, .dynSingle(layer_index)); - } - - pub fn updateAt(kv: KvCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: zml.Tensor, layer_index: zml.Tensor) KvCache { - const k_shape = kv.k.shape().drop(.layer); - return .{ - .k = kv.k.scatterSlices(.{ .layer = layer_index, .k = token_index }, new_k.convert(kv.k.dtype()).transpose(k_shape), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.k), - .v = kv.v.scatterSlices(.{ .layer = layer_index, .k = token_index }, new_v.convert(kv.v.dtype()).transpose(kv.v.shape().drop(.layer)), .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }).reuseBuffer(kv.v), - }; - } - - pub fn reuseBuffer(kv: KvCache, other: KvCache) KvCache { - return .{ - .k = kv.k.reuseBuffer(other.k), - .v = kv.v.reuseBuffer(other.v), - }; - } -}; diff --git a/examples/llm/models/llama/session.zig b/examples/llm/models/llama/session.zig deleted file mode 100644 index 8a6532e6..00000000 --- a/examples/llm/models/llama/session.zig +++ /dev/null @@ -1,222 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const inference = @import("inference.zig"); -const model = @import("model.zig"); - -pub const Session = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - compiled_model: *inference.CompiledModel, - prefill: inference.KernelRunner, - decode: inference.KernelRunner, - kv_cache_buffers: zml.Bufferized(model.KvCache), - token_index_buffers: []zml.Buffer, - rng_buffers: zml.Bufferized(zml.Tensor.Rng), - tokenizer: zml.tokenizer.Tokenizer, - config: *const model.Config, - seqlen: u32, - last_generated_token: u32 = 0, - conversation_id: u64, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - tokenizer: zml.tokenizer.Tokenizer, - compiled_model: *inference.CompiledModel, - model_buffers: *model.Buffers, - ) !Session { - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); - errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); - - const token_index_buffers = try allocator.alloc(zml.Buffer, compiled_model.params.seqlen); - errdefer allocator.free(token_index_buffers); - var initialized_token_index_buffers: usize = 0; - errdefer { - for (token_index_buffers[0..initialized_token_index_buffers]) |*token_index_buffer| { - token_index_buffer.deinit(); - } - } - for (token_index_buffers, 0..) |*token_index_buffer, i| { - token_index_buffer.* = try zml.Buffer.scalar(io, platform, i, .u32); - initialized_token_index_buffers = i + 1; - } - - const conversation_id: u64 = @bitCast(std.Io.Clock.now(.real, io).toMicroseconds()); - - const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); - errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); - - var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); - errdefer prefill.deinit(allocator); - const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); - - return .{ - .allocator = allocator, - .io = io, - .platform = platform, - .compiled_model = compiled_model, - .prefill = prefill, - .decode = decode, - .kv_cache_buffers = kv_cache_buffers, - .token_index_buffers = token_index_buffers, - .rng_buffers = rng_buffers, - .tokenizer = tokenizer, - .config = &compiled_model.loaded_model.parsed_config.value, - .seqlen = @intCast(compiled_model.params.seqlen), - .conversation_id = conversation_id, - }; - } - - pub fn deinit(self: *Session) void { - self.prefill.deinit(self.allocator); - self.decode.deinit(self.allocator); - model.KvCache.deinitBuffer(&self.kv_cache_buffers); - for (self.token_index_buffers) |*token_index_buffer| { - token_index_buffer.deinit(); - } - self.allocator.free(self.token_index_buffers); - zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); - } - - pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { - var encoder = try self.tokenizer.encoder(); - defer encoder.deinit(); - - const start_header = self.tokenizer.tokenId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = self.tokenizer.tokenId("<|end_header_id|>") orelse return error.NoSuchToken; - const eot = self.tokenizer.tokenId("<|eot_id|>") orelse return error.NoSuchToken; - const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; - - var tokens = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); - try tokens.ensureUnusedCapacity(prompt.len); - - const w: *std.Io.Writer = &tokens.writer; - try encoder.appendTokens(w, &.{ self.config.bos_token_id, start_header }); - try encoder.encode(w, "user"); - try encoder.appendTokens(w, &.{ end_header, newline }); - try encoder.encode(w, prompt); - try encoder.appendTokens(w, &.{ eot, newline, start_header }); - try encoder.encode(w, "assistant"); - try encoder.appendTokens(w, &.{ end_header, newline }); - - return @ptrCast(@alignCast(try tokens.toOwnedSlice())); - } - - pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { - var encoder = try self.tokenizer.encoder(); - defer encoder.deinit(); - - const start_header = self.tokenizer.tokenId("<|start_header_id|>") orelse return error.NoSuchToken; - const end_header = self.tokenizer.tokenId("<|end_header_id|>") orelse return error.NoSuchToken; - const eot = self.tokenizer.tokenId("<|eot_id|>") orelse return error.NoSuchToken; - const newline = self.tokenizer.tokenId("\\n") orelse return error.NoSuchToken; - - var tokens = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); - try tokens.ensureUnusedCapacity(prompt.len); - - const w: *std.Io.Writer = &tokens.writer; - try encoder.appendTokens(w, &.{ eot, newline, start_header }); - try encoder.encode(w, "user"); - try encoder.appendTokens(w, &.{ end_header, newline }); - try encoder.encode(w, prompt); - try encoder.appendTokens(w, &.{ eot, newline, start_header }); - try encoder.encode(w, "assistant"); - try encoder.appendTokens(w, &.{ end_header, newline }); - - return @ptrCast(@alignCast(try tokens.toOwnedSlice())); - } - - pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { - const prefill_tokens_slice: zml.Slice = try .alloc(self.allocator, .init(.{self.seqlen}, .u32)); - defer prefill_tokens_slice.free(self.allocator); - @memset(prefill_tokens_slice.items(u32), 0); - @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - - var prefill_tokens_buffer: zml.Buffer = try .fromSlice(self.io, self.platform, prefill_tokens_slice); - defer prefill_tokens_buffer.deinit(); - var active_length_buffer: zml.Buffer = try .scalar(self.io, self.platform, all_tokens.len, .u32); - defer active_length_buffer.deinit(); - - const params = self.compiled_model.params; - var attention_metadata_buffers: zml.Bufferized(zml.attention.Metadata) = switch (params.attention_metadata) { - .stablehlo => try params.attention_metadata.initBuffer(self.io, self.platform), - .flash_attention => |metadata| .{ .flash_attention = switch (metadata) { - .metal => .{ .metal = .{ .num_tokens = try .scalar(self.io, self.platform, all_tokens.len, .u32) } }, - else => try metadata.initBuffer(self.io, self.platform), - } }, - }; - defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); - - var next_token_buffer: zml.Buffer = undefined; - inference.run(&self.prefill, .{ - .io = self.io, - .tokens_buf = &prefill_tokens_buffer, - .next_token_buf = &next_token_buffer, - .token_index_buf = &self.token_index_buffers[0], - .active_length_buf = &active_length_buffer, - .kv_cache_buffers = &self.kv_cache_buffers, - .rng_buffers = &self.rng_buffers, - .attention_metadata_buffers = &attention_metadata_buffers, - }); - - defer next_token_buffer.deinit(); - self.last_generated_token = try next_token_buffer.getValue(u32, self.io); - } - - pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { - var decoder: zml.tokenizer.Tokenizer.Decoder = try self.tokenizer.decoder(); - defer decoder.deinit(); - - const decoder_out_buffer: []u8 = try self.allocator.alloc(u8, 256); - defer self.allocator.free(decoder_out_buffer); - - var last_token_id: u32 = self.last_generated_token; - var current_token_buffer: zml.Buffer = try .fromBytes(self.io, self.platform, .init(.{ .s = 1 }, .u32), @ptrCast(&last_token_id)); - defer current_token_buffer.deinit(); - var active_length_buffer: zml.Buffer = try .scalar(self.io, self.platform, 1, .u32); - defer active_length_buffer.deinit(); - - const params = self.compiled_model.params; - var attention_metadata_buffers = try params.attention_metadata.initBuffer(self.io, self.platform); - defer zml.attention.Metadata.deinitBuffer(&attention_metadata_buffers); - - generation: while (true) { - if (isEosToken(self.config, last_token_id)) break :generation; - - try stdout.writeAll(try decoder.feedOne(last_token_id, decoder_out_buffer)); - try stdout.flush(); - - try all_tokens.append(self.allocator, last_token_id); - if (all_tokens.items.len >= self.seqlen) break :generation; - - inference.run(&self.decode, .{ - .io = self.io, - .tokens_buf = ¤t_token_buffer, - .next_token_buf = ¤t_token_buffer, - .token_index_buf = &self.token_index_buffers[all_tokens.items.len], - .active_length_buf = &active_length_buffer, - .kv_cache_buffers = &self.kv_cache_buffers, - .rng_buffers = &self.rng_buffers, - .attention_metadata_buffers = &attention_metadata_buffers, - }); - last_token_id = try current_token_buffer.getValue(u32, self.io); - } - - try stdout.writeAll(try decoder.finalize(decoder_out_buffer)); - try stdout.flush(); - } -}; - -fn isEosToken(config: *const model.Config, token_id: u32) bool { - return switch (config.eos_token_id) { - .int => |eos| token_id == eos, - .ints => |eos_list| for (eos_list) |eos| { - if (token_id == eos) break true; - } else false, - }; -} diff --git a/examples/llm/models/llama_tests.zig b/examples/llm/models/llama_tests.zig deleted file mode 100644 index 1b5894f1..00000000 --- a/examples/llm/models/llama_tests.zig +++ /dev/null @@ -1,149 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const common = @import("common.zig"); -const llama = @import("llama.zig"); -const model = @import("llama/model.zig"); - -pub const std_options: std.Options = .{ - .log_level = .info, -}; - -const Args = struct { - model: []const u8, - activations: []const u8, - - pub const help = - \\Use llama_tests --model= --activations= - \\ - \\ Validate the LLaMA implementation against activation fixtures. - \\ - \\ Options: - \\ --model= Path to the model repository - \\ --activations= Path to activation safetensors - \\ - ; -}; - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - const io = init.io; - const args = zml.stdx.flags.parse(init.minimal.args, Args); - - const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator, io); - - const repo = try zml.safetensors.resolveModelRepo(io, args.model); - defer repo.close(io); - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - const meshes: common.Meshes = try .init(platform); - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &meshes.all()); - defer store.deinit(); - - var repo_model = try llama.LoadedModel.init(allocator, io, repo, store.view(), .{}); - defer repo_model.deinit(allocator); - - var progress = std.Progress.start(io, .{ .root_name = args.model }); - - var model_buffers = try repo_model.loadBuffers(allocator, io, platform, &store, &progress); - defer repo_model.unloadBuffers(&model_buffers, allocator); - progress.end(); - - try run(allocator, io, platform, args.activations, repo_model.inner, &model_buffers, &meshes.all()); -} - -fn run( - allocator: std.mem.Allocator, - io: std.Io, - platform: *zml.Platform, - activations_path: []const u8, - mdl: model.Model, - model_buffers: *model.Buffers, - meshes: []const *const zml.Mesh, -) !void { - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, activations_path); - defer registry.deinit(); - - var activation_store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, meshes); - defer activation_store.deinit(); - - try testLayer(allocator, io, platform, activation_store.view(), "embed_tokens", mdl.model.embed_tokens, model_buffers.model.embed_tokens, meshes, .{ .absolute_tolerance = 1e-3 }); - - if (mdl.model.layers.len == 0) return; - - const layer = mdl.model.layers[0]; - const layer_buffers = model_buffers.model.layers[0]; - - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.v_proj", layer.self_attn.v_proj, layer_buffers.self_attn.v_proj, meshes, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.q_proj", layer.self_attn.q_proj, layer_buffers.self_attn.q_proj, meshes, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.k_proj", layer.self_attn.k_proj, layer_buffers.self_attn.k_proj, meshes, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.self_attn.o_proj", layer.self_attn.o_proj, layer_buffers.self_attn.o_proj, meshes, .{ .absolute_tolerance = 2e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.mlp", layer.mlp, layer_buffers.mlp, meshes, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.input_layernorm", layer.input_layernorm, layer_buffers.input_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); - try testLayer(allocator, io, platform, activation_store.view(), "layers.0.post_attention_layernorm", layer.post_attention_layernorm, layer_buffers.post_attention_layernorm, meshes, .{ .absolute_tolerance = 1e-2 }); -} - -fn testLayer( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - activation_store: zml.io.TensorStore.View, - name: []const u8, - layer: anytype, - layer_weights: zml.Bufferized(@TypeOf(layer)), - meshes: []const *const zml.Mesh, - opts: zml.testing.CompareOpts, -) !void { - const in_key = try std.fmt.allocPrint(allocator, "{s}.in", .{name}); - defer allocator.free(in_key); - const in_shape = activation_store.getShape(in_key) orelse return error.NotFound; - var in_buffer = try loadBufferFromStore(allocator, io, platform, activation_store, in_key); - defer in_buffer.deinit(); - const in_tensor = zml.Tensor.fromShape(in_shape); - - const out_key = try std.fmt.allocPrint(allocator, "{s}.out", .{name}); - defer allocator.free(out_key); - var out_buffer_expected = try loadBufferFromStore(allocator, io, platform, activation_store, out_key); - defer out_buffer_expected.deinit(); - - // `zml.nn.Linear.forward` takes an explicit output dtype; every other layer here - // is `fn (self, Tensor) Tensor`. - const Layer = @TypeOf(layer); - const Call = struct { - fn forward(l: Layer, x: zml.Tensor) zml.Tensor { - return if (Layer == zml.nn.Linear) l.forward(x, x.dtype()) else Layer.forward(l, x); - } - }; - const exe = try platform.compileFn(allocator, io, Call.forward, .{ layer, in_tensor }, .{ .meshes = meshes }); - defer exe.deinit(); - - var args = try exe.args(allocator); - defer args.deinit(allocator); - args.set(.{ layer_weights, in_buffer }); - - var res = try exe.results(allocator); - defer res.deinit(allocator); - - exe.call(io, args, &res, .{}); - - var out_result = res.get(zml.Buffer); - defer out_result.deinit(); - try zml.testing.expectClose(io, out_result, out_buffer_expected, opts); -} - -fn loadBufferFromStore(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, store: zml.io.TensorStore.View, key: []const u8) !zml.Buffer { - const shape = store.getShape(key) orelse return error.NotFound; - - const host_bytes = try allocator.alloc(u8, shape.byteSize()); - defer allocator.free(host_bytes); - - var io_buffer: [8 * 1024]u8 = undefined; - var reader = try store.getReader(key, io, &io_buffer); - defer reader.deinit(); - - _ = try reader.interface.readSliceAll(host_bytes); - - return zml.Buffer.fromBytes(io, platform, shape, host_bytes); -} diff --git a/examples/llm/models/qwen3_5.zig b/examples/llm/models/qwen3_5.zig deleted file mode 100644 index 4147f695..00000000 --- a/examples/llm/models/qwen3_5.zig +++ /dev/null @@ -1,11 +0,0 @@ -pub const inference = @import("qwen3_5/inference.zig"); -pub const CompilationParameters = inference.CompilationParameters; -pub const CompilationOptions = inference.CompilationParameters; -pub const CompiledModel = inference.CompiledModel; -pub const model = @import("qwen3_5/model.zig"); -pub const Config = model.Config; -pub const Buffers = model.Buffers; -pub const Model = model.Model; -pub const LoadedModel = model.LoadedModel; -pub const session = @import("qwen3_5/session.zig"); -pub const Session = session.Session; diff --git a/examples/llm/models/qwen3_5/inference.zig b/examples/llm/models/qwen3_5/inference.zig deleted file mode 100644 index 50d3da30..00000000 --- a/examples/llm/models/qwen3_5/inference.zig +++ /dev/null @@ -1,320 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const common = @import("../common.zig"); -const model = @import("model.zig"); - -const log = std.log.scoped(.qwen3_5); -const Phase = common.Phase; - -pub const CompilationParameters = struct { - prefill_tokens: zml.Tensor, - decode_tokens: zml.Tensor, - token_index: zml.Tensor, - kv_cache: model.KvCache, - rng: zml.Tensor.Rng, - seqlen: u32, - meshes: common.Meshes, - - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, meshes: common.Meshes) CompilationParameters { - const dtype = mdl.text_model.embed_tokens.weight.dtype(); - return .{ - .prefill_tokens = .init(.{ .b = 1, .s = seqlen }, .u32), - .decode_tokens = .init(.{ .b = 1, .s = 1 }, .u32), - .token_index = .init(.{}, .u32), - .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), - .rng = .init(), - .seqlen = seqlen, - .meshes = meshes, - }; - } -}; - -pub const CompilationOptions = CompilationParameters; - -pub const Args = struct { - io: std.Io, - tokens_buf: *zml.Buffer, - next_token_buf: *zml.Buffer, - token_index_buf: *zml.Buffer, - active_length_buf: *zml.Buffer, - kv_cache_buffers: *zml.Bufferized(model.KvCache), - rng_buffers: *zml.Bufferized(zml.Tensor.Rng), -}; - -pub const CompiledModel = struct { - loaded_model: *const model.LoadedModel, - prefill: KernelExe, - decode: KernelExe, - params: CompilationParameters, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - loaded_model: *const model.LoadedModel, - qwen_model: model.Model, - parameters: CompilationParameters, - progress: *std.Progress.Node, - ) !CompiledModel { - const prefill = try compileKernel(allocator, io, platform, qwen_model, parameters, @intCast(parameters.prefill_tokens.dim(.s)), .prefill, progress); - errdefer prefill.deinit(); - const decode = try compileKernel(allocator, io, platform, qwen_model, parameters, @intCast(parameters.decode_tokens.dim(.s)), .decode, progress); - return .{ - .loaded_model = loaded_model, - .prefill = prefill, - .decode = decode, - .params = parameters, - }; - } - - pub fn deinit(self: *CompiledModel) void { - self.prefill.deinit(); - self.decode.deinit(); - } -}; - -pub const Inference = CompiledModel; - -pub const KernelExe = struct { - embed: zml.FnExe(model.EmbedTokens.forward), - full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn), - linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn), - sample: zml.FnExe(model.Sampler.sampleTokens), - - pub fn deinit(self: *const KernelExe) void { - self.embed.deinit(); - self.full_attention.deinit(); - self.linear_attention.deinit(); - self.sample.deinit(); - } -}; - -pub const LayerRunner = union(enum) { - full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}), - linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}), - - pub fn deinit(self: *LayerRunner, allocator: std.mem.Allocator) void { - switch (self.*) { - inline else => |*runner| runner.deinit(allocator), - } - } -}; - -pub const KernelRunner = struct { - embed: zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}), - layers: []LayerRunner, - sample: zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}), - - pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { - var embed = try zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ - .embedding = .{ .embed_tokens = buffers.text_model.embed_tokens }, - }); - errdefer embed.deinit(allocator); - - const layers = try allocator.alloc(LayerRunner, buffers.text_model.layers.len); - errdefer allocator.free(layers); - var initialized_layers: usize = 0; - errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); - for (layers, buffers.text_model.layers) |*layer, layer_buffers| { - layer.* = switch (layer_buffers.attn) { - .full_attention => .{ .full_attention = try zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}).init( - &exe.full_attention, - allocator, - .{ .layer = layer_buffers }, - ) }, - .linear_attention => .{ .linear_attention = try zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}).init( - &exe.linear_attention, - allocator, - .{ .layer = layer_buffers }, - ) }, - }; - initialized_layers += 1; - } - - var sample = try zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}).init(&exe.sample, allocator, .{ - .sampler = .{ - .norm = buffers.text_model.norm, - .lm_head = buffers.lm_head, - }, - }); - errdefer sample.deinit(allocator); - - return .{ .embed = embed, .layers = layers, .sample = sample }; - } - - pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { - self.embed.deinit(allocator); - for (self.layers) |*layer| layer.deinit(allocator); - allocator.free(self.layers); - self.sample.deinit(allocator); - } -}; - -pub fn run(runner: *KernelRunner, args: Args, layer_index_buffers: []const zml.Buffer) void { - var hidden_buffer: zml.Buffer = undefined; - runner.embed.run(args.io, .{ - .inputs = .{ - .tokens = args.tokens_buf.*, - }, - .outputs = .{ .hidden = &hidden_buffer }, - }, .{}); - defer hidden_buffer.deinit(); - - for (runner.layers, layer_index_buffers) |*layer_runner, layer_index_buffer| { - switch (layer_runner.*) { - .full_attention => |*layer| { - var layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache) = .{ - .k = args.kv_cache_buffers.self_attn.k, - .v = args.kv_cache_buffers.self_attn.v, - .layer_index = layer_index_buffer, - }; - layer.run(args.io, .{ - .inputs = .{ - .hidden = hidden_buffer, - .token_index = args.token_index_buf.*, - .cache = layer_cache, - }, - .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, - }, .{}); - args.kv_cache_buffers.self_attn.k = layer_cache.k; - args.kv_cache_buffers.self_attn.v = layer_cache.v; - layer_cache.layer_index.deinit(); - }, - .linear_attention => |*layer| { - var layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache) = .{ - .conv_state = args.kv_cache_buffers.gated_delta_net.conv_state, - .recurrent_state = args.kv_cache_buffers.gated_delta_net.recurrent_state, - .layer_index = layer_index_buffer, - }; - layer.run(args.io, .{ - .inputs = .{ - .hidden = hidden_buffer, - .active_length = args.active_length_buf.*, - .cache = layer_cache, - }, - .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, - }, .{}); - args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; - args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; - layer_cache.layer_index.deinit(); - }, - } - } - - runner.sample.run(args.io, .{ - .inputs = .{ - .hidden = hidden_buffer, - .tokens = args.tokens_buf.*, - .active_length = args.active_length_buf.*, - .rng = args.rng_buffers.*, - .token_index = args.token_index_buf.*, - }, - .outputs = .{ - .tokens = args.next_token_buf, - .rng = args.rng_buffers, - .token_index = args.token_index_buf, - }, - }, .{}); -} - -fn compileKernel( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - qwen_model: model.Model, - parameters: CompilationOptions, - seqlen: usize, - phase: Phase, - progress: *std.Progress.Node, -) !KernelExe { - const full_index = findFirstLayerIndex(qwen_model.config.text_config.layer_types, .full_attention) orelse return error.MissingFullAttentionLayer; - const linear_index = findFirstLayerIndex(qwen_model.config.text_config.layer_types, .linear_attention) orelse return error.MissingLinearAttentionLayer; - - const embed = try compileEmbed(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress); - errdefer embed.deinit(); - const full_attention = try compileFullAttention(allocator, io, platform, qwen_model, parameters, seqlen, full_index, phase, progress); - errdefer full_attention.deinit(); - const linear_attention = try compileLinearAttention(allocator, io, platform, qwen_model, parameters, seqlen, linear_index, phase, progress); - errdefer linear_attention.deinit(); - const sample = try compileSample(allocator, io, platform, qwen_model, parameters, seqlen, phase, progress); - errdefer sample.deinit(); - return .{ .embed = embed, .full_attention = full_attention, .linear_attention = linear_attention, .sample = sample }; -} - -fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.EmbedTokens.forward) { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("embed tokens"), 1); - defer node.end(); - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "embed tokens", io, from); - return zml.FnExe(model.EmbedTokens.forward).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "embed_tokens") }, .{.{ - .embedding = .{ .embed_tokens = mdl.text_model.embed_tokens }, - .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), - }}); -} - -fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("full attention layer"), 1); - defer node.end(); - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "full attention layer", io, from); - return zml.FnExe(model.TransformerLayer.forwardSelfAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "full_attention_layer") }, .{.{ - .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), - .token_index = parameters.token_index, - .cache = .{ - .k = parameters.kv_cache.self_attn.k, - .v = parameters.kv_cache.self_attn.v, - .layer_index = zml.Tensor.init(.{}, .u32), - }, - }}); -} - -fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, layer_index: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("linear attention layer"), 1); - defer node.end(); - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "linear attention layer", io, from); - return zml.FnExe(model.TransformerLayer.forwardLinearAttn).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "linear_attention_layer") }, .{.{ - .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), - .active_length = zml.Tensor.init(.{}, .u32), - .cache = .{ - .conv_state = parameters.kv_cache.gated_delta_net.conv_state, - .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, - .layer_index = zml.Tensor.init(.{}, .u32), - }, - }}); -} - -fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationOptions, seqlen: usize, phase: Phase, progress: *std.Progress.Node) !zml.FnExe(model.Sampler.sampleTokens) { - progress.increaseEstimatedTotalItems(1); - var node = progress.start(phase.startMessage("sampler"), 1); - defer node.end(); - const from: std.Io.Timestamp = .now(io, .awake); - defer phase.logCompileDone(log, "sampler", io, from); - return zml.FnExe(model.Sampler.sampleTokens).compile(allocator, io, platform, .{ .meshes = ¶meters.meshes.all(), .program_name = phase.programName("qwen3_5", "sampler") }, .{.{ - .sampler = mdl.sampler(), - .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), - .tokens = .init(.{ .b = 1, .s = seqlen }, .u32), - .active_length = .init(.{}, .u32), - .rng = parameters.rng, - .token_index = parameters.token_index, - }}); -} - -fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { - return .fromShape(zml.Shape.init( - .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, - mdl.text_model.embed_tokens.weight.dtype(), - ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); -} - -fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { - for (layer_types, 0..) |layer_type, index| if (layer_type == target) return index; - return null; -} diff --git a/examples/llm/models/qwen3_5/model.zig b/examples/llm/models/qwen3_5/model.zig deleted file mode 100644 index 40174f5c..00000000 --- a/examples/llm/models/qwen3_5/model.zig +++ /dev/null @@ -1,1312 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); -const stdx = zml.stdx; - -const common = @import("../common.zig"); -const inference = @import("inference.zig"); - -const log = std.log.scoped(.qwen3_5); - -pub const Config = struct { - text_config: TextConfig, -}; - -pub const TextConfig = struct { - // General - num_hidden_layers: i64, - layer_types: []const LayerType, - hidden_size: i64, - max_position_embeddings: i64, - rms_norm_eps: f32, - // Self attention - head_dim: i64, - num_attention_heads: i64, - num_key_value_heads: i64, - rope_parameters: RopeParameters, - // Linear attention - linear_conv_kernel_dim: i64, - linear_key_head_dim: i64, - linear_num_key_heads: i64, - linear_num_value_heads: i64, - linear_value_head_dim: i64, -}; - -// Each layer uses either: full attention (SelfAttn) or linear attention (GatedDeltaNet). -pub const LayerType = enum { - linear_attention, - full_attention, -}; - -pub const RopeParameters = struct { - mrope_section: [3]i64, - partial_rotary_factor: f32, - rope_theta: f32, -}; - -pub const LoadedModel = struct { - inner: Model, - parsed_config: std.json.Parsed(Config), - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - store: zml.io.TensorStore.View, - generation: common.GenerationOptions, - ) !LoadedModel { - const parsed_config = try common.parseConfig(Config, allocator, io, repo); - errdefer parsed_config.deinit(); - - const options: Model.GenOptions = .{ - .sampling_strategy = generation.sampling_strategy, - .max_seq_len = parsed_config.value.text_config.max_position_embeddings, - }; - - return .{ - .inner = try .init(allocator, store, parsed_config.value, options), - .parsed_config = parsed_config, - }; - } - - pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { - self.inner.deinit(allocator); - self.parsed_config.deinit(); - } - - pub fn loadBuffers( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - TextModel.unloadBuffers(&buffers.text_model, allocator); - zml.nn.Linear.unloadBuffers(&buffers.lm_head); - } - - pub fn compile( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - backend: zml.attention.Backend, - meshes: common.Meshes, - seqlen: usize, - progress: *std.Progress.Node, - ) !inference.CompiledModel { - _ = backend; - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), meshes); - return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); - } -}; - -pub const Buffers = zml.Bufferized(Model); - -pub const Model = struct { - pub const GenOptions = struct { sampling_strategy: zml.nn.SamplingStrategy = .{}, max_seq_len: i64 }; - - pub const SpecialTokens = struct { - im_start_token_id: u32, - im_end_token_id: u32, - end_of_text_token_id: u32, - }; - - text_model: TextModel, - lm_head: zml.nn.Linear, - - config: Config, - gen_options: GenOptions, - special_tokens: SpecialTokens = .{ - .im_start_token_id = 248045, - .im_end_token_id = 248046, - .end_of_text_token_id = 248044, - }, - - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { - // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer - const lm_head_prefix = b: { - if (store.hasKey("lm_head.weight")) break :b "lm_head"; - break :b "model.language_model.embed_tokens"; - }; - return .{ - .text_model = try .init(allocator, store.withPrefix("model.language_model"), config), - .lm_head = .init( - store.withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), - null, - .d, - ), - .config = config, - .gen_options = gen_options, - }; - } - - pub fn deinit(self: Model, allocator: std.mem.Allocator) void { - self.text_model.deinit(allocator); - } - - pub fn load( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - meshes: []const *const zml.Mesh, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store, meshes); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - TextModel.unloadBuffers(&self.text_model, allocator); - zml.nn.Linear.unloadBuffers(&self.lm_head); - } - pub fn forward( - self: Model, - tokens_: zml.Tensor, - token_index: zml.Tensor, - active_length: zml.Tensor, - kv_cache: KvCache, - rng: zml.Tensor.Rng, - ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { - const tokens = tokens_.withPartialTags(.{.s}); - const text_model_output, const updated_kv_cache = self.text_model.forward(tokens, token_index, active_length, kv_cache); - const result = Sampler.sampleTokens(.{ - .sampler = self.sampler(), - .hidden = text_model_output, - .tokens = tokens, - .active_length = active_length, - .rng = rng, - .token_index = token_index, - }); - return .{ result.tokens, updated_kv_cache, result.rng }; - } - - pub fn sampler(self: Model) Sampler { - return .{ - .norm = self.text_model.norm, - .lm_head = self.lm_head, - .gen_options = self.gen_options, - }; - } -}; - -pub const Sampler = struct { - norm: RmsNorm, - lm_head: zml.nn.Linear, - gen_options: Model.GenOptions, - - pub const Input = struct { - sampler: Sampler, - hidden: zml.Tensor, - tokens: zml.Tensor, - active_length: zml.Tensor, - rng: zml.Tensor.Rng, - token_index: zml.Tensor, - }; - - pub const Output = struct { - tokens: zml.Tensor, - rng: zml.Tensor.Rng, - token_index: zml.Tensor, - }; - - pub fn sampleTokens(input: Input) Output { - const self = input.sampler; - const last_index = input.active_length.convert(.i32).addConstant(-1); - const x = self.norm.forward(input.hidden.slice(.s, .dyn(last_index, 1))); - const logits = self.lm_head.forward(x.withPartialTags(.{.d}), x.dtype()).rename(.{ .dout = .voc }); - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); - const tokens = next_tokens.convert(.u32); - return .{ - .tokens = if (input.tokens.dim(.s) == 1) tokens.reuseBuffer(input.tokens) else tokens, - .rng = new_rng, - .token_index = input.token_index.addConstant(1).reuseBuffer(input.token_index), - }; - } -}; - -pub const EmbedTokens = struct { - embed_tokens: zml.nn.TokenEmbedding, - - pub const Input = struct { - embedding: EmbedTokens, - tokens: zml.Tensor, - }; - - pub const Output = struct { - hidden: zml.Tensor, - }; - - pub fn forward(input: Input) Output { - const tokens = input.tokens.withPartialTags(.{.s}); - return .{ .hidden = input.embedding.embed_tokens.forward(tokens) - .withPartialTags(.{.d}) - .replicate() }; - } -}; - -pub const TextModel = struct { - embed_tokens: zml.nn.TokenEmbedding, - layers: []TransformerLayer, - norm: RmsNorm, - - pub fn init( - allocator: std.mem.Allocator, - store: zml.io.TensorStore.View, - config: Config, - ) !TextModel { - const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); - errdefer allocator.free(layers); - - for (layers, 0..) |*layer, i| { - layer.* = try .init(store.withPrefix("layers").withLayer(i), config, i); - } - - return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, - .layers = layers, - .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), - }; - } - - pub fn deinit(self: TextModel, allocator: std.mem.Allocator) void { - allocator.free(self.layers); - } - - pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer); - } - allocator.free(self.layers); - RmsNorm.unloadBuffers(&self.norm); - } - - pub fn forward( - self: TextModel, - tokens: zml.Tensor, - token_index: zml.Tensor, - active_length: zml.Tensor, - kv_cache: KvCache, - ) struct { zml.Tensor, KvCache } { - var hidden_states = EmbedTokens.forward(.{ - .embedding = .{ .embed_tokens = self.embed_tokens }, - .tokens = tokens, - }).hidden; - - var updated_kv_cache = kv_cache; - for (self.layers, 0..) |layer, i| { - hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i)); - } - - return .{ hidden_states, updated_kv_cache.reuseBuffer(kv_cache) }; - } -}; - -pub const TransformerLayer = struct { - const Attn = union(LayerType) { - linear_attention: GatedDeltaNet, - full_attention: SelfAttn, - }; - - input_layernorm: RmsNorm, - attn: Attn, - mlp: Mlp, - post_attention_layernorm: RmsNorm, - - pub const SelfAttnInput = struct { - layer: TransformerLayer, - hidden: zml.Tensor, - token_index: zml.Tensor, - cache: KvCache.SelfAttnCache, - }; - - pub const SelfAttnOutput = struct { - hidden: zml.Tensor, - cache: KvCache.SelfAttnCache, - }; - - pub const LinearAttnInput = struct { - layer: TransformerLayer, - hidden: zml.Tensor, - active_length: zml.Tensor, - cache: KvCache.GatedDeltaNetCache, - }; - - pub const LinearAttnOutput = struct { - hidden: zml.Tensor, - cache: KvCache.GatedDeltaNetCache, - }; - - pub fn init(store: zml.io.TensorStore.View, config: Config, layer_index: usize) !TransformerLayer { - const is_full_attention = config.text_config.layer_types[layer_index] == .full_attention; - const attn: Attn = switch (is_full_attention) { - true => .{ .full_attention = try .init(store.withPrefix("self_attn"), config) }, - false => .{ .linear_attention = .init(store.withPrefix("linear_attn"), config) }, - }; - return .{ - .input_layernorm = RmsNorm.init(store.withPrefix("input_layernorm"), config.text_config.rms_norm_eps), - .attn = attn, - .mlp = .init(store.withPrefix("mlp")), - .post_attention_layernorm = RmsNorm.init(store.withPrefix("post_attention_layernorm"), config.text_config.rms_norm_eps), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer)) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - switch (self.attn) { - .full_attention => |*self_attn| SelfAttn.unloadBuffers(self_attn), - .linear_attention => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), - } - Mlp.unloadBuffers(&self.mlp); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - } - - pub fn forward( - self: TransformerLayer, - x0: zml.Tensor, - token_index: zml.Tensor, - active_length: zml.Tensor, - kv_cache: KvCache.LayerView, - ) struct { zml.Tensor, KvCache } { - const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); - const normalized_x0 = self.input_layernorm.forward(x0_replicated); - - var attention_output: zml.Tensor = undefined; - var updated_kv_cache: KvCache = kv_cache.parent; - switch (self.attn) { - .full_attention => |*self_attn| { - const cache = switch (kv_cache.cache) { - .self_attn => |cache| cache, - .linear_attn => unreachable, - }; - const result = self_attn.forward(normalized_x0, token_index, cache); - attention_output = result[0]; - updated_kv_cache.self_attn = result[1].reuseBuffer(updated_kv_cache.self_attn); - }, - .linear_attention => |*linear_attn| { - const cache = switch (kv_cache.cache) { - .linear_attn => |cache| cache, - .self_attn => unreachable, - }; - const result = linear_attn.forward(normalized_x0, cache, active_length); - attention_output = result[0]; - updated_kv_cache.gated_delta_net = result[1].reuseBuffer(updated_kv_cache.gated_delta_net); - }, - } - - const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); - const normalized_hidden = self.post_attention_layernorm.forward(x1); - - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); - - return .{ mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), updated_kv_cache }; - } - - pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { - const self = input.layer; - const x0 = input.hidden; - const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); - const normalized_x0 = self.input_layernorm.forward(x0_replicated); - - const self_attn = switch (self.attn) { - .full_attention => |self_attn| self_attn, - .linear_attention => unreachable, - }; - const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); - - const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); - const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); - - return .{ - .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), - .cache = updated_kv_cache, - }; - } - - pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { - const self = input.layer; - const x0 = input.hidden; - const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); - const normalized_x0 = self.input_layernorm.forward(x0_replicated); - - const linear_attn = switch (self.attn) { - .linear_attention => |linear_attn| linear_attn, - .full_attention => unreachable, - }; - const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); - - const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); - const normalized_hidden = self.post_attention_layernorm.forward(x1); - const mlp_output = self.mlp.forward(normalized_hidden).withPartitioning(.model, .{ .d = .replicated }); - - return .{ - .hidden = mlp_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), - .cache = updated_kv_cache, - }; - } -}; - -pub const Mlp = struct { - up_proj: zml.nn.Linear, - gate_proj: zml.nn.Linear, - down_proj: zml.nn.Linear, - - pub fn init(store: zml.io.TensorStore.View) Mlp { - return .{ - .up_proj = .init( - store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), - store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), - .d, - ), - .gate_proj = .init( - store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), - store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), - .d, - ), - .down_proj = .init( - store.withPrefix("down_proj").createTensor("weight", .{ .d, .dout }, .model, .{ .d = .replicated, .dout = .model }), - store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), - .dout, - ), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const up_projed = self.up_proj.forward(x, x.dtype()); - const gate = self.gate_proj.forward(x, x.dtype()); - const hidden = gate.silu().mul(up_projed); - - const output = self.down_proj.forward(hidden, hidden.dtype()); - return output; - } -}; - -pub const SelfAttn = struct { - q_proj: zml.nn.Linear, - k_proj: zml.nn.Linear, - v_proj: zml.nn.Linear, - - q_norm: RmsNorm, - k_norm: RmsNorm, - - num_heads: i64, - num_kv_heads: i64, - head_dim: i64, - rotary_dim: i64, - rotary_embed: TextRotaryEmbedding, - o_proj: zml.nn.Linear, - - fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { - return .init( - store.createTensor("weight", .{ .dout, .d }, .model, partitions), - store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), - .d, - ); - } - - pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { - const rotary_dim: i64 = @intFromFloat(@as(f32, @floatFromInt(config.text_config.head_dim)) * - config.text_config.rope_parameters.partial_rotary_factor); - return .{ - .q_proj = initProj(store.withPrefix("q_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), - .k_proj = initProj(store.withPrefix("k_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), - .v_proj = initProj(store.withPrefix("v_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), - .o_proj = initProj(store.withPrefix("o_proj"), .{ .dout = .replicated, .d = .model }, .{ .dout = .replicated }), - .q_norm = RmsNorm.init(store.withPrefix("q_norm"), config.text_config.rms_norm_eps), - .k_norm = RmsNorm.init(store.withPrefix("k_norm"), config.text_config.rms_norm_eps), - .num_heads = config.text_config.num_attention_heads, - .num_kv_heads = config.text_config.num_key_value_heads, - .head_dim = config.text_config.head_dim, - .rotary_dim = rotary_dim, - .rotary_embed = .init(rotary_dim, config.text_config.rope_parameters.rope_theta, config.text_config.rope_parameters.mrope_section), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - - fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { - const q_proj = self.q_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); - const q, var gate = q_proj.chunkExact(.hd, 2); - gate = gate.merge(.{ .d_out_proj = .{ .h, .hd } }); - return .{ q, gate }; - } - - fn projectKV(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { - const num_kv_heads = b: { - if (self.num_kv_heads > 0) break :b self.num_kv_heads; - break :b self.num_heads; - }; - const k = self.k_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); - const v = self.v_proj.forward(x, x.dtype()).splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); - return .{ k, v }; - } - - fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { - return switch (kv_head_sharding) { - .sharded => |heads| blk: { - const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; - break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); - }, - .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), - }; - } - - fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { - var cache_tensor = tensor.rename(.{ .s = .k }); - return switch (kv_head_sharding) { - .sharded => |heads| blk: { - if (heads.factor != 1) { - cache_tensor = cache_tensor.stutter1d(cache_tensor.axis(.h), heads.factor); - } - break :blk cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); - }, - .replicated => cache_tensor.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), - }; - } - - pub fn forward( - self: SelfAttn, - x: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache.SelfAttnCache, - ) struct { zml.Tensor, KvCache.SelfAttnCache } { - const tp = zml.Compiler.current().mesh(.model); - const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); - - var q, var gate = self.projectQAndGate(x_qkv); - var k, var v = self.projectKV(x_qkv); - const kv_head_sharding = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); - - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); - q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); - k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); - - const dtype = q.dtype(); - const position_ids = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) - .withTags(.{.s}).insertAxes(.s, .{.b}).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64)) - .add(token_index.convert(.i64).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64))); - - const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); - q = self.rotary_embed.applyRope(q, cos, sin); - k = self.rotary_embed.applyRope(k, cos, sin); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); - - const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); - k = new_kv_cache.keys().convert(dtype); - v = new_kv_cache.values().convert(dtype); - q = q.rename(.{ .s = .q }); - k = partitionCachedKv(k, kv_head_sharding); - v = partitionCachedKv(v, kv_head_sharding); - - const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { - .sharded => .head(.model), - .replicated => .head(.replicated), - }; - const attn_output = zml.attention.attention( - q, - k, - v, - token_index, - zml.attention.Metadata.init(.fromBackend(.stablehlo)), - zml.attention.Parameters.init(.fromBackend(.stablehlo)), - zml.Compiler.current().mesh(.model), - head_partitioning, - ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); - - const gated_output = attn_output.mul(gate.sigmoid()); - const projected_output = self.o_proj - .forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()) - .rename(.{ .dout = .d }) - .withPartitioning(tp, .{ .d = .replicated }); - - return .{ projected_output, new_kv_cache }; - } -}; - -pub const TextRotaryEmbedding = struct { - rope_opts: zml.nn.RopeOpts, - rotary_dim: i64, - mrope_section: [3]i64, - - pub fn init(rotary_dim: i64, theta: f32, mrope_section: [3]i64) TextRotaryEmbedding { - return .{ - .rope_opts = .{ - .layout = .real_im_pass, - .scaling = .{ .default = .{ .rope_theta = theta } }, - }, - .rotary_dim = rotary_dim, - .mrope_section = mrope_section, - }; - } - - pub fn getCosAndSin(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { - const inv_freq = zml.nn.invFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); - - const freqs_t = position_ids.convert(.f32).outer(inv_freq); - - const emb = zml.Tensor.concatenate(&.{ freqs_t, freqs_t }, -1); - const cos = emb.cos().convert(dtype); - const sin = emb.sin().convert(dtype); - - return .{ cos, sin }; - } - - pub fn getCosAndSinInterleaved( - self: TextRotaryEmbedding, - position_ids: zml.Tensor, - dtype: zml.DataType, - ) struct { zml.Tensor, zml.Tensor } { - // To be used later in image extension. - const stacked_position_ids = zml.Tensor.stack(&.{ position_ids, position_ids, position_ids }, 0, .g).convert(.f32); - const inv_freq = zml.nn.InvFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); - - var freqs = stacked_position_ids.outer(inv_freq); - var freqs_t, var freqs_h, var freqs_w = freqs.chunkExact(.g, 3); - freqs_t = freqs_t.squeeze(.g); - freqs_h = freqs_h.squeeze(.g); - freqs_w = freqs_w.squeeze(.g); - - const h_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[1] }, .i32), .h).scale(3).addConstant(1); - const w_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[2] }, .i32), .h).scale(3).addConstant(2); - - const h_input = freqs_h.gather(.{ .dh = h_indices }, .{ .indices_are_sorted = true }); - const w_input = freqs_w.gather(.{ .dh = w_indices }, .{ .indices_are_sorted = true }); - freqs_t = freqs_t.scatterSlices(.{ .dh = h_indices }, h_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); - freqs = freqs_t.scatterSlices(.{ .dh = w_indices }, w_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); - - const emb = zml.Tensor.concatenate(&.{ freqs, freqs }, -1); - const cos = emb.cos().convert(dtype); - const sin = emb.sin().convert(dtype); - - return .{ cos, sin }; - } - - fn rotateHalf(x: zml.Tensor) zml.Tensor { - const half_dim = @divExact(x.dim(-1), 2); - const x1 = x.slice(-1, .{ .start = 0, .end = half_dim }); - const x2 = x.slice(-1, .{ .start = half_dim, .end = x.dim(-1) }); - return zml.Tensor.concatenate(&.{ x2.negate(), x1 }, -1); - } - - pub fn applyRope(self: TextRotaryEmbedding, x: zml.Tensor, cos: zml.Tensor, sin: zml.Tensor) zml.Tensor { - const x_rot = x.slice(-1, .{ .start = 0, .end = self.rotary_dim }); - const x_pass = x.slice(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); - - const cos_x = cos.insertAxes(.hd, .{.h}).broad(x_rot.shape()); - const sin_x = sin.insertAxes(.hd, .{.h}).broad(x_rot.shape()); - - const rotated = x_rot.mul(cos_x).add(rotateHalf(x_rot).mul(sin_x)); - - return zml.Tensor.concatenate(&.{ rotated, x_pass }, -1); - } -}; - -pub const GatedDeltaNet = struct { - in_proj_qkv: zml.nn.Linear, - in_proj_z: zml.nn.Linear, - in_proj_b: zml.nn.Linear, - in_proj_a: zml.nn.Linear, - out_proj: zml.nn.Linear, - conv1d_weight: zml.Tensor, - dt_bias: zml.Tensor, - aLog: zml.Tensor, - norm: RmsNormGated, - - num_k_heads: i64, - num_v_heads: i64, - qk_head_repetition: u32, - head_k_dim: i64, - head_v_dim: i64, - conv_kernel_size: i64, - - fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { - return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); - } - - pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { - const qk_head_repetition: u32 = - @intCast(@divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads)); - return .{ - .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv"), .{ .dout = .model, .d = .replicated }), - .in_proj_z = initProj(store.withPrefix("in_proj_z"), .{ .dout = .model, .d = .replicated }), - .in_proj_b = initProj(store.withPrefix("in_proj_b"), .{ .dout = .model, .d = .replicated }), - .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), - .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), - .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), - .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), - .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), - .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), - .num_k_heads = config.text_config.linear_num_key_heads, - .num_v_heads = config.text_config.linear_num_value_heads, - .qk_head_repetition = qk_head_repetition, - .head_k_dim = config.text_config.linear_key_head_dim, - .head_v_dim = config.text_config.linear_value_head_dim, - .conv_kernel_size = config.text_config.linear_conv_kernel_dim, - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { - zml.Buffer.deinitAll(GatedDeltaNet, self); - } - - fn recurrentGatedDeltaRule( - query: zml.Tensor, - key: zml.Tensor, - value: zml.Tensor, - g: zml.Tensor, - beta: zml.Tensor, - initial_state: ?zml.Tensor, - active_length: zml.Tensor, - ) struct { zml.Tensor, zml.Tensor } { - const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); - const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); - const key_norm = zml.nn.normalizeL2(key.rename(.{ .kh = .vh }), 1e-6); - - const query_f32, const key_f32, const value_f32, const alpha_f32, const beta_f32 = .{ - query_norm.convert(.f32).scale(scale).rename(.{ .vh = .h, .khd = .k }), - key_norm.convert(.f32).rename(.{ .vh = .h, .khd = .k }), - value.convert(.f32).rename(.{ .vh = .h, .vhd = .v }), - g.convert(.f32).exp().rename(.{ .vh = .h }), - beta.convert(.f32).rename(.{ .vh = .h }), - }; - - const initial_recurrent_state = b: { - if (initial_state) |state| { - break :b state.convert(.f32).transpose(.{ .b, .vh, .vhd, .khd }).rename(.{ .vh = .h, .vhd = .v, .khd = .k }); - } - - break :b zml.Tensor.constant(zml.DataType.zero(.f32)).broad(zml.Shape.init(.{ - .b = value.dim(.b), - .h = value.dim(.vh), - .v = value.dim(.vhd), - .k = query.dim(.khd), - }, .f32)); - }; - - const result = zml.nn.GatedDeltaNet.forward( - query_f32, - key_f32, - value_f32, - alpha_f32, - beta_f32, - .{ .s = initial_recurrent_state }, - active_length, - ); - - return .{ - result.outputs.rename(.{ .h = .vh, .v = .vhd }).convert(query.dtype()), - result.state.s.transpose(.{ .b, .h, .k, .v }).rename(.{ .h = .vh, .k = .khd, .v = .vhd }), - }; - } - - fn buildUpdatedConvState(input: zml.Tensor, left_pad: i64) zml.Tensor { - const copy_len = @min(input.dim(.s), left_pad); - const tail = input.slice(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); - if (copy_len == left_pad) return tail; - - const padding_shape = zml.Shape.init(.{ .b = input.dim(.b), .s = left_pad - copy_len, .mix = input.dim(.mix) }, input.dtype()); - const padding = zml.Tensor.constant(input.dtype().zero()).broad(padding_shape); - return zml.Tensor.concatenate(&.{ padding, tail }, .s); - } - - fn buildUpdatedConvStateFromPrefix(input: zml.Tensor, left_pad: i64, active_length: zml.Tensor) zml.Tensor { - const padding = zml.Tensor.zeroes(input.shape().setDim(.s, left_pad)); - const padded = zml.Tensor.concatenate(&.{ padding, input }, .s); - return padded.slice(.s, .dyn(active_length.convert(.i64), left_pad)); - } - - pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache, active_length: zml.Tensor) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { - const key_dim = self.num_k_heads * self.head_k_dim; - const value_dim = self.num_v_heads * self.head_v_dim; - const conv_dim = 2 * key_dim + value_dim; - const left_pad = self.conv_kernel_size - 1; - - const x_in = x.withPartitioning(.model, .{ .d = .replicated }); - const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) - .rename(.{ .dout = .mix }) - .withPartitioning(.model, .{ .s = .replicated, .mix = .model }); - const use_cached_state = x.dim(.s) == 1 and left_pad > 0; - const conv_input = b: { - if (use_cached_state) break :b zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s); - break :b projected_qkv; - }; - - const kernel = self.conv1d_weight; - var mixed_qkv = zml.Tensor.conv1d( - conv_input, - kernel, - .{ - .padding = &.{ left_pad, 0 }, - .input_batch_dimension = 0, - .input_feature_dimension = 2, - .input_spatial_dimensions = 1, - .kernel_output_feature_dimension = 0, - .kernel_input_feature_dimension = 1, - .kernel_spatial_dimensions = 2, - .output_batch_dimension = 0, - .output_feature_dimension = 2, - .output_spatial_dimensions = 1, - .feature_group_count = conv_dim, - }, - ) - .silu(); - - if (use_cached_state) { - mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); - } - mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); - - const z = self.in_proj_z.forward(x_in, x_in.dtype()) - .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); - const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }); - const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }); - - const query = mixed_qkv - .slice(.mix, .{ .start = 0, .end = key_dim }) - .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); - const key = mixed_qkv - .slice(.mix, .{ .start = key_dim, .end = 2 * key_dim }) - .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }); - const value = mixed_qkv - .slice(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) - .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }); - - const beta = b.sigmoid(); - const aLog_type = self.aLog.dtype(); - const g = self.aLog.broad(a.shape()).exp().mul(zml.nn.softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())), .{})).negate(); - - const query_for_rule = b: { - if (self.qk_head_repetition == 1) break :b query; - break :b query.stutter1d(query.axis(.kh), self.qk_head_repetition); - }; - const key_for_rule = b: { - if (self.qk_head_repetition == 1) break :b key; - break :b key.stutter1d(key.axis(.kh), self.qk_head_repetition); - }; - - const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( - query_for_rule, - key_for_rule, - value, - g, - beta, - b: { - if (use_cached_state) break :b cache.recurrentState(); - break :b null; - }, - active_length, - ); - - const core_attn_out_normed = self.norm - .forward( - core_attn_out.rename(.{ .vhd = .d }), - z.rename(.{ .vhd = .d }), - ) - .rename(.{ .d = .vhd }); - - const output = self.out_proj - .forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) - .rename(.{ .dout = .d }) - .withPartitioning(.model, .{ .d = .replicated }); - const updated_cache = cache.update( - if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), - last_recurrent_state, - ); - return .{ output, updated_cache }; - } -}; - -pub const RmsNorm = struct { - weight: zml.Tensor, - eps: f32 = 1e-6, - - pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - - pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { - const x_f32 = x.convert(.f32); - const weight_f32 = self.weight.convert(.f32); - - const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); - return normalized.mul(weight_f32.broad(x.shape())).add(normalized).convert(x.dtype()); - } -}; - -pub const RmsNormGated = struct { - weight: zml.Tensor, - eps: f32 = 1e-6, - - pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { - return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { - self.weight.deinit(); - } - - pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { - const x_f32 = x.convert(.f32); - const gate_f32 = gate.convert(.f32); - const weight_f32 = self.weight.convert(.f32); - - const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); - const output = normalized.mul(weight_f32.broad(x.shape())); - - const gated_output = output.mul(gate_f32.silu()); - return gated_output.convert(x.dtype()); - } -}; - -pub const KvCache = struct { - layer_types: []const LayerType, - self_attn: SelfAttnCache, - gated_delta_net: GatedDeltaNetCache, - - pub const SelfAttnCache = struct { - k: zml.Tensor, - v: zml.Tensor, - layer_index: zml.Tensor, - - pub const Buffers = zml.Bufferized(SelfAttnCache); - - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, tp: *const zml.Mesh) SelfAttnCache { - const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); - const kv_shape = zml.Shape.init(.{ - .b = batch_dim, - .layer = num_self_attn_layers, - .s = max_seq_len, - .h = config.text_config.num_key_value_heads, - .hd = config.text_config.head_dim, - }, dtype); - const kv_head_sharding = tp.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); - const sharded_kv_shape = switch (kv_head_sharding) { - .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(tp, .{ .h = .model }), - .replicated => kv_shape.withPartitioning(tp, .{ .h = .replicated }), - }; - return .{ - .k = .fromShape(sharded_kv_shape), - .v = .fromShape(sharded_kv_shape), - .layer_index = .init(.{}, .u32), - }; - } - - pub fn initBuffer(kv: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !SelfAttnCache.Buffers { - return .{ - .k = try zml.Buffer.uninitialized(io, platform, kv.k.shape(), .{}), - .v = try zml.Buffer.uninitialized(io, platform, kv.v.shape(), .{}), - .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), - }; - } - - pub fn deinitBuffer(kv: *SelfAttnCache.Buffers) void { - kv.k.deinit(); - kv.v.deinit(); - kv.layer_index.deinit(); - } - - pub fn keys(kv: SelfAttnCache) zml.Tensor { - return kv.k.slice(.layer, .dynSingle(kv.layer_index)); - } - - pub fn values(kv: SelfAttnCache) zml.Tensor { - return kv.v.slice(.layer, .dynSingle(kv.layer_index)); - } - - pub fn update(kv: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { - const k_shape = kv.k.shape().drop(.layer); - var layer = kv.layer_index; - if (token_index) |idx| { - layer = layer.broad(idx.shape()); - return .{ - .k = kv.k.scatterSlices( - .{ .layer = layer, .s = idx }, - new_k.convert(kv.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(kv.k), - .v = kv.v.scatterSlices( - .{ .layer = layer, .s = idx }, - new_v.convert(kv.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(kv.v), - .layer_index = kv.layer_index, - }; - } - - return .{ - .k = kv.k.scatterSlices( - .{ .layer = layer }, - new_k.convert(kv.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(kv.k), - .v = kv.v.scatterSlices( - .{ .layer = layer }, - new_v.convert(kv.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(kv.v), - .layer_index = kv.layer_index, - }; - } - pub fn atLayer(kv: SelfAttnCache, layer_index: usize) SelfAttnCache { - return .{ - .k = kv.k, - .v = kv.v, - .layer_index = zml.Tensor.scalar(layer_index, .u32), - }; - } - - pub fn reuseBuffer(kv: SelfAttnCache, other: SelfAttnCache) SelfAttnCache { - return .{ - .k = kv.k.reuseBuffer(other.k), - .v = kv.v.reuseBuffer(other.v), - .layer_index = kv.layer_index.reuseBuffer(other.layer_index), - }; - } - }; - - pub const GatedDeltaNetCache = struct { - conv_state: zml.Tensor, - recurrent_state: zml.Tensor, - layer_index: zml.Tensor, - - pub const Buffers = zml.Bufferized(GatedDeltaNetCache); - - pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, tp: *const zml.Mesh) GatedDeltaNetCache { - const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); - const conv_dim = - 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + - config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; - const conv_state_shape = zml.Shape.init(.{ - .b = batch_dim, - .layer = num_linear_attn_layers, - .s = config.text_config.linear_conv_kernel_dim - 1, - .mix = conv_dim, - }, conv_dtype); - const recurrent_state_shape = zml.Shape.init(.{ - .b = batch_dim, - .layer = num_linear_attn_layers, - .vh = config.text_config.linear_num_value_heads, - .khd = config.text_config.linear_key_head_dim, - .vhd = config.text_config.linear_value_head_dim, - }, recurrent_dtype); - const sharded_conv_state_shape = conv_state_shape.withPartitioning(tp, .{ .mix = .model }); - const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(tp, .{ .vh = .model }); - return .{ - .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), - .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), - .layer_index = .init(.{}, .u32), - }; - } - - pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !GatedDeltaNetCache.Buffers { - return .{ - .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), - .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), - .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), - }; - } - - pub fn deinitBuffer(self: *GatedDeltaNetCache.Buffers) void { - self.conv_state.deinit(); - self.recurrent_state.deinit(); - self.layer_index.deinit(); - } - - pub fn convState(self: GatedDeltaNetCache) zml.Tensor { - return self.conv_state.slice(.layer, .dynSingle(self.layer_index)); - } - - pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { - return self.recurrent_state.slice(.layer, .dynSingle(self.layer_index)); - } - - pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { - const conv_state = b: { - if (new_conv_state) |state| { - break :b self.conv_state.scatterSlices( - .{ .layer = self.layer_index }, - state.convert(self.conv_state.dtype()).transpose(self.conv_state.shape().drop(.layer)), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.conv_state); - } - - break :b self.conv_state; - }; - - const recurrent_state = b: { - if (new_recurrent_state) |state| { - break :b self.recurrent_state.scatterSlices( - .{ .layer = self.layer_index }, - state.convert(self.recurrent_state.dtype()).transpose(self.recurrent_state.shape().drop(.layer)), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.recurrent_state); - } - - break :b self.recurrent_state; - }; - - return .{ - .conv_state = conv_state, - .recurrent_state = recurrent_state, - .layer_index = self.layer_index, - }; - } - - pub fn atLayer(self: GatedDeltaNetCache, layer_index: usize) GatedDeltaNetCache { - return .{ - .conv_state = self.conv_state, - .recurrent_state = self.recurrent_state, - .layer_index = zml.Tensor.scalar(layer_index, .u32), - }; - } - - pub fn reuseBuffer(self: GatedDeltaNetCache, other: GatedDeltaNetCache) GatedDeltaNetCache { - return .{ - .conv_state = self.conv_state.reuseBuffer(other.conv_state), - .recurrent_state = self.recurrent_state.reuseBuffer(other.recurrent_state), - .layer_index = self.layer_index.reuseBuffer(other.layer_index), - }; - } - }; - - pub fn init( - config: Config, - batch_dim: i64, - max_seq_len: i64, - cache_dtype: zml.DataType, - recurrent_dtype: zml.DataType, - tp: *const zml.Mesh, - ) KvCache { - return .{ - .layer_types = config.text_config.layer_types, - .self_attn = .init(config, batch_dim, max_seq_len, cache_dtype, tp), - .gated_delta_net = .init(config, batch_dim, cache_dtype, recurrent_dtype, tp), - }; - } - - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { - return .{ - .self_attn = try self.self_attn.initBuffer(io, platform), - .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { - SelfAttnCache.deinitBuffer(&self.self_attn); - GatedDeltaNetCache.deinitBuffer(&self.gated_delta_net); - } - - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { - return .{ - .layer_types = self.layer_types, - .self_attn = self.self_attn.reuseBuffer(other.self_attn), - .gated_delta_net = self.gated_delta_net.reuseBuffer(other.gated_delta_net), - }; - } - - pub const LayerView = struct { - parent: KvCache, - cache: union(enum) { - self_attn: SelfAttnCache, - linear_attn: GatedDeltaNetCache, - }, - }; - - pub fn atLayer(self: KvCache, layer_index: usize) LayerView { - return switch (denseIndex(self.layer_types, layer_index)) { - .full_attention => |dense_index| .{ - .parent = self, - .cache = .{ .self_attn = self.self_attn.atLayer(dense_index.layer_dense_index) }, - }, - .linear_attention => |dense_index| .{ - .parent = self, - .cache = .{ .linear_attn = self.gated_delta_net.atLayer(dense_index.layer_dense_index) }, - }, - }; - } - - fn countLayers(layer_types: []const LayerType, layer_type: LayerType) i64 { - return @intCast(std.mem.countScalar(LayerType, layer_types, layer_type)); - } - - fn denseIndex(layer_types: []const LayerType, layer_index: usize) union(enum) { - full_attention: struct { layer_dense_index: usize }, - linear_attention: struct { layer_dense_index: usize }, - } { - var self_attn_layer_index: usize = 0; - var linear_attn_layer_index: usize = 0; - for (layer_types[0..layer_index]) |layer_type| { - switch (layer_type) { - .full_attention => self_attn_layer_index += 1, - .linear_attention => linear_attn_layer_index += 1, - } - } - return switch (layer_types[layer_index]) { - .full_attention => .{ .full_attention = .{ .layer_dense_index = self_attn_layer_index } }, - .linear_attention => .{ .linear_attention = .{ .layer_dense_index = linear_attn_layer_index } }, - }; - } -}; diff --git a/examples/llm/models/qwen3_5/session.zig b/examples/llm/models/qwen3_5/session.zig deleted file mode 100644 index 5121fd30..00000000 --- a/examples/llm/models/qwen3_5/session.zig +++ /dev/null @@ -1,219 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const inference = @import("inference.zig"); -const model = @import("model.zig"); - -pub const Session = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - compiled_model: *inference.CompiledModel, - prefill: inference.KernelRunner, - decode: inference.KernelRunner, - layer_index_buffers: []zml.Buffer, - kv_cache_buffers: zml.Bufferized(model.KvCache), - rng_buffers: zml.Bufferized(zml.Tensor.Rng), - tokenizer: zml.tokenizer.Tokenizer, - generated_token_slice: zml.Slice, - seqlen: u32, - eos_token_id: u32, - special_tokens: model.Model.SpecialTokens, - think_start: ?u32, - think_end: ?u32, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - tokenizer: zml.tokenizer.Tokenizer, - compiled_model: *inference.CompiledModel, - model_buffers: *model.Buffers, - ) !Session { - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); - errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); - - const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); - errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); - - const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; - const layer_index_buffers = try allocator.alloc(zml.Buffer, layer_types.len); - errdefer allocator.free(layer_index_buffers); - var initialized_layer_index_buffers: usize = 0; - errdefer for (layer_index_buffers[0..initialized_layer_index_buffers]) |*buffer| buffer.deinit(); - var full_attention_index: u32 = 0; - var linear_attention_index: u32 = 0; - for (layer_index_buffers, layer_types) |*buffer, layer_type| { - buffer.* = try switch (layer_type) { - .full_attention => b: { - defer full_attention_index += 1; - break :b zml.Buffer.scalar(io, platform, full_attention_index, .u32); - }, - .linear_attention => b: { - defer linear_attention_index += 1; - break :b zml.Buffer.scalar(io, platform, linear_attention_index, .u32); - }, - }; - initialized_layer_index_buffers += 1; - } - - const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)); - errdefer generated_token_slice.free(allocator); - - var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); - errdefer prefill.deinit(allocator); - const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); - - return .{ - .allocator = allocator, - .io = io, - .platform = platform, - .compiled_model = compiled_model, - .prefill = prefill, - .decode = decode, - .layer_index_buffers = layer_index_buffers, - .kv_cache_buffers = kv_cache_buffers, - .rng_buffers = rng_buffers, - .tokenizer = tokenizer, - .generated_token_slice = generated_token_slice, - .seqlen = compiled_model.params.seqlen, - .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, - .special_tokens = compiled_model.loaded_model.inner.special_tokens, - .think_start = tokenizer.tokenId(""), - .think_end = tokenizer.tokenId(""), - }; - } - - pub fn deinit(self: *Session) void { - self.prefill.deinit(self.allocator); - self.decode.deinit(self.allocator); - for (self.layer_index_buffers) |*buffer| buffer.deinit(); - self.allocator.free(self.layer_index_buffers); - model.KvCache.deinitBuffer(&self.kv_cache_buffers); - zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); - self.generated_token_slice.free(self.allocator); - } - - pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { - return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, true); - } - - pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { - return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, false); - } - - pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { - const prefill_tokens_shape = zml.Shape.init(.{ .b = 1, .s = self.seqlen }, .u32); - const prefill_tokens_slice = try zml.Slice.alloc(self.allocator, prefill_tokens_shape); - defer prefill_tokens_slice.free(self.allocator); - @memset(prefill_tokens_slice.items(u32), 0); - @memcpy(prefill_tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - - var prefill_tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, prefill_tokens_slice); - defer prefill_tokens_buffer.deinit(); - - var prefill_token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); - defer prefill_token_index_buffer.deinit(); - var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); - defer active_length_buffer.deinit(); - - var next_token_buffer: zml.Buffer = undefined; - inference.run(&self.prefill, .{ - .io = self.io, - .tokens_buf = &prefill_tokens_buffer, - .next_token_buf = &next_token_buffer, - .token_index_buf = &prefill_token_index_buffer, - .active_length_buf = &active_length_buffer, - .kv_cache_buffers = &self.kv_cache_buffers, - .rng_buffers = &self.rng_buffers, - }, self.layer_index_buffers); - - defer next_token_buffer.deinit(); - try next_token_buffer.toSlice(self.io, self.generated_token_slice); - } - - pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { - var decoder = try self.tokenizer.decoder(); - defer decoder.deinit(); - - const out_tokens_buffer: []u8 = try self.allocator.alloc(u8, 1024); - defer self.allocator.free(out_tokens_buffer); - - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); - defer current_token_buffer.deinit(); - - var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); - defer token_index_buffer.deinit(); - var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 1), .u32); - defer active_length_buffer.deinit(); - - generation: while (true) { - const token_id = self.generated_token_slice.items(u32)[0]; - if (token_id == self.eos_token_id) break :generation; - - const token = try decoder.feedOne(token_id, out_tokens_buffer); - if (self.think_start) |think_start| if (token_id == think_start) { - try stdout.writeAll("\x1b[2m"); - }; - try stdout.writeAll(token); - if (self.think_end) |think_end| if (token_id == think_end) { - try stdout.writeAll("\x1b[0m"); - }; - try stdout.flush(); - - try all_tokens.append(self.allocator, token_id); - if (all_tokens.items.len >= self.seqlen) break :generation; - - inference.run(&self.decode, .{ - .io = self.io, - .tokens_buf = ¤t_token_buffer, - .next_token_buf = ¤t_token_buffer, - .token_index_buf = &token_index_buffer, - .active_length_buf = &active_length_buffer, - .kv_cache_buffers = &self.kv_cache_buffers, - .rng_buffers = &self.rng_buffers, - }, self.layer_index_buffers); - - try current_token_buffer.toSlice(self.io, self.generated_token_slice); - } - - try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); - try stdout.flush(); - } -}; - -fn tokenizeChatPrompt( - allocator: std.mem.Allocator, - tokenizer: zml.tokenizer.Tokenizer, - prompt: []const u8, - special_tokens: model.Model.SpecialTokens, - is_first_turn: bool, -) ![]const u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenId("<|im_start|>") orelse special_tokens.im_start_token_id; - const im_end = tokenizer.tokenId("<|im_end|>") orelse special_tokens.im_end_token_id; - const newline = tokenizer.tokenId("\\n") orelse return error.NoSuchToken; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, 32); - if (!is_first_turn) { - try tokens.appendSlice(allocator, &.{ im_end, newline }); - } - - try tokens.append(allocator, im_start); - const user_tokens = try encoder.encodeAlloc(allocator, "user\n"); - defer allocator.free(user_tokens); - try tokens.appendSlice(allocator, user_tokens); - const prompt_tokens = try encoder.encodeAlloc(allocator, prompt); - defer allocator.free(prompt_tokens); - try tokens.appendSlice(allocator, prompt_tokens); - try tokens.appendSlice(allocator, &.{ im_end, newline, im_start }); - const assistant_tokens = try encoder.encodeAlloc(allocator, "assistant\n"); - defer allocator.free(assistant_tokens); - try tokens.appendSlice(allocator, assistant_tokens); - - return tokens.toOwnedSlice(allocator); -} diff --git a/examples/llm/models/qwen3_5_moe.zig b/examples/llm/models/qwen3_5_moe.zig deleted file mode 100644 index a97a3207..00000000 --- a/examples/llm/models/qwen3_5_moe.zig +++ /dev/null @@ -1,11 +0,0 @@ -pub const inference = @import("qwen3_5_moe/inference.zig"); -pub const CompilationParameters = inference.CompilationParameters; -pub const CompilationOptions = inference.CompilationParameters; -pub const CompiledModel = inference.CompiledModel; -pub const model = @import("qwen3_5_moe/model.zig"); -pub const Config = model.Config; -pub const Buffers = model.Buffers; -pub const Model = model.Model; -pub const LoadedModel = model.LoadedModel; -pub const session = @import("qwen3_5_moe/session.zig"); -pub const Session = session.Session; diff --git a/examples/llm/models/qwen3_5_moe/inference.zig b/examples/llm/models/qwen3_5_moe/inference.zig deleted file mode 100644 index 896744b3..00000000 --- a/examples/llm/models/qwen3_5_moe/inference.zig +++ /dev/null @@ -1,354 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const common = @import("../common.zig"); -const model = @import("model.zig"); - -const log = std.log.scoped(.qwen3_5_moe); - -pub const CompilationParameters = struct { - kv_cache: model.KvCache, - rng: zml.Tensor.Rng, - moe_backend: zml.moe.Backend, - seqlen: u32, - meshes: common.Meshes, - xla_dump_to: ?[]const u8, - - pub fn init(mdl: model.Model, config: model.Config, seqlen: u32, moe_backend: zml.moe.Backend, meshes: common.Meshes) CompilationParameters { - const dtype = mdl.text_model.embed_tokens.weight.dtype(); - return .{ - .kv_cache = .init(config, 1, seqlen, dtype, .f32, meshes.model), - .rng = .init(), - .moe_backend = moe_backend, - .seqlen = seqlen, - .meshes = meshes, - .xla_dump_to = "/home/ubuntu/xla_dump", - }; - } -}; - -pub const CompilationOptions = CompilationParameters; - -pub const LayerIndexBuffer = union(enum) { - self_attn: zml.Buffer, - linear_attn: zml.Buffer, -}; - -pub const RunArgs = struct { - io: std.Io, - tokens_buffer: *zml.Buffer, - next_token_buffer: *zml.Buffer, - token_index_buffer: *zml.Buffer, - active_length_buffer: *zml.Buffer, - kv_cache_buffers: *zml.Bufferized(model.KvCache), - rng_buffers: *zml.Bufferized(zml.Tensor.Rng), - layer_index_buffers: []const LayerIndexBuffer, -}; - -pub const CompiledModel = struct { - loaded_model: *const model.LoadedModel, - prefill: KernelExe, - decode: KernelExe, - params: CompilationParameters, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - loaded_model: *const model.LoadedModel, - qwen_model: model.Model, - parameters: CompilationParameters, - progress: *std.Progress.Node, - ) !CompiledModel { - const prefill = try compileKernel(allocator, io, platform, qwen_model, parameters, parameters.seqlen, "prefill", progress); - errdefer prefill.deinit(); - const decode = try compileKernel(allocator, io, platform, qwen_model, parameters, 1, "decode", progress); - return .{ - .loaded_model = loaded_model, - .prefill = prefill, - .decode = decode, - .params = parameters, - }; - } - - pub fn deinit(self: *CompiledModel) void { - self.prefill.deinit(); - self.decode.deinit(); - } -}; - -pub const Inference = CompiledModel; - -pub const KernelExe = struct { - embed: zml.FnExe(model.EmbedTokens.forward), - full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn), - linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn), - sample: zml.FnExe(model.Sampler.sampleTokens), - - pub fn deinit(self: *const KernelExe) void { - self.embed.deinit(); - self.full_attention.deinit(); - self.linear_attention.deinit(); - self.sample.deinit(); - } -}; - -pub const LayerRunner = union(enum) { - full_attention: zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}), - linear_attention: zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}), - - pub fn deinit(self: *LayerRunner, allocator: std.mem.Allocator) void { - switch (self.*) { - inline else => |*runner| runner.deinit(allocator), - } - } -}; - -pub const KernelRunner = struct { - embed: zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}), - layers: []LayerRunner, - sample: zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}), - - pub fn init(allocator: std.mem.Allocator, exe: *const KernelExe, buffers: *const model.Buffers) !KernelRunner { - var embed = try zml.FnExe(model.EmbedTokens.forward).Runner(.{.embedding}).init(&exe.embed, allocator, .{ - .embedding = buffers.text_model.embed_tokens, - }); - errdefer embed.deinit(allocator); - - const layers = try allocator.alloc(LayerRunner, buffers.text_model.layers.len); - errdefer allocator.free(layers); - var initialized_layers: usize = 0; - errdefer for (layers[0..initialized_layers]) |*layer| layer.deinit(allocator); - for (layers, buffers.text_model.layers) |*layer, layer_buffers| { - layer.* = switch (layer_buffers.attn) { - .self_attn => .{ .full_attention = try zml.FnExe(model.TransformerLayer.forwardSelfAttn).Runner(.{.layer}).init( - &exe.full_attention, - allocator, - .{ .layer = layer_buffers }, - ) }, - .linear_attn => .{ .linear_attention = try zml.FnExe(model.TransformerLayer.forwardLinearAttn).Runner(.{.layer}).init( - &exe.linear_attention, - allocator, - .{ .layer = layer_buffers }, - ) }, - }; - initialized_layers += 1; - } - - var sample = try zml.FnExe(model.Sampler.sampleTokens).Runner(.{.sampler}).init(&exe.sample, allocator, .{ - .sampler = .{ - .norm = buffers.text_model.norm, - .lm_head = buffers.text_model.lm_head, - }, - }); - errdefer sample.deinit(allocator); - - return .{ .embed = embed, .layers = layers, .sample = sample }; - } - - pub fn deinit(self: *KernelRunner, allocator: std.mem.Allocator) void { - self.embed.deinit(allocator); - for (self.layers) |*layer| layer.deinit(allocator); - allocator.free(self.layers); - self.sample.deinit(allocator); - } -}; - -pub fn run(runner: *KernelRunner, args: RunArgs) void { - var hidden_buffer: zml.Buffer = undefined; - runner.embed.run(args.io, .{ - .inputs = .{ - .tokens = args.tokens_buffer.*, - }, - .outputs = .{ .hidden = &hidden_buffer }, - }, .{}); - defer hidden_buffer.deinit(); - - for (runner.layers, args.layer_index_buffers) |*layer_runner, layer_index_buffer| { - switch (layer_runner.*) { - .full_attention => |*layer| { - const index_buffer = switch (layer_index_buffer) { - .self_attn => |buffer| buffer, - .linear_attn => unreachable, - }; - var layer_cache: zml.Bufferized(model.KvCache.SelfAttnCache) = .{ - .k = args.kv_cache_buffers.self_attn.k, - .v = args.kv_cache_buffers.self_attn.v, - .layer_index = index_buffer, - }; - layer.run(args.io, .{ - .inputs = .{ - .hidden = hidden_buffer, - .token_index = args.token_index_buffer.*, - .cache = layer_cache, - }, - .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, - }, .{}); - args.kv_cache_buffers.self_attn.k = layer_cache.k; - args.kv_cache_buffers.self_attn.v = layer_cache.v; - layer_cache.layer_index.deinit(); - }, - .linear_attention => |*layer| { - const index_buffer = switch (layer_index_buffer) { - .linear_attn => |buffer| buffer, - .self_attn => unreachable, - }; - var layer_cache: zml.Bufferized(model.KvCache.GatedDeltaNetCache) = .{ - .conv_state = args.kv_cache_buffers.gated_delta_net.conv_state, - .recurrent_state = args.kv_cache_buffers.gated_delta_net.recurrent_state, - .layer_index = index_buffer, - }; - layer.run(args.io, .{ - .inputs = .{ - .hidden = hidden_buffer, - .active_length = args.active_length_buffer.*, - .cache = layer_cache, - }, - .outputs = .{ .hidden = &hidden_buffer, .cache = &layer_cache }, - }, .{}); - args.kv_cache_buffers.gated_delta_net.conv_state = layer_cache.conv_state; - args.kv_cache_buffers.gated_delta_net.recurrent_state = layer_cache.recurrent_state; - layer_cache.layer_index.deinit(); - }, - } - } - - runner.sample.run(args.io, .{ - .inputs = .{ - .hidden = hidden_buffer, - .tokens = args.tokens_buffer.*, - .active_length = args.active_length_buffer.*, - .rng = args.rng_buffers.*, - .token_index = args.token_index_buffer.*, - }, - .outputs = .{ - .tokens = args.next_token_buffer, - .rng = args.rng_buffers, - .token_index = args.token_index_buffer, - }, - }, .{}); -} - -fn compileKernel( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - mdl: model.Model, - parameters: CompilationParameters, - seqlen: usize, - phase: []const u8, - progress: *std.Progress.Node, -) !KernelExe { - const full_index = findFirstLayerIndex(mdl.config.text_config.layer_types, .full_attention) orelse return error.MissingFullAttentionLayer; - const linear_index = findFirstLayerIndex(mdl.config.text_config.layer_types, .linear_attention) orelse return error.MissingLinearAttentionLayer; - - var embed_future = try io.concurrent(compileEmbed, .{ allocator, io, platform, mdl, parameters, seqlen, phase, progress }); - errdefer if (embed_future.cancel(io)) |exe| exe.deinit() else |_| {}; - var full_attention_future = try io.concurrent(compileFullAttention, .{ allocator, io, platform, mdl, parameters, seqlen, full_index, phase, progress }); - errdefer if (full_attention_future.cancel(io)) |exe| exe.deinit() else |_| {}; - var linear_attention_future = try io.concurrent(compileLinearAttention, .{ allocator, io, platform, mdl, parameters, seqlen, linear_index, phase, progress }); - errdefer if (linear_attention_future.cancel(io)) |exe| exe.deinit() else |_| {}; - var sample_future = try io.concurrent(compileSample, .{ allocator, io, platform, mdl, parameters, seqlen, phase, progress }); - errdefer if (sample_future.cancel(io)) |exe| exe.deinit() else |_| {}; - - const embed = try embed_future.await(io); - errdefer embed.deinit(); - const full_attention = try full_attention_future.await(io); - errdefer full_attention.deinit(); - const linear_attention = try linear_attention_future.await(io); - errdefer linear_attention.deinit(); - const sample = try sample_future.await(io); - errdefer sample.deinit(); - - return .{ - .embed = embed, - .full_attention = full_attention, - .linear_attention = linear_attention, - .sample = sample, - }; -} - -fn compileEmbed(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.EmbedTokens.forward) { - return compileExe(allocator, io, platform, model.EmbedTokens.forward, .{.{ - .embedding = mdl.text_model.embed_tokens, - .tokens = zml.Tensor.init(.{ .b = 1, .s = seqlen }, .u32), - }}, parameters, progress, phase, "embedding"); -} - -fn compileFullAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardSelfAttn) { - return compileExe(allocator, io, platform, model.TransformerLayer.forwardSelfAttn, .{.{ - .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), - .token_index = zml.Tensor.init(.{}, .u32), - .cache = .{ - .k = parameters.kv_cache.self_attn.k, - .v = parameters.kv_cache.self_attn.v, - .layer_index = zml.Tensor.init(.{}, .u32), - }, - .config = mdl.config, - .moe_backend = parameters.moe_backend, - }}, parameters, progress, phase, "full-attention layer"); -} - -fn compileLinearAttention(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, layer_index: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.TransformerLayer.forwardLinearAttn) { - return compileExe(allocator, io, platform, model.TransformerLayer.forwardLinearAttn, .{.{ - .layer = mdl.text_model.layers[layer_index], - .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), - .active_length = zml.Tensor.init(.{}, .u32), - .cache = .{ - .conv_state = parameters.kv_cache.gated_delta_net.conv_state, - .recurrent_state = parameters.kv_cache.gated_delta_net.recurrent_state, - .layer_index = zml.Tensor.init(.{}, .u32), - }, - .config = mdl.config, - .moe_backend = parameters.moe_backend, - }}, parameters, progress, phase, "linear-attention layer"); -} - -fn compileSample(allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mdl: model.Model, parameters: CompilationParameters, seqlen: usize, phase: []const u8, progress: *std.Progress.Node) !zml.FnExe(model.Sampler.sampleTokens) { - return compileExe(allocator, io, platform, model.Sampler.sampleTokens, .{.{ - .sampler = mdl.text_model.sampler(), - .hidden = hiddenTensor(mdl, seqlen, parameters.meshes.model), - .tokens = .init(.{ .b = 1, .s = seqlen }, .u32), - .active_length = .init(.{}, .u32), - .rng = parameters.rng, - .token_index = zml.Tensor.init(.{}, .u32), - }}, parameters, progress, phase, "sampling"); -} - -fn compileExe( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - comptime function: anytype, - args: std.meta.ArgsTuple(@TypeOf(function)), - parameters: CompilationParameters, - progress: *std.Progress.Node, - phase: []const u8, - component: []const u8, -) !zml.FnExe(function) { - progress.increaseEstimatedTotalItems(1); - const label = try std.fmt.allocPrint(allocator, "Compiling {s} {s}...", .{ phase, component }); - defer allocator.free(label); - var node = progress.start(label, 1); - defer node.end(); - const now: std.Io.Timestamp = .now(io, .awake); - defer log.info("Compiled {s} {s} [{f}]", .{ phase, component, now.untilNow(io, .awake) }); - return zml.FnExe(function).compile(allocator, io, platform, .{ - .meshes = ¶meters.meshes.all(), - .xla_dump_to = parameters.xla_dump_to, - }, args); -} - -fn hiddenTensor(mdl: model.Model, seqlen: usize, mesh: *const zml.Mesh) zml.Tensor { - return .fromShape(zml.Shape.init( - .{ .b = 1, .s = seqlen, .d = mdl.config.text_config.hidden_size }, - mdl.text_model.embed_tokens.weight.dtype(), - ).withPartitioning(mesh, .{ .b = .replicated, .s = .replicated, .d = .replicated })); -} - -fn findFirstLayerIndex(layer_types: []const model.LayerType, target: model.LayerType) ?usize { - for (layer_types, 0..) |layer_type, index| if (layer_type == target) return index; - return null; -} diff --git a/examples/llm/models/qwen3_5_moe/model.zig b/examples/llm/models/qwen3_5_moe/model.zig deleted file mode 100644 index 2a3b0ba8..00000000 --- a/examples/llm/models/qwen3_5_moe/model.zig +++ /dev/null @@ -1,1423 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); -const stdx = zml.stdx; - -const common = @import("../common.zig"); -const inference = @import("inference.zig"); - -const log = std.log.scoped(.qwen3_5_moe); - -pub const Config = struct { - text_config: TextConfig, -}; - -pub const TextConfig = struct { - // General - num_hidden_layers: i64, - layer_types: []const LayerType, - hidden_size: i64, - max_position_embeddings: i64, - rms_norm_eps: f32, - // Self attention - head_dim: i64, - num_attention_heads: i64, - num_key_value_heads: i64, - rope_parameters: RopeParameters, - // Linear attention - linear_conv_kernel_dim: i64, - linear_key_head_dim: i64, - linear_num_key_heads: i64, - linear_num_value_heads: i64, - linear_value_head_dim: i64, - // MoE - num_experts: i64, - num_experts_per_tok: u32, -}; - -// Each layer uses either: full attention (SelfAttn) or linear attention (GatedDeltaNet). -pub const LayerType = enum { - linear_attention, - full_attention, -}; - -pub const RopeParameters = struct { - mrope_section: [3]i64, - partial_rotary_factor: f32, - rope_theta: f32, -}; - -fn partitionProjectedKv(kv: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { - return switch (kv_head_sharding) { - .sharded => |heads| blk: { - const sharded_kv = if (heads.factor != 1) kv.stutter1d(kv.axis(.h), heads.factor) else kv; - break :blk sharded_kv.withPartitioning(.model, .{ .s = .replicated, .h = .model, .hd = .replicated }); - }, - .replicated => kv.withPartitioning(.model, .{ .s = .replicated, .h = .replicated, .hd = .replicated }), - }; -} - -fn partitionCachedKv(tensor: zml.Tensor, kv_head_sharding: zml.Sharding.DimSharding) zml.Tensor { - var kv = tensor.rename(.{ .s = .k }); - return switch (kv_head_sharding) { - .sharded => |heads| blk: { - if (heads.factor != 1) { - kv = kv.stutter1d(kv.axis(.h), heads.factor); - } - break :blk kv.withPartitioning(.model, .{ .k = .replicated, .h = .model, .hd = .replicated }); - }, - .replicated => kv.withPartitioning(.model, .{ .k = .replicated, .h = .replicated, .hd = .replicated }), - }; -} - -pub const LoadedModel = struct { - inner: Model, - parsed_config: std.json.Parsed(Config), - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - repo: std.Io.Dir, - store: zml.io.TensorStore.View, - generation: common.GenerationOptions, - ) !LoadedModel { - const parsed_config = try common.parseConfig(Config, allocator, io, repo); - errdefer parsed_config.deinit(); - - const options: Model.GenOptions = .{ - .sampling_strategy = generation.sampling_strategy, - .max_seq_len = parsed_config.value.text_config.max_position_embeddings, - }; - - return .{ - .inner = try .init(allocator, store, parsed_config.value, options), - .parsed_config = parsed_config, - }; - } - - pub fn deinit(self: *LoadedModel, allocator: std.mem.Allocator) void { - self.inner.deinit(allocator); - self.parsed_config.deinit(); - } - - pub fn loadBuffers( - self: *LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - progress: *std.Progress.Node, - ) !Buffers { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, &self.inner); - errdefer self.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - try loader.load(io, Model, &self.inner, &buffers, store, .{ .progress = progress }); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *const LoadedModel, buffers: *Buffers, allocator: std.mem.Allocator) void { - _ = self; - TextModel.unloadBuffers(&buffers.text_model, allocator); - } - - pub fn compile( - self: *const LoadedModel, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - backend: zml.attention.Backend, - meshes: common.Meshes, - seqlen: usize, - progress: *std.Progress.Node, - ) !inference.CompiledModel { - _ = backend; - const moe_gate_up = self.inner.text_model.layers[0].moe.gate_up_proj; - log.info("Moe dtype : {}", .{moe_gate_up.weight.dtype()}); - const moe_backend = try zml.moe.Backend.auto(platform, moe_gate_up.quantizationScheme(), moe_gate_up.weight.dtype()); - const params = inference.CompilationParameters.init(self.inner, self.parsed_config.value, @intCast(seqlen), moe_backend, meshes); - return inference.CompiledModel.init(allocator, io, platform, self, self.inner, params, progress); - } -}; - -pub const Buffers = zml.Bufferized(Model); - -pub const Model = struct { - pub const GenOptions = struct { sampling_strategy: zml.nn.SamplingStrategy = .{}, max_seq_len: i64 }; - - pub const SpecialTokens = struct { - im_start_token_id: u32, - im_end_token_id: u32, - end_of_text_token_id: u32, - }; - - text_model: TextModel, - - config: Config, - special_tokens: SpecialTokens = .{ - .im_start_token_id = 248045, - .im_end_token_id = 248046, - .end_of_text_token_id = 248044, - }, - - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, gen_options: GenOptions) !Model { - // For some Qwen3.5 versions, the output projection lm_head has a standalone weight tensor, while for others it's the same as the input embedding layer - return .{ - .text_model = try .init(allocator, store.withPrefix("model.language_model"), config, gen_options), - .config = config, - }; - } - - pub fn deinit(self: Model, allocator: std.mem.Allocator) void { - self.text_model.deinit(allocator); - } - - pub fn load( - self: *const Model, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *zml.io.TensorStore, - meshes: []const *const zml.Mesh, - progress: *std.Progress.Node, - ) !zml.Bufferized(Model) { - progress.increaseEstimatedTotalItems(store.view().count()); - const now: std.Io.Timestamp = .now(io, .awake); - - var buffers = try zml.mem.bufferize(allocator, Model, self); - errdefer Model.unloadBuffers(&buffers, allocator); - - var loader: zml.io.Loader = try .init(allocator, platform, .{ - .dma_chunks = 32, - .dma_chunk_size = 256 * zml.MiB, - .parallelism = 16, - }); - defer loader.deinit(); - - loader.load(io, Model, self, &buffers, store, meshes); - try loader.await(io); - - const took = now.untilNow(io, .awake); - const total_bytes: u64 = loader.bytes_loaded.raw; - const bytes_per_sec: u64 = @intFromFloat(@as(f64, @floatFromInt(total_bytes)) / (@as(f64, @floatFromInt(took.nanoseconds)) / std.time.ns_per_s)); - log.info("Loaded weights [{Bi:.2}, {f}, {Bi:.2}/s]", .{ total_bytes, took, bytes_per_sec }); - - return buffers; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Model), allocator: std.mem.Allocator) void { - TextModel.unloadBuffers(&self.text_model, allocator); - } - pub fn forward( - self: Model, - tokens_: zml.Tensor, - token_index: zml.Tensor, - active_length: zml.Tensor, - kv_cache: KvCache, - rng: zml.Tensor.Rng, - moe_backend: zml.moe.Backend, - ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { - const tokens = tokens_.withPartialTags(.{.s}); - const new_tokens, const updated_kv_cache, const new_rng = self.text_model.forward(tokens, token_index, active_length, kv_cache, self.config, rng, moe_backend); - return .{ new_tokens, updated_kv_cache, new_rng }; - } -}; - -//========================Text model======================== - -// Use this intermediate struct to compile sampleTokens without taking the whole TextModel as input -pub const Sampler = struct { - norm: RmsNorm, - lm_head: zml.nn.Linear, - gen_options: Model.GenOptions, - - pub const Input = struct { - sampler: Sampler, - hidden: zml.Tensor, - tokens: zml.Tensor, - active_length: zml.Tensor, - rng: zml.Tensor.Rng, - token_index: zml.Tensor, - }; - - pub const Output = struct { - tokens: zml.Tensor, - rng: zml.Tensor.Rng, - token_index: zml.Tensor, - }; - - pub fn sampleTokens(input: Input) Output { - const self = input.sampler; - const last_index = input.active_length.convert(.i32).addConstant(-1); - const x = self.norm.forward(input.hidden.slice(.s, .dyn(last_index, 1))); - const logits = self.lm_head.forward(x.withPartialTags(.{.d}), x.dtype()).rename(.{ .dout = .voc }); - const next_tokens, const new_rng = zml.nn.sampleTokens(logits, self.gen_options.sampling_strategy, input.rng); - const tokens = next_tokens.convert(.u32); - return .{ - .tokens = if (input.tokens.dim(.s) == 1) tokens.reuseBuffer(input.tokens) else tokens, - .rng = new_rng, - .token_index = input.token_index.addConstant(1).reuseBuffer(input.token_index), - }; - } -}; - -pub const EmbedTokens = struct { - pub const Input = struct { - embedding: zml.nn.TokenEmbedding, - tokens: zml.Tensor, - }; - - pub const Output = struct { - hidden: zml.Tensor, - }; - - pub fn forward(input: Input) Output { - return .{ .hidden = input.embedding.forward(input.tokens) }; - } -}; - -pub const TextModel = struct { - embed_tokens: zml.nn.TokenEmbedding, - layers: []TransformerLayer, - norm: RmsNorm, - lm_head: zml.nn.Linear, - gen_options: Model.GenOptions, - - pub fn init( - allocator: std.mem.Allocator, - store: zml.io.TensorStore.View, - config: Config, - gen_options: Model.GenOptions, - ) !TextModel { - const lm_head_prefix = if (store.root().hasKey("lm_head.weight")) "lm_head" else "model.language_model.embed_tokens"; - - const layers = try allocator.alloc(TransformerLayer, @intCast(config.text_config.num_hidden_layers)); - errdefer allocator.free(layers); - - for (layers, 0..) |*layer, i| { - errdefer for (layers[0..i]) |previous_layer| previous_layer.deinit(allocator); - layer.* = try .init(allocator, store.withPrefix("layers").withLayer(i), config, i); - } - - return .{ - .embed_tokens = .{ .weight = store.createTensor("embed_tokens.weight", .{ .voc, .d }, .model, .{ .voc = .replicated, .d = .model }) }, - .layers = layers, - .norm = RmsNorm.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), - .lm_head = .init(store.root().withPrefix(lm_head_prefix).createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), null, .d), - .gen_options = gen_options, - }; - } - - pub fn deinit(self: TextModel, allocator: std.mem.Allocator) void { - for (self.layers) |layer| { - layer.deinit(allocator); - } - allocator.free(self.layers); - } - - pub fn unloadBuffers(self: *zml.Bufferized(TextModel), allocator: std.mem.Allocator) void { - self.embed_tokens.weight.deinit(); - for (self.layers) |*layer| { - TransformerLayer.unloadBuffers(layer, allocator); - } - allocator.free(self.layers); - RmsNorm.unloadBuffers(&self.norm); - zml.nn.Linear.unloadBuffers(&self.lm_head); - } - - pub fn sampler(self: TextModel) Sampler { - return .{ - .norm = self.norm, - .lm_head = self.lm_head, - .gen_options = self.gen_options, - }; - } - - pub fn forward( - self: TextModel, - tokens: zml.Tensor, - token_index: zml.Tensor, - active_length: zml.Tensor, - kv_cache: KvCache, - config: Config, - rng: zml.Tensor.Rng, - moe_backend: zml.moe.Backend, - ) struct { zml.Tensor, KvCache, zml.Tensor.Rng } { - var hidden_states = EmbedTokens.forward(.{ - .embedding = self.embed_tokens, - .tokens = tokens, - }).hidden; - - var updated_kv_cache = kv_cache; - for (self.layers, 0..) |layer, i| { - hidden_states, updated_kv_cache = layer.forward(hidden_states, token_index, active_length, updated_kv_cache.atLayer(i), config, moe_backend); - } - - const result = Sampler.sampleTokens(.{ - .sampler = self.sampler(), - .hidden = hidden_states, - .tokens = tokens, - .active_length = active_length, - .rng = rng, - .token_index = token_index, - }); - return .{ result.tokens, updated_kv_cache.reuseBuffer(kv_cache), result.rng }; - } -}; - -pub const TransformerLayer = struct { - const Attn = union(enum) { - self_attn: SelfAttn, - linear_attn: GatedDeltaNet, - }; - - input_layernorm: RmsNorm, - attn: Attn, - moe: Moe, - post_attention_layernorm: RmsNorm, - - pub const SelfAttnInput = struct { - layer: TransformerLayer, - hidden: zml.Tensor, - token_index: zml.Tensor, - cache: KvCache.SelfAttnCache, - config: Config, - moe_backend: zml.moe.Backend, - }; - - pub const SelfAttnOutput = struct { - hidden: zml.Tensor, - cache: KvCache.SelfAttnCache, - }; - - pub const LinearAttnInput = struct { - layer: TransformerLayer, - hidden: zml.Tensor, - active_length: zml.Tensor, - cache: KvCache.GatedDeltaNetCache, - config: Config, - moe_backend: zml.moe.Backend, - }; - - pub const LinearAttnOutput = struct { - hidden: zml.Tensor, - cache: KvCache.GatedDeltaNetCache, - }; - - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config, layer_index: usize) !TransformerLayer { - const is_full_attention = config.text_config.layer_types[layer_index] == .full_attention; - return .{ - .input_layernorm = RmsNorm.init(store.withPrefix("input_layernorm"), config.text_config.rms_norm_eps), - .attn = if (is_full_attention) - .{ .self_attn = try .init(store.withPrefix("self_attn"), config) } - else - .{ .linear_attn = .init(store.withPrefix("linear_attn"), config) }, - .moe = try .init(allocator, store.withPrefix("mlp"), config), - .post_attention_layernorm = RmsNorm.init(store.withPrefix("post_attention_layernorm"), config.text_config.rms_norm_eps), - }; - } - - pub fn deinit(self: TransformerLayer, allocator: std.mem.Allocator) void { - self.moe.deinit(allocator); - } - - pub fn unloadBuffers(self: *zml.Bufferized(TransformerLayer), allocator: std.mem.Allocator) void { - RmsNorm.unloadBuffers(&self.input_layernorm); - switch (self.attn) { - .self_attn => |*self_attn| SelfAttn.unloadBuffers(self_attn), - .linear_attn => |*linear_attn| GatedDeltaNet.unloadBuffers(linear_attn), - } - Moe.unloadBuffers(&self.moe, allocator); - RmsNorm.unloadBuffers(&self.post_attention_layernorm); - } - - pub fn forwardSelfAttn(input: SelfAttnInput) SelfAttnOutput { - const self = input.layer; - const x0 = input.hidden; - _ = input.config; - const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); - const normalized_x0 = self.input_layernorm.forward(x0_replicated); - - const self_attn = switch (self.attn) { - .self_attn => |self_attn| self_attn, - .linear_attn => unreachable, - }; - const attention_output, const updated_kv_cache = self_attn.forward(normalized_x0, input.token_index, input.cache); - - const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); - const normalized_hidden = self.post_attention_layernorm.forward(x1); - - const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); - - return .{ - .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), - .cache = updated_kv_cache, - }; - } - - pub fn forwardLinearAttn(input: LinearAttnInput) LinearAttnOutput { - const self = input.layer; - const x0 = input.hidden; - _ = input.config; - const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); - const normalized_x0 = self.input_layernorm.forward(x0_replicated); - - const linear_attn = switch (self.attn) { - .linear_attn => |linear_attn| linear_attn, - .self_attn => unreachable, - }; - const attention_output, const updated_kv_cache = linear_attn.forward(normalized_x0, input.cache, input.active_length); - - const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); - const normalized_hidden = self.post_attention_layernorm.forward(x1); - - const moe_output = self.moe.forward(normalized_hidden, input.moe_backend); - - return .{ - .hidden = moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }).reuseBuffer(x0), - .cache = updated_kv_cache, - }; - } - - pub fn forward( - self: TransformerLayer, - x0: zml.Tensor, - token_index: zml.Tensor, - active_length: zml.Tensor, - kv_cache: KvCache.LayerView, - config: Config, - moe_backend: zml.moe.Backend, - ) struct { zml.Tensor, KvCache } { - _ = config; - const x0_replicated = x0.withPartitioning(.model, .{ .d = .replicated }); - const normalized_x0 = self.input_layernorm.forward(x0_replicated); - - var attention_output: zml.Tensor = undefined; - var updated_kv_cache: KvCache = kv_cache.parent; - switch (self.attn) { - .self_attn => |*self_attn| { - const result = self_attn.forward(normalized_x0, token_index, kv_cache.cache.self_attn); - attention_output = result[0]; - updated_kv_cache.self_attn = result[1]; - }, - .linear_attn => |*linear_attn| { - const result = linear_attn.forward(normalized_x0, kv_cache.cache.linear_attn, active_length); - attention_output = result[0]; - updated_kv_cache.gated_delta_net = result[1]; - }, - } - - const x1 = attention_output.add(x0_replicated).withPartitioning(.model, .{ .d = .replicated }); - const normalized_hidden = self.post_attention_layernorm.forward(x1); - - const moe_output = self.moe.forward(normalized_hidden, moe_backend); - - return .{ moe_output.add(x1).withPartitioning(.model, .{ .d = .replicated }), updated_kv_cache }; - } -}; - -pub const SelfAttn = struct { - q_proj: zml.nn.Linear, - q_proj_scale: ?zml.Tensor, - k_proj: zml.nn.Linear, - k_proj_scale: ?zml.Tensor, - v_proj: zml.nn.Linear, - v_proj_scale: ?zml.Tensor, - - q_norm: RmsNorm, - k_norm: RmsNorm, - - num_heads: i64, - num_kv_heads: i64, - head_dim: i64, - rotary_dim: i64, - rotary_embed: TextRotaryEmbedding, - o_proj: zml.nn.Linear, - - o_proj_scale: ?zml.Tensor, - - fn initProj(store: zml.io.TensorStore.View, partitions: anytype, bias_partitions: anytype) zml.nn.Linear { - return .init( - store.createTensor("weight", .{ .dout, .d }, .model, partitions), - store.maybeCreateTensor("bias", .{.dout}, .model, bias_partitions), - .d, - ); - } - - pub fn init(store: zml.io.TensorStore.View, config: Config) !SelfAttn { - const rotary_dim: i64 = @intFromFloat(@as(f32, @floatFromInt(config.text_config.head_dim)) * - config.text_config.rope_parameters.partial_rotary_factor); - return .{ - .q_proj = initProj(store.withPrefix("q_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), - .q_proj_scale = null, - .k_proj = initProj(store.withPrefix("k_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), - .k_proj_scale = null, - .v_proj = initProj(store.withPrefix("v_proj"), .{ .dout = .model, .d = .replicated }, .{ .dout = .model }), - .v_proj_scale = null, - .o_proj = initProj(store.withPrefix("o_proj"), .{ .dout = .replicated, .d = .model }, .{ .dout = .replicated }), - .o_proj_scale = null, - .q_norm = RmsNorm.init(store.withPrefix("q_norm"), config.text_config.rms_norm_eps), - .k_norm = RmsNorm.init(store.withPrefix("k_norm"), config.text_config.rms_norm_eps), - .num_heads = config.text_config.num_attention_heads, - .num_kv_heads = config.text_config.num_key_value_heads, - .head_dim = config.text_config.head_dim, - .rotary_dim = rotary_dim, - .rotary_embed = .init(rotary_dim, config.text_config.rope_parameters.rope_theta, config.text_config.rope_parameters.mrope_section), - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(SelfAttn)) void { - zml.Buffer.deinitAll(SelfAttn, self); - } - - fn projectQAndGate(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { - const q_proj = self.q_proj.forward(x, x.dtype()) - .splitAxis(.dout, .{ .h = self.num_heads, .hd = 2 * self.head_dim }); - const q, var gate = q_proj.chunkExact(.hd, 2); - gate = gate.merge(.{ .d_out_proj = .{ .h, .hd } }); - return .{ q, gate }; - } - - fn projectKV(self: SelfAttn, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { - const num_kv_heads = if (self.num_kv_heads > 0) self.num_kv_heads else self.num_heads; - const k = self.k_proj.forward(x, x.dtype()) - .splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); - const v = self.v_proj.forward(x, x.dtype()) - .splitAxis(.dout, .{ .h = num_kv_heads, .hd = self.head_dim }); - return .{ k, v }; - } - - pub fn forward( - self: SelfAttn, - x: zml.Tensor, - token_index: zml.Tensor, - kv_cache: KvCache.SelfAttnCache, - ) struct { zml.Tensor, KvCache.SelfAttnCache } { - const tp = zml.Compiler.current().mesh(.model); - const x_qkv = x.withPartitioning(tp, .{ .d = .replicated }); - var q, var gate = self.projectQAndGate(x_qkv); - var k, var v = self.projectKV(x_qkv); - const kv_head_sharding = tp.shardableDim(k.dim(.h), .model, q.dim(.h)); - - q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); - gate = gate.withPartitioning(tp, .{ .s = .replicated, .d_out_proj = .model }); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); - q = self.q_norm.forward(q.rename(.{ .hd = .d })).rename(.{ .d = .hd }); - k = self.k_norm.forward(k.rename(.{ .hd = .d })).rename(.{ .d = .hd }); - - const dtype = q.dtype(); - const position_ids: zml.Tensor = zml.Tensor.arange(.{ .end = x.dim(.s) }, .i64) - .withTags(.{.s}).insertAxes(.s, .{.b}).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64)) - .add(token_index.convert(.i64).broad(zml.Shape.init(.{ .b = x.dim(.b), .s = x.dim(.s) }, .i64))); - - const cos, const sin = self.rotary_embed.getCosAndSin(position_ids, dtype); - q = self.rotary_embed.applyRope(q, cos, sin); - k = self.rotary_embed.applyRope(k, cos, sin); - q = q.withPartitioning(tp, .{ .s = .replicated, .h = .model, .hd = .replicated }); - k = partitionProjectedKv(k, kv_head_sharding); - v = partitionProjectedKv(v, kv_head_sharding); - - const new_kv_cache = kv_cache.update(k, v, token_index.convert(.u32)); - k = new_kv_cache.keys().convert(dtype); - v = new_kv_cache.values().convert(dtype); - q = q.rename(.{ .s = .q }).withPartitioning(tp, .{ .q = .replicated, .h = .model, .hd = .replicated }); - k = partitionCachedKv(k, kv_head_sharding); - v = partitionCachedKv(v, kv_head_sharding); - - const head_partitioning: zml.attention.Partitioning = switch (kv_head_sharding) { - .sharded => .head(.model), - .replicated => .head(.replicated), - }; - const attn_output = zml.attention.attention( - q, - k, - v, - token_index, - zml.attention.Metadata.init(.fromBackend(.stablehlo)), - zml.attention.Parameters.init(.fromBackend(.stablehlo)), - zml.Compiler.current().mesh(.model), - head_partitioning, - ).rename(.{ .q = .s }).merge(.{ .d_out_proj = .{ .h, .hd } }); - - const gated_output = attn_output.mul(gate.sigmoid()); - const projected_output = self.o_proj.forward(gated_output.rename(.{ .d_out_proj = .d }), gated_output.dtype()).rename(.{ .dout = .d }).withPartitioning(tp, .{ .d = .replicated }); - - return .{ projected_output, new_kv_cache }; - } -}; - -pub const Mlp = struct { - up_proj: zml.nn.Linear, - up_proj_scale: ?zml.Tensor, - gate_proj: zml.nn.Linear, - gate_proj_scale: ?zml.Tensor, - down_proj: zml.nn.Linear, - down_proj_scale: ?zml.Tensor, - - pub fn init(store: zml.io.TensorStore.View) Mlp { - return .{ - .up_proj = .init( - store.withPrefix("up_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), - store.withPrefix("up_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), - .d, - ), - .up_proj_scale = null, - .gate_proj = .init( - store.withPrefix("gate_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .model, .d = .replicated }), - store.withPrefix("gate_proj").maybeCreateTensor("bias", .{.dout}, .model, .{ .dout = .model }), - .d, - ), - .gate_proj_scale = null, - .down_proj = .init( - store.withPrefix("down_proj").createTensor("weight", .{ .dout, .d }, .model, .{ .dout = .replicated, .d = .model }), - store.withPrefix("down_proj").maybeCreateTensor("bias", .{.d}, .model, .{ .d = .replicated }), - .d, - ), - .down_proj_scale = null, - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Mlp)) void { - zml.Buffer.deinitAll(Mlp, self); - } - - pub fn forward(self: Mlp, x: zml.Tensor) zml.Tensor { - const up_projed = self.up_proj.forward(x, x.dtype()); - const gate = self.gate_proj.forward(x, x.dtype()); - const hidden = gate.silu().mul(up_projed).rename(.{ .dout = .d }); - - const output = self.down_proj.forward(hidden, hidden.dtype()); - return output; - } -}; - -const Router = struct { - router: zml.nn.Linear, - num_experts_per_tok: u32, - - pub fn init(store: zml.io.TensorStore.View, num_experts_per_tok: u32) Router { - return .{ - .router = .init( - store.createTensor("weight", .{ .expert, .d }, .replicated, .{ .expert = .replicated, .d = .replicated }), - store.maybeCreateTensor("bias", .{.expert}, .replicated, .{ .expert = .replicated }), - .d, - ), - .num_experts_per_tok = num_experts_per_tok, - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(Router)) void { - zml.Buffer.deinitAll(Router, self); - } - - pub fn forward(self: Router, x: zml.Tensor) struct { zml.Tensor, zml.Tensor } { - const router_logits = self.router.forward(x, .f32); - const routing = router_logits.topK(.{ .topk = .expert }, self.num_experts_per_tok, .{}); - const topk_ids = routing.indices.convert(.i32); - const router_scores = routing.values.softmax(.topk); - return .{ router_scores, topk_ids }; - } -}; - -pub const Moe = struct { - shared_expert: Mlp, - shared_expert_gate: zml.nn.Linear, - gate_up_proj: zml.nn.Linear, - down_proj: zml.nn.Linear, - router: Router, - - pub fn init(allocator: std.mem.Allocator, store: zml.io.TensorStore.View, config: Config) !Moe { - _ = allocator; - const experts_store = store.withPrefix("experts"); - const gate_up_proj_tensor = experts_store.createTensor( - "gate_up_proj", - .{ .expert, .dout, .d }, - .experts, - .{ .expert = .experts, .dout = .replicated, .d = .replicated }, - ); - const down_proj_tensor = experts_store.createTensor( - "down_proj", - .{ .expert, .d, .dout }, - .experts, - .{ .expert = .experts, .d = .replicated, .dout = .replicated }, - ); - - return .{ - .shared_expert = Mlp.init(store.withPrefix("shared_expert")), - .shared_expert_gate = .init( - store.withPrefix("shared_expert_gate").createTensor("weight", .{ .dout, .d }, .replicated, .{ .dout = .replicated, .d = .replicated }), - store.withPrefix("shared_expert_gate").maybeCreateTensor("bias", .{.dout}, .replicated, .{ .dout = .replicated }), - .d, - ), - .gate_up_proj = .init(gate_up_proj_tensor, null, .d), - .down_proj = .init(down_proj_tensor, null, .dout), - .router = Router.init(store.withPrefix("gate"), config.text_config.num_experts_per_tok), - }; - } - - pub fn deinit(self: Moe, allocator: std.mem.Allocator) void { - _ = self; - _ = allocator; - } - - pub fn forward(self: Moe, x: zml.Tensor, moe_backend: zml.moe.Backend) zml.Tensor { - const routing_scores, const topk_ids = self.router.forward(x); - - const moe_output = zml.moe.forwardMoe( - x.merge(.{ .t = .{ .b, .s } }), - topk_ids.merge(.{ .t = .{ .b, .s } }), - routing_scores.merge(.{ .t = .{ .b, .s } }), - self.gate_up_proj, - self.down_proj, - moe_backend, - zml.Compiler.current().mesh(.experts), - .expert(.experts), - .{ - .activation = .{ .swiglu = .{} }, - .quantize_input = false, - .routing_weight_placement = .after_down, - }, - ).reshape(x.shape()); - - const shared_gate = self.shared_expert_gate.forward(x, x.dtype()).sigmoid().broad(x.shape()); - const shared = self.shared_expert.forward(x).rename(.{ .dout = .d }).mul(shared_gate).withPartitioning(.model, .{ - .b = .replicated, - .s = .replicated, - .d = .replicated, - }); - - return moe_output.add(shared); - } - - pub fn unloadBuffers(self: *zml.Bufferized(Moe), allocator: std.mem.Allocator) void { - _ = allocator; - zml.Buffer.deinitAll(Moe, self); - } -}; - -pub const TextRotaryEmbedding = struct { - rope_opts: zml.nn.RopeOpts, - rotary_dim: i64, - mrope_section: [3]i64, - - pub fn init(rotary_dim: i64, theta: f32, mrope_section: [3]i64) TextRotaryEmbedding { - return .{ - .rope_opts = .{ - .layout = .real_im_pass, - .scaling = .{ .default = .{ .rope_theta = theta } }, - }, - .rotary_dim = rotary_dim, - .mrope_section = mrope_section, - }; - } - - pub fn getCosAndSin(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { - const inv_freq = zml.nn.invFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); - - const freqs_t = position_ids.convert(.f32).outer(inv_freq); - - const emb = zml.Tensor.concatenate(&.{ freqs_t, freqs_t }, -1); - const cos = emb.cos().convert(dtype); - const sin = emb.sin().convert(dtype); - - return .{ cos, sin }; - } - - pub fn getCosAndSinInterleaved(self: TextRotaryEmbedding, position_ids: zml.Tensor, dtype: zml.DataType) struct { zml.Tensor, zml.Tensor } { // To be used later in image extension - const stacked_position_ids = zml.Tensor.stack(&.{ position_ids, position_ids, position_ids }, 0, .g).convert(.f32); - const inv_freq = zml.nn.InvFreq(self.rotary_dim, self.rope_opts).withTags(.{.hd}); - - var freqs = stacked_position_ids.outer(inv_freq); - var freqs_t, var freqs_h, var freqs_w = freqs.chunkExact(.g, 3); - freqs_t = freqs_t.squeeze(.g); - freqs_h = freqs_h.squeeze(.g); - freqs_w = freqs_w.squeeze(.g); - - const h_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[1] }, .i32), .h).scale(3).addConstant(1); - const w_indices = zml.Tensor.iota(zml.Shape.init(.{ .h = self.mrope_section[2] }, .i32), .h).scale(3).addConstant(2); - - const h_input = freqs_h.gather(.{ .dh = h_indices }, .{ .indices_are_sorted = true }); - const w_input = freqs_w.gather(.{ .dh = w_indices }, .{ .indices_are_sorted = true }); - freqs_t = freqs_t.scatterSlices(.{ .dh = h_indices }, h_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); - freqs = freqs_t.scatterSlices(.{ .dh = w_indices }, w_input, .{ .update_fn = zml.Tensor.ScatterOpts.override }); - - const emb = zml.Tensor.concatenate(&.{ freqs, freqs }, -1); - const cos = emb.cos().convert(dtype); - const sin = emb.sin().convert(dtype); - - return .{ cos, sin }; - } - - fn rotateHalf(x: zml.Tensor) zml.Tensor { - const half_dim = @divExact(x.dim(-1), 2); - const x1 = x.slice(-1, .{ .start = 0, .end = half_dim }); - const x2 = x.slice(-1, .{ .start = half_dim, .end = x.dim(-1) }); - return zml.Tensor.concatenate(&.{ x2.negate(), x1 }, -1); - } - - pub fn applyRope(self: TextRotaryEmbedding, x: zml.Tensor, cos: zml.Tensor, sin: zml.Tensor) zml.Tensor { - const x_rot = x.slice(-1, .{ .start = 0, .end = self.rotary_dim }); - const x_pass = x.slice(-1, .{ .start = self.rotary_dim, .end = x.dim(-1) }); - - const cos_x = cos.insertAxes(.hd, .{.h}).broad(x_rot.shape()); - const sin_x = sin.insertAxes(.hd, .{.h}).broad(x_rot.shape()); - - const rotated = x_rot.mul(cos_x).add(rotateHalf(x_rot).mul(sin_x)); - - return zml.Tensor.concatenate(&.{ rotated, x_pass }, -1); - } -}; - -pub const GatedDeltaNet = struct { - in_proj_qkv: zml.nn.Linear, - in_proj_qkv_scale: ?zml.Tensor, - in_proj_z: zml.nn.Linear, - in_proj_z_scale: ?zml.Tensor, - in_proj_b: zml.nn.Linear, - in_proj_a: zml.nn.Linear, - out_proj: zml.nn.Linear, - out_proj_scale: ?zml.Tensor, - conv1d_weight: zml.Tensor, - dt_bias: zml.Tensor, - aLog: zml.Tensor, - norm: RmsNormGated, - - num_k_heads: i64, - num_v_heads: i64, - qk_head_repetition: u32, - head_k_dim: i64, - head_v_dim: i64, - conv_kernel_size: i64, - - fn initProj(store: zml.io.TensorStore.View, partitions: anytype) zml.nn.Linear { - return .init(store.createTensor("weight", .{ .dout, .d }, .model, partitions), null, .d); - } - - pub fn init(store: zml.io.TensorStore.View, config: Config) GatedDeltaNet { - const qk_head_repetition: u32 = - @intCast(@divExact(config.text_config.linear_num_value_heads, config.text_config.linear_num_key_heads)); - return .{ - .in_proj_qkv = initProj(store.withPrefix("in_proj_qkv"), .{ .dout = .model, .d = .replicated }), - .in_proj_qkv_scale = null, - .in_proj_z = initProj(store.withPrefix("in_proj_z"), .{ .dout = .model, .d = .replicated }), - .in_proj_z_scale = null, - .in_proj_b = initProj(store.withPrefix("in_proj_b"), .{ .dout = .model, .d = .replicated }), - .in_proj_a = initProj(store.withPrefix("in_proj_a"), .{ .dout = .model, .d = .replicated }), - .out_proj = initProj(store.withPrefix("out_proj"), .{ .dout = .replicated, .d = .model }), - .out_proj_scale = null, - .conv1d_weight = store.withPrefix("conv1d").createTensor("weight", .{ .out, .in, .kernel_size }, .model, .{ .out = .model, .in = .replicated, .kernel_size = .replicated }), - .dt_bias = store.createTensor("dt_bias", .{.vh}, .model, .{ .vh = .model }), - .aLog = store.createTensor("A_log", .{.vh}, .model, .{ .vh = .model }), - .norm = RmsNormGated.init(store.withPrefix("norm"), config.text_config.rms_norm_eps), - .num_k_heads = config.text_config.linear_num_key_heads, - .num_v_heads = config.text_config.linear_num_value_heads, - .qk_head_repetition = qk_head_repetition, - .head_k_dim = config.text_config.linear_key_head_dim, - .head_v_dim = config.text_config.linear_value_head_dim, - .conv_kernel_size = config.text_config.linear_conv_kernel_dim, - }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(GatedDeltaNet)) void { - zml.Buffer.deinitAll(GatedDeltaNet, self); - } - - fn recurrentGatedDeltaRule( - query: zml.Tensor, - key: zml.Tensor, - value: zml.Tensor, - g: zml.Tensor, - beta: zml.Tensor, - initial_state: ?zml.Tensor, - active_length: zml.Tensor, - ) struct { zml.Tensor, zml.Tensor } { - const scale: f32 = 1.0 / @sqrt(@as(f32, @floatFromInt(query.dim(.khd)))); - const query_norm = zml.nn.normalizeL2(query.rename(.{ .kh = .vh }), 1e-6); - const key_norm = zml.nn.normalizeL2(key.rename(.{ .kh = .vh }), 1e-6); - - const query_f32, const key_f32, const value_f32, const alpha_f32, const beta_f32 = .{ - query_norm.convert(.f32).scale(scale).rename(.{ .vh = .h, .khd = .k }), - key_norm.convert(.f32).rename(.{ .vh = .h, .khd = .k }), - value.convert(.f32).rename(.{ .vh = .h, .vhd = .v }), - g.convert(.f32).exp().rename(.{ .vh = .h }), - beta.convert(.f32).rename(.{ .vh = .h }), - }; - - const initial_recurrent_state = if (initial_state) |state| - state.convert(.f32).transpose(.{ .b, .vh, .vhd, .khd }).rename(.{ .vh = .h, .vhd = .v, .khd = .k }) - else - zml.Tensor.constant(zml.DataType.zero(.f32)).broad(zml.Shape.init(.{ - .b = value.dim(.b), - .h = value.dim(.vh), - .v = value.dim(.vhd), - .k = query.dim(.khd), - }, .f32)); - - const result = zml.nn.GatedDeltaNet.forward( - query_f32, - key_f32, - value_f32, - alpha_f32, - beta_f32, - .{ .s = initial_recurrent_state }, - active_length, - ); - - return .{ - result.outputs.rename(.{ .h = .vh, .v = .vhd }).convert(query.dtype()), - result.state.s.transpose(.{ .b, .h, .k, .v }).rename(.{ .h = .vh, .k = .khd, .v = .vhd }), - }; - } - - fn buildUpdatedConvState(input: zml.Tensor, left_pad: i64) zml.Tensor { - const copy_len = @min(input.dim(.s), left_pad); - const tail = input.slice(.s, .{ .start = input.dim(.s) - copy_len, .end = input.dim(.s) }); - if (copy_len == left_pad) return tail; - - const padding_shape = zml.Shape.init(.{ .b = input.dim(.b), .s = left_pad - copy_len, .mix = input.dim(.mix) }, input.dtype()); - const padding = zml.Tensor.constant(input.dtype().zero()).broad(padding_shape); - return zml.Tensor.concatenate(&.{ padding, tail }, .s); - } - - fn buildUpdatedConvStateFromPrefix(input: zml.Tensor, left_pad: i64, active_length: zml.Tensor) zml.Tensor { - const padding = zml.Tensor.zeroes(input.shape().setDim(.s, left_pad)); - const padded = zml.Tensor.concatenate(&.{ padding, input }, .s); - return padded.slice(.s, .dyn(active_length.convert(.i64), left_pad)); - } - - pub fn forward(self: GatedDeltaNet, x: zml.Tensor, cache: KvCache.GatedDeltaNetCache, active_length: zml.Tensor) struct { zml.Tensor, KvCache.GatedDeltaNetCache } { - const key_dim = self.num_k_heads * self.head_k_dim; - const value_dim = self.num_v_heads * self.head_v_dim; - const conv_dim = 2 * key_dim + value_dim; - const left_pad = self.conv_kernel_size - 1; - - const x_in = x.withPartitioning(.model, .{ .d = .replicated }); - const projected_qkv = self.in_proj_qkv.forward(x_in, x_in.dtype()) - .rename(.{ .dout = .mix }).withPartitioning(.model, .{ .s = .replicated, .mix = .model }); - const use_cached_state = x.dim(.s) == 1 and left_pad > 0; - const conv_input = if (use_cached_state) - zml.Tensor.concatenate(&.{ cache.convState(), projected_qkv }, .s) - else - projected_qkv; - - const kernel = self.conv1d_weight; - var mixed_qkv = zml.Tensor.conv1d( - conv_input, - kernel, - .{ - .padding = &.{ left_pad, 0 }, - .input_batch_dimension = 0, - .input_feature_dimension = 2, - .input_spatial_dimensions = 1, - .kernel_output_feature_dimension = 0, - .kernel_input_feature_dimension = 1, - .kernel_spatial_dimensions = 2, - .output_batch_dimension = 0, - .output_feature_dimension = 2, - .output_spatial_dimensions = 1, - .feature_group_count = conv_dim, - }, - ) - .silu(); - - if (use_cached_state) { - mixed_qkv = mixed_qkv.slice(.s, .{ .start = mixed_qkv.dim(.s) - 1, .end = mixed_qkv.dim(.s) }); - } - mixed_qkv = mixed_qkv.withPartitioning(.model, .{ .s = .replicated, .mix = .model }); - - const z = self.in_proj_z.forward(x_in, x_in.dtype()) - .splitAxis(.dout, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) - .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); - const b = self.in_proj_b.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); - const a = self.in_proj_a.forward(x_in, x_in.dtype()).rename(.{ .dout = .vh }).withPartitioning(.model, .{ .s = .replicated, .vh = .model }); - - const query = mixed_qkv - .slice(.mix, .{ .start = 0, .end = key_dim }) - .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) - .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); - const key = mixed_qkv - .slice(.mix, .{ .start = key_dim, .end = 2 * key_dim }) - .splitAxis(.mix, .{ .kh = self.num_k_heads, .khd = self.head_k_dim }) - .withPartitioning(.model, .{ .s = .replicated, .kh = .model, .khd = .replicated }); - const value = mixed_qkv - .slice(.mix, .{ .start = 2 * key_dim, .end = 2 * key_dim + value_dim }) - .splitAxis(.mix, .{ .vh = self.num_v_heads, .vhd = self.head_v_dim }) - .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); - - const beta = b.sigmoid(); - const aLog_type = self.aLog.dtype(); - const g = self.aLog.broad(a.shape()).exp().mul(zml.nn.softplus(a.convert(aLog_type).add(self.dt_bias.convert(aLog_type).broad(a.shape())), .{})).negate(); - - const query_for_rule = if (self.qk_head_repetition == 1) query else query.stutter1d(query.axis(.kh), self.qk_head_repetition); - const key_for_rule = if (self.qk_head_repetition == 1) key else key.stutter1d(key.axis(.kh), self.qk_head_repetition); - - const core_attn_out, const last_recurrent_state = recurrentGatedDeltaRule( - query_for_rule, - key_for_rule, - value, - g, - beta, - if (use_cached_state) cache.recurrentState() else null, - active_length, - ); - - const core_attn_out_normed = self.norm - .forward( - core_attn_out.rename(.{ .vhd = .d }), - z.rename(.{ .vhd = .d }), - ) - .rename(.{ .d = .vhd }) - .withPartitioning(.model, .{ .s = .replicated, .vh = .model, .vhd = .replicated }); - - const output = self.out_proj.forward(core_attn_out_normed.merge(.{ .d = .{ .vh, .vhd } }), core_attn_out_normed.dtype()) - .rename(.{ .dout = .d }).withPartitioning(.model, .{ .d = .replicated }); - const updated_cache = cache.update( - if (use_cached_state) buildUpdatedConvState(conv_input, left_pad) else buildUpdatedConvStateFromPrefix(projected_qkv, left_pad, active_length), - last_recurrent_state, - ); - return .{ output, updated_cache }; - } -}; - -pub const RmsNorm = struct { - weight: zml.Tensor, - eps: f32 = 1e-6, - - pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNorm { - return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(RmsNorm)) void { - self.weight.deinit(); - } - - pub fn forward(self: RmsNorm, x: zml.Tensor) zml.Tensor { - const x_f32 = x.convert(.f32); - const weight_f32 = self.weight.convert(.f32); - - const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); - return normalized.mul(weight_f32.broad(x.shape())).add(normalized).convert(x.dtype()); - } -}; - -pub const RmsNormGated = struct { - weight: zml.Tensor, - eps: f32 = 1e-6, - - pub fn init(store: zml.io.TensorStore.View, eps: f32) RmsNormGated { - return .{ .weight = store.createTensor("weight", .{.d}, .model, .{ .d = .replicated }), .eps = eps }; - } - - pub fn unloadBuffers(self: *zml.Bufferized(RmsNormGated)) void { - self.weight.deinit(); - } - - pub fn forward(self: RmsNormGated, x: zml.Tensor, gate: zml.Tensor) zml.Tensor { - const x_f32 = x.convert(.f32); - const gate_f32 = gate.convert(.f32); - const weight_f32 = self.weight.convert(.f32); - - const normalized = zml.nn.rmsNorm(x_f32, .d, self.eps); - const output = normalized.mul(weight_f32.broad(x.shape())); - - const gated_output = output.mul(gate_f32.silu()); - return gated_output.convert(x.dtype()); - } -}; - -pub const KvCache = struct { - layer_types: []const LayerType, - self_attn: SelfAttnCache, - gated_delta_net: GatedDeltaNetCache, - - pub const SelfAttnCache = struct { - k: zml.Tensor, - v: zml.Tensor, - layer_index: zml.Tensor, - - pub fn init(config: Config, batch_dim: i64, max_seq_len: i64, dtype: zml.DataType, model_meshe: *const zml.Mesh) SelfAttnCache { - const num_self_attn_layers = countLayers(config.text_config.layer_types, .full_attention); - const kv_shape = zml.Shape.init(.{ - .b = batch_dim, - .layer = num_self_attn_layers, - .s = max_seq_len, - .h = config.text_config.num_key_value_heads, - .hd = config.text_config.head_dim, - }, dtype); - const kv_head_sharding = model_meshe.shardableDim(kv_shape.dim(.h), .model, config.text_config.num_attention_heads); - const sharded_kv_shape = switch (kv_head_sharding) { - .sharded => |heads| kv_shape.setDim(.h, heads.dim).withPartitioning(model_meshe, .{ .h = .model }), - .replicated => kv_shape.withPartitioning(model_meshe, .{ .h = .replicated }), - }; - return .{ - .k = zml.Tensor.fromShape(sharded_kv_shape), - .v = zml.Tensor.fromShape(sharded_kv_shape), - .layer_index = .init(.{}, .u32), - }; - } - - pub fn initBuffer(self: SelfAttnCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(SelfAttnCache) { - return .{ - .k = try zml.Buffer.uninitialized(io, platform, self.k.shape(), .{}), - .v = try zml.Buffer.uninitialized(io, platform, self.v.shape(), .{}), - .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(SelfAttnCache)) void { - self.k.deinit(); - self.v.deinit(); - self.layer_index.deinit(); - } - - pub fn keys(self: SelfAttnCache) zml.Tensor { - return self.k.slice(.layer, .dynSingle(self.layer_index)); - } - - pub fn values(self: SelfAttnCache) zml.Tensor { - return self.v.slice(.layer, .dynSingle(self.layer_index)); - } - - pub fn update(self: SelfAttnCache, new_k: zml.Tensor, new_v: zml.Tensor, token_index: ?zml.Tensor) SelfAttnCache { - const k_shape = self.k.shape().drop(.layer); - var layer = self.layer_index; - layer = if (token_index) |idx| layer.broad(idx.shape()) else layer; - - return if (token_index) |idx| .{ - .k = self.k.scatterSlices( - .{ .layer = layer, .s = idx }, - new_k.convert(self.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer, .s = idx }, - new_v.convert(self.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), - .layer_index = self.layer_index, - } else .{ - .k = self.k.scatterSlices( - .{ .layer = layer }, - new_k.convert(self.k.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.k), - .v = self.v.scatterSlices( - .{ .layer = layer }, - new_v.convert(self.v.dtype()).transpose(k_shape), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.v), - .layer_index = self.layer_index, - }; - } - - pub fn atLayer(self: SelfAttnCache, layer_index: usize) SelfAttnCache { - return .{ - .k = self.k, - .v = self.v, - .layer_index = zml.Tensor.scalar(layer_index, .u32), - }; - } - - pub fn reuseBuffer(self: SelfAttnCache, other: SelfAttnCache) SelfAttnCache { - return .{ - .k = self.k.reuseBuffer(other.k), - .v = self.v.reuseBuffer(other.v), - .layer_index = self.layer_index.reuseBuffer(other.layer_index), - }; - } - }; - - pub const GatedDeltaNetCache = struct { - conv_state: zml.Tensor, - recurrent_state: zml.Tensor, - layer_index: zml.Tensor, - - pub fn init(config: Config, batch_dim: i64, conv_dtype: zml.DataType, recurrent_dtype: zml.DataType, model_meshe: *const zml.Mesh) GatedDeltaNetCache { - const num_linear_attn_layers = countLayers(config.text_config.layer_types, .linear_attention); - const conv_dim = 2 * config.text_config.linear_num_key_heads * config.text_config.linear_key_head_dim + config.text_config.linear_num_value_heads * config.text_config.linear_value_head_dim; - const conv_state_shape = zml.Shape.init(.{ - .b = batch_dim, - .layer = num_linear_attn_layers, - .s = config.text_config.linear_conv_kernel_dim - 1, - .mix = conv_dim, - }, conv_dtype); - const recurrent_state_shape = zml.Shape.init(.{ - .b = batch_dim, - .layer = num_linear_attn_layers, - .vh = config.text_config.linear_num_value_heads, - .khd = config.text_config.linear_key_head_dim, - .vhd = config.text_config.linear_value_head_dim, - }, recurrent_dtype); - const sharded_conv_state_shape = conv_state_shape.withPartitioning(model_meshe, .{ .mix = .model }); - const sharded_recurrent_state_shape = recurrent_state_shape.withPartitioning(model_meshe, .{ .vh = .model }); - return .{ - .conv_state = zml.Tensor.fromShape(sharded_conv_state_shape), - .recurrent_state = zml.Tensor.fromShape(sharded_recurrent_state_shape), - .layer_index = .init(.{}, .u32), - }; - } - - pub fn initBuffer(self: GatedDeltaNetCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(GatedDeltaNetCache) { - return .{ - .conv_state = try zml.Buffer.uninitialized(io, platform, self.conv_state.shape(), .{}), - .recurrent_state = try zml.Buffer.uninitialized(io, platform, self.recurrent_state.shape(), .{}), - .layer_index = try zml.Buffer.scalar(io, platform, 0, .u32), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(GatedDeltaNetCache)) void { - self.conv_state.deinit(); - self.recurrent_state.deinit(); - self.layer_index.deinit(); - } - - pub fn convState(self: GatedDeltaNetCache) zml.Tensor { - return self.conv_state.slice(.layer, .dynSingle(self.layer_index)); - } - - pub fn recurrentState(self: GatedDeltaNetCache) zml.Tensor { - return self.recurrent_state.slice(.layer, .dynSingle(self.layer_index)); - } - - pub fn update(self: GatedDeltaNetCache, new_conv_state: ?zml.Tensor, new_recurrent_state: ?zml.Tensor) GatedDeltaNetCache { - const conv_state = if (new_conv_state) |state| - self.conv_state.scatterSlices( - .{ .layer = self.layer_index }, - state.convert(self.conv_state.dtype()).transpose(self.conv_state.shape().drop(.layer)), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.conv_state) - else - self.conv_state; - - const recurrent_state = if (new_recurrent_state) |state| - self.recurrent_state.scatterSlices( - .{ .layer = self.layer_index }, - state.convert(self.recurrent_state.dtype()).transpose(self.recurrent_state.shape().drop(.layer)), - .{ .indices_are_sorted = true, .update_fn = zml.Tensor.ScatterOpts.override }, - ).reuseBuffer(self.recurrent_state) - else - self.recurrent_state; - - return .{ - .conv_state = conv_state, - .recurrent_state = recurrent_state, - .layer_index = self.layer_index, - }; - } - - pub fn atLayer(self: GatedDeltaNetCache, layer_index: usize) GatedDeltaNetCache { - return .{ - .conv_state = self.conv_state, - .recurrent_state = self.recurrent_state, - .layer_index = zml.Tensor.scalar(layer_index, .u32), - }; - } - - pub fn reuseBuffer(self: GatedDeltaNetCache, other: GatedDeltaNetCache) GatedDeltaNetCache { - return .{ - .conv_state = self.conv_state.reuseBuffer(other.conv_state), - .recurrent_state = self.recurrent_state.reuseBuffer(other.recurrent_state), - .layer_index = self.layer_index.reuseBuffer(other.layer_index), - }; - } - }; - - pub fn init( - config: Config, - batch_dim: i64, - max_seq_len: i64, - cache_dtype: zml.DataType, - recurrent_dtype: zml.DataType, - model_meshe: *const zml.Mesh, - ) KvCache { - return .{ - .layer_types = config.text_config.layer_types, - .self_attn = SelfAttnCache.init(config, batch_dim, max_seq_len, cache_dtype, model_meshe), - .gated_delta_net = GatedDeltaNetCache.init(config, batch_dim, cache_dtype, recurrent_dtype, model_meshe), - }; - } - - pub fn initBuffer(self: KvCache, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(KvCache) { - return .{ - .self_attn = try self.self_attn.initBuffer(io, platform), - .gated_delta_net = try self.gated_delta_net.initBuffer(io, platform), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(KvCache)) void { - SelfAttnCache.deinitBuffer(&self.self_attn); - GatedDeltaNetCache.deinitBuffer(&self.gated_delta_net); - } - - pub const LayerView = struct { - parent: KvCache, - cache: union(enum) { - self_attn: SelfAttnCache, - linear_attn: GatedDeltaNetCache, - }, - }; - - pub fn atLayer(self: KvCache, layer_index: usize) LayerView { - return switch (getDenseIndex(self.layer_types, layer_index)) { - .full_attention => |dense_index| .{ - .parent = self, - .cache = .{ .self_attn = self.self_attn.atLayer(dense_index.layer_dense_index) }, - }, - .linear_attention => |dense_index| .{ - .parent = self, - .cache = .{ .linear_attn = self.gated_delta_net.atLayer(dense_index.layer_dense_index) }, - }, - }; - } - - pub fn reuseBuffer(self: KvCache, other: KvCache) KvCache { - return .{ - .layer_types = self.layer_types, - .self_attn = self.self_attn.reuseBuffer(other.self_attn), - .gated_delta_net = self.gated_delta_net.reuseBuffer(other.gated_delta_net), - }; - } - - fn countLayers(layer_types: []const LayerType, layer_type: LayerType) i64 { - var count: i64 = 0; - for (layer_types) |registered_layer_type| { - if (registered_layer_type == layer_type) count += 1; - } - return count; - } - - fn getDenseIndex(layer_types: []const LayerType, layer_index: usize) union(enum) { - full_attention: struct { layer_dense_index: usize }, - linear_attention: struct { layer_dense_index: usize }, - } { - var self_attn_layer_index: usize = 0; - var linear_attn_layer_index: usize = 0; - for (layer_types[0..layer_index]) |layer_type| { - switch (layer_type) { - .full_attention => self_attn_layer_index += 1, - .linear_attention => linear_attn_layer_index += 1, - } - } - return switch (layer_types[layer_index]) { - .full_attention => .{ .full_attention = .{ .layer_dense_index = self_attn_layer_index } }, - .linear_attention => .{ .linear_attention = .{ .layer_dense_index = linear_attn_layer_index } }, - }; - } -}; diff --git a/examples/llm/models/qwen3_5_moe/session.zig b/examples/llm/models/qwen3_5_moe/session.zig deleted file mode 100644 index 6a819f26..00000000 --- a/examples/llm/models/qwen3_5_moe/session.zig +++ /dev/null @@ -1,223 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const inference = @import("inference.zig"); -const model = @import("model.zig"); - -pub const Session = struct { - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - compiled_model: *inference.CompiledModel, - prefill: inference.KernelRunner, - decode: inference.KernelRunner, - kv_cache_buffers: zml.Bufferized(model.KvCache), - rng_buffers: zml.Bufferized(zml.Tensor.Rng), - layer_index_buffers: []inference.LayerIndexBuffer, - generated_token_slice: zml.Slice, - tokenizer: zml.tokenizer.Tokenizer, - seqlen: u32, - eos_token_id: u32, - special_tokens: model.Model.SpecialTokens, - think_start: ?u32, - think_end: ?u32, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - tokenizer: zml.tokenizer.Tokenizer, - compiled_model: *inference.CompiledModel, - model_buffers: *model.Buffers, - ) !Session { - var kv_cache_buffers = try compiled_model.params.kv_cache.initBuffer(io, platform); - errdefer model.KvCache.deinitBuffer(&kv_cache_buffers); - - const seed: u128 = @intCast(std.Io.Clock.now(.real, io).toNanoseconds()); - var rng_buffers = try zml.Tensor.Rng.init().initBuffer(io, platform, seed); - errdefer zml.Tensor.Rng.deinitBuffer(&rng_buffers); - - const layer_types = compiled_model.loaded_model.inner.config.text_config.layer_types; - const layer_index_buffers = try allocator.alloc(inference.LayerIndexBuffer, layer_types.len); - errdefer allocator.free(layer_index_buffers); - var initialized_layer_index_buffers: usize = 0; - errdefer for (layer_index_buffers[0..initialized_layer_index_buffers]) |*layer_index_buffer| { - switch (layer_index_buffer.*) { - .self_attn => |*buffer| buffer.deinit(), - .linear_attn => |*buffer| buffer.deinit(), - } - }; - - var self_attn_layer_index: u32 = 0; - var linear_attn_layer_index: u32 = 0; - for (layer_types, layer_index_buffers) |layer_type, *layer_index_buffer| { - layer_index_buffer.* = switch (layer_type) { - .full_attention => b: { - defer self_attn_layer_index += 1; - break :b .{ .self_attn = try .scalar(io, platform, self_attn_layer_index, .u32) }; - }, - .linear_attention => b: { - defer linear_attn_layer_index += 1; - break :b .{ .linear_attn = try .scalar(io, platform, linear_attn_layer_index, .u32) }; - }, - }; - initialized_layer_index_buffers += 1; - } - - const generated_token_slice = try zml.Slice.alloc(allocator, zml.Shape.init(.{ .b = 1, .s = 1 }, .u32)); - errdefer generated_token_slice.free(allocator); - - var prefill = try inference.KernelRunner.init(allocator, &compiled_model.prefill, model_buffers); - errdefer prefill.deinit(allocator); - const decode = try inference.KernelRunner.init(allocator, &compiled_model.decode, model_buffers); - - return .{ - .allocator = allocator, - .io = io, - .platform = platform, - .compiled_model = compiled_model, - .prefill = prefill, - .decode = decode, - .kv_cache_buffers = kv_cache_buffers, - .rng_buffers = rng_buffers, - .layer_index_buffers = layer_index_buffers, - .generated_token_slice = generated_token_slice, - .tokenizer = tokenizer, - .seqlen = compiled_model.params.seqlen, - .eos_token_id = compiled_model.loaded_model.inner.special_tokens.end_of_text_token_id, - .special_tokens = compiled_model.loaded_model.inner.special_tokens, - .think_start = tokenizer.tokenId(""), - .think_end = tokenizer.tokenId(""), - }; - } - - pub fn deinit(self: *Session) void { - self.prefill.deinit(self.allocator); - self.decode.deinit(self.allocator); - model.KvCache.deinitBuffer(&self.kv_cache_buffers); - zml.Tensor.Rng.deinitBuffer(&self.rng_buffers); - for (self.layer_index_buffers) |*layer_index_buffer| { - switch (layer_index_buffer.*) { - .self_attn => |*buffer| buffer.deinit(), - .linear_attn => |*buffer| buffer.deinit(), - } - } - self.allocator.free(self.layer_index_buffers); - self.generated_token_slice.free(self.allocator); - } - - pub fn tokenizePrompt(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { - return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, true); - } - - pub fn tokenizeTurn(self: *const Session, allocator: std.mem.Allocator, prompt: []const u8) ![]const u32 { - return tokenizeChatPrompt(allocator, self.tokenizer, prompt, self.special_tokens, false); - } - - pub fn runPrefill(self: *Session, all_tokens: []const u32) !void { - const tokens_slice = try zml.Slice.alloc(self.allocator, .init(.{ .b = 1, .s = self.seqlen }, .u32)); - defer tokens_slice.free(self.allocator); - @memset(tokens_slice.items(u32), 0); - @memcpy(tokens_slice.items(u32)[0..all_tokens.len], all_tokens); - - var tokens_buffer = try zml.Buffer.fromSlice(self.io, self.platform, tokens_slice); - defer tokens_buffer.deinit(); - var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 0), .u32); - defer token_index_buffer.deinit(); - var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.len)), .u32); - defer active_length_buffer.deinit(); - - var next_token_buffer: zml.Buffer = undefined; - inference.run(&self.prefill, .{ - .io = self.io, - .tokens_buffer = &tokens_buffer, - .next_token_buffer = &next_token_buffer, - .token_index_buffer = &token_index_buffer, - .active_length_buffer = &active_length_buffer, - .kv_cache_buffers = &self.kv_cache_buffers, - .rng_buffers = &self.rng_buffers, - .layer_index_buffers = self.layer_index_buffers, - }); - - defer next_token_buffer.deinit(); - try next_token_buffer.toSlice(self.io, self.generated_token_slice); - } - - pub fn runDecode(self: *Session, all_tokens: *std.ArrayList(u32), stdout: *std.Io.Writer) !void { - var decoder = try self.tokenizer.decoder(); - defer decoder.deinit(); - const out_tokens_buffer = try self.allocator.alloc(u8, 1024); - defer self.allocator.free(out_tokens_buffer); - - var current_token_buffer = try zml.Buffer.fromSlice(self.io, self.platform, self.generated_token_slice); - defer current_token_buffer.deinit(); - var token_index_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, @intCast(all_tokens.items.len)), .u32); - defer token_index_buffer.deinit(); - var active_length_buffer = try zml.Buffer.scalar(self.io, self.platform, @as(u32, 1), .u32); - defer active_length_buffer.deinit(); - - generation: while (true) { - const token_id = self.generated_token_slice.items(u32)[0]; - if (token_id == self.eos_token_id) break :generation; - - const token = try decoder.feedOne(token_id, out_tokens_buffer); - if (self.think_start) |think_start| if (token_id == think_start) try stdout.writeAll("\x1b[2m"); - try stdout.writeAll(token); - if (self.think_end) |think_end| if (token_id == think_end) try stdout.writeAll("\x1b[0m"); - try stdout.flush(); - - try all_tokens.append(self.allocator, token_id); - if (all_tokens.items.len >= self.seqlen) break :generation; - - inference.run(&self.decode, .{ - .io = self.io, - .tokens_buffer = ¤t_token_buffer, - .next_token_buffer = ¤t_token_buffer, - .token_index_buffer = &token_index_buffer, - .active_length_buffer = &active_length_buffer, - .kv_cache_buffers = &self.kv_cache_buffers, - .rng_buffers = &self.rng_buffers, - .layer_index_buffers = self.layer_index_buffers, - }); - try current_token_buffer.toSlice(self.io, self.generated_token_slice); - } - - try stdout.writeAll(try decoder.finalize(out_tokens_buffer)); - try stdout.flush(); - } -}; - -fn tokenizeChatPrompt(allocator: std.mem.Allocator, tokenizer: zml.tokenizer.Tokenizer, prompt: []const u8, special_tokens: model.Model.SpecialTokens, is_first_turn: bool) ![]const u32 { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const im_start = tokenizer.tokenId("<|im_start|>") orelse special_tokens.im_start_token_id; - const im_end = tokenizer.tokenId("<|im_end|>") orelse special_tokens.im_end_token_id; - - var tokens: std.ArrayList(u32) = try .initCapacity(allocator, prompt.len + 32); - if (!is_first_turn) { - try tokens.append(allocator, im_end); - const newline = try encoder.encodeAlloc(allocator, "\n"); - defer allocator.free(newline); - try tokens.appendSlice(allocator, newline); - } - - try tokens.append(allocator, im_start); - const user = try encoder.encodeAlloc(allocator, "user\n"); - defer allocator.free(user); - try tokens.appendSlice(allocator, user); - const prompt_encoded = try encoder.encodeAlloc(allocator, prompt); - defer allocator.free(prompt_encoded); - try tokens.appendSlice(allocator, prompt_encoded); - try tokens.append(allocator, im_end); - const newline = try encoder.encodeAlloc(allocator, "\n"); - defer allocator.free(newline); - try tokens.appendSlice(allocator, newline); - try tokens.append(allocator, im_start); - const assistant = try encoder.encodeAlloc(allocator, "assistant\n"); - defer allocator.free(assistant); - try tokens.appendSlice(allocator, assistant); - - return tokens.toOwnedSlice(allocator); -} diff --git a/examples/mnist/BUILD.bazel b/examples/mnist/BUILD.bazel index f0ddf811..ad10f9f1 100644 --- a/examples/mnist/BUILD.bazel +++ b/examples/mnist/BUILD.bazel @@ -1,9 +1,5 @@ -load("@bazel_lib//lib:transitions.bzl", "platform_transition_filegroup") -load("@bazel_lib//lib:expand_template.bzl", "expand_template") -load("@rules_oci//oci:defs.bzl", "oci_image", "oci_load", "oci_push") +load("@bazel_skylib//rules:build_test.bzl", "build_test") load("@rules_zig//zig:defs.bzl", "zig_binary") -load("@tar.bzl//tar:mtree.bzl", "mtree_spec") -load("@tar.bzl//tar:tar.bzl", "tar") zig_binary( name = "mnist", @@ -23,99 +19,7 @@ zig_binary( ], ) -zig_binary( - name = "mnist_dp", - args = [ - "$(location @mnist//:mnist.safetensors)", - "$(location @mnist//:t10k-images.idx3-ubyte)", - ], - data = [ - "@mnist//:mnist.safetensors", - "@mnist//:t10k-images.idx3-ubyte", - ], - main = "mnist_data_parallel.zig", - srcs = ["mnist.zig"], - visibility = ["//visibility:public"], - deps = [ - "//bazel", - "//zml", - ], -) - -mtree_spec( - name = "mtree", - srcs = [":mnist"], - tags = ["manual"], -) - -tar( - name = "archive", - srcs = [":mnist"], - args = [ - "--options", - "zstd:compression-level=9", - ], - compress = "zstd", - mtree = ":mtree", - tags = ["manual"], -) - -expand_template( - name = "entrypoint", - data = [ - ":mnist", - "@mnist//:mnist.safetensors", - "@mnist//:t10k-images.idx3-ubyte", - ], - substitutions = { - ":model": "$(rlocationpath @mnist//:mnist.safetensors)", - ":data": "$(rlocationpath @mnist//:t10k-images.idx3-ubyte)", - }, - template = [ - "./{}/mnist".format(package_name()), - "/{}/mnist.runfiles/:model".format(package_name()), - "/{}/mnist.runfiles/:data".format(package_name()), - ], - tags = ["manual"], -) - -# The actual docker image, with entrypoint, created from tar archive -oci_image( - name = "image_", - base = "@distroless_cc_debian12", - entrypoint = ":entrypoint", - target_compatible_with = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], - tars = [":archive"], - tags = ["manual"], -) - -# We always want to create the image for Linux -platform_transition_filegroup( - name = "image", - srcs = [":image_"], - target_platform = "@zml//platforms:linux_amd64", - tags = ["manual"], -) - -# Load will immediately load the image (eg: docker load) -oci_load( - name = "load", - image = ":image", - repo_tags = [ - "distroless/mnist:latest", - ], - tags = ["manual"], -) - -# Bazel target for pushing the Linux image to our docker registry -oci_push( - name = "push", - image = ":image", - remote_tags = ["latest"], - # override with -- --repository foo.bar/org/image - repository = "index.docker.io/steeve/mnist", - tags = ["manual"], +build_test( + name = "test", + targets = [":mnist"], ) diff --git a/examples/mnist/mnist.zig b/examples/mnist/mnist.zig index f5842af8..696de3af 100644 --- a/examples/mnist/mnist.zig +++ b/examples/mnist/mnist.zig @@ -2,13 +2,12 @@ const std = @import("std"); const log = std.log; const zml = @import("zml"); +const stdx = zml.stdx; pub const std_options: std.Options = .{ .log_level = .info, }; -pub const Img = [28 * 28]u8; - /// Model definition const Mnist = struct { fc1: Layer, @@ -20,8 +19,8 @@ const Mnist = struct { pub fn init(store: zml.io.TensorStore.View) Layer { return .{ - .weight = store.createReplicatedTensor("weight", .{ .d_out, .d }), - .bias = store.createReplicatedTensor("bias", .{.d_out}), + .weight = store.createTensorWithTags("weight", .{ .d_out, .d }), + .bias = store.createTensorWithTags("bias", .{.d_out}), }; } @@ -44,16 +43,12 @@ const Mnist = struct { platform: *const zml.Platform, store: *const zml.io.TensorStore, ) !zml.Bufferized(Mnist) { - var buffers = try zml.mem.bufferize(allocator, Mnist, self); - errdefer unloadBuffers(&buffers); - - var loader: zml.io.Loader = try .init(allocator, platform, .default); - errdefer loader.deinit(); - - try loader.load(io, Mnist, self, &buffers, store, .{}); - try loader.await(io); - - return buffers; + return zml.io.load(Mnist, self, allocator, io, platform, .{ + .store = store, + .parallelism = 1, + .dma_chunks = 1, + .dma_chunk_size = 16 * 1024 * 1024, + }); } pub fn unloadBuffers(self: *zml.Bufferized(Mnist)) void { @@ -67,10 +62,7 @@ const Mnist = struct { pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { var x = input.flatten().convert(.f32).withTags(.{.d}); const layers: []const Layer = &.{ self.fc1, self.fc2 }; - const compiler = zml.Compiler.current(); - for (0.., layers) |i, layer| { - compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); - defer compiler.popLocation(); + for (layers) |layer| { x = layer.forward(x); } return x.argMax(0).indices.convert(.u8); @@ -91,22 +83,22 @@ pub fn main(init: std.process.Init) !void { var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); defer registry.deinit(); - // Auto-select platform - const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator, io); - // Init model - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); + var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry); defer store.deinit(); const mnist_model: Mnist = .init(store.view()); + // Auto-select platform + const platform: *zml.Platform = try .auto(allocator, io, .{}); + defer platform.deinit(allocator); + // // Compile model const input: zml.Tensor = .init(.{ 28, 28 }, .u8); var exe = blk: { log.info("Compiling model....", .{}); const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Compiled model [{f}]", .{start.untilNow(io, .awake)}); - break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}, .{}); + defer log.info("✅ Compiled model [{D}]", .{stdx.fmt.fmtDuration(start.untilNow(io, .awake))}); + break :blk try platform.compile(allocator, io, mnist_model, .forward, .{input}); }; defer exe.deinit(); @@ -114,10 +106,10 @@ pub fn main(init: std.process.Init) !void { var mnist_buffers = blk: { log.info("Transfering weights....", .{}); const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Transferred weights [{f}]", .{ - start.untilNow(io, .awake), + defer log.info("✅ Transferred weights [{D}]", .{ + stdx.fmt.fmtDuration(start.untilNow(io, .awake)), }); - break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); + break :blk try mnist_model.load(allocator, io, platform, &store); }; defer Mnist.unloadBuffers(&mnist_buffers); @@ -138,16 +130,16 @@ pub fn main(init: std.process.Init) !void { // inference - can be looped const idx = rng.random().uintLessThan(u64, 10000); - var sample: Img align(16) = undefined; - _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * @sizeOf(Img))); + var sample: [28 * 28]u8 align(16) = undefined; + _ = try dataset.readPositionalAll(io, &sample, 16 + (idx * 28 * 28)); var input_buffer: zml.Buffer = try .fromSlice(io, platform, zml.Slice.init(input.shape(), &sample)); defer input_buffer.deinit(); - printDigit(&sample); + printDigit(sample); args.set(.{ mnist_buffers, input_buffer }); - exe.call(io, args, &results, .{}); + exe.call(args, &results); var result: zml.Buffer = results.get(zml.Buffer); defer result.deinit(); @@ -160,14 +152,14 @@ pub fn main(init: std.process.Init) !void { , .{digits[try result.getValue(u8, io)]}); } -pub fn printDigit(digit_flat: *const Img) void { - const digit: *const [28][28]u8 = @ptrCast(digit_flat); +fn printDigit(digit: [28 * 28]u8) void { var buffer: [28][30][2]u8 = undefined; for (0..28) |y| { buffer[y][0] = .{ '|', ' ' }; buffer[y][29] = .{ '|', '\n' }; for (1..29) |x| { - const val = digit[y][x - 1]; + const idx = (y * 28) + (x - 1); + const val = digit[idx]; buffer[y][x] = blk: { if (val > 240) break :blk .{ '*', '*' }; if (val > 225) break :blk .{ 'o', 'o' }; @@ -186,7 +178,7 @@ pub fn printDigit(digit_flat: *const Img) void { , .{std.mem.asBytes(&buffer)}); } -pub const digits = [_][]const u8{ +const digits = [_][]const u8{ \\ | ### | \\ | # # | \\ | # # | diff --git a/examples/mnist/mnist_data_parallel.zig b/examples/mnist/mnist_data_parallel.zig deleted file mode 100644 index b7805e49..00000000 --- a/examples/mnist/mnist_data_parallel.zig +++ /dev/null @@ -1,206 +0,0 @@ -//! Data-parallel version of mnist.zig, -//! Separated to kept `mnist.zig` minimal - -const std = @import("std"); - -const zml = @import("zml"); - -const base = @import("mnist.zig"); - -const log = std.log.scoped(.mnist); - -pub const std_options: std.Options = .{ - .log_level = .info, -}; - -/// Model definition -const Mnist = struct { - fc1: Layer, - fc2: Layer, - - pub fn init(store: zml.io.TensorStore.View) Mnist { - return .{ - // Layer 1 is sharded following it's output axis - .fc1 = .{ - .weight = store.createTensor("fc1.weight", .{ .d_out, .d }, .dp_mp, .{ .d_out = .model }), - .bias = store.createTensor("fc1.bias", .{.d_out}, .dp_mp, .{ .d_out = .model }), - }, - // Layer 2 is sharded following it's input axis (and bias is fully replicated) - .fc2 = .{ - .weight = store.createTensor("fc2.weight", .{ .d_out, .d }, .dp_mp, .{ .d = .model }), - .bias = store.createReplicatedTensor("fc2.bias", .{.d_out}), - }, - }; - } - - pub fn load( - self: *const Mnist, - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - store: *const zml.io.TensorStore, - ) !zml.Bufferized(Mnist) { - var buffers = try zml.mem.bufferize(allocator, Mnist, self); - errdefer zml.Buffer.deinitAll(Mnist, &buffers); - - var loader: zml.io.Loader = try .init(allocator, platform, .default); - errdefer loader.deinit(); - - try loader.load(io, Mnist, self, &buffers, store, .{}); - try loader.await(io); - - return buffers; - } - - /// just two linear layers + relu activation - pub fn forward(self: Mnist, input: zml.Tensor) zml.Tensor { - var x = input.onMemory(.host_pinned).toMemory(.device).merge(.{ .d = .{ .x, .y } }).convert(.f32); - - const layers: []const Layer = &.{ self.fc1, self.fc2 }; - - const compiler = zml.Compiler.current(); - - for (0.., layers) |i, layer| { - compiler.pushLocationFmt(@src(), "layer[{d}]", .{i}); - defer compiler.popLocation(); - x = layer.forward(x); - } - - return x.argMax(.d).indices.convert(.u8) - .withPartitioning(.dp_mp, .{ .b = .data, .d = .replicated }) - .toMemory(.host_pinned); - } - - const Layer = struct { - weight: zml.Tensor, - bias: zml.Tensor, - - pub fn forward(self: Layer, input: zml.Tensor) zml.Tensor { - log.info("Layer(input={f}, w={f}, b={f}) -> {f}", .{ input, self.weight, self.bias, input.dot(self.weight, .d) }); - const x = input.dot(self.weight, .d); - return x.add(self.bias.broad(x.shape())).relu().rename(.{ .d_out = .d }); - } - }; -}; - -pub fn main(init: std.process.Init) !void { - const arena = init.arena; - const allocator = init.gpa; - const io = init.io; - - // Parse program args - const process_args = try init.minimal.args.toSlice(arena.allocator()); - const model_path = process_args[1]; - const t10kfilename = process_args[2]; - - // Read model shapes. - var registry: zml.safetensors.TensorRegistry = try .fromPath(allocator, io, model_path); - defer registry.deinit(); - - // Auto-select platform - const platform: *zml.Platform = try .auto(allocator, io, .{}); - defer platform.deinit(allocator, io); - - // Decide how to partition the devices of our hardware. - // Here we use a mixed Data parallel / Model parallel - const dp_mp = try platform.registerMesh("dp_mp", try .init( - .{ .data, .model }, - &.{ .low_bandwidth, .high_bandwidth }, - &.{ 2, -1 }, - )); - log.info("{f}", .{dp_mp}); - - var profiler = try platform.profiler(allocator, io, .{ .session_id = "mnist" }); - defer profiler.deinit(); - - var store: zml.io.TensorStore = .fromRegistry(allocator, ®istry, &.{dp_mp}); - defer store.deinit(); - - const mnist_model: Mnist = .init(store.view()); - - // Compile model - const bs: u32 = 4; - const input: zml.Tensor = .withPartitioning(.init(.{ .b = bs, .x = 28, .y = 28 }, .u8), dp_mp, .{ .b = .data }); - - var exe = exe: { - log.info("Compiling model....", .{}); - const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Compiled model [{f}]", .{start.untilNow(io, .awake)}); - break :exe try platform.compile( - allocator, - io, - mnist_model, - .forward, - .{input}, - .{ - .program_name = "mnist", - .meshes = &.{dp_mp}, - .xla_dump_to = "/tmp/zml/mnist_dp", - }, - ); - }; - defer exe.deinit(); - - // Load buffers - var mnist_buffers = blk: { - log.info("Transfering weights....", .{}); - const start: std.Io.Timestamp = .now(io, .awake); - defer log.info("✅ Transferred weights [{f}]", .{ - start.untilNow(io, .awake), - }); - break :blk try mnist_model.load(init.arena.allocator(), io, platform, &store); - }; - defer zml.Buffer.deinitAll(Mnist, &mnist_buffers); - - var args = try exe.args(allocator); - defer args.deinit(allocator); - - var results = try exe.results(allocator); - defer results.deinit(allocator); - - // Load a random digit image from the dataset. - const dataset = try std.Io.Dir.openFile(.cwd(), io, t10kfilename, .{ .mode = .read_only }); - defer dataset.close(io); - - var rng: std.Random.DefaultPrng = blk: { - const now: std.Io.Timestamp = .now(io, .awake); - break :blk .init(@intCast(now.toMilliseconds())); - }; - - // inference - can be looped - var batch: []align(16) base.Img = try allocator.alignedAlloc(base.Img, .@"16", bs); - defer allocator.free(batch); - - for (0..bs) |i| { - const rand_id = rng.random().uintLessThan(u64, 10000); - _ = try dataset.readPositionalAll(io, @ptrCast(&batch[i]), 16 + (rand_id * @sizeOf(base.Img))); - } - - // This performs a host to host pinned memcpy. - // TODO: show how to allocate a Host pinned buffer and write into it. - var input_buffer: zml.Buffer = try .fromBytesOpts(io, platform, input.shape(), std.mem.sliceAsBytes(batch), .{ .memory = .host_pinned }); - defer input_buffer.deinit(); - - for (batch[0..]) |*sample| base.printDigit(sample); - - args.set(.{ mnist_buffers, input_buffer }); - - try profiler.start(); - exe.call(io, args, &results, .{}); - if (try profiler.stop()) |report| { - log.info("Wrote profiler files to {s} and {s}", .{ report.protobuf_path, report.perfetto_path }); - } - var recognized_digits_d: zml.Buffer = results.get(zml.Buffer); - defer recognized_digits_d.deinit(); - - const recognized_digits = try recognized_digits_d.getValue([bs]u8, io); - for (0..bs) |i| { - log.info( - \\✅ Shard {d} RECOGNIZED DIGIT: - \\ +-------------+ - \\{s} - \\ +-------------+ - \\ - , .{ i, base.digits[recognized_digits[i]] }); - } -} diff --git a/examples/sharding/BUILD.bazel b/examples/sharding/BUILD.bazel deleted file mode 100644 index 81f6de89..00000000 --- a/examples/sharding/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -load("@bazel_skylib//rules:build_test.bzl", "build_test") -load("@rules_zig//zig:defs.bzl", "zig_binary") - -zig_binary( - name = "sharding", - args = [], - data = [], - main = "main.zig", - deps = ["//zml"], - visibility = ["//visibility:public"], -) - -build_test( - name = "test", - targets = [":sharding"], -) diff --git a/examples/sharding/main.zig b/examples/sharding/main.zig deleted file mode 100644 index 34a78bfe..00000000 --- a/examples/sharding/main.zig +++ /dev/null @@ -1,263 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const log = std.log.scoped(.mesh); - -pub const std_options: std.Options = .{ - .log_level = .info, - .log_scope_levels = &.{ - .{ .scope = .@"zml/compiler", .level = .debug }, - }, -}; - -const CliMesh = enum { - auto, - mock, -}; - -const CliArgs = struct { - partitioner: ?zml.Sharding.Partitioner = null, - mesh: CliMesh = .auto, -}; - -const DemoModel = struct { - w: zml.Tensor, // {feature, hidden} - b: zml.Tensor, // {hidden} - - pub fn init(w: zml.Tensor, b: zml.Tensor) DemoModel { - return .{ .w = w, .b = b }; - } - - pub fn forward(self: DemoModel, input: zml.Tensor) zml.Tensor { - const compiler = zml.Compiler.current(); - - const x = input.convert(.f32); - var y = x.dot(self.w, .feature); - y = y.add(self.b.broad(y.shape())); - y = y.withPartitioning(.dp_mp, .{ .batch = .data, .hidden = .model }); - y.print("dense_out"); - - const gate = y.scale(0.01).sigmoid(); - - const spec: zml.Sharding.Partitioning.Logical = .parse(.{ .batch = .data, .hidden = .model }); - - return zml.ops.manualComputation( - (struct { - y: zml.Tensor, - gate: zml.Tensor, - - fn body(ctx: @This(), output_sharded: zml.Shape) zml.Tensor { - const local_result = ctx.y.relu().mul(ctx.gate); - return local_result.reshape(output_sharded); - } - }).body, - .{ .y = y, .gate = gate }, - y.shape(), - compiler.mesh(.dp_mp), - .{ - .in_specs = .{ .y = spec, .gate = spec }, - .out_specs = spec, - }, - ); - } -}; - -fn usage() void { - std.debug.print( - \\Usage: sharding [--partitioner=shardy|gspmd] [--mesh=auto|mock] - \\ - \\Simple sharding demo: - \\ - explicit partitioning on data/model axes - \\ - tensor print - \\ - manualComputation with multiple inputs and one output - \\ - \\Mesh modes: - \\ - auto: physical_mesh=auto, inferred cpu device_count=2 - \\ - mock: physical_mesh=mock topology, inferred cpu device_count=9 - \\ - , .{}); -} - -fn parseArgs(init: std.process.Init) !CliArgs { - var args: CliArgs = .{}; - var it = init.minimal.args.iterate(); - _ = it.next(); // program name - - while (it.next()) |arg| { - if (std.mem.eql(u8, arg, "-h") or std.mem.eql(u8, arg, "--help")) { - usage(); - std.process.exit(0); - } else if (std.mem.startsWith(u8, arg, "--partitioner=")) { - const value = arg["--partitioner=".len..]; - if (std.mem.eql(u8, value, "shardy")) { - args.partitioner = .shardy; - } else if (std.mem.eql(u8, value, "gspmd")) { - args.partitioner = .gspmd; - } else { - std.debug.print("error: unknown partitioner '{s}'\n\n", .{value}); - usage(); - return error.InvalidPartitioner; - } - } else if (std.mem.startsWith(u8, arg, "--mesh=")) { - const value = arg["--mesh=".len..]; - if (std.mem.eql(u8, value, "auto")) { - args.mesh = .auto; - } else if (std.mem.eql(u8, value, "mock")) { - args.mesh = .mock; - } else { - std.debug.print("error: unknown mesh mode '{s}'\n\n", .{value}); - usage(); - return error.InvalidMeshMode; - } - } else { - std.debug.print("error: unknown argument '{s}'\n\n", .{arg}); - usage(); - return error.InvalidArgument; - } - } - return args; -} - -fn buildMockMesh( - allocator: std.mem.Allocator, - target: zml.Target, - devices: []const zml.platform.Device, -) !zml.Sharding.PhysicalMesh { - if (devices.len < 8) return error.NotEnoughDevicesForMockMesh; - const topology: zml.Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ - .axis(.link_y, .{ .mesh = .torus }, &.{ - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[3]), .device(devices[1]), - }), - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[2]), .device(devices[0]), - }), - }), - .axis(.link_y, .{ .mesh = .torus }, &.{ - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[4]), .device(devices[5]), - }), - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[6]), .device(devices[7]), - }), - }), - }); - return zml.Sharding.PhysicalMesh.fromTree(allocator, target, topology); -} - -fn createSequenceBuffer( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const zml.Platform, - shape: zml.Shape, - start: f32, -) !zml.Buffer { - const slice = try zml.Slice.alloc(allocator, shape); - defer slice.free(allocator); - - for (slice.items(f32), 0..) |*e, i| { - e.* = start + @as(f32, @floatFromInt(i)); - } - - return zml.Buffer.fromSlice(io, platform, slice); -} - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - const io = init.io; - var args = try parseArgs(init); - - const create_options: zml.platform.CreateOptions = switch (args.mesh) { - .auto => .{}, - .mock => .{ - .cpu = .{ .device_count = 8 }, - .physical_mesh = .{ .custom = buildMockMesh }, - }, - }; - - var platform: *zml.Platform = try .auto(allocator, io, create_options); - defer platform.deinit(allocator, io); - - log.info("\n{f}", .{platform.fmtVerbose()}); - - var profiler_options: zml.Platform.ProfilerOptions = .defaults; - profiler_options.repository_path = "/tmp/xprof"; - profiler_options.session_id = "profiling"; - - var profiler = try platform.profiler(allocator, io, profiler_options); - defer profiler.deinit(); - - try profiler.start(); - defer { - _ = profiler.stop() catch unreachable; - } - - if (args.partitioner) |partitioner| { - log.info("Partitioner: {s}", .{@tagName(partitioner)}); - } else { - args.partitioner = .fromTarget(platform.target); - log.info("Partitioner: {s} (default)", .{@tagName(args.partitioner.?)}); - } - log.info("{f}", .{platform.physical_mesh}); - - const mesh: *const zml.Mesh = try platform.registerMesh( - "dp_mp", - .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), - ); - - log.info("{f}", .{mesh.logical}); - log.info("{f}", .{mesh}); - - const input_shape = zml.Shape.init(.{ .batch = 16, .feature = 32 }, .f32) - .withPartitioning(mesh, .{ .batch = .data, .feature = .replicated }); - const w_shape = zml.Shape.init(.{ .feature = 32, .hidden = 64 }, .f32) - .withPartitioning(mesh, .{ .feature = .replicated, .hidden = .model }); - const b_shape = zml.Shape.init(.{ .hidden = 64 }, .f32) - .withPartitioning(mesh, .{ .hidden = .model }); - - const input: zml.Tensor = zml.Tensor.fromShape(input_shape); - const w: zml.Tensor = zml.Tensor.fromShape(w_shape); - const b: zml.Tensor = zml.Tensor.fromShape(b_shape); - const model: DemoModel = .init(w, b); - - var exe = try platform.compile( - allocator, - io, - model, - .forward, - .{input}, - .{ - .partitioner = args.partitioner, - .meshes = &.{mesh}, - }, - ); - defer exe.deinit(); - - var w_buf = try createSequenceBuffer(allocator, io, platform, w_shape, 0.0); - defer w_buf.deinit(); - var b_buf = try createSequenceBuffer(allocator, io, platform, b_shape, 100.0); - defer b_buf.deinit(); - var input_buf = try createSequenceBuffer(allocator, io, platform, input_shape, 1000.0); - defer input_buf.deinit(); - - log.info("input placement: {f}", .{input_buf}); - log.info("weight placement: {f}", .{w_buf}); - log.info("bias placement: {f}", .{b_buf}); - - var exe_args = try exe.args(allocator); - defer exe_args.deinit(allocator); - var exe_results = try exe.results(allocator); - defer exe_results.deinit(allocator); - exe_args.set(.{ w_buf, b_buf, input_buf }); - - exe.call(io, exe_args, &exe_results, .{}); - var out = exe_results.get(zml.Buffer); - defer out.deinit(); - - const out_slice = try out.toSliceAlloc(allocator, io); - defer out_slice.free(allocator); - const out_items = out_slice.items(f32); - const preview_len = @min(out_items.len, @as(usize, 16)); - log.info("output preview: {any}", .{out_items[0..preview_len]}); -} diff --git a/examples/io/BUILD.bazel b/examples/vfs/BUILD.bazel similarity index 68% rename from examples/io/BUILD.bazel rename to examples/vfs/BUILD.bazel index d471f74d..aa3a533f 100644 --- a/examples/io/BUILD.bazel +++ b/examples/vfs/BUILD.bazel @@ -5,7 +5,11 @@ zig_binary( name = "playground", args = [], data = [], - main = "main.zig", + main = "playground.zig", deps = ["//zml"], - visibility = ["//visibility:public"], +) + +build_test( + name = "test", + targets = [":mnist"], ) diff --git a/examples/vfs/playground.zig b/examples/vfs/playground.zig new file mode 100644 index 00000000..1c5a1e47 --- /dev/null +++ b/examples/vfs/playground.zig @@ -0,0 +1,168 @@ +const std = @import("std"); + +const zml = @import("zml"); +const stdx = zml.stdx; + +const log = std.log.scoped(.vfs); + +pub const std_options: std.Options = .{ + .log_level = .info, +}; + +// -- ls hf://openai/gpt-oss-20b@6cee5e8 +// -- ls hf://Qwen/Qwen3-235B-A22B-Instruct-2507 +// -- ls hf://meta-llama/Llama-3.1-8B-Instruct@0e9e39f +// -- cat https://iprs.fly.dev +pub fn main() !void { + var debug_allocator: ?std.heap.DebugAllocator(.{}) = null; + const allocator = if (@import("builtin").mode == .Debug) blk: { + debug_allocator = .init; + break :blk debug_allocator.?.allocator(); + } else std.heap.c_allocator; + defer if (debug_allocator) |*da| std.debug.assert(da.deinit() == .ok); + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const Command = enum { cat, ls, stat, realpath }; + + var it = std.process.args(); + _ = it.next(); // skip program name + const command: Command = std.meta.stringToEnum(Command, it.next() orelse return error.MissingCommand) orelse return error.CommandInvalid; + const path = it.next() orelse return error.MissingPath; + + var http_client: std.http.Client = .{ + .allocator = allocator, + .io = threaded.io(), + .connection_pool = .{ + .free_size = threaded.async_limit.toInt() orelse 16, + }, + }; + + try http_client.initDefaultProxies(allocator); + defer http_client.deinit(); + + var vfs_file: zml.io.VFS.File = .init(allocator, threaded.io(), .{}); + defer vfs_file.deinit(); + + var vfs_https: zml.io.VFS.HTTP = try .init(allocator, threaded.io(), &http_client, .https); + defer vfs_https.deinit(); + + var hf_vfs: zml.io.VFS.HF = try .auto(allocator, threaded.io(), &http_client); + defer hf_vfs.deinit(); + + var s3_vfs: zml.io.VFS.S3 = try .fromEnv(allocator, threaded.io(), &http_client); + defer s3_vfs.deinit(); + + var vfs: zml.io.VFS = try .init(allocator, threaded.io()); + defer vfs.deinit(); + + try vfs.register("file", vfs_file.io()); + try vfs.register("https", vfs_https.io()); + try vfs.register("hf", hf_vfs.io()); + try vfs.register("s3", s3_vfs.io()); + + const io = vfs.io(); + + const buffer = try allocator.alloc(u8, 16 * 1024 * 1024); + defer allocator.free(buffer); + + var stdout_writer = std.Io.File.stdout().writer(io, buffer); + defer stdout_writer.interface.flush() catch {}; + + switch (command) { + .cat => { + var file = try std.Io.Dir.openFile(.cwd(), io, path, .{ .mode = .read_only }); + defer file.close(io); + + var reader = file.reader(io, &.{}); + const read = try reader.interface.streamRemaining(&stdout_writer.interface); + _ = try stdout_writer.interface.write("\n"); + + try stdout_writer.interface.print("Wrote {B:.2} to stdout from {s}\n", .{ read, path }); + }, + .ls => { + var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{ .iterate = true }); + defer dir.close(io); + + const dir_stat = try dir.stat(io); + try stdout_writer.interface.print("{s} - {B:.2}\n", .{ path, dir_stat.size }); + + var counts: TreeCounts = .{}; + try printTree(io, &stdout_writer.interface, dir, "", 10, &counts); + try stdout_writer.interface.print("\n{d} directories, {d} files\n", .{ counts.dirs, counts.files }); + }, + .stat => { + const stat = std.Io.Dir.statFile(.cwd(), io, path, .{}) catch |err| blk: { + if (err == error.IsDir) { + var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{}); + defer dir.close(io); + + break :blk try dir.stat(io); + } + return err; + }; + + try stdout_writer.interface.print("{s}: {B:.2} ({s})\n", .{ path, stat.size, @tagName(stat.kind) }); + }, + .realpath => { + var dir = try std.Io.Dir.openDir(.cwd(), io, path, .{}); + defer dir.close(io); + + var real_path_buf: [256]u8 = undefined; + const len = try dir.realPath(io, &real_path_buf); + + try stdout_writer.interface.print("{s}\n", .{real_path_buf[0..len]}); + }, + } +} + +const TreeCounts = struct { + dirs: usize = 0, + files: usize = 0, +}; + +fn printTree(io: std.Io, writer: *std.Io.Writer, dir: std.Io.Dir, prefix: []const u8, max_depth: usize, counts: *TreeCounts) !void { + if (max_depth == 0) return; + + var entries: stdx.BoundedArray(std.Io.Dir.Entry, 1024) = .{}; + var it = dir.iterate(); + while (try it.next(io)) |entry| { + entries.appendAssumeCapacity(entry); + } + + for (entries.constSlice(), 0..) |entry, idx| { + const is_last = (idx == entries.len - 1); + const connector = if (is_last) "└── " else "├── "; + const extension = if (is_last) " " else "│ "; + + const size: u64 = switch (entry.kind) { + .file => blk: { + const stat = try dir.statFile(io, entry.name, .{}); + break :blk stat.size; + }, + .directory => blk: { + var sub_dir = dir.openDir(io, entry.name, .{}) catch break :blk 0; + defer sub_dir.close(io); + + const stat = try sub_dir.stat(io); + break :blk stat.size; + }, + else => 0, + }; + + try writer.print("{s}{s}{s} - {B:.2}\n", .{ prefix, connector, entry.name, size }); + + if (entry.kind == .directory) { + counts.dirs += 1; + var sub_dir = dir.openDir(io, entry.name, .{ .iterate = true }) catch continue; + defer sub_dir.close(io); + + var new_prefix_buf: [4096]u8 = undefined; + const new_prefix = std.fmt.bufPrint(&new_prefix_buf, "{s}{s}", .{ prefix, extension }) catch continue; + try printTree(io, writer, sub_dir, new_prefix, max_depth - 1, counts); + } else { + counts.files += 1; + } + } +} diff --git a/ffi/BUILD.bazel b/ffi/BUILD.bazel index f53c70ed..1bf8ca7f 100644 --- a/ffi/BUILD.bazel +++ b/ffi/BUILD.bazel @@ -3,7 +3,6 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "cc", - tags = ["manual"], hdrs = [ "zig_allocator.h", "zig_slice.h", @@ -13,7 +12,6 @@ cc_library( zig_library( name = "zig", - tags = ["manual"], srcs = [ "zig_allocator.zig", "zig_slice.zig", diff --git a/ffi/zig_allocator.zig b/ffi/zig_allocator.zig index 52c000bd..35f5fc33 100644 --- a/ffi/zig_allocator.zig +++ b/ffi/zig_allocator.zig @@ -1,5 +1,4 @@ const std = @import("std"); - const c = @import("c"); pub const ZigAllocator = struct { @@ -13,7 +12,7 @@ pub const ZigAllocator = struct { pub fn alloc(ctx: ?*const anyopaque, elem: usize, nelems: usize, alignment: usize) callconv(.c) ?*anyopaque { const self: *const std.mem.Allocator = @ptrCast(@alignCast(ctx)); - const ret = self.rawAlloc(elem * nelems, .fromByteUnits(alignment), @returnAddress()) orelse return null; + const ret = self.rawAlloc(elem * nelems, std.math.log2_int(usize, alignment), @returnAddress()) orelse return null; return @ptrCast(ret); } @@ -21,6 +20,6 @@ pub const ZigAllocator = struct { const self: *const std.mem.Allocator = @ptrCast(@alignCast(ctx)); const memory: [*c]u8 = @ptrCast(ptr); const size = elem * nelems; - self.rawFree(memory[0..size], .fromByteUnits(alignment), @returnAddress()); + self.rawFree(memory[0..size], std.math.log2_int(usize, alignment), @returnAddress()); } }; diff --git a/mlir/.clang-format b/mlir/.clang-format deleted file mode 100644 index 9b3aa8b7..00000000 --- a/mlir/.clang-format +++ /dev/null @@ -1 +0,0 @@ -BasedOnStyle: LLVM diff --git a/mlir/BUILD.bazel b/mlir/BUILD.bazel index 8333c579..87982a7e 100644 --- a/mlir/BUILD.bazel +++ b/mlir/BUILD.bazel @@ -4,9 +4,12 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") cc_library( name = "c", hdrs = ["mlir.h"], - tags = ["manual"], + visibility = ["//mlir:__subpackages__"], deps = [ + "@llvm-project//mlir:CAPIArith", "@llvm-project//mlir:CAPIIR", + "@llvm-project//mlir:CAPIMath", + "@llvm-project//mlir:CAPISCF", "@llvm-project//mlir:CAPITransforms", ], ) @@ -14,7 +17,6 @@ cc_library( zig_library( name = "mlir", main = "mlir.zig", - tags = ["manual"], visibility = ["//visibility:public"], deps = [ ":c", diff --git a/mlir/dialects/cf/BUILD.bazel b/mlir/dialects/BUILD.bazel similarity index 53% rename from mlir/dialects/cf/BUILD.bazel rename to mlir/dialects/BUILD.bazel index 3d308f63..26cca662 100644 --- a/mlir/dialects/cf/BUILD.bazel +++ b/mlir/dialects/BUILD.bazel @@ -1,23 +1,23 @@ load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") zig_library( - name = "cf", - import_name = "mlir/dialects/cf", - main = "cf.zig", - tags = ["manual"], + name = "dialects", + srcs = [ + "func.zig", + ], + import_name = "mlir/dialects", + main = "dialects.zig", visibility = ["//visibility:public"], deps = [ "//mlir", + "//mlir/dialects/stablehlo", "//stdx", - "@llvm-project//mlir:CAPICF", ], ) zig_test( name = "test", - main = "test.zig", deps = [ - ":cf", - "//mlir", + ":dialects", ], ) diff --git a/mlir/dialects/README.md b/mlir/dialects/README.md deleted file mode 100644 index 7df47c5c..00000000 --- a/mlir/dialects/README.md +++ /dev/null @@ -1,269 +0,0 @@ -# MLIR dialects - -One Bazel package per MLIR dialect that ZML emits. The rules come in three -tiers: - -- **Guarantees** hold for every package. Tests and the conformance checks - below enforce them, and callers rely on them. -- **Conventions** are the defaults. Follow them in new code; existing code - that differs is fine unless it confuses callers. -- **Exceptions** list the known, justified departures from the conventions. - -References: `cuda/`, `nvvm/` and `cute_nvgpu/` (generated from ODS, with -their own TableGen and C API), `gpu/`, `arith/` and `func/` (upstream -dialects, with a small C API for what upstream lacks). - -## Guarantees - -- **Registration.** The package exposes `registerDialect(registry)`, which - calls `registry.registerDialect("")` with a string literal, where - `` is the second argument of - `MLIR_DECLARE_CAPI_DIALECT_REGISTRATION`. Tests and consumers register - through `.registerDialect`, never through raw strings or - `DialectHandle.fromString`. A DSL exposes `registerDialects(registry)`, - which calls the `registerDialect` of each package it emits. -- **Operation list.** A package that builds operations has, in its - `test.zig`, a sorted `const operation_names: []const []const u8` naming - every operation its builders create. `test "every bound operation is - registered"` checks each name against the registry, and checks that - unregistered dialects are not allowed. A builder for an operation the pinned dialect does not register - does not exist; a DSL method builds it from the operations the Python DSL - lowers it to instead. -- **Bindings, not text.** A dialect package builds attributes and types - through C API constructors (`get`/`getChecked`), never by printing and - parsing text. A type or attribute the C API lacks gets a binding (a C - constructor and a Zig wrapper) before anything uses it. `mlir.Type.parse` - and `mlir.Attribute.parse` are for tests. Consumers use the bindings too. -- **Everything compiles.** `test.zig` runs `std.testing.refAllDecls` on the - package and on each of its wrapper types, so no declaration hides behind - lazy analysis. -- **Argument order.** A builder takes `ctx`, the required operands, the - result types (omitted when inferred), the required attributes, then the - optional arguments (an `opts` struct), then `location`, which is always - last. -- **Verification.** A builder verifies the operation it creates. One that - cannot (its region is filled later, or it only verifies inside a parent) - passes `.verify = false` and says so in its doc comment. -- **C API safety.** Every declaration is `MLIR_CAPI_EXPORTED`. Constructors - load the dialect, return a null handle for a null context, and verifying - constructors (`getChecked`) return a null handle for invalid input. No - `alwayslink` and no static initializers: the Zig side calls the - registration function by name, so the linker keeps what it uses. -- **Hygiene.** LLVM file banners and include-path guards (below), clean under - `mlir/.clang-format` and the Bazel toolchain's `zig fmt` (0.17), tests only - in `test.zig`, and the BUILD essentials: `capi` and the `zig_library` - tagged `manual`, an explicit `srcs` list in `td_files`, no `glob(`. - -## Conventions - -### Package layout - -- `mlir/dialects//` is imported as `@import("mlir/dialects/")`, - main file `.zig`. Generated builders go in `ops.zig`, exposed as - `.ops`, with hand-written helpers in `.zig`. -- The C API is `capi.h` / `capi.cc`. Dialects written here use - `{Dialect,Ops,Types,Attributes,Enums}.{td,h,cpp}`; vendored - sources keep their upstream file names. -- A generated file starts with a header naming its source and saying it is - generated. The package that generates it has a `REGENERATING.md` of agent - instructions (sources, generated vs maintained files, steps, checks) that - points here for conventions. Generator scripts are not committed. - -### BUILD.bazel - -Targets in this order: `capi` (`cc_library`, deps `:dialect`, `:headers`, -`@llvm-project//mlir:CAPIIR`), the `zig_library` (public, deps `:capi`, -`//mlir`), `test` (`zig_test`, `main = "test.zig"`, plus `:capi` when -`test.zig` imports `c`), then `td_files` (`includes = ["/"]`, direct td -deps such as `@llvm-project//mlir:OpBaseTdFiles`), the `gentbl_cc_library` -targets the dialect needs (`dialect_inc_gen`, `enums_inc_gen`, -`attrs_inc_gen`, `types_inc_gen`, `ops_inc_gen`), `headers` and `dialect`. -Every dep a source includes or imports is listed directly. Format with -`./tools/buildifier.sh`. - -### C and C++ - -- The first line is an 80-column LLVM banner: - `//===- capi.h - C API for NVIDIA's cuda dialect -------------------*- C -*-===//`. - The tag is `*- C -*-` for C headers, `*- C++ -*-` for C++ headers and - `*- tablegen -*-` for `.td`; `.cc`/`.cpp`/`.inc` banners end in `---===//`. - An optional `//` description follows, then a blank line. -- The guard is the include path in upper case with `/` and `.` as `_`, no - `ZML_` prefix (`MLIR_DIALECTS_CUDA_CUDADIALECT_H`, `..._TD`), closed with - `#endif // `. -- Includes follow LLVM order: own header, package headers, `mlir/` and - `llvm/`, then system headers. `capi.h` includes `mlir-c/IR.h` (which - provides `stdbool` and `stdint`) and the upstream `mlir-c/Dialect/.h` it - extends, inside `extern "C"` under `#ifdef __cplusplus`. -- Hand-written C APIs use fixed-width integers (`uint32_t`, `int64_t`), - parameters `MlirContext context`, `MlirAttribute attr`, `MlirType type`, - and C++ locals `MLIRContext *ctx`. `capi.cc` uses `static` helpers, not - anonymous namespaces, and one static helper loads the dialect - (`loadGPU` in `gpu/capi.cc`). - -### Names and where they come from - -`` is the dialect's C prefix, the first argument of -`MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(, )`: `Cuda`, `CuteNVGPU`, -`Triton`, `ROCDL`. Everything else is named as ODS names it: - -| ODS | C API | Zig | -|---|---|---| -| `AttrDef` (class `NameAttr`) | `mlirAttributeIsAName`, `mlirNameAttrGet` | `NameAttr`, `NameAttr.get(ctx, .{ ... })` | -| `TypeDef` (class `NameType`) | `mlirTypeIsAName`, `mlirNameTypeGet` | `NameType` | -| parameter `$elementType` | `mlirNameAttrGetElementType` | field `.elementType`, accessor `elementType()` | -| list parameter `$archs` | `...GetNumArchs`, `...GetArchs(attr, pos)` | `numArchs()`, `arch(i)` | -| `EnumAttr` / `I32EnumAttr` | `...AttrGet(context, value)`, `...AttrGetValue` | Zig enum, `XAttr.get(ctx, .{ .value = ... })`, `value()` | -| op mnemonic `fast_sin.f32` | generic operation API | builder `fast_sin_f32` | - -- Predicates have no `Attr`/`Type` suffix (`mlirTypeIsACudaEvent`). -- C getters keep MLIR's `Get`. Zig accessors are nouns, as in the rest - of ZML (`value()`, `elementType()`, `numAxes()`, `axis(i)`), never - `get`-prefixed. Predicates keep `is`/`has`. -- Enum fields use the IR spelling (`@"and"`, `@"false"`). -- A builder is named after its mnemonic with dots turned into underscores, - case kept. Its parameters are the ODS operand and attribute names, and its - result types `_type` (`_types` when variadic). -- A name that is a Zig keyword, or that collides with a member, a declaration - of the file or a builder local, takes a trailing underscore (`for_`, - `return_`, `ptr_()`, `format_()`). It is quoted only when that name is taken - too (`TupleType.@"type"(i)`, next to the `type_()` upcast). -- Alternative constructors are `mlirNameAttr
Get` in C, as MLIR's - `mlirDenseElementsAttrSplatGet`, and `get(ctx, ...)` in Zig. - -### Zig API - -```zig -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("cuda"); -} - -/// `#cuda.executable`: Format of a compiled executable -pub const ExecutableAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACudaExecutable; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const InitArgs = struct { format: ExecutableFormat }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { ... } - pub fn attribute(self: *const Self) *const mlir.Attribute { ... } - pub fn format_(self: *const Self) ExecutableFormat { ... } -}; -``` - -- Wrappers use `mlir.Methods` with `c.MlirAttribute` or `c.MlirType`, upcast - with `attribute()` or `type_()`, and build with `get(ctx, InitArgs)`, a - struct even for one field. `get` returns `mlir.Error!*const X` when the C - getter can return null, a plain pointer otherwise. -- Free helpers are `Attr(ctx, ...)` and `Type(ctx, ...)`. -- Optional builder arguments go in `opts`: generated builders declare it - inline (`opts: struct { ... }`), hand-written ones may name it `Opts`. - Its fields are the optional operands, result types and optional, defaulted - or unit attributes, each defaulting to `null` (or the ODS default), so - callers pass `.{}`. -- Sorted imports (`std` first), no unused declarations. - -### Comments - -- Porting adds no explanatory comments; doc comments on public API stay - short. -- A generated wrapper's doc comment is the IR spelling, a colon and the ODS - summary (``/// `#cuda.executable`: Format of a compiled executable``); a - generated builder's is the IR name, a period, then the summary. - -### Tests - -Beyond the guaranteed `refAllDecls` and registration tests: a -`testContext()` that registers the package and its dependencies and calls -`mlir.Context.init(.{ .registry = registry, .threading = false })` and -`loadAllAvailableDialects()`; attributes and types round-tripped (build with -`get`, print with `{f}`, reparse, compare with `eql`); operations built and -verified. - -## Exceptions - -- Generated `.td` files start with a "generated" header instead of a banner. -- Generated C APIs keep ODS's integer types (`int`) and repeat the null check - and dialect load in every constructor. -- `cute_ir/` and `cute_nvgpu/` keep their generated builders in `.zig`; - `llvm/` keeps its builders in a private `ops.zig`. -- `cuda/`, `cute_nvgpu/` and `nvvm/` are regenerated through - `cute_ir/REGENERATING.md`, `fly_rocdl/` through `fly/REGENERATING.md`. -- `cute_ir:capi` and `cute_nvgpu:capi` each depend on the other's `:dialect`: - their C++ sources use each other's types. -- `cf`, `math`, `memref`, `scf`, `shardy` and `stablehlo` have no `capi` of - their own and bind upstream's C API target directly; `llvm`'s `capi` only - adds what upstream's leaves out (loop annotations). -- Hand-written builders with many parameters take an `Args` struct - (`func.func`, `gpu.func`, `ttir.func`, `cuda.kernel`, `cuda_tile.entry`, - `air.kernel`, `memref.subview`, `vector.transfer_read`, ...); - function-like ones carry `location` inside it. -- Some hand-written `Opts` hold required or non-null defaulted fields where - named arguments read better (`stablehlo.PadOpts`, - `mosaic_tpu.LoadOpts.sublane_mask`, `affine.ForOpts.step`). -- stablehlo, shardy, cuda_tile and ttir getters return plain pointers (their - C calls cannot fail); `ttir.pointerType` and `ttir.tensorDescType` take no - `ctx` (the C API derives it from the element type). -- Enum cases whose IR spelling is not an identifier use the C++ symbol - (rocdl's `volatile_op`), as do the generated cute_nvgpu and ttir enums. -- fly's attributes and cute_ir's algebra attributes have named constructors - (`getStatic`, `getDynamic`, `getTuple`, ...); fly's generated builders use - snake_case parameters. -- tcl mirrors the Furiosa plugin's file names and C++ style. One - `graph(op)` builder covers the 17 `tcl.graph.*` operations that share a - shape, and `loop`, `yield` and `copy` build `tcl.graph.for`, `yield` and - `copy`. Its `Tactic` and `ReduceMode` enums map to string attribute values, - its C API covers what ZML emits, and its operations are verified through - `dsl/tcl`'s tests. - -## Porting checklist - -1. Choose ``, the C prefix and the handle; create the package with the - BUILD targets above. -2. Vendor or write the `.td`/C++ sources and list every `.td` in `td_files`. -3. Write `capi.h`/`capi.cc` for registration and whatever upstream's C API - lacks: per attribute and type, a predicate, a `Get` and the parameter - getters. -4. Bind in `.zig`, or generate `ops.zig`: `registerDialect`, enums, - wrappers, helpers, builders. -5. Write `test.zig`, with the `operation_names` list. -6. If anything is generated, add a `REGENERATING.md`. -7. Make consumers depend on `//mlir/dialects/` and call - `.registerDialect(registry)`. -8. Run the verification below. - -## Verification - -Run from the repository root, with `ZIG` set to the Bazel toolchain's `zig` -(`AGENTS.md` shows how to find its standard library; `zig` is next to it). - -```bash -#!/usr/bin/env bash -d=mlir/dialects/ -bazel test "//$d:test" && bazel test //mlir/... && bazel build //... -./tools/buildifier.sh -mode=check -r "$PWD/$d" -"$ZIG" fmt --check "$d" -src=(); while IFS= read -r f; do src+=("$f"); done < <(find "$d" -maxdepth 1 \( -name '*.h' -o -name '*.cc' -o -name '*.cpp' -o -name '*.td' -o -name '*.inc' \)) -cc=(); for f in "${src[@]}"; do [[ $f == *.td || $f == *.inc ]] || cc+=("$f"); done -[ ${#cc[@]} -eq 0 ] || clang-format --dry-run --Werror --style=file:mlir/.clang-format "${cc[@]}" - -# Conformance: prints nothing on a conforming package. -for f in "${src[@]}"; do l=$(head -1 "$f"); [ ${#l} -eq 80 ] || grep -q generated <<<"$l" || echo "banner: $f"; done -grep -rln --include='*.h' --include='*.td' 'ZML_' "$d" -grep -nE 'alwayslink|glob\(' "$d/BUILD.bazel" -[ -f "$d/capi.cc" ] && grep -nE 'extern "C"|^namespace \{' "$d/capi.cc" -[ -f "$d/capi.h" ] && grep -nw unsigned "$d/capi.h" -grep -rnE --include='*.zig' 'pub fn get[A-Z][A-Za-z]*\(self' "$d" -grep -rnE --include='*.zig' --exclude=test.zig '(Type|Attribute)\.parse\(' "$d" -grep -rln --include='*.zig' --exclude=test.zig '^test ' "$d" -grep -rqF --include='*.zig' 'pub fn registerDialect' "$d" || echo "missing: registerDialect" -if grep -rqF --include='*.zig' '.make(' "$d"; then - grep -qF 'const operation_names' "$d/test.zig" || echo "missing: operation_names" - grep -qF 'test "every bound operation is registered"' "$d/test.zig" || echo "missing: registration test" -fi -git grep -n 'DialectHandle.fromString' -- dsl zml examples -exit 0 -``` diff --git a/mlir/dialects/affine/BUILD.bazel b/mlir/dialects/affine/BUILD.bazel deleted file mode 100644 index 06f31849..00000000 --- a/mlir/dialects/affine/BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - "@llvm-project//mlir:AffineDialect", - "@llvm-project//mlir:CAPIIR", - ], -) - -zig_library( - name = "affine", - import_name = "mlir/dialects/affine", - main = "affine.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":affine", - "//mlir", - ], -) diff --git a/mlir/dialects/affine/affine.zig b/mlir/dialects/affine/affine.zig deleted file mode 100644 index c808619c..00000000 --- a/mlir/dialects/affine/affine.zig +++ /dev/null @@ -1,413 +0,0 @@ -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("affine"); -} - -// ============================================================================= -// affine.apply / affine.min / affine.max -// ============================================================================= - -/// affine.apply — apply a 1-result affine map to a list of dim+symbol -/// operands. Result is `index`. -pub fn apply( - ctx: *mlir.Context, - map: *const mlir.AffineMap, - map_operands: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "affine.apply", .{ - .operands = .{ .flat = map_operands }, - .results = .{ .flat = &.{.index(ctx)} }, - .attributes = &.{ - .named(ctx, "map", .affineMap(map)), - }, - .location = location, - }); -} - -/// affine.min — minimum of all results of `map` evaluated at `map_operands`. -pub fn min( - ctx: *mlir.Context, - map: *const mlir.AffineMap, - map_operands: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "affine.min", .{ - .operands = .{ .flat = map_operands }, - .results = .{ .flat = &.{.index(ctx)} }, - .attributes = &.{ - .named(ctx, "map", .affineMap(map)), - }, - .location = location, - }); -} - -/// affine.max — maximum of all results of `map`. -pub fn max( - ctx: *mlir.Context, - map: *const mlir.AffineMap, - map_operands: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "affine.max", .{ - .operands = .{ .flat = map_operands }, - .results = .{ .flat = &.{.index(ctx)} }, - .attributes = &.{ - .named(ctx, "map", .affineMap(map)), - }, - .location = location, - }); -} - -// ============================================================================= -// affine.for / affine.yield -// ============================================================================= - -pub const ForOpts = struct { - /// Constant lower bound. Pass `null` when supplying a `lower_bound_map`. - lower_bound: ?i64 = null, - /// Constant upper bound. Pass `null` when supplying an `upper_bound_map`. - upper_bound: ?i64 = null, - /// Optional map-based bounds. When non-null they take precedence over the - /// constant counterpart. - lower_bound_map: ?*const mlir.AffineMap = null, - upper_bound_map: ?*const mlir.AffineMap = null, - /// Operands feeding `lower_bound_map` (in dim+symbol order). - lower_bound_operands: []const *const mlir.Value = &.{}, - /// Operands feeding `upper_bound_map`. - upper_bound_operands: []const *const mlir.Value = &.{}, - /// Loop step (positive, defaults to 1). - step: i64 = 1, - /// Loop-carried initial values. - inits: []const *const mlir.Value = &.{}, -}; - -/// affine.for — `for %i = lower to upper step S iter_args(...)`. The body -/// block takes `index, *init_arg_types` and must terminate with `affine.yield`. -pub fn for_( - ctx: *mlir.Context, - body: *mlir.Block, - opts: ForOpts, - location: *const mlir.Location, -) *mlir.Operation { - // Resolve final maps. - const lb_map = opts.lower_bound_map orelse blk: { - const cst = mlir.AffineExpr.constant(ctx, opts.lower_bound orelse 0); - break :blk mlir.AffineMap.get(ctx, 0, 0, &.{cst}); - }; - const ub_map = opts.upper_bound_map orelse blk: { - const cst = mlir.AffineExpr.constant(ctx, opts.upper_bound orelse 0); - break :blk mlir.AffineMap.get(ctx, 0, 0, &.{cst}); - }; - - var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; - operands.appendSliceAssumeCapacity(opts.lower_bound_operands); - operands.appendSliceAssumeCapacity(opts.upper_bound_operands); - operands.appendSliceAssumeCapacity(opts.inits); - - const seg_sizes = [3]i32{ - @intCast(opts.lower_bound_operands.len), - @intCast(opts.upper_bound_operands.len), - @intCast(opts.inits.len), - }; - - var result_types: stdx.BoundedArray(*const mlir.Type, 64) = .empty; - for (opts.inits) |v| result_types.appendAssumeCapacity(v.type_()); - - return mlir.Operation.make(ctx, "affine.for", .{ - .operands = .{ .flat = operands.constSlice() }, - .results = .{ .flat = result_types.constSlice() }, - .blocks = &.{body}, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "lowerBoundMap", .affineMap(lb_map)), - .named(ctx, "upperBoundMap", .affineMap(ub_map)), - .named(ctx, "step", .int(ctx, .i64, opts.step)), - }, - // The for body may reference outer SSA values that only become - // reachable once it's appended to its parent — defer to module-level - // verify (same trick used by scf.for). - .verify = false, - .location = location, - }); -} - -/// affine.yield — terminator for affine.for / affine.if / affine.parallel. -pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "affine.yield", .{ - .operands = .{ .flat = values }, - .verify = false, - .location = location, - }); -} - -// ============================================================================= -// affine.if -// ============================================================================= - -/// affine.if — `if condition[ ... ] { then } else { else }`. `condition` is an -/// IntegerSetAttribute (a conjunction of affine constraints). The result types -/// are inferred from the yielded values; pass `&.{}` for the no-result form. -pub fn if_( - ctx: *mlir.Context, - condition: *const mlir.IntegerSet, - operands: []const *const mlir.Value, - result_types: []const *const mlir.Type, - then_block: *mlir.Block, - opts: struct { - else_block: ?*mlir.Block = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var state: mlir.OperationState = .init("affine.if", location); - state.addOperands(operands); - state.addResults(result_types); - state.addAttributes(&.{ - .named(ctx, "condition", .integerSet(condition)), - }); - - const then_region = mlir.Region.init(); - then_region.appendOwnedBlock(then_block); - const else_region = mlir.Region.init(); - if (opts.else_block) |b| else_region.appendOwnedBlock(b); - state.addOwnedRegions(&.{ then_region, else_region }); - - return mlir.Operation.init(&state) catch @panic("Failed to create affine.if"); -} - -// ============================================================================= -// affine.load / affine.store / affine.vector_load / affine.vector_store -// ============================================================================= - -/// affine.load — load with affine indexing. -pub fn load( - ctx: *mlir.Context, - memref: *const mlir.Value, - map: *const mlir.AffineMap, - map_operands: []const *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - operands_buf.appendAssumeCapacity(memref); - operands_buf.appendSliceAssumeCapacity(map_operands); - - return mlir.Operation.make(ctx, "affine.load", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "map", .affineMap(map)), - }, - .location = location, - }); -} - -pub fn store( - ctx: *mlir.Context, - value: *const mlir.Value, - memref: *const mlir.Value, - map: *const mlir.AffineMap, - map_operands: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ value, memref }); - operands_buf.appendSliceAssumeCapacity(map_operands); - - return mlir.Operation.make(ctx, "affine.store", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .attributes = &.{ - .named(ctx, "map", .affineMap(map)), - }, - .location = location, - }); -} - -pub fn vector_load( - ctx: *mlir.Context, - memref: *const mlir.Value, - map: *const mlir.AffineMap, - map_operands: []const *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - operands_buf.appendAssumeCapacity(memref); - operands_buf.appendSliceAssumeCapacity(map_operands); - - return mlir.Operation.make(ctx, "affine.vector_load", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "map", .affineMap(map)), - }, - .location = location, - }); -} - -pub fn vector_store( - ctx: *mlir.Context, - value: *const mlir.Value, - memref: *const mlir.Value, - map: *const mlir.AffineMap, - map_operands: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ value, memref }); - operands_buf.appendSliceAssumeCapacity(map_operands); - - return mlir.Operation.make(ctx, "affine.vector_store", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .attributes = &.{ - .named(ctx, "map", .affineMap(map)), - }, - .location = location, - }); -} - -// ============================================================================= -// affine.parallel -// ============================================================================= - -pub const ParallelArgs = struct { - /// Per-loop-dim reductions, one entry per result. Use the same kinds as - /// `AtomicRMWKind` in `mlir/dialects/memref` (the dialect uses - /// `arith::AtomicRMWKind`). Pass `&.{}` for parallel loops with no - /// loop-carried reductions. - reductions: []const i64 = &.{}, - /// Map producing all lower-bound values; a single dim's bound is the - /// max of the results in its `lower_bounds_groups[i]` group. - lower_bounds_map: *const mlir.AffineMap, - /// One i32 per loop dim — number of map results that contribute to the - /// lower bound of that dim. Sum must equal `lower_bounds_map.numResults()`. - lower_bounds_groups: []const i32, - upper_bounds_map: *const mlir.AffineMap, - upper_bounds_groups: []const i32, - /// Per-dim integer steps (default 1). - steps: []const i64, - /// Operands feeding both bound maps. - map_operands: []const *const mlir.Value = &.{}, -}; - -/// affine.parallel — multi-dim parallel band. `body_block` takes one `index` -/// argument per loop dim and must terminate with `affine.yield`. -pub fn parallel( - ctx: *mlir.Context, - args: ParallelArgs, - body_block: *mlir.Block, - result_types: []const *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - // I32ElementsAttr — dense<[...]> : tensor. - const lb_groups_ty = mlir.RankedTensorType.get( - &.{@intCast(args.lower_bounds_groups.len)}, - .int(ctx, .i32), - null, - ).shaped(); - const ub_groups_ty = mlir.RankedTensorType.get( - &.{@intCast(args.upper_bounds_groups.len)}, - .int(ctx, .i32), - null, - ).shaped(); - - // I64SmallVectorArrayAttr — built as ArrayAttr>. - var steps_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; - for (args.steps) |s| { - steps_attrs.appendAssumeCapacity(.int(ctx, .i64, s)); - } - - // Reductions are AtomicRMWKindAttr (I64EnumAttr). We let callers pass - // raw int values to avoid coupling against the enum table. - var reductions_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; - for (args.reductions) |k| { - reductions_attrs.appendAssumeCapacity(.int(ctx, .i64, k)); - } - - return mlir.Operation.make(ctx, "affine.parallel", .{ - .operands = .{ .flat = args.map_operands }, - .results = .{ .flat = result_types }, - .blocks = &.{body_block}, - .attributes = &.{ - .named(ctx, "reductions", .array(ctx, reductions_attrs.constSlice())), - .named(ctx, "lowerBoundsMap", .affineMap(args.lower_bounds_map)), - .named(ctx, "lowerBoundsGroups", .denseElements(lb_groups_ty, args.lower_bounds_groups)), - .named(ctx, "upperBoundsMap", .affineMap(args.upper_bounds_map)), - .named(ctx, "upperBoundsGroups", .denseElements(ub_groups_ty, args.upper_bounds_groups)), - .named(ctx, "steps", .array(ctx, steps_attrs.constSlice())), - }, - .verify = false, - .location = location, - }); -} - -// ============================================================================= -// affine.delinearize_index / affine.linearize_index -// ============================================================================= - -pub const DelinearizeArgs = struct { - static_basis: []const i64, - dynamic_basis: []const *const mlir.Value = &.{}, -}; - -pub fn delinearize_index( - ctx: *mlir.Context, - linear_index: *const mlir.Value, - args: DelinearizeArgs, - result_types: []const *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - operands_buf.appendAssumeCapacity(linear_index); - operands_buf.appendSliceAssumeCapacity(args.dynamic_basis); - - return mlir.Operation.make(ctx, "affine.delinearize_index", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = result_types }, - .attributes = &.{ - .named(ctx, "static_basis", .denseArray(ctx, .i64, args.static_basis)), - }, - .location = location, - }); -} - -pub const LinearizeArgs = struct { - multi_index: []const *const mlir.Value, - static_basis: []const i64, - dynamic_basis: []const *const mlir.Value = &.{}, - disjoint: bool = false, -}; - -pub fn linearize_index( - ctx: *mlir.Context, - args: LinearizeArgs, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; - operands_buf.appendSliceAssumeCapacity(args.multi_index); - operands_buf.appendSliceAssumeCapacity(args.dynamic_basis); - - const seg_sizes = [2]i32{ - @intCast(args.multi_index.len), - @intCast(args.dynamic_basis.len), - }; - - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "static_basis", .denseArray(ctx, .i64, args.static_basis)), - }); - if (args.disjoint) { - attrs.appendAssumeCapacity(.named(ctx, "disjoint", .unit(ctx))); - } - - return mlir.Operation.make(ctx, "affine.linearize_index", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} diff --git a/mlir/dialects/affine/capi.cc b/mlir/dialects/affine/capi.cc deleted file mode 100644 index 2110365f..00000000 --- a/mlir/dialects/affine/capi.cc +++ /dev/null @@ -1,9 +0,0 @@ -//===- capi.cc - C API for the affine dialect -----------------------------===// - -#include "capi.h" - -#include "mlir/CAPI/Registration.h" -#include "mlir/Dialect/Affine/IR/AffineOps.h" - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Affine, affine, - mlir::affine::AffineDialect) diff --git a/mlir/dialects/affine/capi.h b/mlir/dialects/affine/capi.h deleted file mode 100644 index 5ae66467..00000000 --- a/mlir/dialects/affine/capi.h +++ /dev/null @@ -1,18 +0,0 @@ -//===- capi.h - C API for the affine dialect ----------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_AFFINE_CAPI_H -#define MLIR_DIALECTS_AFFINE_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Affine, affine); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_AFFINE_CAPI_H diff --git a/mlir/dialects/affine/test.zig b/mlir/dialects/affine/test.zig deleted file mode 100644 index 4248d4c3..00000000 --- a/mlir/dialects/affine/test.zig +++ /dev/null @@ -1,40 +0,0 @@ -const std = @import("std"); - -const affine = @import("mlir/dialects/affine"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "affine.apply", - "affine.delinearize_index", - "affine.for", - "affine.if", - "affine.linearize_index", - "affine.load", - "affine.max", - "affine.min", - "affine.parallel", - "affine.store", - "affine.vector_load", - "affine.vector_store", - "affine.yield", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(affine); - inline for (comptime std.meta.declarations(affine)) |decl| { - const T = @field(affine, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - affine.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/air/AIRAttributes.cpp b/mlir/dialects/air/AIRAttributes.cpp deleted file mode 100644 index 287cb26b..00000000 --- a/mlir/dialects/air/AIRAttributes.cpp +++ /dev/null @@ -1,140 +0,0 @@ -//===- AIRAttributes.cpp - Metal AIR attributes ---------------------------===// - -#include "mlir/dialects/air/AIRAttributes.h" - -#include "mlir/dialects/air/AIRDialect.h" - -#include "mlir/Dialect/LLVMIR/LLVMTypes.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/DialectImplementation.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/MathExtras.h" - -#include - -using namespace mlir; -using namespace mlir::metal::air; - -#include "mlir/dialects/air/AIREnums.cpp.inc" -#define GET_ATTRDEF_CLASSES -#include "mlir/dialects/air/AIRAttributes.cpp.inc" - -void AIRDialect::registerAttributes() { - addAttributes< -#define GET_ATTRDEF_LIST -#include "mlir/dialects/air/AIRAttributes.cpp.inc" - >(); -} - -static bool isValidName(StringRef name) { - return !name.empty() && !name.contains('\0'); -} - -// The size of a Metal scalar or vector, where 3-element vectors take the -// space of 4. A bool vector is an i8 vector in memory, as LLVM packs i1 -// vectors into bits. -static std::optional metalSize(Type type) { - uint64_t elements = 1; - if (auto vector = dyn_cast(type)) { - if (vector.isScalable() || vector.getRank() != 1 || - vector.getElementType().isInteger(1)) - return std::nullopt; - elements = vector.getNumElements(); - if (elements < 2 || elements > 4) - return std::nullopt; - if (elements == 3) - elements = 4; - type = vector.getElementType(); - } - // bool, char, short, int, long, half, bfloat and float. - if (!type.isInteger(1) && !type.isInteger(8) && !type.isInteger(16) && - !type.isInteger(32) && !type.isInteger(64) && !type.isF16() && - !type.isBF16() && !type.isF32()) - return std::nullopt; - return elements * llvm::divideCeil(type.getIntOrFloatBitWidth(), 8); -} - -LogicalResult -StructFieldAttr::verify(llvm::function_ref emitError, - uint64_t offset, uint64_t size, uint64_t count, - StringRef typeName, StringRef name) { - if (!size) - return emitError() << "expected a positive struct field size"; - if (!isValidName(typeName) || !isValidName(name)) - return emitError() << "expected nonempty field and type names"; - return success(); -} - -static LogicalResult -verifyStructFields(llvm::function_ref emitError, - LLVM::LLVMStructType type, uint64_t typeSize, - ArrayRef fields) { - if (type.isOpaque() || type.isPacked()) - return emitError() << "expected a sized, unpacked struct"; - ArrayRef body = type.getBody(); - if (body.size() != fields.size()) - return emitError() << "expected one field per struct element"; - uint64_t end = 0; - for (auto [member, field] : llvm::zip_equal(body, fields)) { - Type element = member; - uint64_t count = 0; - if (auto array = dyn_cast(element)) { - count = array.getNumElements(); - element = array.getElementType(); - } - std::optional size = metalSize(element); - if (!size || *size != field.getSize() || count != field.getCount()) - return emitError() << "field " << field.getName() - << " does not match the struct element " << element; - if (field.getOffset() < end || field.getOffset() % *size != 0) - return emitError() << "field " << field.getName() - << " is misplaced at offset " << field.getOffset(); - end = field.getOffset() + *size * std::max(count, 1); - } - if (end > typeSize) - return emitError() << "struct fields overflow the buffer type size"; - return success(); -} - -LogicalResult BufferAttr::verify( - llvm::function_ref emitError, unsigned locationIndex, - AddressSpace addressSpace, Access access, Type elementType, - StringRef typeName, uint64_t typeSize, uint64_t typeAlignment, - uint64_t bufferSize, bool noalias, ArrayRef fields) { - if (addressSpace == AddressSpace::thread) - return emitError() << "buffers are not in thread memory"; - if (addressSpace == AddressSpace::constant && access != Access::read) - return emitError() << "constant buffers are read-only"; - if (addressSpace == AddressSpace::threadgroup && access != Access::read_write) - return emitError() << "threadgroup buffers are read_write"; - if (!elementType) - return emitError() << "expected a buffer element type"; - if (!isValidName(typeName)) - return emitError() << "expected a nonempty Metal type name"; - if (!typeSize || !llvm::isPowerOf2_64(typeAlignment) || - typeSize % typeAlignment != 0) - return emitError() << "expected a positive ABI size divisible by a " - "power-of-two alignment"; - if (auto structType = dyn_cast(elementType)) { - if (addressSpace != AddressSpace::constant) - return emitError() << "struct buffers must be constant"; - if (failed(verifyStructFields(emitError, structType, typeSize, fields))) - return failure(); - } else { - std::optional size = metalSize(elementType); - if (!size) - return emitError() << "expected a scalar, vector or struct element type"; - if (*size != typeSize) - return emitError() << "ABI size " << typeSize << " does not match " - << elementType; - if (!fields.empty()) - return emitError() << "only struct buffers have fields"; - } - if (bufferSize && (addressSpace != AddressSpace::constant || - bufferSize < typeSize || bufferSize % typeAlignment)) - return emitError() << "only constant buffers have a buffer size, which " - "must cover the element and respect its alignment"; - return success(); -} diff --git a/mlir/dialects/air/AIRAttributes.h b/mlir/dialects/air/AIRAttributes.h deleted file mode 100644 index 4a31eb77..00000000 --- a/mlir/dialects/air/AIRAttributes.h +++ /dev/null @@ -1,12 +0,0 @@ -//===- AIRAttributes.h - Metal AIR attributes -------------------*- C++ -*-===// - -#ifndef MLIR_DIALECTS_AIR_AIRATTRIBUTES_H -#define MLIR_DIALECTS_AIR_AIRATTRIBUTES_H - -#include "mlir/dialects/air/AIREnums.h" - -#include "mlir/IR/Attributes.h" -#define GET_ATTRDEF_CLASSES -#include "mlir/dialects/air/AIRAttributes.h.inc" - -#endif // MLIR_DIALECTS_AIR_AIRATTRIBUTES_H diff --git a/mlir/dialects/air/AIRAttributes.td b/mlir/dialects/air/AIRAttributes.td deleted file mode 100644 index 3e4852d0..00000000 --- a/mlir/dialects/air/AIRAttributes.td +++ /dev/null @@ -1,66 +0,0 @@ -//===- AIRAttributes.td - Metal AIR attributes -------------*- tablegen -*-===// - -#ifndef MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD -#define MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD - -include "mlir/dialects/air/AIRDialect.td" -include "mlir/dialects/air/AIREnums.td" - -def AIR_BuiltinAttr : EnumAttr { - let summary = "Kernel argument builtin"; - let description = [{ - Attach as `air.builtin` to a kernel argument. The position and size - builtins are `uint` or `ushort` scalars or 2- and 3-element vectors - (`i32`, `i16`, `vector<3xi32>`, ...); the index builtins are `uint` or - `ushort` scalars. - }]; -} - -def AIR_MemFlagsAttr : EnumAttr { - let summary = "Memory a barrier orders"; -} - -def AIR_StructField : AttrDef { - let mnemonic = "struct_field"; - let summary = "Field of a constant buffer struct"; - let description = [{ - One `air.struct_type_info` entry: the byte offset, the element size, - the array length (zero for a non-array field), the Metal type name and - the field name. - }]; - let parameters = (ins "uint64_t":$offset, - "uint64_t":$size, - DefaultValuedParameter<"uint64_t", "0">:$count, - StringRefParameter<>:$typeName, - StringRefParameter<>:$name); - let assemblyFormat = "`<` struct(params) `>`"; - let genVerifyDecl = 1; -} - -def AIR_Buffer : AttrDef { - let mnemonic = "buffer"; - let summary = "Kernel buffer argument"; - let description = [{ - Attach as `air.buffer` to a kernel argument of type `!llvm.ptr` in - `addressSpace`. `elementType` is the pointee (a scalar, a vector, or an - `!llvm.struct` described by `fields`), `typeName` its Metal spelling, - and `typeSize` and `typeAlignment` its size and alignment in bytes. - `locationIndex` is the `[[buffer(n)]]` or `[[threadgroup(n)]]` index. - A constant reference records its size as `bufferSize`. `noalias` is a - `__restrict__` pointer. - }]; - let parameters = (ins "unsigned":$locationIndex, - EnumParameter:$addressSpace, - EnumParameter:$access, - "::mlir::Type":$elementType, - StringRefParameter<>:$typeName, - "uint64_t":$typeSize, - "uint64_t":$typeAlignment, - DefaultValuedParameter<"uint64_t", "0">:$bufferSize, - DefaultValuedParameter<"bool", "false">:$noalias, - OptionalArrayRefParameter<"StructFieldAttr">:$fields); - let assemblyFormat = "`<` struct(params) `>`"; - let genVerifyDecl = 1; -} - -#endif // MLIR_DIALECTS_AIR_AIRATTRIBUTES_TD diff --git a/mlir/dialects/air/AIRDialect.cpp b/mlir/dialects/air/AIRDialect.cpp deleted file mode 100644 index 7ad56427..00000000 --- a/mlir/dialects/air/AIRDialect.cpp +++ /dev/null @@ -1,168 +0,0 @@ -//===- AIRDialect.cpp - Metal AIR dialect ---------------------------------===// - -#include "mlir/dialects/air/AIRDialect.h" - -#include "mlir/dialects/air/AIRAttributes.h" -#include "mlir/dialects/air/AIROps.h" - -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/DialectImplementation.h" -#include "mlir/Interfaces/FunctionInterfaces.h" -#include "llvm/ADT/DenseSet.h" - -#include - -using namespace mlir; -using namespace mlir::metal::air; - -#include "mlir/dialects/air/AIRDialect.cpp.inc" - -void AIRDialect::initialize() { - registerAttributes(); - addOperations< -#define GET_OP_LIST -#include "mlir/dialects/air/AIROps.cpp.inc" - >(); -} - -static bool isKernelArgumentAttribute(StringRef name) { - return name == AIRDialect::getBuiltinAttrName() || - name == AIRDialect::getBufferAttrName() || - name == AIRDialect::getArgNameAttrName(); -} - -// Position and size builtins may be 2- or 3-element vectors; index builtins -// are scalars. -static bool isVectorBuiltin(Builtin builtin) { - switch (builtin) { - case Builtin::thread_position_in_grid: - case Builtin::threadgroup_position_in_grid: - case Builtin::thread_position_in_threadgroup: - case Builtin::threads_per_threadgroup: - case Builtin::threadgroups_per_grid: - case Builtin::threads_per_grid: - return true; - default: - return false; - } -} - -static LogicalResult verifyKernelArgument(FunctionOpInterface function, - unsigned index) { - Operation *op = function.getOperation(); - Attribute builtin = - function.getArgAttr(index, AIRDialect::getBuiltinAttrName()); - Attribute buffer = - function.getArgAttr(index, AIRDialect::getBufferAttrName()); - if (bool(builtin) == bool(buffer)) - return op->emitOpError() - << "argument " << index - << " must have exactly one of air.builtin or air.buffer"; - if (Attribute name = - function.getArgAttr(index, AIRDialect::getArgNameAttrName())) - if (auto string = dyn_cast(name); - !string || string.getValue().empty()) - return op->emitOpError("air.arg_name must be a nonempty string"); - Type type = function.getArgumentTypes()[index]; - if (builtin) { - auto metadata = dyn_cast(builtin); - if (!metadata) - return op->emitOpError("air.builtin must be an #air.builtin"); - if (auto vector = dyn_cast(type)) { - if (!isVectorBuiltin(metadata.getValue()) || vector.isScalable() || - vector.getRank() != 1 || - (vector.getNumElements() != 2 && vector.getNumElements() != 3)) - return op->emitOpError() - << "builtin " << stringifyBuiltin(metadata.getValue()) - << " cannot be a " << type; - type = vector.getElementType(); - } - if (!type.isSignlessInteger(32) && !type.isSignlessInteger(16)) - return op->emitOpError("AIR builtins are uint or ushort scalars or " - "vectors"); - return success(); - } - auto metadata = dyn_cast(buffer); - if (!metadata) - return op->emitOpError("air.buffer must be an #air.buffer"); - auto pointer = dyn_cast(type); - if (!pointer || - pointer.getAddressSpace() != unsigned(metadata.getAddressSpace())) - return op->emitOpError() - << "argument " << index - << " must be an !llvm.ptr in the address space of its air.buffer"; - return success(); -} - -LogicalResult AIRDialect::verifyOperationAttribute(Operation *op, - NamedAttribute attr) { - StringRef name = attr.getName().getValue(); - if (isKernelArgumentAttribute(name)) - return op->emitOpError() << name << " belongs on a kernel argument"; - if (name == AIRDialect::getFastMathEnableAttrName()) { - if (!isa(op) || !isa(attr.getValue())) - return op->emitOpError() << name << " is a unit attribute of a module"; - return success(); - } - if (name != AIRDialect::getKernelAttrName()) - return op->emitOpError() << "unknown AIR attribute " << name; - auto function = dyn_cast(op); - if (!function || !isa(attr.getValue())) - return op->emitOpError( - "air.kernel requires a function and a unit attribute"); - if (failed(op->getName().verifyInvariants(op))) - return failure(); - if (function.isExternal()) - return op->emitOpError("AIR kernels need a body"); - if (function.getNumResults() != 0) - return op->emitOpError("AIR kernels cannot return values"); - if (auto llvmFunction = dyn_cast(op)) - if (llvmFunction.isVarArg()) - return op->emitOpError("AIR kernels cannot be variadic"); - llvm::DenseSet builtins; - llvm::DenseSet> bindings; - for (unsigned i = 0; i < function.getNumArguments(); ++i) { - if (failed(verifyKernelArgument(function, i))) - return failure(); - if (auto builtin = function.getArgAttrOfType( - i, AIRDialect::getBuiltinAttrName())) { - if (!builtins.insert(builtin.getValue()).second) - return op->emitOpError() << "duplicate AIR builtin " - << stringifyBuiltin(builtin.getValue()); - continue; - } - auto buffer = function.getArgAttrOfType( - i, AIRDialect::getBufferAttrName()); - // Device and constant buffers share bindings; threadgroup buffers do not. - auto key = std::make_pair( - unsigned(buffer.getAddressSpace() == AddressSpace::threadgroup), - buffer.getLocationIndex()); - if (!bindings.insert(key).second) - return op->emitOpError() - << "duplicate AIR buffer binding " << buffer.getLocationIndex(); - } - return success(); -} - -LogicalResult AIRDialect::verifyRegionArgAttribute(Operation *op, - unsigned regionIndex, - unsigned argIndex, - NamedAttribute attr) { - StringRef name = attr.getName().getValue(); - if (!isKernelArgumentAttribute(name)) - return op->emitOpError() << name << " is not an AIR argument attribute"; - auto function = dyn_cast(op); - if (!function || regionIndex != 0 || - !op->getAttrOfType(AIRDialect::getKernelAttrName())) - return op->emitOpError() - << name << " requires an air.kernel function argument"; - return verifyKernelArgument(function, argIndex); -} - -LogicalResult AIRDialect::verifyRegionResultAttribute(Operation *op, unsigned, - unsigned, - NamedAttribute attr) { - return op->emitOpError() << attr.getName() - << " cannot annotate a function result"; -} diff --git a/mlir/dialects/air/AIRDialect.h b/mlir/dialects/air/AIRDialect.h deleted file mode 100644 index e28ec0bc..00000000 --- a/mlir/dialects/air/AIRDialect.h +++ /dev/null @@ -1,9 +0,0 @@ -//===- AIRDialect.h - Metal AIR dialect -------------------------*- C++ -*-===// - -#ifndef MLIR_DIALECTS_AIR_AIRDIALECT_H -#define MLIR_DIALECTS_AIR_AIRDIALECT_H - -#include "mlir/IR/Dialect.h" -#include "mlir/dialects/air/AIRDialect.h.inc" - -#endif // MLIR_DIALECTS_AIR_AIRDIALECT_H diff --git a/mlir/dialects/air/AIRDialect.td b/mlir/dialects/air/AIRDialect.td deleted file mode 100644 index 96264d76..00000000 --- a/mlir/dialects/air/AIRDialect.td +++ /dev/null @@ -1,53 +0,0 @@ -//===- AIRDialect.td - Metal AIR dialect -------------------*- tablegen -*-===// - -#ifndef MLIR_DIALECTS_AIR_AIRDIALECT_TD -#define MLIR_DIALECTS_AIR_AIRDIALECT_TD - -include "mlir/IR/AttrTypeBase.td" -include "mlir/IR/OpBase.td" - -def AIR_Dialect : Dialect { - let name = "air"; - let cppNamespace = "::mlir::metal::air"; - let summary = "Apple Metal AIR intrinsics and kernel ABI metadata"; - let description = [{ - The Metal-specific part of the LLVM IR Apple's Metal compiler emits (AIR), - not AMD's asynchronous AIR dialect. Each operation is an `air.*` - intrinsic named as Apple names it, without the type suffix its operand - and result types determine: `air.fast_exp %x : f32` is - `@air.fast_exp.f32`, `air.max.s %a, %b : vector<2xi32>` is - `@air.max.s.v2i32`. Kernels are `func.func` with `air.kernel`, and their - arguments carry the `!air.kernel` descriptors as `air.builtin` or - `air.buffer`. Arithmetic, memory and control flow use the arith, llvm - and scf dialects; pointers are `!llvm.ptr` in AIR's address spaces. - }]; - let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; - let useDefaultAttributePrinterParser = 1; - let hasOperationAttrVerify = 1; - let hasRegionArgAttrVerify = 1; - let hasRegionResultAttrVerify = 1; - let extraClassDeclaration = [{ - void registerAttributes(); - - static constexpr ::llvm::StringLiteral getKernelAttrName() { - return ::llvm::StringLiteral("air.kernel"); - } - static constexpr ::llvm::StringLiteral getBuiltinAttrName() { - return ::llvm::StringLiteral("air.builtin"); - } - static constexpr ::llvm::StringLiteral getBufferAttrName() { - return ::llvm::StringLiteral("air.buffer"); - } - static constexpr ::llvm::StringLiteral getArgNameAttrName() { - return ::llvm::StringLiteral("air.arg_name"); - } - static constexpr ::llvm::StringLiteral getFastMathEnableAttrName() { - return ::llvm::StringLiteral("air.fast_math_enable"); - } - }]; -} - -class AIR_Op traits = []> - : Op; - -#endif // MLIR_DIALECTS_AIR_AIRDIALECT_TD diff --git a/mlir/dialects/air/AIREnums.h b/mlir/dialects/air/AIREnums.h deleted file mode 100644 index 385f2041..00000000 --- a/mlir/dialects/air/AIREnums.h +++ /dev/null @@ -1,10 +0,0 @@ -//===- AIREnums.h - Metal AIR enums -----------------------------*- C++ -*-===// - -#ifndef MLIR_DIALECTS_AIR_AIRENUMS_H -#define MLIR_DIALECTS_AIR_AIRENUMS_H - -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/dialects/air/AIREnums.h.inc" -#include "llvm/ADT/StringExtras.h" - -#endif // MLIR_DIALECTS_AIR_AIRENUMS_H diff --git a/mlir/dialects/air/AIREnums.td b/mlir/dialects/air/AIREnums.td deleted file mode 100644 index 61e9c25a..00000000 --- a/mlir/dialects/air/AIREnums.td +++ /dev/null @@ -1,54 +0,0 @@ -//===- AIREnums.td - Metal AIR enums -----------------------*- tablegen -*-===// - -#ifndef MLIR_DIALECTS_AIR_AIRENUMS_TD -#define MLIR_DIALECTS_AIR_AIRENUMS_TD - -include "mlir/IR/EnumAttr.td" - -def AIR_Builtin : I32EnumAttr<"Builtin", "Kernel argument builtin", [ - I32EnumAttrCase<"thread_position_in_grid", 0>, - I32EnumAttrCase<"threadgroup_position_in_grid", 1>, - I32EnumAttrCase<"thread_position_in_threadgroup", 2>, - I32EnumAttrCase<"threads_per_threadgroup", 3>, - I32EnumAttrCase<"threadgroups_per_grid", 4>, - I32EnumAttrCase<"threads_per_grid", 5>, - I32EnumAttrCase<"thread_index_in_simdgroup", 6>, - I32EnumAttrCase<"simdgroup_index_in_threadgroup", 7>, - I32EnumAttrCase<"threads_per_simdgroup", 8>, - I32EnumAttrCase<"simdgroups_per_threadgroup", 9>, - I32EnumAttrCase<"thread_index_in_threadgroup", 10>, -]> { - let cppNamespace = "::mlir::metal::air"; - let genSpecializedAttr = 0; -} - -def AIR_AddressSpace : I32EnumAttr<"AddressSpace", "Address space", [ - I32EnumAttrCase<"thread", 0>, - I32EnumAttrCase<"device", 1>, - I32EnumAttrCase<"constant", 2>, - I32EnumAttrCase<"threadgroup", 3>, -]> { - let cppNamespace = "::mlir::metal::air"; - let genSpecializedAttr = 0; -} - -def AIR_Access : I32EnumAttr<"Access", "Buffer access", [ - I32EnumAttrCase<"read", 0>, - I32EnumAttrCase<"read_write", 1>, -]> { - let cppNamespace = "::mlir::metal::air"; - let genSpecializedAttr = 0; -} - -def AIR_MemFlags : I32BitEnumAttr<"MemFlags", "Barrier memory flags", [ - I32BitEnumAttrCaseNone<"none">, - I32BitEnumAttrCaseBit<"device", 0>, - I32BitEnumAttrCaseBit<"threadgroup", 1>, - I32BitEnumAttrCaseBit<"texture", 2>, -]> { - let separator = "|"; - let cppNamespace = "::mlir::metal::air"; - let genSpecializedAttr = 0; -} - -#endif // MLIR_DIALECTS_AIR_AIRENUMS_TD diff --git a/mlir/dialects/air/AIROps.cpp b/mlir/dialects/air/AIROps.cpp deleted file mode 100644 index 85e48837..00000000 --- a/mlir/dialects/air/AIROps.cpp +++ /dev/null @@ -1,64 +0,0 @@ -//===- AIROps.cpp - Metal AIR operations ----------------------------------===// - -#include "mlir/dialects/air/AIROps.h" - -#include "mlir/dialects/air/AIRDialect.h" - -#include "mlir/IR/Builders.h" -#include "mlir/IR/OpImplementation.h" -#include "mlir/IR/TypeUtilities.h" - -using namespace mlir; -using namespace mlir::metal::air; - -#define GET_OP_CLASSES -#include "mlir/dialects/air/AIROps.cpp.inc" - -LogicalResult DotOp::verify() { - if (getElementTypeOrSelf(getX().getType()) != getResult().getType()) - return emitOpError("expects the operands' element type as result"); - return success(); -} - -static LogicalResult verifyConvert(Operation *op, bool toOrFromInteger) { - Type input = op->getOperand(0).getType(); - Type result = op->getResult(0).getType(); - if (failed(verifyCompatibleShape(input, result)) || - isa(input) != isa(result)) - return op->emitOpError("expects operands and results of the same shape"); - bool inputInt = isa(getElementTypeOrSelf(input)); - bool resultInt = isa(getElementTypeOrSelf(result)); - if (!toOrFromInteger && (inputInt || resultInt || input == result)) - return op->emitOpError("converts between distinct float types"); - if (!toOrFromInteger && !isa(input) && - (input.isF32() || result.isF32())) - return op->emitOpError("scalar f32 conversions are LLVM casts in Apple's " - "compiler"); - if (toOrFromInteger && inputInt == resultInt) - return op->emitOpError("converts between an integer and a float type"); - if (resultInt && cast(op) - .getFastMathFlagsAttr() - .getValue() != arith::FastMathFlags::none) - return op->emitOpError("integer results take no fastmath flags"); - return success(); -} - -LogicalResult ConvertOp::verify() { return verifyConvert(*this, false); } -LogicalResult ConvertSOp::verify() { return verifyConvert(*this, true); } -LogicalResult ConvertUOp::verify() { return verifyConvert(*this, true); } - -LogicalResult SimdgroupMatrixMultiplyAccumulateOp::verify() { - auto element = [](Value value) { - return cast(value.getType()).getElementType(); - }; - Type d = element(getResult()), a = element(getA()), b = element(getB()), - c = element(getC()); - // The combinations Apple's compiler emits for simdgroup_multiply_accumulate. - bool mixed = d.isF32() && c.isF32() && (a.isF32() || a == b) && - (b.isF16() || b.isBF16() || b.isF32()); - bool half = d.isF16() && a.isF16() && b.isF16() && c.isF16(); - if (!mixed && !half) - return emitOpError("unsupported element types ") - << a << " * " << b << " + " << c << " -> " << d; - return success(); -} diff --git a/mlir/dialects/air/AIROps.h b/mlir/dialects/air/AIROps.h deleted file mode 100644 index baa5a517..00000000 --- a/mlir/dialects/air/AIROps.h +++ /dev/null @@ -1,16 +0,0 @@ -//===- AIROps.h - Metal AIR operations --------------------------*- C++ -*-===// - -#ifndef MLIR_DIALECTS_AIR_AIROPS_H -#define MLIR_DIALECTS_AIR_AIROPS_H - -#include "mlir/dialects/air/AIRAttributes.h" - -#include "mlir/Dialect/Arith/IR/Arith.h" -#include "mlir/Dialect/LLVMIR/LLVMTypes.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/OpDefinition.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" -#define GET_OP_CLASSES -#include "mlir/dialects/air/AIROps.h.inc" - -#endif // MLIR_DIALECTS_AIR_AIROPS_H diff --git a/mlir/dialects/air/AIROps.td b/mlir/dialects/air/AIROps.td deleted file mode 100644 index 6d072437..00000000 --- a/mlir/dialects/air/AIROps.td +++ /dev/null @@ -1,352 +0,0 @@ -//===- AIROps.td - Metal AIR operations --------------------*- tablegen -*-===// - -#ifndef MLIR_DIALECTS_AIR_AIROPS_TD -#define MLIR_DIALECTS_AIR_AIROPS_TD - -include "mlir/Dialect/Arith/IR/ArithBase.td" -include "mlir/Dialect/Arith/IR/ArithOpsInterfaces.td" -include "mlir/Dialect/LLVMIR/LLVMOpBase.td" -include "mlir/Interfaces/InferTypeOpInterface.td" -include "mlir/Interfaces/SideEffectInterfaces.td" -include "mlir/dialects/air/AIRAttributes.td" - -class AIR_VectorOf types> - : FixedVectorOfRankAndLengthAndType<[1], [2, 3, 4], types>; -class AIR_ScalarOrVectorOf types> - : AnyTypeOf])>; - -def AIR_FloatLike : AIR_ScalarOrVectorOf<[F16, F32]>; -def AIR_F32Like : AIR_ScalarOrVectorOf<[F32]>; -def AIR_IntLike : AIR_ScalarOrVectorOf<[I8, I16, I32, I64]>; -def AIR_SimdIntLike : AIR_ScalarOrVectorOf<[I8, I16, I32]>; -def AIR_ConvertLike : AIR_ScalarOrVectorOf<[F16, BF16, F32, I8, I16, I32, I64]>; -def AIR_Matrix : FixedVectorOfRankAndLengthAndType<[1], [64], [F16, BF16, F32]>; -def AIR_MatrixVector : FixedVectorOfShapeAndType<[2], I64>; -def AIR_MatrixPointer : AnyTypeOf<[LLVM_PointerInAddressSpace<1>, - LLVM_PointerInAddressSpace<3>]>; - -def AIR_FastMath : DefaultValuedAttr; - -//===----------------------------------------------------------------------===// -// Math -//===----------------------------------------------------------------------===// - -class AIR_ElementwiseOp traits = []> - : AIR_Op { - let summary = "`" # mnemonic # "` on each element"; - let description = "Calls `@air." # mnemonic # ".`."; -} - -class AIR_FloatUnaryOp - : AIR_ElementwiseOp]> { - let arguments = (ins type:$x, AIR_FastMath:$fastmath); - let results = (outs type:$result); - let assemblyFormat = - "$x (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; -} - -class AIR_FloatBinaryOp - : AIR_ElementwiseOp]> { - let arguments = (ins type:$x, type:$y, AIR_FastMath:$fastmath); - let results = (outs type:$result); - let assemblyFormat = - "$x `,` $y (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; -} - -class AIR_FloatTernaryOp - : AIR_ElementwiseOp]> { - let arguments = (ins type:$x, type:$y, type:$z, AIR_FastMath:$fastmath); - let results = (outs type:$result); - let assemblyFormat = [{ - $x `,` $y `,` $z (`fastmath` `` $fastmath^)? attr-dict `:` type($result) - }]; -} - -class AIR_IntUnaryOp : AIR_ElementwiseOp { - let arguments = (ins AIR_IntLike:$x); - let results = (outs AIR_IntLike:$result); - let assemblyFormat = "$x attr-dict `:` type($result)"; -} - -class AIR_IntBinaryOp : AIR_ElementwiseOp { - let arguments = (ins AIR_IntLike:$x, AIR_IntLike:$y); - let results = (outs AIR_IntLike:$result); - let assemblyFormat = "$x `,` $y attr-dict `:` type($result)"; -} - -class AIR_IntTernaryOp : AIR_ElementwiseOp { - let arguments = (ins AIR_IntLike:$x, AIR_IntLike:$y, AIR_IntLike:$z); - let results = (outs AIR_IntLike:$result); - let assemblyFormat = "$x `,` $y `,` $z attr-dict `:` type($result)"; -} - -// `exp` is `metal::precise::exp`; `fast_exp` is `metal::fast::exp`, which -// `exp` selects under fast math. Only f32 has fast variants. -def AIR_ExpOp : AIR_FloatUnaryOp<"exp">; -def AIR_FastExpOp : AIR_FloatUnaryOp<"fast_exp", AIR_F32Like>; -def AIR_Exp2Op : AIR_FloatUnaryOp<"exp2">; -def AIR_FastExp2Op : AIR_FloatUnaryOp<"fast_exp2", AIR_F32Like>; -def AIR_LogOp : AIR_FloatUnaryOp<"log">; -def AIR_FastLogOp : AIR_FloatUnaryOp<"fast_log", AIR_F32Like>; -def AIR_Log2Op : AIR_FloatUnaryOp<"log2">; -def AIR_FastLog2Op : AIR_FloatUnaryOp<"fast_log2", AIR_F32Like>; -def AIR_TanhOp : AIR_FloatUnaryOp<"tanh">; -def AIR_FastTanhOp : AIR_FloatUnaryOp<"fast_tanh", AIR_F32Like>; -def AIR_SqrtOp : AIR_FloatUnaryOp<"sqrt">; -def AIR_FastSqrtOp : AIR_FloatUnaryOp<"fast_sqrt", AIR_F32Like>; -def AIR_RsqrtOp : AIR_FloatUnaryOp<"rsqrt">; -def AIR_FastRsqrtOp : AIR_FloatUnaryOp<"fast_rsqrt", AIR_F32Like>; -def AIR_FabsOp : AIR_FloatUnaryOp<"fabs">; -def AIR_FastFabsOp : AIR_FloatUnaryOp<"fast_fabs", AIR_F32Like>; -def AIR_CeilOp : AIR_FloatUnaryOp<"ceil">; -def AIR_FastCeilOp : AIR_FloatUnaryOp<"fast_ceil", AIR_F32Like>; -def AIR_FloorOp : AIR_FloatUnaryOp<"floor">; -def AIR_FastFloorOp : AIR_FloatUnaryOp<"fast_floor", AIR_F32Like>; -def AIR_RoundOp : AIR_FloatUnaryOp<"round">; -def AIR_FastRoundOp : AIR_FloatUnaryOp<"fast_round", AIR_F32Like>; -def AIR_TruncOp : AIR_FloatUnaryOp<"trunc">; -def AIR_FastTruncOp : AIR_FloatUnaryOp<"fast_trunc", AIR_F32Like>; -def AIR_SinOp : AIR_FloatUnaryOp<"sin">; -def AIR_FastSinOp : AIR_FloatUnaryOp<"fast_sin", AIR_F32Like>; -def AIR_CosOp : AIR_FloatUnaryOp<"cos">; -def AIR_FastCosOp : AIR_FloatUnaryOp<"fast_cos", AIR_F32Like>; - -def AIR_FmaxOp : AIR_FloatBinaryOp<"fmax">; -def AIR_FastFmaxOp : AIR_FloatBinaryOp<"fast_fmax", AIR_F32Like>; -def AIR_FminOp : AIR_FloatBinaryOp<"fmin">; -def AIR_FastFminOp : AIR_FloatBinaryOp<"fast_fmin", AIR_F32Like>; -def AIR_PowOp : AIR_FloatBinaryOp<"pow">; -def AIR_FastPowOp : AIR_FloatBinaryOp<"fast_pow", AIR_F32Like>; -def AIR_FmodOp : AIR_FloatBinaryOp<"fmod">; -def AIR_FastFmodOp : AIR_FloatBinaryOp<"fast_fmod", AIR_F32Like>; - -def AIR_FmaOp : AIR_FloatTernaryOp<"fma">; -def AIR_ClampOp : AIR_FloatTernaryOp<"clamp">; -def AIR_FastClampOp : AIR_FloatTernaryOp<"fast_clamp", AIR_F32Like>; - -def AIR_DotOp : AIR_Op<"dot", [Pure, AllTypesMatch<["x", "y"]>, - DeclareOpInterfaceMethods]> { - let summary = "Dot product of two float vectors"; - let description = "Calls `@air.dot.`."; - let arguments = (ins AIR_VectorOf<[F16, F32]>:$x, AIR_VectorOf<[F16, F32]>:$y, - AIR_FastMath:$fastmath); - let results = (outs AnyTypeOf<[F16, F32]>:$result); - let assemblyFormat = [{ - $x `,` $y (`fastmath` `` $fastmath^)? attr-dict `:` type($x) `->` - type($result) - }]; - let hasVerifier = 1; -} - -// `.s` and `.u` are the signed and unsigned overloads of MSL's integer -// functions. -def AIR_MaxSOp : AIR_IntBinaryOp<"max.s">; -def AIR_MaxUOp : AIR_IntBinaryOp<"max.u">; -def AIR_MinSOp : AIR_IntBinaryOp<"min.s">; -def AIR_MinUOp : AIR_IntBinaryOp<"min.u">; -def AIR_AbsSOp : AIR_IntUnaryOp<"abs.s">; -def AIR_ClampSOp : AIR_IntTernaryOp<"clamp.s">; -def AIR_ClampUOp : AIR_IntTernaryOp<"clamp.u">; - -//===----------------------------------------------------------------------===// -// Conversions -//===----------------------------------------------------------------------===// - -class AIR_ConvertBaseOp - : AIR_Op]> { - let summary = summaryText; - let description = [{ - Calls `@air.convert....`, - where floats are `f` and the integer side is `s` or `u` as the mnemonic - says. Scalar f32 to and from f16 or bf16, and integer to integer, are - LLVM casts in Apple's compiler. - }]; - let arguments = (ins AIR_ConvertLike:$x, AIR_FastMath:$fastmath); - let results = (outs AIR_ConvertLike:$result); - let assemblyFormat = [{ - $x (`fastmath` `` $fastmath^)? attr-dict `:` type($x) `to` type($result) - }]; - let hasVerifier = 1; -} - -def AIR_ConvertOp : AIR_ConvertBaseOp<"convert", "Float to float conversion">; -def AIR_ConvertSOp : AIR_ConvertBaseOp<"convert.s", - "Conversion between floats and signed integers">; -def AIR_ConvertUOp : AIR_ConvertBaseOp<"convert.u", - "Conversion between floats and unsigned integers">; - -//===----------------------------------------------------------------------===// -// SIMD-group functions -//===----------------------------------------------------------------------===// - -// SIMD-group functions read the other lanes, so they are not Pure: unknown -// effects keep them from being speculated or moved across divergent control. -class AIR_SimdReduceOp traits = []> - : AIR_Op { - let summary = "`" # mnemonic # "` across the SIMD-group"; - let description = "Calls `@air." # mnemonic # ".`."; - let results = (outs type:$result); -} - -class AIR_SimdFloatReduceOp - : AIR_SimdReduceOp]> { - let arguments = (ins AIR_FloatLike:$x, AIR_FastMath:$fastmath); - let assemblyFormat = - "$x (`fastmath` `` $fastmath^)? attr-dict `:` type($result)"; -} - -class AIR_SimdIntReduceOp - : AIR_SimdReduceOp { - let arguments = (ins AIR_SimdIntLike:$x); - let assemblyFormat = "$x attr-dict `:` type($result)"; -} - -class AIR_SimdShuffleOp traits = []> - : AIR_Op]> { - let summary = "`" # mnemonic # "` across the SIMD-group"; - let description = [{ - Calls `@air.}] # mnemonic # [{.`. `lane` is the source lane, the - XOR mask or the lane delta, as for the MSL function. - }]; - let results = (outs type:$result); -} - -class AIR_SimdFloatShuffleOp - : AIR_SimdShuffleOp]> { - let arguments = (ins AIR_FloatLike:$data, I16:$lane, AIR_FastMath:$fastmath); - let assemblyFormat = [{ - $data `,` $lane (`fastmath` `` $fastmath^)? attr-dict `:` type($result) - }]; -} - -class AIR_SimdIntShuffleOp - : AIR_SimdShuffleOp { - let arguments = (ins AIR_SimdIntLike:$data, I16:$lane); - let assemblyFormat = "$data `,` $lane attr-dict `:` type($result)"; -} - -def AIR_SimdSumOp : AIR_SimdFloatReduceOp<"simd_sum">; -def AIR_SimdSumSOp : AIR_SimdIntReduceOp<"simd_sum.s">; -def AIR_SimdSumUOp : AIR_SimdIntReduceOp<"simd_sum.u">; -def AIR_SimdMaxOp : AIR_SimdFloatReduceOp<"simd_max">; -def AIR_SimdMaxSOp : AIR_SimdIntReduceOp<"simd_max.s">; -def AIR_SimdMaxUOp : AIR_SimdIntReduceOp<"simd_max.u">; -def AIR_SimdMinOp : AIR_SimdFloatReduceOp<"simd_min">; -def AIR_SimdMinSOp : AIR_SimdIntReduceOp<"simd_min.s">; -def AIR_SimdMinUOp : AIR_SimdIntReduceOp<"simd_min.u">; -def AIR_SimdPrefixExclusiveSumOp - : AIR_SimdFloatReduceOp<"simd_prefix_exclusive_sum">; - -def AIR_SimdShuffleOp : AIR_SimdFloatShuffleOp<"simd_shuffle">; -def AIR_SimdShuffleSOp : AIR_SimdIntShuffleOp<"simd_shuffle.s">; -def AIR_SimdShuffleUOp : AIR_SimdIntShuffleOp<"simd_shuffle.u">; -def AIR_SimdShuffleXorOp : AIR_SimdFloatShuffleOp<"simd_shuffle_xor">; -def AIR_SimdShuffleXorSOp : AIR_SimdIntShuffleOp<"simd_shuffle_xor.s">; -def AIR_SimdShuffleXorUOp : AIR_SimdIntShuffleOp<"simd_shuffle_xor.u">; -def AIR_SimdShuffleDownOp : AIR_SimdFloatShuffleOp<"simd_shuffle_down">; -def AIR_SimdShuffleDownSOp : AIR_SimdIntShuffleOp<"simd_shuffle_down.s">; -def AIR_SimdShuffleDownUOp : AIR_SimdIntShuffleOp<"simd_shuffle_down.u">; -def AIR_SimdBroadcastOp : AIR_SimdFloatShuffleOp<"simd_broadcast">; -def AIR_SimdBroadcastSOp : AIR_SimdIntShuffleOp<"simd_broadcast.s">; -def AIR_SimdBroadcastUOp : AIR_SimdIntShuffleOp<"simd_broadcast.u">; - -class AIR_SimdVoteOp : AIR_Op { - let summary = summaryText; - let description = "Calls `@air." # mnemonic # "`."; - let arguments = (ins I1:$x); - let results = (outs I1:$result); - let assemblyFormat = "$x attr-dict"; -} - -def AIR_SimdAnyOp : AIR_SimdVoteOp<"simd_any", - "Whether `x` is true in any active lane">; -def AIR_SimdAllOp : AIR_SimdVoteOp<"simd_all", - "Whether `x` is true in every active lane">; - -//===----------------------------------------------------------------------===// -// Barriers -//===----------------------------------------------------------------------===// - -def AIR_WgBarrierOp : AIR_Op<"wg.barrier"> { - let summary = "threadgroup_barrier"; - let description = "Calls `@air.wg.barrier(i32 mem_flags, i32 1)`."; - let arguments = (ins AIR_MemFlagsAttr:$mem_flags); - let assemblyFormat = "$mem_flags attr-dict"; -} - -def AIR_SimdgroupBarrierOp : AIR_Op<"simdgroup.barrier"> { - let summary = "simdgroup_barrier"; - let description = "Calls `@air.simdgroup.barrier(i32 mem_flags, i32 4)`."; - let arguments = (ins AIR_MemFlagsAttr:$mem_flags); - let assemblyFormat = "$mem_flags attr-dict"; -} - -//===----------------------------------------------------------------------===// -// SIMD-group matrices -//===----------------------------------------------------------------------===// - -// A simdgroup_matrix is a vector<64xT>; thread_elements() are its -// elements 0 and 1. -def AIR_SimdgroupMatrixLoadOp : AIR_Op<"simdgroup_matrix_8x8_load", [ - DeclareOpInterfaceMethods]> { - let summary = "simdgroup_load"; - let description = [{ - Calls `@air.simdgroup_matrix_8x8_load.v64.p
`. - `simdgroup_load(m, p, ld, origin, transpose)` passes `extents` (ld, 8) - and `strides` (1, ld), swapped when transposed. - }]; - let arguments = (ins AIR_MatrixPointer:$pointer, - AIR_MatrixVector:$extents, - AIR_MatrixVector:$strides, - AIR_MatrixVector:$origin, - AIR_FastMath:$fastmath); - let results = (outs AIR_Matrix:$result); - let assemblyFormat = [{ - $pointer `,` $extents `,` $strides `,` $origin (`fastmath` `` $fastmath^)? - attr-dict `:` qualified(type($pointer)) `->` type($result) - }]; -} - -def AIR_SimdgroupMatrixStoreOp : AIR_Op<"simdgroup_matrix_8x8_store"> { - let summary = "simdgroup_store"; - let description = [{ - Calls `@air.simdgroup_matrix_8x8_store.v64.p
`, with - the operands of `simdgroup_matrix_8x8_load`. - }]; - let arguments = (ins AIR_Matrix:$value, - AIR_MatrixPointer:$pointer, - AIR_MatrixVector:$extents, - AIR_MatrixVector:$strides, - AIR_MatrixVector:$origin); - let assemblyFormat = [{ - $value `,` $pointer `,` $extents `,` $strides `,` $origin attr-dict `:` - type($value) `,` qualified(type($pointer)) - }]; -} - -def AIR_SimdgroupMatrixMultiplyAccumulateOp - : AIR_Op<"simdgroup_matrix_8x8_multiply_accumulate", [ - DeclareOpInterfaceMethods]> { - let summary = "simdgroup_multiply_accumulate"; - let description = [{ - Calls `@air.simdgroup_matrix_8x8_multiply_accumulate.v64.v64.v64.v64`: - `result = a * b + c`. `simdgroup_multiply` passes a zero `c`. - }]; - let arguments = (ins AIR_Matrix:$a, AIR_Matrix:$b, AIR_Matrix:$c, - AIR_FastMath:$fastmath); - let results = (outs AIR_Matrix:$result); - let assemblyFormat = [{ - $a `,` $b `,` $c (`fastmath` `` $fastmath^)? attr-dict `:` type($a) `,` - type($b) `,` type($c) `->` type($result) - }]; - let hasVerifier = 1; -} - -#endif // MLIR_DIALECTS_AIR_AIROPS_TD diff --git a/mlir/dialects/air/BUILD.bazel b/mlir/dialects/air/BUILD.bazel deleted file mode 100644 index 9fb6ed6c..00000000 --- a/mlir/dialects/air/BUILD.bazel +++ /dev/null @@ -1,179 +0,0 @@ -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - ":dialect", - ":headers", - "@llvm-project//mlir:CAPIIR", - ], -) - -zig_library( - name = "air", - import_name = "mlir/dialects/air", - main = "air.zig", - tags = ["manual"], - deps = [ - ":capi", - "//mlir", - "//mlir/dialects/arith", - "//mlir/dialects/func", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":air", - "//mlir", - "//mlir/dialects/arith", - "//mlir/dialects/func", - "//mlir/dialects/llvm", - ], -) - -td_library( - name = "td_files", - srcs = [ - "AIRAttributes.td", - "AIRDialect.td", - "AIREnums.td", - "AIROps.td", - ], - includes = ["/"], - deps = [ - "@llvm-project//mlir:ArithOpsInterfacesTdFiles", - "@llvm-project//mlir:ArithOpsTdFiles", - "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", - "@llvm-project//mlir:LLVMOpsTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - ], -) - -gentbl_cc_library( - name = "dialect_inc_gen", - tbl_outs = [ - ( - [ - "-gen-dialect-decls", - "-dialect=air", - ], - "AIRDialect.h.inc", - ), - ( - [ - "-gen-dialect-defs", - "-dialect=air", - ], - "AIRDialect.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "AIRDialect.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "enums_inc_gen", - tbl_outs = [ - ( - ["-gen-enum-decls"], - "AIREnums.h.inc", - ), - ( - ["-gen-enum-defs"], - "AIREnums.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "AIREnums.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "attrs_inc_gen", - tbl_outs = [ - ( - [ - "-gen-attrdef-decls", - "-attrdefs-dialect=air", - ], - "AIRAttributes.h.inc", - ), - ( - [ - "-gen-attrdef-defs", - "-attrdefs-dialect=air", - ], - "AIRAttributes.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "AIRAttributes.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "ops_inc_gen", - tbl_outs = [ - ( - ["-gen-op-decls"], - "AIROps.h.inc", - ), - ( - ["-gen-op-defs"], - "AIROps.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "AIROps.td", - deps = [":td_files"], -) - -cc_library( - name = "headers", - hdrs = [ - "AIRAttributes.h", - "AIRDialect.h", - "AIREnums.h", - "AIROps.h", - ], - deps = [ - ":attrs_inc_gen", - ":dialect_inc_gen", - ":enums_inc_gen", - ":ops_inc_gen", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:LLVMDialect", - "@llvm-project//mlir:SideEffectInterfaces", - ], -) - -cc_library( - name = "dialect", - srcs = [ - "AIRAttributes.cpp", - "AIRDialect.cpp", - "AIROps.cpp", - ], - deps = [ - ":headers", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:FunctionInterfaces", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:LLVMDialect", - ], -) diff --git a/mlir/dialects/air/README.md b/mlir/dialects/air/README.md deleted file mode 100644 index cd8a0531..00000000 --- a/mlir/dialects/air/README.md +++ /dev/null @@ -1,90 +0,0 @@ -# Metal AIR dialect - -The Metal-specific part of AIR, the LLVM IR Apple's Metal compiler emits (not -AMD's asynchronous AIR dialect). Namespace `air`, C++ namespace -`mlir::metal::air`, C API prefix `mlirMetalAIR`. `dsl/metal` emits it, and -XLA's `__gpu$xla.gpu.air` custom call lowers it, from a vendored copy, to a -metallib. - -## Operations - -Each operation is an `air.*` intrinsic, named as Apple's compiler names it -minus the type suffix its operand and result types determine: - -| Operation | Intrinsic | -|---|---| -| `air.fast_exp %x : vector<2xf32>` | `@air.fast_exp.v2f32` | -| `air.max.u %a, %b : i32` | `@air.max.u.i32` | -| `air.simd_shuffle_xor %x, %lane : f16` | `@air.simd_shuffle_xor.f16` | -| `air.convert.s %x : i32 to f32` | `@air.convert.f.f32.s.i32` | -| `air.simdgroup_matrix_8x8_load %p, ... : !llvm.ptr<3> -> vector<64xbf16>` | `@air.simdgroup_matrix_8x8_load.v64bf16.p3bf16` | -| `air.wg.barrier ` | `@air.wg.barrier(i32 2, i32 1)` | - -- Float math: `exp`, `exp2`, `log`, `log2`, `tanh`, `sqrt`, `rsqrt`, `fabs`, - `ceil`, `floor`, `round`, `trunc`, `sin`, `cos`, `fmax`, `fmin`, `pow`, - `fmod`, `fma`, `clamp` and `dot` on f16 and f32. The `fast_` variants are - f32 only: under fast math (MSL's default) `exp(x)` is `fast_exp`, and - `metal::precise::exp` is `exp`. -- Integer math: `max`, `min`, `clamp` and `abs`, as `.s` and `.u` - overloads. -- SIMD-group functions: `simd_sum`, `simd_max`, `simd_min`, - `simd_prefix_exclusive_sum`, `simd_shuffle`, `simd_shuffle_xor`, - `simd_shuffle_down` and `simd_broadcast` on floats, the integer ones as - `.s` and `.u`, and `simd_any`, `simd_all`. There are no bfloat overloads. -- `convert` between float types, and `convert.s`, `convert.u` between floats - and integers. Apple's compiler uses LLVM casts for scalar f32 to and from - f16 or bf16, and between integers. -- `wg.barrier` and `simdgroup.barrier` (`threadgroup_barrier`, - `simdgroup_barrier`), with `#air.mem_flags`. -- `simdgroup_matrix_8x8_load`, `_store` and `_multiply_accumulate`. A - `simdgroup_matrix` is a `vector<64xT>`, and `thread_elements()` its - elements 0 and 1. - -Float operations carry `fastmath` flags, which become the call's flags (Apple's -compiler marks every float call `fast` under fast math). SIMD-group, barrier -and matrix operations have unknown effects, so they are not speculated or -moved across divergent control flow. Arithmetic, memory and control flow use -the arith, cf, scf, vector and llvm dialects, with pointers as `!llvm.ptr` in -AIR's address spaces: 0 thread, 1 device, 2 constant, 3 threadgroup (a buffer -is never in thread memory). - -## Kernels - -A kernel is a void `func.func` with `air.kernel`. Each argument carries one -`!air.kernel` descriptor: - -- `air.builtin = #air.builtin`: the position and - size builtins are `uint` or `ushort` scalars or 2- and 3-element vectors, - the index builtins (`thread_index_in_simdgroup`, ...) scalars. -- `air.buffer = #air.buffer<...>` on an `!llvm.ptr`: the `[[buffer(n)]]` or - `[[threadgroup(n)]]` index, the address space, the access (`const` pointers - are `read`), the pointee type with its Metal name, size and alignment, the - size of a constant reference, `noalias` for `__restrict__`, and the fields - of a constant struct (`air.struct_type_info`). Device and constant buffers - share indices; threadgroup buffers have their own. -- `air.arg_name`, optionally, the argument's name. - -`air.fast_math_enable` on the module records that it was compiled with fast -math (`air.compile.fast_math_enable`). - -## Evidence - -Names, signatures, argument descriptors and flags follow what Apple's Metal -compiler 32023.830 (the `@metal_toolchain` repository) emits for MSL probes and -for XLA's Metal kernels, with `metal -std=metal4.0 -S -emit-llvm` (fast math, --O2, as XLA compiles them). Only the overloads it emits are accepted. - -## Build and test - -```sh -bazel build //mlir/dialects/air //mlir/dialects/air:capi -bazel test //mlir/dialects/air:test -``` - -The Zig library is `:air`, the C++ dialect `:dialect` and its C API `:capi`. -`registerDialect(registry)` registers AIR, which loads the LLVM dialect; -`kernel` builds a `func.func`, so callers also register `func`. Attributes are -built through the C API (`BuiltinAttr`, `MemFlagsAttr`, `StructFieldAttr`, -`BufferAttr`), kernel arguments through `builtinArgumentAttr` and -`bufferArgumentAttr`, and every operation has a builder named after it -(`fast_exp`, `max_u`, `wg_barrier`, `simdgroup_matrix_8x8_load`). diff --git a/mlir/dialects/air/air.zig b/mlir/dialects/air/air.zig deleted file mode 100644 index 2cdf2d25..00000000 --- a/mlir/dialects/air/air.zig +++ /dev/null @@ -1,536 +0,0 @@ -//! Metal AIR intrinsics and kernel ABI annotations. A builder emits the -//! `air.*` intrinsic it names, minus the type suffix its types select: -//! `fast_exp` on f32 is `@air.fast_exp.f32`, `max_u` on `vector<2xi32>` is -//! `@air.max.u.v2i32`. Arithmetic, memory and control flow use the arith, -//! llvm and scf dialects, with pointers as `!llvm.ptr` in AIR's address spaces. -const std = @import("std"); - -const arith = @import("mlir/dialects/arith"); -const c = @import("c"); -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("air"); -} - -pub const Builtin = enum(u32) { - thread_position_in_grid, - threadgroup_position_in_grid, - thread_position_in_threadgroup, - threads_per_threadgroup, - threadgroups_per_grid, - threads_per_grid, - thread_index_in_simdgroup, - simdgroup_index_in_threadgroup, - threads_per_simdgroup, - simdgroups_per_threadgroup, - thread_index_in_threadgroup, -}; - -pub const AddressSpace = enum(u32) { thread = 0, device = 1, constant = 2, threadgroup = 3 }; - -pub const Access = enum(u32) { read, read_write }; - -pub const MemFlags = packed struct(u32) { - device: bool = false, - threadgroup: bool = false, - texture: bool = false, - _padding: u29 = 0, - - pub const none: MemFlags = .{}; -}; - -/// `#air.builtin`: Kernel argument builtin -pub const BuiltinAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAMetalAIRBuiltin; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { value: Builtin }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirMetalAIRBuiltinAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn value(self: *const Self) Builtin { - return @fromBackingInt(@intCast(c.mlirMetalAIRBuiltinAttrGetValue(self.ptr()))); - } -}; - -/// `#air.mem_flags`: Memory a barrier orders -pub const MemFlagsAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAMetalAIRMemFlags; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { value: MemFlags }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirMetalAIRMemFlagsAttrGet(ctx.ptr(), @bitCast(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn value(self: *const Self) MemFlags { - return @bitCast(c.mlirMetalAIRMemFlagsAttrGetValue(self.ptr())); - } -}; - -/// `#air.struct_field`: Field of a constant buffer struct -pub const StructFieldAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAMetalAIRStructField; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - offset: u64, - size: u64, - /// The array length, zero for a non-array field. - count: u64 = 0, - typeName: []const u8, - name: []const u8, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirMetalAIRStructFieldAttrGet(ctx.ptr(), args.offset, args.size, args.count, mlir.stringRef(args.typeName), mlir.stringRef(args.name)).ptr orelse return error.InvalidMlir); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn offset(self: *const Self) u64 { - return c.mlirMetalAIRStructFieldAttrGetOffset(self.ptr()); - } - - pub fn size(self: *const Self) u64 { - return c.mlirMetalAIRStructFieldAttrGetSize(self.ptr()); - } - - pub fn count(self: *const Self) u64 { - return c.mlirMetalAIRStructFieldAttrGetCount(self.ptr()); - } - - pub fn typeName(self: *const Self) []const u8 { - return mlir.string(c.mlirMetalAIRStructFieldAttrGetTypeName(self.ptr())); - } - - pub fn name(self: *const Self) []const u8 { - return mlir.string(c.mlirMetalAIRStructFieldAttrGetName(self.ptr())); - } -}; - -/// `#air.buffer`: Kernel buffer argument -pub const BufferAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAMetalAIRBuffer; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - locationIndex: u32, - addressSpace: AddressSpace, - access: Access, - elementType: *const mlir.Type, - typeName: []const u8, - typeSize: u64, - typeAlignment: u64, - /// The size of a constant reference; zero omits it. - bufferSize: u64 = 0, - noalias_: bool = false, - fields: []const *const StructFieldAttr = &.{}, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirMetalAIRBufferAttrGet( - ctx.ptr(), - args.locationIndex, - @backingInt(args.addressSpace), - @backingInt(args.access), - args.elementType.ptr(), - mlir.stringRef(args.typeName), - args.typeSize, - args.typeAlignment, - args.bufferSize, - args.noalias_, - @intCast(args.fields.len), - @ptrCast(args.fields.ptr), - ).ptr orelse return error.InvalidMlir); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn locationIndex(self: *const Self) u32 { - return c.mlirMetalAIRBufferAttrGetLocationIndex(self.ptr()); - } - - pub fn addressSpace(self: *const Self) AddressSpace { - return @fromBackingInt(@intCast(c.mlirMetalAIRBufferAttrGetAddressSpace(self.ptr()))); - } - - pub fn access(self: *const Self) Access { - return @fromBackingInt(@intCast(c.mlirMetalAIRBufferAttrGetAccess(self.ptr()))); - } - - pub fn elementType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirMetalAIRBufferAttrGetElementType(self.ptr()).ptr); - } - - pub fn typeName(self: *const Self) []const u8 { - return mlir.string(c.mlirMetalAIRBufferAttrGetTypeName(self.ptr())); - } - - pub fn typeSize(self: *const Self) u64 { - return c.mlirMetalAIRBufferAttrGetTypeSize(self.ptr()); - } - - pub fn typeAlignment(self: *const Self) u64 { - return c.mlirMetalAIRBufferAttrGetTypeAlignment(self.ptr()); - } - - pub fn bufferSize(self: *const Self) u64 { - return c.mlirMetalAIRBufferAttrGetBufferSize(self.ptr()); - } - - pub fn noalias_(self: *const Self) bool { - return c.mlirMetalAIRBufferAttrGetNoalias(self.ptr()); - } - - pub fn numFields(self: *const Self) usize { - return @intCast(c.mlirMetalAIRBufferAttrGetNumFields(self.ptr())); - } - - pub fn field(self: *const Self, i: usize) *const StructFieldAttr { - return @ptrCast(c.mlirMetalAIRBufferAttrGetField(self.ptr(), @intCast(i)).ptr); - } -}; - -pub const ArgumentOpts = struct { - /// `air.arg_name`, the argument's name in reflection. - name: ?[]const u8 = null, -}; - -/// The argument dictionary of a kernel argument bound to `builtin`. -pub fn builtinArgumentAttr(ctx: *mlir.Context, builtin: Builtin, opts: ArgumentOpts) mlir.Error!*const mlir.Attribute { - return argumentAttr(ctx, .named(ctx, "air.builtin", (try BuiltinAttr.get(ctx, .{ .value = builtin })).attribute()), opts); -} - -/// The argument dictionary of a kernel buffer argument. -pub fn bufferArgumentAttr(ctx: *mlir.Context, args: BufferAttr.InitArgs, opts: ArgumentOpts) mlir.Error!*const mlir.Attribute { - return argumentAttr(ctx, .named(ctx, "air.buffer", (try BufferAttr.get(ctx, args)).attribute()), opts); -} - -fn argumentAttr(ctx: *mlir.Context, annotation: mlir.NamedAttribute, opts: ArgumentOpts) *const mlir.Attribute { - if (opts.name) |name| return .dict(ctx, &.{ annotation, .named(ctx, "air.arg_name", .string(ctx, name)) }); - return .dict(ctx, &.{annotation}); -} - -/// Marks `module` as compiled with fast math (`air.compile.fast_math_enable`). -pub fn enableFastMath(ctx: *mlir.Context, module: *mlir.Operation) void { - module.setAttributeByName("air.fast_math_enable", .unit(ctx)); -} - -pub const KernelArgs = struct { - name: []const u8, - block: *mlir.Block, - args_attributes: []const *const mlir.Attribute, - location: *const mlir.Location, - verify: bool = true, -}; - -/// A `func.func` with `air.kernel`, one argument dictionary per block -/// argument. Takes ownership of `block`; set `verify = false` while the body -/// is incomplete. -pub fn kernel(ctx: *mlir.Context, args: KernelArgs) *mlir.Operation { - std.debug.assert(args.args_attributes.len == args.block.numArguments()); - return func.func(ctx, .{ - .name = args.name, - .block = args.block, - .args_attributes = args.args_attributes, - .results = &.{}, - .extra_attributes = &.{.named(ctx, "air.kernel", .unit(ctx))}, - .location = args.location, - .verify = args.verify, - }); -} - -pub const FloatOpts = struct { - fastmath: arith.FastMathFlags = .none, -}; - -/// The `fastmath` attribute of a float op, stored in the caller's `buf`. -fn fastmathAttrs(buf: *[1]mlir.NamedAttribute, ctx: *mlir.Context, fastmath: arith.FastMathFlags) []const mlir.NamedAttribute { - if (fastmath == arith.FastMathFlags.none) return &.{}; - const attr = arith.FastMathAttr.get(ctx, .{ .value = fastmath }) catch unreachable; - buf[0] = .named(ctx, "fastmath", attr.attribute()); - return buf; -} - -fn floatUnary(comptime name: []const u8) type { - return struct { - pub fn call(ctx: *mlir.Context, x: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: [1]mlir.NamedAttribute = undefined; - return mlir.Operation.make(ctx, name, .{ - .operands = .{ .flat = &.{x} }, - .results = .{ .flat = &.{x.type_()} }, - .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), - .location = location, - }); - } - }; -} - -fn floatBinary(comptime name: []const u8) type { - return struct { - pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: [1]mlir.NamedAttribute = undefined; - return mlir.Operation.make(ctx, name, .{ - .operands = .{ .flat = &.{ x, y } }, - .results = .{ .flat = &.{x.type_()} }, - .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), - .location = location, - }); - } - }; -} - -fn floatTernary(comptime name: []const u8) type { - return struct { - pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, z: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: [1]mlir.NamedAttribute = undefined; - return mlir.Operation.make(ctx, name, .{ - .operands = .{ .flat = &.{ x, y, z } }, - .results = .{ .flat = &.{x.type_()} }, - .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), - .location = location, - }); - } - }; -} - -fn intUnary(comptime name: []const u8) type { - return struct { - pub fn call(ctx: *mlir.Context, x: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, name, .{ - .operands = .{ .flat = &.{x} }, - .results = .{ .flat = &.{x.type_()} }, - .location = location, - }); - } - }; -} - -fn intBinary(comptime name: []const u8) type { - return struct { - pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, name, .{ - .operands = .{ .flat = &.{ x, y } }, - .results = .{ .flat = &.{x.type_()} }, - .location = location, - }); - } - }; -} - -fn intTernary(comptime name: []const u8) type { - return struct { - pub fn call(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, z: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, name, .{ - .operands = .{ .flat = &.{ x, y, z } }, - .results = .{ .flat = &.{x.type_()} }, - .location = location, - }); - } - }; -} - -fn floatShuffle(comptime name: []const u8) type { - return struct { - pub fn call(ctx: *mlir.Context, data: *const mlir.Value, lane: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: [1]mlir.NamedAttribute = undefined; - return mlir.Operation.make(ctx, name, .{ - .operands = .{ .flat = &.{ data, lane } }, - .results = .{ .flat = &.{data.type_()} }, - .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), - .location = location, - }); - } - }; -} - -fn intShuffle(comptime name: []const u8) type { - return struct { - pub fn call(ctx: *mlir.Context, data: *const mlir.Value, lane: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, name, .{ - .operands = .{ .flat = &.{ data, lane } }, - .results = .{ .flat = &.{data.type_()} }, - .location = location, - }); - } - }; -} - -fn convertOp(comptime name: []const u8) type { - return struct { - pub fn call(ctx: *mlir.Context, x: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: [1]mlir.NamedAttribute = undefined; - return mlir.Operation.make(ctx, name, .{ - .operands = .{ .flat = &.{x} }, - .results = .{ .flat = &.{result_type} }, - .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), - .location = location, - }); - } - }; -} - -fn barrier(comptime name: []const u8) type { - return struct { - pub fn call(ctx: *mlir.Context, mem_flags: MemFlags, location: *const mlir.Location) *mlir.Operation { - const flags = MemFlagsAttr.get(ctx, .{ .value = mem_flags }) catch unreachable; - return mlir.Operation.make(ctx, name, .{ - .attributes = &.{.named(ctx, "mem_flags", flags.attribute())}, - .location = location, - }); - } - }; -} - -pub const exp = floatUnary("air.exp").call; -pub const fast_exp = floatUnary("air.fast_exp").call; -pub const exp2 = floatUnary("air.exp2").call; -pub const fast_exp2 = floatUnary("air.fast_exp2").call; -pub const log = floatUnary("air.log").call; -pub const fast_log = floatUnary("air.fast_log").call; -pub const log2 = floatUnary("air.log2").call; -pub const fast_log2 = floatUnary("air.fast_log2").call; -pub const tanh = floatUnary("air.tanh").call; -pub const fast_tanh = floatUnary("air.fast_tanh").call; -pub const sqrt = floatUnary("air.sqrt").call; -pub const fast_sqrt = floatUnary("air.fast_sqrt").call; -pub const rsqrt = floatUnary("air.rsqrt").call; -pub const fast_rsqrt = floatUnary("air.fast_rsqrt").call; -pub const fabs = floatUnary("air.fabs").call; -pub const fast_fabs = floatUnary("air.fast_fabs").call; -pub const ceil = floatUnary("air.ceil").call; -pub const fast_ceil = floatUnary("air.fast_ceil").call; -pub const floor = floatUnary("air.floor").call; -pub const fast_floor = floatUnary("air.fast_floor").call; -pub const round = floatUnary("air.round").call; -pub const fast_round = floatUnary("air.fast_round").call; -pub const trunc = floatUnary("air.trunc").call; -pub const fast_trunc = floatUnary("air.fast_trunc").call; -pub const sin = floatUnary("air.sin").call; -pub const fast_sin = floatUnary("air.fast_sin").call; -pub const cos = floatUnary("air.cos").call; -pub const fast_cos = floatUnary("air.fast_cos").call; -pub const fmax = floatBinary("air.fmax").call; -pub const fast_fmax = floatBinary("air.fast_fmax").call; -pub const fmin = floatBinary("air.fmin").call; -pub const fast_fmin = floatBinary("air.fast_fmin").call; -pub const pow = floatBinary("air.pow").call; -pub const fast_pow = floatBinary("air.fast_pow").call; -pub const fmod = floatBinary("air.fmod").call; -pub const fast_fmod = floatBinary("air.fast_fmod").call; -pub const fma = floatTernary("air.fma").call; -pub const clamp = floatTernary("air.clamp").call; -pub const fast_clamp = floatTernary("air.fast_clamp").call; -pub const abs_s = intUnary("air.abs.s").call; -pub const max_s = intBinary("air.max.s").call; -pub const max_u = intBinary("air.max.u").call; -pub const min_s = intBinary("air.min.s").call; -pub const min_u = intBinary("air.min.u").call; -pub const clamp_s = intTernary("air.clamp.s").call; -pub const clamp_u = intTernary("air.clamp.u").call; -pub const simd_sum = floatUnary("air.simd_sum").call; -pub const simd_sum_s = intUnary("air.simd_sum.s").call; -pub const simd_sum_u = intUnary("air.simd_sum.u").call; -pub const simd_max = floatUnary("air.simd_max").call; -pub const simd_max_s = intUnary("air.simd_max.s").call; -pub const simd_max_u = intUnary("air.simd_max.u").call; -pub const simd_min = floatUnary("air.simd_min").call; -pub const simd_min_s = intUnary("air.simd_min.s").call; -pub const simd_min_u = intUnary("air.simd_min.u").call; -pub const simd_prefix_exclusive_sum = floatUnary("air.simd_prefix_exclusive_sum").call; -pub const simd_shuffle = floatShuffle("air.simd_shuffle").call; -pub const simd_shuffle_s = intShuffle("air.simd_shuffle.s").call; -pub const simd_shuffle_u = intShuffle("air.simd_shuffle.u").call; -pub const simd_shuffle_xor = floatShuffle("air.simd_shuffle_xor").call; -pub const simd_shuffle_xor_s = intShuffle("air.simd_shuffle_xor.s").call; -pub const simd_shuffle_xor_u = intShuffle("air.simd_shuffle_xor.u").call; -pub const simd_shuffle_down = floatShuffle("air.simd_shuffle_down").call; -pub const simd_shuffle_down_s = intShuffle("air.simd_shuffle_down.s").call; -pub const simd_shuffle_down_u = intShuffle("air.simd_shuffle_down.u").call; -pub const simd_broadcast = floatShuffle("air.simd_broadcast").call; -pub const simd_broadcast_s = intShuffle("air.simd_broadcast.s").call; -pub const simd_broadcast_u = intShuffle("air.simd_broadcast.u").call; -pub const simd_any = intUnary("air.simd_any").call; -pub const simd_all = intUnary("air.simd_all").call; -pub const convert = convertOp("air.convert").call; -pub const convert_s = convertOp("air.convert.s").call; -pub const convert_u = convertOp("air.convert.u").call; -pub const wg_barrier = barrier("air.wg.barrier").call; -pub const simdgroup_barrier = barrier("air.simdgroup.barrier").call; - -pub fn dot(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: [1]mlir.NamedAttribute = undefined; - return mlir.Operation.make(ctx, "air.dot", .{ - .operands = .{ .flat = &.{ x, y } }, - .results = .{ .flat = &.{x.type_().isA(mlir.ShapedType).?.elementType()} }, - .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), - .location = location, - }); -} - -/// `simdgroup_load`: `extents`, `strides` and `origin` are `vector<2xi64>`. -pub fn simdgroup_matrix_8x8_load(ctx: *mlir.Context, pointer: *const mlir.Value, extents: *const mlir.Value, strides: *const mlir.Value, origin: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: [1]mlir.NamedAttribute = undefined; - return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_load", .{ - .operands = .{ .flat = &.{ pointer, extents, strides, origin } }, - .results = .{ .flat = &.{result_type} }, - .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), - .location = location, - }); -} - -pub fn simdgroup_matrix_8x8_store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir.Value, extents: *const mlir.Value, strides: *const mlir.Value, origin: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_store", .{ - .operands = .{ .flat = &.{ value, pointer, extents, strides, origin } }, - .location = location, - }); -} - -/// `simdgroup_multiply_accumulate`: `a * b + c`. -pub fn simdgroup_matrix_8x8_multiply_accumulate(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, result_type: *const mlir.Type, opts: FloatOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: [1]mlir.NamedAttribute = undefined; - return mlir.Operation.make(ctx, "air.simdgroup_matrix_8x8_multiply_accumulate", .{ - .operands = .{ .flat = &.{ a, b, c_ } }, - .results = .{ .flat = &.{result_type} }, - .attributes = fastmathAttrs(&attrs, ctx, opts.fastmath), - .location = location, - }); -} diff --git a/mlir/dialects/air/capi.cc b/mlir/dialects/air/capi.cc deleted file mode 100644 index 64df5d20..00000000 --- a/mlir/dialects/air/capi.cc +++ /dev/null @@ -1,189 +0,0 @@ -//===- capi.cc - C API for the Metal AIR dialect --------------------------===// - -#include "capi.h" - -#include "mlir/dialects/air/AIRAttributes.h" -#include "mlir/dialects/air/AIRDialect.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" -#include "mlir/CAPI/Support.h" -#include "mlir/IR/Location.h" -#include "llvm/ADT/SmallVector.h" - -#include - -using namespace mlir; -using namespace mlir::metal::air; - -// Registration - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(MetalAIR, air, AIRDialect) - -static MLIRContext *loadAIR(MlirContext context) { - MLIRContext *ctx = unwrap(context); - if (ctx) - ctx->getOrLoadDialect(); - return ctx; -} - -static MlirAttribute invalid(MLIRContext *ctx, const Twine &message) { - emitError(UnknownLoc::get(ctx), message); - return {nullptr}; -} - -// Attributes - -bool mlirAttributeIsAMetalAIRBuiltin(MlirAttribute attr) { - return isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirMetalAIRBuiltinAttrGet(MlirContext context, uint32_t value) { - MLIRContext *ctx = loadAIR(context); - if (!ctx) - return {nullptr}; - std::optional builtin = symbolizeBuiltin(value); - if (!builtin) - return invalid(ctx, "unknown AIR builtin " + Twine(value)); - return wrap(BuiltinAttr::get(ctx, *builtin)); -} - -uint32_t mlirMetalAIRBuiltinAttrGetValue(MlirAttribute attr) { - return static_cast(cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAMetalAIRMemFlags(MlirAttribute attr) { - return isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirMetalAIRMemFlagsAttrGet(MlirContext context, uint32_t value) { - MLIRContext *ctx = loadAIR(context); - if (!ctx) - return {nullptr}; - std::optional flags = symbolizeMemFlags(value); - if (!flags) - return invalid(ctx, "unknown AIR memory flags " + Twine(value)); - return wrap(MemFlagsAttr::get(ctx, *flags)); -} - -uint32_t mlirMetalAIRMemFlagsAttrGetValue(MlirAttribute attr) { - return static_cast(cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAMetalAIRStructField(MlirAttribute attr) { - return isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirMetalAIRStructFieldAttrGet(MlirContext context, - uint64_t offset, uint64_t size, - uint64_t count, - MlirStringRef typeName, - MlirStringRef name) { - MLIRContext *ctx = loadAIR(context); - if (!ctx) - return {nullptr}; - return wrap(StructFieldAttr::getChecked(UnknownLoc::get(ctx), ctx, offset, - size, count, unwrap(typeName), - unwrap(name))); -} - -uint64_t mlirMetalAIRStructFieldAttrGetOffset(MlirAttribute attr) { - return cast(unwrap(attr)).getOffset(); -} - -uint64_t mlirMetalAIRStructFieldAttrGetSize(MlirAttribute attr) { - return cast(unwrap(attr)).getSize(); -} - -uint64_t mlirMetalAIRStructFieldAttrGetCount(MlirAttribute attr) { - return cast(unwrap(attr)).getCount(); -} - -MlirStringRef mlirMetalAIRStructFieldAttrGetTypeName(MlirAttribute attr) { - return wrap(cast(unwrap(attr)).getTypeName()); -} - -MlirStringRef mlirMetalAIRStructFieldAttrGetName(MlirAttribute attr) { - return wrap(cast(unwrap(attr)).getName()); -} - -bool mlirAttributeIsAMetalAIRBuffer(MlirAttribute attr) { - return isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirMetalAIRBufferAttrGet( - MlirContext context, uint32_t locationIndex, uint32_t addressSpace, - uint32_t access, MlirType elementType, MlirStringRef typeName, - uint64_t typeSize, uint64_t typeAlignment, uint64_t bufferSize, - bool noalias, intptr_t numFields, MlirAttribute const *fields) { - MLIRContext *ctx = loadAIR(context); - if (!ctx) - return {nullptr}; - std::optional space = symbolizeAddressSpace(addressSpace); - std::optional mode = symbolizeAccess(access); - if (!space || !mode) - return invalid(ctx, "unknown AIR buffer address space or access"); - Type element = unwrap(elementType); - if (!element || element.getContext() != ctx) - return invalid(ctx, - "buffer element type must belong to the attribute context"); - if (numFields < 0 || (numFields > 0 && !fields)) - return invalid(ctx, "invalid AIR struct fields"); - SmallVector structFields; - for (intptr_t i = 0; i < numFields; ++i) { - auto field = dyn_cast_or_null(unwrap(fields[i])); - if (!field || field.getContext() != ctx) - return invalid(ctx, "AIR struct fields must be #air.struct_field of " - "the attribute context"); - structFields.push_back(field); - } - return wrap(BufferAttr::getChecked( - UnknownLoc::get(ctx), ctx, locationIndex, *space, *mode, element, - unwrap(typeName), typeSize, typeAlignment, bufferSize, noalias, - ArrayRef(structFields))); -} - -uint32_t mlirMetalAIRBufferAttrGetLocationIndex(MlirAttribute attr) { - return cast(unwrap(attr)).getLocationIndex(); -} - -uint32_t mlirMetalAIRBufferAttrGetAddressSpace(MlirAttribute attr) { - return static_cast( - cast(unwrap(attr)).getAddressSpace()); -} - -uint32_t mlirMetalAIRBufferAttrGetAccess(MlirAttribute attr) { - return static_cast(cast(unwrap(attr)).getAccess()); -} - -MlirType mlirMetalAIRBufferAttrGetElementType(MlirAttribute attr) { - return wrap(cast(unwrap(attr)).getElementType()); -} - -MlirStringRef mlirMetalAIRBufferAttrGetTypeName(MlirAttribute attr) { - return wrap(cast(unwrap(attr)).getTypeName()); -} - -uint64_t mlirMetalAIRBufferAttrGetTypeSize(MlirAttribute attr) { - return cast(unwrap(attr)).getTypeSize(); -} - -uint64_t mlirMetalAIRBufferAttrGetTypeAlignment(MlirAttribute attr) { - return cast(unwrap(attr)).getTypeAlignment(); -} - -uint64_t mlirMetalAIRBufferAttrGetBufferSize(MlirAttribute attr) { - return cast(unwrap(attr)).getBufferSize(); -} - -bool mlirMetalAIRBufferAttrGetNoalias(MlirAttribute attr) { - return cast(unwrap(attr)).getNoalias(); -} - -intptr_t mlirMetalAIRBufferAttrGetNumFields(MlirAttribute attr) { - return cast(unwrap(attr)).getFields().size(); -} - -MlirAttribute mlirMetalAIRBufferAttrGetField(MlirAttribute attr, intptr_t pos) { - return wrap(cast(unwrap(attr)).getFields()[pos]); -} diff --git a/mlir/dialects/air/capi.h b/mlir/dialects/air/capi.h deleted file mode 100644 index f91b9f64..00000000 --- a/mlir/dialects/air/capi.h +++ /dev/null @@ -1,81 +0,0 @@ -//===- capi.h - C API for the Metal AIR dialect -------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_AIR_CAPI_H -#define MLIR_DIALECTS_AIR_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Registration - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(MetalAIR, air); - -// Attributes. Invalid parameters emit a diagnostic and return a null handle; -// typed getters require a handle of their attribute kind. - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRBuiltin(MlirAttribute attr); -/// `value` is a `mlir::metal::air::Builtin`. -MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRBuiltinAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirMetalAIRBuiltinAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRMemFlags(MlirAttribute attr); -/// `value` is a `mlir::metal::air::MemFlags` bit set. -MLIR_CAPI_EXPORTED MlirAttribute -mlirMetalAIRMemFlagsAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirMetalAIRMemFlagsAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRStructField(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRStructFieldAttrGet( - MlirContext context, uint64_t offset, uint64_t size, uint64_t count, - MlirStringRef typeName, MlirStringRef name); -MLIR_CAPI_EXPORTED uint64_t -mlirMetalAIRStructFieldAttrGetOffset(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint64_t -mlirMetalAIRStructFieldAttrGetSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint64_t -mlirMetalAIRStructFieldAttrGetCount(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirMetalAIRStructFieldAttrGetTypeName(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirMetalAIRStructFieldAttrGetName(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAMetalAIRBuffer(MlirAttribute attr); -/// `addressSpace` is a `mlir::metal::air::AddressSpace` and `access` a -/// `mlir::metal::air::Access`. Sizes are bytes; a zero `bufferSize` omits it. -/// `elementType` and `fields` must belong to `context`. -MLIR_CAPI_EXPORTED MlirAttribute mlirMetalAIRBufferAttrGet( - MlirContext context, uint32_t locationIndex, uint32_t addressSpace, - uint32_t access, MlirType elementType, MlirStringRef typeName, - uint64_t typeSize, uint64_t typeAlignment, uint64_t bufferSize, - bool noalias, intptr_t numFields, MlirAttribute const *fields); -MLIR_CAPI_EXPORTED uint32_t -mlirMetalAIRBufferAttrGetLocationIndex(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t -mlirMetalAIRBufferAttrGetAddressSpace(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t mlirMetalAIRBufferAttrGetAccess(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirType -mlirMetalAIRBufferAttrGetElementType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirMetalAIRBufferAttrGetTypeName(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint64_t -mlirMetalAIRBufferAttrGetTypeSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint64_t -mlirMetalAIRBufferAttrGetTypeAlignment(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint64_t -mlirMetalAIRBufferAttrGetBufferSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool mlirMetalAIRBufferAttrGetNoalias(MlirAttribute attr); -MLIR_CAPI_EXPORTED intptr_t -mlirMetalAIRBufferAttrGetNumFields(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirMetalAIRBufferAttrGetField(MlirAttribute attr, intptr_t pos); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_AIR_CAPI_H diff --git a/mlir/dialects/air/test.zig b/mlir/dialects/air/test.zig deleted file mode 100644 index 57f0427e..00000000 --- a/mlir/dialects/air/test.zig +++ /dev/null @@ -1,439 +0,0 @@ -const std = @import("std"); - -const air = @import("mlir/dialects/air"); -const arith = @import("mlir/dialects/arith"); -const func = @import("mlir/dialects/func"); -const llvm = @import("mlir/dialects/llvm"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "air.abs.s", - "air.ceil", - "air.clamp", - "air.clamp.s", - "air.clamp.u", - "air.convert", - "air.convert.s", - "air.convert.u", - "air.cos", - "air.dot", - "air.exp", - "air.exp2", - "air.fabs", - "air.fast_ceil", - "air.fast_clamp", - "air.fast_cos", - "air.fast_exp", - "air.fast_exp2", - "air.fast_fabs", - "air.fast_floor", - "air.fast_fmax", - "air.fast_fmin", - "air.fast_fmod", - "air.fast_log", - "air.fast_log2", - "air.fast_pow", - "air.fast_round", - "air.fast_rsqrt", - "air.fast_sin", - "air.fast_sqrt", - "air.fast_tanh", - "air.fast_trunc", - "air.floor", - "air.fma", - "air.fmax", - "air.fmin", - "air.fmod", - "air.log", - "air.log2", - "air.max.s", - "air.max.u", - "air.min.s", - "air.min.u", - "air.pow", - "air.round", - "air.rsqrt", - "air.simd_all", - "air.simd_any", - "air.simd_broadcast", - "air.simd_broadcast.s", - "air.simd_broadcast.u", - "air.simd_max", - "air.simd_max.s", - "air.simd_max.u", - "air.simd_min", - "air.simd_min.s", - "air.simd_min.u", - "air.simd_prefix_exclusive_sum", - "air.simd_shuffle", - "air.simd_shuffle.s", - "air.simd_shuffle.u", - "air.simd_shuffle_down", - "air.simd_shuffle_down.s", - "air.simd_shuffle_down.u", - "air.simd_shuffle_xor", - "air.simd_shuffle_xor.s", - "air.simd_shuffle_xor.u", - "air.simd_sum", - "air.simd_sum.s", - "air.simd_sum.u", - "air.simdgroup.barrier", - "air.simdgroup_matrix_8x8_load", - "air.simdgroup_matrix_8x8_multiply_accumulate", - "air.simdgroup_matrix_8x8_store", - "air.sin", - "air.sqrt", - "air.tanh", - "air.trunc", - "air.wg.barrier", -}; - -test { - std.testing.refAllDecls(air); - inline for (.{ air.BuiltinAttr, air.MemFlagsAttr, air.StructFieldAttr, air.BufferAttr }) |T| std.testing.refAllDecls(T); -} - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - inline for (.{ air, arith, func, llvm }) |dialect| dialect.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -fn expectRoundTrip(ctx: *mlir.Context, value: anytype) !void { - var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer writer.deinit(); - try writer.writer.print("{f}", .{value}); - try std.testing.expect((try mlir.Attribute.parse(ctx, writer.written())).eql(value.attribute())); -} - -test "every bound operation is registered" { - const ctx = try testContext(); - defer ctx.deinit(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} - -test "AIR attributes use the C API" { - const ctx = try testContext(); - defer ctx.deinit(); - inline for (std.meta.tags(air.Builtin)) |builtin| { - const attr = try air.BuiltinAttr.get(ctx, .{ .value = builtin }); - try std.testing.expectEqual(builtin, attr.value()); - try expectRoundTrip(ctx, attr); - } - for ([_]air.MemFlags{ .none, .{ .device = true }, .{ .threadgroup = true }, .{ .device = true, .threadgroup = true, .texture = true } }) |flags| { - const attr = try air.MemFlagsAttr.get(ctx, .{ .value = flags }); - try std.testing.expectEqual(flags, attr.value()); - try expectRoundTrip(ctx, attr); - } - try std.testing.expectError(error.InvalidMlir, air.MemFlagsAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); - - const field = try air.StructFieldAttr.get(ctx, .{ .offset = 4, .size = 4, .count = 3, .typeName = "float", .name = "s" }); - try std.testing.expectEqual(4, field.offset()); - try std.testing.expectEqual(4, field.size()); - try std.testing.expectEqual(3, field.count()); - try std.testing.expectEqualStrings("float", field.typeName()); - try std.testing.expectEqualStrings("s", field.name()); - try expectRoundTrip(ctx, field); - try std.testing.expectError(error.InvalidMlir, air.StructFieldAttr.get(ctx, .{ .offset = 0, .size = 0, .typeName = "int", .name = "n" })); - - const i32_type: *const mlir.Type = .int(ctx, .i32); - const f32_type: *const mlir.Type = .float(ctx, .f32); - const device: air.BufferAttr.InitArgs = .{ - .locationIndex = 1, - .addressSpace = .device, - .access = .read, - .elementType = .vector(&.{4}, .float(ctx, .bf16)), - .typeName = "bfloat4", - .typeSize = 8, - .typeAlignment = 8, - .noalias_ = true, - }; - const buffer = try air.BufferAttr.get(ctx, device); - try std.testing.expectEqual(1, buffer.locationIndex()); - try std.testing.expectEqual(air.AddressSpace.device, buffer.addressSpace()); - try std.testing.expectEqual(air.Access.read, buffer.access()); - try std.testing.expect(buffer.elementType().eql(device.elementType)); - try std.testing.expectEqualStrings("bfloat4", buffer.typeName()); - try std.testing.expectEqual(8, buffer.typeSize()); - try std.testing.expectEqual(8, buffer.typeAlignment()); - try std.testing.expectEqual(0, buffer.bufferSize()); - try std.testing.expect(buffer.noalias_()); - try std.testing.expectEqual(0, buffer.numFields()); - try expectRoundTrip(ctx, buffer); - - // `struct A { int n; float s[3]; }`, as Apple's compiler describes it. - const fields = [_]*const air.StructFieldAttr{ - try air.StructFieldAttr.get(ctx, .{ .offset = 0, .size = 4, .typeName = "int", .name = "n" }), - field, - }; - const constant: air.BufferAttr.InitArgs = .{ - .locationIndex = 0, - .addressSpace = .constant, - .access = .read, - .elementType = llvm.structType(ctx, &.{ i32_type, llvm.arrayType(f32_type, 3) }), - .typeName = "A", - .typeSize = 16, - .typeAlignment = 4, - .bufferSize = 16, - .fields = &fields, - }; - const structured = try air.BufferAttr.get(ctx, constant); - try std.testing.expectEqual(2, structured.numFields()); - try std.testing.expect(structured.field(1).eql(field)); - try expectRoundTrip(ctx, structured); - - var invalid = device; - invalid.typeSize = 16; - try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); - invalid = device; - invalid.addressSpace = .constant; - invalid.access = .read_write; - try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); - invalid = device; - invalid.addressSpace = .thread; - try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); - invalid = device; - invalid.addressSpace = .threadgroup; - try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); - invalid = device; - invalid.bufferSize = 8; - try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); - invalid = device; - invalid.fields = fields[0..1]; - try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); - invalid = constant; - invalid.fields = fields[0..1]; - try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); - invalid = constant; - invalid.addressSpace = .device; - try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(ctx, invalid)); - - const other = try testContext(); - defer other.deinit(); - try std.testing.expectError(error.InvalidMlir, air.BufferAttr.get(other, device)); -} - -test "all AIR operations construct and verify" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc: *const mlir.Location = .unknown(ctx); - const f32x4: *const mlir.Type = .vector(&.{4}, .float(ctx, .f32)); - const i32x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i32)); - const i64x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i64)); - const block = mlir.Block.init( - &.{ .float(ctx, .f32), .float(ctx, .f16), f32x4, .int(ctx, .i32), i32x2, .int(ctx, .i16), .int(ctx, .i1), llvm.pointerType(ctx, air.AddressSpace.threadgroup), i64x2, .vector(&.{64}, .float(ctx, .f16)), .vector(&.{64}, .float(ctx, .f32)), .vector(&.{4}, .float(ctx, .bf16)) }, - &.{ loc, loc, loc, loc, loc, loc, loc, loc, loc, loc, loc, loc }, - ); - defer block.deinit(); - const f = block.argument(0); - const h = block.argument(1); - const v = block.argument(2); - const i = block.argument(3); - const iv = block.argument(4); - const lane = block.argument(5); - const b = block.argument(6); - const fast: air.FloatOpts = .{ .fastmath = .fast }; - var ops: std.ArrayList(*mlir.Operation) = .empty; - defer ops.deinit(std.testing.allocator); - inline for (.{ air.exp, air.exp2, air.log, air.log2, air.tanh, air.sqrt, air.rsqrt, air.fabs, air.ceil, air.floor, air.round, air.trunc, air.sin, air.cos, air.simd_sum, air.simd_max, air.simd_min, air.simd_prefix_exclusive_sum }) |op| { - try ops.append(std.testing.allocator, op(ctx, h, .{}, loc).appendTo(block)); - try ops.append(std.testing.allocator, op(ctx, v, fast, loc).appendTo(block)); - } - inline for (.{ air.fast_exp, air.fast_exp2, air.fast_log, air.fast_log2, air.fast_tanh, air.fast_sqrt, air.fast_rsqrt, air.fast_fabs, air.fast_ceil, air.fast_floor, air.fast_round, air.fast_trunc, air.fast_sin, air.fast_cos }) |op| { - try ops.append(std.testing.allocator, op(ctx, f, fast, loc).appendTo(block)); - try ops.append(std.testing.allocator, op(ctx, v, fast, loc).appendTo(block)); - } - inline for (.{ air.fmax, air.fmin, air.pow, air.fmod, air.fast_fmax, air.fast_fmin, air.fast_pow, air.fast_fmod }) |op| { - try ops.append(std.testing.allocator, op(ctx, f, f, fast, loc).appendTo(block)); - } - inline for (.{ air.fma, air.clamp, air.fast_clamp }) |op| { - try ops.append(std.testing.allocator, op(ctx, v, v, v, fast, loc).appendTo(block)); - } - try ops.append(std.testing.allocator, air.dot(ctx, v, v, fast, loc).appendTo(block)); - try ops.append(std.testing.allocator, air.abs_s(ctx, i, loc).appendTo(block)); - inline for (.{ air.max_s, air.max_u, air.min_s, air.min_u }) |op| { - try ops.append(std.testing.allocator, op(ctx, iv, iv, loc).appendTo(block)); - } - inline for (.{ air.clamp_s, air.clamp_u }) |op| { - try ops.append(std.testing.allocator, op(ctx, i, i, i, loc).appendTo(block)); - } - inline for (.{ air.simd_sum_s, air.simd_sum_u, air.simd_max_s, air.simd_max_u, air.simd_min_s, air.simd_min_u }) |op| { - try ops.append(std.testing.allocator, op(ctx, i, loc).appendTo(block)); - try ops.append(std.testing.allocator, op(ctx, lane, loc).appendTo(block)); - } - inline for (.{ air.simd_shuffle, air.simd_shuffle_xor, air.simd_shuffle_down, air.simd_broadcast }) |op| { - try ops.append(std.testing.allocator, op(ctx, v, lane, fast, loc).appendTo(block)); - } - inline for (.{ air.simd_shuffle_s, air.simd_shuffle_u, air.simd_shuffle_xor_s, air.simd_shuffle_xor_u, air.simd_shuffle_down_s, air.simd_shuffle_down_u, air.simd_broadcast_s, air.simd_broadcast_u }) |op| { - try ops.append(std.testing.allocator, op(ctx, iv, lane, loc).appendTo(block)); - } - inline for (.{ air.simd_any, air.simd_all }) |op| { - try ops.append(std.testing.allocator, op(ctx, b, loc).appendTo(block)); - } - try ops.append(std.testing.allocator, air.convert(ctx, block.argument(11), f32x4, fast, loc).appendTo(block)); - try ops.append(std.testing.allocator, air.convert(ctx, h, .float(ctx, .bf16), .{}, loc).appendTo(block)); - try ops.append(std.testing.allocator, air.convert_s(ctx, i, .float(ctx, .f32), fast, loc).appendTo(block)); - try ops.append(std.testing.allocator, air.convert_u(ctx, f, .int(ctx, .i32), .{}, loc).appendTo(block)); - inline for (.{ air.wg_barrier, air.simdgroup_barrier }) |op| { - try ops.append(std.testing.allocator, op(ctx, .{ .threadgroup = true }, loc).appendTo(block)); - } - const shape = block.argument(8); - const matrix = air.simdgroup_matrix_8x8_load(ctx, block.argument(7), shape, shape, shape, .vector(&.{64}, .float(ctx, .f16)), fast, loc).appendTo(block); - const product = air.simdgroup_matrix_8x8_multiply_accumulate(ctx, matrix.result(0), block.argument(9), block.argument(10), .vector(&.{64}, .float(ctx, .f32)), fast, loc).appendTo(block); - const stored = air.simdgroup_matrix_8x8_store(ctx, product.result(0), block.argument(7), shape, shape, shape, loc).appendTo(block); - try ops.appendSlice(std.testing.allocator, &.{ matrix, product, stored }); - for (ops.items) |op| try std.testing.expect(op.verify()); -} - -test "AIR operations round trip through their custom assembly" { - const ctx = try testContext(); - defer ctx.deinit(); - const source = - \\func.func @ops(%f: f32, %v: vector<4xbf16>, %i: i32, %s: i16, %b: i1, %p: !llvm.ptr<3>, %d: vector<2xi64>, %m: vector<64xf32>) { - \\ %0 = air.fast_exp %f fastmath : f32 - \\ %1 = air.tanh %f : f32 - \\ %2 = air.fma %f, %f, %f fastmath : f32 - \\ %3 = air.max.s %i, %i : i32 - \\ %4 = air.simd_shuffle_xor %f, %s fastmath : f32 - \\ %5 = air.simd_sum.u %i : i32 - \\ %6 = air.simd_any %b - \\ %7 = air.convert %v fastmath : vector<4xbf16> to vector<4xf32> - \\ %8 = air.convert.s %i fastmath : i32 to f32 - \\ %9 = air.dot %7, %7 fastmath : vector<4xf32> -> f32 - \\ air.wg.barrier - \\ air.simdgroup.barrier - \\ %10 = air.simdgroup_matrix_8x8_load %p, %d, %d, %d fastmath : !llvm.ptr<3> -> vector<64xf32> - \\ %11 = air.simdgroup_matrix_8x8_multiply_accumulate %10, %m, %m fastmath : vector<64xf32>, vector<64xf32>, vector<64xf32> -> vector<64xf32> - \\ air.simdgroup_matrix_8x8_store %11, %p, %d, %d, %d : vector<64xf32>, !llvm.ptr<3> - \\ return - \\} - ; - const module = try mlir.Module.parse(ctx, source); - defer module.deinit(); - var first: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer first.deinit(); - try first.writer.print("{f}", .{module.operation()}); - try std.testing.expect(std.mem.find(u8, first.written(), "\"air.") == null); - const reparsed = try mlir.Module.parse(ctx, first.written()); - defer reparsed.deinit(); - var second: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer second.deinit(); - try second.writer.print("{f}", .{reparsed.operation()}); - try std.testing.expectEqualStrings(first.written(), second.written()); -} - -test "AIR verifier rejects types Apple's compiler has no intrinsic for" { - const ctx = try testContext(); - defer ctx.deinit(); - const rejected = [_][]const u8{ - "func.func @f(%x: f16) { %0 = air.fast_exp %x : f16\n return }", - "func.func @f(%x: bf16) { %0 = air.exp %x : bf16\n return }", - "func.func @f(%x: f32) { %0 = air.max.s %x, %x : f32\n return }", - "func.func @f(%x: i64) { %0 = air.simd_sum.s %x : i64\n return }", - "func.func @f(%x: vector<8xf32>) { %0 = air.simd_sum %x : vector<8xf32>\n return }", - "func.func @f(%x: f32) { %0 = air.convert %x : f32 to f32\n return }", - "func.func @f(%x: i32) { %0 = air.convert %x : i32 to f32\n return }", - "func.func @f(%x: f32) { %0 = air.convert.s %x : f32 to f16\n return }", - "func.func @f(%x: f32) { %0 = air.convert.u %x fastmath : f32 to i32\n return }", - "func.func @f(%x: vector<4xf32>) { %0 = air.convert %x : vector<4xf32> to vector<2xf16>\n return }", - "func.func @f(%x: vector<4xf32>) { %0 = air.dot %x, %x : vector<4xf32> -> f16\n return }", - "func.func @f(%p: !llvm.ptr<2>, %d: vector<2xi64>) { %0 = air.simdgroup_matrix_8x8_load %p, %d, %d, %d : !llvm.ptr<2> -> vector<64xf32>\n return }", - "func.func @f(%a: vector<64xbf16>, %b: vector<64xf16>, %c: vector<64xf32>) { %0 = air.simdgroup_matrix_8x8_multiply_accumulate %a, %b, %c : vector<64xbf16>, vector<64xf16>, vector<64xf32> -> vector<64xf32>\n return }", - "func.func @f(%a: vector<64xf16>, %c: vector<64xf32>) { %0 = air.simdgroup_matrix_8x8_multiply_accumulate %a, %a, %c : vector<64xf16>, vector<64xf16>, vector<64xf32> -> vector<64xf16>\n return }", - }; - for (rejected) |source| try std.testing.expectError(error.InvalidMlir, mlir.Module.parse(ctx, source)); -} - -test "AIR verifier takes the 8-bit integers Apple's compiler emits for char" { - const ctx = try testContext(); - defer ctx.deinit(); - const module = try mlir.Module.parse(ctx, - \\func.func @f(%c: i8, %v: vector<2xi8>, %f: f32, %w: vector<2xf32>, %l: i16) { - \\ %0 = air.convert.s %c fastmath : i8 to f32 - \\ %1 = air.convert.u %f : f32 to i8 - \\ %2 = air.convert.s %v fastmath : vector<2xi8> to vector<2xf32> - \\ %3 = air.convert.s %w : vector<2xf32> to vector<2xi8> - \\ %4 = air.max.s %c, %c : i8 - \\ %5 = air.min.u %v, %v : vector<2xi8> - \\ %6 = air.abs.s %c : i8 - \\ %7 = air.clamp.u %c, %c, %c : i8 - \\ %8 = air.simd_shuffle.s %c, %l : i8 - \\ %9 = air.simd_sum.s %c : i8 - \\ %10 = air.simd_max.u %c : i8 - \\ return - \\} - ); - module.deinit(); -} - -test "AIR kernel ABI" { - const ctx = try testContext(); - defer ctx.deinit(); - const device = "#air.buffer"; - const threadgroup = "#air.buffer"; - const accepted = [_][]const u8{ - "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ ", air.arg_name = \"a\"}, %b: !llvm.ptr<3> {air.buffer = " ++ threadgroup ++ "}, %t: vector<3xi32> {air.builtin = #air.builtin}, %l: i16 {air.builtin = #air.builtin}, %s: i32 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", - "module attributes {air.fast_math_enable} {}", - }; - const rejected = [_][]const u8{ - "func.func @k(%a: i32) attributes {air.kernel} { return }", - "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ ", air.builtin = #air.builtin}) attributes {air.kernel} { return }", - "func.func @k(%a: !llvm.ptr<3> {air.buffer = " ++ device ++ "}) attributes {air.kernel} { return }", - "func.func @k(%a: !llvm.ptr<1> {air.buffer = " ++ device ++ "}, %b: !llvm.ptr<1> {air.buffer = " ++ device ++ "}) attributes {air.kernel} { return }", - "func.func @k(%a: vector<3xi32> {air.builtin = #air.builtin}) attributes {air.kernel} { return }", - "func.func @k(%a: vector<4xi32> {air.builtin = #air.builtin}) attributes {air.kernel} { return }", - "func.func @k(%a: i64 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", - "func.func @k(%a: i32 {air.builtin = #air.builtin}, %b: i32 {air.builtin = #air.builtin}) attributes {air.kernel} { return }", - "func.func @k(%a: i32 {air.builtin = #air.builtin, air.arg_name = \"\"}) attributes {air.kernel} { return }", - "func.func @k() -> i32 attributes {air.kernel} { %c = arith.constant 0 : i32\n return %c : i32 }", - "func.func @h(%a: i32 {air.builtin = #air.builtin}) { return }", - "func.func @h() attributes {air.fast_math_enable} { return }", - "func.func @h() attributes {air.unknown} { return }", - }; - for (accepted) |source| { - const module = try mlir.Module.parse(ctx, source); - module.deinit(); - } - for (rejected) |source| try std.testing.expectError(error.InvalidMlir, mlir.Module.parse(ctx, source)); -} - -test "AIR kernel builder" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc: *const mlir.Location = .unknown(ctx); - const module: *mlir.Module = .init(loc); - defer module.deinit(); - const block = mlir.Block.init(&.{ llvm.pointerType(ctx, air.AddressSpace.device), .vector(&.{3}, .int(ctx, .i32)) }, &.{ loc, loc }); - _ = air.wg_barrier(ctx, .{ .threadgroup = true }, loc).appendTo(block); - _ = func.returns(ctx, &.{}, loc).appendTo(block); - _ = air.kernel(ctx, .{ - .name = "example", - .block = block, - .location = loc, - .args_attributes = &.{ - try air.bufferArgumentAttr(ctx, .{ - .locationIndex = 0, - .addressSpace = .device, - .access = .read_write, - .elementType = .float(ctx, .f32), - .typeName = "float", - .typeSize = 4, - .typeAlignment = 4, - }, .{ .name = "out" }), - try air.builtinArgumentAttr(ctx, .thread_position_in_grid, .{}), - }, - }).appendTo(module.body()); - air.enableFastMath(ctx, module.operation()); - try std.testing.expect(module.operation().verify()); - var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer writer.deinit(); - try writer.writer.print("{f}", .{module.operation()}); - const parsed = try mlir.Module.parse(ctx, writer.written()); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); - try std.testing.expect(std.mem.find(u8, writer.written(), "air.arg_name = \"out\"") != null); -} diff --git a/mlir/dialects/arith/BUILD.bazel b/mlir/dialects/arith/BUILD.bazel deleted file mode 100644 index 90c84ff0..00000000 --- a/mlir/dialects/arith/BUILD.bazel +++ /dev/null @@ -1,36 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:CAPIIR", - ], -) - -zig_library( - name = "arith", - import_name = "mlir/dialects/arith", - main = "arith.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//stdx", - "@llvm-project//mlir:CAPIArith", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":arith", - "//mlir", - ], -) diff --git a/mlir/dialects/arith/arith.zig b/mlir/dialects/arith/arith.zig deleted file mode 100644 index c45ce5b5..00000000 --- a/mlir/dialects/arith/arith.zig +++ /dev/null @@ -1,495 +0,0 @@ -const c = @import("c"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("arith"); -} - -// ============================================================================= -// Predicate enums (match ArithBase.td exactly) -// ============================================================================= - -/// Arith cmpi predicates. Values match mlir::arith::CmpIPredicate (I64EnumAttr). -pub const CmpIPredicate = enum(i64) { - eq = 0, - ne = 1, - slt = 2, - sle = 3, - sgt = 4, - sge = 5, - ult = 6, - ule = 7, - ugt = 8, - uge = 9, -}; - -/// Arith cmpf predicates. Values match mlir::arith::CmpFPredicate. -pub const CmpFPredicate = enum(i64) { - false = 0, - oeq = 1, - ogt = 2, - oge = 3, - olt = 4, - ole = 5, - one = 6, - ord = 7, - ueq = 8, - ugt = 9, - uge = 10, - ult = 11, - ule = 12, - une = 13, - uno = 14, - true = 15, -}; - -// ============================================================================= -// Flag attributes -// ============================================================================= - -/// `mlir::arith::FastMathFlags`, one bit per flag. -pub const FastMathFlags = packed struct(u32) { - reassoc: bool = false, - nnan: bool = false, - ninf: bool = false, - nsz: bool = false, - arcp: bool = false, - contract: bool = false, - afn: bool = false, - _padding: u25 = 0, - - pub const none: FastMathFlags = .{}; - pub const fast: FastMathFlags = .{ .reassoc = true, .nnan = true, .ninf = true, .nsz = true, .arcp = true, .contract = true, .afn = true }; -}; - -/// `#arith.fastmath<...>`. -pub const FastMathAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAArithFastMath; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "fastmath"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: FastMathFlags, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirArithFastMathAttrGet(ctx.ptr(), @bitCast(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) FastMathFlags { - return @bitCast(c.mlirArithFastMathAttrGetValue(self.ptr())); - } -}; - -/// `fastmath = #arith.fastmath`: the fast-math flags of a float -/// operation. -pub fn fastmath(ctx: *mlir.Context, flags: FastMathFlags) mlir.Error!mlir.NamedAttribute { - return .named(ctx, "fastmath", (try FastMathAttr.get(ctx, .{ .value = flags })).attribute()); -} - -/// `mlir::arith::IntegerOverflowFlags`, one bit per flag. -pub const IntegerOverflowFlags = packed struct(u32) { - nsw: bool = false, - nuw: bool = false, - _padding: u30 = 0, - - pub const none: IntegerOverflowFlags = .{}; -}; - -/// `#arith.overflow<...>`. -pub const IntegerOverflowAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAArithIntegerOverflow; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "overflow"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: IntegerOverflowFlags, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirArithIntegerOverflowAttrGet(ctx.ptr(), @bitCast(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) IntegerOverflowFlags { - return @bitCast(c.mlirArithIntegerOverflowAttrGetValue(self.ptr())); - } -}; - -/// `overflowFlags = #arith.overflow`: the overflow flags of an -/// integer operation. -pub fn overflowFlags(ctx: *mlir.Context, flags: IntegerOverflowFlags) mlir.Error!mlir.NamedAttribute { - return .named(ctx, "overflowFlags", (try IntegerOverflowAttr.get(ctx, .{ .value = flags })).attribute()); -} - -// ============================================================================= -// Constants -// ============================================================================= - -/// arith.constant for an integer scalar. `value` is reinterpreted to `int_type`'s bit width. -pub fn constant_int( - ctx: *mlir.Context, - value: i64, - int_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "arith.constant", .{ - .results = .{ .flat = &.{int_type} }, - .attributes = &.{.named(ctx, "value", .intFromType(@ptrCast(int_type), value))}, - .location = location, - }); -} - -/// arith.constant for a float scalar of the given FloatTypes kind. -pub fn constant_float( - ctx: *mlir.Context, - value: f64, - comptime ft: mlir.FloatTypes, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "arith.constant", .{ - .results = .{ .flat = &.{.float(ctx, ft)} }, - .attributes = &.{.named(ctx, "value", .float(ctx, ft, value))}, - .location = location, - }); -} - -/// arith.constant for an `index`-typed integer. -pub fn constant_index(ctx: *mlir.Context, value: i64, location: *const mlir.Location) *mlir.Operation { - const idx_ty = mlir.Type.index(ctx); - const attr: *const mlir.Attribute = @ptrCast(c.mlirIntegerAttrGet(idx_ty.ptr(), value).ptr); - return mlir.Operation.make(ctx, "arith.constant", .{ - .results = .{ .flat = &.{idx_ty} }, - .attributes = &.{.named(ctx, "value", attr)}, - .location = location, - }); -} - -// ============================================================================= -// Binary ops (generated from shared template) -// ============================================================================= - -fn binaryOp(comptime op_name: []const u8) type { - return struct { - pub fn call( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - location: *const mlir.Location, - ) *mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .location = location, - }); - } - }; -} - -// Integer arithmetic -pub const addi = binaryOp("arith.addi").call; -pub const subi = binaryOp("arith.subi").call; -pub const muli = binaryOp("arith.muli").call; -pub const divsi = binaryOp("arith.divsi").call; -pub const divui = binaryOp("arith.divui").call; -pub const remsi = binaryOp("arith.remsi").call; -pub const remui = binaryOp("arith.remui").call; - -// Integer bitwise / shifts -pub const andi = binaryOp("arith.andi").call; -pub const ori = binaryOp("arith.ori").call; -pub const xori = binaryOp("arith.xori").call; -pub const shli = binaryOp("arith.shli").call; -pub const shrsi = binaryOp("arith.shrsi").call; -pub const shrui = binaryOp("arith.shrui").call; - -// Integer min/max -pub const maxsi = binaryOp("arith.maxsi").call; -pub const maxui = binaryOp("arith.maxui").call; -pub const minsi = binaryOp("arith.minsi").call; -pub const minui = binaryOp("arith.minui").call; - -// Ceil/floor div -pub const ceildivsi = binaryOp("arith.ceildivsi").call; -pub const ceildivui = binaryOp("arith.ceildivui").call; -pub const floordivsi = binaryOp("arith.floordivsi").call; - -// Float arithmetic -pub const addf = binaryOp("arith.addf").call; -pub const subf = binaryOp("arith.subf").call; -pub const mulf = binaryOp("arith.mulf").call; -pub const divf = binaryOp("arith.divf").call; -pub const remf = binaryOp("arith.remf").call; - -// Float min/max -pub const maximumf = binaryOp("arith.maximumf").call; -pub const maxnumf = binaryOp("arith.maxnumf").call; -pub const minimumf = binaryOp("arith.minimumf").call; -pub const minnumf = binaryOp("arith.minnumf").call; - -// ============================================================================= -// Unary ops -// ============================================================================= - -pub fn negf(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "arith.negf", .{ - .operands = .{ .flat = &.{src} }, - .result_type_inference = true, - .location = location, - }); -} - -/// arith.flush_denormals — clamp subnormals to zero (SameOperandsAndResultType). -pub fn flush_denormals(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "arith.flush_denormals", .{ - .operands = .{ .flat = &.{src} }, - .result_type_inference = true, - .location = location, - }); -} - -/// arith.constant for a dense splat: builds `arith.constant dense : T` -/// where `T` is a ranked tensor type. Useful for tensor-typed zero/one constants. -/// For float splats, use `constant_dense_splat_f`; for ints, `constant_dense_splat_i`. -pub fn constant_dense_splat_f( - ctx: *mlir.Context, - value: f64, - tensor_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - const elem_c = c.mlirShapedTypeGetElementType(tensor_type.ptr()); - const elem_attr = c.mlirFloatAttrDoubleGet(ctx.ptr(), elem_c, value); - const dense_c = c.mlirDenseElementsAttrSplatGet(tensor_type.ptr(), elem_attr); - const dense_attr: *const mlir.Attribute = @ptrCast(dense_c.ptr); - return mlir.Operation.make(ctx, "arith.constant", .{ - .results = .{ .flat = &.{tensor_type} }, - .attributes = &.{.named(ctx, "value", dense_attr)}, - .location = location, - }); -} - -pub fn constant_dense_splat_i( - ctx: *mlir.Context, - value: i64, - tensor_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - const elem_c = c.mlirShapedTypeGetElementType(tensor_type.ptr()); - const elem_attr = c.mlirIntegerAttrGet(elem_c, value); - const dense_c = c.mlirDenseElementsAttrSplatGet(tensor_type.ptr(), elem_attr); - const dense_attr: *const mlir.Attribute = @ptrCast(dense_c.ptr); - return mlir.Operation.make(ctx, "arith.constant", .{ - .results = .{ .flat = &.{tensor_type} }, - .attributes = &.{.named(ctx, "value", dense_attr)}, - .location = location, - }); -} - -// ============================================================================= -// Extended-result binary ops — return two values -// ============================================================================= - -/// arith.addui_extended — unsigned add, returns (sum, overflow: i1 / i1-tensor). -/// The overflow type is typically i1 (scalar) or tensor<...xi1> matching lhs shape. -pub fn addui_extended( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - overflow_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "arith.addui_extended", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{ lhs.type_(), overflow_type } }, - .location = location, - }); -} - -/// arith.mulsi_extended — signed multiply, returns (low, high) both same type as lhs. -pub fn mulsi_extended( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "arith.mulsi_extended", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{ lhs.type_(), lhs.type_() } }, - .location = location, - }); -} - -/// arith.mului_extended — unsigned multiply, returns (low, high) both same type as lhs. -pub fn mului_extended( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "arith.mului_extended", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{ lhs.type_(), lhs.type_() } }, - .location = location, - }); -} - -// ============================================================================= -// Cast ops (1 operand, explicit result type) -// ============================================================================= - -fn castOp(comptime op_name: []const u8) type { - return struct { - pub fn call( - ctx: *mlir.Context, - value: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, - ) *mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = .{ .flat = &.{value} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); - } - }; -} - -pub const extsi = castOp("arith.extsi").call; -pub const extui = castOp("arith.extui").call; -pub const extf = castOp("arith.extf").call; -pub const trunci = castOp("arith.trunci").call; -pub const truncf = castOp("arith.truncf").call; -pub const sitofp = castOp("arith.sitofp").call; -pub const uitofp = castOp("arith.uitofp").call; -pub const fptosi = castOp("arith.fptosi").call; -pub const fptoui = castOp("arith.fptoui").call; -pub const bitcast = castOp("arith.bitcast").call; -pub const index_cast = castOp("arith.index_cast").call; -pub const index_castui = castOp("arith.index_castui").call; - -// ============================================================================= -// convertf — same-bitwidth float cast (e.g. f16↔bf16). Not expressible via -// arith.extf or arith.truncf. -// ============================================================================= - -/// Matches `Arith_RoundingModeAttr` (I32EnumAttr; see arith_dialect.td). -pub const RoundingMode = enum(i32) { - to_nearest_even = 0, - downward = 1, - upward = 2, - toward_zero = 3, - to_nearest_away = 4, -}; - -pub const ConvertFOpts = struct { - rounding: ?RoundingMode = null, -}; - -pub fn convertf( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - opts: ConvertFOpts, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); - } - return mlir.Operation.make(ctx, "arith.convertf", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// arith.scaling_truncf — OCP MXFP downcast with per-block scale factors. -pub fn scaling_truncf( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - result_type: *const mlir.Type, - opts: ConvertFOpts, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "roundingmode", .int(ctx, .i32, @backingInt(rm)))); - } - return mlir.Operation.make(ctx, "arith.scaling_truncf", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -// ============================================================================= -// Compare -// ============================================================================= - -pub fn cmpi( - ctx: *mlir.Context, - predicate: CmpIPredicate, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "arith.cmpi", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .attributes = &.{ - .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), - }, - .location = location, - }); -} - -pub fn cmpf( - ctx: *mlir.Context, - predicate: CmpFPredicate, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "arith.cmpf", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .attributes = &.{ - .named(ctx, "predicate", .int(ctx, .i64, @backingInt(predicate))), - }, - .location = location, - }); -} - -// ============================================================================= -// Select -// ============================================================================= - -/// arith.select — result type matches the true/false operand type. -pub fn select( - ctx: *mlir.Context, - cond: *const mlir.Value, - t: *const mlir.Value, - f: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "arith.select", .{ - .operands = .{ .flat = &.{ cond, t, f } }, - .results = .{ .flat = &.{t.type_()} }, - .location = location, - }); -} diff --git a/mlir/dialects/arith/capi.cc b/mlir/dialects/arith/capi.cc deleted file mode 100644 index 3d7359c2..00000000 --- a/mlir/dialects/arith/capi.cc +++ /dev/null @@ -1,47 +0,0 @@ -//===- capi.cc - C API for the arith dialect ------------------------------===// - -#include "capi.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/Dialect/Arith/IR/Arith.h" - -bool mlirAttributeIsAArithFastMath(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirArithFastMathAttrGet(MlirContext context, uint32_t value) { - mlir::MLIRContext *ctx = unwrap(context); - std::optional flags = - mlir::arith::symbolizeFastMathFlags(value); - if (!ctx || !flags) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::arith::FastMathFlagsAttr::get(ctx, *flags)); -} - -uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAArithIntegerOverflow(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} - -MlirAttribute mlirArithIntegerOverflowAttrGet(MlirContext context, - uint32_t value) { - mlir::MLIRContext *ctx = unwrap(context); - std::optional flags = - mlir::arith::symbolizeIntegerOverflowFlags(value); - if (!ctx || !flags) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::arith::IntegerOverflowFlagsAttr::get(ctx, *flags)); -} - -uint32_t mlirArithIntegerOverflowAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} diff --git a/mlir/dialects/arith/capi.h b/mlir/dialects/arith/capi.h deleted file mode 100644 index ec8962af..00000000 --- a/mlir/dialects/arith/capi.h +++ /dev/null @@ -1,33 +0,0 @@ -//===- capi.h - C API for the arith dialect -----------------------*- C -*-===// -// -// C entry points for the `arith` dialect's flag attributes, which upstream's -// C API leaves out. - -#ifndef MLIR_DIALECTS_ARITH_CAPI_H -#define MLIR_DIALECTS_ARITH_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// #arith.fastmath<...>: `value` is the `FastMathFlags` bit set; `Get` returns -// a null attribute when it has bits outside the enum. -MLIR_CAPI_EXPORTED bool mlirAttributeIsAArithFastMath(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirArithFastMathAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirArithFastMathAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool -mlirAttributeIsAArithIntegerOverflow(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirArithIntegerOverflowAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirArithIntegerOverflowAttrGetValue(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_ARITH_CAPI_H diff --git a/mlir/dialects/arith/test.zig b/mlir/dialects/arith/test.zig deleted file mode 100644 index 844898ca..00000000 --- a/mlir/dialects/arith/test.zig +++ /dev/null @@ -1,142 +0,0 @@ -const std = @import("std"); - -const arith = @import("mlir/dialects/arith"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "arith.addf", - "arith.addi", - "arith.addui_extended", - "arith.andi", - "arith.bitcast", - "arith.ceildivsi", - "arith.ceildivui", - "arith.cmpf", - "arith.cmpi", - "arith.constant", - "arith.convertf", - "arith.divf", - "arith.divsi", - "arith.divui", - "arith.extf", - "arith.extsi", - "arith.extui", - "arith.floordivsi", - "arith.flush_denormals", - "arith.fptosi", - "arith.fptoui", - "arith.index_cast", - "arith.index_castui", - "arith.maximumf", - "arith.maxnumf", - "arith.maxsi", - "arith.maxui", - "arith.minimumf", - "arith.minnumf", - "arith.minsi", - "arith.minui", - "arith.mulf", - "arith.muli", - "arith.mulsi_extended", - "arith.mului_extended", - "arith.negf", - "arith.ori", - "arith.remf", - "arith.remsi", - "arith.remui", - "arith.scaling_truncf", - "arith.select", - "arith.shli", - "arith.shrsi", - "arith.shrui", - "arith.sitofp", - "arith.subf", - "arith.subi", - "arith.truncf", - "arith.trunci", - "arith.uitofp", - "arith.xori", -}; - -fn expectPrints(expected: []const u8, value: anytype) !void { - var buf: [128]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(expected, w.buffered()); -} - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(arith); - inline for (comptime std.meta.declarations(arith)) |decl| { - const T = @field(arith, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "arith fastmath attribute is built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - const cases = [_]struct { arith.FastMathFlags, []const u8 }{ - .{ .none, "#arith.fastmath" }, - .{ .fast, "#arith.fastmath" }, - .{ .{ .reassoc = true }, "#arith.fastmath" }, - .{ .{ .nnan = true, .ninf = true, .afn = true }, "#arith.fastmath" }, - .{ .{ .nsz = true, .arcp = true, .contract = true }, "#arith.fastmath" }, - }; - for (cases) |case| { - const attr = try arith.FastMathAttr.get(ctx, .{ .value = case[0] }); - try std.testing.expectEqual(case[0], attr.value()); - try std.testing.expect(attr.attribute().isA(arith.FastMathAttr) != null); - try expectPrints(case[1], attr); - } - try std.testing.expectError(error.InvalidMlir, arith.FastMathAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); -} - -test "fastmath names the attribute of a float operation" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - var buf: [64]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{try arith.fastmath(ctx, .fast)})}); - try std.testing.expectEqualStrings("{fastmath = #arith.fastmath}", w.buffered()); -} - -test "overflowFlags names the attribute of an integer operation" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - var buf: [64]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{try arith.overflowFlags(ctx, .{ .nsw = true })})}); - try std.testing.expectEqualStrings("{overflowFlags = #arith.overflow}", w.buffered()); -} - -test "arith overflow attribute is built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - const cases = [_]struct { arith.IntegerOverflowFlags, []const u8 }{ - .{ .none, "#arith.overflow" }, - .{ .{ .nsw = true }, "#arith.overflow" }, - .{ .{ .nsw = true, .nuw = true }, "#arith.overflow" }, - }; - for (cases) |case| { - const attr = try arith.IntegerOverflowAttr.get(ctx, .{ .value = case[0] }); - try std.testing.expectEqual(case[0], attr.value()); - try std.testing.expect(attr.attribute().isA(arith.IntegerOverflowAttr) != null); - try expectPrints(case[1], attr); - } - try std.testing.expectError(error.InvalidMlir, arith.IntegerOverflowAttr.get(ctx, .{ .value = .{ ._padding = 1 } })); -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - arith.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/cf/cf.zig b/mlir/dialects/cf/cf.zig deleted file mode 100644 index 9a63297c..00000000 --- a/mlir/dialects/cf/cf.zig +++ /dev/null @@ -1,168 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("cf"); -} - -// ============================================================================= -// cf.assert -// ============================================================================= - -/// cf.assert %cond, "message" — aborts execution with `msg` when `cond` is -/// false. `cond` is i1. -pub fn assert( - ctx: *mlir.Context, - cond: *const mlir.Value, - msg: []const u8, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cf.assert", .{ - .operands = .{ .flat = &.{cond} }, - .attributes = &.{ - .named(ctx, "msg", .string(ctx, msg)), - }, - .location = location, - }); -} - -// ============================================================================= -// cf.br -// ============================================================================= - -/// cf.br ^dest(operands...) — unconditional branch. Terminator of its block. -/// The types of `dest_operands` must match the block arguments of `dest`. -pub fn br( - ctx: *mlir.Context, - dest: *mlir.Block, - dest_operands: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - _ = ctx; - var state: mlir.OperationState = .init("cf.br", location); - state.addOperands(dest_operands); - var successors: [1]*const mlir.Block = .{dest}; - state.addSuccessors(&successors); - return mlir.Operation.init(&state) catch @panic("Failed to create cf.br"); -} - -// ============================================================================= -// cf.cond_br -// ============================================================================= - -/// cf.cond_br %cond, ^trueDest(trueOps...), ^falseDest(falseOps...) — branch -/// on an i1. Terminator of its block. -/// -/// Operand layout (flat) is `[cond, *trueOperands, *falseOperands]` with an -/// `operandSegmentSizes = [1, |trueOperands|, |falseOperands|]` attribute per -/// the `AttrSizedOperandSegments` trait on the op. -/// -/// `branch_weights` (optional) lets profile-guided lowering bias the branch; -/// when provided, its length must be 2 (true weight, false weight). -pub fn cond_br( - ctx: *mlir.Context, - cond: *const mlir.Value, - true_dest: *mlir.Block, - true_operands: []const *const mlir.Value, - false_dest: *mlir.Block, - false_operands: []const *const mlir.Value, - opts: struct { - branch_weights: ?[]const i32 = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 128) = .empty; - operands_buf.appendAssumeCapacity(cond); - operands_buf.appendSliceAssumeCapacity(true_operands); - operands_buf.appendSliceAssumeCapacity(false_operands); - - const seg_sizes = [3]i32{ 1, @intCast(true_operands.len), @intCast(false_operands.len) }; - - var state: mlir.OperationState = .init("cf.cond_br", location); - state.addOperands(operands_buf.constSlice()); - var successors: [2]*const mlir.Block = .{ true_dest, false_dest }; - state.addSuccessors(&successors); - - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); - if (opts.branch_weights) |w| { - std.debug.assert(w.len == 2); - attrs.appendAssumeCapacity(.named(ctx, "branch_weights", .denseArray(ctx, .i32, w))); - } - state.addAttributes(attrs.constSlice()); - - return mlir.Operation.init(&state) catch @panic("Failed to create cf.cond_br"); -} - -// ============================================================================= -// cf.switch -// ============================================================================= - -/// cf.switch %flag : iN, [ default: ^def(defOps...), caseN: ^bbN(opsN...) ] -/// -/// `flag` is any integer type. `case_values` are matched signed; the matching -/// case's block is jumped to (with the corresponding operand segment -/// forwarded). If no case matches, `default_dest` is taken with -/// `default_operands`. -/// -/// Operand layout: `[flag, *defaultOperands, *caseOperands[0], ..., -/// *caseOperands[N-1]]`. `case_operand_segments[i]` carries the length of -/// each case's operand segment. The `AttrSizedOperandSegments` trait tracks -/// the top-level split between `defaultOperands` and the lumped -/// `caseOperands`. -pub fn switch_( - ctx: *mlir.Context, - flag: *const mlir.Value, - default_dest: *mlir.Block, - default_operands: []const *const mlir.Value, - case_values: []const i32, - case_dests: []const *mlir.Block, - case_operands: []const []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - std.debug.assert(case_values.len == case_dests.len); - std.debug.assert(case_values.len == case_operands.len); - - var operands_buf: stdx.BoundedArray(*const mlir.Value, 256) = .empty; - operands_buf.appendAssumeCapacity(flag); - operands_buf.appendSliceAssumeCapacity(default_operands); - - var case_seg_sizes: stdx.BoundedArray(i32, 64) = .empty; - var case_total: i32 = 0; - for (case_operands) |segment| { - operands_buf.appendSliceAssumeCapacity(segment); - case_seg_sizes.appendAssumeCapacity(@intCast(segment.len)); - case_total += @intCast(segment.len); - } - - // Top-level AttrSizedOperandSegments: [flag=1, defaultOperands, caseOperands lumped]. - const top_seg_sizes = [3]i32{ 1, @intCast(default_operands.len), case_total }; - - var state: mlir.OperationState = .init("cf.switch", location); - state.addOperands(operands_buf.constSlice()); - - var successors_buf: stdx.BoundedArray(*const mlir.Block, 64) = .empty; - successors_buf.appendAssumeCapacity(default_dest); - for (case_dests) |d| successors_buf.appendAssumeCapacity(d); - state.addSuccessors(@constCast(successors_buf.constSlice())); - - // case_values is stored as a DenseIntElementsAttr (tensor); - // the td uses `AnyIntElementsAttr`, and the parser prints decimal literals - // matching the flag's bit width. We use i32 for the common path (matches - // Python Triton's flag type for pid compares). - const shaped = mlir.RankedTensorType.get( - &.{@intCast(case_values.len)}, - .int(ctx, .i32), - null, - ).shaped(); - - state.addAttributes(&.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &top_seg_sizes)), - .named(ctx, "case_values", .denseElements(shaped, case_values)), - .named(ctx, "case_operand_segments", .denseArray(ctx, .i32, case_seg_sizes.constSlice())), - }); - - return mlir.Operation.init(&state) catch @panic("Failed to create cf.switch"); -} diff --git a/mlir/dialects/cf/test.zig b/mlir/dialects/cf/test.zig deleted file mode 100644 index 5a91f8f4..00000000 --- a/mlir/dialects/cf/test.zig +++ /dev/null @@ -1,31 +0,0 @@ -const std = @import("std"); - -const cf = @import("mlir/dialects/cf"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "cf.assert", - "cf.br", - "cf.cond_br", - "cf.switch", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(cf); - inline for (comptime std.meta.declarations(cf)) |decl| { - const T = @field(cf, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - cf.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/cuda/BUILD.bazel b/mlir/dialects/cuda/BUILD.bazel deleted file mode 100644 index bcfabc4a..00000000 --- a/mlir/dialects/cuda/BUILD.bazel +++ /dev/null @@ -1,193 +0,0 @@ -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - ":dialect", - ":headers", - "@llvm-project//mlir:CAPIIR", - ], -) - -zig_library( - name = "cuda", - srcs = [ - "ops.zig", - ], - import_name = "mlir/dialects/cuda", - main = "cuda.zig", - tags = ["manual"], - deps = [ - ":capi", - "//mlir", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":cuda", - "//mlir", - ], -) - -td_library( - name = "td_files", - srcs = [ - "CudaAttrs.td", - "CudaDialect.td", - "CudaEnums.td", - "CudaOps.td", - "CudaTypes.td", - ], - includes = ["/"], - deps = [ - "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - ], -) - -gentbl_cc_library( - name = "dialect_inc_gen", - tbl_outs = [ - ( - [ - "-gen-dialect-decls", - "-dialect=cuda", - ], - "CudaDialect.h.inc", - ), - ( - [ - "-gen-dialect-defs", - "-dialect=cuda", - ], - "CudaDialect.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CudaDialect.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "enums_inc_gen", - tbl_outs = [ - ( - ["-gen-enum-decls"], - "CudaEnums.h.inc", - ), - ( - ["-gen-enum-defs"], - "CudaEnums.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CudaEnums.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "attrs_inc_gen", - tbl_outs = [ - ( - [ - "-gen-attrdef-decls", - "-attrdefs-dialect=cuda", - ], - "CudaAttrs.h.inc", - ), - ( - [ - "-gen-attrdef-defs", - "-attrdefs-dialect=cuda", - ], - "CudaAttrs.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CudaAttrs.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "types_inc_gen", - tbl_outs = [ - ( - [ - "-gen-typedef-decls", - "-typedefs-dialect=cuda", - ], - "CudaTypes.h.inc", - ), - ( - [ - "-gen-typedef-defs", - "-typedefs-dialect=cuda", - ], - "CudaTypes.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CudaTypes.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "ops_inc_gen", - tbl_outs = [ - ( - ["-gen-op-decls"], - "CudaOps.h.inc", - ), - ( - ["-gen-op-defs"], - "CudaOps.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CudaOps.td", - deps = [":td_files"], -) - -cc_library( - name = "headers", - hdrs = ["CudaDialect.h"], - deps = [ - ":attrs_inc_gen", - ":dialect_inc_gen", - ":enums_inc_gen", - ":ops_inc_gen", - ":types_inc_gen", - "@llvm-project//mlir:BytecodeOpInterface", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:LLVMDialect", - "@llvm-project//mlir:SideEffectInterfaces", - ], -) - -cc_library( - name = "dialect", - srcs = [ - "CudaDialect.cpp", - "CudaInference.cpp", - ], - deps = [ - ":headers", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:LLVMDialect", - ], -) diff --git a/mlir/dialects/cuda/CudaAttrs.td b/mlir/dialects/cuda/CudaAttrs.td deleted file mode 100644 index 4d5e6f08..00000000 --- a/mlir/dialects/cuda/CudaAttrs.td +++ /dev/null @@ -1,68 +0,0 @@ -//===- CudaAttrs.td - CUDA host attributes of the CuTe DSL -*- tablegen -*-===// -// -// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler -// (`_cutlass_ir.cu13` parser messages, its mnemonics, and probing its -// attribute parser and printer); do not edit by hand. -// -// Every cuda attribute the compiler's parser knows. `device_attributes` and -// `func_attributes` print as `#cuda`: the compiler -// parses any dictionary there and prints back only the entries it keeps, -// whose keys were not recovered; these keep the dictionary as written, and -// print a space before it, which the compiler parses too. -// -//===----------------------------------------------------------------------===// - -#ifndef MLIR_DIALECTS_CUDA_CUDAATTRS_TD -#define MLIR_DIALECTS_CUDA_CUDAATTRS_TD - -include "mlir/dialects/cuda/CudaDialect.td" -include "mlir/dialects/cuda/CudaEnums.td" - -defvar Cuda_Ns = "::mlir::cutlass_compiler::cuda"; - -def Cuda_AssumeKernelAttr : Cuda_Attr<"AssumeKernel", "assume_kernel_attr"> { - let summary = "Whether `cuda.launch` may assume its callee is a kernel"; - let description = "`#cuda.assume_kernel_attr`"; - let parameters = (ins "bool":$value); - let assemblyFormat = "`<` $value `>`"; -} - -def Cuda_ComputeTargetAttr : Cuda_Attr<"ComputeTarget", "compute_target"> { - let summary = "Representation, portability and architectures of a compiled executable"; - let description = "`#cuda.compute_target`"; - let parameters = (ins - AttrParameter:$representation, - AttrParameter:$portability, - ArrayRefParameter:$archs - ); - let assemblyFormat = "`<` $representation `,` $portability `,` `[` $archs `]` `>`"; -} - -def Cuda_DevMaxSharedMemoryOptinAttr - : Cuda_Attr<"DevMaxSharedMemoryOptin", "dev_max_shared_memory_optin"> { - let summary = "The device's opt-in maximum of shared memory per block"; - let description = "`#cuda.dev_max_shared_memory_optin`"; -} - -def Cuda_DeviceAttributesAttr : Cuda_Attr<"DeviceAttributes", "device_attributes"> { - let summary = "CUDA device attributes"; - let description = "`#cuda`"; - let parameters = (ins "::mlir::DictionaryAttr":$values); - let assemblyFormat = "$values"; -} - -def Cuda_ExecutableAttr : Cuda_Attr<"Executable", "executable"> { - let summary = "Format of a compiled executable"; - let description = "`#cuda.executable`"; - let parameters = (ins AttrParameter:$format); - let assemblyFormat = "`<` $format `>`"; -} - -def Cuda_FuncAttributesAttr : Cuda_Attr<"FuncAttributes", "func_attributes"> { - let summary = "CUDA function attributes of a kernel"; - let description = "`#cuda`"; - let parameters = (ins "::mlir::DictionaryAttr":$values); - let assemblyFormat = "$values"; -} - -#endif // MLIR_DIALECTS_CUDA_CUDAATTRS_TD diff --git a/mlir/dialects/cuda/CudaDialect.cpp b/mlir/dialects/cuda/CudaDialect.cpp deleted file mode 100644 index 0ce2910d..00000000 --- a/mlir/dialects/cuda/CudaDialect.cpp +++ /dev/null @@ -1,38 +0,0 @@ -//===- CudaDialect.cpp - NVIDIA's cuda dialect of the CuTe DSL ------------===// - -#include "mlir/dialects/cuda/CudaDialect.h" - -#include "mlir/IR/Builders.h" -#include "mlir/IR/DialectImplementation.h" -#include "llvm/ADT/TypeSwitch.h" - -using namespace mlir; -using namespace mlir::cutlass_compiler::cuda; - -#include "mlir/dialects/cuda/CudaDialect.cpp.inc" - -#include "mlir/dialects/cuda/CudaEnums.cpp.inc" - -#define GET_ATTRDEF_CLASSES -#include "mlir/dialects/cuda/CudaAttrs.cpp.inc" - -#define GET_TYPEDEF_CLASSES -#include "mlir/dialects/cuda/CudaTypes.cpp.inc" - -#define GET_OP_CLASSES -#include "mlir/dialects/cuda/CudaOps.cpp.inc" - -void CudaDialect::initialize() { - addAttributes< -#define GET_ATTRDEF_LIST -#include "mlir/dialects/cuda/CudaAttrs.cpp.inc" - >(); - addTypes< -#define GET_TYPEDEF_LIST -#include "mlir/dialects/cuda/CudaTypes.cpp.inc" - >(); - addOperations< -#define GET_OP_LIST -#include "mlir/dialects/cuda/CudaOps.cpp.inc" - >(); -} diff --git a/mlir/dialects/cuda/CudaDialect.h b/mlir/dialects/cuda/CudaDialect.h deleted file mode 100644 index a96cdd17..00000000 --- a/mlir/dialects/cuda/CudaDialect.h +++ /dev/null @@ -1,30 +0,0 @@ -//===- CudaDialect.h - NVIDIA's cuda dialect of the CuTe DSL ----*- C++ -*-===// - -#ifndef MLIR_DIALECTS_CUDA_CUDADIALECT_H -#define MLIR_DIALECTS_CUDA_CUDADIALECT_H - -#include "mlir/Bytecode/BytecodeOpInterface.h" -#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/LLVMIR/LLVMTypes.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/DialectImplementation.h" -#include "mlir/IR/OpImplementation.h" -#include "mlir/Interfaces/InferTypeOpInterface.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" - -#include "mlir/dialects/cuda/CudaDialect.h.inc" - -#include "mlir/dialects/cuda/CudaEnums.h.inc" - -#define GET_ATTRDEF_CLASSES -#include "mlir/dialects/cuda/CudaAttrs.h.inc" - -#define GET_TYPEDEF_CLASSES -#include "mlir/dialects/cuda/CudaTypes.h.inc" - -#define GET_OP_CLASSES -#include "mlir/dialects/cuda/CudaOps.h.inc" - -#endif // MLIR_DIALECTS_CUDA_CUDADIALECT_H diff --git a/mlir/dialects/cuda/CudaDialect.td b/mlir/dialects/cuda/CudaDialect.td deleted file mode 100644 index 939e2d85..00000000 --- a/mlir/dialects/cuda/CudaDialect.td +++ /dev/null @@ -1,38 +0,0 @@ -//===- CudaDialect.td - CUDA host dialect of the CuTe DSL --*- tablegen -*-===// - -#ifndef MLIR_DIALECTS_CUDA_CUDADIALECT_TD -#define MLIR_DIALECTS_CUDA_CUDADIALECT_TD - -include "mlir/IR/AttrTypeBase.td" -include "mlir/IR/DialectBase.td" -include "mlir/IR/OpBase.td" - -def Cuda_Dialect : Dialect { - let name = "cuda"; - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let summary = "NVIDIA's CUDA host-program dialect used by CuTe-DSL"; - let description = [{ - This dialect mirrors the operation, type, attribute and enum registry of - the proprietary `cuda` dialect embedded in the nvidia-cutlass-dsl 4.8.0 - compiler: kernels, launch configurations and the CUDA runtime and driver - calls a host function makes. - }]; - let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; - let useDefaultTypePrinterParser = 1; - let useDefaultAttributePrinterParser = 1; -} - -class Cuda_Op traits = []> - : Op; - -class Cuda_Type - : TypeDef { - let mnemonic = typeMnemonic; -} - -class Cuda_Attr - : AttrDef { - let mnemonic = attrMnemonic; -} - -#endif // MLIR_DIALECTS_CUDA_CUDADIALECT_TD diff --git a/mlir/dialects/cuda/CudaEnums.td b/mlir/dialects/cuda/CudaEnums.td deleted file mode 100644 index f7980167..00000000 --- a/mlir/dialects/cuda/CudaEnums.td +++ /dev/null @@ -1,336 +0,0 @@ -// The cuda dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 enum -// bindings (`_cuda_enum_gen.py`); do not edit. -// -// Case values and spellings are the compiler's. An enum the bindings build -// as a dialect attribute also has its EnumAttr, printed `#` -// or, where the compiler prints it so, `#nvvm`. - -#ifndef MLIR_DIALECTS_CUDA_CUDAENUMS_TD -#define MLIR_DIALECTS_CUDA_CUDAENUMS_TD - -include "mlir/dialects/cuda/CudaDialect.td" -include "mlir/IR/EnumAttr.td" - -def Cuda_ArchPortabilityEnum : I32EnumAttr<"ArchPortability", "Enumerated type describing portability", [ - I32EnumAttrCase<"portable", 0, "portable">, - I32EnumAttrCase<"conditional", 1, "conditional">, - I32EnumAttrCase<"family", 2, "family"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_CUDeviceAttrEnum : I32EnumAttr<"CUDeviceAttr", "Enumerated type describing CUDA device attributes can be set through Cuda Dialect", [ - I32EnumAttrCase<"maxThreadsPerBlock", 1, "maxThreadsPerBlock">, - I32EnumAttrCase<"maxBlockDimX", 2, "maxBlockDimX">, - I32EnumAttrCase<"maxBlockDimY", 3, "maxBlockDimY">, - I32EnumAttrCase<"maxBlockDimZ", 4, "maxBlockDimZ">, - I32EnumAttrCase<"maxGridDimX", 5, "maxGridDimX">, - I32EnumAttrCase<"maxGridDimY", 6, "maxGridDimY">, - I32EnumAttrCase<"maxGridDimZ", 7, "maxGridDimZ">, - I32EnumAttrCase<"maxSharedMemoryPerBlock", 8, "maxSharedMemoryPerBlock">, - I32EnumAttrCase<"totalConstantMemory", 9, "totalConstantMemory">, - I32EnumAttrCase<"warpSize", 10, "warpSize">, - I32EnumAttrCase<"maxPitch", 11, "maxPitch">, - I32EnumAttrCase<"maxRegistersPerBlock", 12, "maxRegistersPerBlock">, - I32EnumAttrCase<"clockRate", 13, "clockRate">, - I32EnumAttrCase<"textureAlignment", 14, "textureAlignment">, - I32EnumAttrCase<"gpuOverlap", 15, "gpuOverlap">, - I32EnumAttrCase<"multiProcessorCount", 16, "multiProcessorCount">, - I32EnumAttrCase<"kernelExecTimeout", 17, "kernelExecTimeout">, - I32EnumAttrCase<"integrated", 18, "integrated">, - I32EnumAttrCase<"canMapHostMemory", 19, "canMapHostMemory">, - I32EnumAttrCase<"computeMode", 20, "computeMode">, - I32EnumAttrCase<"maxTexture1DWidth", 21, "maxTexture1DWidth">, - I32EnumAttrCase<"maxTexture2DWidth", 22, "maxTexture2DWidth">, - I32EnumAttrCase<"maxTexture2DHeight", 23, "maxTexture2DHeight">, - I32EnumAttrCase<"maxTexture3DWidth", 24, "maxTexture3DWidth">, - I32EnumAttrCase<"maxTexture3DHeight", 25, "maxTexture3DHeight">, - I32EnumAttrCase<"maxTexture3DDepth", 26, "maxTexture3DDepth">, - I32EnumAttrCase<"maxTexture2DLayeredWidth", 27, "maxTexture2DLayeredWidth">, - I32EnumAttrCase<"maxTexture2DLayeredHeight", 28, "maxTexture2DLayeredHeight">, - I32EnumAttrCase<"maxTexture2DLayeredLayers", 29, "maxTexture2DLayeredLayers">, - I32EnumAttrCase<"surfaceAlignment", 30, "surfaceAlignment">, - I32EnumAttrCase<"concurrentKernels", 31, "concurrentKernels">, - I32EnumAttrCase<"eccEnabled", 32, "eccEnabled">, - I32EnumAttrCase<"pciBusId", 33, "pciBusId">, - I32EnumAttrCase<"pciDeviceId", 34, "pciDeviceId">, - I32EnumAttrCase<"tccDriver", 35, "tccDriver">, - I32EnumAttrCase<"memoryClockRate", 36, "memoryClockRate">, - I32EnumAttrCase<"globalMemoryBusWidth", 37, "globalMemoryBusWidth">, - I32EnumAttrCase<"l2CacheSize", 38, "l2CacheSize">, - I32EnumAttrCase<"maxThreadsPerMultiProcessor", 39, "maxThreadsPerMultiProcessor">, - I32EnumAttrCase<"asyncEngineCount", 40, "asyncEngineCount">, - I32EnumAttrCase<"unifiedAddressing", 41, "unifiedAddressing">, - I32EnumAttrCase<"maxTexture1DLayeredWidth", 42, "maxTexture1DLayeredWidth">, - I32EnumAttrCase<"maxTexture1DLayeredLayers", 43, "maxTexture1DLayeredLayers">, - I32EnumAttrCase<"maxTexture2DGatherWidth", 45, "maxTexture2DGatherWidth">, - I32EnumAttrCase<"maxTexture2DGatherHeight", 46, "maxTexture2DGatherHeight">, - I32EnumAttrCase<"maxTexture3DWidthAlt", 47, "maxTexture3DWidthAlt">, - I32EnumAttrCase<"maxTexture3DHeightAlt", 48, "maxTexture3DHeightAlt">, - I32EnumAttrCase<"maxTexture3DDepthAlt", 49, "maxTexture3DDepthAlt">, - I32EnumAttrCase<"pciDomainId", 50, "pciDomainId">, - I32EnumAttrCase<"texturePitchAlignment", 51, "texturePitchAlignment">, - I32EnumAttrCase<"maxTextureCubemapWidth", 52, "maxTextureCubemapWidth">, - I32EnumAttrCase<"maxTextureCubemapLayeredWidth", 53, "maxTextureCubemapLayeredWidth">, - I32EnumAttrCase<"maxTextureCubemapLayeredLayers", 54, "maxTextureCubemapLayeredLayers">, - I32EnumAttrCase<"maxSurface1DWidth", 55, "maxSurface1DWidth">, - I32EnumAttrCase<"maxSurface2DWidth", 56, "maxSurface2DWidth">, - I32EnumAttrCase<"maxSurface2DHeight", 57, "maxSurface2DHeight">, - I32EnumAttrCase<"maxSurface3DWidth", 58, "maxSurface3DWidth">, - I32EnumAttrCase<"maxSurface3DHeight", 59, "maxSurface3DHeight">, - I32EnumAttrCase<"maxSurface3DDepth", 60, "maxSurface3DDepth">, - I32EnumAttrCase<"maxSurface1DLayeredWidth", 61, "maxSurface1DLayeredWidth">, - I32EnumAttrCase<"maxSurface1DLayeredLayers", 62, "maxSurface1DLayeredLayers">, - I32EnumAttrCase<"maxSurface2DLayeredWidth", 63, "maxSurface2DLayeredWidth">, - I32EnumAttrCase<"maxSurface2DLayeredHeight", 64, "maxSurface2DLayeredHeight">, - I32EnumAttrCase<"maxSurface2DLayeredLayers", 65, "maxSurface2DLayeredLayers">, - I32EnumAttrCase<"maxSurfaceCubemapWidth", 66, "maxSurfaceCubemapWidth">, - I32EnumAttrCase<"maxSurfaceCubemapLayeredWidth", 67, "maxSurfaceCubemapLayeredWidth">, - I32EnumAttrCase<"maxSurfaceCubemapLayeredLayers", 68, "maxSurfaceCubemapLayeredLayers">, - I32EnumAttrCase<"maxTexture1DLinearWidth", 69, "maxTexture1DLinearWidth">, - I32EnumAttrCase<"maxTexture2DLinearWidth", 70, "maxTexture2DLinearWidth">, - I32EnumAttrCase<"maxTexture2DLinearHeight", 71, "maxTexture2DLinearHeight">, - I32EnumAttrCase<"maxTexture2DLinearPitch", 72, "maxTexture2DLinearPitch">, - I32EnumAttrCase<"maxTexture2DMipmappedWidth", 73, "maxTexture2DMipmappedWidth">, - I32EnumAttrCase<"maxTexture2DMipmappedHeight", 74, "maxTexture2DMipmappedHeight">, - I32EnumAttrCase<"computeCapabilityMajor", 75, "computeCapabilityMajor">, - I32EnumAttrCase<"computeCapabilityMinor", 76, "computeCapabilityMinor">, - I32EnumAttrCase<"maxTexture1DMipmappedWidth", 77, "maxTexture1DMipmappedWidth">, - I32EnumAttrCase<"streamPrioritiesSupported", 78, "streamPrioritiesSupported">, - I32EnumAttrCase<"globalL1CacheSupported", 79, "globalL1CacheSupported">, - I32EnumAttrCase<"localL1CacheSupported", 80, "localL1CacheSupported">, - I32EnumAttrCase<"maxSharedMemoryPerMultiprocessor", 81, "maxSharedMemoryPerMultiprocessor">, - I32EnumAttrCase<"maxRegistersPerMultiprocessor", 82, "maxRegistersPerMultiprocessor">, - I32EnumAttrCase<"managedMemory", 83, "managedMemory">, - I32EnumAttrCase<"isMultiGpuBoard", 84, "isMultiGpuBoard">, - I32EnumAttrCase<"multiGpuBoardGroupID", 85, "multiGpuBoardGroupID">, - I32EnumAttrCase<"hostNativeAtomicSupported", 86, "hostNativeAtomicSupported">, - I32EnumAttrCase<"singleToDoublePrecisionPerfRatio", 87, "singleToDoublePrecisionPerfRatio">, - I32EnumAttrCase<"pageableMemoryAccess", 88, "pageableMemoryAccess">, - I32EnumAttrCase<"concurrentManagedAccess", 89, "concurrentManagedAccess">, - I32EnumAttrCase<"computePreemptionSupported", 90, "computePreemptionSupported">, - I32EnumAttrCase<"canUseHostPointerForRegisteredMem", 91, "canUseHostPointerForRegisteredMem">, - I32EnumAttrCase<"reserved92", 92, "reserved92">, - I32EnumAttrCase<"reserved93", 93, "reserved93">, - I32EnumAttrCase<"reserved94", 94, "reserved94">, - I32EnumAttrCase<"cooperativeLaunch", 95, "cooperativeLaunch">, - I32EnumAttrCase<"reserved96", 96, "reserved96">, - I32EnumAttrCase<"maxSharedMemoryPerBlockOptin", 97, "maxSharedMemoryPerBlockOptin">, - I32EnumAttrCase<"canFlushRemoteWrites", 98, "canFlushRemoteWrites">, - I32EnumAttrCase<"hostRegisterSupported", 99, "hostRegisterSupported">, - I32EnumAttrCase<"pageableMemoryAccessUsesHostPageTables", 100, "pageableMemoryAccessUsesHostPageTables">, - I32EnumAttrCase<"directManagedMemAccessFromHost", 101, "directManagedMemAccessFromHost">, - I32EnumAttrCase<"maxBlocksPerMultiprocessor", 106, "maxBlocksPerMultiprocessor">, - I32EnumAttrCase<"maxPersistingL2CacheSize", 108, "maxPersistingL2CacheSize">, - I32EnumAttrCase<"maxAccessPolicyWindowSize", 109, "maxAccessPolicyWindowSize">, - I32EnumAttrCase<"reservedSharedMemoryPerBlock", 111, "reservedSharedMemoryPerBlock">, - I32EnumAttrCase<"sparseCudaArraySupported", 112, "sparseCudaArraySupported">, - I32EnumAttrCase<"hostRegisterReadOnlySupported", 113, "hostRegisterReadOnlySupported">, - I32EnumAttrCase<"timelineSemaphoreInteropSupported", 114, "timelineSemaphoreInteropSupported">, - I32EnumAttrCase<"memoryPoolsSupported", 115, "memoryPoolsSupported">, - I32EnumAttrCase<"gpuDirectRdmaSupported", 116, "gpuDirectRdmaSupported">, - I32EnumAttrCase<"gpuDirectRdmaFlushWritesOptions", 117, "gpuDirectRdmaFlushWritesOptions">, - I32EnumAttrCase<"gpuDirectRdmaWritesOrdering", 118, "gpuDirectRdmaWritesOrdering">, - I32EnumAttrCase<"memoryPoolSupportedHandleTypes", 119, "memoryPoolSupportedHandleTypes">, - I32EnumAttrCase<"clusterLaunch", 120, "clusterLaunch">, - I32EnumAttrCase<"deferredMappingCudaArraySupported", 121, "deferredMappingCudaArraySupported">, - I32EnumAttrCase<"reserved122", 122, "reserved122">, - I32EnumAttrCase<"reserved123", 123, "reserved123">, - I32EnumAttrCase<"reserved124", 124, "reserved124">, - I32EnumAttrCase<"ipcEventSupport", 125, "ipcEventSupport">, - I32EnumAttrCase<"memSyncDomainCount", 126, "memSyncDomainCount">, - I32EnumAttrCase<"reserved127", 127, "reserved127">, - I32EnumAttrCase<"reserved128", 128, "reserved128">, - I32EnumAttrCase<"reserved129", 129, "reserved129">, - I32EnumAttrCase<"numaConfig", 130, "numaConfig">, - I32EnumAttrCase<"numaId", 131, "numaId">, - I32EnumAttrCase<"reserved132", 132, "reserved132">, - I32EnumAttrCase<"mpsEnabled", 133, "mpsEnabled">, - I32EnumAttrCase<"hostNumaId", 134, "hostNumaId">, - I32EnumAttrCase<"d3d12CigSupported", 135, "d3d12CigSupported">, - I32EnumAttrCase<"vulkanCigSupported", 138, "vulkanCigSupported">, - I32EnumAttrCase<"gpuPciDeviceId", 139, "gpuPciDeviceId">, - I32EnumAttrCase<"gpuPciSubsystemId", 140, "gpuPciSubsystemId">, - I32EnumAttrCase<"reserved141", 141, "reserved141">, - I32EnumAttrCase<"hostNumaMemoryPoolsSupported", 142, "hostNumaMemoryPoolsSupported">, - I32EnumAttrCase<"hostNumaMultinodeIpcSupported", 143, "hostNumaMultinodeIpcSupported">, - I32EnumAttrCase<"hostMemoryPoolsSupported", 144, "hostMemoryPoolsSupported">, - I32EnumAttrCase<"reserved145", 145, "reserved145">, - I32EnumAttrCase<"onlyPartialHostNativeAtomicSupported", 147, "onlyPartialHostNativeAtomicSupported">, - I32EnumAttrCase<"cudaDevAttrOversizedSharedMemoryPerBlock", 150, "cudaDevAttrOversizedSharedMemoryPerBlock"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_CUFunctionAttributeEnum : I32EnumAttr<"CUFunctionAttribute", "Enumerated type describing CUDA function attributes can be set through Cuda Dialect", [ - I32EnumAttrCase<"max_threads_per_block", 0, "max_threads_per_block">, - I32EnumAttrCase<"shared_size_bytes", 1, "shared_size_bytes">, - I32EnumAttrCase<"const_size_bytes", 2, "const_size_bytes">, - I32EnumAttrCase<"local_size_bytes", 3, "local_size_bytes">, - I32EnumAttrCase<"num_regs", 4, "num_regs">, - I32EnumAttrCase<"ptx_version", 5, "ptx_version">, - I32EnumAttrCase<"binary_version", 6, "binary_version">, - I32EnumAttrCase<"cache_mode_ca", 7, "cache_mode_ca">, - I32EnumAttrCase<"max_dynamic_shared_size_bytes", 8, "max_dynamic_shared_size_bytes">, - I32EnumAttrCase<"preferred_shared_memory_carveout", 9, "preferred_shared_memory_carveout">, - I32EnumAttrCase<"cluster_size_must_be_set", 10, "cluster_size_must_be_set">, - I32EnumAttrCase<"required_cluster_width", 11, "required_cluster_width">, - I32EnumAttrCase<"required_cluster_height", 12, "required_cluster_height">, - I32EnumAttrCase<"required_cluster_depth", 13, "required_cluster_depth">, - I32EnumAttrCase<"non_portable_cluster_size_allowed", 14, "non_portable_cluster_size_allowed">, - I32EnumAttrCase<"cluster_scheduling_policy_preference", 15, "cluster_scheduling_policy_preference"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_CudaABIEnum : I32EnumAttr<"CudaABI", "Enumerated of the host library ABI", [ - I32EnumAttrCase<"LP64", 0, "LP64">, - I32EnumAttrCase<"ILP64", 1, "ILP64">, - I32EnumAttrCase<"LLP64", 2, "LLP64"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_CudaAccessPropertyEnum : I32EnumAttr<"CudaAccessProperty", "Specifies performance hint with cudaAccessPolicyWindow for hitProp and missProp members", [ - I32EnumAttrCase<"cudaAccessPropertyNormal", 0, "cudaAccessPropertyNormal">, - I32EnumAttrCase<"cudaAccessPropertyStreaming", 1, "cudaAccessPropertyStreaming">, - I32EnumAttrCase<"cudaAccessPropertyPersisting", 2, "cudaAccessPropertyPersisting"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_CudaClusterSchedulingPolicyEnum : I32EnumAttr<"CudaClusterSchedulingPolicy", "Enumerated type describing CUDA cluster scheduling policy", [ - I32EnumAttrCase<"cudaClusterSchedulingPolicyDefault", 0, "cudaClusterSchedulingPolicyDefault">, - I32EnumAttrCase<"cudaClusterSchedulingPolicySpread", 1, "cudaClusterSchedulingPolicySpread">, - I32EnumAttrCase<"cudaClusterSchedulingPolicyLoadBalancing", 2, "cudaClusterSchedulingPolicyLoadBalancing"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_CudaFuncAttributeEnum : I32EnumAttr<"CudaFuncAttribute", "Enumerated type describing CUDA function attributes can be set through Cuda Dialect", [ - I32EnumAttrCase<"cudaFuncAttributeSharedMemoryMode", 16, "cudaFuncAttributeSharedMemoryMode"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_CudaLaunchAttributeIDEnum : I32EnumAttr<"CudaLaunchAttributeID", "Enumerated type mirroring CudaLaunchAttributeID from CUDA Runtime API", [ - I32EnumAttrCase<"cudaLaunchAttributeIgnore", 0, "cudaLaunchAttributeIgnore">, - I32EnumAttrCase<"cudaLaunchAttributeAccessPolicyWindow", 1, "cudaLaunchAttributeAccessPolicyWindow">, - I32EnumAttrCase<"cudaLaunchAttributeCooperative", 2, "cudaLaunchAttributeCooperative">, - I32EnumAttrCase<"cudaLaunchAttributeSynchronizationPolicy", 3, "cudaLaunchAttributeSynchronizationPolicy">, - I32EnumAttrCase<"cudaLaunchAttributeClusterDimension", 4, "cudaLaunchAttributeClusterDimension">, - I32EnumAttrCase<"cudaLaunchAttributeClusterSchedulingPolicyPreference", 5, "cudaLaunchAttributeClusterSchedulingPolicyPreference">, - I32EnumAttrCase<"cudaLaunchAttributeProgrammaticStreamSerialization", 6, "cudaLaunchAttributeProgrammaticStreamSerialization">, - I32EnumAttrCase<"cudaLaunchAttributeProgrammaticEvent", 7, "cudaLaunchAttributeProgrammaticEvent">, - I32EnumAttrCase<"cudaLaunchAttributePriority", 8, "cudaLaunchAttributePriority">, - I32EnumAttrCase<"cudaLaunchAttributeMemSyncDomainMap", 9, "cudaLaunchAttributeMemSyncDomainMap">, - I32EnumAttrCase<"cudaLaunchAttributeMemSyncDomain", 10, "cudaLaunchAttributeMemSyncDomain">, - I32EnumAttrCase<"cudaLaunchAttributePreferredClusterDimension", 11, "cudaLaunchAttributePreferredClusterDimension">, - I32EnumAttrCase<"cudaLaunchAttributeLaunchCompletionEvent", 12, "cudaLaunchAttributeLaunchCompletionEvent">, - I32EnumAttrCase<"cudaLaunchAttributeDeviceUpdatableKernelNode", 13, "cudaLaunchAttributeDeviceUpdatableKernelNode">, - I32EnumAttrCase<"cudaLaunchAttributePreferredSharedMemoryCarveout", 14, "cudaLaunchAttributePreferredSharedMemoryCarveout">, - I32EnumAttrCase<"cudaLaunchAttributeNvlinkUtilCentricScheduling", 16, "cudaLaunchAttributeNvlinkUtilCentricScheduling"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_CudaLaunchMemSyncDomainEnum : I32EnumAttr<"CudaLaunchMemSyncDomain", "Enumerated type describing CUDA memory sync domain", [ - I32EnumAttrCase<"cudaLaunchMemSyncDomainDefault", 0, "cudaLaunchMemSyncDomainDefault">, - I32EnumAttrCase<"cudaLaunchMemSyncDomainRemote", 1, "cudaLaunchMemSyncDomainRemote"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_CudaSharedMemoryModeEnum : I32EnumAttr<"CudaSharedMemoryMode", "Enumerated type describing CUDA shared memory modes", [ - I32EnumAttrCase<"cudaSharedMemoryModeDefault", 0, "cudaSharedMemoryModeDefault">, - I32EnumAttrCase<"cudaSharedMemoryModeRequirePortable", 1, "cudaSharedMemoryModeRequirePortable">, - I32EnumAttrCase<"cudaSharedMemoryModeAllowNonPortable", 2, "cudaSharedMemoryModeAllowNonPortable">, - I32EnumAttrCase<"cudaSharedMemoryModeAllowOversizedSharedMemory", 3, "cudaSharedMemoryModeAllowOversizedSharedMemory">, - I32EnumAttrCase<"cudaSharedMemoryModePreferOversizedSharedMemory", 4, "cudaSharedMemoryModePreferOversizedSharedMemory"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_CudaSynchronizationPolicyEnum : I32EnumAttr<"CudaSynchronizationPolicy", "Enumerated type describing CUDA synchronization policy", [ - I32EnumAttrCase<"cudaSyncPolicyAuto", 0, "cudaSyncPolicyAuto">, - I32EnumAttrCase<"cudaSyncPolicySpin", 1, "cudaSyncPolicySpin">, - I32EnumAttrCase<"cudaSyncPolicyYield", 2, "cudaSyncPolicyYield">, - I32EnumAttrCase<"cudaSyncPolicyBlockingSync", 3, "cudaSyncPolicyBlockingSync"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_ExecutableFormatEnum : I32EnumAttr<"ExecutableFormat", "Enumerated type describing the format of an executable", [ - I32EnumAttrCase<"text", 0, "text">, - I32EnumAttrCase<"elf", 1, "elf">, - I32EnumAttrCase<"bitcode", 2, "bitcode">, - I32EnumAttrCase<"fatbin", 3, "fatbin"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_ExecutableRepresentationEnum : I32EnumAttr<"ExecutableRepresentation", "Enumerated type describing the representation of an executable", [ - I32EnumAttrCase<"sass", 0, "sass">, - I32EnumAttrCase<"ptx", 2, "ptx">, - I32EnumAttrCase<"nvvm", 3, "nvvm">, - I32EnumAttrCase<"ltoir", 4, "ltoir">, - I32EnumAttrCase<"mlir", 5, "mlir">, - I32EnumAttrCase<"llvm", 6, "llvm">, - I32EnumAttrCase<"hostobj", 7, "hostobj"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_GpuArchitectureEnum : I32EnumAttr<"GpuArchitecture", "Enumerated type denoting a GPU Architecture", [ - I32EnumAttrCase<"sm_80", 80, "sm_80">, - I32EnumAttrCase<"sm_86", 86, "sm_86">, - I32EnumAttrCase<"sm_87", 87, "sm_87">, - I32EnumAttrCase<"sm_88", 88, "sm_88">, - I32EnumAttrCase<"sm_89", 89, "sm_89">, - I32EnumAttrCase<"sm_90", 90, "sm_90">, - I32EnumAttrCase<"sm_100", 100, "sm_100">, - I32EnumAttrCase<"sm_101", 101, "sm_101">, - I32EnumAttrCase<"sm_103", 103, "sm_103">, - I32EnumAttrCase<"sm_107", 107, "sm_107">, - I32EnumAttrCase<"sm_110", 110, "sm_110">, - I32EnumAttrCase<"sm_120", 120, "sm_120">, - I32EnumAttrCase<"sm_121", 121, "sm_121"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_MemcpyKindEnum : I32EnumAttr<"MemcpyKind", "Enumerated type describing memcpy directions", [ - I32EnumAttrCase<"HostToHost", 0, "HostToHost">, - I32EnumAttrCase<"HostToDevice", 1, "HostToDevice">, - I32EnumAttrCase<"DeviceToHost", 2, "DeviceToHost">, - I32EnumAttrCase<"DeviceToDevice", 3, "DeviceToDevice">, - I32EnumAttrCase<"Default", 4, "Default"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -def Cuda_StreamCaptureModeEnumEnum : I32EnumAttr<"StreamCaptureModeEnum", "Enumerated type describing CUDA stream capture modes", [ - I32EnumAttrCase<"Global", 0, "Global">, - I32EnumAttrCase<"ThreadLocal", 1, "ThreadLocal">, - I32EnumAttrCase<"Relaxed", 2, "Relaxed"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cuda"; - let genSpecializedAttr = 0; -} - -#endif // MLIR_DIALECTS_CUDA_CUDAENUMS_TD diff --git a/mlir/dialects/cuda/CudaInference.cpp b/mlir/dialects/cuda/CudaInference.cpp deleted file mode 100644 index 1289679e..00000000 --- a/mlir/dialects/cuda/CudaInference.cpp +++ /dev/null @@ -1,17 +0,0 @@ -//===- CudaInference.cpp - result types the compiler infers ---------------===// -// -// Generated from the nvidia-cutlass-dsl 4.8.0 compiler (its inference -// probed by varying operand types); do not edit. An operation without a -// recovered rule fails inference, so the result types written stand. - -#include "mlir/dialects/cuda/CudaDialect.h" - -using namespace mlir; -using namespace ::mlir::cutlass_compiler::cuda; - -LogicalResult LaunchCfgCreateFromStreamOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} diff --git a/mlir/dialects/cuda/CudaOps.td b/mlir/dialects/cuda/CudaOps.td deleted file mode 100644 index 991814c1..00000000 --- a/mlir/dialects/cuda/CudaOps.td +++ /dev/null @@ -1,3072 +0,0 @@ -// The cuda dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. -// -// Sources: the operation bindings (`_cuda_ops_gen.py`: names, classes, -// docstrings, operand/result/attribute structure, inference) and the live -// compiler (`_cutlass_ir` loaded from Python): each operand and result -// constraint carries the verifier's wording and accepts every type class the -// verifier let through when probed with one instance of every type; traits -// and interfaces are the registered operation's; memory effects come from -// loop-invariant code motion (hoisted: Pure) and CSE (merged: at most -// reads). Operations whose effects include writes or could not be told -// apart carry none (unknown effects). The compiler's own verifiers (operand -// combinations, shapes, instruction legality) are left to it. -// -// Operations keep the generic assembly form. -// -// Result types the compiler infers without a recovered rule (written types stand): -// cuda.launch_cfg.create_from_stream -// -// Constraints not recovered: - -#ifndef MLIR_DIALECTS_CUDA_CUDAOPS_TD -#define MLIR_DIALECTS_CUDA_CUDAOPS_TD - -include "mlir/dialects/cuda/CudaAttrs.td" -include "mlir/dialects/cuda/CudaEnums.td" -include "mlir/dialects/cuda/CudaTypes.td" - -include "mlir/Interfaces/InferTypeOpInterface.td" -include "mlir/Interfaces/SideEffectInterfaces.td" - -// Operand and result constraints, in the compiler's words. -def Cuda_AnyCUDAStructType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeDim3Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeExtentType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeKernelNodeParamsType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeMemcpy3DParamsType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimePitchedPtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimePosType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeUuidType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueCooperativeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueMemSyncDomainType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePriorityType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSyncPolicyType>($_self)">, "any CUDA struct type">; -def Cuda_AnyType : Type, "any type">; -def Cuda_CUDACudaDevicePropStructTypeValueTyped : Type($_self)">, "CUDA cudaDeviceProp struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeDevicePropType::get($_builder.getContext())">; -def Cuda_CUDACudaDevicePropStructTypeValueTyped2 : Type($_self)">, "CUDA cudaDeviceProp struct type (value-typed).">; -def Cuda_CUDACudaExtentStructTypeValueTyped : Type($_self)">, "CUDA cudaExtent struct type (value-typed).">; -def Cuda_CUDACudaExtentStructTypeValueTyped2 : Type($_self)">, "CUDA cudaExtent struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeExtentType::get($_builder.getContext())">; -def Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped : Type($_self)">, "CUDA cudaKernelNodeParams struct type (value-typed).">; -def Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped : Type($_self)">, "CUDA cudaMemcpy3DParms struct type (value-typed).">; -def Cuda_CUDACudaPitchedPtrStructTypeValueTyped : Type($_self)">, "CUDA cudaPitchedPtr struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimePitchedPtrType::get($_builder.getContext())">; -def Cuda_CUDACudaPitchedPtrStructTypeValueTyped2 : Type($_self)">, "CUDA cudaPitchedPtr struct type (value-typed).">; -def Cuda_CUDACudaPosStructTypeValueTyped : Type($_self)">, "CUDA cudaPos struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimePosType::get($_builder.getContext())">; -def Cuda_CUDACudaPosStructTypeValueTyped2 : Type($_self)">, "CUDA cudaPos struct type (value-typed).">; -def Cuda_CUDACudaUUIDTStructTypeValue : Type($_self)">, "CUDA cudaUUID_t struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeUuidType::get($_builder.getContext())">; -def Cuda_CUDACudaUUIDTStructTypeValue2 : Type($_self)">, "CUDA cudaUUID_t struct type (value-typed).">; -def Cuda_CUDADim3StructTypeValueTyped : Type($_self)">, "CUDA dim3 struct type (value-typed).">; -def Cuda_CUDADim3StructTypeValueTyped2 : Type($_self)">, "CUDA dim3 struct type (value-typed).">, BuildableType<"::mlir::cutlass_compiler::cuda::RuntimeDim3Type::get($_builder.getContext())">; -def Cuda_CUDAGraph : Type($_self)">, "CUDA Graph">; -def Cuda_CUDAGraph2 : Type($_self)">, "CUDA Graph">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphType::get($_builder.getContext())">; -def Cuda_CUDAGraphExecution : Type($_self)">, "CUDA Graph Execution">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphExecType::get($_builder.getContext())">; -def Cuda_CUDAGraphExecution2 : Type($_self)">, "CUDA Graph Execution">; -def Cuda_CUDAGraphNodeHandle : Type($_self)">, "CUDA Graph Node handle">, BuildableType<"::mlir::cutlass_compiler::cuda::GraphNodeType::get($_builder.getContext())">; -def Cuda_CUDAGraphNodeHandle2 : Type($_self)">, "CUDA Graph Node handle">; -def Cuda_CUDAKernelHandle : Type($_self)">, "CUDA Kernel handle">, BuildableType<"::mlir::cutlass_compiler::cuda::KernelType::get($_builder.getContext())">; -def Cuda_CUDAKernelHandle2 : Type($_self)">, "CUDA Kernel handle">; -def Cuda_CUDALibraryHandle : Type($_self)">, "CUDA Library handle">; -def Cuda_CUDALibraryHandle2 : Type($_self)">, "CUDA Library handle">, BuildableType<"::mlir::cutlass_compiler::cuda::LibraryType::get($_builder.getContext())">; -def Cuda_CUDAModuleHandle : Type($_self)">, "CUDA Module handle">, BuildableType<"::mlir::cutlass_compiler::cuda::ModuleType::get($_builder.getContext())">; -def Cuda_CUDAResultType : Type($_self)">, "CUDA Result type">, BuildableType<"::mlir::cutlass_compiler::cuda::ResultType::get($_builder.getContext())">; -def Cuda_CUDAResultType2 : Type($_self)">, "CUDA Result type">; -def Cuda_CUDAResultTypeOrCUDAStream : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::StreamType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::ResultType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::EventType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::GraphDeviceNodeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::TensorMapType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::IntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::LaunchConfigType>($_self) || ($_self.getDialect().getNamespace() == \"ptr\" || (::llvm::isa<::mlir::OpaqueType>($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\"))">, "CUDA Result type or CUDA Stream or `Event` represents CUDA event or CUDA Tensor Map type or Representation of CUDA's extensible launch configuration mirroring cuda runtime API or `GraphDeviceNode` represents CUDA device node handle for device-side node update or `cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth or 32-bit signless integer or 64-bit signless integer or pointer type or LLVM pointer type">; -def Cuda_CUDAStream : Type($_self)">, "CUDA Stream">; -def Cuda_CUDAStream2 : Type($_self)">, "CUDA Stream">, BuildableType<"::mlir::cutlass_compiler::cuda::StreamType::get($_builder.getContext())">; -def Cuda_CUDATensorMapType : Type($_self)">, "CUDA Tensor Map type">, BuildableType<"::mlir::cutlass_compiler::cuda::TensorMapType::get($_builder.getContext())">; -def Cuda_CudaIntIsAIntegerType : Type($_self)">, "`cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth">; -def Cuda_EventRepresentsCUDAEvent : Type($_self)">, "`Event` represents CUDA event">, BuildableType<"::mlir::cutlass_compiler::cuda::EventType::get($_builder.getContext())">; -def Cuda_EventRepresentsCUDAEvent2 : Type($_self)">, "`Event` represents CUDA event">; -def Cuda_FixedLengthVectorOf32Bit : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 3">; -def Cuda_FixedLengthVectorOf32Bit2 : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 2">; -def Cuda_FixedLengthVectorOf8Bit : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 8">; -def Cuda_FixedLengthVectorOf8Bit2 : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 256">; -def Cuda_FixedLengthVectorOf8Bit3 : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 16">; -def Cuda_GraphDeviceNodeRepresentsCUDADeviceNodeHandle : Type($_self)">, "`GraphDeviceNode` represents CUDA device node handle for device-side node update">; -def Cuda_LLVMPointerInAddressSpace0 : Type($_self)">, "LLVM pointer in address space 0">; -def Cuda_LLVMPointerType : Type($_self)">, "LLVM pointer type">; -def Cuda_PointerType : Type($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\"))">, "pointer type">; -def Cuda_RepresentationOfCUDASExtensibleLaunch : Type($_self)">, "Representation of CUDA's extensible launch configuration mirroring cuda runtime API">; -def Cuda_T1BitSignlessInteger : Type, "1-bit signless integer">, BuildableType<"$_builder.getIntegerType(1)">; -def Cuda_T32BitFloat : Type($_self)">, "32-bit float">, BuildableType<"::mlir::Float32Type::get($_builder.getContext())">; -def Cuda_T32BitFloat2 : Type($_self)">, "32-bit float">; -def Cuda_T32BitSignlessInteger : Type, "32-bit signless integer">; -def Cuda_T32BitSignlessInteger2 : Type, "32-bit signless integer">, BuildableType<"$_builder.getIntegerType(32)">; -def Cuda_T64BitSignlessInteger : Type, "64-bit signless integer">; -def Cuda_T64BitSignlessInteger2 : Type, "64-bit signless integer">, BuildableType<"$_builder.getIntegerType(64)">; -def Cuda_VariadicOf32BitSignlessInteger : Type, "variadic of 32-bit signless integer">; -def Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueMemSyncDomainType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValuePriorityType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cuda::RuntimeLaunchAttributeValueSyncPolicyType>($_self)">, "Variant `cooperative` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `syncPolicy` of CUDA `cudaLaunchAttributeValue` union (payload: enum cudaSynchronizationPolicy). or Variant `clusterSchedulingPolicyPreference` of CUDA `cudaLaunchAttributeValue` union (payload: enum cudaClusterSchedulingPolicy). or Variant `programmaticStreamSerializationAllowed` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `priority` of CUDA `cudaLaunchAttributeValue` union (payload: int). or Variant `memSyncDomain` of CUDA `cudaLaunchAttributeValue` union (payload: cudaLaunchMemSyncDomain). or Variant `sharedMemCarveout` of CUDA `cudaLaunchAttributeValue` union (payload: unsigned int). or Variant `nvlinkUtilCentricScheduling` of CUDA `cudaLaunchAttributeValue` union (payload: unsigned int). or Variant `portableClusterSizeMode` of CUDA `cudaLaunchAttributeValue` union (payload: cudaLaunchAttributePortableClusterMode). or Variant `sharedMemoryMode` of CUDA `cudaLaunchAttributeValue` union (payload: cudaSharedMemoryMode).">; - -// Attribute constraints the bindings name but do not describe. -def Cuda_DenseResourceElementsAttr : Attr($_self)">, "dense resource elements attribute"> { - let storageType = "::mlir::DenseResourceElementsAttr"; - let returnType = "::mlir::DenseResourceElementsAttr"; - let convertFromStorage = "$_self"; -} - -def Cuda_FuncFunction_typeAttr : Attr($_self) && (::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of function type"> { - let storageType = "::mlir::TypeAttr"; - let returnType = "::mlir::TypeAttr"; - let convertFromStorage = "$_self"; -} - -def Cuda_KernelFunction_typeAttr : Attr($_self) && (::llvm::isa<::mlir::FunctionType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of function type"> { - let storageType = "::mlir::TypeAttr"; - let returnType = "::mlir::TypeAttr"; - let convertFromStorage = "$_self"; -} - -def Cuda_IntegerAttr : Attr($_self)">, "integer attribute"> { - let storageType = "::mlir::IntegerAttr"; - let returnType = "::mlir::IntegerAttr"; - let convertFromStorage = "$_self"; -} - -// Operations whose result types are inferred in C++ (CudaInference.cpp). -// Verification compares them with the inferred ones where the rule is -// recovered; elsewhere the types written stand, as the compiler accepted them. -class Cuda_InferredOp traits = []> - : Cuda_Op]> { - let extraClassDefinition = [{ - ::llvm::LogicalResult $cppClass::refineReturnTypes( - ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, - ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, - ::mlir::PropertyRef properties, ::mlir::RegionRange regions, - ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { - ::llvm::SmallVector<::mlir::Type, 4> inferred; - if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, - attributes, properties, regions, - inferred))) - return ::mlir::success(); - if (!isCompatibleReturnTypes(inferred, returnTypes)) - return ::mlir::emitOptionalError( - location, "'", getOperationName(), "' op inferred type(s) ", - inferred, " are incompatible with return type(s) of operation ", - returnTypes); - return ::mlir::success(); - } - }]; -} - -def Cuda_AddressOfOp : Cuda_Op<"addressof"> { - let arguments = (ins - SymbolRefAttr:$symname - ); - let results = (outs - Cuda_PointerType:$result - ); -} -def Cuda_BinaryOp : Cuda_Op<"binary", [NoTerminator]> { - let description = [{ - This operation represents a CUDA binary, which contains a number of functions and - a serialized representation of the binary. - - Examples: - - ```mlir - - cuda.binary @binary_name <...> { - cuda.kernel @function_name function_type - } - ``` - }]; - let arguments = (ins - SymbolNameAttr:$sym_name, - Cuda_DenseResourceElementsAttr:$binary - ); - let results = (outs); - let regions = (region AnyRegion:$body); -} -def Cuda_CastOp : Cuda_Op<"cast", [Pure]> { - let description = [{ - Casts between CUDA types and standard dialect or built-in types. - The source/destination types are statically checked and the IR - is invalid if the cast is not possible. - - Examples: - ```mlir - - %res_i32 = cuda.cast %res : cuda.result -> i32 - %res1 = cuda.cast %res_i32 : i32 -> cuda.result - - %s_ptr = cuda.cast %hstream : cuda.stream -> ptr.ptr - %s_i64 = cuda.cast %hstream : cuda.stream -> i64 - ``` - }]; - let arguments = (ins - Cuda_CUDAResultTypeOrCUDAStream:$src - ); - let results = (outs - Cuda_CUDAResultTypeOrCUDAStream:$dst - ); -} -def Cuda_ComposeOp : Cuda_Op<"compose"> { - let description = [{ - Generic constructor for any auto-generated CUDA struct type. The result - type identifies which struct is being built; operands are the struct's - non-padding fields in declaration order. The verifier checks operand - count and per-operand type against the result type's - `CudaStructTypeInterface::getOperandFieldTypes()`. - - Lowers to `llvm.mlir.undef` + a chain of `llvm.insertvalue` (one per - operand-mapped slot from `getOperandFieldIndices()`). Pure value SSA; - no memory. - - Example: - ```mlir - %pos = cuda.compose %x, %y, %z - : i64, i64, i64 -> !cuda.runtime.pos - ``` - }]; - let arguments = (ins - Variadic:$fields - ); - let results = (outs - Cuda_AnyCUDAStructType:$result - ); -} -def Cuda_DecomposeOp : Cuda_Op<"decompose"> { - let description = [{ - Generic accessor: returns each non-padding field of a CUDA struct value - as an SSA value, in declaration order. The verifier checks result count - and per-result type against the source type's - `CudaStructTypeInterface::getOperandFieldTypes()`. - - Lowers to a chain of `llvm.extractvalue` (one per operand-mapped slot - from `getOperandFieldIndices()`). Pure value SSA; no memory. - - Example: - ```mlir - %x, %y, %z = cuda.decompose %pos - : !cuda.runtime.pos -> i64, i64, i64 - ``` - }]; - let arguments = (ins - Cuda_AnyCUDAStructType:$source - ); - let results = (outs - Optional:$fields - ); -} -def Cuda_DeviceSyncOp : Cuda_Op<"device.sync"> { - let description = [{ - Operation to synchronize with the device. See cudaDeviceSynchronize() - }]; - let arguments = (ins); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_DriverEventCreateOp : Cuda_Op<"driver.event.create"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_T32BitSignlessInteger:$Flags - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_EventRepresentsCUDAEvent:$phEvent - ); -} -def Cuda_DriverEventDestroyOp : Cuda_Op<"driver.event.destroy"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$hEvent - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_DriverEventElapsedTimeOp : Cuda_Op<"driver.event.elapsed_time"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$hStart, - Cuda_EventRepresentsCUDAEvent2:$hEnd - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_T32BitFloat:$pMilliseconds - ); -} -def Cuda_DriverEventQueryOp : Cuda_Op<"driver.event.query", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$hEvent - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_DriverEventRecordOp : Cuda_Op<"driver.event.record"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$hEvent, - Cuda_CUDAStream:$hStream - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_DriverEventRecordWithFlagsOp : Cuda_Op<"driver.event.record_with_flags"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$hEvent, - Cuda_CUDAStream:$hStream, - Cuda_T32BitSignlessInteger:$flags - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_DriverEventSynchronizeOp : Cuda_Op<"driver.event.synchronize"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$hEvent - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_DriverLibraryGetGlobalOp : Cuda_Op<"driver.library.get_global", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library, - Cuda_PointerType:$name_arg - ); - let results = (outs - Cuda_CUDAResultType2:$cuda_result, - Cuda_PointerType:$dptr, - Cuda_T64BitSignlessInteger:$bytes - ); -} -def Cuda_DriverLibraryGetKernelOp : Cuda_Op<"driver.library.get_kernel", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library, - Cuda_PointerType:$name_arg - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDAKernelHandle:$pKernel - ); -} -def Cuda_DriverLibraryGetKernelCountOp : Cuda_Op<"driver.library.get_kernel_count", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$lib - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_T32BitSignlessInteger2:$count - ); -} -def Cuda_DriverLibraryGetManagedOp : Cuda_Op<"driver.library.get_managed", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library, - Cuda_PointerType:$name_arg - ); - let results = (outs - Cuda_CUDAResultType2:$cuda_result, - Cuda_PointerType:$dptr, - Cuda_T64BitSignlessInteger:$bytes - ); -} -def Cuda_DriverLibraryGetModuleOp : Cuda_Op<"driver.library.get_module", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDAModuleHandle:$pMod - ); -} -def Cuda_DriverLibraryGetUnifiedFunctionOp : Cuda_Op<"driver.library.get_unified_function", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library, - Cuda_PointerType:$symbol - ); - let results = (outs - Cuda_CUDAResultType2:$cuda_result, - Cuda_PointerType:$fptr - ); -} -def Cuda_DriverLibraryLoadDataOp : Cuda_Op<"driver.library.load_data"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_PointerType:$code_arg, - Cuda_PointerType:$jitOptions, - Cuda_PointerType:$jitOptionsValues, - Cuda_T32BitSignlessInteger:$numJitOptions, - Cuda_PointerType:$libraryOptions, - Cuda_PointerType:$libraryOptionValues, - Cuda_T32BitSignlessInteger:$numLibraryOptions - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDALibraryHandle2:$library - ); -} -def Cuda_DriverLibraryLoadFromFileOp : Cuda_Op<"driver.library.load_from_file"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_PointerType:$fileName, - Cuda_PointerType:$jitOptions, - Cuda_PointerType:$jitOptionsValues, - Cuda_T32BitSignlessInteger:$numJitOptions, - Cuda_PointerType:$libraryOptions, - Cuda_PointerType:$libraryOptionValues, - Cuda_T32BitSignlessInteger:$numLibraryOptions - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDALibraryHandle2:$library - ); -} -def Cuda_DriverLibraryUnloadOp : Cuda_Op<"driver.library.unload"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_DriverTensorMapEncodeIm2colOp : Cuda_Op<"driver.tensor_map.encode_im2col"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_T32BitSignlessInteger:$tensorDataType, - Cuda_T32BitSignlessInteger:$tensorRank, - Cuda_PointerType:$globalAddress, - Cuda_PointerType:$globalDim, - Cuda_PointerType:$globalStrides, - Cuda_PointerType:$pixelBoxLowerCorner, - Cuda_PointerType:$pixelBoxUpperCorner, - Cuda_T32BitSignlessInteger:$channelsPerPixel, - Cuda_T32BitSignlessInteger:$pixelsPerColumn, - Cuda_PointerType:$elementStrides, - Cuda_T32BitSignlessInteger:$interleave, - Cuda_T32BitSignlessInteger:$swizzle, - Cuda_T32BitSignlessInteger:$l2Promotion, - Cuda_T32BitSignlessInteger:$oobFill - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDATensorMapType:$tensorMap - ); -} -def Cuda_DriverTensorMapEncodeIm2colWideOp : Cuda_Op<"driver.tensor_map.encode_im2col_wide"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_T32BitSignlessInteger:$tensorDataType, - Cuda_T32BitSignlessInteger:$tensorRank, - Cuda_PointerType:$globalAddress, - Cuda_PointerType:$globalDim, - Cuda_PointerType:$globalStrides, - Cuda_T32BitSignlessInteger:$pixelBoxLowerCornerWidth, - Cuda_T32BitSignlessInteger:$pixelBoxUpperCornerWidth, - Cuda_T32BitSignlessInteger:$channelsPerPixel, - Cuda_T32BitSignlessInteger:$pixelsPerColumn, - Cuda_PointerType:$elementStrides, - Cuda_T32BitSignlessInteger:$interleave, - Cuda_T32BitSignlessInteger:$mode, - Cuda_T32BitSignlessInteger:$swizzle, - Cuda_T32BitSignlessInteger:$l2Promotion, - Cuda_T32BitSignlessInteger:$oobFill - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDATensorMapType:$tensorMap - ); -} -def Cuda_DriverTensorMapReplaceAddressOp : Cuda_Op<"driver.tensor_map.replace_address"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_PointerType:$globalAddress - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDATensorMapType:$tensorMap - ); -} -def Cuda_EventCreateOp : Cuda_Op<"event.create"> { - let description = [{ - Operation to create a CUDA event. - }]; - let arguments = (ins); - let results = (outs - Cuda_CUDAResultType:$result, - Cuda_EventRepresentsCUDAEvent:$event - ); -} -def Cuda_EventDestroyOp : Cuda_Op<"event.destroy"> { - let description = [{ - Operation to destroy a CUDA event. - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$event - ); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_EventElapsedTimeOp : Cuda_Op<"event.elapsed_time"> { - let description = [{ - Operation to compute the elapsed time between events - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$start_event, - Cuda_EventRepresentsCUDAEvent2:$end_event - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_T32BitFloat:$elapsed_time - ); -} -def Cuda_EventRecordOp : Cuda_Op<"event.record"> { - let description = [{ - Operation to record an event in a CUDA stream. - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$event, - Optional:$stream - ); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_FuncOp : Cuda_Op<"func"> { - let description = [{ - This operation represents a CUDA device function which can be called from - another `cuda.func` or a `cuda.kernel`. It cannot be launched from the host. - - Examples: - - ```mlir - - cuda.module @module_name { - - cuda.func @device_func(%arg0 : i32) -> i32 { - %0 = arith.constant 0 : i32 - cuda.return %0 : i32 - } - - } - ``` - }]; - let arguments = (ins - SymbolNameAttr:$sym_name, - Cuda_FuncFunction_typeAttr:$function_type, - OptionalAttr:$arg_attrs, - OptionalAttr:$res_attrs - ); - let results = (outs); - let regions = (region AnyRegion:$body); -} -def Cuda_FuncGetNameOp : Cuda_Op<"func.get_name"> { - let arguments = (ins - Cuda_CUDAKernelHandle2:$kernel - ); - let results = (outs - Cuda_CUDAResultType2:$result, - Cuda_PointerType:$name - ); -} -def Cuda_GraphDestroyOp : Cuda_Op<"graph.destroy"> { - let description = [{ - Operation to destroy a CUDA graph. - }]; - let arguments = (ins - Cuda_CUDAGraph:$graph - ); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_GraphInstantiateOp : Cuda_Op<"graph.instantiate"> { - let description = [{ - Operation to instantiate a CUDA graph. - }]; - let arguments = (ins - Cuda_CUDAGraph:$graph - ); - let results = (outs - Cuda_CUDAResultType:$result, - Cuda_CUDAGraphExecution:$graph_exec - ); -} -def Cuda_GraphLaunchOp : Cuda_Op<"graph.launch"> { - let description = [{ - Operation to launch a CUDA graph. - }]; - let arguments = (ins - Cuda_CUDAGraphExecution2:$graph_exec, - Cuda_CUDAStream:$stream - ); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_GraphExecDestroyOp : Cuda_Op<"graph_exec.destroy"> { - let description = [{ - Operation to destroy a CUDA graph execution. - }]; - let arguments = (ins - Cuda_CUDAGraphExecution2:$graph_exec - ); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_KernelOp : Cuda_Op<"kernel"> { - let description = [{ - This operation represents a CUDA kernel which may be launched as a grid. It - differs from a `cuda.func`. - - Examples: - - ```mlir - - cuda.module @module_name { - - cuda.func @kernel(%arg0 : i32) - attributes { - cu_attrs = { - num_regs = 128, - max_dynamic_shared_size_bytes = 2048 - // ... add more cuda function attributes here - } - } - } - - ... - - %res = cuda.launch @module_name::@device_func <<< %grid, %block, %cluster, %smem, %stream >>>(%0) - - %0 = cuda.result.succeeded(%res) - scf.if %0 { - - } - else { - - } - ``` - }]; - let arguments = (ins - SymbolNameAttr:$sym_name, - Cuda_KernelFunction_typeAttr:$function_type, - OptionalAttr:$arg_attrs, - OptionalAttr:$res_attrs, - OptionalAttr:$cu_func_attrs - ); - let results = (outs); - let regions = (region AnyRegion:$body); -} -def Cuda_LaunchOp : Cuda_Op<"launch", [AttrSizedOperandSegments]> { - let description = [{ - Operation to launch a `cuda.kernel` on the device. - - Examples: - - ```mlir - - cuda.module @module_name { - - cuda.func @kernel(%arg0 : i32) { - - } - - } - - ... - - %0 = ... // kernel arguments - - %res = cuda.launch @module_name::@device_func<<< %grid, %block, %smem, %stream >>>(%0) : !cuda.launch_cfg, ... -> !cuda.result - - // If calling with an Op that implements a FunctionOpInterface - %res = cuda.launch @module_name::@device_func<<< %grid, %block, %smem, %stream >>>(%0) : !cuda.launch_cfg, ... -> !cuda.result - - %res2 = cuda.result.succeeded(%res) - scf.if %res2 { - - } - else { - - } - ``` - }]; - let arguments = (ins - Variadic:$grid, - Variadic:$block, - Cuda_T64BitSignlessInteger:$dynamicSmem, - Cuda_CUDAStream:$cudaStream, - Variadic:$inputs, - SymbolRefAttr:$callee, - OptionalAttr:$arg_attrs, - OptionalAttr:$res_attrs, - OptionalAttr:$assume_kernel_attr - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_LaunchCfgAccessPolicyWindowOp : Cuda_Op<"launch_cfg.access_policy_window"> { - let description = [{ - Sets accessPolicyWindow for a launch_config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create(...) - cuda.launch_cfg.access_policy_window [%cfg0] ( - base_ptr=%basePtr, - hitProp=%hitProp, - hitRatio=%hitRatio, - missProp=%missProp, - numBytes=%numBytes - ) : !cuda.launch_cfg, !ptr.ptr<#ptr.generic_space>, f32, i64 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_PointerType:$base_ptr, - Cuda_T32BitFloat2:$hitRatio, - Cuda_T64BitSignlessInteger:$num_bytes, - Cuda_CudaAccessPropertyEnum:$hitProp, - Cuda_CudaAccessPropertyEnum:$missProp - ); - let results = (outs); -} -def Cuda_LaunchCfgClusterDimOp : Cuda_Op<"launch_cfg.cluster_dim"> { - let description = [{ - Sets clusterDim for a launch_config - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create(...) - cuda.launch_cfg.cluster_dim [%cfg0] (1, 2, 8) : !cuda.launch_cfg, i32, i32, i32 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_T32BitSignlessInteger:$clusterDimX, - Cuda_T32BitSignlessInteger:$clusterDimY, - Cuda_T32BitSignlessInteger:$clusterDimZ - ); - let results = (outs); -} -def Cuda_LaunchCfgClusterSchedulingPolicyOp : Cuda_Op<"launch_cfg.cluster_scheduling_policy"> { - let description = [{ - Sets clusterSchedulingPolicyPreference for a launch_config - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.cluster_scheduling_policy [%cfg0] cudaClusterSchedulingPolicyLoadBalancing - : !cuda.launch_cfg - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_CudaClusterSchedulingPolicyEnum:$policy - ); - let results = (outs); -} -def Cuda_LaunchCfgCooperativeOp : Cuda_Op<"launch_cfg.cooperative"> { - let description = [{ - Sets cooperative attribute to a launch_config. Nonzero indicates a cooperative kernel. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.cooperative [%cfg0] 1 - : !cuda.launch_cfg, i32 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_T32BitSignlessInteger:$cooperative - ); - let results = (outs); -} -def Cuda_LaunchCfgCreateOp : Cuda_Op<"launch_cfg.create"> { - let description = [{ - Creates a launch_config object with the specified runtime values. - The result is a Cuda_LaunchConfig type that contains all the runtime data. - - Example: - ```mlir - %cfg = cuda.launch_cfg.create( - blockDim=(%block_x, %block_y, %block_z), - dynamicSmemBytes=%dynamic_smem_bytes, - gridDim=(%grid_x, %grid_y, %grid_z), - stream=%hstream - ) : i32, i32, i32, i64, i32, i32, i32, !cuda.stream -> !cuda.launch_cfg - ``` - }]; - let arguments = (ins - Cuda_T32BitSignlessInteger:$blockDimX, - Cuda_T32BitSignlessInteger:$blockDimY, - Cuda_T32BitSignlessInteger:$blockDimZ, - Cuda_T64BitSignlessInteger:$dynamicSmemBytes, - Cuda_T32BitSignlessInteger:$gridDimX, - Cuda_T32BitSignlessInteger:$gridDimY, - Cuda_T32BitSignlessInteger:$gridDimZ, - Cuda_CUDAStream:$stream, - Cuda_IntegerAttr:$maxNumAttrs - ); - let results = (outs - Cuda_RepresentationOfCUDASExtensibleLaunch:$result - ); -} -def Cuda_LaunchCfgCreateFromStreamOp : Cuda_InferredOp<"launch_cfg.create_from_stream"> { - let description = [{ - Creates a launch_config object initialized with a CUDA stream and space for - the specified number of launch attributes. Grid dimensions, block - dimensions, and dynamic shared memory bytes can be supplied later with - `cuda.launch_cfg.grid_dimensions`. - - Example: - ```mlir - %cfg = cuda.launch_cfg.create_from_stream( - stream=%hstream - ) : !cuda.stream - ``` - }]; - let arguments = (ins - Cuda_CUDAStream:$stream, - Cuda_IntegerAttr:$maxNumAttrs - ); - let results = (outs - Cuda_RepresentationOfCUDASExtensibleLaunch:$result - ); -} -def Cuda_LaunchCfgDeviceUpdatableKernelNodeOp : Cuda_Op<"launch_cfg.device_updatable_kernel_node"> { - let description = [{ - Sets deviceUpdatableKernelNode attribute to a launch_config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.device_updatable_kernel_node [%cfg0] (devNode=%devNode, deviceUpdatable=%deviceUpdatable) - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_GraphDeviceNodeRepresentsCUDADeviceNodeHandle:$devNode, - Cuda_T32BitSignlessInteger:$deviceUpdatable - ); - let results = (outs); -} -def Cuda_LaunchCfgGridDimensionsOp : Cuda_Op<"launch_cfg.grid_dimensions"> { - let description = [{ - Sets the fixed launch dimensions for a launch_config object. This updates - the direct `cudaLaunchConfig_t` fields, not the launch attribute array. - - Example: - ```mlir - cuda.launch_cfg.grid_dimensions [%cfg0] - grid(%gx, %gy, %gz), block(%bx, %by, %bz), dynamicSmem(%smem) - : !cuda.launch_cfg, i32, i32, i32, i32, i32, i32, i64 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_T32BitSignlessInteger:$gridDimX, - Cuda_T32BitSignlessInteger:$gridDimY, - Cuda_T32BitSignlessInteger:$gridDimZ, - Cuda_T32BitSignlessInteger:$blockDimX, - Cuda_T32BitSignlessInteger:$blockDimY, - Cuda_T32BitSignlessInteger:$blockDimZ, - Cuda_T64BitSignlessInteger:$dynamicSmemBytes - ); - let results = (outs); -} -def Cuda_LaunchCfgLaunchCompletionEventOp : Cuda_Op<"launch_cfg.launch_completion_event"> { - let description = [{ - Sets launchCompletionEvent attribute to a launch_config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.launch_completion_event [%cfg0] (event=%event, flags=%flags) - : !cuda.launch_cfg, !cuda.event, i32 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_EventRepresentsCUDAEvent2:$event, - Cuda_T32BitSignlessInteger:$flags - ); - let results = (outs); -} -def Cuda_LaunchCfgMemSyncDomainOp : Cuda_Op<"launch_cfg.mem_sync_domain"> { - let description = [{ - Sets memSyncDomain attribute to a launch_config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.mem_sync_domain [%cfg0] cudaMemSyncDomainRemote - : !cuda.launch_cfg - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_CudaLaunchMemSyncDomainEnum:$memSyncDomain - ); - let results = (outs); -} -def Cuda_LaunchCfgMemSyncDomainMapOp : Cuda_Op<"launch_cfg.mem_sync_domain_map"> { - let description = [{ - Sets memSyncDomainMap attribute to a launch_config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.mem_sync_domain_map [%cfg0] (domain_default=0, domain_remote=1) - : !cuda.launch_cfg, i32, i32 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_T32BitSignlessInteger:$domain_default, - Cuda_T32BitSignlessInteger:$domain_remote - ); - let results = (outs); -} -def Cuda_LaunchCfgNvlinkUtilCentricSchedulingOp : Cuda_Op<"launch_cfg.nvlink_util_centric_scheduling"> { - let description = [{ - Sets nvlinkUtilCentricScheduling attribute to a launch_config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.nvlink_util_centric_scheduling [%cfg0] 1 : !cuda.launch_cfg, i32 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_T32BitSignlessInteger:$nvlinkSchedulingEnabled - ); - let results = (outs); -} -def Cuda_LaunchCfgPreferredClusterDimOp : Cuda_Op<"launch_cfg.preferred_cluster_dim"> { - let description = [{ - Sets preferredClusterDim attribute to a launch_config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.preferred_cluster_dim [%cfg0] (1, 1, 8) - : !cuda.launch_cfg, i32, i32, i32 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_T32BitSignlessInteger:$preferredClusterDimX, - Cuda_T32BitSignlessInteger:$preferredClusterDimY, - Cuda_T32BitSignlessInteger:$preferredClusterDimZ - ); - let results = (outs); -} -def Cuda_LaunchCfgPrintOp : Cuda_Op<"launch_cfg.print"> { - let description = [{ - Prints a cuda launch config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create(...) - cuda.launch_cfg.print [%cfg0] : !cuda.launch_cfg - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg - ); - let results = (outs); -} -def Cuda_LaunchCfgPriorityOp : Cuda_Op<"launch_cfg.priority"> { - let description = [{ - Sets priority attribute to a launch_config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.priority [%cfg0] 1 - : !cuda.launch_cfg, i32 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_T32BitSignlessInteger:$priority - ); - let results = (outs); -} -def Cuda_LaunchCfgProgrammaticEventOp : Cuda_Op<"launch_cfg.programmatic_event"> { - let description = [{ - Sets programmaticEvent attribute to a launch_config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.programmatic_event [%cfg0] ( - event=%event, - flags=%flags, - triggerAtBlockStart=%triggerAtBlockStart) - : !cuda.launch_cfg, !cuda.event, i32, i32 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_EventRepresentsCUDAEvent2:$event, - Cuda_T32BitSignlessInteger:$flags, - Cuda_T32BitSignlessInteger:$triggerAtBlockStart - ); - let results = (outs); -} -def Cuda_LaunchCfgProgrammaticStreamSerializationAllowedOp : Cuda_Op<"launch_cfg.programmatic_stream_serialization_allowed"> { - let description = [{ - Sets programmaticStreamSerializationAllowed attribute to a launch_config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.programmatic_stream_serialization_allowed [%cfg0] 1 - : !cuda.launch_cfg, i32 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_T32BitSignlessInteger:$programmaticStreamSerializationAllowed - ); - let results = (outs); -} -def Cuda_LaunchCfgSharedMemCarveoutOp : Cuda_Op<"launch_cfg.shared_mem_carveout"> { - let description = [{ - Sets sharedMemCarveout attribute to a launch_config. - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.shared_mem_carveout [%cfg0] 1024 - : !cuda.launch_cfg, i32 - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_T32BitSignlessInteger:$sharedMemCarveout - ); - let results = (outs); -} -def Cuda_LaunchCfgSyncPolicyOp : Cuda_Op<"launch_cfg.sync_policy"> { - let description = [{ - Sets syncPolicy for a launch_config - - Example: - ```mlir - %cfg0 = cuda.launch_cfg.create... - cuda.launch_cfg.sync_policy [%cfg0] cudaSynchronizationPolicyDefault - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Cuda_CudaSynchronizationPolicyEnum:$syncPolicy - ); - let results = (outs); -} -def Cuda_LaunchExOp : Cuda_Op<"launch_ex"> { - let description = [{ - Operation to launch a `cuda.kernel` on the device using extensible launch API. - - Examples: - - ```mlir - - cuda.module @module_name { - - cuda.func @kernel(%arg0 : i32) { - - } - - } - - ... - %0 = ... // kernel arguments - - %res = cuda.launch_ex @module_name::@device_func<%cfg>(%0) : !cuda.launch_cfg, ... -> !cuda.result - - // If calling with an Op that implements a FunctionOpInterface - %res = cuda.launch_ex @module_name::@device_func<%cfg>(%0) : !cuda.launch_cfg, ... -> !cuda.result - - %res2 = cuda.result.succeeded(%res) - scf.if %res2 { - - } - else { - - } - ``` - }]; - let arguments = (ins - Cuda_RepresentationOfCUDASExtensibleLaunch:$launch_cfg, - Variadic:$inputs, - SymbolRefAttr:$callee, - OptionalAttr:$arg_attrs, - OptionalAttr:$res_attrs, - OptionalAttr:$assume_kernel_attr - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_LaunchKernelOp : Cuda_Op<"launch_kernel", [AttrSizedOperandSegments]> { - let description = [{ - Operation to launch a kernel from a kernel pointer on the device. This launch API - directly follows the CUDA C++ runtime API, passing all kernel arguments as a pointer - to a struct. - - Example: - - ```mlir - %res2 = cuda.launch_kernel %kernel, (%one32, %one32, %one32), (%one32, %one32, %one32), %zero64, %stream, %zeroPtr : !ptr.ptr<#ptr.generic_space> -> !cuda.result - - ``` - }]; - let arguments = (ins - Cuda_CUDAKernelHandle2:$kernel, - Variadic:$grid, - Variadic:$block, - Cuda_T64BitSignlessInteger:$dynamicSmem, - Cuda_CUDAStream:$cudaStream, - Cuda_PointerType:$kernel_args, - OptionalAttr:$arg_attrs, - OptionalAttr:$res_attrs - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_LibraryGetKernelOp : Cuda_Op<"library.get_kernel"> { - let description = [{ - Return a kernel handle object if found in the given library. - - ## Example - - ``` - %result_0, %kernel = cuda.library.get_kernel %l1, %name : !cuda.library, !ptr.ptr<#ptr.generic_space> -> !cuda.result, !cuda.kernel - ``` - - ## Behavior - - This op executes the CUDA API function cudaGetKernel. If the kernel is not found the cuda result will be `cudaErrorSymbolNotFound`. If the `kernel_name` is not a valid, null-terminated ASCII string, the behavior is UNDEFINED. Upon a result of `cudaSuccess`, `kernel` contains a valid kernel handle object. Is the result is not `cudaSuccess`, and use of the `kernel_name` handle is UNDEFINED. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library, - Cuda_PointerType:$kernel_name - ); - let results = (outs - Cuda_CUDAResultType:$result, - Cuda_CUDAKernelHandle:$kernel - ); -} -def Cuda_LibraryLoadDataOp : Cuda_Op<"library.load_data"> { - let description = [{ - Load a library from a pointer. - - ## Example - - ``` - %result,%l1 = cuda.library.load_data %modptr : !ptr.ptr<#ptr.generic_space> -> !cuda.result, !cuda.library - ``` - ## Behavior - - This op execute the CUDA API function cudaLibraryLoadData. Whether the library is loaded immediately depends on the module load environment. - }]; - let arguments = (ins - Cuda_PointerType:$data - ); - let results = (outs - Cuda_CUDAResultType:$result, - Cuda_CUDALibraryHandle2:$library - ); -} -def Cuda_LibraryUnloadOp : Cuda_Op<"library.unload"> { - let arguments = (ins - Cuda_CUDALibraryHandle:$library - ); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_MemoryAllocOp : Cuda_Op<"memory.alloc"> { - let description = [{ - Operation to allocate device memory. Calls `cuMemAlloc()`. - - Examples: - ```mlir - - %bytes = arith.constant 0 : i64 - %res, %device_mem = cuda.memory.alloc(%bytes) : i64 -> cuda.result, cuda.ptr - - %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) - - %res1 = cuda.memory.free(%device_mem) : i64 -> cuda.result - ``` - }]; - let arguments = (ins - Cuda_T64BitSignlessInteger:$bytes - ); - let results = (outs - Cuda_CUDAResultType2:$cuda_result, - Cuda_PointerType:$ptr - ); -} -def Cuda_MemoryAllocAsyncOp : Cuda_Op<"memory.alloc_async"> { - let description = [{ - Operation to allocate device memory asynchronously. Calls `cuMemAllocAsync()`. - - Examples: - ```mlir - - %res, %device_mem = cuda.memory.alloc_async(%bytes, %stream) : i64, cuda.ptr -> cuda.result - - cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) - - $res1 = cuda.memory.free_async(%device_mem, %stream) : i64, !cuda.stream -> cuda.result - ``` - }]; - let arguments = (ins - Cuda_T64BitSignlessInteger:$bytes, - Cuda_CUDAStream:$stream - ); - let results = (outs - Cuda_CUDAResultType2:$cuda_result, - Cuda_PointerType:$ptr - ); -} -def Cuda_MemoryAllocHostOp : Cuda_Op<"memory.alloc_host"> { - let description = [{ - Operation to allocate host memory. Calls `cuMemAllocHost()`. - - Examples: - ```mlir - - %bytes = arith.constant 0 : i64 - %host_mem, %res = cuda.memory.alloc_host(%bytes) : (i64) -> (cuda.ptr, cuda.result) - - %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%host_mem) - - %res1 = cuda.memory.free_host(%host_mem) - ``` - }]; - let arguments = (ins - Cuda_T64BitSignlessInteger:$bytes - ); - let results = (outs - Cuda_CUDAResultType2:$cuda_result, - Cuda_PointerType:$ptr - ); -} -def Cuda_MemoryAllocManagedOp : Cuda_Op<"memory.alloc_managed"> { - let description = [{ - Operation to allocate managed memory. - - Examples: - ```mlir - - %bytes = arith.constant 0 : i64 - %managed_mem, %res = cuda.memory.alloc_managed(%bytes) : (i64) -> (cuda.ptr, cuda.result) - - %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%managed_mem) - - %res1 = cuda.memory.free(%managed_mem) - ``` - }]; - let arguments = (ins - Cuda_T64BitSignlessInteger:$bytes - ); - let results = (outs - Cuda_CUDAResultType2:$cuda_result, - Cuda_PointerType:$ptr - ); -} -def Cuda_MemoryCopyOp : Cuda_Op<"memory.copy"> { - let description = [{ - Operation to copy memory between host and device. - - Examples: - ```mlir - - %bytes = arith.constant 1024 : i64 - - %device_mem, %res = cuda.memory.alloc %bytes : i64 -> (!cuda.result, !ptr.ptr) - - %res = cuda.memory.copy %device_mem, %host_mem, %bytes [HostToDevice] - - %res = cuda.launch @module::@kernel<<< (%gridX, %gridY, %gridZ), (%blockX, %blockY, %blockZ), %smem, %stream >>>(%device_mem) - - %res = cuda.memory.copy %host_mem, %device_mem, %bytes [DeviceToHost] - - %res = cuda.memory.free %device_mem : !ptr.ptr -> !cuda.result - ``` - }]; - let arguments = (ins - Cuda_PointerType:$src, - Cuda_PointerType:$dst, - Cuda_T64BitSignlessInteger:$bytes, - Cuda_MemcpyKindEnum:$kind - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_MemoryCopyAsyncOp : Cuda_Op<"memory.copy_async"> { - let description = [{ - Operation to copy memory between host and device asynchronously. - - Examples: - ```mlir - - %bytes = arith.constant 1024 : i64 - - %device_mem, %res = cuda.memory.alloc %bytes : i64 -> (!cuda.result, !ptr.ptr) - - %res = cuda.memory.copy_async %device_mem, %host_mem, %bytes [HostToDevice] - - %res = cuda.launch @module::@kernel<<< (%gridX, %gridY, %gridZ), (%blockX, %blockY, %blockZ), %smem, %stream >>>(%device_mem) - - %res = cuda.memory.copy_async %host_mem, %device_mem, %bytes [DeviceToHost] - - %res = cuda.memory.free %device_mem : !ptr.ptr -> !cuda.result - ``` - }]; - let arguments = (ins - Cuda_PointerType:$src, - Cuda_PointerType:$dst, - Cuda_T64BitSignlessInteger:$bytes, - Optional:$stream, - Cuda_MemcpyKindEnum:$kind - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_MemoryFreeOp : Cuda_Op<"memory.free"> { - let description = [{ - Operation to free allocated device memory. - - Examples: - ```mlir - - %bytes = arith.constant 0 : i64 - %res, %device_mem = cuda.memory.alloc(%bytes) : i64 -> cuda.result, cuda.ptr - - %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) - - %res1 = cuda.memory.free(%device_mem) : i64 -> cuda.result - ``` - }]; - let arguments = (ins - Cuda_PointerType:$ptr - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_MemoryFreeAsyncOp : Cuda_Op<"memory.free_async"> { - let description = [{ - Operation to free allocated device memory asynchronously. - - Examples: - ```mlir - - %res, %device_mem = cuda.memory.alloc_async(%bytes, %stream) : i64, cuda.ptr -> cuda.result - - cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%device_mem) - - $res1 = cuda.memory.free_async(%device_mem, %stream) : i64, !cuda.stream -> cuda.result - ``` - - Preserves semantics of [cuMemAllocAsync](https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MALLOC__ASYNC.html#group__CUDA__MALLOC__ASYNC_1g13413273e84a641bce1929eae9e6501f) - }]; - let arguments = (ins - Cuda_PointerType:$ptr, - Cuda_CUDAStream:$stream - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_MemoryFreeHostOp : Cuda_Op<"memory.free_host"> { - let description = [{ - Operation to free allocated host memory. - - Examples: - ```mlir - - %bytes = arith.constant 0 : i64 - %host_mem, %res = cuda.memory.alloc_host(%bytes) : (i64) -> (cuda.ptr, cuda.result) - - %res0 = cuda.launch @module::@kernel<<< %grid, %block, %cluster, %smem, %stream >>>(%host_mem) - - %res1 = cuda.memory.free_host(%host_mem) - ``` - }]; - let arguments = (ins - Cuda_PointerType:$ptr - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_MemorySetOp : Cuda_Op<"memory.set"> { - let description = [{ - Operation to fill memory with a uniform value. - }]; - let arguments = (ins - Cuda_PointerType:$ptr, - Cuda_T32BitSignlessInteger:$value, - Cuda_T64BitSignlessInteger:$bytes - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_MemorySetAsyncOp : Cuda_Op<"memory.set_async"> { - let description = [{ - Operation to fill memory with a uniform value asynchronously. - }]; - let arguments = (ins - Cuda_PointerType:$ptr, - Cuda_T32BitSignlessInteger:$value, - Cuda_T64BitSignlessInteger:$bytes, - Optional:$stream - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_OccupancyMaxActiveBlocks : Cuda_Op<"occupancy_max_active_blocks_per_multiprocessor"> { - let description = [{ - Returns in `num_blocks` the maximum number of active blocks per streaming multiplrocessor for the device function. - - Example: - - ```mlir - %one64 = arith.constant 1 : i64 - %one32 = arith.constant 1 : i32 - %res, %num_blocks = cuda.occupancy_max_active_blocks_per_multiprocessor @funcname, %one32, %one64 : (i32, i64) -> (!cuda.result, i32) - ``` - }]; - let arguments = (ins - Cuda_CudaIntIsAIntegerType:$block_size, - Cuda_T64BitSignlessInteger:$dynamic_smem_size, - SymbolRefAttr:$func - ); - let results = (outs - Cuda_CUDAResultType:$res, - Cuda_T32BitSignlessInteger2:$num_blocks - ); -} -def Cuda_ProfilerStart : Cuda_Op<"profiler.start"> { - let description = [{ - Starts the CUDA profiler. - }]; - let arguments = (ins); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_ProfilerStop : Cuda_Op<"profiler.stop"> { - let description = [{ - Stops the CUDA profiler. - }]; - let arguments = (ins); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_ResultAssertSuccessOp : Cuda_Op<"result.assert_success"> { - let description = [{ - Asserts that a `cuda.result` indicates success. - }]; - let arguments = (ins - Cuda_CUDAResultType2:$cuda_result - ); - let results = (outs); -} -def Cuda_ResultFailedOp : Cuda_Op<"result.failed"> { - let description = [{ - Determines if a `cuda.result` indicates failure. - }]; - let arguments = (ins); - let results = (outs); -} -def Cuda_ResultPrintOnErrorOp : Cuda_Op<"result.print_on_error"> { - let description = [{ - Prints a message on error. - }]; - let arguments = (ins - Cuda_CUDAResultType2:$cuda_result - ); - let results = (outs); -} -def Cuda_ResultSucceededOp : Cuda_Op<"result.succeeded"> { - let description = [{ - Determines if a `cuda.result` indicates success. - }]; - let arguments = (ins - Cuda_CUDAResultType2:$cuda_result - ); - let results = (outs - Cuda_T1BitSignlessInteger:$success - ); -} -def Cuda_ReturnOp : Cuda_Op<"return", [Terminator]> { - let description = [{ - Returns a value from a `cuda.kernel` or `cuda.func` region. - - Note, the emitter seems to omit the dialect and simply prints `return` in the - custom assembly syntax. - - Examples: - - ```mlir - - cuda.module @module_name { - - cuda.func @device_func(%arg0 : i32) -> i32 { - %0 = arith.constant 0 : i32 - cuda.return %0 : i32 - } - - } - ``` - }]; - let arguments = (ins - Variadic:$inputs - ); - let results = (outs); -} -def Cuda_ReturnIfErrorOp : Cuda_Op<"return_if_error"> { - let description = [{ - The first operand must be the !cuda.result to check. - The subsequent operands are the values to be returned if an error exists. - This op is first lowered to an intermediate op `cuda.return_with_error`, - before final conversion to LLVM in a subsequent pass `cuda-finalize-error-handling`. - - Examples: - ```mlir - func.func @host_func() -> i32 { - %ten = arith.constant 10 : i32 - %cuda_result = cuda.launch @module::@kernel... - // Create an undefined/poison value for the error return path. - // Users can also input any arbitrary value for the error return path. - %poison = llvm.mlir.poison : i32 - // return poison if cuda_result is not cudaSuccess - %res_i32 = cuda.cast %cuda_result : !cuda.result -> i32 - cuda.return_if_error %res_i32, %poison : i32, i32 - - // return 10 if execution reaches here - return %ten : i32 - } - ``` - }]; - let arguments = (ins - Variadic:$inputs - ); - let results = (outs); -} -def Cuda_RuntimeGetDevicePropertiesOp : Cuda_Op<"runtime.device.properties"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_T32BitSignlessInteger:$device - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDACudaDevicePropStructTypeValueTyped:$prop - ); -} -def Cuda_RuntimeDevicePropGetAccessPolicyMaxWindowSizeOp : Cuda_Op<"runtime.device_prop.get_access_policy_max_window_size", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetAsyncEngineCountOp : Cuda_Op<"runtime.device_prop.get_async_engine_count", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetCanMapHostMemoryOp : Cuda_Op<"runtime.device_prop.get_can_map_host_memory", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetCanUseHostPointerForRegisteredMemOp : Cuda_Op<"runtime.device_prop.get_can_use_host_pointer_for_registered_mem", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetClusterLaunchOp : Cuda_Op<"runtime.device_prop.get_cluster_launch", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetComputePreemptionSupportedOp : Cuda_Op<"runtime.device_prop.get_compute_preemption_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetConcurrentKernelsOp : Cuda_Op<"runtime.device_prop.get_concurrent_kernels", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetConcurrentManagedAccessOp : Cuda_Op<"runtime.device_prop.get_concurrent_managed_access", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetCooperativeLaunchOp : Cuda_Op<"runtime.device_prop.get_cooperative_launch", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetDeferredMappingCudaArraySupportedOp : Cuda_Op<"runtime.device_prop.get_deferred_mapping_cuda_array_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetDeviceNumaConfigOp : Cuda_Op<"runtime.device_prop.get_device_numa_config", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetDeviceNumaIdOp : Cuda_Op<"runtime.device_prop.get_device_numa_id", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetDirectManagedMemAccessFromHostOp : Cuda_Op<"runtime.device_prop.get_direct_managed_mem_access_from_host", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetECCEnabledOp : Cuda_Op<"runtime.device_prop.get_ecc_enabled", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetGlobalL1CacheSupportedOp : Cuda_Op<"runtime.device_prop.get_global_l1cache_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetGpuDirectRDMAFlushWritesOptionsOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetGpuDirectRDMASupportedOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetGpuDirectRDMAWritesOrderingOp : Cuda_Op<"runtime.device_prop.get_gpu_direct_rdma_writes_ordering", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetGpuPciDeviceIDOp : Cuda_Op<"runtime.device_prop.get_gpu_pci_device_id", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetGpuPciSubsystemIDOp : Cuda_Op<"runtime.device_prop.get_gpu_pci_subsystem_id", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetHostNativeAtomicSupportedOp : Cuda_Op<"runtime.device_prop.get_host_native_atomic_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetHostNumaIdOp : Cuda_Op<"runtime.device_prop.get_host_numa_id", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetHostNumaMultinodeIpcSupportedOp : Cuda_Op<"runtime.device_prop.get_host_numa_multinode_ipc_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetHostRegisterReadOnlySupportedOp : Cuda_Op<"runtime.device_prop.get_host_register_read_only_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetHostRegisterSupportedOp : Cuda_Op<"runtime.device_prop.get_host_register_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetIntegratedOp : Cuda_Op<"runtime.device_prop.get_integrated", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetIpcEventSupportedOp : Cuda_Op<"runtime.device_prop.get_ipc_event_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetIsMultiGpuBoardOp : Cuda_Op<"runtime.device_prop.get_is_multi_gpu_board", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetL2CacheSizeOp : Cuda_Op<"runtime.device_prop.get_l_2cache_size", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetLocalL1CacheSupportedOp : Cuda_Op<"runtime.device_prop.get_local_l1cache_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetLuidOp : Cuda_Op<"runtime.device_prop.get_luid", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf8Bit:$result - ); -} -def Cuda_RuntimeDevicePropGetLuidDeviceNodeMaskOp : Cuda_Op<"runtime.device_prop.get_luid_device_node_mask", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMajorOp : Cuda_Op<"runtime.device_prop.get_major", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetManagedMemoryOp : Cuda_Op<"runtime.device_prop.get_managed_memory", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxBlocksPerMultiProcessorOp : Cuda_Op<"runtime.device_prop.get_max_blocks_per_multi_processor", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxGridSizeOp : Cuda_Op<"runtime.device_prop.get_max_grid_size", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxSurface1DOp : Cuda_Op<"runtime.device_prop.get_max_surface_1d", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxSurface1DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_1d_layered", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxSurface2DOp : Cuda_Op<"runtime.device_prop.get_max_surface_2d", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxSurface2DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_2d_layered", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxSurface3DOp : Cuda_Op<"runtime.device_prop.get_max_surface_3d", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxSurfaceCubemapOp : Cuda_Op<"runtime.device_prop.get_max_surface_cubemap", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxSurfaceCubemapLayeredOp : Cuda_Op<"runtime.device_prop.get_max_surface_cubemap_layered", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTexture1DOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTexture1DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d_layered", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTexture1DMipmapOp : Cuda_Op<"runtime.device_prop.get_max_texture_1d_mipmap", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTexture2DOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTexture2DGatherOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_gather", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTexture2DLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_layered", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTexture2DLinearOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_linear", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTexture2DMipmapOp : Cuda_Op<"runtime.device_prop.get_max_texture_2d_mipmap", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTexture3DOp : Cuda_Op<"runtime.device_prop.get_max_texture_3d", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTexture3DAltOp : Cuda_Op<"runtime.device_prop.get_max_texture_3d_alt", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTextureCubemapOp : Cuda_Op<"runtime.device_prop.get_max_texture_cubemap", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxTextureCubemapLayeredOp : Cuda_Op<"runtime.device_prop.get_max_texture_cubemap_layered", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxThreadsDimOp : Cuda_Op<"runtime.device_prop.get_max_threads_dim", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf32Bit:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxThreadsPerBlockOp : Cuda_Op<"runtime.device_prop.get_max_threads_per_block", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMaxThreadsPerMultiProcessorOp : Cuda_Op<"runtime.device_prop.get_max_threads_per_multi_processor", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMemPitchOp : Cuda_Op<"runtime.device_prop.get_mem_pitch", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMemoryBusWidthOp : Cuda_Op<"runtime.device_prop.get_memory_bus_width", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMemoryPoolSupportedHandleTypesOp : Cuda_Op<"runtime.device_prop.get_memory_pool_supported_handle_types", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMemoryPoolsSupportedOp : Cuda_Op<"runtime.device_prop.get_memory_pools_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMinorOp : Cuda_Op<"runtime.device_prop.get_minor", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMpsEnabledOp : Cuda_Op<"runtime.device_prop.get_mps_enabled", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMultiGpuBoardGroupIDOp : Cuda_Op<"runtime.device_prop.get_multi_gpu_board_group_id", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetMultiProcessorCountOp : Cuda_Op<"runtime.device_prop.get_multi_processor_count", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetNameOp : Cuda_Op<"runtime.device_prop.get_name", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf8Bit2:$result - ); -} -def Cuda_RuntimeDevicePropGetPageableMemoryAccessOp : Cuda_Op<"runtime.device_prop.get_pageable_memory_access", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetPageableMemoryAccessUsesHostPageTablesOp : Cuda_Op<"runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetPciBusIDOp : Cuda_Op<"runtime.device_prop.get_pci_bus_id", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetPciDeviceIDOp : Cuda_Op<"runtime.device_prop.get_pci_device_id", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetPciDomainIDOp : Cuda_Op<"runtime.device_prop.get_pci_domain_id", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetPersistingL2CacheMaxSizeOp : Cuda_Op<"runtime.device_prop.get_persisting_l2cache_max_size", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetRegsPerBlockOp : Cuda_Op<"runtime.device_prop.get_regs_per_block", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetRegsPerMultiprocessorOp : Cuda_Op<"runtime.device_prop.get_regs_per_multiprocessor", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetReservedSharedMemPerBlockOp : Cuda_Op<"runtime.device_prop.get_reserved_shared_mem_per_block", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetSharedMemPerBlockOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_block", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetSharedMemPerBlockOptinOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_block_optin", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetSharedMemPerMultiprocessorOp : Cuda_Op<"runtime.device_prop.get_shared_mem_per_multiprocessor", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetSparseCudaArraySupportedOp : Cuda_Op<"runtime.device_prop.get_sparse_cuda_array_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetStreamPrioritiesSupportedOp : Cuda_Op<"runtime.device_prop.get_stream_priorities_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetSurfaceAlignmentOp : Cuda_Op<"runtime.device_prop.get_surface_alignment", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetTccDriverOp : Cuda_Op<"runtime.device_prop.get_tcc_driver", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetTextureAlignmentOp : Cuda_Op<"runtime.device_prop.get_texture_alignment", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetTexturePitchAlignmentOp : Cuda_Op<"runtime.device_prop.get_texture_pitch_alignment", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetTimelineSemaphoreInteropSupportedOp : Cuda_Op<"runtime.device_prop.get_timeline_semaphore_interop_supported", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetTotalConstMemOp : Cuda_Op<"runtime.device_prop.get_total_const_mem", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetTotalGlobalMemOp : Cuda_Op<"runtime.device_prop.get_total_global_mem", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetUnifiedAddressingOp : Cuda_Op<"runtime.device_prop.get_unified_addressing", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetUnifiedFunctionPointersOp : Cuda_Op<"runtime.device_prop.get_unified_function_pointers", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDevicePropGetUuidOp : Cuda_Op<"runtime.device_prop.get_uuid", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_CUDACudaUUIDTStructTypeValue:$result - ); -} -def Cuda_RuntimeDevicePropGetWarpSizeOp : Cuda_Op<"runtime.device_prop.get_warp_size", [Pure]> { - let arguments = (ins - Cuda_CUDACudaDevicePropStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDim3GetXOp : Cuda_Op<"runtime.dim_3.get_x", [Pure]> { - let arguments = (ins - Cuda_CUDADim3StructTypeValueTyped:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDim3GetYOp : Cuda_Op<"runtime.dim_3.get_y", [Pure]> { - let arguments = (ins - Cuda_CUDADim3StructTypeValueTyped:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeDim3GetZOp : Cuda_Op<"runtime.dim_3.get_z", [Pure]> { - let arguments = (ins - Cuda_CUDADim3StructTypeValueTyped:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeEventCreateWithFlagsOp : Cuda_Op<"runtime.event.create_with_flags"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_T32BitSignlessInteger:$flags - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_EventRepresentsCUDAEvent:$event - ); -} -def Cuda_RuntimeEventQueryOp : Cuda_Op<"runtime.event.query", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$event - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_RuntimeEventRecordWithFlagsOp : Cuda_Op<"runtime.event.record_with_flags"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$event, - Cuda_CUDAStream:$stream, - Cuda_T32BitSignlessInteger:$flags - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_RuntimeEventSynchronizeOp : Cuda_Op<"runtime.event.synchronize"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_EventRepresentsCUDAEvent2:$event - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_RuntimeExtentGetDepthOp : Cuda_Op<"runtime.extent.get_depth", [Pure]> { - let arguments = (ins - Cuda_CUDACudaExtentStructTypeValueTyped:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeExtentGetHeightOp : Cuda_Op<"runtime.extent.get_height", [Pure]> { - let arguments = (ins - Cuda_CUDACudaExtentStructTypeValueTyped:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeExtentGetWidthOp : Cuda_Op<"runtime.extent.get_width", [Pure]> { - let arguments = (ins - Cuda_CUDACudaExtentStructTypeValueTyped:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeGraphAddKernelNodeOp : Cuda_Op<"runtime.graph.add_kernel_node"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDAGraph:$graph, - Cuda_PointerType:$pDependencies, - Cuda_T64BitSignlessInteger:$numDependencies, - Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$pNodeParams - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDAGraphNodeHandle:$pGraphNode - ); -} -def Cuda_RuntimeGraphCreateOp : Cuda_Op<"runtime.graph.create"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_T32BitSignlessInteger:$flags - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDAGraph2:$pGraph - ); -} -def Cuda_RuntimeGraphKernelNodeGetAttributeOp : Cuda_Op<"runtime.graph.kernel_node_get_attribute", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDAGraphNodeHandle2:$hNode, - Cuda_T32BitSignlessInteger:$attr - ); - let results = (outs - Cuda_CUDAResultType2:$cuda_result, - Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion:$value_out - ); -} -def Cuda_RuntimeGraphKernelNodeSetAttributeOp : Cuda_Op<"runtime.graph.kernel_node_set_attribute"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDAGraphNodeHandle2:$hNode, - Cuda_T32BitSignlessInteger:$attr, - Cuda_VariantCooperativeOfCUDACudaLaunchAttributeValueUnion:$value - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_RuntimeKernelNodeParamsGetBlockDimOp : Cuda_Op<"runtime.kernel_node_params.get_block_dim", [Pure]> { - let arguments = (ins - Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_CUDADim3StructTypeValueTyped2:$result - ); -} -def Cuda_RuntimeKernelNodeParamsGetExtraOp : Cuda_Op<"runtime.kernel_node_params.get_extra", [Pure]> { - let arguments = (ins - Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_LLVMPointerType:$result - ); -} -def Cuda_RuntimeKernelNodeParamsGetFuncOp : Cuda_Op<"runtime.kernel_node_params.get_func", [Pure]> { - let arguments = (ins - Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_LLVMPointerType:$result - ); -} -def Cuda_RuntimeKernelNodeParamsGetGridDimOp : Cuda_Op<"runtime.kernel_node_params.get_grid_dim", [Pure]> { - let arguments = (ins - Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_CUDADim3StructTypeValueTyped2:$result - ); -} -def Cuda_RuntimeKernelNodeParamsGetKernelParamsOp : Cuda_Op<"runtime.kernel_node_params.get_kernel_params", [Pure]> { - let arguments = (ins - Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_LLVMPointerType:$result - ); -} -def Cuda_RuntimeKernelNodeParamsGetSharedMemBytesOp : Cuda_Op<"runtime.kernel_node_params.get_shared_mem_bytes", [Pure]> { - let arguments = (ins - Cuda_CUDACudaKernelNodeParamsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeLibraryGetGlobalOp : Cuda_Op<"runtime.library.get_global", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library, - Cuda_PointerType:$name_arg - ); - let results = (outs - Cuda_CUDAResultType2:$cuda_result, - Cuda_PointerType:$dptr, - Cuda_T64BitSignlessInteger:$bytes - ); -} -def Cuda_RuntimeLibraryGetKernelOp : Cuda_Op<"runtime.library.get_kernel", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library, - Cuda_PointerType:$name_arg - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDAKernelHandle:$pKernel - ); -} -def Cuda_RuntimeLibraryGetKernelCountOp : Cuda_Op<"runtime.library.get_kernel_count", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$lib - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_T32BitSignlessInteger2:$count - ); -} -def Cuda_RuntimeLibraryGetManagedOp : Cuda_Op<"runtime.library.get_managed", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library, - Cuda_PointerType:$name_arg - ); - let results = (outs - Cuda_CUDAResultType2:$cuda_result, - Cuda_PointerType:$dptr, - Cuda_T64BitSignlessInteger:$bytes - ); -} -def Cuda_RuntimeLibraryGetUnifiedFunctionOp : Cuda_Op<"runtime.library.get_unified_function", [MemoryEffects<[MemRead]>]> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library, - Cuda_PointerType:$symbol - ); - let results = (outs - Cuda_CUDAResultType2:$cuda_result, - Cuda_PointerType:$fptr - ); -} -def Cuda_RuntimeLibraryLoadDataOp : Cuda_Op<"runtime.library.load_data"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_PointerType:$code_arg, - Cuda_PointerType:$jitOptions, - Cuda_PointerType:$jitOptionsValues, - Cuda_T32BitSignlessInteger:$numJitOptions, - Cuda_PointerType:$libraryOptions, - Cuda_PointerType:$libraryOptionValues, - Cuda_T32BitSignlessInteger:$numLibraryOptions - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDALibraryHandle2:$library - ); -} -def Cuda_RuntimeLibraryLoadFromFileOp : Cuda_Op<"runtime.library.load_from_file"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_PointerType:$fileName, - Cuda_PointerType:$jitOptions, - Cuda_PointerType:$jitOptionsValues, - Cuda_T32BitSignlessInteger:$numJitOptions, - Cuda_PointerType:$libraryOptions, - Cuda_PointerType:$libraryOptionValues, - Cuda_T32BitSignlessInteger:$numLibraryOptions - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDALibraryHandle2:$library - ); -} -def Cuda_RuntimeLibraryUnloadOp : Cuda_Op<"runtime.library.unload"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDALibraryHandle:$library - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_RuntimeMemcpy3DParamsGetDstArrayOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_array", [Pure]> { - let arguments = (ins - Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_PointerType:$result - ); -} -def Cuda_RuntimeMemcpy3DParamsGetDstPosOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_pos", [Pure]> { - let arguments = (ins - Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_CUDACudaPosStructTypeValueTyped:$result - ); -} -def Cuda_RuntimeMemcpy3DParamsGetDstPtrOp : Cuda_Op<"runtime.memcpy_3d_params.get_dst_ptr", [Pure]> { - let arguments = (ins - Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_CUDACudaPitchedPtrStructTypeValueTyped:$result - ); -} -def Cuda_RuntimeMemcpy3DParamsGetExtentOp : Cuda_Op<"runtime.memcpy_3d_params.get_extent", [Pure]> { - let arguments = (ins - Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_CUDACudaExtentStructTypeValueTyped2:$result - ); -} -def Cuda_RuntimeMemcpy3DParamsGetKindOp : Cuda_Op<"runtime.memcpy_3d_params.get_kind", [Pure]> { - let arguments = (ins - Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_T32BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeMemcpy3DParamsGetSrcArrayOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_array", [Pure]> { - let arguments = (ins - Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_PointerType:$result - ); -} -def Cuda_RuntimeMemcpy3DParamsGetSrcPosOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_pos", [Pure]> { - let arguments = (ins - Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_CUDACudaPosStructTypeValueTyped:$result - ); -} -def Cuda_RuntimeMemcpy3DParamsGetSrcPtrOp : Cuda_Op<"runtime.memcpy_3d_params.get_src_ptr", [Pure]> { - let arguments = (ins - Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$source - ); - let results = (outs - Cuda_CUDACudaPitchedPtrStructTypeValueTyped:$result - ); -} -def Cuda_RuntimeMemcpy3DOp : Cuda_Op<"runtime.memory.cpy3d"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$p - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_RuntimeMemcpy3DAsyncOp : Cuda_Op<"runtime.memory.cpy3d_async"> { - let description = [{ - Base class for all CUDA driver call builder operations. It provides a common - interface for building CUDA driver calls, including the ability to specify - the API to be called, the arguments, and the expected results. - This class is not intended to be used directly, but rather as a base class - for specific CUDA driver call builder operations. - }]; - let arguments = (ins - Cuda_CUDACudaMemcpy3DParmsStructTypeValueTyped:$p, - Cuda_CUDAStream:$stream - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result - ); -} -def Cuda_RuntimePitchedPtrGetPitchOp : Cuda_Op<"runtime.pitched_ptr.get_pitch", [Pure]> { - let arguments = (ins - Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimePitchedPtrGetPtrOp : Cuda_Op<"runtime.pitched_ptr.get_ptr", [Pure]> { - let arguments = (ins - Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_LLVMPointerType:$result - ); -} -def Cuda_RuntimePitchedPtrGetXsizeOp : Cuda_Op<"runtime.pitched_ptr.get_xsize", [Pure]> { - let arguments = (ins - Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimePitchedPtrGetYsizeOp : Cuda_Op<"runtime.pitched_ptr.get_ysize", [Pure]> { - let arguments = (ins - Cuda_CUDACudaPitchedPtrStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimePosGetXOp : Cuda_Op<"runtime.pos.get_x", [Pure]> { - let arguments = (ins - Cuda_CUDACudaPosStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimePosGetYOp : Cuda_Op<"runtime.pos.get_y", [Pure]> { - let arguments = (ins - Cuda_CUDACudaPosStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimePosGetZOp : Cuda_Op<"runtime.pos.get_z", [Pure]> { - let arguments = (ins - Cuda_CUDACudaPosStructTypeValueTyped2:$source - ); - let results = (outs - Cuda_T64BitSignlessInteger2:$result - ); -} -def Cuda_RuntimeUuidGetBytesOp : Cuda_Op<"runtime.uuid.get_bytes", [Pure]> { - let arguments = (ins - Cuda_CUDACudaUUIDTStructTypeValue2:$source - ); - let results = (outs - Cuda_FixedLengthVectorOf8Bit3:$result - ); -} -def Cuda_StreamAddCallbackOp : Cuda_Op<"stream.add_callback"> { - let description = [{ - Enqueues a host-side callback on the given CUDA stream. The callback is provided - as a symbol to a `func.func` with ABI: - `func @cb(!ptr.ptr<#ptr.generic_space>, i32, !ptr.ptr<#ptr.generic_space>) -> ()` - i.e., (stream, status, userData) -> void. - }]; - let arguments = (ins - Cuda_CUDAStream:$stream, - Cuda_PointerType:$user_data, - FlatSymbolRefAttr:$callback - ); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_StreamBeginCaptureOp : Cuda_Op<"stream.begin_capture"> { - let description = [{ - Operation to begin capturing a CUDA stream. - }]; - let arguments = (ins - Cuda_CUDAStream:$stream, - Cuda_StreamCaptureModeEnumEnum:$mode - ); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_StreamCreateOp : Cuda_Op<"stream.create"> { - let description = [{ - Operation to create a CUDA stream. - }]; - let arguments = (ins); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDAStream2:$stream - ); -} -def Cuda_StreamDestroyOp : Cuda_Op<"stream.destroy"> { - let description = [{ - Operation to destroy a CUDA stream. - }]; - let arguments = (ins - Cuda_CUDAStream:$stream - ); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_StreamEndCaptureOp : Cuda_Op<"stream.end_capture"> { - let description = [{ - Operation to end capturing a CUDA stream. - }]; - let arguments = (ins - Cuda_CUDAStream:$stream - ); - let results = (outs - Cuda_CUDAResultType:$result, - Cuda_CUDAGraph2:$graph - ); -} -def Cuda_StreamSynchronizeOp : Cuda_Op<"stream.synchronize", [MemoryEffects<[MemRead]>]> { - let description = [{ - Operation to synchronize with a CUDA Stream. - }]; - let arguments = (ins - Cuda_CUDAStream:$stream, - Optional:$event - ); - let results = (outs - Cuda_CUDAResultType:$result - ); -} -def Cuda_TensorMapEncodeTiledOp : Cuda_Op<"tensor_map_encode_tiled"> { - let description = [{ - Creates a descriptor for Tensor Memory Access (TMA) object specified by the parameters - describing a tiled region and returns it in tensorMap. - - This operation corresponds to the CUDA Driver API function: - `cuTensorMapEncodeTiled` - - Tensor map objects are only supported on devices of compute capability 9.0 or higher. - The tensorMap address must be aligned to 64 bytes. - - All parameters are provided as runtime SSA values. - }]; - let arguments = (ins - Cuda_T32BitSignlessInteger:$tensorDataType, - Cuda_T32BitSignlessInteger:$tensorRank, - Cuda_PointerType:$globalAddress, - Cuda_LLVMPointerInAddressSpace0:$globalDims, - Cuda_LLVMPointerInAddressSpace0:$globalStrides, - Cuda_LLVMPointerInAddressSpace0:$boxDims, - Cuda_LLVMPointerInAddressSpace0:$elementStrides, - Cuda_T32BitSignlessInteger:$interleave, - Cuda_T32BitSignlessInteger:$swizzle, - Cuda_T32BitSignlessInteger:$l2Promotion, - Cuda_T32BitSignlessInteger:$oobFill - ); - let results = (outs - Cuda_CUDAResultType:$cuda_result, - Cuda_CUDATensorMapType:$cuTensorMap - ); -} - -#endif // MLIR_DIALECTS_CUDA_CUDAOPS_TD diff --git a/mlir/dialects/cuda/CudaTypes.td b/mlir/dialects/cuda/CudaTypes.td deleted file mode 100644 index d149651e..00000000 --- a/mlir/dialects/cuda/CudaTypes.td +++ /dev/null @@ -1,212 +0,0 @@ -//===- CudaTypes.td - CUDA host types of the CuTe DSL ------*- tablegen -*-===// -// -// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler (its typed -// Python type classes, its mnemonics, `_cutlass_ir.cu13` parser messages and -// the verifier's wording for each type); do not edit by hand. -// -// Every cuda type the compiler's parser knows. Only `launch_cfg` has a -// parameter; the C++ class names not given by the typed Python classes -// (`!cuda.int`, the graph handles, `arch_set`, `kernel`, `module` and the -// launch attribute values) follow the mnemonic. -// -//===----------------------------------------------------------------------===// - -#ifndef MLIR_DIALECTS_CUDA_CUDATYPES_TD -#define MLIR_DIALECTS_CUDA_CUDATYPES_TD - -include "mlir/dialects/cuda/CudaDialect.td" - -def Cuda_ArchSetType : Cuda_Type<"ArchSet", "arch_set"> { - let summary = "`!cuda.arch_set`"; - let description = "`!cuda.arch_set`"; - let assemblyFormat = ""; -} - -def Cuda_EventType : Cuda_Type<"Event", "event"> { - let summary = "`Event` represents CUDA event"; - let description = "`!cuda.event`"; - let assemblyFormat = ""; -} - -def Cuda_GraphType : Cuda_Type<"Graph", "graph"> { - let summary = "CUDA Graph"; - let description = "`!cuda.graph`"; - let assemblyFormat = ""; -} - -def Cuda_GraphDeviceNodeType : Cuda_Type<"GraphDeviceNode", "graph_device_node"> { - let summary = "`GraphDeviceNode` represents CUDA device node handle for device-side node update"; - let description = "`!cuda.graph_device_node`"; - let assemblyFormat = ""; -} - -def Cuda_GraphExecType : Cuda_Type<"GraphExec", "graph_exec"> { - let summary = "CUDA Graph Execution"; - let description = "`!cuda.graph_exec`"; - let assemblyFormat = ""; -} - -def Cuda_GraphNodeType : Cuda_Type<"GraphNode", "graph_node"> { - let summary = "CUDA Graph Node handle"; - let description = "`!cuda.graph_node`"; - let assemblyFormat = ""; -} - -def Cuda_IntType : Cuda_Type<"Int", "int"> { - let summary = "`cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth"; - let description = "`!cuda.int`"; - let assemblyFormat = ""; -} - -def Cuda_KernelType : Cuda_Type<"Kernel", "kernel"> { - let summary = "CUDA Kernel handle"; - let description = "`!cuda.kernel`"; - let assemblyFormat = ""; -} - -def Cuda_LaunchConfigType : Cuda_Type<"LaunchConfig", "launch_cfg"> { - let summary = "Representation of CUDA's extensible launch configuration mirroring cuda runtime API"; - let description = "`!cuda.launch_cfg`"; - let parameters = (ins "uint32_t":$maxNumAttrs); - let assemblyFormat = "`<` `max_attrs` `=` $maxNumAttrs `>`"; -} - -def Cuda_LibraryType : Cuda_Type<"Library", "library"> { - let summary = "CUDA Library handle"; - let description = "`!cuda.library`"; - let assemblyFormat = ""; -} - -def Cuda_ModuleType : Cuda_Type<"Module", "module"> { - let summary = "CUDA Module handle"; - let description = "`!cuda.module`"; - let assemblyFormat = ""; -} - -def Cuda_ResultType : Cuda_Type<"Result", "result"> { - let summary = "CUDA Result type"; - let description = "`!cuda.result`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeDevicePropType : Cuda_Type<"RuntimeDeviceProp", "runtime.device_prop"> { - let summary = "CUDA cudaDeviceProp struct type (value-typed)."; - let description = "`!cuda.runtime.device_prop`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeDim3Type : Cuda_Type<"RuntimeDim3", "runtime.dim_3"> { - let summary = "CUDA dim3 struct type (value-typed)."; - let description = "`!cuda.runtime.dim_3`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeExtentType : Cuda_Type<"RuntimeExtent", "runtime.extent"> { - let summary = "CUDA cudaExtent struct type (value-typed)."; - let description = "`!cuda.runtime.extent`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeKernelNodeParamsType : Cuda_Type<"RuntimeKernelNodeParams", "runtime.kernel_node_params"> { - let summary = "CUDA cudaKernelNodeParams struct type (value-typed)."; - let description = "`!cuda.runtime.kernel_node_params`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType : Cuda_Type<"RuntimeLaunchAttributeValueClusterSchedulingPolicyPreference", "runtime.launch_attribute_value.cluster_scheduling_policy_preference"> { - let summary = "`!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`"; - let description = "`!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeLaunchAttributeValueCooperativeType : Cuda_Type<"RuntimeLaunchAttributeValueCooperative", "runtime.launch_attribute_value.cooperative"> { - let summary = "`!cuda.runtime.launch_attribute_value.cooperative`"; - let description = "`!cuda.runtime.launch_attribute_value.cooperative`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeLaunchAttributeValueMemSyncDomainType : Cuda_Type<"RuntimeLaunchAttributeValueMemSyncDomain", "runtime.launch_attribute_value.mem_sync_domain"> { - let summary = "`!cuda.runtime.launch_attribute_value.mem_sync_domain`"; - let description = "`!cuda.runtime.launch_attribute_value.mem_sync_domain`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType : Cuda_Type<"RuntimeLaunchAttributeValueNvlinkUtilCentricScheduling", "runtime.launch_attribute_value.nvlink_util_centric_scheduling"> { - let summary = "`!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`"; - let description = "`!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeLaunchAttributeValuePortableClusterSizeModeType : Cuda_Type<"RuntimeLaunchAttributeValuePortableClusterSizeMode", "runtime.launch_attribute_value.portable_cluster_size_mode"> { - let summary = "`!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`"; - let description = "`!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeLaunchAttributeValuePriorityType : Cuda_Type<"RuntimeLaunchAttributeValuePriority", "runtime.launch_attribute_value.priority"> { - let summary = "`!cuda.runtime.launch_attribute_value.priority`"; - let description = "`!cuda.runtime.launch_attribute_value.priority`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType : Cuda_Type<"RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed", "runtime.launch_attribute_value.programmatic_stream_serialization_allowed"> { - let summary = "`!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`"; - let description = "`!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeLaunchAttributeValueSharedMemCarveoutType : Cuda_Type<"RuntimeLaunchAttributeValueSharedMemCarveout", "runtime.launch_attribute_value.shared_mem_carveout"> { - let summary = "`!cuda.runtime.launch_attribute_value.shared_mem_carveout`"; - let description = "`!cuda.runtime.launch_attribute_value.shared_mem_carveout`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeLaunchAttributeValueSharedMemoryModeType : Cuda_Type<"RuntimeLaunchAttributeValueSharedMemoryMode", "runtime.launch_attribute_value.shared_memory_mode"> { - let summary = "`!cuda.runtime.launch_attribute_value.shared_memory_mode`"; - let description = "`!cuda.runtime.launch_attribute_value.shared_memory_mode`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeLaunchAttributeValueSyncPolicyType : Cuda_Type<"RuntimeLaunchAttributeValueSyncPolicy", "runtime.launch_attribute_value.sync_policy"> { - let summary = "`!cuda.runtime.launch_attribute_value.sync_policy`"; - let description = "`!cuda.runtime.launch_attribute_value.sync_policy`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeMemcpy3DParamsType : Cuda_Type<"RuntimeMemcpy3DParams", "runtime.memcpy_3d_params"> { - let summary = "CUDA cudaMemcpy3DParms struct type (value-typed)."; - let description = "`!cuda.runtime.memcpy_3d_params`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimePitchedPtrType : Cuda_Type<"RuntimePitchedPtr", "runtime.pitched_ptr"> { - let summary = "CUDA cudaPitchedPtr struct type (value-typed)."; - let description = "`!cuda.runtime.pitched_ptr`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimePosType : Cuda_Type<"RuntimePos", "runtime.pos"> { - let summary = "CUDA cudaPos struct type (value-typed)."; - let description = "`!cuda.runtime.pos`"; - let assemblyFormat = ""; -} - -def Cuda_RuntimeUuidType : Cuda_Type<"RuntimeUuid", "runtime.uuid"> { - let summary = "CUDA cudaUUID_t struct type (value-typed)."; - let description = "`!cuda.runtime.uuid`"; - let assemblyFormat = ""; -} - -def Cuda_StreamType : Cuda_Type<"Stream", "stream"> { - let summary = "CUDA Stream"; - let description = "`!cuda.stream`"; - let assemblyFormat = ""; -} - -def Cuda_TensorMapType : Cuda_Type<"TensorMap", "tensor_map"> { - let summary = "CUDA Tensor Map type"; - let description = "`!cuda.tensor_map`"; - let assemblyFormat = ""; -} - -#endif // MLIR_DIALECTS_CUDA_CUDATYPES_TD diff --git a/mlir/dialects/cuda/capi.cc b/mlir/dialects/cuda/capi.cc deleted file mode 100644 index 4cdff5ee..00000000 --- a/mlir/dialects/cuda/capi.cc +++ /dev/null @@ -1,559 +0,0 @@ -//===- capi.cc - C API for NVIDIA's cuda dialect --------------------------===// - -#include "capi.h" - -#include "mlir/dialects/cuda/CudaDialect.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" -#include "mlir/CAPI/Support.h" - -// Registration - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Cuda, cuda, - mlir::cutlass_compiler::cuda::CudaDialect) - -// Attributes - -namespace cuda = mlir::cutlass_compiler::cuda; - -// Unwraps `handle` into `out`: false when it is of another context or not a -// T, or when it is null and not optional. -template -static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { - auto value = unwrap(handle); - if (!value) - return optional; - if (value.getContext() != ctx) - return false; - out = llvm::dyn_cast(value); - return static_cast(out); -} - -bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaAssumeKernelAttrGet(MlirContext context, bool value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::AssumeKernelAttr::get(ctx, value)); -} -bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getValue(); -} - -bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaComputeTargetAttrGet(MlirContext context, - uint32_t representation, - uint32_t portability, - intptr_t numArchs, - const uint32_t *archs) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto representationValue = - ::mlir::cutlass_compiler::cuda::symbolizeExecutableRepresentation( - representation); - if (!representationValue) - return {nullptr}; - auto portabilityValue = - ::mlir::cutlass_compiler::cuda::symbolizeArchPortability(portability); - if (!portabilityValue) - return {nullptr}; - llvm::SmallVector<::mlir::cutlass_compiler::cuda::GpuArchitecture> archsValue; - for (intptr_t i = 0; i < numArchs; ++i) { - auto value = - ::mlir::cutlass_compiler::cuda::symbolizeGpuArchitecture(archs[i]); - if (!value) - return {nullptr}; - archsValue.push_back(*value); - } - return wrap(cuda::ComputeTargetAttr::get(ctx, *representationValue, - *portabilityValue, archsValue)); -} -uint32_t mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getRepresentation()); -} -uint32_t mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getPortability()); -} -intptr_t mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getArchs().size(); -} -uint32_t mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos) { - return static_cast( - llvm::cast(unwrap(attr)).getArchs()[pos]); -} - -bool mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::DevMaxSharedMemoryOptinAttr::get(ctx)); -} - -bool mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaDeviceAttributesAttrGet(MlirContext context, - MlirAttribute values) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::DictionaryAttr valuesValue; - if (!unwrapAs(ctx, values, false, valuesValue)) - return {nullptr}; - return wrap(cuda::DeviceAttributesAttr::get(ctx, valuesValue)); -} -MlirAttribute mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getValues())); -} - -bool mlirAttributeIsACudaExecutable(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, uint32_t format) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto formatValue = - ::mlir::cutlass_compiler::cuda::symbolizeExecutableFormat(format); - if (!formatValue) - return {nullptr}; - return wrap(cuda::ExecutableAttr::get(ctx, *formatValue)); -} -uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getFormat()); -} - -bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCudaFuncAttributesAttrGet(MlirContext context, - MlirAttribute values) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::DictionaryAttr valuesValue; - if (!unwrapAs(ctx, values, false, valuesValue)) - return {nullptr}; - return wrap(cuda::FuncAttributesAttr::get(ctx, valuesValue)); -} -MlirAttribute mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getValues())); -} - -// Types - -bool mlirTypeIsACudaArchSet(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaArchSetTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::ArchSetType::get(ctx)); -} - -bool mlirTypeIsACudaEvent(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaEventTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::EventType::get(ctx)); -} - -bool mlirTypeIsACudaGraphDeviceNode(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaGraphDeviceNodeTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::GraphDeviceNodeType::get(ctx)); -} - -bool mlirTypeIsACudaGraphExec(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaGraphExecTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::GraphExecType::get(ctx)); -} - -bool mlirTypeIsACudaGraphNode(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaGraphNodeTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::GraphNodeType::get(ctx)); -} - -bool mlirTypeIsACudaGraph(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaGraphTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::GraphType::get(ctx)); -} - -bool mlirTypeIsACudaInt(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaIntTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::IntType::get(ctx)); -} - -bool mlirTypeIsACudaKernel(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaKernelTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::KernelType::get(ctx)); -} - -bool mlirTypeIsACudaLaunchConfig(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, - uint32_t maxNumAttrs) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::LaunchConfigType::get(ctx, maxNumAttrs)); -} -uint32_t mlirCudaLaunchConfigTypeGetMaxNumAttrs(MlirType type) { - return llvm::cast(unwrap(type)).getMaxNumAttrs(); -} - -bool mlirTypeIsACudaLibrary(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaLibraryTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::LibraryType::get(ctx)); -} - -bool mlirTypeIsACudaModule(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaModuleTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::ModuleType::get(ctx)); -} - -bool mlirTypeIsACudaResult(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaResultTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::ResultType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeDeviceProp(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaRuntimeDevicePropTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeDevicePropType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeDim3(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaRuntimeDim3TypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeDim3Type::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeExtent(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaRuntimeExtentTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeExtentType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeKernelNodeParams(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaRuntimeKernelNodeParamsTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeKernelNodeParamsType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference( - MlirType type) { - return llvm::isa_and_nonnull< - cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType>( - unwrap(type)); -} -MlirType -mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet( - MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap( - cuda::RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType:: - get(ctx)); -} - -bool mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative(MlirType type) { - return llvm::isa_and_nonnull< - cuda::RuntimeLaunchAttributeValueCooperativeType>(unwrap(type)); -} -MlirType -mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeLaunchAttributeValueCooperativeType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain(MlirType type) { - return llvm::isa_and_nonnull< - cuda::RuntimeLaunchAttributeValueMemSyncDomainType>(unwrap(type)); -} -MlirType -mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeLaunchAttributeValueMemSyncDomainType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling( - MlirType type) { - return llvm::isa_and_nonnull< - cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType>( - unwrap(type)); -} -MlirType mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet( - MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap( - cuda::RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType::get( - ctx)); -} - -bool mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode( - MlirType type) { - return llvm::isa_and_nonnull< - cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType>( - unwrap(type)); -} -MlirType mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet( - MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap( - cuda::RuntimeLaunchAttributeValuePortableClusterSizeModeType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeLaunchAttributeValuePriority(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType -mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeLaunchAttributeValuePriorityType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed( - MlirType type) { - return llvm::isa_and_nonnull< - cuda:: - RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType>( - unwrap(type)); -} -MlirType -mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet( - MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap( - cuda:: - RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType:: - get(ctx)); -} - -bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout( - MlirType type) { - return llvm::isa_and_nonnull< - cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType>(unwrap(type)); -} -MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet( - MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeLaunchAttributeValueSharedMemCarveoutType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode(MlirType type) { - return llvm::isa_and_nonnull< - cuda::RuntimeLaunchAttributeValueSharedMemoryModeType>(unwrap(type)); -} -MlirType mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet( - MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeLaunchAttributeValueSharedMemoryModeType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType -mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeLaunchAttributeValueSyncPolicyType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeMemcpy3DParams(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaRuntimeMemcpy3DParamsTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeMemcpy3DParamsType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimePitchedPtr(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaRuntimePitchedPtrTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimePitchedPtrType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimePos(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaRuntimePosTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimePosType::get(ctx)); -} - -bool mlirTypeIsACudaRuntimeUuid(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaRuntimeUuidTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::RuntimeUuidType::get(ctx)); -} - -bool mlirTypeIsACudaStream(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaStreamTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::StreamType::get(ctx)); -} - -bool mlirTypeIsACudaTensorMap(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCudaTensorMapTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cuda::TensorMapType::get(ctx)); -} diff --git a/mlir/dialects/cuda/capi.h b/mlir/dialects/cuda/capi.h deleted file mode 100644 index 126a87d8..00000000 --- a/mlir/dialects/cuda/capi.h +++ /dev/null @@ -1,253 +0,0 @@ -//===- capi.h - C API for NVIDIA's cuda dialect -------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_CUDA_CAPI_H -#define MLIR_DIALECTS_CUDA_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Registration - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cuda, cuda); - -// Attributes - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles. Enums are their integer -// values (the dialect's *Enums.td). Getters require a value accepted by the -// matching IsA. - -// `#cuda.assume_kernel_attr`: Whether `cuda.launch` may assume its callee is a -// kernel -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaAssumeKernel(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaAssumeKernelAttrGet(MlirContext context, bool value); -MLIR_CAPI_EXPORTED bool mlirCudaAssumeKernelAttrGetValue(MlirAttribute attr); - -// `#cuda.compute_target`: Representation, portability and architectures of a -// compiled executable -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaComputeTarget(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaComputeTargetAttrGet( - MlirContext context, uint32_t representation, uint32_t portability, - intptr_t numArchs, const uint32_t *archs); -MLIR_CAPI_EXPORTED uint32_t -mlirCudaComputeTargetAttrGetRepresentation(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t -mlirCudaComputeTargetAttrGetPortability(MlirAttribute attr); -MLIR_CAPI_EXPORTED intptr_t -mlirCudaComputeTargetAttrGetNumArchs(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t -mlirCudaComputeTargetAttrGetArchs(MlirAttribute attr, intptr_t pos); - -// `#cuda.dev_max_shared_memory_optin`: The device's opt-in maximum of shared -// memory per block -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACudaDevMaxSharedMemoryOptin(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaDevMaxSharedMemoryOptinAttrGet(MlirContext context); - -// `#cuda.device_attributes`: CUDA device attributes -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACudaDeviceAttributes(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaDeviceAttributesAttrGet(MlirContext context, MlirAttribute values); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaDeviceAttributesAttrGetValues(MlirAttribute attr); - -// `#cuda.executable`: Format of a compiled executable -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaExecutable(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaExecutableAttrGet(MlirContext context, - uint32_t format); -MLIR_CAPI_EXPORTED uint32_t mlirCudaExecutableAttrGetFormat(MlirAttribute attr); - -// `#cuda.func_attributes`: CUDA function attributes of a kernel -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaFuncAttributes(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaFuncAttributesAttrGet(MlirContext context, MlirAttribute values); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaFuncAttributesAttrGetValues(MlirAttribute attr); - -// Types - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles. Enums are their integer -// values (the dialect's *Enums.td). Getters require a value accepted by the -// matching IsA. - -// `!cuda.arch_set`: `!cuda.arch_set` -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaArchSet(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaArchSetTypeGet(MlirContext context); - -// `!cuda.event`: `Event` represents CUDA event -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaEvent(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaEventTypeGet(MlirContext context); - -// `!cuda.graph_device_node`: `GraphDeviceNode` represents CUDA device node -// handle for device-side node update -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphDeviceNode(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaGraphDeviceNodeTypeGet(MlirContext context); - -// `!cuda.graph_exec`: CUDA Graph Execution -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphExec(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaGraphExecTypeGet(MlirContext context); - -// `!cuda.graph_node`: CUDA Graph Node handle -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraphNode(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaGraphNodeTypeGet(MlirContext context); - -// `!cuda.graph`: CUDA Graph -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaGraph(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaGraphTypeGet(MlirContext context); - -// `!cuda.int`: `cuda.int` is a integer type with a bitwidth matching the -// underlying ABI integer bitwidth -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaInt(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaIntTypeGet(MlirContext context); - -// `!cuda.kernel`: CUDA Kernel handle -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaKernel(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaKernelTypeGet(MlirContext context); - -// `!cuda.launch_cfg`: Representation of CUDA's extensible launch configuration -// mirroring cuda runtime API -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLaunchConfig(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaLaunchConfigTypeGet(MlirContext context, - uint32_t maxNumAttrs); -MLIR_CAPI_EXPORTED uint32_t -mlirCudaLaunchConfigTypeGetMaxNumAttrs(MlirType type); - -// `!cuda.library`: CUDA Library handle -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaLibrary(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaLibraryTypeGet(MlirContext context); - -// `!cuda.module`: CUDA Module handle -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaModule(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaModuleTypeGet(MlirContext context); - -// `!cuda.result`: CUDA Result type -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaResult(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaResultTypeGet(MlirContext context); - -// `!cuda.runtime.device_prop`: CUDA cudaDeviceProp struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDeviceProp(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeDevicePropTypeGet(MlirContext context); - -// `!cuda.runtime.dim_3`: CUDA dim3 struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeDim3(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeDim3TypeGet(MlirContext context); - -// `!cuda.runtime.extent`: CUDA cudaExtent struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeExtent(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeExtentTypeGet(MlirContext context); - -// `!cuda.runtime.kernel_node_params`: CUDA cudaKernelNodeParams struct type -// (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeKernelNodeParams(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeKernelNodeParamsTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference` -MLIR_CAPI_EXPORTED bool -mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference( - MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet( - MlirContext context); - -// `!cuda.runtime.launch_attribute_value.cooperative` -MLIR_CAPI_EXPORTED bool -mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.mem_sync_domain` -MLIR_CAPI_EXPORTED bool -mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling` -MLIR_CAPI_EXPORTED bool -mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling( - MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet( - MlirContext context); - -// `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode` -MLIR_CAPI_EXPORTED bool -mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode( - MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet( - MlirContext context); - -// `!cuda.runtime.launch_attribute_value.priority` -MLIR_CAPI_EXPORTED bool -mlirTypeIsACudaRuntimeLaunchAttributeValuePriority(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed` -MLIR_CAPI_EXPORTED bool -mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed( - MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet( - MlirContext context); - -// `!cuda.runtime.launch_attribute_value.shared_mem_carveout` -MLIR_CAPI_EXPORTED bool -mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet( - MlirContext context); - -// `!cuda.runtime.launch_attribute_value.shared_memory_mode` -MLIR_CAPI_EXPORTED bool -mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(MlirContext context); - -// `!cuda.runtime.launch_attribute_value.sync_policy` -MLIR_CAPI_EXPORTED bool -mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(MlirContext context); - -// `!cuda.runtime.memcpy_3d_params`: CUDA cudaMemcpy3DParms struct type -// (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeMemcpy3DParams(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimeMemcpy3DParamsTypeGet(MlirContext context); - -// `!cuda.runtime.pitched_ptr`: CUDA cudaPitchedPtr struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePitchedPtr(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaRuntimePitchedPtrTypeGet(MlirContext context); - -// `!cuda.runtime.pos`: CUDA cudaPos struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimePos(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimePosTypeGet(MlirContext context); - -// `!cuda.runtime.uuid`: CUDA cudaUUID_t struct type (value-typed). -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaRuntimeUuid(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaRuntimeUuidTypeGet(MlirContext context); - -// `!cuda.stream`: CUDA Stream -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaStream(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaStreamTypeGet(MlirContext context); - -// `!cuda.tensor_map`: CUDA Tensor Map type -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTensorMap(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaTensorMapTypeGet(MlirContext context); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_CUDA_CAPI_H diff --git a/mlir/dialects/cuda/cuda.zig b/mlir/dialects/cuda/cuda.zig deleted file mode 100644 index 2c74c920..00000000 --- a/mlir/dialects/cuda/cuda.zig +++ /dev/null @@ -1,104 +0,0 @@ -//! Builders for the CUDA host-program operations CuTe kernels emit, over the -//! bindings of NVIDIA's `cuda` dialect (`mlir/dialects/cuda`, `ops`), -//! which the CuTe-DSL compiler registers. Each builder fixes the attributes the -//! kernels do not choose. - -const mlir = @import("mlir"); - -pub const ops = @import("ops.zig"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("cuda"); -} - -pub fn streamType(ctx: *mlir.Context) *const mlir.Type { - return (ops.StreamType.get(ctx, .{}) catch unreachable).type_(); -} - -pub fn resultType(ctx: *mlir.Context) *const mlir.Type { - return (ops.ResultType.get(ctx, .{}) catch unreachable).type_(); -} - -pub fn launchConfigType(ctx: *mlir.Context, max_attrs: u32) *const mlir.Type { - return (ops.LaunchConfigType.get(ctx, .{ .maxNumAttrs = max_attrs }) catch unreachable).type_(); -} - -/// `#cuda.assume_kernel_attr`, whether `cuda.launch_ex` may assume its -/// callee is a kernel. -pub fn assumeKernelAttr(ctx: *mlir.Context, value: bool) *const mlir.Attribute { - return (ops.AssumeKernelAttr.get(ctx, .{ .value = value }) catch unreachable).attribute(); -} - -pub const KernelArgs = struct { - name: []const u8, - function_type: *const mlir.Type, - body: *mlir.Block, - arg_attrs: ?*const mlir.Attribute = null, - extra_attributes: []const mlir.NamedAttribute = &.{}, - location: *const mlir.Location, -}; - -/// `cuda.kernel`; `body` is filled afterwards. -pub fn kernel(ctx: *mlir.Context, args: KernelArgs) *mlir.Operation { - const op = ops.kernel(ctx, .string(ctx, args.name), .typeAttr(args.function_type), .{args.body}, .{ .arg_attrs = args.arg_attrs }, args.location); - for (args.extra_attributes) |attr| op.setAttribute(attr); - return op; -} - -pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.return_(ctx, values, location); -} - -pub fn cast(ctx: *mlir.Context, dst: *const mlir.Type, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.cast(ctx, src, dst, location); -} - -pub const LaunchConfigCreateArgs = struct { - max_attrs: u32, - block: [3]*const mlir.Value, - dynamic_smem: *const mlir.Value, - grid: [3]*const mlir.Value, - stream: *const mlir.Value, - location: *const mlir.Location, -}; - -pub fn launch_cfg_create(ctx: *mlir.Context, args: LaunchConfigCreateArgs) *mlir.Operation { - return ops.launch_cfg_create( - ctx, - args.block[0], - args.block[1], - args.block[2], - args.dynamic_smem, - args.grid[0], - args.grid[1], - args.grid[2], - args.stream, - launchConfigType(ctx, args.max_attrs), - .int(ctx, .i32, args.max_attrs), - args.location, - ); -} - -pub fn launch_cfg_cluster_dim(ctx: *mlir.Context, config: *const mlir.Value, dims: [3]*const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.launch_cfg_cluster_dim(ctx, config, dims[0], dims[1], dims[2], location); -} - -pub fn launch_cfg_cooperative(ctx: *mlir.Context, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.launch_cfg_cooperative(ctx, config, enabled, location); -} - -pub fn launch_cfg_programmatic_stream_serialization_allowed(ctx: *mlir.Context, config: *const mlir.Value, enabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.launch_cfg_programmatic_stream_serialization_allowed(ctx, config, enabled, location); -} - -pub const LaunchExArgs = struct { - config: *const mlir.Value, - inputs: []const *const mlir.Value, - callee: *const mlir.Attribute, - assume_kernel_attr: ?*const mlir.Attribute = null, - location: *const mlir.Location, -}; - -pub fn launch_ex(ctx: *mlir.Context, args: LaunchExArgs) *mlir.Operation { - return ops.launch_ex(ctx, args.config, args.inputs, resultType(ctx), args.callee, .{ .assume_kernel_attr = args.assume_kernel_attr }, args.location); -} diff --git a/mlir/dialects/cuda/ops.zig b/mlir/dialects/cuda/ops.zig deleted file mode 100644 index baeb0ded..00000000 --- a/mlir/dialects/cuda/ops.zig +++ /dev/null @@ -1,3346 +0,0 @@ -//! Zig bindings of NVIDIA's CUDA host dialect of the CuTe DSL (nvidia-cutlass-dsl 4.8.0), generated -//! from the dialect's .td files; do not edit. - -const c = @import("c"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -/// The C dialect handle, `mlirGetDialectHandle__cuda__`. -pub const dialect_handle = "cuda"; - -// Enums, generated from the dialect's .td files; values are the compiler's. - -/// Enumerated type describing portability. -pub const ArchPortability = enum(u32) { - portable = 0, - conditional = 1, - family = 2, -}; - -/// Enumerated type describing CUDA device attributes can be set through Cuda Dialect. -pub const CUDeviceAttr = enum(u32) { - maxThreadsPerBlock = 1, - maxBlockDimX = 2, - maxBlockDimY = 3, - maxBlockDimZ = 4, - maxGridDimX = 5, - maxGridDimY = 6, - maxGridDimZ = 7, - maxSharedMemoryPerBlock = 8, - totalConstantMemory = 9, - warpSize = 10, - maxPitch = 11, - maxRegistersPerBlock = 12, - clockRate = 13, - textureAlignment = 14, - gpuOverlap = 15, - multiProcessorCount = 16, - kernelExecTimeout = 17, - integrated = 18, - canMapHostMemory = 19, - computeMode = 20, - maxTexture1DWidth = 21, - maxTexture2DWidth = 22, - maxTexture2DHeight = 23, - maxTexture3DWidth = 24, - maxTexture3DHeight = 25, - maxTexture3DDepth = 26, - maxTexture2DLayeredWidth = 27, - maxTexture2DLayeredHeight = 28, - maxTexture2DLayeredLayers = 29, - surfaceAlignment = 30, - concurrentKernels = 31, - eccEnabled = 32, - pciBusId = 33, - pciDeviceId = 34, - tccDriver = 35, - memoryClockRate = 36, - globalMemoryBusWidth = 37, - l2CacheSize = 38, - maxThreadsPerMultiProcessor = 39, - asyncEngineCount = 40, - unifiedAddressing = 41, - maxTexture1DLayeredWidth = 42, - maxTexture1DLayeredLayers = 43, - maxTexture2DGatherWidth = 45, - maxTexture2DGatherHeight = 46, - maxTexture3DWidthAlt = 47, - maxTexture3DHeightAlt = 48, - maxTexture3DDepthAlt = 49, - pciDomainId = 50, - texturePitchAlignment = 51, - maxTextureCubemapWidth = 52, - maxTextureCubemapLayeredWidth = 53, - maxTextureCubemapLayeredLayers = 54, - maxSurface1DWidth = 55, - maxSurface2DWidth = 56, - maxSurface2DHeight = 57, - maxSurface3DWidth = 58, - maxSurface3DHeight = 59, - maxSurface3DDepth = 60, - maxSurface1DLayeredWidth = 61, - maxSurface1DLayeredLayers = 62, - maxSurface2DLayeredWidth = 63, - maxSurface2DLayeredHeight = 64, - maxSurface2DLayeredLayers = 65, - maxSurfaceCubemapWidth = 66, - maxSurfaceCubemapLayeredWidth = 67, - maxSurfaceCubemapLayeredLayers = 68, - maxTexture1DLinearWidth = 69, - maxTexture2DLinearWidth = 70, - maxTexture2DLinearHeight = 71, - maxTexture2DLinearPitch = 72, - maxTexture2DMipmappedWidth = 73, - maxTexture2DMipmappedHeight = 74, - computeCapabilityMajor = 75, - computeCapabilityMinor = 76, - maxTexture1DMipmappedWidth = 77, - streamPrioritiesSupported = 78, - globalL1CacheSupported = 79, - localL1CacheSupported = 80, - maxSharedMemoryPerMultiprocessor = 81, - maxRegistersPerMultiprocessor = 82, - managedMemory = 83, - isMultiGpuBoard = 84, - multiGpuBoardGroupID = 85, - hostNativeAtomicSupported = 86, - singleToDoublePrecisionPerfRatio = 87, - pageableMemoryAccess = 88, - concurrentManagedAccess = 89, - computePreemptionSupported = 90, - canUseHostPointerForRegisteredMem = 91, - reserved92 = 92, - reserved93 = 93, - reserved94 = 94, - cooperativeLaunch = 95, - reserved96 = 96, - maxSharedMemoryPerBlockOptin = 97, - canFlushRemoteWrites = 98, - hostRegisterSupported = 99, - pageableMemoryAccessUsesHostPageTables = 100, - directManagedMemAccessFromHost = 101, - maxBlocksPerMultiprocessor = 106, - maxPersistingL2CacheSize = 108, - maxAccessPolicyWindowSize = 109, - reservedSharedMemoryPerBlock = 111, - sparseCudaArraySupported = 112, - hostRegisterReadOnlySupported = 113, - timelineSemaphoreInteropSupported = 114, - memoryPoolsSupported = 115, - gpuDirectRdmaSupported = 116, - gpuDirectRdmaFlushWritesOptions = 117, - gpuDirectRdmaWritesOrdering = 118, - memoryPoolSupportedHandleTypes = 119, - clusterLaunch = 120, - deferredMappingCudaArraySupported = 121, - reserved122 = 122, - reserved123 = 123, - reserved124 = 124, - ipcEventSupport = 125, - memSyncDomainCount = 126, - reserved127 = 127, - reserved128 = 128, - reserved129 = 129, - numaConfig = 130, - numaId = 131, - reserved132 = 132, - mpsEnabled = 133, - hostNumaId = 134, - d3d12CigSupported = 135, - vulkanCigSupported = 138, - gpuPciDeviceId = 139, - gpuPciSubsystemId = 140, - reserved141 = 141, - hostNumaMemoryPoolsSupported = 142, - hostNumaMultinodeIpcSupported = 143, - hostMemoryPoolsSupported = 144, - reserved145 = 145, - onlyPartialHostNativeAtomicSupported = 147, - cudaDevAttrOversizedSharedMemoryPerBlock = 150, -}; - -/// Enumerated type describing CUDA function attributes can be set through Cuda Dialect. -pub const CUFunctionAttribute = enum(u32) { - max_threads_per_block = 0, - shared_size_bytes = 1, - const_size_bytes = 2, - local_size_bytes = 3, - num_regs = 4, - ptx_version = 5, - binary_version = 6, - cache_mode_ca = 7, - max_dynamic_shared_size_bytes = 8, - preferred_shared_memory_carveout = 9, - cluster_size_must_be_set = 10, - required_cluster_width = 11, - required_cluster_height = 12, - required_cluster_depth = 13, - non_portable_cluster_size_allowed = 14, - cluster_scheduling_policy_preference = 15, -}; - -/// Enumerated of the host library ABI. -pub const CudaABI = enum(u32) { - LP64 = 0, - ILP64 = 1, - LLP64 = 2, -}; - -/// Specifies performance hint with cudaAccessPolicyWindow for hitProp and missProp members. -pub const CudaAccessProperty = enum(u32) { - cudaAccessPropertyNormal = 0, - cudaAccessPropertyStreaming = 1, - cudaAccessPropertyPersisting = 2, -}; - -/// Enumerated type describing CUDA cluster scheduling policy. -pub const CudaClusterSchedulingPolicy = enum(u32) { - cudaClusterSchedulingPolicyDefault = 0, - cudaClusterSchedulingPolicySpread = 1, - cudaClusterSchedulingPolicyLoadBalancing = 2, -}; - -/// Enumerated type describing CUDA function attributes can be set through Cuda Dialect. -pub const CudaFuncAttribute = enum(u32) { - cudaFuncAttributeSharedMemoryMode = 16, -}; - -/// Enumerated type mirroring CudaLaunchAttributeID from CUDA Runtime API. -pub const CudaLaunchAttributeID = enum(u32) { - cudaLaunchAttributeIgnore = 0, - cudaLaunchAttributeAccessPolicyWindow = 1, - cudaLaunchAttributeCooperative = 2, - cudaLaunchAttributeSynchronizationPolicy = 3, - cudaLaunchAttributeClusterDimension = 4, - cudaLaunchAttributeClusterSchedulingPolicyPreference = 5, - cudaLaunchAttributeProgrammaticStreamSerialization = 6, - cudaLaunchAttributeProgrammaticEvent = 7, - cudaLaunchAttributePriority = 8, - cudaLaunchAttributeMemSyncDomainMap = 9, - cudaLaunchAttributeMemSyncDomain = 10, - cudaLaunchAttributePreferredClusterDimension = 11, - cudaLaunchAttributeLaunchCompletionEvent = 12, - cudaLaunchAttributeDeviceUpdatableKernelNode = 13, - cudaLaunchAttributePreferredSharedMemoryCarveout = 14, - cudaLaunchAttributeNvlinkUtilCentricScheduling = 16, -}; - -/// Enumerated type describing CUDA memory sync domain. -pub const CudaLaunchMemSyncDomain = enum(u32) { - cudaLaunchMemSyncDomainDefault = 0, - cudaLaunchMemSyncDomainRemote = 1, -}; - -/// Enumerated type describing CUDA shared memory modes. -pub const CudaSharedMemoryMode = enum(u32) { - cudaSharedMemoryModeDefault = 0, - cudaSharedMemoryModeRequirePortable = 1, - cudaSharedMemoryModeAllowNonPortable = 2, - cudaSharedMemoryModeAllowOversizedSharedMemory = 3, - cudaSharedMemoryModePreferOversizedSharedMemory = 4, -}; - -/// Enumerated type describing CUDA synchronization policy. -pub const CudaSynchronizationPolicy = enum(u32) { - cudaSyncPolicyAuto = 0, - cudaSyncPolicySpin = 1, - cudaSyncPolicyYield = 2, - cudaSyncPolicyBlockingSync = 3, -}; - -/// Enumerated type describing the format of an executable. -pub const ExecutableFormat = enum(u32) { - text = 0, - elf = 1, - bitcode = 2, - fatbin = 3, -}; - -/// Enumerated type describing the representation of an executable. -pub const ExecutableRepresentation = enum(u32) { - sass = 0, - ptx = 2, - nvvm = 3, - ltoir = 4, - mlir = 5, - llvm = 6, - hostobj = 7, -}; - -/// Enumerated type denoting a GPU Architecture. -pub const GpuArchitecture = enum(u32) { - sm_80 = 80, - sm_86 = 86, - sm_87 = 87, - sm_88 = 88, - sm_89 = 89, - sm_90 = 90, - sm_100 = 100, - sm_101 = 101, - sm_103 = 103, - sm_107 = 107, - sm_110 = 110, - sm_120 = 120, - sm_121 = 121, -}; - -/// Enumerated type describing memcpy directions. -pub const MemcpyKind = enum(u32) { - HostToHost = 0, - HostToDevice = 1, - DeviceToHost = 2, - DeviceToDevice = 3, - Default = 4, -}; - -/// Enumerated type describing CUDA stream capture modes. -pub const StreamCaptureModeEnum = enum(u32) { - Global = 0, - ThreadLocal = 1, - Relaxed = 2, -}; - -// Types and attributes, generated from the dialect's .td files. - -/// `#cuda.assume_kernel_attr`: Whether `cuda.launch` may assume its callee is a kernel -pub const AssumeKernelAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACudaAssumeKernel; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "assume_kernel_attr"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: bool, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaAssumeKernelAttrGet(ctx.ptr(), args.value); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) bool { - return c.mlirCudaAssumeKernelAttrGetValue(self.ptr()); - } -}; - -/// `#cuda.compute_target`: Representation, portability and architectures of a compiled executable -pub const ComputeTargetAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACudaComputeTarget; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "compute_target"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - representation: ExecutableRepresentation, - portability: ArchPortability, - archs: []const GpuArchitecture, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaComputeTargetAttrGet(ctx.ptr(), @backingInt(args.representation), @backingInt(args.portability), @intCast(args.archs.len), @ptrCast(args.archs.ptr)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn representation(self: *const Self) ExecutableRepresentation { - return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetRepresentation(self.ptr()))); - } - pub fn portability(self: *const Self) ArchPortability { - return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetPortability(self.ptr()))); - } - pub fn numArchs(self: *const Self) usize { - return @intCast(c.mlirCudaComputeTargetAttrGetNumArchs(self.ptr())); - } - pub fn arch(self: *const Self, pos: usize) GpuArchitecture { - return @fromBackingInt(@intCast(c.mlirCudaComputeTargetAttrGetArchs(self.ptr(), @intCast(pos)))); - } -}; - -/// `#cuda.dev_max_shared_memory_optin`: The device's opt-in maximum of shared memory per block -pub const DevMaxSharedMemoryOptinAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACudaDevMaxSharedMemoryOptin; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "dev_max_shared_memory_optin"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaDevMaxSharedMemoryOptinAttrGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `#cuda.device_attributes`: CUDA device attributes -pub const DeviceAttributesAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACudaDeviceAttributes; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "device_attributes"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - values: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaDeviceAttributesAttrGet(ctx.ptr(), args.values.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn values(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCudaDeviceAttributesAttrGetValues(self.ptr()).ptr.?); - } -}; - -/// `#cuda.executable`: Format of a compiled executable -pub const ExecutableAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACudaExecutable; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "executable"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - format: ExecutableFormat, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaExecutableAttrGet(ctx.ptr(), @backingInt(args.format)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn format_(self: *const Self) ExecutableFormat { - return @fromBackingInt(@intCast(c.mlirCudaExecutableAttrGetFormat(self.ptr()))); - } -}; - -/// `#cuda.func_attributes`: CUDA function attributes of a kernel -pub const FuncAttributesAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACudaFuncAttributes; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "func_attributes"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - values: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaFuncAttributesAttrGet(ctx.ptr(), args.values.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn values(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCudaFuncAttributesAttrGetValues(self.ptr()).ptr.?); - } -}; - -/// `!cuda.arch_set`: `!cuda.arch_set` -pub const ArchSetType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaArchSet; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "arch_set"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaArchSetTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.event`: `Event` represents CUDA event -pub const EventType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaEvent; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "event"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaEventTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.graph_device_node`: `GraphDeviceNode` represents CUDA device node handle for device-side node update -pub const GraphDeviceNodeType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaGraphDeviceNode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "graph_device_node"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaGraphDeviceNodeTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.graph_exec`: CUDA Graph Execution -pub const GraphExecType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaGraphExec; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "graph_exec"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaGraphExecTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.graph_node`: CUDA Graph Node handle -pub const GraphNodeType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaGraphNode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "graph_node"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaGraphNodeTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.graph`: CUDA Graph -pub const GraphType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaGraph; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "graph"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaGraphTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.int`: `cuda.int` is a integer type with a bitwidth matching the underlying ABI integer bitwidth -pub const IntType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaInt; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "int"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaIntTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.kernel`: CUDA Kernel handle -pub const KernelType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaKernel; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "kernel"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaKernelTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.launch_cfg`: Representation of CUDA's extensible launch configuration mirroring cuda runtime API -pub const LaunchConfigType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaLaunchConfig; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "launch_cfg"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - maxNumAttrs: u32, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaLaunchConfigTypeGet(ctx.ptr(), args.maxNumAttrs); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn maxNumAttrs(self: *const Self) u32 { - return c.mlirCudaLaunchConfigTypeGetMaxNumAttrs(self.ptr()); - } -}; - -/// `!cuda.library`: CUDA Library handle -pub const LibraryType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaLibrary; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "library"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaLibraryTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.module`: CUDA Module handle -pub const ModuleType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaModule; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "module"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaModuleTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.result`: CUDA Result type -pub const ResultType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaResult; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "result"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaResultTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.device_prop`: CUDA cudaDeviceProp struct type (value-typed). -pub const RuntimeDevicePropType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeDeviceProp; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.device_prop"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeDevicePropTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.dim_3`: CUDA dim3 struct type (value-typed). -pub const RuntimeDim3Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeDim3; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.dim_3"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeDim3TypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.extent`: CUDA cudaExtent struct type (value-typed). -pub const RuntimeExtentType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeExtent; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.extent"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeExtentTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.kernel_node_params`: CUDA cudaKernelNodeParams struct type (value-typed). -pub const RuntimeKernelNodeParamsType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeKernelNodeParams; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.kernel_node_params"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeKernelNodeParamsTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference`: `!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference` -pub const RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreference; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.launch_attribute_value.cluster_scheduling_policy_preference"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.launch_attribute_value.cooperative`: `!cuda.runtime.launch_attribute_value.cooperative` -pub const RuntimeLaunchAttributeValueCooperativeType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueCooperative; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.launch_attribute_value.cooperative"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeLaunchAttributeValueCooperativeTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.launch_attribute_value.mem_sync_domain`: `!cuda.runtime.launch_attribute_value.mem_sync_domain` -pub const RuntimeLaunchAttributeValueMemSyncDomainType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueMemSyncDomain; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.launch_attribute_value.mem_sync_domain"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeLaunchAttributeValueMemSyncDomainTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling`: `!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling` -pub const RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueNvlinkUtilCentricScheduling; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.launch_attribute_value.nvlink_util_centric_scheduling"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode`: `!cuda.runtime.launch_attribute_value.portable_cluster_size_mode` -pub const RuntimeLaunchAttributeValuePortableClusterSizeModeType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValuePortableClusterSizeMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.launch_attribute_value.portable_cluster_size_mode"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeLaunchAttributeValuePortableClusterSizeModeTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.launch_attribute_value.priority`: `!cuda.runtime.launch_attribute_value.priority` -pub const RuntimeLaunchAttributeValuePriorityType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValuePriority; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.launch_attribute_value.priority"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeLaunchAttributeValuePriorityTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed`: `!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed` -pub const RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowed; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.launch_attribute_value.programmatic_stream_serialization_allowed"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.launch_attribute_value.shared_mem_carveout`: `!cuda.runtime.launch_attribute_value.shared_mem_carveout` -pub const RuntimeLaunchAttributeValueSharedMemCarveoutType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemCarveout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.launch_attribute_value.shared_mem_carveout"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeLaunchAttributeValueSharedMemCarveoutTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.launch_attribute_value.shared_memory_mode`: `!cuda.runtime.launch_attribute_value.shared_memory_mode` -pub const RuntimeLaunchAttributeValueSharedMemoryModeType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSharedMemoryMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.launch_attribute_value.shared_memory_mode"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeLaunchAttributeValueSharedMemoryModeTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.launch_attribute_value.sync_policy`: `!cuda.runtime.launch_attribute_value.sync_policy` -pub const RuntimeLaunchAttributeValueSyncPolicyType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeLaunchAttributeValueSyncPolicy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.launch_attribute_value.sync_policy"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeLaunchAttributeValueSyncPolicyTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.memcpy_3d_params`: CUDA cudaMemcpy3DParms struct type (value-typed). -pub const RuntimeMemcpy3DParamsType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeMemcpy3DParams; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.memcpy_3d_params"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeMemcpy3DParamsTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.pitched_ptr`: CUDA cudaPitchedPtr struct type (value-typed). -pub const RuntimePitchedPtrType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimePitchedPtr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.pitched_ptr"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimePitchedPtrTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.pos`: CUDA cudaPos struct type (value-typed). -pub const RuntimePosType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimePos; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.pos"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimePosTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.runtime.uuid`: CUDA cudaUUID_t struct type (value-typed). -pub const RuntimeUuidType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaRuntimeUuid; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "runtime.uuid"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaRuntimeUuidTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.stream`: CUDA Stream -pub const StreamType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaStream; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "stream"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaStreamTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cuda.tensor_map`: CUDA Tensor Map type -pub const TensorMapType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaTensorMap; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "tensor_map"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCudaTensorMapTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -// Operation builders, generated from the operations' .td. -/// `cuda.addressof`. Result types are explicit; attributes use mlir.Attribute. -pub fn addressof(ctx: *mlir.Context, result_type: *const mlir.Type, symname: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.addressof", .{ - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "symname", symname), - }, - .location = location, - }); -} - -/// `cuda.binary`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. -pub fn binary(ctx: *mlir.Context, sym_name: *const mlir.Attribute, binary_: *const mlir.Attribute, blocks: [1]*mlir.Block, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.binary", .{ - .attributes = &.{ - .named(ctx, "sym_name", sym_name), - .named(ctx, "binary", binary_), - }, - .blocks = &blocks, - .verify = false, - .location = location, - }); -} - -/// `cuda.cast`. Result types are explicit; attributes use mlir.Attribute. -pub fn cast(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.cast", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cuda.compose`. Result types are explicit; attributes use mlir.Attribute. -pub fn compose(ctx: *mlir.Context, fields: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.compose", .{ - .operands = .{ .flat = fields }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.decompose`. Result types are explicit; attributes use mlir.Attribute. -pub fn decompose( - ctx: *mlir.Context, - source: *const mlir.Value, - opts: struct { - fields_type: ?*const mlir.Type = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (opts.fields_type) |value| results.appendAssumeCapacity(value); - return mlir.Operation.make(ctx, "cuda.decompose", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = results.constSlice() }, - .location = location, - }); -} - -/// `cuda.device.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn device_sync(ctx: *mlir.Context, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.device.sync", .{ - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.driver.event.create`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_event_create(ctx: *mlir.Context, Flags: *const mlir.Value, cuda_result_type: *const mlir.Type, phEvent_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.event.create", .{ - .operands = .{ .flat = &.{Flags} }, - .results = .{ .flat = &.{ cuda_result_type, phEvent_type } }, - .location = location, - }); -} - -/// `cuda.driver.event.destroy`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_event_destroy(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.event.destroy", .{ - .operands = .{ .flat = &.{hEvent} }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.driver.event.elapsed_time`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_event_elapsed_time(ctx: *mlir.Context, hStart: *const mlir.Value, hEnd: *const mlir.Value, cuda_result_type: *const mlir.Type, pMilliseconds_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.event.elapsed_time", .{ - .operands = .{ .flat = &.{ hStart, hEnd } }, - .results = .{ .flat = &.{ cuda_result_type, pMilliseconds_type } }, - .location = location, - }); -} - -/// `cuda.driver.event.query`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_event_query(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.event.query", .{ - .operands = .{ .flat = &.{hEvent} }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.driver.event.record`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_event_record(ctx: *mlir.Context, hEvent: *const mlir.Value, hStream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.event.record", .{ - .operands = .{ .flat = &.{ hEvent, hStream } }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.driver.event.record_with_flags`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_event_record_with_flags(ctx: *mlir.Context, hEvent: *const mlir.Value, hStream: *const mlir.Value, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.event.record_with_flags", .{ - .operands = .{ .flat = &.{ hEvent, hStream, flags } }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.driver.event.synchronize`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_event_synchronize(ctx: *mlir.Context, hEvent: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.event.synchronize", .{ - .operands = .{ .flat = &.{hEvent} }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.driver.library.get_global`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_library_get_global(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.library.get_global", .{ - .operands = .{ .flat = &.{ library, name_arg } }, - .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, - .location = location, - }); -} - -/// `cuda.driver.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, pKernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.library.get_kernel", .{ - .operands = .{ .flat = &.{ library, name_arg } }, - .results = .{ .flat = &.{ cuda_result_type, pKernel_type } }, - .location = location, - }); -} - -/// `cuda.driver.library.get_kernel_count`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_library_get_kernel_count(ctx: *mlir.Context, lib: *const mlir.Value, cuda_result_type: *const mlir.Type, count_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.library.get_kernel_count", .{ - .operands = .{ .flat = &.{lib} }, - .results = .{ .flat = &.{ cuda_result_type, count_type } }, - .location = location, - }); -} - -/// `cuda.driver.library.get_managed`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_library_get_managed(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.library.get_managed", .{ - .operands = .{ .flat = &.{ library, name_arg } }, - .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, - .location = location, - }); -} - -/// `cuda.driver.library.get_module`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_library_get_module(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, pMod_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.library.get_module", .{ - .operands = .{ .flat = &.{library} }, - .results = .{ .flat = &.{ cuda_result_type, pMod_type } }, - .location = location, - }); -} - -/// `cuda.driver.library.get_unified_function`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_library_get_unified_function(ctx: *mlir.Context, library: *const mlir.Value, symbol: *const mlir.Value, cuda_result_type: *const mlir.Type, fptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.library.get_unified_function", .{ - .operands = .{ .flat = &.{ library, symbol } }, - .results = .{ .flat = &.{ cuda_result_type, fptr_type } }, - .location = location, - }); -} - -/// `cuda.driver.library.load_data`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_library_load_data(ctx: *mlir.Context, code_arg: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.library.load_data", .{ - .operands = .{ .flat = &.{ code_arg, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, - .results = .{ .flat = &.{ cuda_result_type, library_type } }, - .location = location, - }); -} - -/// `cuda.driver.library.load_from_file`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_library_load_from_file(ctx: *mlir.Context, fileName: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.library.load_from_file", .{ - .operands = .{ .flat = &.{ fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, - .results = .{ .flat = &.{ cuda_result_type, library_type } }, - .location = location, - }); -} - -/// `cuda.driver.library.unload`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_library_unload(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.library.unload", .{ - .operands = .{ .flat = &.{library} }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.driver.tensor_map.encode_im2col`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_tensor_map_encode_im2col(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDim: *const mlir.Value, globalStrides: *const mlir.Value, pixelBoxLowerCorner: *const mlir.Value, pixelBoxUpperCorner: *const mlir.Value, channelsPerPixel: *const mlir.Value, pixelsPerColumn: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.tensor_map.encode_im2col", .{ - .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCorner, pixelBoxUpperCorner, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, swizzle, l2Promotion, oobFill } }, - .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, - .location = location, - }); -} - -/// `cuda.driver.tensor_map.encode_im2col_wide`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_tensor_map_encode_im2col_wide(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDim: *const mlir.Value, globalStrides: *const mlir.Value, pixelBoxLowerCornerWidth: *const mlir.Value, pixelBoxUpperCornerWidth: *const mlir.Value, channelsPerPixel: *const mlir.Value, pixelsPerColumn: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, mode: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.tensor_map.encode_im2col_wide", .{ - .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDim, globalStrides, pixelBoxLowerCornerWidth, pixelBoxUpperCornerWidth, channelsPerPixel, pixelsPerColumn, elementStrides, interleave, mode, swizzle, l2Promotion, oobFill } }, - .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, - .location = location, - }); -} - -/// `cuda.driver.tensor_map.replace_address`. Result types are explicit; attributes use mlir.Attribute. -pub fn driver_tensor_map_replace_address(ctx: *mlir.Context, globalAddress: *const mlir.Value, cuda_result_type: *const mlir.Type, tensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.driver.tensor_map.replace_address", .{ - .operands = .{ .flat = &.{globalAddress} }, - .results = .{ .flat = &.{ cuda_result_type, tensorMap_type } }, - .location = location, - }); -} - -/// `cuda.event.create`. Result types are explicit; attributes use mlir.Attribute. -pub fn event_create(ctx: *mlir.Context, result_type: *const mlir.Type, event_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.event.create", .{ - .results = .{ .flat = &.{ result_type, event_type } }, - .location = location, - }); -} - -/// `cuda.event.destroy`. Result types are explicit; attributes use mlir.Attribute. -pub fn event_destroy(ctx: *mlir.Context, event: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.event.destroy", .{ - .operands = .{ .flat = &.{event} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.event.elapsed_time`. Result types are explicit; attributes use mlir.Attribute. -pub fn event_elapsed_time(ctx: *mlir.Context, start_event: *const mlir.Value, end_event: *const mlir.Value, cuda_result_type: *const mlir.Type, elapsed_time_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.event.elapsed_time", .{ - .operands = .{ .flat = &.{ start_event, end_event } }, - .results = .{ .flat = &.{ cuda_result_type, elapsed_time_type } }, - .location = location, - }); -} - -/// `cuda.event.record`. Result types are explicit; attributes use mlir.Attribute. -pub fn event_record( - ctx: *mlir.Context, - event: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - stream: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.event.record", .{ - .operands = .{ .flat = if (opts.stream) |value| &.{ event, value } else &.{event} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.func`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. -pub fn func( - ctx: *mlir.Context, - sym_name: *const mlir.Attribute, - function_type: *const mlir.Attribute, - blocks: [1]*mlir.Block, - opts: struct { - arg_attrs: ?*const mlir.Attribute = null, - res_attrs: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "sym_name", sym_name)); - attributes.appendAssumeCapacity(.named(ctx, "function_type", function_type)); - if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); - if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); - return mlir.Operation.make(ctx, "cuda.func", .{ - .attributes = attributes.constSlice(), - .blocks = &blocks, - .verify = false, - .location = location, - }); -} - -/// `cuda.func.get_name`. Result types are explicit; attributes use mlir.Attribute. -pub fn func_get_name(ctx: *mlir.Context, kernel_: *const mlir.Value, result_type: *const mlir.Type, name_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.func.get_name", .{ - .operands = .{ .flat = &.{kernel_} }, - .results = .{ .flat = &.{ result_type, name_type } }, - .location = location, - }); -} - -/// `cuda.graph.destroy`. Result types are explicit; attributes use mlir.Attribute. -pub fn graph_destroy(ctx: *mlir.Context, graph: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.graph.destroy", .{ - .operands = .{ .flat = &.{graph} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.graph.instantiate`. Result types are explicit; attributes use mlir.Attribute. -pub fn graph_instantiate(ctx: *mlir.Context, graph: *const mlir.Value, result_type: *const mlir.Type, graph_exec_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.graph.instantiate", .{ - .operands = .{ .flat = &.{graph} }, - .results = .{ .flat = &.{ result_type, graph_exec_type } }, - .location = location, - }); -} - -/// `cuda.graph.launch`. Result types are explicit; attributes use mlir.Attribute. -pub fn graph_launch(ctx: *mlir.Context, graph_exec: *const mlir.Value, stream: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.graph.launch", .{ - .operands = .{ .flat = &.{ graph_exec, stream } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.graph_exec.destroy`. Result types are explicit; attributes use mlir.Attribute. -pub fn graph_exec_destroy(ctx: *mlir.Context, graph_exec: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.graph_exec.destroy", .{ - .operands = .{ .flat = &.{graph_exec} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.kernel`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: the blocks are usually filled afterwards. -pub fn kernel( - ctx: *mlir.Context, - sym_name: *const mlir.Attribute, - function_type: *const mlir.Attribute, - blocks: [1]*mlir.Block, - opts: struct { - arg_attrs: ?*const mlir.Attribute = null, - res_attrs: ?*const mlir.Attribute = null, - cu_func_attrs: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "sym_name", sym_name)); - attributes.appendAssumeCapacity(.named(ctx, "function_type", function_type)); - if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); - if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); - if (opts.cu_func_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "cu_func_attrs", value)); - return mlir.Operation.make(ctx, "cuda.kernel", .{ - .attributes = attributes.constSlice(), - .blocks = &blocks, - .verify = false, - .location = location, - }); -} - -/// `cuda.launch`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch( - ctx: *mlir.Context, - grid: []const *const mlir.Value, - block: []const *const mlir.Value, - dynamicSmem: *const mlir.Value, - cudaStream: *const mlir.Value, - inputs: []const *const mlir.Value, - cuda_result_type: *const mlir.Type, - callee: *const mlir.Attribute, - opts: struct { - arg_attrs: ?*const mlir.Attribute = null, - res_attrs: ?*const mlir.Attribute = null, - assume_kernel_attr: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "callee", callee)); - if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); - if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); - if (opts.assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); - return mlir.Operation.make(ctx, "cuda.launch", .{ - .operands = .{ .variadic = &.{ - grid, - block, - &.{dynamicSmem}, - &.{cudaStream}, - inputs, - } }, - .results = .{ .flat = &.{cuda_result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cuda.launch_cfg.access_policy_window`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_access_policy_window(ctx: *mlir.Context, launch_cfg: *const mlir.Value, base_ptr: *const mlir.Value, hitRatio: *const mlir.Value, num_bytes: *const mlir.Value, hitProp: *const mlir.Attribute, missProp: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.access_policy_window", .{ - .operands = .{ .flat = &.{ launch_cfg, base_ptr, hitRatio, num_bytes } }, - .attributes = &.{ - .named(ctx, "hitProp", hitProp), - .named(ctx, "missProp", missProp), - }, - .location = location, - }); -} - -/// `cuda.launch_cfg.cluster_dim`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_cluster_dim(ctx: *mlir.Context, launch_cfg: *const mlir.Value, clusterDimX: *const mlir.Value, clusterDimY: *const mlir.Value, clusterDimZ: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.cluster_dim", .{ - .operands = .{ .flat = &.{ launch_cfg, clusterDimX, clusterDimY, clusterDimZ } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.cluster_scheduling_policy`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_cluster_scheduling_policy(ctx: *mlir.Context, launch_cfg: *const mlir.Value, policy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.cluster_scheduling_policy", .{ - .operands = .{ .flat = &.{launch_cfg} }, - .attributes = &.{ - .named(ctx, "policy", policy), - }, - .location = location, - }); -} - -/// `cuda.launch_cfg.cooperative`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_cooperative(ctx: *mlir.Context, launch_cfg: *const mlir.Value, cooperative: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.cooperative", .{ - .operands = .{ .flat = &.{ launch_cfg, cooperative } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.create`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_create(ctx: *mlir.Context, blockDimX: *const mlir.Value, blockDimY: *const mlir.Value, blockDimZ: *const mlir.Value, dynamicSmemBytes: *const mlir.Value, gridDimX: *const mlir.Value, gridDimY: *const mlir.Value, gridDimZ: *const mlir.Value, stream: *const mlir.Value, result_type: *const mlir.Type, maxNumAttrs: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.create", .{ - .operands = .{ .flat = &.{ blockDimX, blockDimY, blockDimZ, dynamicSmemBytes, gridDimX, gridDimY, gridDimZ, stream } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "maxNumAttrs", maxNumAttrs), - }, - .location = location, - }); -} - -/// `cuda.launch_cfg.create_from_stream`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_create_from_stream(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, maxNumAttrs: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.create_from_stream", .{ - .operands = .{ .flat = &.{stream} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "maxNumAttrs", maxNumAttrs), - }, - .location = location, - }); -} - -/// `cuda.launch_cfg.device_updatable_kernel_node`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_device_updatable_kernel_node(ctx: *mlir.Context, launch_cfg: *const mlir.Value, devNode: *const mlir.Value, deviceUpdatable: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.device_updatable_kernel_node", .{ - .operands = .{ .flat = &.{ launch_cfg, devNode, deviceUpdatable } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.grid_dimensions`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_grid_dimensions(ctx: *mlir.Context, launch_cfg: *const mlir.Value, gridDimX: *const mlir.Value, gridDimY: *const mlir.Value, gridDimZ: *const mlir.Value, blockDimX: *const mlir.Value, blockDimY: *const mlir.Value, blockDimZ: *const mlir.Value, dynamicSmemBytes: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.grid_dimensions", .{ - .operands = .{ .flat = &.{ launch_cfg, gridDimX, gridDimY, gridDimZ, blockDimX, blockDimY, blockDimZ, dynamicSmemBytes } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.launch_completion_event`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_launch_completion_event(ctx: *mlir.Context, launch_cfg: *const mlir.Value, event: *const mlir.Value, flags: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.launch_completion_event", .{ - .operands = .{ .flat = &.{ launch_cfg, event, flags } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.mem_sync_domain`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_mem_sync_domain(ctx: *mlir.Context, launch_cfg: *const mlir.Value, memSyncDomain: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.mem_sync_domain", .{ - .operands = .{ .flat = &.{launch_cfg} }, - .attributes = &.{ - .named(ctx, "memSyncDomain", memSyncDomain), - }, - .location = location, - }); -} - -/// `cuda.launch_cfg.mem_sync_domain_map`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_mem_sync_domain_map(ctx: *mlir.Context, launch_cfg: *const mlir.Value, domain_default: *const mlir.Value, domain_remote: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.mem_sync_domain_map", .{ - .operands = .{ .flat = &.{ launch_cfg, domain_default, domain_remote } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.nvlink_util_centric_scheduling`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_nvlink_util_centric_scheduling(ctx: *mlir.Context, launch_cfg: *const mlir.Value, nvlinkSchedulingEnabled: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.nvlink_util_centric_scheduling", .{ - .operands = .{ .flat = &.{ launch_cfg, nvlinkSchedulingEnabled } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.preferred_cluster_dim`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_preferred_cluster_dim(ctx: *mlir.Context, launch_cfg: *const mlir.Value, preferredClusterDimX: *const mlir.Value, preferredClusterDimY: *const mlir.Value, preferredClusterDimZ: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.preferred_cluster_dim", .{ - .operands = .{ .flat = &.{ launch_cfg, preferredClusterDimX, preferredClusterDimY, preferredClusterDimZ } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.print`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_print(ctx: *mlir.Context, launch_cfg: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.print", .{ - .operands = .{ .flat = &.{launch_cfg} }, - .location = location, - }); -} - -/// `cuda.launch_cfg.priority`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_priority(ctx: *mlir.Context, launch_cfg: *const mlir.Value, priority: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.priority", .{ - .operands = .{ .flat = &.{ launch_cfg, priority } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.programmatic_event`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_programmatic_event(ctx: *mlir.Context, launch_cfg: *const mlir.Value, event: *const mlir.Value, flags: *const mlir.Value, triggerAtBlockStart: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.programmatic_event", .{ - .operands = .{ .flat = &.{ launch_cfg, event, flags, triggerAtBlockStart } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.programmatic_stream_serialization_allowed`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_programmatic_stream_serialization_allowed(ctx: *mlir.Context, launch_cfg: *const mlir.Value, programmaticStreamSerializationAllowed: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.programmatic_stream_serialization_allowed", .{ - .operands = .{ .flat = &.{ launch_cfg, programmaticStreamSerializationAllowed } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.shared_mem_carveout`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_shared_mem_carveout(ctx: *mlir.Context, launch_cfg: *const mlir.Value, sharedMemCarveout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.shared_mem_carveout", .{ - .operands = .{ .flat = &.{ launch_cfg, sharedMemCarveout } }, - .location = location, - }); -} - -/// `cuda.launch_cfg.sync_policy`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_cfg_sync_policy(ctx: *mlir.Context, launch_cfg: *const mlir.Value, syncPolicy: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.launch_cfg.sync_policy", .{ - .operands = .{ .flat = &.{launch_cfg} }, - .attributes = &.{ - .named(ctx, "syncPolicy", syncPolicy), - }, - .location = location, - }); -} - -/// `cuda.launch_ex`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_ex( - ctx: *mlir.Context, - launch_cfg: *const mlir.Value, - inputs: []const *const mlir.Value, - cuda_result_type: *const mlir.Type, - callee: *const mlir.Attribute, - opts: struct { - arg_attrs: ?*const mlir.Attribute = null, - res_attrs: ?*const mlir.Attribute = null, - assume_kernel_attr: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; - operands.appendAssumeCapacity(launch_cfg); - operands.appendSliceAssumeCapacity(inputs); - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "callee", callee)); - if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); - if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); - if (opts.assume_kernel_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "assume_kernel_attr", value)); - return mlir.Operation.make(ctx, "cuda.launch_ex", .{ - .operands = .{ .flat = operands.constSlice() }, - .results = .{ .flat = &.{cuda_result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cuda.launch_kernel`. Result types are explicit; attributes use mlir.Attribute. -pub fn launch_kernel( - ctx: *mlir.Context, - kernel_: *const mlir.Value, - grid: []const *const mlir.Value, - block: []const *const mlir.Value, - dynamicSmem: *const mlir.Value, - cudaStream: *const mlir.Value, - kernel_args: *const mlir.Value, - cuda_result_type: *const mlir.Type, - opts: struct { - arg_attrs: ?*const mlir.Attribute = null, - res_attrs: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.arg_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", value)); - if (opts.res_attrs) |value| attributes.appendAssumeCapacity(.named(ctx, "res_attrs", value)); - return mlir.Operation.make(ctx, "cuda.launch_kernel", .{ - .operands = .{ .variadic = &.{ - &.{kernel_}, - grid, - block, - &.{dynamicSmem}, - &.{cudaStream}, - &.{kernel_args}, - } }, - .results = .{ .flat = &.{cuda_result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cuda.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. -pub fn library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, kernel_name: *const mlir.Value, result_type: *const mlir.Type, kernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.library.get_kernel", .{ - .operands = .{ .flat = &.{ library, kernel_name } }, - .results = .{ .flat = &.{ result_type, kernel_type } }, - .location = location, - }); -} - -/// `cuda.library.load_data`. Result types are explicit; attributes use mlir.Attribute. -pub fn library_load_data(ctx: *mlir.Context, data: *const mlir.Value, result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.library.load_data", .{ - .operands = .{ .flat = &.{data} }, - .results = .{ .flat = &.{ result_type, library_type } }, - .location = location, - }); -} - -/// `cuda.library.unload`. Result types are explicit; attributes use mlir.Attribute. -pub fn library_unload(ctx: *mlir.Context, library: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.library.unload", .{ - .operands = .{ .flat = &.{library} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.memory.alloc`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_alloc(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.memory.alloc", .{ - .operands = .{ .flat = &.{bytes} }, - .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, - .location = location, - }); -} - -/// `cuda.memory.alloc_async`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_alloc_async(ctx: *mlir.Context, bytes: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.memory.alloc_async", .{ - .operands = .{ .flat = &.{ bytes, stream } }, - .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, - .location = location, - }); -} - -/// `cuda.memory.alloc_host`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_alloc_host(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.memory.alloc_host", .{ - .operands = .{ .flat = &.{bytes} }, - .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, - .location = location, - }); -} - -/// `cuda.memory.alloc_managed`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_alloc_managed(ctx: *mlir.Context, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.memory.alloc_managed", .{ - .operands = .{ .flat = &.{bytes} }, - .results = .{ .flat = &.{ cuda_result_type, ptr_type } }, - .location = location, - }); -} - -/// `cuda.memory.copy`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_copy(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.memory.copy", .{ - .operands = .{ .flat = &.{ src, dst, bytes } }, - .results = .{ .flat = &.{cuda_result_type} }, - .attributes = &.{ - .named(ctx, "kind", kind), - }, - .location = location, - }); -} - -/// `cuda.memory.copy_async`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_copy_async( - ctx: *mlir.Context, - src: *const mlir.Value, - dst: *const mlir.Value, - bytes: *const mlir.Value, - cuda_result_type: *const mlir.Type, - kind: *const mlir.Attribute, - opts: struct { - stream: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.memory.copy_async", .{ - .operands = .{ .flat = if (opts.stream) |value| &.{ src, dst, bytes, value } else &.{ src, dst, bytes } }, - .results = .{ .flat = &.{cuda_result_type} }, - .attributes = &.{ - .named(ctx, "kind", kind), - }, - .location = location, - }); -} - -/// `cuda.memory.free`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_free(ctx: *mlir.Context, ptr: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.memory.free", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.memory.free_async`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_free_async(ctx: *mlir.Context, ptr: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.memory.free_async", .{ - .operands = .{ .flat = &.{ ptr, stream } }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.memory.free_host`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_free_host(ctx: *mlir.Context, ptr: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.memory.free_host", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.memory.set`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_set(ctx: *mlir.Context, ptr: *const mlir.Value, value_: *const mlir.Value, bytes: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.memory.set", .{ - .operands = .{ .flat = &.{ ptr, value_, bytes } }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.memory.set_async`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_set_async( - ctx: *mlir.Context, - ptr: *const mlir.Value, - value_: *const mlir.Value, - bytes: *const mlir.Value, - cuda_result_type: *const mlir.Type, - opts: struct { - stream: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.memory.set_async", .{ - .operands = .{ .flat = if (opts.stream) |value| &.{ ptr, value_, bytes, value } else &.{ ptr, value_, bytes } }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.occupancy_max_active_blocks_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. -pub fn occupancy_max_active_blocks_per_multiprocessor(ctx: *mlir.Context, block_size: *const mlir.Value, dynamic_smem_size: *const mlir.Value, res_type: *const mlir.Type, num_blocks_type: *const mlir.Type, func_: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.occupancy_max_active_blocks_per_multiprocessor", .{ - .operands = .{ .flat = &.{ block_size, dynamic_smem_size } }, - .results = .{ .flat = &.{ res_type, num_blocks_type } }, - .attributes = &.{ - .named(ctx, "func", func_), - }, - .location = location, - }); -} - -/// `cuda.profiler.start`. Result types are explicit; attributes use mlir.Attribute. -pub fn profiler_start(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.profiler.start", .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.profiler.stop`. Result types are explicit; attributes use mlir.Attribute. -pub fn profiler_stop(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.profiler.stop", .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.result.assert_success`. Result types are explicit; attributes use mlir.Attribute. -pub fn result_assert_success(ctx: *mlir.Context, cuda_result: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.result.assert_success", .{ - .operands = .{ .flat = &.{cuda_result} }, - .location = location, - }); -} - -/// `cuda.result.failed`. Result types are explicit; attributes use mlir.Attribute. -pub fn result_failed(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.result.failed", .{ - .location = location, - }); -} - -/// `cuda.result.print_on_error`. Result types are explicit; attributes use mlir.Attribute. -pub fn result_print_on_error(ctx: *mlir.Context, cuda_result: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.result.print_on_error", .{ - .operands = .{ .flat = &.{cuda_result} }, - .location = location, - }); -} - -/// `cuda.result.succeeded`. Result types are explicit; attributes use mlir.Attribute. -pub fn result_succeeded(ctx: *mlir.Context, cuda_result: *const mlir.Value, success_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.result.succeeded", .{ - .operands = .{ .flat = &.{cuda_result} }, - .results = .{ .flat = &.{success_type} }, - .location = location, - }); -} - -/// `cuda.return`. Result types are explicit; attributes use mlir.Attribute. Not verified on creation: a terminator is only one in its block. -pub fn return_(ctx: *mlir.Context, inputs: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.return", .{ - .operands = .{ .flat = inputs }, - .verify = false, - .location = location, - }); -} - -/// `cuda.return_if_error`. Result types are explicit; attributes use mlir.Attribute. -pub fn return_if_error(ctx: *mlir.Context, inputs: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.return_if_error", .{ - .operands = .{ .flat = inputs }, - .location = location, - }); -} - -/// `cuda.runtime.device.properties`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_properties(ctx: *mlir.Context, device: *const mlir.Value, cuda_result_type: *const mlir.Type, prop_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device.properties", .{ - .operands = .{ .flat = &.{device} }, - .results = .{ .flat = &.{ cuda_result_type, prop_type } }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_access_policy_max_window_size`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_access_policy_max_window_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_access_policy_max_window_size", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_async_engine_count`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_async_engine_count(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_async_engine_count", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_can_map_host_memory`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_can_map_host_memory(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_can_map_host_memory", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_can_use_host_pointer_for_registered_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_cluster_launch`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_cluster_launch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_cluster_launch", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_compute_preemption_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_compute_preemption_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_compute_preemption_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_concurrent_kernels`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_concurrent_kernels(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_concurrent_kernels", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_concurrent_managed_access`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_concurrent_managed_access(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_concurrent_managed_access", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_cooperative_launch`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_cooperative_launch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_cooperative_launch", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_deferred_mapping_cuda_array_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_device_numa_config`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_device_numa_config(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_device_numa_config", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_device_numa_id`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_device_numa_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_device_numa_id", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_direct_managed_mem_access_from_host`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_direct_managed_mem_access_from_host(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_direct_managed_mem_access_from_host", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_ecc_enabled`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_ecc_enabled(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_ecc_enabled", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_global_l1cache_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_global_l1cache_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_global_l1cache_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_gpu_direct_rdma_flush_writes_options(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_gpu_direct_rdma_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_gpu_direct_rdma_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_gpu_direct_rdma_writes_ordering(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_gpu_pci_device_id`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_gpu_pci_device_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_pci_device_id", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_gpu_pci_subsystem_id`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_gpu_pci_subsystem_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_gpu_pci_subsystem_id", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_host_native_atomic_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_host_native_atomic_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_native_atomic_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_host_numa_id`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_host_numa_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_numa_id", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_host_numa_multinode_ipc_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_host_register_read_only_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_host_register_read_only_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_register_read_only_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_host_register_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_host_register_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_host_register_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_integrated`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_integrated(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_integrated", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_ipc_event_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_ipc_event_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_ipc_event_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_is_multi_gpu_board`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_is_multi_gpu_board(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_is_multi_gpu_board", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_l_2cache_size`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_l_2cache_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_l_2cache_size", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_local_l1cache_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_local_l1cache_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_local_l1cache_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_luid`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_luid(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_luid", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_luid_device_node_mask`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_luid_device_node_mask(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_luid_device_node_mask", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_major`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_major(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_major", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_managed_memory`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_managed_memory(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_managed_memory", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_blocks_per_multi_processor`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_blocks_per_multi_processor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_blocks_per_multi_processor", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_grid_size`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_grid_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_grid_size", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_surface_1d`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_surface_1d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_1d", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_surface_1d_layered`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_surface_1d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_1d_layered", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_surface_2d`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_surface_2d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_2d", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_surface_2d_layered`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_surface_2d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_2d_layered", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_surface_3d`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_surface_3d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_3d", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_surface_cubemap`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_surface_cubemap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_cubemap", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_surface_cubemap_layered`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_surface_cubemap_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_surface_cubemap_layered", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_1d`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_1d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_1d_layered`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_1d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d_layered", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_1d_mipmap`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_1d_mipmap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_1d_mipmap", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_2d`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_2d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_2d_gather`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_2d_gather(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_gather", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_2d_layered`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_2d_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_layered", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_2d_linear`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_2d_linear(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_linear", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_2d_mipmap`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_2d_mipmap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_2d_mipmap", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_3d`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_3d(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_3d", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_3d_alt`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_3d_alt(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_3d_alt", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_cubemap`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_cubemap(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_cubemap", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_texture_cubemap_layered`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_texture_cubemap_layered(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_texture_cubemap_layered", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_threads_dim`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_threads_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_dim", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_threads_per_block`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_threads_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_per_block", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_max_threads_per_multi_processor`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_max_threads_per_multi_processor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_max_threads_per_multi_processor", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_mem_pitch`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_mem_pitch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_mem_pitch", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_memory_bus_width`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_memory_bus_width(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_bus_width", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_memory_pool_supported_handle_types`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_memory_pool_supported_handle_types(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_pool_supported_handle_types", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_memory_pools_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_memory_pools_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_memory_pools_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_minor`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_minor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_minor", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_mps_enabled`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_mps_enabled(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_mps_enabled", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_multi_gpu_board_group_id`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_multi_gpu_board_group_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_multi_gpu_board_group_id", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_multi_processor_count`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_multi_processor_count(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_multi_processor_count", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_name`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_name(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_name", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_pageable_memory_access`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_pageable_memory_access(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pageable_memory_access", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_pageable_memory_access_uses_host_page_tables(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_pci_bus_id`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_pci_bus_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_bus_id", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_pci_device_id`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_pci_device_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_device_id", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_pci_domain_id`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_pci_domain_id(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_pci_domain_id", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_persisting_l2cache_max_size`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_persisting_l2cache_max_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_persisting_l2cache_max_size", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_regs_per_block`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_regs_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_regs_per_block", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_regs_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_regs_per_multiprocessor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_regs_per_multiprocessor", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_reserved_shared_mem_per_block`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_reserved_shared_mem_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_reserved_shared_mem_per_block", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_shared_mem_per_block`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_shared_mem_per_block(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_block", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_shared_mem_per_block_optin`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_shared_mem_per_block_optin(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_block_optin", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_shared_mem_per_multiprocessor`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_shared_mem_per_multiprocessor(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_shared_mem_per_multiprocessor", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_sparse_cuda_array_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_sparse_cuda_array_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_sparse_cuda_array_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_stream_priorities_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_stream_priorities_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_stream_priorities_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_surface_alignment`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_surface_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_surface_alignment", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_tcc_driver`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_tcc_driver(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_tcc_driver", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_texture_alignment`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_texture_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_texture_alignment", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_texture_pitch_alignment`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_texture_pitch_alignment(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_texture_pitch_alignment", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_timeline_semaphore_interop_supported`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_timeline_semaphore_interop_supported(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_timeline_semaphore_interop_supported", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_total_const_mem`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_total_const_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_total_const_mem", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_total_global_mem`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_total_global_mem(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_total_global_mem", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_unified_addressing`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_unified_addressing(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_unified_addressing", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_unified_function_pointers`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_unified_function_pointers(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_unified_function_pointers", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_uuid`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_uuid(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_uuid", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.device_prop.get_warp_size`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_device_prop_get_warp_size(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.device_prop.get_warp_size", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.dim_3.get_x`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_dim_3_get_x(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_x", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.dim_3.get_y`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_dim_3_get_y(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_y", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.dim_3.get_z`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_dim_3_get_z(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.dim_3.get_z", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.event.create_with_flags`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_event_create_with_flags(ctx: *mlir.Context, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, event_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.event.create_with_flags", .{ - .operands = .{ .flat = &.{flags} }, - .results = .{ .flat = &.{ cuda_result_type, event_type } }, - .location = location, - }); -} - -/// `cuda.runtime.event.query`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_event_query(ctx: *mlir.Context, event: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.event.query", .{ - .operands = .{ .flat = &.{event} }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.event.record_with_flags`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_event_record_with_flags(ctx: *mlir.Context, event: *const mlir.Value, stream: *const mlir.Value, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.event.record_with_flags", .{ - .operands = .{ .flat = &.{ event, stream, flags } }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.event.synchronize`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_event_synchronize(ctx: *mlir.Context, event: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.event.synchronize", .{ - .operands = .{ .flat = &.{event} }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.extent.get_depth`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_extent_get_depth(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.extent.get_depth", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.extent.get_height`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_extent_get_height(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.extent.get_height", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.extent.get_width`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_extent_get_width(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.extent.get_width", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.graph.add_kernel_node`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_graph_add_kernel_node(ctx: *mlir.Context, graph: *const mlir.Value, pDependencies: *const mlir.Value, numDependencies: *const mlir.Value, pNodeParams: *const mlir.Value, cuda_result_type: *const mlir.Type, pGraphNode_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.graph.add_kernel_node", .{ - .operands = .{ .flat = &.{ graph, pDependencies, numDependencies, pNodeParams } }, - .results = .{ .flat = &.{ cuda_result_type, pGraphNode_type } }, - .location = location, - }); -} - -/// `cuda.runtime.graph.create`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_graph_create(ctx: *mlir.Context, flags: *const mlir.Value, cuda_result_type: *const mlir.Type, pGraph_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.graph.create", .{ - .operands = .{ .flat = &.{flags} }, - .results = .{ .flat = &.{ cuda_result_type, pGraph_type } }, - .location = location, - }); -} - -/// `cuda.runtime.graph.kernel_node_get_attribute`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_graph_kernel_node_get_attribute(ctx: *mlir.Context, hNode: *const mlir.Value, attr: *const mlir.Value, cuda_result_type: *const mlir.Type, value_out_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.graph.kernel_node_get_attribute", .{ - .operands = .{ .flat = &.{ hNode, attr } }, - .results = .{ .flat = &.{ cuda_result_type, value_out_type } }, - .location = location, - }); -} - -/// `cuda.runtime.graph.kernel_node_set_attribute`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_graph_kernel_node_set_attribute(ctx: *mlir.Context, hNode: *const mlir.Value, attr: *const mlir.Value, value_: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.graph.kernel_node_set_attribute", .{ - .operands = .{ .flat = &.{ hNode, attr, value_ } }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.kernel_node_params.get_block_dim`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_kernel_node_params_get_block_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_block_dim", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.kernel_node_params.get_extra`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_kernel_node_params_get_extra(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_extra", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.kernel_node_params.get_func`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_kernel_node_params_get_func(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_func", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.kernel_node_params.get_grid_dim`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_kernel_node_params_get_grid_dim(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_grid_dim", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.kernel_node_params.get_kernel_params`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_kernel_node_params_get_kernel_params(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_kernel_params", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.kernel_node_params.get_shared_mem_bytes`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_kernel_node_params_get_shared_mem_bytes(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.kernel_node_params.get_shared_mem_bytes", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.library.get_global`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_library_get_global(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.library.get_global", .{ - .operands = .{ .flat = &.{ library, name_arg } }, - .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, - .location = location, - }); -} - -/// `cuda.runtime.library.get_kernel`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_library_get_kernel(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, pKernel_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.library.get_kernel", .{ - .operands = .{ .flat = &.{ library, name_arg } }, - .results = .{ .flat = &.{ cuda_result_type, pKernel_type } }, - .location = location, - }); -} - -/// `cuda.runtime.library.get_kernel_count`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_library_get_kernel_count(ctx: *mlir.Context, lib: *const mlir.Value, cuda_result_type: *const mlir.Type, count_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.library.get_kernel_count", .{ - .operands = .{ .flat = &.{lib} }, - .results = .{ .flat = &.{ cuda_result_type, count_type } }, - .location = location, - }); -} - -/// `cuda.runtime.library.get_managed`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_library_get_managed(ctx: *mlir.Context, library: *const mlir.Value, name_arg: *const mlir.Value, cuda_result_type: *const mlir.Type, dptr_type: *const mlir.Type, bytes_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.library.get_managed", .{ - .operands = .{ .flat = &.{ library, name_arg } }, - .results = .{ .flat = &.{ cuda_result_type, dptr_type, bytes_type } }, - .location = location, - }); -} - -/// `cuda.runtime.library.get_unified_function`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_library_get_unified_function(ctx: *mlir.Context, library: *const mlir.Value, symbol: *const mlir.Value, cuda_result_type: *const mlir.Type, fptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.library.get_unified_function", .{ - .operands = .{ .flat = &.{ library, symbol } }, - .results = .{ .flat = &.{ cuda_result_type, fptr_type } }, - .location = location, - }); -} - -/// `cuda.runtime.library.load_data`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_library_load_data(ctx: *mlir.Context, code_arg: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.library.load_data", .{ - .operands = .{ .flat = &.{ code_arg, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, - .results = .{ .flat = &.{ cuda_result_type, library_type } }, - .location = location, - }); -} - -/// `cuda.runtime.library.load_from_file`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_library_load_from_file(ctx: *mlir.Context, fileName: *const mlir.Value, jitOptions: *const mlir.Value, jitOptionsValues: *const mlir.Value, numJitOptions: *const mlir.Value, libraryOptions: *const mlir.Value, libraryOptionValues: *const mlir.Value, numLibraryOptions: *const mlir.Value, cuda_result_type: *const mlir.Type, library_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.library.load_from_file", .{ - .operands = .{ .flat = &.{ fileName, jitOptions, jitOptionsValues, numJitOptions, libraryOptions, libraryOptionValues, numLibraryOptions } }, - .results = .{ .flat = &.{ cuda_result_type, library_type } }, - .location = location, - }); -} - -/// `cuda.runtime.library.unload`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_library_unload(ctx: *mlir.Context, library: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.library.unload", .{ - .operands = .{ .flat = &.{library} }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.memcpy_3d_params.get_dst_array`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_memcpy_3d_params_get_dst_array(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_array", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.memcpy_3d_params.get_dst_pos`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_memcpy_3d_params_get_dst_pos(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_pos", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.memcpy_3d_params.get_dst_ptr`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_memcpy_3d_params_get_dst_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_dst_ptr", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.memcpy_3d_params.get_extent`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_memcpy_3d_params_get_extent(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_extent", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.memcpy_3d_params.get_kind`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_memcpy_3d_params_get_kind(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_kind", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.memcpy_3d_params.get_src_array`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_memcpy_3d_params_get_src_array(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_array", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.memcpy_3d_params.get_src_pos`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_memcpy_3d_params_get_src_pos(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_pos", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.memcpy_3d_params.get_src_ptr`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_memcpy_3d_params_get_src_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.memcpy_3d_params.get_src_ptr", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.memory.cpy3d`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_memory_cpy3d(ctx: *mlir.Context, p: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.memory.cpy3d", .{ - .operands = .{ .flat = &.{p} }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.memory.cpy3d_async`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_memory_cpy3d_async(ctx: *mlir.Context, p: *const mlir.Value, stream: *const mlir.Value, cuda_result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.memory.cpy3d_async", .{ - .operands = .{ .flat = &.{ p, stream } }, - .results = .{ .flat = &.{cuda_result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.pitched_ptr.get_pitch`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_pitched_ptr_get_pitch(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_pitch", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.pitched_ptr.get_ptr`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_pitched_ptr_get_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_ptr", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.pitched_ptr.get_xsize`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_pitched_ptr_get_xsize(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_xsize", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.pitched_ptr.get_ysize`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_pitched_ptr_get_ysize(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.pitched_ptr.get_ysize", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.pos.get_x`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_pos_get_x(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.pos.get_x", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.pos.get_y`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_pos_get_y(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.pos.get_y", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.pos.get_z`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_pos_get_z(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.pos.get_z", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.runtime.uuid.get_bytes`. Result types are explicit; attributes use mlir.Attribute. -pub fn runtime_uuid_get_bytes(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.runtime.uuid.get_bytes", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.stream.add_callback`. Result types are explicit; attributes use mlir.Attribute. -pub fn stream_add_callback(ctx: *mlir.Context, stream: *const mlir.Value, user_data: *const mlir.Value, result_type: *const mlir.Type, callback: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.stream.add_callback", .{ - .operands = .{ .flat = &.{ stream, user_data } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "callback", callback), - }, - .location = location, - }); -} - -/// `cuda.stream.begin_capture`. Result types are explicit; attributes use mlir.Attribute. -pub fn stream_begin_capture(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.stream.begin_capture", .{ - .operands = .{ .flat = &.{stream} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "mode", mode), - }, - .location = location, - }); -} - -/// `cuda.stream.create`. Result types are explicit; attributes use mlir.Attribute. -pub fn stream_create(ctx: *mlir.Context, cuda_result_type: *const mlir.Type, stream_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.stream.create", .{ - .results = .{ .flat = &.{ cuda_result_type, stream_type } }, - .location = location, - }); -} - -/// `cuda.stream.destroy`. Result types are explicit; attributes use mlir.Attribute. -pub fn stream_destroy(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.stream.destroy", .{ - .operands = .{ .flat = &.{stream} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.stream.end_capture`. Result types are explicit; attributes use mlir.Attribute. -pub fn stream_end_capture(ctx: *mlir.Context, stream: *const mlir.Value, result_type: *const mlir.Type, graph_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.stream.end_capture", .{ - .operands = .{ .flat = &.{stream} }, - .results = .{ .flat = &.{ result_type, graph_type } }, - .location = location, - }); -} - -/// `cuda.stream.synchronize`. Result types are explicit; attributes use mlir.Attribute. -pub fn stream_synchronize( - ctx: *mlir.Context, - stream: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - event: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.stream.synchronize", .{ - .operands = .{ .flat = if (opts.event) |value| &.{ stream, value } else &.{stream} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cuda.tensor_map_encode_tiled`. Result types are explicit; attributes use mlir.Attribute. -pub fn tensor_map_encode_tiled(ctx: *mlir.Context, tensorDataType: *const mlir.Value, tensorRank: *const mlir.Value, globalAddress: *const mlir.Value, globalDims: *const mlir.Value, globalStrides: *const mlir.Value, boxDims: *const mlir.Value, elementStrides: *const mlir.Value, interleave: *const mlir.Value, swizzle: *const mlir.Value, l2Promotion: *const mlir.Value, oobFill: *const mlir.Value, cuda_result_type: *const mlir.Type, cuTensorMap_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cuda.tensor_map_encode_tiled", .{ - .operands = .{ .flat = &.{ tensorDataType, tensorRank, globalAddress, globalDims, globalStrides, boxDims, elementStrides, interleave, swizzle, l2Promotion, oobFill } }, - .results = .{ .flat = &.{ cuda_result_type, cuTensorMap_type } }, - .location = location, - }); -} diff --git a/mlir/dialects/cuda/test.zig b/mlir/dialects/cuda/test.zig deleted file mode 100644 index b17f50cd..00000000 --- a/mlir/dialects/cuda/test.zig +++ /dev/null @@ -1,324 +0,0 @@ -const std = @import("std"); - -const cuda = @import("mlir/dialects/cuda"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "cuda.addressof", - "cuda.binary", - "cuda.cast", - "cuda.compose", - "cuda.decompose", - "cuda.device.sync", - "cuda.driver.event.create", - "cuda.driver.event.destroy", - "cuda.driver.event.elapsed_time", - "cuda.driver.event.query", - "cuda.driver.event.record", - "cuda.driver.event.record_with_flags", - "cuda.driver.event.synchronize", - "cuda.driver.library.get_global", - "cuda.driver.library.get_kernel", - "cuda.driver.library.get_kernel_count", - "cuda.driver.library.get_managed", - "cuda.driver.library.get_module", - "cuda.driver.library.get_unified_function", - "cuda.driver.library.load_data", - "cuda.driver.library.load_from_file", - "cuda.driver.library.unload", - "cuda.driver.tensor_map.encode_im2col", - "cuda.driver.tensor_map.encode_im2col_wide", - "cuda.driver.tensor_map.replace_address", - "cuda.event.create", - "cuda.event.destroy", - "cuda.event.elapsed_time", - "cuda.event.record", - "cuda.func", - "cuda.func.get_name", - "cuda.graph.destroy", - "cuda.graph.instantiate", - "cuda.graph.launch", - "cuda.graph_exec.destroy", - "cuda.kernel", - "cuda.launch", - "cuda.launch_cfg.access_policy_window", - "cuda.launch_cfg.cluster_dim", - "cuda.launch_cfg.cluster_scheduling_policy", - "cuda.launch_cfg.cooperative", - "cuda.launch_cfg.create", - "cuda.launch_cfg.create_from_stream", - "cuda.launch_cfg.device_updatable_kernel_node", - "cuda.launch_cfg.grid_dimensions", - "cuda.launch_cfg.launch_completion_event", - "cuda.launch_cfg.mem_sync_domain", - "cuda.launch_cfg.mem_sync_domain_map", - "cuda.launch_cfg.nvlink_util_centric_scheduling", - "cuda.launch_cfg.preferred_cluster_dim", - "cuda.launch_cfg.print", - "cuda.launch_cfg.priority", - "cuda.launch_cfg.programmatic_event", - "cuda.launch_cfg.programmatic_stream_serialization_allowed", - "cuda.launch_cfg.shared_mem_carveout", - "cuda.launch_cfg.sync_policy", - "cuda.launch_ex", - "cuda.launch_kernel", - "cuda.library.get_kernel", - "cuda.library.load_data", - "cuda.library.unload", - "cuda.memory.alloc", - "cuda.memory.alloc_async", - "cuda.memory.alloc_host", - "cuda.memory.alloc_managed", - "cuda.memory.copy", - "cuda.memory.copy_async", - "cuda.memory.free", - "cuda.memory.free_async", - "cuda.memory.free_host", - "cuda.memory.set", - "cuda.memory.set_async", - "cuda.occupancy_max_active_blocks_per_multiprocessor", - "cuda.profiler.start", - "cuda.profiler.stop", - "cuda.result.assert_success", - "cuda.result.failed", - "cuda.result.print_on_error", - "cuda.result.succeeded", - "cuda.return", - "cuda.return_if_error", - "cuda.runtime.device.properties", - "cuda.runtime.device_prop.get_access_policy_max_window_size", - "cuda.runtime.device_prop.get_async_engine_count", - "cuda.runtime.device_prop.get_can_map_host_memory", - "cuda.runtime.device_prop.get_can_use_host_pointer_for_registered_mem", - "cuda.runtime.device_prop.get_cluster_launch", - "cuda.runtime.device_prop.get_compute_preemption_supported", - "cuda.runtime.device_prop.get_concurrent_kernels", - "cuda.runtime.device_prop.get_concurrent_managed_access", - "cuda.runtime.device_prop.get_cooperative_launch", - "cuda.runtime.device_prop.get_deferred_mapping_cuda_array_supported", - "cuda.runtime.device_prop.get_device_numa_config", - "cuda.runtime.device_prop.get_device_numa_id", - "cuda.runtime.device_prop.get_direct_managed_mem_access_from_host", - "cuda.runtime.device_prop.get_ecc_enabled", - "cuda.runtime.device_prop.get_global_l1cache_supported", - "cuda.runtime.device_prop.get_gpu_direct_rdma_flush_writes_options", - "cuda.runtime.device_prop.get_gpu_direct_rdma_supported", - "cuda.runtime.device_prop.get_gpu_direct_rdma_writes_ordering", - "cuda.runtime.device_prop.get_gpu_pci_device_id", - "cuda.runtime.device_prop.get_gpu_pci_subsystem_id", - "cuda.runtime.device_prop.get_host_native_atomic_supported", - "cuda.runtime.device_prop.get_host_numa_id", - "cuda.runtime.device_prop.get_host_numa_multinode_ipc_supported", - "cuda.runtime.device_prop.get_host_register_read_only_supported", - "cuda.runtime.device_prop.get_host_register_supported", - "cuda.runtime.device_prop.get_integrated", - "cuda.runtime.device_prop.get_ipc_event_supported", - "cuda.runtime.device_prop.get_is_multi_gpu_board", - "cuda.runtime.device_prop.get_l_2cache_size", - "cuda.runtime.device_prop.get_local_l1cache_supported", - "cuda.runtime.device_prop.get_luid", - "cuda.runtime.device_prop.get_luid_device_node_mask", - "cuda.runtime.device_prop.get_major", - "cuda.runtime.device_prop.get_managed_memory", - "cuda.runtime.device_prop.get_max_blocks_per_multi_processor", - "cuda.runtime.device_prop.get_max_grid_size", - "cuda.runtime.device_prop.get_max_surface_1d", - "cuda.runtime.device_prop.get_max_surface_1d_layered", - "cuda.runtime.device_prop.get_max_surface_2d", - "cuda.runtime.device_prop.get_max_surface_2d_layered", - "cuda.runtime.device_prop.get_max_surface_3d", - "cuda.runtime.device_prop.get_max_surface_cubemap", - "cuda.runtime.device_prop.get_max_surface_cubemap_layered", - "cuda.runtime.device_prop.get_max_texture_1d", - "cuda.runtime.device_prop.get_max_texture_1d_layered", - "cuda.runtime.device_prop.get_max_texture_1d_mipmap", - "cuda.runtime.device_prop.get_max_texture_2d", - "cuda.runtime.device_prop.get_max_texture_2d_gather", - "cuda.runtime.device_prop.get_max_texture_2d_layered", - "cuda.runtime.device_prop.get_max_texture_2d_linear", - "cuda.runtime.device_prop.get_max_texture_2d_mipmap", - "cuda.runtime.device_prop.get_max_texture_3d", - "cuda.runtime.device_prop.get_max_texture_3d_alt", - "cuda.runtime.device_prop.get_max_texture_cubemap", - "cuda.runtime.device_prop.get_max_texture_cubemap_layered", - "cuda.runtime.device_prop.get_max_threads_dim", - "cuda.runtime.device_prop.get_max_threads_per_block", - "cuda.runtime.device_prop.get_max_threads_per_multi_processor", - "cuda.runtime.device_prop.get_mem_pitch", - "cuda.runtime.device_prop.get_memory_bus_width", - "cuda.runtime.device_prop.get_memory_pool_supported_handle_types", - "cuda.runtime.device_prop.get_memory_pools_supported", - "cuda.runtime.device_prop.get_minor", - "cuda.runtime.device_prop.get_mps_enabled", - "cuda.runtime.device_prop.get_multi_gpu_board_group_id", - "cuda.runtime.device_prop.get_multi_processor_count", - "cuda.runtime.device_prop.get_name", - "cuda.runtime.device_prop.get_pageable_memory_access", - "cuda.runtime.device_prop.get_pageable_memory_access_uses_host_page_tables", - "cuda.runtime.device_prop.get_pci_bus_id", - "cuda.runtime.device_prop.get_pci_device_id", - "cuda.runtime.device_prop.get_pci_domain_id", - "cuda.runtime.device_prop.get_persisting_l2cache_max_size", - "cuda.runtime.device_prop.get_regs_per_block", - "cuda.runtime.device_prop.get_regs_per_multiprocessor", - "cuda.runtime.device_prop.get_reserved_shared_mem_per_block", - "cuda.runtime.device_prop.get_shared_mem_per_block", - "cuda.runtime.device_prop.get_shared_mem_per_block_optin", - "cuda.runtime.device_prop.get_shared_mem_per_multiprocessor", - "cuda.runtime.device_prop.get_sparse_cuda_array_supported", - "cuda.runtime.device_prop.get_stream_priorities_supported", - "cuda.runtime.device_prop.get_surface_alignment", - "cuda.runtime.device_prop.get_tcc_driver", - "cuda.runtime.device_prop.get_texture_alignment", - "cuda.runtime.device_prop.get_texture_pitch_alignment", - "cuda.runtime.device_prop.get_timeline_semaphore_interop_supported", - "cuda.runtime.device_prop.get_total_const_mem", - "cuda.runtime.device_prop.get_total_global_mem", - "cuda.runtime.device_prop.get_unified_addressing", - "cuda.runtime.device_prop.get_unified_function_pointers", - "cuda.runtime.device_prop.get_uuid", - "cuda.runtime.device_prop.get_warp_size", - "cuda.runtime.dim_3.get_x", - "cuda.runtime.dim_3.get_y", - "cuda.runtime.dim_3.get_z", - "cuda.runtime.event.create_with_flags", - "cuda.runtime.event.query", - "cuda.runtime.event.record_with_flags", - "cuda.runtime.event.synchronize", - "cuda.runtime.extent.get_depth", - "cuda.runtime.extent.get_height", - "cuda.runtime.extent.get_width", - "cuda.runtime.graph.add_kernel_node", - "cuda.runtime.graph.create", - "cuda.runtime.graph.kernel_node_get_attribute", - "cuda.runtime.graph.kernel_node_set_attribute", - "cuda.runtime.kernel_node_params.get_block_dim", - "cuda.runtime.kernel_node_params.get_extra", - "cuda.runtime.kernel_node_params.get_func", - "cuda.runtime.kernel_node_params.get_grid_dim", - "cuda.runtime.kernel_node_params.get_kernel_params", - "cuda.runtime.kernel_node_params.get_shared_mem_bytes", - "cuda.runtime.library.get_global", - "cuda.runtime.library.get_kernel", - "cuda.runtime.library.get_kernel_count", - "cuda.runtime.library.get_managed", - "cuda.runtime.library.get_unified_function", - "cuda.runtime.library.load_data", - "cuda.runtime.library.load_from_file", - "cuda.runtime.library.unload", - "cuda.runtime.memcpy_3d_params.get_dst_array", - "cuda.runtime.memcpy_3d_params.get_dst_pos", - "cuda.runtime.memcpy_3d_params.get_dst_ptr", - "cuda.runtime.memcpy_3d_params.get_extent", - "cuda.runtime.memcpy_3d_params.get_kind", - "cuda.runtime.memcpy_3d_params.get_src_array", - "cuda.runtime.memcpy_3d_params.get_src_pos", - "cuda.runtime.memcpy_3d_params.get_src_ptr", - "cuda.runtime.memory.cpy3d", - "cuda.runtime.memory.cpy3d_async", - "cuda.runtime.pitched_ptr.get_pitch", - "cuda.runtime.pitched_ptr.get_ptr", - "cuda.runtime.pitched_ptr.get_xsize", - "cuda.runtime.pitched_ptr.get_ysize", - "cuda.runtime.pos.get_x", - "cuda.runtime.pos.get_y", - "cuda.runtime.pos.get_z", - "cuda.runtime.uuid.get_bytes", - "cuda.stream.add_callback", - "cuda.stream.begin_capture", - "cuda.stream.create", - "cuda.stream.destroy", - "cuda.stream.end_capture", - "cuda.stream.synchronize", - "cuda.tensor_map_encode_tiled", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(cuda); - std.testing.refAllDecls(cuda.ops); - inline for (comptime std.meta.declarations(cuda.ops)) |decl| { - const T = @field(cuda.ops, decl); - if (@TypeOf(T) == type) std.testing.refAllDecls(T); - } -} - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - cuda.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -test "every bound operation is registered" { - const ctx = try testContext(); - defer ctx.deinit(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} - -test "cuda types and attributes print as the compiler does" { - @setEvalBranchQuota(100_000); - const ctx = try testContext(); - defer ctx.deinit(); - // The compiler's printed forms, which these must print byte for byte. - inline for (.{ - .{ cuda.ops.AssumeKernelAttr, "#cuda.assume_kernel_attr" }, - .{ cuda.ops.ComputeTargetAttr, "#cuda.compute_target" }, - .{ cuda.ops.DevMaxSharedMemoryOptinAttr, "#cuda.dev_max_shared_memory_optin" }, - .{ cuda.ops.ExecutableAttr, "#cuda.executable" }, - .{ cuda.ops.LaunchConfigType, "!cuda.launch_cfg" }, - .{ cuda.ops.StreamType, "!cuda.stream" }, - .{ cuda.ops.ResultType, "!cuda.result" }, - .{ cuda.ops.IntType, "!cuda.int" }, - .{ cuda.ops.RuntimeDim3Type, "!cuda.runtime.dim_3" }, - .{ cuda.ops.RuntimeLaunchAttributeValueSyncPolicyType, "!cuda.runtime.launch_attribute_value.sync_policy" }, - .{ cuda.ops.RuntimeLaunchAttributeValueClusterSchedulingPolicyPreferenceType, "!cuda.runtime.launch_attribute_value.cluster_scheduling_policy_preference" }, - .{ cuda.ops.RuntimeLaunchAttributeValueNvlinkUtilCentricSchedulingType, "!cuda.runtime.launch_attribute_value.nvlink_util_centric_scheduling" }, - .{ cuda.ops.RuntimeLaunchAttributeValueProgrammaticStreamSerializationAllowedType, "!cuda.runtime.launch_attribute_value.programmatic_stream_serialization_allowed" }, - }) |example| { - const T = example[0]; - const is_type = @hasDecl(T, "type_"); - const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); - const value = parsed.isA(T) orelse return error.TestUnexpectedResult; - try std.testing.expect((try rebuildEnums(T, ctx, value)).eql(value)); - var buf: [512]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(example[1], w.buffered()); - } - // The compiler prints these without the space before the dictionary; it - // parses both. - inline for (.{ - .{ cuda.ops.DeviceAttributesAttr, "#cuda" }, - .{ cuda.ops.FuncAttributesAttr, "#cuda" }, - }) |example| { - const parsed = try mlir.Attribute.parse(ctx, example[1]); - const value = parsed.isA(example[0]) orelse return error.TestUnexpectedResult; - try std.testing.expect((try rebuildEnums(example[0], ctx, value)).eql(value)); - } -} - -/// `rebuild` for parameters that are slices of enums. -fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { - var args: T.InitArgs = undefined; - var buffers: [4][16]u32 = undefined; - inline for (@typeInfo(T.InitArgs).@"struct".field_names, 0..) |field, i| { - const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; - const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; - const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; - const info = @typeInfo(@FieldType(T.InitArgs, field)); - if (info == .pointer and info.pointer.size == .slice and @typeInfo(info.pointer.child) == .@"enum") { - const E = info.pointer.child; - const count = @field(T, "num" ++ name)(value); - const elements: []E = @ptrCast(buffers[i][0..count]); - for (elements, 0..) |*element, pos| element.* = @field(T, noun[0 .. noun.len - 1])(value, pos); - @field(args, field) = elements; - } else { - @field(args, field) = @field(T, accessor)(value); - } - } - return T.get(ctx, args); -} diff --git a/mlir/dialects/cuda_tile/BUILD.bazel b/mlir/dialects/cuda_tile/BUILD.bazel deleted file mode 100644 index 92d622cf..00000000 --- a/mlir/dialects/cuda_tile/BUILD.bazel +++ /dev/null @@ -1,38 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -# The TileView type interface, which NVIDIA's C API does not expose. -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - "@cuda_tile//:CudaTileDialect", - "@llvm-project//mlir:CAPIIR", - ], -) - -zig_library( - name = "cuda_tile", - import_name = "mlir/dialects/cuda_tile", - main = "cuda_tile.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//stdx", - "@cuda_tile//:cuda_tile_capi", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":capi", - ":cuda_tile", - "//mlir", - ], -) diff --git a/mlir/dialects/cuda_tile/capi.cc b/mlir/dialects/cuda_tile/capi.cc deleted file mode 100644 index 879e3757..00000000 --- a/mlir/dialects/cuda_tile/capi.cc +++ /dev/null @@ -1,204 +0,0 @@ -//===- capi.cc - C API for the cuda_tile dialect --------------------------===// - -#include "capi.h" - -#include "cuda_tile/Dialect/CudaTile/IR/Attributes.h" -#include "cuda_tile/Dialect/CudaTile/IR/Dialect.h" -#include "cuda_tile/Dialect/CudaTile/IR/Types.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/Diagnostics.h" - -#include - -using namespace mlir::cuda_tile; - -static mlir::MLIRContext *loadCudaTile(MlirContext context) { - mlir::MLIRContext *ctx = unwrap(context); - if (ctx) - ctx->getOrLoadDialect(); - return ctx; -} - -static std::optional optional(bool has, int64_t value) { - return has ? std::optional(value) : std::nullopt; -} - -//===----------------------------------------------------------------------===// -// TileView -//===----------------------------------------------------------------------===// - -bool mlirTypeIsACudaTileTileView(MlirType type) { - return llvm::isa(unwrap(type)); -} - -MlirType mlirCudaTileTileViewTypeGetViewTileType(MlirType type) { - return wrap(llvm::cast(unwrap(type)).getViewTileType()); -} - -intptr_t mlirCudaTileTileViewTypeGetViewIndexRank(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getViewIndexRank()); -} - -//===----------------------------------------------------------------------===// -// GatherScatterViewType -//===----------------------------------------------------------------------===// - -bool mlirTypeIsACudaTileGatherScatterView(MlirType type) { - return llvm::isa(unwrap(type)); -} - -MlirType mlirCudaTileGatherScatterViewTypeGet( - MlirContext context, intptr_t rank, const int32_t *tileShape, - MlirType tensorView, uint32_t sparseDim, MlirAttribute paddingValue) { - mlir::MLIRContext *ctx = loadCudaTile(context); - if (!ctx) - return {nullptr}; - auto view = llvm::dyn_cast_or_null(unwrap(tensorView)); - if (!view) - return {nullptr}; - PaddingValueAttr padding; - if (!mlirAttributeIsNull(paddingValue)) { - padding = llvm::dyn_cast(unwrap(paddingValue)); - if (!padding) - return {nullptr}; - } - auto shape = mlir::DenseI32ArrayAttr::get( - ctx, llvm::ArrayRef(tileShape, rank)); - return wrap(GatherScatterViewType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, shape, - view, sparseDim, padding)); -} - -MlirAttribute mlirCudaTileGatherScatterViewTypeGetTileShape(MlirType type) { - return wrap(llvm::cast(unwrap(type)).getTileShape()); -} - -MlirType mlirCudaTileGatherScatterViewTypeGetTensorView(MlirType type) { - return wrap(llvm::cast(unwrap(type)).getTensorView()); -} - -uint32_t mlirCudaTileGatherScatterViewTypeGetSparseDim(MlirType type) { - return llvm::cast(unwrap(type)).getSparseDim(); -} - -MlirAttribute mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type) { - return wrap( - llvm::cast(unwrap(type)).getPaddingValue()); -} - -//===----------------------------------------------------------------------===// -// OptimizationHintsAttr -//===----------------------------------------------------------------------===// - -MlirAttribute mlirCudaTileOptimizationHintsAttrGet(MlirContext context, - MlirAttribute value) { - mlir::MLIRContext *ctx = loadCudaTile(context); - if (!ctx) - return {nullptr}; - auto dict = llvm::dyn_cast_or_null(unwrap(value)); - if (!dict) - return {nullptr}; - return wrap(OptimizationHintsAttr::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, dict)); -} - -MlirAttribute mlirCudaTileOptimizationHintsAttrGetValue(MlirAttribute attr) { - return wrap(llvm::cast(unwrap(attr)).getValue()); -} - -//===----------------------------------------------------------------------===// -// DivByAttr -//===----------------------------------------------------------------------===// - -bool mlirAttributeIsACudaTileDivBy(MlirAttribute attr) { - return llvm::isa(unwrap(attr)); -} - -MlirAttribute mlirCudaTileDivByAttrGet(MlirContext context, uint64_t divisor, - bool hasEvery, int64_t every, - bool hasAlong, int64_t along) { - mlir::MLIRContext *ctx = loadCudaTile(context); - // The printer writes `every` and `along` together. - if (!ctx || hasEvery != hasAlong) - return {nullptr}; - return wrap(DivByAttr::get(ctx, divisor, optional(hasEvery, every), - optional(hasAlong, along))); -} - -uint64_t mlirCudaTileDivByAttrGetDivisor(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getDivisor(); -} - -bool mlirCudaTileDivByAttrHasEvery(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getEvery().has_value(); -} - -int64_t mlirCudaTileDivByAttrGetEvery(MlirAttribute attr) { - return *llvm::cast(unwrap(attr)).getEvery(); -} - -bool mlirCudaTileDivByAttrHasAlong(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getAlong().has_value(); -} - -int64_t mlirCudaTileDivByAttrGetAlong(MlirAttribute attr) { - return *llvm::cast(unwrap(attr)).getAlong(); -} - -//===----------------------------------------------------------------------===// -// SameElementsAttr -//===----------------------------------------------------------------------===// - -bool mlirAttributeIsACudaTileSameElements(MlirAttribute attr) { - return llvm::isa(unwrap(attr)); -} - -MlirAttribute mlirCudaTileSameElementsAttrGet(MlirContext context, - intptr_t rank, - const int64_t *values) { - mlir::MLIRContext *ctx = loadCudaTile(context); - if (!ctx) - return {nullptr}; - return wrap(SameElementsAttr::get( - ctx, mlir::DenseI64ArrayAttr::get( - ctx, llvm::ArrayRef(values, rank)))); -} - -MlirAttribute mlirCudaTileSameElementsAttrGetValues(MlirAttribute attr) { - return wrap(llvm::cast(unwrap(attr)).getValues()); -} - -//===----------------------------------------------------------------------===// -// BoundedAttr -//===----------------------------------------------------------------------===// - -bool mlirAttributeIsACudaTileBounded(MlirAttribute attr) { - return llvm::isa(unwrap(attr)); -} - -MlirAttribute mlirCudaTileBoundedAttrGet(MlirContext context, bool hasLb, - int64_t lb, bool hasUb, int64_t ub) { - mlir::MLIRContext *ctx = loadCudaTile(context); - if (!ctx) - return {nullptr}; - return wrap(BoundedAttr::get(ctx, optional(hasLb, lb), optional(hasUb, ub))); -} - -bool mlirCudaTileBoundedAttrHasLb(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getLb().has_value(); -} - -int64_t mlirCudaTileBoundedAttrGetLb(MlirAttribute attr) { - return *llvm::cast(unwrap(attr)).getLb(); -} - -bool mlirCudaTileBoundedAttrHasUb(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getUb().has_value(); -} - -int64_t mlirCudaTileBoundedAttrGetUb(MlirAttribute attr) { - return *llvm::cast(unwrap(attr)).getUb(); -} diff --git a/mlir/dialects/cuda_tile/capi.h b/mlir/dialects/cuda_tile/capi.h deleted file mode 100644 index 6f299c3f..00000000 --- a/mlir/dialects/cuda_tile/capi.h +++ /dev/null @@ -1,84 +0,0 @@ -//===- capi.h - C API for the cuda_tile dialect -------------------*- C -*-===// -// -// C entry points for what NVIDIA's cuda_tile C API leaves out: the -// `TileView` type interface, shared by the partition, strided and -// gather/scatter views, the gather/scatter view type, the general -// `optimization_hints` constructor and the `assume` predicates. - -#ifndef MLIR_DIALECTS_CUDA_TILE_CAPI_H -#define MLIR_DIALECTS_CUDA_TILE_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Constructors return null for invalid parameters; optional attribute -// parameters take null handles, and std::optional integers a `has` flag. -// Getters require a value accepted by the matching IsA. - -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTileTileView(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaTileTileViewTypeGetViewTileType(MlirType type); -MLIR_CAPI_EXPORTED intptr_t -mlirCudaTileTileViewTypeGetViewIndexRank(MlirType type); - -// `!cuda_tile.gather_scatter_view`. `paddingValue` is a -// `#cuda_tile.padding_value` attribute, or null for none. -MLIR_CAPI_EXPORTED bool mlirTypeIsACudaTileGatherScatterView(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCudaTileGatherScatterViewTypeGet( - MlirContext context, intptr_t rank, const int32_t *tileShape, - MlirType tensorView, uint32_t sparseDim, MlirAttribute paddingValue); -// A DenseI32ArrayAttr. -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaTileGatherScatterViewTypeGetTileShape(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCudaTileGatherScatterViewTypeGetTensorView(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCudaTileGatherScatterViewTypeGetSparseDim(MlirType type); -// Null when the view has no padding value. -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaTileGatherScatterViewTypeGetPaddingValue(MlirType type); - -// `#cuda_tile.optimization_hints` over its dictionary of per-architecture -// dictionaries (IsA: NVIDIA's mlirCudaTileAttributeIsAOptimizationHintsAttr). -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaTileOptimizationHintsAttrGet(MlirContext context, MlirAttribute value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaTileOptimizationHintsAttrGetValue(MlirAttribute attr); - -// `#cuda_tile.div_by`. `every` and `along` are both present or both absent. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaTileDivBy(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaTileDivByAttrGet(MlirContext context, uint64_t divisor, bool hasEvery, - int64_t every, bool hasAlong, int64_t along); -MLIR_CAPI_EXPORTED uint64_t mlirCudaTileDivByAttrGetDivisor(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool mlirCudaTileDivByAttrHasEvery(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirCudaTileDivByAttrGetEvery(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool mlirCudaTileDivByAttrHasAlong(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirCudaTileDivByAttrGetAlong(MlirAttribute attr); - -// `#cuda_tile.same_elements`. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACudaTileSameElements(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaTileSameElementsAttrGet( - MlirContext context, intptr_t rank, const int64_t *values); -// A DenseI64ArrayAttr. -MLIR_CAPI_EXPORTED MlirAttribute -mlirCudaTileSameElementsAttrGetValues(MlirAttribute attr); - -// `#cuda_tile.bounded`, each bound optional. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACudaTileBounded(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCudaTileBoundedAttrGet( - MlirContext context, bool hasLb, int64_t lb, bool hasUb, int64_t ub); -MLIR_CAPI_EXPORTED bool mlirCudaTileBoundedAttrHasLb(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirCudaTileBoundedAttrGetLb(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool mlirCudaTileBoundedAttrHasUb(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirCudaTileBoundedAttrGetUb(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_CUDA_TILE_CAPI_H diff --git a/mlir/dialects/cuda_tile/cuda_tile.zig b/mlir/dialects/cuda_tile/cuda_tile.zig deleted file mode 100644 index ddf0a0d5..00000000 --- a/mlir/dialects/cuda_tile/cuda_tile.zig +++ /dev/null @@ -1,1747 +0,0 @@ -//! Zig bindings for NVIDIA's `cuda_tile` MLIR dialect (CUDA Tile IR): one -//! function per op in `Ops.td` order. Types and attributes go through the -//! dialect's C API, completed by `capi.h` where it has no getter. - -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -const dialect_namespace = "cuda_tile"; - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("cuda_tile"); -} - -fn stringRef(s: []const u8) c.MlirStringRef { - return .{ .data = s.ptr, .length = s.len }; -} - -fn typeFromC(t: c.MlirType) *const mlir.Type { - return @ptrCast(t.ptr orelse @panic("cuda_tile: null MlirType")); -} - -fn attrFromC(a: c.MlirAttribute) *const mlir.Attribute { - return @ptrCast(a.ptr orelse @panic("cuda_tile: null MlirAttribute")); -} - -const null_attr: c.MlirAttribute = .{ .ptr = null }; - -/// `!cuda_tile.ptr` — a typed global-memory pointer. -pub const PointerType = opaque { - const M = mlir.Methods(PointerType, c.MlirType); - - pub const isAFn = c.mlirCudaTileTypeIsAPointerType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - pub fn get(ctx: *mlir.Context, pointee_type: *const mlir.Type) *const PointerType { - return @ptrCast(typeFromC(c.mlirCudaTilePointerTypeGet(ctx.ptr(), pointee_type.ptr()))); - } - - pub fn pointee(self: *const PointerType) *const mlir.Type { - return typeFromC(c.mlirCudaTilePointerTypeGetPointeeType(self.ptr())); - } - - pub fn type_(self: *const PointerType) *const mlir.Type { - return @ptrCast(self); - } -}; - -pub fn pointerType(ctx: *mlir.Context, pointee_type: *const mlir.Type) *const mlir.Type { - return @ptrCast(PointerType.get(ctx, pointee_type)); -} - -/// `!cuda_tile.tile` — every SSA value that carries data. Rank 0 is -/// the scalar. Dimensions must be powers of two. -pub const TileType = opaque { - const M = mlir.Methods(TileType, c.MlirType); - - pub const isAFn = c.mlirCudaTileTypeIsATileType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - shape: []const i64, - elem: *const mlir.Type, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const TileType { - return @ptrCast(typeFromC(c.mlirCudaTileTileTypeGet(ctx.ptr(), @intCast(args.shape.len), args.shape.ptr, args.elem.ptr()))); - } - - pub fn elementType(self: *const TileType) *const mlir.Type { - return typeFromC(c.mlirCudaTileTileTypeGetElementType(self.ptr())); - } - - pub fn rank(self: *const TileType) usize { - return @intCast(c.mlirCudaTileTileTypeGetRank(self.ptr())); - } - - pub fn dimension(self: *const TileType, i: usize) i64 { - return c.mlirCudaTileTileTypeGetDimSize(self.ptr(), @intCast(i)); - } - - pub fn type_(self: *const TileType) *const mlir.Type { - return @ptrCast(self); - } -}; - -pub fn tileType(ctx: *mlir.Context, shape: []const i64, elem: *const mlir.Type) *const mlir.Type { - return @ptrCast(TileType.get(ctx, .{ .shape = shape, .elem = elem })); -} - -/// `!cuda_tile.token` — the ordering token every `*_tko` op produces. -pub const TokenType = opaque { - const M = mlir.Methods(TokenType, c.MlirType); - - pub const isAFn = c.mlirCudaTileTypeIsATokenType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - pub fn get(ctx: *mlir.Context) *const TokenType { - return @ptrCast(typeFromC(c.mlirCudaTileTokenTypeGet(ctx.ptr()))); - } - - pub fn type_(self: *const TokenType) *const mlir.Type { - return @ptrCast(self); - } -}; - -pub fn tokenType(ctx: *mlir.Context) *const mlir.Type { - return @ptrCast(TokenType.get(ctx)); -} - -/// `!cuda_tile.tensor_view`. `dynamic()` marks a -/// dimension or stride that arrives as an operand of `make_tensor_view`. -pub const TensorViewType = opaque { - const M = mlir.Methods(TensorViewType, c.MlirType); - - pub const isAFn = c.mlirCudaTileTypeIsATensorViewType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - pub fn dynamic() i64 { - return c.mlirCudaTileTensorViewTypeGetDynamicSize(); - } - - pub const InitArgs = struct { - elem: *const mlir.Type, - shape: []const i64, - strides: []const i64, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const TensorViewType { - const shape = args.shape; - const strides = args.strides; - return @ptrCast(typeFromC(c.mlirCudaTileTensorViewTypeGet( - ctx.ptr(), - args.elem.ptr(), - @intCast(shape.len), - shape.ptr, - @intCast(strides.len), - strides.ptr, - ))); - } - - pub fn elementType(self: *const TensorViewType) *const mlir.Type { - return typeFromC(c.mlirCudaTileTensorViewTypeGetElementType(self.ptr())); - } - - pub fn rank(self: *const TensorViewType) usize { - return @intCast(c.mlirCudaTileTensorViewTypeGetRank(self.ptr())); - } - - pub fn dimension(self: *const TensorViewType, i: usize) i64 { - return c.mlirCudaTileTensorViewTypeGetDimSize(self.ptr(), @intCast(i)); - } - - pub fn stride(self: *const TensorViewType, i: usize) i64 { - return c.mlirCudaTileTensorViewTypeGetStride(self.ptr(), @intCast(i)); - } - - pub fn type_(self: *const TensorViewType) *const mlir.Type { - return @ptrCast(self); - } -}; - -pub fn tensorViewType(ctx: *mlir.Context, elem: *const mlir.Type, shape: []const i64, strides: []const i64) *const mlir.Type { - return @ptrCast(TensorViewType.get(ctx, .{ .elem = elem, .shape = shape, .strides = strides })); -} - -const DimMap = stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK); - -fn identityDimMap(dim_map: []const i32, rank: usize) DimMap { - var out: DimMap = .empty; - if (dim_map.len > 0) { - out.appendSliceAssumeCapacity(dim_map); - } else { - for (0..rank) |i| out.appendAssumeCapacity(@intCast(i)); - } - return out; -} - -/// `!cuda_tile.partition_view, dim_map=[...]>`. -/// The tile shape, padding and dim_map all live in the type; the op that -/// builds one has no attributes of its own. -pub const PartitionViewType = opaque { - const M = mlir.Methods(PartitionViewType, c.MlirType); - - pub const isAFn = c.mlirCudaTileTypeIsAPartitionViewType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - /// An empty `dim_map` is the identity; the type requires one entry per - /// tile dimension. - pub const InitArgs = struct { - tile_shape: []const i32, - tensor_view: *const mlir.Type, - dim_map: []const i32 = &.{}, - padding: ?PaddingValue = null, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const PartitionViewType { - const tile_shape = args.tile_shape; - const tensor_view = args.tensor_view; - const dim_map = args.dim_map; - const tile_attr: *const mlir.Attribute = .denseArray(ctx, .i32, tile_shape); - const map = identityDimMap(dim_map, tile_shape.len); - return @ptrCast(typeFromC(c.mlirCudaTilePartitionViewTypeGet( - ctx.ptr(), - tile_attr.ptr(), - tensor_view.ptr(), - @intCast(map.len), - map.constSlice().ptr, - if (args.padding) |p| p.attribute(ctx).ptr() else null_attr, - ))); - } - - pub fn tensorView(self: *const PartitionViewType) *const mlir.Type { - return typeFromC(c.mlirCudaTilePartitionViewTypeGetTensorView(self.ptr())); - } - - pub fn viewTileType(self: *const PartitionViewType) *const mlir.Type { - return typeFromC(c.mlirCudaTilePartitionViewTypeGetViewTileType(self.ptr())); - } - - pub fn viewIndexRank(self: *const PartitionViewType) usize { - return @intCast(c.mlirCudaTilePartitionViewTypeGetViewIndexRank(self.ptr())); - } - - pub fn type_(self: *const PartitionViewType) *const mlir.Type { - return @ptrCast(self); - } -}; - -/// `!cuda_tile.strided_view` (13.3). -pub const StridedViewType = opaque { - const M = mlir.Methods(StridedViewType, c.MlirType); - - pub const isAFn = c.mlirCudaTileTypeIsAStridedViewType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - tile_shape: []const i32, - traversal_strides: []const i32, - tensor_view: *const mlir.Type, - dim_map: []const i32 = &.{}, - padding: ?PaddingValue = null, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const StridedViewType { - const tile_shape = args.tile_shape; - const traversal_strides = args.traversal_strides; - const tensor_view = args.tensor_view; - const dim_map = args.dim_map; - const tile_attr: *const mlir.Attribute = .denseArray(ctx, .i32, tile_shape); - const strides_attr: *const mlir.Attribute = .denseArray(ctx, .i32, traversal_strides); - const map = identityDimMap(dim_map, tile_shape.len); - return @ptrCast(typeFromC(c.mlirCudaTileStridedViewTypeGet( - ctx.ptr(), - tile_attr.ptr(), - strides_attr.ptr(), - tensor_view.ptr(), - @intCast(map.len), - map.constSlice().ptr, - if (args.padding) |p| p.attribute(ctx).ptr() else null_attr, - ))); - } - - pub fn viewTileType(self: *const StridedViewType) *const mlir.Type { - return typeFromC(c.mlirCudaTileStridedViewTypeGetViewTileType(self.ptr())); - } - - pub fn viewIndexRank(self: *const StridedViewType) usize { - return @intCast(c.mlirCudaTileStridedViewTypeGetViewIndexRank(self.ptr())); - } - - pub fn type_(self: *const StridedViewType) *const mlir.Type { - return @ptrCast(self); - } -}; - -/// `!cuda_tile.gather_scatter_view, sparse_dim=N>` -/// (13.3): gathers or scatters along `sparseDim`, one tile per index. -pub const GatherScatterViewType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACudaTileGatherScatterView; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "gather_scatter_view"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - tileShape: []const i32, - tensorView: *const TensorViewType, - sparseDim: u32, - paddingValue: ?PaddingValue = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaTileGatherScatterViewTypeGet( - ctx.ptr(), - @intCast(args.tileShape.len), - args.tileShape.ptr, - args.tensorView.ptr(), - args.sparseDim, - if (args.paddingValue) |p| p.attribute(ctx).ptr() else null_attr, - ); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn tileShape(self: *const Self) *const mlir.DenseArrayAttribute(.i32) { - return @ptrCast(attrFromC(c.mlirCudaTileGatherScatterViewTypeGetTileShape(self.ptr()))); - } - pub fn tensorView(self: *const Self) *const TensorViewType { - return @ptrCast(typeFromC(c.mlirCudaTileGatherScatterViewTypeGetTensorView(self.ptr()))); - } - pub fn sparseDim(self: *const Self) u32 { - return c.mlirCudaTileGatherScatterViewTypeGetSparseDim(self.ptr()); - } - pub fn paddingValue(self: *const Self) ?PaddingValue { - const attr = c.mlirCudaTileGatherScatterViewTypeGetPaddingValue(self.ptr()); - if (attr.ptr == null) return null; - const name = c.mlirCudaTilePaddingValueAttrGetValue(attr); - return std.meta.stringToEnum(PaddingValue, name.data[0..name.length]).?; - } -}; - -pub fn gatherScatterViewType( - ctx: *mlir.Context, - tile_shape: []const i64, - tensor_view: *const mlir.Type, - sparse_dim: u32, - opts: struct { - padding: ?PaddingValue = null, - }, -) *const mlir.Type { - var shape: stdx.BoundedArray(i32, mlir.ShapedType.MAX_RANK) = .empty; - for (tile_shape) |d| shape.appendAssumeCapacity(@intCast(d)); - const view = GatherScatterViewType.get(ctx, .{ - .tileShape = shape.constSlice(), - .tensorView = tensor_view.isA(TensorViewType) orelse std.debug.panic("cuda_tile: {f} is not a tensor_view", .{tensor_view}), - .sparseDim = sparse_dim, - .paddingValue = opts.padding, - }) catch std.debug.panic("cuda_tile: invalid gather_scatter_view over {f}", .{tensor_view}); - return view.type_(); -} - -/// The `tile<...>` type a load through `view` yields, for any view kind — -/// through the dialect's `TileView` type interface (`capi.cc`), so -/// the gather/scatter view, which has no getters in NVIDIA's C API, works too. -pub fn tileTypeOfView(view: *const mlir.Type) *const mlir.Type { - if (!c.mlirTypeIsACudaTileTileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); - return typeFromC(c.mlirCudaTileTileViewTypeGetViewTileType(view.ptr())); -} - -pub fn indexRankOfView(view: *const mlir.Type) usize { - if (!c.mlirTypeIsACudaTileTileView(view.ptr())) std.debug.panic("cuda_tile: {f} is not a tiled view", .{view}); - return @intCast(c.mlirCudaTileTileViewTypeGetViewIndexRank(view.ptr())); -} - -// Enum attributes: built through the C getter from the tag name, so a typo is -// a null attribute rather than a wrong case. - -fn enumAttribute(comptime getter: anytype, ctx: *mlir.Context, name: []const u8) *const mlir.Attribute { - return attrFromC(getter(ctx.ptr(), stringRef(name))); -} - -pub const RoundingMode = enum { - nearest_even, - zero, - negative_inf, - positive_inf, - approx, - full, - nearest_int_to_zero, - - pub fn attribute(self: RoundingMode, ctx: *mlir.Context) *const mlir.Attribute { - return enumAttribute(c.mlirCudaTileRoundingModeAttrGet, ctx, @tagName(self)); - } -}; - -pub const Signedness = enum { - signed, - unsigned, - - pub fn attribute(self: Signedness, ctx: *mlir.Context) *const mlir.Attribute { - return enumAttribute(c.mlirCudaTileSignednessAttrGet, ctx, @tagName(self)); - } -}; - -pub const IntegerOverflow = enum { - none, - no_signed_wrap, - no_unsigned_wrap, - no_wrap, - - pub fn attribute(self: IntegerOverflow, ctx: *mlir.Context) *const mlir.Attribute { - return enumAttribute(c.mlirCudaTileIntegerOverflowAttrGet, ctx, @tagName(self)); - } -}; - -pub const ComparisonOrdering = enum { - unordered, - ordered, - - pub fn attribute(self: ComparisonOrdering, ctx: *mlir.Context) *const mlir.Attribute { - return enumAttribute(c.mlirCudaTileComparisonOrderingAttrGet, ctx, @tagName(self)); - } -}; - -pub const ComparisonPredicate = enum { - equal, - not_equal, - less_than, - less_than_or_equal, - greater_than, - greater_than_or_equal, - - pub fn attribute(self: ComparisonPredicate, ctx: *mlir.Context) *const mlir.Attribute { - return enumAttribute(c.mlirCudaTileComparisonPredicateAttrGet, ctx, @tagName(self)); - } -}; - -pub const AtomicRMWMode = enum { - @"and", - @"or", - xor, - add, - addf, - max, - min, - umax, - umin, - xchg, - - pub fn attribute(self: AtomicRMWMode, ctx: *mlir.Context) *const mlir.Attribute { - return enumAttribute(c.mlirCudaTileAtomicRMWModeAttrGet, ctx, @tagName(self)); - } -}; - -pub const MemoryScope = enum { - tl_blk, - device, - sys, - - pub fn attribute(self: MemoryScope, ctx: *mlir.Context) *const mlir.Attribute { - return enumAttribute(c.mlirCudaTileMemoryScopeAttrGet, ctx, @tagName(self)); - } -}; - -pub const MemoryOrdering = enum { - weak, - relaxed, - acquire, - release, - acq_rel, - - pub fn attribute(self: MemoryOrdering, ctx: *mlir.Context) *const mlir.Attribute { - return enumAttribute(c.mlirCudaTileMemoryOrderingSemanticsAttrGet, ctx, @tagName(self)); - } -}; - -pub const PaddingValue = enum { - zero, - neg_zero, - nan, - pos_inf, - neg_inf, - - pub fn attribute(self: PaddingValue, ctx: *mlir.Context) *const mlir.Attribute { - return enumAttribute(c.mlirCudaTilePaddingValueAttrGet, ctx, @tagName(self)); - } -}; - -pub const SymbolVisibility = enum { - public, - private, - - pub fn attribute(self: SymbolVisibility, ctx: *mlir.Context) *const mlir.Attribute { - return enumAttribute(c.mlirCudaTileSymbolVisibilityAttrGet, ctx, @tagName(self)); - } -}; - -// optimization_hints: one dictionary of hints per architecture. - -pub const Arch = enum { default, sm_80, sm_86, sm_87, sm_88, sm_89, sm_90, sm_100, sm_103, sm_110, sm_120, sm_121 }; - -/// Hints `entry` accepts: `num_cta_in_cga` (power of two <= 16), -/// `num_worker_warps_per_cta` (4 or 8), `occupancy`. -pub const EntryHint = struct { - arch: Arch = .default, - num_cta_in_cga: ?i32 = null, - num_worker_warps_per_cta: ?i32 = null, - occupancy: ?i32 = null, -}; - -/// Hints `load_view_tko` / `store_view_tko` accept. -pub const LoadStoreHint = struct { - arch: Arch = .default, - allow_tma: ?bool = null, - latency: ?i32 = null, -}; - -/// Hints `load_ptr_tko` / `store_ptr_tko` accept: `latency` only, the -/// pointer arm cannot use TMA. (The dialect drops unsupported keys silently.) -pub const PtrLoadStoreHint = struct { - arch: Arch = .default, - latency: ?i32 = null, -}; - -/// `#cuda_tile.optimization_hints`: a -/// dictionary from architecture name to a dictionary of hints. -pub const OptimizationHintsAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirCudaTileAttributeIsAOptimizationHintsAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "optimization_hints"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: *const mlir.DictionaryAttribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaTileOptimizationHintsAttrGet(ctx.ptr(), args.value.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) *const mlir.DictionaryAttribute { - return @ptrCast(attrFromC(c.mlirCudaTileOptimizationHintsAttrGetValue(self.ptr()))); - } -}; - -/// The hint dictionary of one architecture: the non-null fields of `hint` -/// but `arch`, integers as `i64` like the textual form. -fn hintDictionary(ctx: *mlir.Context, hint: anytype) *const mlir.Attribute { - const field_names = @typeInfo(@TypeOf(hint)).@"struct".field_names; - var entries: stdx.BoundedArray(mlir.NamedAttribute, field_names.len) = .empty; - inline for (field_names) |field_name| { - if (comptime std.mem.eql(u8, field_name, "arch")) continue; - if (@field(hint, field_name)) |v| { - const value: *const mlir.Attribute = switch (@TypeOf(v)) { - bool => .boolean(ctx, v), - else => .int(ctx, .i64, v), - }; - entries.appendAssumeCapacity(.named(ctx, field_name, value)); - } - } - return .dict(ctx, entries.constSlice()); -} - -/// `#cuda_tile.optimization_hints` from a slice of `EntryHint`, -/// `LoadStoreHint` or `PtrLoadStoreHint`. -pub fn optimizationHintsAttr(ctx: *mlir.Context, hints: anytype) *const mlir.Attribute { - var entries: stdx.BoundedArray(mlir.NamedAttribute, std.meta.fieldNames(Arch).len) = .empty; - for (hints) |h| entries.appendAssumeCapacity(.named(ctx, @tagName(h.arch), hintDictionary(ctx, h))); - const value: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, entries.constSlice())); - const attr = OptimizationHintsAttr.get(ctx, .{ .value = value }) catch - std.debug.panic("cuda_tile: invalid optimization_hints {f}", .{value}); - return attr.attribute(); -} - -// assume predicates. - -/// `#cuda_tile.div_by`, or `` for a -/// per-group claim; `every` and `along` are both set or both null. -pub const DivByAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACudaTileDivBy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "div_by"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - divisor: u64, - every: ?i64 = null, - along: ?i64 = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaTileDivByAttrGet( - ctx.ptr(), - args.divisor, - args.every != null, - args.every orelse 0, - args.along != null, - args.along orelse 0, - ); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn divisor(self: *const Self) u64 { - return c.mlirCudaTileDivByAttrGetDivisor(self.ptr()); - } - pub fn every(self: *const Self) ?i64 { - return if (c.mlirCudaTileDivByAttrHasEvery(self.ptr())) c.mlirCudaTileDivByAttrGetEvery(self.ptr()) else null; - } - pub fn along(self: *const Self) ?i64 { - return if (c.mlirCudaTileDivByAttrHasAlong(self.ptr())) c.mlirCudaTileDivByAttrGetAlong(self.ptr()) else null; - } -}; - -/// `along` defaults to 0 when only `every` is given. -pub fn divByAttr( - ctx: *mlir.Context, - divisor: u64, - opts: struct { - every: ?i64 = null, - along: ?i64 = null, - }, -) *const mlir.Attribute { - const attr = DivByAttr.get(ctx, .{ - .divisor = divisor, - .every = opts.every, - .along = if (opts.every != null) opts.along orelse 0 else null, - }) catch unreachable; - return attr.attribute(); -} - -/// `#cuda_tile.same_elements<[...]>`: per dimension, the size of the groups -/// of equal elements. -pub const SameElementsAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACudaTileSameElements; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "same_elements"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - values: []const i64, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaTileSameElementsAttrGet(ctx.ptr(), @intCast(args.values.len), args.values.ptr); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn values(self: *const Self) *const mlir.DenseArrayAttribute(.i64) { - return @ptrCast(attrFromC(c.mlirCudaTileSameElementsAttrGetValues(self.ptr()))); - } -}; - -pub fn sameElementsAttr(ctx: *mlir.Context, values: []const i64) *const mlir.Attribute { - const attr = SameElementsAttr.get(ctx, .{ .values = values }) catch unreachable; - return attr.attribute(); -} - -/// `#cuda_tile.bounded`, either side `?` when null. -pub const BoundedAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACudaTileBounded; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "bounded"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - lb: ?i64 = null, - ub: ?i64 = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCudaTileBoundedAttrGet(ctx.ptr(), args.lb != null, args.lb orelse 0, args.ub != null, args.ub orelse 0); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn lb(self: *const Self) ?i64 { - return if (c.mlirCudaTileBoundedAttrHasLb(self.ptr())) c.mlirCudaTileBoundedAttrGetLb(self.ptr()) else null; - } - pub fn ub(self: *const Self) ?i64 { - return if (c.mlirCudaTileBoundedAttrHasUb(self.ptr())) c.mlirCudaTileBoundedAttrGetUb(self.ptr()) else null; - } -}; - -pub fn boundedAttr( - ctx: *mlir.Context, - opts: struct { - lb: ?i64 = null, - ub: ?i64 = null, - }, -) *const mlir.Attribute { - const attr = BoundedAttr.get(ctx, .{ .lb = opts.lb, .ub = opts.ub }) catch unreachable; - return attr.attribute(); -} - -// Constant values: DenseElementsAttr over a tile type, which is a ShapedType. - -pub fn splatAttr(tile_ty: *const mlir.Type, elem_attr: *const mlir.Attribute) *const mlir.Attribute { - return attrFromC(c.mlirDenseElementsAttrSplatGet(tile_ty.ptr(), elem_attr.ptr())); -} - -pub fn intElemAttr(elem_ty: *const mlir.Type, value: i64) *const mlir.Attribute { - return attrFromC(c.mlirIntegerAttrGet(elem_ty.ptr(), value)); -} - -pub fn floatElemAttr(ctx: *mlir.Context, elem_ty: *const mlir.Type, value: f64) *const mlir.Attribute { - return attrFromC(c.mlirFloatAttrDoubleGet(ctx.ptr(), elem_ty.ptr(), value)); -} - -/// A dense `DenseElementsAttr` over `tile_ty` from a raw element buffer; the -/// element type's bit width must be a multiple of 8. -pub fn denseAttr(tile_ty: *const mlir.Type, values: anytype) *const mlir.Attribute { - return .denseElements(@ptrCast(tile_ty), values); -} - -const MakeArgs = mlir.Operation.MakeArgs; - -fn opName(comptime mnemonic: []const u8) []const u8 { - return dialect_namespace ++ "." ++ mnemonic; -} - -/// Attribute lists are passed by value so a helper's temporary outlives the -/// slice handed to `Operation.make`. -const Attrs = stdx.BoundedArray(mlir.NamedAttribute, 8); - -fn attrsOf(list: []const mlir.NamedAttribute) Attrs { - var attrs: Attrs = .empty; - attrs.appendSliceAssumeCapacity(list); - return attrs; -} - -fn unary(ctx: *mlir.Context, comptime mnemonic: []const u8, source: *const mlir.Value, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName(mnemonic), .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{source.type_()} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -fn binary(ctx: *mlir.Context, comptime mnemonic: []const u8, lhs: *const mlir.Value, rhs: *const mlir.Value, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName(mnemonic), .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{lhs.type_()} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -fn convert(ctx: *mlir.Context, comptime mnemonic: []const u8, source: *const mlir.Value, result_type: *const mlir.Type, attrs: Attrs, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName(mnemonic), .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -fn roundingAttrs(ctx: *mlir.Context, rounding: RoundingMode, flush_to_zero: bool) Attrs { - var attrs: Attrs = .empty; - attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", rounding.attribute(ctx))); - if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); - return attrs; -} - -fn overflowAttrs(ctx: *mlir.Context, overflow: IntegerOverflow) Attrs { - var attrs: Attrs = .empty; - if (overflow != .none) attrs.appendAssumeCapacity(.named(ctx, "overflow", overflow.attribute(ctx))); - return attrs; -} - -fn ftzAttrs(ctx: *mlir.Context, flush_to_zero: bool) Attrs { - var attrs: Attrs = .empty; - if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); - return attrs; -} - -fn i1Like(ctx: *mlir.Context, v: *const mlir.Value) *const mlir.Type { - const t = v.type_().isA(TileType) orelse std.debug.panic("cuda_tile: {f} is not a tile", .{v.type_()}); - var shape: stdx.BoundedArray(i64, mlir.ShapedType.MAX_RANK) = .empty; - for (0..t.rank()) |i| shape.appendAssumeCapacity(t.dimension(i)); - return tileType(ctx, shape.constSlice(), .int(ctx, .i1)); -} - -pub fn absf(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "absf", source, .empty, location); -} - -pub fn addf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "addf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); -} - -pub fn subf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "subf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); -} - -pub fn mulf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "mulf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); -} - -pub fn divf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "divf", lhs, rhs, roundingAttrs(ctx, rounding, flush_to_zero), location); -} - -pub fn remf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "remf", lhs, rhs, .empty, location); -} - -pub fn negf(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "negf", source, .empty, location); -} - -pub fn fma(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("fma"), .{ - .operands = .{ .flat = &.{ lhs, rhs, acc } }, - .results = .{ .flat = &.{acc.type_()} }, - .attributes = roundingAttrs(ctx, rounding, flush_to_zero).constSlice(), - .location = location, - }); -} - -fn minMaxAttrs(ctx: *mlir.Context, propagate_nan: bool, flush_to_zero: bool) Attrs { - var attrs: Attrs = .empty; - if (propagate_nan) attrs.appendAssumeCapacity(.named(ctx, "propagate_nan", .unit(ctx))); - if (flush_to_zero) attrs.appendAssumeCapacity(.named(ctx, "flush_to_zero", .unit(ctx))); - return attrs; -} - -pub fn maxf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, propagate_nan: bool, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "maxf", lhs, rhs, minMaxAttrs(ctx, propagate_nan, flush_to_zero), location); -} - -pub fn minf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, propagate_nan: bool, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "minf", lhs, rhs, minMaxAttrs(ctx, propagate_nan, flush_to_zero), location); -} - -pub fn pow(ctx: *mlir.Context, source: *const mlir.Value, exponent: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "fpowf", source, exponent, .empty, location); -} - -pub fn atan2(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "atan2", x, y, .empty, location); -} - -/// `rounding` null keeps the op's default (13.3 attribute). -pub fn exp( - ctx: *mlir.Context, - source: *const mlir.Value, - opts: struct { - rounding: ?RoundingMode = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: Attrs = .empty; - if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); - return unary(ctx, "exp", source, attrs, location); -} - -pub fn exp2(ctx: *mlir.Context, source: *const mlir.Value, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "exp2", source, ftzAttrs(ctx, flush_to_zero), location); -} - -pub fn log(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "log", source, .empty, location); -} - -pub fn log2(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "log2", source, .empty, location); -} - -pub fn sqrt(ctx: *mlir.Context, source: *const mlir.Value, rounding: RoundingMode, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "sqrt", source, roundingAttrs(ctx, rounding, flush_to_zero), location); -} - -pub fn rsqrt(ctx: *mlir.Context, source: *const mlir.Value, flush_to_zero: bool, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "rsqrt", source, ftzAttrs(ctx, flush_to_zero), location); -} - -pub fn sin(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "sin", source, .empty, location); -} - -pub fn cos(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "cos", source, .empty, location); -} - -pub fn tan(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "tan", source, .empty, location); -} - -pub fn sinh(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "sinh", source, .empty, location); -} - -pub fn cosh(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "cosh", source, .empty, location); -} - -/// `rounding` null keeps the op's default (13.2 attribute). Only `approx` -/// and `full` are legal; the upstream verifier reports the others without -/// failing, so they are refused here. -pub fn tanh( - ctx: *mlir.Context, - source: *const mlir.Value, - opts: struct { - rounding: ?RoundingMode = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: Attrs = .empty; - if (opts.rounding) |r| { - switch (r) { - .approx, .full => {}, - else => std.debug.panic("cuda_tile.tanh: rounding mode {s} is not allowed (only approx or full)", .{@tagName(r)}), - } - attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); - } - return unary(ctx, "tanh", source, attrs, location); -} - -pub fn ceil(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "ceil", source, .empty, location); -} - -pub fn floor(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "floor", source, .empty, location); -} - -pub fn cmpf(ctx: *mlir.Context, predicate: ComparisonPredicate, ordering: ComparisonOrdering, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("cmpf"), .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{i1Like(ctx, lhs)} }, - .attributes = &.{ - .named(ctx, "comparison_predicate", predicate.attribute(ctx)), - .named(ctx, "comparison_ordering", ordering.attribute(ctx)), - }, - .location = location, - }); -} - -pub fn mmaf(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, fast_acc: bool, location: *const mlir.Location) *mlir.Operation { - var attrs: Attrs = .empty; - if (fast_acc) attrs.appendAssumeCapacity(.named(ctx, "fast_acc", .unit(ctx))); - return mlir.Operation.make(ctx, opName("mmaf"), .{ - .operands = .{ .flat = &.{ lhs, rhs, acc } }, - .results = .{ .flat = &.{acc.type_()} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// Block-scaled MMA (13.3): scales are plain tiles in logical M/N/K layout. -pub fn mmaf_scaled(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, lhs_scale: *const mlir.Value, rhs_scale: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("mmaf_scaled"), .{ - .operands = .{ .flat = &.{ lhs, rhs, acc, lhs_scale, rhs_scale } }, - .results = .{ .flat = &.{acc.type_()} }, - .location = location, - }); -} - -pub fn absi(ctx: *mlir.Context, source: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "absi", source, .empty, location); -} - -pub fn addi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "addi", lhs, rhs, overflowAttrs(ctx, overflow), location); -} - -pub fn subi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "subi", lhs, rhs, overflowAttrs(ctx, overflow), location); -} - -pub fn muli(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "muli", lhs, rhs, overflowAttrs(ctx, overflow), location); -} - -pub fn mulhii(ctx: *mlir.Context, x: *const mlir.Value, y: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "mulhii", x, y, .empty, location); -} - -/// `rounding` null keeps the op's default (truncation). -pub fn divi( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - signedness: Signedness, - opts: struct { - rounding: ?RoundingMode = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: Attrs = .empty; - attrs.appendAssumeCapacity(.named(ctx, "signedness", signedness.attribute(ctx))); - if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding", r.attribute(ctx))); - return binary(ctx, "divi", lhs, rhs, attrs, location); -} - -fn signedBinary(ctx: *mlir.Context, comptime mnemonic: []const u8, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, mnemonic, lhs, rhs, attrsOf(&.{.named(ctx, "signedness", signedness.attribute(ctx))}), location); -} - -pub fn remi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { - return signedBinary(ctx, "remi", lhs, rhs, signedness, location); -} - -pub fn maxi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { - return signedBinary(ctx, "maxi", lhs, rhs, signedness, location); -} - -pub fn mini(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { - return signedBinary(ctx, "mini", lhs, rhs, signedness, location); -} - -pub fn negi(ctx: *mlir.Context, source: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { - return unary(ctx, "negi", source, overflowAttrs(ctx, overflow), location); -} - -pub fn shli(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "shli", lhs, rhs, overflowAttrs(ctx, overflow), location); -} - -pub fn shri(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, signedness: Signedness, location: *const mlir.Location) *mlir.Operation { - return signedBinary(ctx, "shri", lhs, rhs, signedness, location); -} - -pub fn cmpi(ctx: *mlir.Context, predicate: ComparisonPredicate, signedness: Signedness, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("cmpi"), .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{i1Like(ctx, lhs)} }, - .attributes = &.{ - .named(ctx, "comparison_predicate", predicate.attribute(ctx)), - .named(ctx, "signedness", signedness.attribute(ctx)), - }, - .location = location, - }); -} - -pub fn mmai(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, acc: *const mlir.Value, signedness_lhs: Signedness, signedness_rhs: Signedness, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("mmai"), .{ - .operands = .{ .flat = &.{ lhs, rhs, acc } }, - .results = .{ .flat = &.{acc.type_()} }, - .attributes = &.{ - .named(ctx, "signedness_lhs", signedness_lhs.attribute(ctx)), - .named(ctx, "signedness_rhs", signedness_rhs.attribute(ctx)), - }, - .location = location, - }); -} - -pub fn andi(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "andi", lhs, rhs, .empty, location); -} - -pub fn ori(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "ori", lhs, rhs, .empty, location); -} - -pub fn xori(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "xori", lhs, rhs, .empty, location); -} - -pub fn bitcast(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "bitcast", source, result_type, .empty, location); -} - -pub fn exti(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "exti", from, to, attrsOf(&.{.named(ctx, "signedness", signedness.attribute(ctx))}), location); -} - -pub fn trunci(ctx: *mlir.Context, from: *const mlir.Value, to: *const mlir.Type, overflow: IntegerOverflow, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "trunci", from, to, overflowAttrs(ctx, overflow), location); -} - -/// `rounding` null keeps the op's default (nearest_even). -pub fn ftof( - ctx: *mlir.Context, - from: *const mlir.Value, - to: *const mlir.Type, - opts: struct { - rounding: ?RoundingMode = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: Attrs = .empty; - if (opts.rounding) |r| attrs.appendAssumeCapacity(.named(ctx, "rounding_mode", r.attribute(ctx))); - return convert(ctx, "ftof", from, to, attrs, location); -} - -pub fn ftoi(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, rounding: RoundingMode, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "ftoi", from, to, attrsOf(&.{ - .named(ctx, "signedness", signedness.attribute(ctx)), - .named(ctx, "rounding_mode", rounding.attribute(ctx)), - }), location); -} - -pub fn itof(ctx: *mlir.Context, from: *const mlir.Value, signedness: Signedness, rounding: RoundingMode, to: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "itof", from, to, attrsOf(&.{ - .named(ctx, "signedness", signedness.attribute(ctx)), - .named(ctx, "rounding_mode", rounding.attribute(ctx)), - }), location); -} - -pub fn int_to_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "int_to_ptr", source, result_type, .empty, location); -} - -pub fn ptr_to_int(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "ptr_to_int", source, result_type, .empty, location); -} - -pub fn ptr_to_ptr(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "ptr_to_ptr", source, result_type, .empty, location); -} - -/// 13.3: sub-byte tile -> i8 tile. -pub fn pack(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "pack", source, result_type, .empty, location); -} - -/// 13.3: i8 tile -> sub-byte tile. -pub fn unpack(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "unpack", source, result_type, .empty, location); -} - -pub fn constant(ctx: *mlir.Context, value: *const mlir.Attribute, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("constant"), .{ - .results = .{ .flat = &.{result_type} }, - .attributes = &.{.named(ctx, "value", value)}, - .location = location, - }); -} - -pub fn iota(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("iota"), .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn reshape(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "reshape", source, result_type, .empty, location); -} - -pub fn broadcast(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "broadcast", source, result_type, .empty, location); -} - -pub fn cat(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, dim: i64, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("cat"), .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{.named(ctx, "dim", .int(ctx, .i64, dim))}, - .location = location, - }); -} - -pub fn extract(ctx: *mlir.Context, source: *const mlir.Value, indices: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, mlir.ShapedType.MAX_RANK + 1) = .empty; - operands.appendAssumeCapacity(source); - operands.appendSliceAssumeCapacity(indices); - return mlir.Operation.make(ctx, opName("extract"), .{ - .operands = .{ .flat = operands.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn permute(ctx: *mlir.Context, source: *const mlir.Value, permutation: []const i32, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, "permute", source, result_type, attrsOf(&.{.named(ctx, "permutation", .denseArray(ctx, .i32, permutation))}), location); -} - -pub fn select(ctx: *mlir.Context, cond: *const mlir.Value, val_if_true: *const mlir.Value, val_if_false: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("select"), .{ - .operands = .{ .flat = &.{ cond, val_if_true, val_if_false } }, - .results = .{ .flat = &.{val_if_true.type_()} }, - .location = location, - }); -} - -pub fn offset(ctx: *mlir.Context, ptr: *const mlir.Value, off: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return binary(ctx, "offset", ptr, off, .empty, location); -} - -fn threeI32(ctx: *mlir.Context, comptime mnemonic: []const u8, location: *const mlir.Location) *mlir.Operation { - const i32_tile = tileType(ctx, &.{}, .int(ctx, .i32)); - return mlir.Operation.make(ctx, opName(mnemonic), .{ - .results = .{ .flat = &.{ i32_tile, i32_tile, i32_tile } }, - .location = location, - }); -} - -pub fn get_tile_block_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return threeI32(ctx, "get_tile_block_id", location); -} - -pub fn get_num_tile_blocks(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return threeI32(ctx, "get_num_tile_blocks", location); -} - -pub const GlobalOpts = struct { - alignment: ?i64 = null, - constant: bool = false, - visibility: ?SymbolVisibility = null, -}; - -pub fn global(ctx: *mlir.Context, name: []const u8, value: *const mlir.Attribute, opts: GlobalOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: Attrs = .empty; - attrs.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, name))); - attrs.appendAssumeCapacity(.named(ctx, "value", value)); - if (opts.alignment) |a| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); - if (opts.constant) attrs.appendAssumeCapacity(.named(ctx, "constant", .unit(ctx))); - if (opts.visibility) |v| attrs.appendAssumeCapacity(.named(ctx, "symbol_visibility", v.attribute(ctx))); - return mlir.Operation.make(ctx, opName("global"), .{ - .attributes = attrs.constSlice(), - .verify = false, - .location = location, - }); -} - -pub fn get_global(ctx: *mlir.Context, name: []const u8, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("get_global"), .{ - .results = .{ .flat = &.{result_type} }, - .attributes = &.{.named(ctx, "name", .flatSymbolRef(ctx, name))}, - // Symbol lookup needs the enclosing module; verified with it. - .verify = false, - .location = location, - }); -} - -/// `cuda_tile.module @name { ... }`. `body` holds the entries and globals. -pub fn module(ctx: *mlir.Context, name: []const u8, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("module"), .{ - .attributes = &.{.named(ctx, "sym_name", .string(ctx, name))}, - .blocks = &.{body}, - .verify = false, - .location = location, - }); -} - -pub const EntryArgs = struct { - name: []const u8, - /// The entry block; its arguments are the kernel parameters, every one a - /// rank-0 tile. - block: *mlir.Block, - arg_attrs: ?[]const *const mlir.Attribute = null, - optimization_hints: ?*const mlir.Attribute = null, - location: *const mlir.Location, -}; - -/// `entry @name(...) { ... }` — the kernel. Returns nothing; results reach -/// the host through pointer arguments. -pub fn entry(ctx: *mlir.Context, args: EntryArgs) *mlir.Operation { - var arg_types: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (args.block.numArguments() > arg_types.capacity()) std.debug.panic("cuda_tile.entry: {d} parameters, at most {d} supported", .{ args.block.numArguments(), arg_types.capacity() }); - for (0..args.block.numArguments()) |i| arg_types.appendAssumeCapacity(args.block.argument(i).type_()); - - var attrs: Attrs = .empty; - attrs.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, args.name))); - attrs.appendAssumeCapacity(.named(ctx, "function_type", .typeAttr(.function(ctx, arg_types.constSlice(), &.{})))); - if (args.arg_attrs) |a| attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, a))); - if (args.optimization_hints) |h| attrs.appendAssumeCapacity(.named(ctx, "optimization_hints", h)); - - return mlir.Operation.make(ctx, opName("entry"), .{ - .attributes = attrs.constSlice(), - .blocks = &.{args.block}, - .verify = false, - .location = args.location, - }); -} - -fn terminator(ctx: *mlir.Context, comptime mnemonic: []const u8, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName(mnemonic), .{ - .operands = .{ .flat = values }, - .verify = false, - .location = location, - }); -} - -pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return terminator(ctx, "return", values, location); -} - -pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return terminator(ctx, "yield", values, location); -} - -pub fn continue_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return terminator(ctx, "continue", values, location); -} - -pub fn break_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return terminator(ctx, "break", values, location); -} - -/// `for %iv in (%lb to %ub, step %step) iter_values(...)`. `body` takes -/// `[iv, inits...]` and ends with `continue`. -pub fn for_(ctx: *mlir.Context, lower: *const mlir.Value, upper: *const mlir.Value, step: *const mlir.Value, inits: []const *const mlir.Value, body: *mlir.Block, unsigned_cmp: bool, location: *const mlir.Location) *mlir.Operation { - // Built on the state directly: any number of carried values, and outer - // SSA uses only dominate once the op sits in its block, so no per-op - // verify. - var state: mlir.OperationState = .init(opName("for"), location); - state.addOperands(&.{ lower, upper, step }); - state.addOperands(inits); - for (inits) |v| state.addResults(&.{v.type_()}); - if (unsigned_cmp) state.addAttributes(&.{.named(ctx, "unsignedCmp", .unit(ctx))}); - const region = mlir.Region.init(); - region.appendOwnedBlock(body); - state.addOwnedRegions(&.{region}); - return mlir.Operation.init(&state) catch @panic("Failed to create cuda_tile.for"); -} - -/// `if %cond -> (types) { ... } else { ... }`. Branches end with `yield` -/// (or `continue`/`break`/`return` when they leave the enclosing scope). -pub fn if_( - ctx: *mlir.Context, - cond: *const mlir.Value, - result_types: []const *const mlir.Type, - then_block: *mlir.Block, - opts: struct { - else_block: ?*mlir.Block = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - _ = ctx; - var state: mlir.OperationState = .init(opName("if"), location); - state.addOperands(&.{cond}); - state.addResults(result_types); - const then_region = mlir.Region.init(); - then_region.appendOwnedBlock(then_block); - const else_region = mlir.Region.init(); - if (opts.else_block) |b| else_region.appendOwnedBlock(b); - state.addOwnedRegions(&.{ then_region, else_region }); - return mlir.Operation.init(&state) catch @panic("Failed to create cuda_tile.if"); -} - -/// An unstructured loop over `inits`; the body takes them as arguments and -/// leaves through `continue` (next iteration) or `break` (the results). -pub fn loop(ctx: *mlir.Context, inits: []const *const mlir.Value, result_types: []const *const mlir.Type, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("loop"), .{ - .operands = .{ .flat = inits }, - .results = .{ .flat = result_types }, - .blocks = &.{body}, - .verify = false, - .location = location, - }); -} - -/// `reduce %operands dim=D identities=[...]` — `body` takes `2N` arguments -/// (element, accumulator per operand) and yields `N` accumulators. Results -/// drop dimension `dim`. -pub fn reduce(ctx: *mlir.Context, operands: []const *const mlir.Value, dim: i32, identities: []const *const mlir.Attribute, body: *mlir.Block, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("reduce"), .{ - .operands = .{ .flat = operands }, - .results = .{ .flat = result_types }, - .blocks = &.{body}, - .attributes = &.{ - .named(ctx, "dim", .int(ctx, .i32, dim)), - .named(ctx, "identities", .array(ctx, identities)), - }, - .verify = false, - .location = location, - }); -} - -/// `scan %operands dim=D reverse=R identities=[...]` — like `reduce`, but the -/// results keep the operand shape. -pub fn scan(ctx: *mlir.Context, operands: []const *const mlir.Value, dim: i32, reverse: bool, identities: []const *const mlir.Attribute, body: *mlir.Block, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("scan"), .{ - .operands = .{ .flat = operands }, - .results = .{ .flat = result_types }, - .blocks = &.{body}, - .attributes = &.{ - .named(ctx, "dim", .int(ctx, .i32, dim)), - .named(ctx, "reverse", .boolean(ctx, reverse)), - .named(ctx, "identities", .array(ctx, identities)), - }, - .verify = false, - .location = location, - }); -} - -pub fn assert_(ctx: *mlir.Context, condition: *const mlir.Value, message: []const u8, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("assert"), .{ - .operands = .{ .flat = &.{condition} }, - .attributes = &.{.named(ctx, "message", .string(ctx, message))}, - .location = location, - }); -} - -/// `%r = assume #predicate, %value` — a refined copy of `value`. -pub fn assume(ctx: *mlir.Context, value: *const mlir.Value, predicate: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("assume"), .{ - .operands = .{ .flat = &.{value} }, - .results = .{ .flat = &.{value.type_()} }, - .attributes = &.{.named(ctx, "predicate", predicate)}, - .location = location, - }); -} - -/// `make_tensor_view %base, shape = [...], strides = [...]`. Static extents -/// live in `result_type`; the operands fill its `?` slots in order. -pub fn make_tensor_view(ctx: *mlir.Context, base: *const mlir.Value, dynamic_shape: []const *const mlir.Value, dynamic_strides: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - var segs: Segments = .{}; - segs.add(&.{base}); - segs.add(dynamic_shape); - segs.add(dynamic_strides); - return mlir.Operation.make(ctx, opName("make_tensor_view"), .{ - .operands = .{ .flat = segs.values.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{segs.attr(ctx)}, - .location = location, - }); -} - -fn viewFromTensorView(ctx: *mlir.Context, comptime mnemonic: []const u8, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return convert(ctx, mnemonic, tensor_view, result_type, .empty, location); -} - -/// The tile shape, padding and dim_map are all in `result_type`. -pub fn make_partition_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return viewFromTensorView(ctx, "make_partition_view", tensor_view, result_type, location); -} - -pub fn make_strided_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return viewFromTensorView(ctx, "make_strided_view", tensor_view, result_type, location); -} - -pub fn make_gather_scatter_view(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return viewFromTensorView(ctx, "make_gather_scatter_view", tensor_view, result_type, location); -} - -/// One rank-0 integer tile per view index dimension. -pub fn get_index_space_shape(ctx: *mlir.Context, view: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("get_index_space_shape"), .{ - .operands = .{ .flat = &.{view} }, - .results = .{ .flat = result_types }, - .location = location, - }); -} - -/// One rank-0 integer tile per tensor dimension. -pub fn get_tensor_shape(ctx: *mlir.Context, tensor_view: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("get_tensor_shape"), .{ - .operands = .{ .flat = &.{tensor_view} }, - .results = .{ .flat = result_types }, - .location = location, - }); -} - -pub const MemOpts = struct { - ordering: MemoryOrdering = .weak, - scope: ?MemoryScope = null, - token: ?*const mlir.Value = null, - optimization_hints: ?*const mlir.Attribute = null, -}; - -fn memAttrs(ctx: *mlir.Context, opts: MemOpts) Attrs { - var attrs: Attrs = .empty; - attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", opts.ordering.attribute(ctx))); - if (opts.scope) |s| attrs.appendAssumeCapacity(.named(ctx, "memory_scope", s.attribute(ctx))); - if (opts.optimization_hints) |h| attrs.appendAssumeCapacity(.named(ctx, "optimization_hints", h)); - return attrs; -} - -/// An optional operand as a 0- or 1-element segment. Held in a local so the -/// slice handed to `Operation.make` stays valid. -const Seg = struct { - buf: [1]*const mlir.Value = undefined, - len: usize = 0, - - fn slice(self: *const Seg) []const *const mlir.Value { - return self.buf[0..self.len]; - } -}; - -fn optional(v: ?*const mlir.Value) Seg { - var s: Seg = .{}; - if (v) |x| { - s.buf[0] = x; - s.len = 1; - } - return s; -} - -/// Operands of an `AttrSizedOperandSegments` op, flat plus the -/// `operandSegmentSizes` the op's properties expect (a DenseI32ArrayAttr). -const Segments = struct { - values: stdx.BoundedArray(*const mlir.Value, 128) = .empty, - sizes: stdx.BoundedArray(i32, 8) = .empty, - - fn add(self: *Segments, segment: []const *const mlir.Value) void { - self.values.appendSliceAssumeCapacity(segment); - self.sizes.appendAssumeCapacity(@intCast(segment.len)); - } - - fn attr(self: *const Segments, ctx: *mlir.Context) mlir.NamedAttribute { - return .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, self.sizes.constSlice())); - } -}; - -/// Two results: the tile (the view's tile type) and the token. -pub fn load_view_tko(ctx: *mlir.Context, view: *const mlir.Value, index: []const *const mlir.Value, opts: MemOpts, location: *const mlir.Location) *mlir.Operation { - const tok = optional(opts.token); - var segs: Segments = .{}; - segs.add(&.{view}); - segs.add(index); - segs.add(tok.slice()); - var attrs = memAttrs(ctx, opts); - // Preserve bounds checks unless the caller has proved a coordinate in bounds. - const inbounds: [mlir.ShapedType.MAX_RANK]i32 = @splat(0); - attrs.appendAssumeCapacity(.named(ctx, "inbounds", .denseArray(ctx, .bool, inbounds[0..index.len]))); - attrs.appendAssumeCapacity(segs.attr(ctx)); - return mlir.Operation.make(ctx, opName("load_view_tko"), .{ - .operands = .{ .flat = segs.values.constSlice() }, - .results = .{ .flat = &.{ tileTypeOfView(view.type_()), tokenType(ctx) } }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// One result: the token. -pub fn store_view_tko(ctx: *mlir.Context, tile: *const mlir.Value, view: *const mlir.Value, index: []const *const mlir.Value, opts: MemOpts, location: *const mlir.Location) *mlir.Operation { - const tok = optional(opts.token); - var segs: Segments = .{}; - segs.add(&.{tile}); - segs.add(&.{view}); - segs.add(index); - segs.add(tok.slice()); - var attrs = memAttrs(ctx, opts); - const inbounds: [mlir.ShapedType.MAX_RANK]i32 = @splat(0); - attrs.appendAssumeCapacity(.named(ctx, "inbounds", .denseArray(ctx, .bool, inbounds[0..index.len]))); - attrs.appendAssumeCapacity(segs.attr(ctx)); - return mlir.Operation.make(ctx, opName("store_view_tko"), .{ - .operands = .{ .flat = segs.values.constSlice() }, - .results = .{ .flat = &.{tokenType(ctx)} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -pub const LoadPtrOpts = struct { - mask: ?*const mlir.Value = null, - padding: ?*const mlir.Value = null, - mem: MemOpts = .{}, -}; - -/// Gather through a tile of pointers. Two results: the loaded tile and the -/// token. `result_type` is the pointee tile. -pub fn load_ptr_tko(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, opts: LoadPtrOpts, location: *const mlir.Location) *mlir.Operation { - const mask = optional(opts.mask); - const padding = optional(opts.padding); - const tok = optional(opts.mem.token); - var segs: Segments = .{}; - segs.add(&.{source}); - segs.add(mask.slice()); - segs.add(padding.slice()); - segs.add(tok.slice()); - var attrs = memAttrs(ctx, opts.mem); - attrs.appendAssumeCapacity(segs.attr(ctx)); - return mlir.Operation.make(ctx, opName("load_ptr_tko"), .{ - .operands = .{ .flat = segs.values.constSlice() }, - .results = .{ .flat = &.{ result_type, tokenType(ctx) } }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -pub const StorePtrOpts = struct { - mask: ?*const mlir.Value = null, - mem: MemOpts = .{}, -}; - -/// Scatter through a tile of pointers. One result: the token. -pub fn store_ptr_tko(ctx: *mlir.Context, destination: *const mlir.Value, value: *const mlir.Value, opts: StorePtrOpts, location: *const mlir.Location) *mlir.Operation { - const mask = optional(opts.mask); - const tok = optional(opts.mem.token); - var segs: Segments = .{}; - segs.add(&.{destination}); - segs.add(&.{value}); - segs.add(mask.slice()); - segs.add(tok.slice()); - var attrs = memAttrs(ctx, opts.mem); - attrs.appendAssumeCapacity(segs.attr(ctx)); - return mlir.Operation.make(ctx, opName("store_ptr_tko"), .{ - .operands = .{ .flat = segs.values.constSlice() }, - .results = .{ .flat = &.{tokenType(ctx)} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -pub const AtomicOpts = struct { - ordering: MemoryOrdering = .acq_rel, - scope: MemoryScope = .device, - mask: ?*const mlir.Value = null, - token: ?*const mlir.Value = null, -}; - -fn atomicAttrs(ctx: *mlir.Context, opts: AtomicOpts) Attrs { - var attrs: Attrs = .empty; - attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", opts.ordering.attribute(ctx))); - attrs.appendAssumeCapacity(.named(ctx, "memory_scope", opts.scope.attribute(ctx))); - return attrs; -} - -/// Two results: the old values and the token. -pub fn atomic_cas_tko(ctx: *mlir.Context, pointers: *const mlir.Value, cmp: *const mlir.Value, val: *const mlir.Value, opts: AtomicOpts, location: *const mlir.Location) *mlir.Operation { - const mask = optional(opts.mask); - const tok = optional(opts.token); - var segs: Segments = .{}; - segs.add(&.{pointers}); - segs.add(&.{cmp}); - segs.add(&.{val}); - segs.add(mask.slice()); - segs.add(tok.slice()); - var attrs = atomicAttrs(ctx, opts); - attrs.appendAssumeCapacity(segs.attr(ctx)); - return mlir.Operation.make(ctx, opName("atomic_cas_tko"), .{ - .operands = .{ .flat = segs.values.constSlice() }, - .results = .{ .flat = &.{ val.type_(), tokenType(ctx) } }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// Two results: the old values and the token. -pub fn atomic_rmw_tko(ctx: *mlir.Context, pointers: *const mlir.Value, mode: AtomicRMWMode, arg: *const mlir.Value, opts: AtomicOpts, location: *const mlir.Location) *mlir.Operation { - const mask = optional(opts.mask); - const tok = optional(opts.token); - var segs: Segments = .{}; - segs.add(&.{pointers}); - segs.add(&.{arg}); - segs.add(mask.slice()); - segs.add(tok.slice()); - var attrs = atomicAttrs(ctx, opts); - attrs.appendAssumeCapacity(.named(ctx, "mode", mode.attribute(ctx))); - attrs.appendAssumeCapacity(segs.attr(ctx)); - return mlir.Operation.make(ctx, opName("atomic_rmw_tko"), .{ - .operands = .{ .flat = segs.values.constSlice() }, - .results = .{ .flat = &.{ arg.type_(), tokenType(ctx) } }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// `atomic_red_view_tko` is relaxed-only, block- or device-scoped, and has -/// no mask operand. -pub const RedViewOpts = struct { - scope: MemoryScope = .device, - token: ?*const mlir.Value = null, -}; - -/// 13.3: an atomic reduction through an unpadded tiled view. One result: -/// the token. `xchg` is not a reduction. -pub fn atomic_red_view_tko(ctx: *mlir.Context, view: *const mlir.Value, index: []const *const mlir.Value, mode: AtomicRMWMode, value: *const mlir.Value, opts: RedViewOpts, location: *const mlir.Location) *mlir.Operation { - if (mode == .xchg) @panic("cuda_tile.atomic_red_view_tko: xchg is not a reduction"); - if (opts.scope == .sys) @panic("cuda_tile.atomic_red_view_tko: only tl_blk and device scopes are allowed"); - const tok = optional(opts.token); - var segs: Segments = .{}; - segs.add(&.{view}); - segs.add(index); - segs.add(&.{value}); - segs.add(tok.slice()); - var attrs: Attrs = .empty; - attrs.appendAssumeCapacity(.named(ctx, "memory_ordering_semantics", MemoryOrdering.relaxed.attribute(ctx))); - attrs.appendAssumeCapacity(.named(ctx, "memory_scope", opts.scope.attribute(ctx))); - attrs.appendAssumeCapacity(.named(ctx, "mode", mode.attribute(ctx))); - attrs.appendAssumeCapacity(segs.attr(ctx)); - return mlir.Operation.make(ctx, opName("atomic_red_view_tko"), .{ - .operands = .{ .flat = segs.values.constSlice() }, - .results = .{ .flat = &.{tokenType(ctx)} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// 13.3: `num_elem` elements of the pointee, as a rank-0 pointer tile. -pub fn alloca(ctx: *mlir.Context, num_elem: i64, alignment: i64, is_global: bool, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - var attrs: Attrs = .empty; - attrs.appendAssumeCapacity(.named(ctx, "num_elem", .int(ctx, .i64, num_elem))); - attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, alignment))); - if (is_global) attrs.appendAssumeCapacity(.named(ctx, "global", .unit(ctx))); - return mlir.Operation.make(ctx, opName("alloca"), .{ - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -pub fn make_token(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("make_token"), .{ - .results = .{ .flat = &.{tokenType(ctx)} }, - .location = location, - }); -} - -pub fn join_tokens(ctx: *mlir.Context, tokens: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, opName("join_tokens"), .{ - .operands = .{ .flat = tokens }, - .results = .{ .flat = &.{tokenType(ctx)} }, - .location = location, - }); -} - -/// Device-side printf. One result: the token. -pub fn print_tko( - ctx: *mlir.Context, - str: []const u8, - args: []const *const mlir.Value, - opts: struct { - token: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - const tok = optional(opts.token); - var segs: Segments = .{}; - segs.add(args); - segs.add(tok.slice()); - return mlir.Operation.make(ctx, opName("print_tko"), .{ - .operands = .{ .flat = segs.values.constSlice() }, - .results = .{ .flat = &.{tokenType(ctx)} }, - .attributes = &.{ .named(ctx, "str", .string(ctx, str)), segs.attr(ctx) }, - .location = location, - }); -} - -// ============================================================================= -// Tests -// ============================================================================= diff --git a/mlir/dialects/cuda_tile/test.zig b/mlir/dialects/cuda_tile/test.zig deleted file mode 100644 index 52ae9480..00000000 --- a/mlir/dialects/cuda_tile/test.zig +++ /dev/null @@ -1,391 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const cuda_tile = @import("mlir/dialects/cuda_tile"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "cuda_tile.absf", - "cuda_tile.absi", - "cuda_tile.addf", - "cuda_tile.addi", - "cuda_tile.alloca", - "cuda_tile.andi", - "cuda_tile.assert", - "cuda_tile.assume", - "cuda_tile.atan2", - "cuda_tile.atomic_cas_tko", - "cuda_tile.atomic_red_view_tko", - "cuda_tile.atomic_rmw_tko", - "cuda_tile.bitcast", - "cuda_tile.break", - "cuda_tile.broadcast", - "cuda_tile.cat", - "cuda_tile.ceil", - "cuda_tile.cmpf", - "cuda_tile.cmpi", - "cuda_tile.constant", - "cuda_tile.continue", - "cuda_tile.cos", - "cuda_tile.cosh", - "cuda_tile.divf", - "cuda_tile.divi", - "cuda_tile.entry", - "cuda_tile.exp", - "cuda_tile.exp2", - "cuda_tile.exti", - "cuda_tile.extract", - "cuda_tile.floor", - "cuda_tile.fma", - "cuda_tile.for", - "cuda_tile.fpowf", - "cuda_tile.ftof", - "cuda_tile.ftoi", - "cuda_tile.get_global", - "cuda_tile.get_index_space_shape", - "cuda_tile.get_num_tile_blocks", - "cuda_tile.get_tensor_shape", - "cuda_tile.get_tile_block_id", - "cuda_tile.global", - "cuda_tile.if", - "cuda_tile.int_to_ptr", - "cuda_tile.iota", - "cuda_tile.itof", - "cuda_tile.join_tokens", - "cuda_tile.load_ptr_tko", - "cuda_tile.load_view_tko", - "cuda_tile.log", - "cuda_tile.log2", - "cuda_tile.loop", - "cuda_tile.make_gather_scatter_view", - "cuda_tile.make_partition_view", - "cuda_tile.make_strided_view", - "cuda_tile.make_tensor_view", - "cuda_tile.make_token", - "cuda_tile.maxf", - "cuda_tile.maxi", - "cuda_tile.minf", - "cuda_tile.mini", - "cuda_tile.mmaf", - "cuda_tile.mmaf_scaled", - "cuda_tile.mmai", - "cuda_tile.module", - "cuda_tile.mulf", - "cuda_tile.mulhii", - "cuda_tile.muli", - "cuda_tile.negf", - "cuda_tile.negi", - "cuda_tile.offset", - "cuda_tile.ori", - "cuda_tile.pack", - "cuda_tile.permute", - "cuda_tile.print_tko", - "cuda_tile.ptr_to_int", - "cuda_tile.ptr_to_ptr", - "cuda_tile.reduce", - "cuda_tile.remf", - "cuda_tile.remi", - "cuda_tile.reshape", - "cuda_tile.return", - "cuda_tile.rsqrt", - "cuda_tile.scan", - "cuda_tile.select", - "cuda_tile.shli", - "cuda_tile.shri", - "cuda_tile.sin", - "cuda_tile.sinh", - "cuda_tile.sqrt", - "cuda_tile.store_ptr_tko", - "cuda_tile.store_view_tko", - "cuda_tile.subf", - "cuda_tile.subi", - "cuda_tile.tan", - "cuda_tile.tanh", - "cuda_tile.trunci", - "cuda_tile.unpack", - "cuda_tile.xori", - "cuda_tile.yield", -}; - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - cuda_tile.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(cuda_tile); - inline for (comptime std.meta.declarations(cuda_tile)) |decl| { - const T = @field(cuda_tile, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "dialect registers and types round-trip" { - const ctx = try testContext(); - defer ctx.deinit(); - - try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.entry")); - try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.load_view_tko")); - try std.testing.expect(ctx.isRegisteredOperation("cuda_tile.mmaf")); - - const f32_ty = mlir.Type.float(ctx, .f32); - const tile = cuda_tile.tileType(ctx, &.{ 64, 64 }, f32_ty); - try std.testing.expect(tile.eql(try mlir.Type.parse(ctx, "!cuda_tile.tile<64x64xf32>"))); - try std.testing.expectEqual(@as(usize, 2), tile.isA(cuda_tile.TileType).?.rank()); - try std.testing.expectEqual(@as(i64, 64), tile.isA(cuda_tile.TileType).?.dimension(1)); - - const ptr = cuda_tile.pointerType(ctx, f32_ty); - try std.testing.expect(ptr.eql(try mlir.Type.parse(ctx, "!cuda_tile.ptr"))); - try std.testing.expect(ptr.isA(cuda_tile.PointerType).?.pointee().eql(f32_ty)); - - const dyn = cuda_tile.TensorViewType.dynamic(); - const tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{ dyn, 32 }, &.{ dyn, 1 }); - try std.testing.expect(tv.eql(try mlir.Type.parse(ctx, "!cuda_tile.tensor_view"))); - - const static_tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{16}, &.{1}); - const pv: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{2}, .tensor_view = static_tv, .padding = .zero })); - try std.testing.expect(pv.eql(try mlir.Type.parse(ctx, "!cuda_tile.partition_view>"))); - try std.testing.expect(cuda_tile.tileTypeOfView(pv).eql(cuda_tile.tileType(ctx, &.{2}, f32_ty))); - try std.testing.expectEqual(@as(usize, 1), cuda_tile.indexRankOfView(pv)); - - try std.testing.expect(cuda_tile.tokenType(ctx).eql(try mlir.Type.parse(ctx, "!cuda_tile.token"))); - - const gather_tv = cuda_tile.tensorViewType(ctx, f32_ty, &.{ 64, 32 }, &.{ 32, 1 }); - const gsv = cuda_tile.gatherScatterViewType(ctx, &.{ 8, 16 }, gather_tv, 0, .{ .padding = .zero }); - try std.testing.expect(gsv.eql(try mlir.Type.parse(ctx, "!cuda_tile.gather_scatter_view, sparse_dim=0>"))); - const typed_gsv = gsv.isA(cuda_tile.GatherScatterViewType).?; - const gsv_tile = typed_gsv.tileShape(); - var gsv_shape: [2]i32 = undefined; - try std.testing.expectEqual(gsv_shape.len, gsv_tile.numElements()); - for (&gsv_shape, 0..) |*d, i| d.* = gsv_tile.element(i); - const rebuilt_gsv = try cuda_tile.GatherScatterViewType.get(ctx, .{ - .tileShape = &gsv_shape, - .tensorView = typed_gsv.tensorView(), - .sparseDim = typed_gsv.sparseDim(), - .paddingValue = typed_gsv.paddingValue(), - }); - try std.testing.expect(rebuilt_gsv.eql(typed_gsv)); - try std.testing.expect(cuda_tile.tileTypeOfView(gsv).eql(cuda_tile.tileType(ctx, &.{ 8, 16 }, f32_ty))); - try std.testing.expect((try cuda_tile.GatherScatterViewType.get(ctx, .{ - .tileShape = &.{ 8, 16 }, - .tensorView = gather_tv.isA(cuda_tile.TensorViewType).?, - .sparseDim = 1, - })).paddingValue() == null); - try std.testing.expectError(error.InvalidMlir, cuda_tile.GatherScatterViewType.get(ctx, .{ - .tileShape = &.{8}, - .tensorView = gather_tv.isA(cuda_tile.TensorViewType).?, - .sparseDim = 0, - })); -} - -test "enum and hint attributes" { - const ctx = try testContext(); - defer ctx.deinit(); - - try std.testing.expect(cuda_tile.RoundingMode.nearest_even.attribute(ctx).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.rounding"))); - const padding = cuda_tile.PaddingValue.neg_inf.attribute(ctx); - const padding_text = c.mlirCudaTilePaddingValueAttrGetValue(padding.ptr()); - try std.testing.expectEqualStrings("neg_inf", padding_text.data[0..padding_text.length]); - - const hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.EntryHint{ - .{ .arch = .sm_100, .num_cta_in_cga = 2 }, - .{ .arch = .sm_120, .num_cta_in_cga = 2, .occupancy = 2 }, - }); - try std.testing.expect(hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); - const typed_hints = hints.isA(cuda_tile.OptimizationHintsAttr).?; - try std.testing.expect((try cuda_tile.OptimizationHintsAttr.get(ctx, .{ .value = typed_hints.value() })).eql(typed_hints)); - const ls_hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.LoadStoreHint{.{ .arch = .sm_100, .allow_tma = false, .latency = 3 }}); - try std.testing.expect(ls_hints.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.optimization_hints"))); - const not_a_dict: *const mlir.DictionaryAttribute = @ptrCast(mlir.Attribute.dict(ctx, &.{.named(ctx, "sm_100", .int(ctx, .i64, 1))})); - try std.testing.expectError(error.InvalidMlir, cuda_tile.OptimizationHintsAttr.get(ctx, .{ .value = not_a_dict })); - - const div = cuda_tile.divByAttr(ctx, 16, .{}); - try std.testing.expect(div.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<16>"))); - const div_every = cuda_tile.divByAttr(ctx, 4, .{ .every = 4, .along = 1 }); - try std.testing.expect(div_every.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<4, every 4 along 1>"))); - try std.testing.expect(cuda_tile.divByAttr(ctx, 32, .{ .every = 4 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.div_by<32, every 4 along 0>"))); - const typed_div = div_every.isA(cuda_tile.DivByAttr).?; - try std.testing.expect((try cuda_tile.DivByAttr.get(ctx, .{ - .divisor = typed_div.divisor(), - .every = typed_div.every(), - .along = typed_div.along(), - })).eql(typed_div)); - try std.testing.expect(div.isA(cuda_tile.DivByAttr).?.every() == null); - try std.testing.expectError(error.InvalidMlir, cuda_tile.DivByAttr.get(ctx, .{ .divisor = 4, .every = 4 })); - - const same = cuda_tile.sameElementsAttr(ctx, &.{ 2, 4 }); - try std.testing.expect(same.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.same_elements<[2, 4]>"))); - const same_values = same.isA(cuda_tile.SameElementsAttr).?.values(); - try std.testing.expectEqual(@as(usize, 2), same_values.numElements()); - try std.testing.expectEqual(@as(i64, 4), same_values.element(1)); - - const lower = cuda_tile.boundedAttr(ctx, .{ .lb = 0 }); - try std.testing.expect(lower.eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<0, ?>"))); - try std.testing.expect(cuda_tile.boundedAttr(ctx, .{ .ub = 7 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded"))); - try std.testing.expect(cuda_tile.boundedAttr(ctx, .{ .lb = -3, .ub = 7 }).eql(try mlir.Attribute.parse(ctx, "#cuda_tile.bounded<-3, 7>"))); - const typed_lower = lower.isA(cuda_tile.BoundedAttr).?; - try std.testing.expectEqual(@as(?i64, 0), typed_lower.lb()); - try std.testing.expect(typed_lower.ub() == null); -} - -test "entry round-trip through print and parse" { - const ctx = try testContext(); - defer ctx.deinit(); - - const loc: *const mlir.Location = .unknown(ctx); - const module_op: *mlir.Module = .init(loc); - defer module_op.deinit(); - - const f32_ty = mlir.Type.float(ctx, .f32); - const ptr_tile = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, f32_ty)); - const block = mlir.Block.init(&.{ ptr_tile, ptr_tile }, &.{ loc, loc }); - - // out[i] = in[i] + 1 over 128 elements, the shape of cuda-tile's README example. - const idx_ty = cuda_tile.tileType(ctx, &.{128}, .int(ctx, .i32)); - const offsets = cuda_tile.iota(ctx, idx_ty, loc).appendTo(block); - const ptr1 = cuda_tile.tileType(ctx, &.{1}, cuda_tile.pointerType(ctx, f32_ty)); - const ptr128 = cuda_tile.tileType(ctx, &.{128}, cuda_tile.pointerType(ctx, f32_ty)); - const in_r = cuda_tile.reshape(ctx, block.argument(0), ptr1, loc).appendTo(block); - const in_b = cuda_tile.broadcast(ctx, in_r.result(0), ptr128, loc).appendTo(block); - const in_p = cuda_tile.offset(ctx, in_b.result(0), offsets.result(0), loc).appendTo(block); - const f32_128 = cuda_tile.tileType(ctx, &.{128}, f32_ty); - const loaded = cuda_tile.load_ptr_tko(ctx, in_p.result(0), f32_128, .{}, loc).appendTo(block); - const one = cuda_tile.constant(ctx, cuda_tile.splatAttr(f32_128, cuda_tile.floatElemAttr(ctx, f32_ty, 1.0)), f32_128, loc).appendTo(block); - const sum = cuda_tile.addf(ctx, loaded.result(0), one.result(0), .nearest_even, false, loc).appendTo(block); - const out_r = cuda_tile.reshape(ctx, block.argument(1), ptr1, loc).appendTo(block); - const out_b = cuda_tile.broadcast(ctx, out_r.result(0), ptr128, loc).appendTo(block); - const out_p = cuda_tile.offset(ctx, out_b.result(0), offsets.result(0), loc).appendTo(block); - _ = cuda_tile.store_ptr_tko(ctx, out_p.result(0), sum.result(0), .{ .mem = .{ .token = loaded.result(1) } }, loc).appendTo(block); - _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); - - const ct_body = mlir.Block.init(&.{}, &.{}); - _ = cuda_tile.entry(ctx, .{ .name = "add_one", .block = block, .location = loc }).appendTo(ct_body); - _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); - - try std.testing.expect(module_op.operation().verify()); - - var al: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer al.deinit(); - try al.writer.print("{f}", .{module_op.operation()}); - const text = al.written(); - try std.testing.expect(std.mem.indexOf(u8, text, "cuda_tile.module @m") != null); - try std.testing.expect(std.mem.indexOf(u8, text, "entry @add_one(") != null); - try std.testing.expect(std.mem.indexOf(u8, text, "load_ptr_tko weak") != null); - try std.testing.expect(std.mem.indexOf(u8, text, "addf") != null); - - const parsed = try mlir.Module.parse(ctx, text); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "views, control flow and mma verify" { - const ctx = try testContext(); - defer ctx.deinit(); - - const loc: *const mlir.Location = .unknown(ctx); - const module_op: *mlir.Module = .init(loc); - defer module_op.deinit(); - - const bf16_ty = mlir.Type.float(ctx, .bf16); - const f32_ty = mlir.Type.float(ctx, .f32); - const i32_ty = mlir.Type.int(ctx, .i32); - const bf16_ptr = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, bf16_ty)); - const f32_ptr = cuda_tile.tileType(ctx, &.{}, cuda_tile.pointerType(ctx, f32_ty)); - const block = mlir.Block.init(&.{ bf16_ptr, bf16_ptr, f32_ptr }, &.{ loc, loc, loc }); - - // C[128x128] = A[128x64] @ B[64x128] in 64-wide K tiles. - const tv_a = cuda_tile.make_tensor_view(ctx, block.argument(0), &.{}, &.{}, cuda_tile.tensorViewType(ctx, bf16_ty, &.{ 128, 64 }, &.{ 64, 1 }), loc).appendTo(block); - const tv_b = cuda_tile.make_tensor_view(ctx, block.argument(1), &.{}, &.{}, cuda_tile.tensorViewType(ctx, bf16_ty, &.{ 64, 128 }, &.{ 128, 1 }), loc).appendTo(block); - const tv_c = cuda_tile.make_tensor_view(ctx, block.argument(2), &.{}, &.{}, cuda_tile.tensorViewType(ctx, f32_ty, &.{ 128, 128 }, &.{ 128, 1 }), loc).appendTo(block); - const pv_a: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_a.result(0).type_(), .padding = .zero })); - const pv_b: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_b.result(0).type_(), .padding = .zero })); - const pv_c: *const mlir.Type = @ptrCast(cuda_tile.PartitionViewType.get(ctx, .{ .tile_shape = &.{ 64, 64 }, .tensor_view = tv_c.result(0).type_() })); - const va = cuda_tile.make_partition_view(ctx, tv_a.result(0), pv_a, loc).appendTo(block); - const vb = cuda_tile.make_partition_view(ctx, tv_b.result(0), pv_b, loc).appendTo(block); - const vc = cuda_tile.make_partition_view(ctx, tv_c.result(0), pv_c, loc).appendTo(block); - - const bid = cuda_tile.get_tile_block_id(ctx, loc).appendTo(block); - const i32_tile = cuda_tile.tileType(ctx, &.{}, i32_ty); - const zero = cuda_tile.constant(ctx, cuda_tile.splatAttr(i32_tile, cuda_tile.intElemAttr(i32_ty, 0)), i32_tile, loc).appendTo(block); - const one = cuda_tile.constant(ctx, cuda_tile.splatAttr(i32_tile, cuda_tile.intElemAttr(i32_ty, 1)), i32_tile, loc).appendTo(block); - const acc_ty = cuda_tile.tileType(ctx, &.{ 64, 64 }, f32_ty); - const acc0 = cuda_tile.constant(ctx, cuda_tile.splatAttr(acc_ty, cuda_tile.floatElemAttr(ctx, f32_ty, 0.0)), acc_ty, loc).appendTo(block); - - const body = mlir.Block.init(&.{ i32_tile, acc_ty }, &.{ loc, loc }); - const at = cuda_tile.load_view_tko(ctx, va.result(0), &.{ bid.result(0), body.argument(0) }, .{}, loc).appendTo(body); - const bt = cuda_tile.load_view_tko(ctx, vb.result(0), &.{ body.argument(0), bid.result(1) }, .{}, loc).appendTo(body); - try std.testing.expect(at.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); - const acc = cuda_tile.mmaf(ctx, at.result(0), bt.result(0), body.argument(1), false, loc).appendTo(body); - _ = cuda_tile.continue_(ctx, &.{acc.result(0)}, loc).appendTo(body); - const loop_op = cuda_tile.for_(ctx, zero.result(0), one.result(0), one.result(0), &.{acc0.result(0)}, body, false, loc).appendTo(block); - - // if %cond -> (tile) { yield } else { yield } - const cond = cuda_tile.cmpi(ctx, .equal, .signed, bid.result(0), zero.result(0), loc).appendTo(block); - const then_block = mlir.Block.init(&.{}, &.{}); - _ = cuda_tile.yield(ctx, &.{loop_op.result(0)}, loc).appendTo(then_block); - const else_block = mlir.Block.init(&.{}, &.{}); - _ = cuda_tile.yield(ctx, &.{acc0.result(0)}, loc).appendTo(else_block); - const if_op = cuda_tile.if_(ctx, cond.result(0), &.{acc_ty}, then_block, .{ .else_block = else_block }, loc).appendTo(block); - - const store = cuda_tile.store_view_tko(ctx, if_op.result(0), vc.result(0), &.{ bid.result(0), bid.result(1) }, .{}, loc).appendTo(block); - try std.testing.expect(store.attributeByName("inbounds").?.eql(.denseArray(ctx, .bool, &.{ 0, 0 }))); - _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); - - const ct_body = mlir.Block.init(&.{}, &.{}); - _ = cuda_tile.entry(ctx, .{ - .name = "gemm", - .block = block, - .optimization_hints = cuda_tile.optimizationHintsAttr(ctx, &[_]cuda_tile.EntryHint{.{ .arch = .sm_120, .num_worker_warps_per_cta = 4 }}), - .location = loc, - }).appendTo(ct_body); - _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); - - try std.testing.expect(module_op.operation().verify()); - - var al: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer al.deinit(); - try al.writer.print("{f}", .{module_op.operation()}); - const text = al.written(); - try std.testing.expect(std.mem.indexOf(u8, text, "optimization_hints=") != null); - try std.testing.expect(std.mem.indexOf(u8, text, "load_view_tko weak") != null); - try std.testing.expect(std.mem.indexOf(u8, text, "continue ") != null); - - const parsed = try mlir.Module.parse(ctx, text); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "the verifier rejects what the dialect forbids" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc: *const mlir.Location = .unknown(ctx); - const f32_ty = mlir.Type.float(ctx, .f32); - - // An entry with a rank-1 argument. (Per-op negatives go through - // `Operation.try_make`, whose logged error the test runner counts as a - // failure, so the module-level verifier is the oracle here.) - const module_op: *mlir.Module = .init(loc); - defer module_op.deinit(); - const block = mlir.Block.init(&.{cuda_tile.tileType(ctx, &.{4}, f32_ty)}, &.{loc}); - _ = cuda_tile.return_(ctx, &.{}, loc).appendTo(block); - const ct_body = mlir.Block.init(&.{}, &.{}); - _ = cuda_tile.entry(ctx, .{ .name = "bad", .block = block, .location = loc }).appendTo(ct_body); - _ = cuda_tile.module(ctx, "m", ct_body, loc).appendTo(module_op.body()); - try std.testing.expect(!module_op.operation().verify()); -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - cuda_tile.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/cute_ir/BUILD.bazel b/mlir/dialects/cute_ir/BUILD.bazel deleted file mode 100644 index c584509f..00000000 --- a/mlir/dialects/cute_ir/BUILD.bazel +++ /dev/null @@ -1,211 +0,0 @@ -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - ":dialect", - ":headers", - "//mlir/dialects/cute_nvgpu:dialect", - "@llvm-project//mlir:CAPIIR", - ], -) - -zig_library( - name = "cute_ir", - import_name = "mlir/dialects/cute_ir", - main = "cute_ir.zig", - tags = ["manual"], - deps = [ - ":capi", - "//mlir", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":cute_ir", - "//mlir", - "//mlir/dialects/cute_nvgpu", - ], -) - -td_library( - name = "td_files", - srcs = [ - "CuteAttrsAll.td", - "CuteAttrsCompiler.td", - "CuteEnums.td", - "CuteOps.td", - "CuteTypesAll.td", - "CuteTypesCompiler.td", - ], - includes = ["/"], - deps = [ - "@cute_ir//:cute_ir_dialect_td", - "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - ], -) - -gentbl_cc_library( - name = "enums_inc_gen", - tbl_outs = [ - ( - ["-gen-enum-decls"], - "CuteEnums.h.inc", - ), - ( - ["-gen-enum-defs"], - "CuteEnums.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteEnums.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "attrs_inc_gen", - tbl_outs = [ - ( - [ - "-gen-attrdef-decls", - "-attrdefs-dialect=cute", - ], - "CuteAttrDecls.h.inc", - ), - ( - [ - "-gen-attrdef-defs", - "-attrdefs-dialect=cute", - ], - "CuteAttrs.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteAttrsAll.td", - deps = [":td_files"], -) - -genrule( - name = "attrs_h_inc", - srcs = ["CuteAttrDecls.h.inc"], - outs = ["CuteAttrs.h.inc"], - cmd = "(echo '#ifdef GET_ATTRDEF_CLASSES'; " + - "echo '#include \"cute_ir/Dialect/Cute/IR/CuteEnums.h.inc\"'; " + - "echo '#endif'; cat $<) > $@", -) - -gentbl_cc_library( - name = "types_inc_gen", - tbl_outs = [ - ( - [ - "-gen-typedef-decls", - "-typedefs-dialect=cute", - ], - "CuteTypes.h.inc", - ), - ( - [ - "-gen-typedef-defs", - "-typedefs-dialect=cute", - ], - "CuteTypes.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteTypesAll.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "ops_inc_gen", - tbl_outs = [ - ( - ["-gen-op-decls"], - "CuteOps.h.inc", - ), - ( - ["-gen-op-defs"], - "CuteOps.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteOps.td", - deps = [":td_files"], -) - -cc_library( - name = "headers", - hdrs = [ - "CuteAttrs.h.inc", - "CuteInference.h", - ], - deps = [ - ":attrs_inc_gen", - ":enums_inc_gen", - ":ops_inc_gen", - ":types_inc_gen", - ":upstream_generated_headers", - "@cute_ir//:cute_ir_dialect_header", - "@cute_ir//:cutegen", - "@llvm-project//mlir:DataLayoutInterfaces", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:SideEffectInterfaces", - ], -) - -cc_library( - name = "dialect", - srcs = [ - "CuteInference.cpp", - "CuteOps.cpp", - "CuteTypes.cpp", - "@cute_ir//:cute_ir/lib/Dialect/IR/CuteAttributes.cpp", - "@cute_ir//:cute_ir/lib/Dialect/IR/CuteDialect.cpp", - ], - deps = [ - ":headers", - "//mlir/dialects/cute_nvgpu:headers", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:LLVMDialect", - "@llvm-project//mlir:MathDialect", - "@llvm-project//mlir:NVVMDialect", - "@llvm-project//mlir:SCFDialect", - "@llvm-project//mlir:UBDialect", - ], -) - -# Upstream CuTe sources retain their generated-header include paths. -cc_library( - name = "upstream_generated_headers", - hdrs = [ - "CuteAttrDecls.h.inc", - "CuteAttrs.cpp.inc", - "CuteAttrs.h.inc", - "CuteEnums.cpp.inc", - "CuteEnums.h.inc", - "CuteOps.cpp.inc", - "CuteOps.h.inc", - "CuteTypes.cpp.inc", - "CuteTypes.h.inc", - ], - include_prefix = "cute_ir/Dialect/Cute/IR", - strip_include_prefix = ".", -) diff --git a/mlir/dialects/cute_ir/CuteAttrsAll.td b/mlir/dialects/cute_ir/CuteAttrsAll.td deleted file mode 100644 index 3f42a1cb..00000000 --- a/mlir/dialects/cute_ir/CuteAttrsAll.td +++ /dev/null @@ -1,14 +0,0 @@ -//===- CuteAttrsAll.td - all cute attributes ---------------*- tablegen -*-===// -// -// The dialect has one generated attribute parser and printer, so the -// release's attributes, the compiler's and its enum attributes are generated -// together from here. - -#ifndef MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD -#define MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD - -include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" -include "mlir/dialects/cute_ir/CuteAttrsCompiler.td" -include "mlir/dialects/cute_ir/CuteEnums.td" - -#endif // MLIR_DIALECTS_CUTE_IR_CUTEATTRSALL_TD diff --git a/mlir/dialects/cute_ir/CuteAttrsCompiler.td b/mlir/dialects/cute_ir/CuteAttrsCompiler.td deleted file mode 100644 index 874763a5..00000000 --- a/mlir/dialects/cute_ir/CuteAttrsCompiler.td +++ /dev/null @@ -1,220 +0,0 @@ -// The cute dialect's attributes beyond the release, generated from the -// nvidia-cutlass-dsl 4.8.0 release; do not edit. -// -// Sources: the compiler library (`_cutlass_ir`): its generated parser -// messages give every class, its parameters in declaration order and their -// C++ types; the live compiler, loaded from Python, gives the syntax (what it -// prints for what it parses) and the verifier messages. -// -// The fragment parameters are `FragmentAttrInterface` in the compiler, an -// interface neither the release nor this tree defines: they are plain -// attributes here (the cute_nvgpu `*_frg` attributes). -// -// The reduction_op enum attribute is in CuteEnums.td. - -#ifndef MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD -#define MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD - -include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" - -def Cute_BitLayoutAttr : Cute_Attr<"BitLayout", "bitlayout"> { - let summary = "Bit layout of a pointer's elements within a storage chunk"; - let description = [{ - A two-mode layout from (element bit, element index) to the bit offset in - a storage chunk. Used by `!cute.ptr` (`bitlayout<"...">`). - - ```mlir - #cute.bitlayout<<"(8,4):(1,8)">> - ``` - }]; - let parameters = (ins - Cute_Param<"LayoutAttr", "(element bit, element index) -> bit">:$layout - ); - let assemblyFormat = "`<` $layout `>`"; - let genVerifyDecl = 1; -} - -def Cute_CopyAtomAttr : Cute_Attr<"CopyAtom", "copy_atom"> { - let summary = "Thread and value layouts of a copy atom"; - let description = [{ - ```mlir - #cute.copy_atom, layout_tv = (<"(1,8):(0,1)">, <"(1,8):(0,1)">, <"(1,8):(0,1)">)> - ``` - }]; - let parameters = (ins - Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, - Cute_Param<"LayoutAttr", "source thread-value layout">:$layout_src, - Cute_Param<"LayoutAttr", "destination thread-value layout">:$layout_dst, - Cute_Param<"LayoutAttr", "reference thread-value layout">:$layout_ref - ); - let assemblyFormat = [{ - `<` `thr_id` `=` $thr_id `,` `layout_tv` `=` - `(` $layout_src `,` $layout_dst `,` $layout_ref `)` `>` - }]; -} - -def Cute_CopyAtomV2Attr : Cute_Attr<"CopyAtomV2", "copy_atom_v2"> { - let summary = "Thread and value layouts and fragments of a V2 copy atom"; - let description = [{ - ```mlir - #cute.copy_atom_v2, layout_tv = (<"(1,8):(0,1)">, <"(1,8):(0,1)">), frg = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> - ``` - }]; - let parameters = (ins - Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, - Cute_Param<"LayoutAttr", "source thread-value layout">:$layoutSrc_TV, - Cute_Param<"LayoutAttr", "destination thread-value layout">:$layoutDst_TV, - Cute_Param<"mlir::Attribute", "source fragment">:$frgSrc, - Cute_Param<"mlir::Attribute", "destination fragment">:$frgDst - ); - let assemblyFormat = [{ - `<` `thr_id` `=` $thr_id `,` `layout_tv` `=` - `(` $layoutSrc_TV `,` $layoutDst_TV `)` `,` - `frg` `=` `(` $frgSrc `,` $frgDst `)` `>` - }]; -} - -def Cute_MmaAtomAttr : Cute_Attr<"MmaAtom", "mma_atom"> { - let summary = "Shapes, thread-value layouts and fragments of an MMA atom"; - let description = [{ - ```mlir - #cute.mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> - ``` - }]; - let parameters = (ins - Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, - Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, - Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, - Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, - Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, - Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, - Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, - Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, - Cute_Param<"mlir::Attribute", "A fragment">:$frgA, - Cute_Param<"mlir::Attribute", "B fragment">:$frgB, - Cute_Param<"mlir::Attribute", "C fragment">:$frgC - ); - let assemblyFormat = [{ - `<` $shape_MNK `,` $thr_id `,` - `layout_TV` `=` `(` $layoutA_TV `,` $layoutB_TV `,` $layoutC_TV `)` `,` - `shapes` `=` `(` $shapeA_MK `,` $shapeB_NK `,` $shapeC_MN `)` - `frgs` `=` `(` $frgA `,` $frgB `,` $frgC `)` `>` - }]; -} - -def Cute_MxMmaAtomAttr : Cute_Attr<"MxMmaAtom", "mx_mma_atom"> { - let summary = "MMA atom attribute with scale-factor operands"; - let description = [{ - Operands in the order A, SFA, B, SFB, C. - - ```mlir - #cute.mx_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">, <"1">, <"2">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> - ``` - }]; - let parameters = (ins - Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, - Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, - Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, - Cute_Param<"LayoutAttr", "SFA thread-value layout">:$layoutSFA_TV, - Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, - Cute_Param<"LayoutAttr", "SFB thread-value layout">:$layoutSFB_TV, - Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, - Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, - Cute_Param<"ShapeAttr", "SFA shape (MK)">:$shapeSFA_MK, - Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, - Cute_Param<"ShapeAttr", "SFB shape (NK)">:$shapeSFB_NK, - Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, - Cute_Param<"mlir::Attribute", "A fragment">:$frgA, - Cute_Param<"mlir::Attribute", "SFA fragment">:$frgSFA, - Cute_Param<"mlir::Attribute", "B fragment">:$frgB, - Cute_Param<"mlir::Attribute", "SFB fragment">:$frgSFB, - Cute_Param<"mlir::Attribute", "C fragment">:$frgC - ); - let assemblyFormat = [{ - `<` $shape_MNK `,` $thr_id `,` - `layout_TV` `=` `(` $layoutA_TV `,` $layoutSFA_TV `,` $layoutB_TV `,` - $layoutSFB_TV `,` $layoutC_TV `)` `,` - `shapes` `=` `(` $shapeA_MK `,` $shapeSFA_MK `,` $shapeB_NK `,` - $shapeSFB_NK `,` $shapeC_MN `)` - `frgs` `=` `(` $frgA `,` $frgSFA `,` $frgB `,` $frgSFB `,` $frgC `)` `>` - }]; -} - -def Cute_SparseMmaAtomAttr : Cute_Attr<"SparseMmaAtom", "sparse_mma_atom"> { - let summary = "MMA atom attribute with a sparsity-metadata operand"; - let description = [{ - Operands in the order A, B, C, E. - - ```mlir - #cute.sparse_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"1:0">, <"1:0">, <"1:0">), shapes = (<"(16,16)">, <"(8,16)">, <"(16,8)">, <"1">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> - ``` - }]; - let parameters = (ins - Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, - Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, - Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, - Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, - Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, - Cute_Param<"LayoutAttr", "E thread-value layout">:$layoutE_TV, - Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, - Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, - Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, - Cute_Param<"ShapeAttr", "E shape (MK)">:$shapeE_MK, - Cute_Param<"mlir::Attribute", "A fragment">:$frgA, - Cute_Param<"mlir::Attribute", "B fragment">:$frgB, - Cute_Param<"mlir::Attribute", "C fragment">:$frgC, - Cute_Param<"mlir::Attribute", "E fragment">:$frgE - ); - let assemblyFormat = [{ - `<` $shape_MNK `,` $thr_id `,` - `layout_TV` `=` `(` $layoutA_TV `,` $layoutB_TV `,` $layoutC_TV `,` - $layoutE_TV `)` `,` - `shapes` `=` `(` $shapeA_MK `,` $shapeB_NK `,` $shapeC_MN `,` $shapeE_MK `)` - `frgs` `=` `(` $frgA `,` $frgB `,` $frgC `,` $frgE `)` `>` - }]; -} - -def Cute_SparseMxMmaAtomAttr - : Cute_Attr<"SparseMxMmaAtom", "sparse_mx_mma_atom"> { - let summary = "MMA atom attribute with scale-factor and metadata operands"; - let description = [{ - Operands in the order A, SFA, B, SFB, C, E. - - ```mlir - #cute.sparse_mx_mma_atom<<"(16,8,16)">, <"32:1">, layout_TV = (<"1:0">, <"2:0">, <"3:0">, <"4:0">, <"5:0">, <"6:0">), shapes = (<"1">, <"2">, <"3">, <"4">, <"5">, <"6">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)> - ``` - }]; - let parameters = (ins - Cute_Param<"ShapeAttr", "MNK shape">:$shape_MNK, - Cute_Param<"LayoutAttr", "thread id layout">:$thr_id, - Cute_Param<"LayoutAttr", "A thread-value layout">:$layoutA_TV, - Cute_Param<"LayoutAttr", "SFA thread-value layout">:$layoutSFA_TV, - Cute_Param<"LayoutAttr", "B thread-value layout">:$layoutB_TV, - Cute_Param<"LayoutAttr", "SFB thread-value layout">:$layoutSFB_TV, - Cute_Param<"LayoutAttr", "C thread-value layout">:$layoutC_TV, - Cute_Param<"LayoutAttr", "E thread-value layout">:$layoutE_TV, - Cute_Param<"ShapeAttr", "A shape (MK)">:$shapeA_MK, - Cute_Param<"ShapeAttr", "SFA shape (MK)">:$shapeSFA_MK, - Cute_Param<"ShapeAttr", "B shape (NK)">:$shapeB_NK, - Cute_Param<"ShapeAttr", "SFB shape (NK)">:$shapeSFB_NK, - Cute_Param<"ShapeAttr", "C shape (MN)">:$shapeC_MN, - Cute_Param<"ShapeAttr", "E shape (MK)">:$shapeE_MK, - Cute_Param<"mlir::Attribute", "A fragment">:$frgA, - Cute_Param<"mlir::Attribute", "SFA fragment">:$frgSFA, - Cute_Param<"mlir::Attribute", "B fragment">:$frgB, - Cute_Param<"mlir::Attribute", "SFB fragment">:$frgSFB, - Cute_Param<"mlir::Attribute", "C fragment">:$frgC, - Cute_Param<"mlir::Attribute", "E fragment">:$frgE - ); - let assemblyFormat = [{ - `<` $shape_MNK `,` $thr_id `,` - `layout_TV` `=` `(` $layoutA_TV `,` $layoutSFA_TV `,` $layoutB_TV `,` - $layoutSFB_TV `,` $layoutC_TV `,` $layoutE_TV `)` `,` - `shapes` `=` `(` $shapeA_MK `,` $shapeSFA_MK `,` $shapeB_NK `,` - $shapeSFB_NK `,` $shapeC_MN `,` $shapeE_MK `)` - `frgs` `=` `(` $frgA `,` $frgSFA `,` $frgB `,` $frgSFB `,` $frgC `,` - $frgE `)` `>` - }]; -} - -#endif // MLIR_DIALECTS_CUTE_IR_CUTEATTRSCOMPILER_TD diff --git a/mlir/dialects/cute_ir/CuteEnums.td b/mlir/dialects/cute_ir/CuteEnums.td deleted file mode 100644 index 68018cc1..00000000 --- a/mlir/dialects/cute_ir/CuteEnums.td +++ /dev/null @@ -1,70 +0,0 @@ -// The cute dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 -// enum bindings (`_cute_enum_gen.py`); do not edit. -// -// Case values and spellings are the compiler's. An enum the bindings build -// as a dialect attribute also has its EnumAttr, printed `#`. - -#ifndef MLIR_DIALECTS_CUTE_IR_CUTEENUMS_TD -#define MLIR_DIALECTS_CUTE_IR_CUTEENUMS_TD - -include "cute_ir/Dialect/Cute/IR/CuteDialect.td" -include "mlir/IR/EnumAttr.td" - -def Cute_AddressSpaceEnum : I32EnumAttr<"AddressSpace", "Address spaces for CuTe memrefs and pointers", [ - I32EnumAttrCase<"generic", 0, "generic">, - I32EnumAttrCase<"gmem", 1, "gmem">, - I32EnumAttrCase<"smem", 3, "smem">, - I32EnumAttrCase<"rmem", 5, "rmem">, - I32EnumAttrCase<"tmem", 6, "tmem">, - I32EnumAttrCase<"dsmem", 7, "dsmem">, - I32EnumAttrCase<"cmem", 4, "cmem"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute"; - let genSpecializedAttr = 0; -} - -def Cute_CopyOperandEnum : I32EnumAttr<"CopyOperand", "Copy operand ID", [ - I32EnumAttrCase<"S", 0, "S">, - I32EnumAttrCase<"D", 1, "D"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute"; - let genSpecializedAttr = 0; -} - -def Cute_MajorModeEnum : I32EnumAttr<"MajorMode", "Major mode for MMA operations", [ - I32EnumAttrCase<"k", 0, "k">, - I32EnumAttrCase<"mn", 1, "mn"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute"; - let genSpecializedAttr = 0; -} - -def Cute_MmaOperandEnum : I32EnumAttr<"MmaOperand", "MMA operand ID", [ - I32EnumAttrCase<"A", 0, "A">, - I32EnumAttrCase<"B", 1, "B">, - I32EnumAttrCase<"C", 2, "C">, - I32EnumAttrCase<"D", 3, "D">, - I32EnumAttrCase<"SFA", 4, "SFA">, - I32EnumAttrCase<"SFB", 5, "SFB">, - I32EnumAttrCase<"E", 6, "E"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute"; - let genSpecializedAttr = 0; -} - -def Cute_ReductionOpEnum : I32EnumAttr<"ReductionOp", "Op for cute reduce operations", [ - I32EnumAttrCase<"ADD", 0, "ADD">, - I32EnumAttrCase<"MUL", 1, "MUL">, - I32EnumAttrCase<"MIN", 2, "MIN">, - I32EnumAttrCase<"MAX", 3, "MAX"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute"; - let genSpecializedAttr = 0; -} - -def Cute_ReductionOpAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute"; - let assemblyFormat = "`<` $value `>`"; -} - -#endif // MLIR_DIALECTS_CUTE_IR_CUTEENUMS_TD diff --git a/mlir/dialects/cute_ir/CuteInference.cpp b/mlir/dialects/cute_ir/CuteInference.cpp deleted file mode 100644 index 2f0415e4..00000000 --- a/mlir/dialects/cute_ir/CuteInference.cpp +++ /dev/null @@ -1,3600 +0,0 @@ -//===- CuteInference.cpp - result types the DSL compiler would infer ------===// -// -// Every operation the DSL compiler infers result types for, inferred here -// over cutegen, so a module built in this tree carries the types the compiler -// expects. Where the compiler's rule is not recovered for some inputs (other -// atoms, tilings or layouts than those covered), inference fails rather than -// guesses. test:dialect_inference_test holds these to the compiler's choice. - -#include "mlir/dialects/cute_ir/CuteInference.h" - -#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" - -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/Matchers.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/TypeSwitch.h" - -#include -#include -#include -#include -#include -#include -#include - -using namespace mlir; -using namespace mlir::cutlass_compiler::cute; -namespace cg = cutegen; -namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; -// The dialect's memref, not the builtin one. -using CuteMemRefType = ::mlir::cutlass_compiler::cute::MemRefType; - -namespace { - -// Composed layouts that turned out plain are typed as plain layouts, as the -// compiler types them. -Type layoutType(MLIRContext *ctx, cg::layout l) { - return LayoutType::get(ctx, std::move(l)); -} -Type layoutType(MLIRContext *ctx, cg::composed_layout l) { - if (l.is_normal_layout()) - return LayoutType::get(ctx, l.layout_b()); - return ComposedLayoutType::get(ctx, std::move(l)); -} - -// `f` runs over the algebra value of a layout or composed layout type; the -// result keeps the input's kind (see rebuild for views). -template Type mapLayout(MLIRContext *ctx, Type layout, F &&f) { - if (auto l = llvm::dyn_cast(layout)) { - auto r = f(l.getRef()); - if (!cg::is_valid(r)) - return {}; - return layoutType(ctx, std::move(r)); - } - if (auto l = llvm::dyn_cast(layout)) { - auto r = f(l.getRef()); - if (!cg::is_valid(r)) - return {}; - return ComposedLayoutType::get(ctx, std::move(r)); - } - return {}; -} - -// What carries a layout: the layout itself, a memref (pointer + layout) or a -// coordinate tensor (arithmetic tuple + layout). -struct Carrier { - Type layout; - PtrType ptr; // memref - IntTupleType iter; // coord_tensor - Type desc; // cute_nvgpu.smem_desc_view -}; - -std::optional carrierOf(Type t) { - if (llvm::isa(t)) - return Carrier{t, {}, {}, {}}; - if (auto m = llvm::dyn_cast(t)) - return Carrier{m.getLayout(), m.getPtr(), {}, {}}; - if (auto c = llvm::dyn_cast(t)) - return Carrier{c.getLayout(), {}, c.getArithTuple(), {}}; - if (auto v = llvm::dyn_cast(t)) - return Carrier{ - LayoutType::get(t.getContext(), v.getLayout()), {}, {}, v.getDesc()}; - return std::nullopt; -} - -Type rebuild(MLIRContext *ctx, const Carrier &c, Type layout) { - if (!layout) - return {}; - // A view's layout is plain when its composed layout turned out plain. - if (auto l = llvm::dyn_cast(layout); - l && (c.ptr || c.iter || c.desc)) - layout = layoutType(ctx, l.getRef()); - if (c.ptr) - return CuteMemRefType::get(ctx, c.ptr, layout); - if (c.iter) - return CoordTensorType::get(ctx, c.iter, layout); - if (c.desc) { - auto l = llvm::dyn_cast(layout); - return l ? cute_nvgpu::SmemDescViewType::get(ctx, c.desc, l.getAttr()) - : Type(); - } - return layout; -} - -// The bit width of an element type; tf32 counts as its 32-bit storage. -int64_t bitsOfType(Type t) { - if (t.isTF32()) - return 32; - return t.isIntOrFloat() ? t.getIntOrFloatBitWidth() : 0; -} - -// The alignment a pointer keeps after moving by `offset` elements: the gcd -// with what is known of the offset, a static value or a divisibility; the -// element's natural alignment when less than a byte is known. -uint64_t movedAlignment(PtrType ptr, const cg::int_tuple &offset) { - Type elem = ptr.getValueType(); - uint64_t bits = elem && elem.isIntOrFloat() ? bitsOfType(elem) : 8; - uint64_t natural = PtrType::getNaturalAlignment(elem); - uint64_t align = ptr.getAlignment() ? ptr.getAlignment() : natural; - uint64_t known = 0; - if (cg::holds_int(offset)) { - int64_t v = offset.as_int64(); - if (v == 0) - return ptr.getAlignment(); - known = static_cast(v < 0 ? -v : v); - } else if (cg::holds_dynamic_int(offset)) { - known = static_cast( - std::get(offset).get_properties().divisibility); - } else { - return natural; - } - uint64_t result = std::gcd(align * 8, known * bits) / 8; - return result ? result : natural; -} - -PtrType movedPtr(PtrType ptr, const cg::int_tuple &offset) { - return PtrType::get(ptr.getContext(), ptr.getValueType(), - ptr.getAddressSpace(), movedAlignment(ptr, offset), - ptr.getSwizzle(), ptr.getBitlayout()); -} - -// A coordinate with its underscores read as zero, for evaluating a layout. -std::optional zeroUnderscores(const cg::coord &c) { - std::string text = cg::to_string(c); - std::replace(text.begin(), text.end(), '_', '0'); - return cg::from_string(text); -} - -// Slicing a carrier: the layout is sliced and the iterator moves by the -// layout evaluated at the coordinate. -Type sliceCarrier(MLIRContext *ctx, const Carrier &c, const cg::coord &coord) { - cg::int_tuple offset; - Type layout; - if (auto l = llvm::dyn_cast(c.layout)) { - auto [sliced, off] = cg::slice_and_offset(coord, l.getRef()); - if (!cg::is_valid(sliced)) - return {}; - layout = layoutType(ctx, std::move(sliced)); - offset = off; - } else if (auto l = llvm::dyn_cast(c.layout)) { - // The composed layout's offset takes the move; the iterator stays. - auto sliced = cg::slice(coord, l.getRef()); - if (!cg::is_valid(sliced)) - return {}; - return rebuild(ctx, c, layoutType(ctx, std::move(sliced))); - } else { - return {}; - } - if (c.ptr) - return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); - if (c.iter) { - auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); - if (!cg::is_valid(sum)) - return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), - layout); - } - // A descriptor iterator does not move. - return rebuild(ctx, c, layout); -} - -// The compiler writes a stride of 0 on every mode of extent 1 that a divide -// produces; cutegen keeps the stride the algebra gave it. -cg::layout zeroUnitModes(cg::layout l) { - if (cg::holds_leaf(l.shape())) { - if (cg::holds_int(l.shape()) && l.shape().as_int64() == 1) { - if (auto unit = cg::from_string("1:0")) - return *unit; - } - return l; - } - // Shapes and strides are congruent, so walk both spellings together. - auto shape = cg::to_string(l.shape()); - auto stride = cg::to_string(l.stride()); - auto fs = cg::flatten(l.shape()); - auto ft = cg::flatten(l.stride()); - if (cg::rank(fs) != cg::rank(ft)) - return l; - bool changed = false; - for (size_t i = 0; i < cg::rank(fs); ++i) { - if (cg::holds_int(fs[i]) && fs[i].as_int64() == 1 && - !(cg::holds_int(ft[i]) && ft[i].as_int64() == 0)) { - changed = true; - } - } - if (!changed) - return l; - // Rewrite the stride text leaf by leaf, in flattened order. - std::string out; - size_t leaf = 0; - auto stridesAreLeaves = [&](size_t i) { - return cg::holds_int(fs[i]) && fs[i].as_int64() == 1; - }; - size_t i = 0; - while (i < stride.size()) { - char ch = stride[i]; - if (ch == '(' || ch == ')' || ch == ',') { - out += ch; - ++i; - continue; - } - size_t j = i; - int depth = 0; - while (j < stride.size() && - !((stride[j] == ',' || stride[j] == ')') && depth == 0)) { - if (stride[j] == '<' || stride[j] == '{') - ++depth; - if (stride[j] == '>' || stride[j] == '}') - --depth; - ++j; - } - out += stridesAreLeaves(leaf) ? std::string("0") : stride.substr(i, j - i); - ++leaf; - i = j; - } - auto rebuilt = cg::from_string(shape + ":" + out); - return rebuilt ? *rebuilt : l; -} - -const cg::layout &outerOf(const cg::layout &l) { return l; } -cg::layout outerOf(const cg::composed_layout &l) { return l.layout_b(); } - -cg::composed_layout zeroUnitModes(cg::composed_layout l) { - if (l.is_a_swizzle()) - return cg::composed_layout(l.swizzle_a(), l.offset(), - zeroUnitModes(l.layout_b())); - return cg::composed_layout(l.layout_a(), l.offset(), - zeroUnitModes(l.layout_b())); -} - -template -LogicalResult infer(std::optional loc, SmallVectorImpl &out, - F &&f) { - Type t = f(); - if (!t) - return emitOptionalError(loc, "cannot infer the result type"); - out.push_back(t); - return success(); -} - -// `like`'s nesting with its leaves replaced, in order, by `leaves`. -template -std::string nestLike(const T &like, llvm::ArrayRef leaves, - size_t &i) { - if (!cg::holds_vector(like)) - return i < leaves.size() ? leaves[i++] : "x"; - std::string s = "("; - for (size_t k = 0; k < cg::rank(like); ++k) - s += (k ? "," : "") + nestLike(like[k], leaves, i); - return s + ")"; -} - -template void leavesOf(const T &t, std::vector &out) { - if (!cg::holds_vector(t)) - return out.push_back(cg::to_string(t)); - for (size_t k = 0; k < cg::rank(t); ++k) - leavesOf(t[k], out); -} - -bool isOne(const std::string &leaf) { return leaf == "1"; } - -// The plain layout of a layout, or the outer layout of a composed one. -std::optional outerLayout(Type t) { - if (auto l = llvm::dyn_cast(t)) - return l.getRef(); - if (auto c = llvm::dyn_cast(t)) - return c.getRef().layout_b(); - return std::nullopt; -} - -// The plain layout of a layout or swizzled composed layout. -std::optional swizzledOuter(Type t) { - if (auto l = llvm::dyn_cast(t)) - return l.getRef(); - auto c = llvm::dyn_cast(t); - if (c && c.getRef().is_a_swizzle()) - return c.getRef().layout_b(); - return std::nullopt; -} - -// A dynamic integer's spelling: `?`, with its width when 64 and its -// divisibility when above 1. -std::string dynamicInt(int64_t div, bool wide) { - std::string props = std::string(wide ? "i64" : "") + - (wide && div > 1 ? " " : "") + - (div > 1 ? "div=" + std::to_string(div) : ""); - return props.empty() ? "?" : "?{" + props + "}"; -} - -// idx2crd of a scalar `idx` over `shape` (column-major), keeping what a -// dynamic index's divisibility tells of each coordinate, as the compiler -// does (cutegen drops it): a divisible mode's coordinate is 0; a mode whose -// extent the divisibility divides keeps it; the rest know nothing. -std::optional splitIndex(const cg::int_tuple &idx, - const cg::shape &shape) { - std::vector leaves, crd; - leavesOf(shape, leaves); - bool known = cg::holds_int(idx); - if (!known && !cg::holds_dynamic_int(idx)) - return std::nullopt; - int64_t v = known ? idx.as_int64() : 0, div = 1; - bool wide = false; - if (!known) { - const auto &p = std::get(idx).get_properties(); - div = p.divisibility; - wide = p.width == 64; - } - for (size_t i = 0; i < leaves.size(); ++i) { - int64_t n = 0; - bool staticExtent = !StringRef(leaves[i]).getAsInteger(10, n) && n > 0; - bool last = i + 1 == leaves.size(); - if (known) { - if (!staticExtent) - return std::nullopt; - crd.push_back(std::to_string(last ? v : v % n)); - v /= n; - } else if (last) { - crd.push_back(dynamicInt(div, wide)); - } else if (staticExtent && div % n == 0) { - crd.push_back("0"); - div /= n; - } else { - crd.push_back(dynamicInt(staticExtent && n % div == 0 ? div : 1, wide)); - div = 1; - } - } - size_t i = 0; - return nestLike(shape, crd, i); -} -} // namespace - -// ---------------------------------------------------------------- cute - -LogicalResult AddOffsetOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto offset = llvm::dyn_cast(a.getOffset().getType()); - if (!offset) - return emitOptionalError(loc, "offset must be an int tuple"); - return infer(loc, out, [&]() -> Type { - Type src = a.getSrc().getType(); - if (auto p = llvm::dyn_cast(src)) - return movedPtr(p, offset.getRef()); - if (auto it = llvm::dyn_cast(src)) { - auto sum = cg::arith_tuple_sum(it.getArithTuple().getRef(), - offset.getRef()); - if (!cg::is_valid(sum)) - return {}; - return ArithTupleIteratorType::get(ctx, IntTupleType::get(ctx, sum)); - } - return {}; - }); -} - -namespace { -// Pads `input` to `rank` modes with `element`, after (append) or before. -template -LogicalResult inferToRank(MLIRContext *ctx, std::optional loc, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, SmallVectorImpl &out, - bool append) { - typename Op::Adaptor a(operands, attrs, props); - int32_t rank = a.getRank(); - Type in = a.getInput().getType(), e = a.getElement().getType(); - return infer(loc, out, [&]() -> Type { - if (rank <= 0) - return {}; - return llvm::TypeSwitch(in) - .Case([&](auto ty) -> Type { - using T = decltype(ty); - // A composed layout is padded with plain layouts. - using E = std::conditional_t, - LayoutType, T>; - auto el = llvm::dyn_cast(e); - if (!el) - return {}; - auto r = append - ? cg::append_to_rank_N(rank, ty.getRef(), el.getRef()) - : cg::prepend_to_rank_N(rank, ty.getRef(), el.getRef()); - if (!cg::is_valid(r)) - return {}; - return T::get(ctx, std::move(r)); - }) - .Default([](Type) -> Type { return {}; }); - }); -} -} // namespace - -LogicalResult AppendToRankOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferToRank(ctx, loc, operands, attrs, props, out, - true); -} - -LogicalResult PrependToRankOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferToRank(ctx, loc, operands, attrs, props, out, - false); -} - -Type mlir::cutlass_compiler::cute::assumeResultType(Type src) { - auto constrained = llvm::dyn_cast(src); - for (unsigned width : {64u, 32u}) - if (src.isInteger(width) || - (constrained && constrained.getWidth() == width)) - return ConstrainedIntType::get(src.getContext(), 1, width, false); - return {}; -} - -LogicalResult CeilDivOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - Type in = a.getInput().getType(), tiler = a.getTiler().getType(); - return infer(loc, out, [&]() -> Type { - // cutegen divides shapes; int tuples go through shape and back. - cg::shape shape; - bool tuple = false; - if (auto s = llvm::dyn_cast(in)) { - shape = s.getRef(); - } else if (auto t = llvm::dyn_cast(in)) { - shape = cg::rec_var_cast(t.getRef()); - tuple = true; - } else { - return {}; - } - cg::shape r; - if (auto t = llvm::dyn_cast(tiler)) { - r = cg::ceil_div(shape, t.getRef()); - } else if (auto t = llvm::dyn_cast(tiler)) { - r = cg::ceil_div(shape, t.getRef()); - } else if (auto t = llvm::dyn_cast(tiler)) { - r = cg::ceil_div(shape, cg::rec_var_cast(t.getRef())); - } else { - return {}; - } - if (!cg::is_valid(r)) - return {}; - if (tuple) - return IntTupleType::get(ctx, cg::rec_var_cast(r)); - return ShapeType::get(ctx, std::move(r)); - }); -} - -LogicalResult CoalesceOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto c = carrierOf(a.getInput().getType()); - if (!c) - return emitOptionalError(loc, "input carries no layout"); - CoordType profile; - if (a.getTargetProfile()) - profile = llvm::dyn_cast(a.getTargetProfile().getType()); - return infer(loc, out, [&]() -> Type { - return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { - return profile ? cg::coalesce(l, profile.getRef()) - : cg::coalesce(l); - })); - }); -} - -LogicalResult ComposedGetOuterOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - auto c = llvm::dyn_cast(operands[0].getType()); - if (!c) - return {}; - return LayoutType::get(ctx, c.getRef().layout_b()); - }); -} - -LogicalResult CompositionOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto c = carrierOf(a.getLhs().getType()); - if (!c) - return emitOptionalError(loc, "lhs carries no layout"); - Type rhs = a.getRhs().getType(); - return infer(loc, out, [&]() -> Type { - return rebuild(ctx, *c, - llvm::TypeSwitch(rhs) - .Case([&](auto r) { - return mapLayout(ctx, c->layout, [&](const auto &l) { - return cg::composition(l, r.getRef()); - }); - }) - .Default([](Type) -> Type { return {}; })); - }); -} - -LogicalResult Crd2IdxOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto coord = llvm::dyn_cast(a.getCoord().getType()); - if (!coord) - return emitOptionalError(loc, "coord must be a coord"); - return infer(loc, out, [&]() -> Type { - auto c = zeroUnderscores(coord.getRef()); - if (!c) - return {}; - Type layout = a.getLayout().getType(); - cg::int_tuple idx; - if (auto l = llvm::dyn_cast(layout)) { - idx = cg::layout_eval(*c, l.getRef()); - } else if (auto s = llvm::dyn_cast(layout)) { - idx = cg::crd2idx(*c, s.getRef()); - } else { - return {}; - } - if (!cg::is_valid(idx)) - return {}; - return IntTupleType::get(ctx, std::move(idx)); - }); -} - -LogicalResult DereferenceArithTupleIteratorOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - auto it = llvm::dyn_cast(operands[0].getType()); - return it ? it.getArithTuple() : Type{}; - }); -} - -LogicalResult DiceOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, PropertyRef props, - RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - const cg::coord &coord = a.getCoordAttr().getRef(); - Type in = a.getInput().getType(); - return infer(loc, out, [&]() -> Type { - if (auto c = carrierOf(in)) { - return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { - return cg::dice(coord, l); - })); - } - return llvm::TypeSwitch(in) - .Case( - [&](auto ty) -> Type { - auto r = cg::dice(coord, ty.getRef()); - if (!cg::is_valid(r)) - return {}; - return decltype(ty)::get(ctx, std::move(r)); - }) - .Default([](Type) -> Type { return {}; }); - }); -} - -Type mlir::cutlass_compiler::cute::fastDivmodComputeResultType(Type divisor) { - auto d = llvm::dyn_cast(divisor); - return d ? IntegerType::get(divisor.getContext(), d.getWidth()) : Type(); -} - -Type mlir::cutlass_compiler::cute::fastDivmodCreateDivisorResultType( - Type divisor) { - auto i = llvm::dyn_cast(divisor); - return i ? FastDivmodDivisorType::get(divisor.getContext(), i.getWidth(), - false) - : Type(); -} - -LogicalResult FilterZerosOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto c = carrierOf(a.getInput().getType()); - if (!c) - return emitOptionalError(loc, "input carries no layout"); - return infer(loc, out, [&]() -> Type { - return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { - return cg::filter_zeros(l); - })); - }); -} - -namespace { -// zeroUnitModes on all but the last `keep` modes (of mode 1 when `zipped`): -// the modes a divide leaves untouched keep their strides. -cg::layout zeroUnitModesHead(const cg::layout &l, size_t keep, bool zipped) { - if (!keep || !cg::holds_vector(l.shape())) - return zeroUnitModes(l); - if (zipped) { - if (cg::rank(l.shape()) != 2) - return zeroUnitModes(l); - return cg::make_layout( - std::vector{zeroUnitModes(cg::get(l, 0)), - zeroUnitModesHead(cg::get(l, 1), keep, false)}); - } - size_t n = cg::rank(l.shape()); - if (keep > n) - return l; - std::vector modes; - for (size_t i = 0; i < n; ++i) - modes.push_back(i < n - keep ? zeroUnitModes(cg::get(l, i)) - : cg::get(l, i)); - return cg::make_layout(modes); -} -cg::composed_layout zeroUnitModesHead(const cg::composed_layout &l, size_t keep, - bool zipped) { - cg::layout b = zeroUnitModesHead(l.layout_b(), keep, zipped); - if (l.is_a_swizzle()) - return cg::composed_layout(l.swizzle_a(), l.offset(), b); - return cg::composed_layout(l.layout_a(), l.offset(), b); -} - -template -LogicalResult inferDivide(MLIRContext *ctx, std::optional loc, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, SmallVectorImpl &out, - F &÷, bool zipped = false) { - typename Op::Adaptor a(operands, attrs, props); - auto c = carrierOf(a.getInput().getType()); - if (!c) - return emitOptionalError(loc, "input carries no layout"); - Type tiler = a.getTiler().getType(); - return infer(loc, out, [&]() -> Type { - return rebuild( - ctx, *c, - llvm::TypeSwitch(tiler) - .template Case([&](auto t) { - // Unit modes the tiler passes through (`_` or extent 1) keep - // their nonzero strides, which zeroUnitModes does not model: not - // covered. - if constexpr (std::is_same_v) { - auto outer = outerLayout(c->layout); - for (size_t i = 0; outer && cg::holds_vector(t.getRef()) && - i < cg::rank(t.getRef()) && - cg::holds_vector(outer->shape()) && - i < cg::rank(outer->shape()); - ++i) { - std::string m = cg::to_string(t.getRef()[i]); - auto tl = cg::from_string(m); - bool pass = m == "_" || (tl && cg::holds_int(cg::size(*tl)) && - cg::size(*tl).as_int64() == 1 && - cg::to_string(tl->stride()) != "0"); - std::vector shapes, strides; - leavesOf(outer->shape()[i], shapes); - leavesOf(outer->stride()[i], strides); - for (size_t k = 0; - pass && k < shapes.size() && k < strides.size(); ++k) - if (shapes[k] == "1" && strides[k] != "0") - return Type(); - } - } - return mapLayout(ctx, c->layout, [&](const auto &l) { - // A tile or shape tiler leaves the input's trailing modes - // alone. - size_t keep = 0; - if constexpr (!std::is_same_v) { - size_t in = cg::rank(outerOf(l).shape()), - r = cg::rank(t.getRef()); - if (cg::holds_vector(outerOf(l).shape()) && r < in) - keep = in - r; - } - return zeroUnitModesHead(divide(l, t.getRef()), keep, zipped); - }); - }) - .Default([](Type) -> Type { return {}; })); - }); -} -} // namespace - -LogicalResult FlatDivideOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferDivide( - ctx, loc, operands, attrs, props, out, - [](const auto &l, const auto &t) { return cg::flat_divide(l, t); }); -} - -LogicalResult LogicalDivideOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferDivide( - ctx, loc, operands, attrs, props, out, - [](const auto &l, const auto &t) { return cg::logical_divide(l, t); }); -} - -LogicalResult TiledDivideOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferDivide( - ctx, loc, operands, attrs, props, out, - [](const auto &l, const auto &t) { return cg::tiled_divide(l, t); }); -} - -LogicalResult ZippedDivideOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferDivide( - ctx, loc, operands, attrs, props, out, - [](const auto &l, const auto &t) { return cg::zipped_divide(l, t); }, - true); -} - -Type mlir::cutlass_compiler::cute::getResultType(Type in, - ArrayRef mode) { - MLIRContext *ctx = in.getContext(); - return llvm::TypeSwitch(in) - .Case([&](auto ty) -> Type { - auto r = cg::get(ty.getRef(), mode); - if (!cg::is_valid(r)) - return {}; - if constexpr (std::is_same_v) - return layoutType(ctx, std::move(r)); - else - return decltype(ty)::get(ctx, std::move(r)); - }) - .Default([](Type) -> Type { return {}; }); -} - -LogicalResult GetIterOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - Type src = operands[0].getType(); - if (auto m = llvm::dyn_cast(src)) - return m.getPtr(); - if (auto c = llvm::dyn_cast(src)) - return ArithTupleIteratorType::get(ctx, c.getArithTuple()); - if (auto v = llvm::dyn_cast(src)) - return v.getDesc(); - return {}; - }); -} - -LogicalResult GetLayoutOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - auto c = carrierOf(operands[0].getType()); - return c ? c->layout : Type{}; - }); -} - -LogicalResult GetLeavesOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - return llvm::TypeSwitch(operands[0].getType()) - .Case( - [&](auto ty) -> LogicalResult { - auto flat = cg::flatten(ty.getRef()); - for (size_t i = 0; i < cg::rank(flat); ++i) - out.push_back(decltype(ty)::get(ctx, flat[i])); - return success(); - }) - .Case([&](TileType ty) -> LogicalResult { - // A tile's leaves: its layouts, and `_` as a tile. - auto flat = cg::flatten(ty.getRef()); - for (size_t i = 0; i < cg::rank(flat); ++i) { - flat[i].visit([&](const auto &e) { - using T = std::decay_t; - if constexpr (cg::is_layout::value) - out.push_back(LayoutType::get(ctx, e)); - else if constexpr (!std::is_same_v>) - out.push_back(TileType::get(ctx, flat[i])); - }); - } - return success(); - }) - .Default([&](Type t) -> LogicalResult { - return emitOptionalError(loc, "cannot take the leaves of ", t); - }); -} - -LogicalResult GetScalarsOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - bool onlyDynamic = a.getOnlyDynamic(); - return llvm::TypeSwitch(a.getCuteValue().getType()) - .Case([&](auto ty) -> LogicalResult { - SmallVector scalars; - cg::collect_scalar_types(ctx, ty.getRef(), scalars, onlyDynamic); - out.append(scalars.begin(), scalars.end()); - return success(); - }) - .Default([&](Type t) -> LogicalResult { - return emitOptionalError(loc, "cannot take the scalars of ", t); - }); -} - -LogicalResult GetShapeOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - Type in = operands[0].getType(); - if (auto t = llvm::dyn_cast(in)) - return ShapeType::get(ctx, cg::extract_shape_from_tile(t.getRef())); - auto c = carrierOf(in); - if (!c) - return {}; - if (auto l = llvm::dyn_cast(c->layout)) - return ShapeType::get(ctx, l.getRef().shape()); - if (auto l = llvm::dyn_cast(c->layout)) - return ShapeType::get(ctx, l.getRef().shape()); - return {}; - }); -} - -LogicalResult GroupModesOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto c = carrierOf(a.getInput().getType()); - if (!c) - return emitOptionalError(loc, "input carries no layout"); - int begin = static_cast(a.getBegin()), - end = static_cast(a.getEnd()); - return infer(loc, out, [&]() -> Type { - return rebuild(ctx, *c, mapLayout(ctx, c->layout, [&](const auto &l) { - return cg::group(begin, end, l); - })); - }); -} - -LogicalResult LocalTileOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto c = carrierOf(a.getInput().getType()); - auto tile = llvm::dyn_cast(a.getTile().getType()); - auto coord = llvm::dyn_cast(a.getCoord().getType()); - if (!c || !tile || !coord) - return emitOptionalError(loc, - "expects a layout carrier, a tile and a coord"); - return infer(loc, out, [&]() -> Type { - // CuTe's inner_partition: divide by the tile, take the coordinate in the - // rest modes, and flatten the two modes into one rank. - cg::tile tiler = tile.getRef(); - cg::coord crd = coord.getRef(); - if (CoordAttr proj = a.getProjAttr()) { - tiler = cg::dice(proj.getRef(), tiler); - crd = cg::dice(proj.getRef(), crd); - } - auto layout = llvm::dyn_cast(c->layout); - if (!layout) - return {}; - cg::layout zipped = cg::zipped_divide(layout.getRef(), tiler); - if (!cg::is_valid(zipped)) - return {}; - cg::layout tiles = cg::get(zipped, 0), rest = cg::get(zipped, 1); - auto [sliced, offset] = cg::slice_and_offset(crd, rest); - if (!cg::is_valid(sliced)) - return {}; - // One rank: the tile modes, then what is left of the rest modes. - std::vector modes; - for (size_t i = 0; i < cg::rank(tiles.shape()); ++i) - modes.push_back(cg::get(tiles, i)); - if (cg::holds_leaf(sliced.shape())) { - modes.push_back(sliced); - } else { - for (size_t i = 0; i < cg::rank(sliced.shape()); ++i) - modes.push_back(cg::get(sliced, i)); - } - // The input's modes beyond the tiler that the coordinate keeps are - // untouched and keep their strides. - size_t keep = 0, in = cg::rank(layout.getRef().shape()), - r = cg::rank(tiler); - for (size_t i = r; cg::holds_vector(crd) && i < in && i < cg::rank(crd); - ++i) - keep += cg::to_string(crd[i]) == "_"; - cg::layout flat = zeroUnitModesHead(cg::make_layout(modes), keep, false); - Type result = layoutType(ctx, std::move(flat)); - if (c->ptr) - return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); - if (c->iter) { - auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); - if (!cg::is_valid(sum)) - return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), - result); - } - return result; - }); -} - -Type mlir::cutlass_compiler::cute::makeArithTupleIterResultType(Type tuple) { - auto t = llvm::dyn_cast(tuple); - return t ? ArithTupleIteratorType::get(tuple.getContext(), t) : Type(); -} - -LogicalResult MakeComposedLayoutOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto offset = llvm::dyn_cast(a.getOffset().getType()); - auto outer = llvm::dyn_cast(a.getOuter().getType()); - if (!offset || !outer) - return emitOptionalError(loc, "expects an int tuple offset and a layout"); - return infer(loc, out, [&]() -> Type { - Type inner = a.getInner().getType(); - if (auto l = llvm::dyn_cast(inner)) - return ComposedLayoutType::get( - ctx, - cg::composed_layout(l.getRef(), offset.getRef(), outer.getRef())); - if (auto s = llvm::dyn_cast(inner)) - return ComposedLayoutType::get( - ctx, - cg::composed_layout(s.getRef(), offset.getRef(), outer.getRef())); - return {}; - }); -} - -LogicalResult MakeIdentityLayoutOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - auto s = llvm::dyn_cast(operands[0].getType()); - if (!s) - return {}; - cg::layout l = cg::make_identity_layout(s.getRef()); - return cg::is_valid(l) ? LayoutType::get(ctx, std::move(l)) : Type{}; - }); -} - -LogicalResult MakeOrderedLayoutOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto shape = llvm::dyn_cast(a.getShape().getType()); - auto order = llvm::dyn_cast(a.getOrder().getType()); - if (!shape || !order) - return emitOptionalError(loc, "expects a shape and an order"); - return infer(loc, out, [&]() -> Type { - cg::layout l(shape.getRef(), - cg::compact_order(shape.getRef(), order.getRef())); - return cg::is_valid(l) ? LayoutType::get(ctx, std::move(l)) : Type{}; - }); -} - -Type mlir::cutlass_compiler::cute::makeViewResultType(Type iter, Type layout) { - MLIRContext *ctx = iter.getContext(); - if (!llvm::isa(layout)) - return {}; - if (auto p = llvm::dyn_cast(iter)) - return CuteMemRefType::get(ctx, p, layout); - if (auto it = llvm::dyn_cast(iter)) - return CoordTensorType::get(ctx, it.getArithTuple(), layout); - auto plain = llvm::dyn_cast(layout); - if (plain && llvm::isa(iter)) - return cute_nvgpu::SmemDescViewType::get(ctx, iter, plain.getAttr()); - return {}; -} - -LogicalResult -MemRefLoadOp::inferReturnTypes(MLIRContext *ctx, std::optional loc, - ValueRange operands, DictionaryAttr, PropertyRef, - RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - auto m = llvm::dyn_cast(operands[0].getType()); - return m ? m.getValueType() : Type{}; - }); -} - -LogicalResult MemRefLoadVecOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - return infer(loc, out, [&]() -> Type { - auto m = llvm::dyn_cast(a.getSrc().getType()); - if (!m) - return {}; - auto l = llvm::dyn_cast(m.getLayout()); - if (!l) - return {}; - auto n = cg::size(l.getRef()); - if (!cg::holds_int(n)) - return {}; - return VectorType::get({n.as_int64()}, m.getValueType()); - }); -} - -LogicalResult SizeOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, PropertyRef props, - RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - ArrayRef mode; - if (auto m = a.getModeAttr()) - mode = m.asArrayRef(); - Type in = a.getInput().getType(); - if (auto c = carrierOf(in)) - in = c->layout; - return infer(loc, out, [&]() -> Type { - return llvm::TypeSwitch(in) - .Case( - [&](auto ty) -> Type { - auto r = cg::rec_var_cast( - cg::size(cg::get(ty.getRef(), mode))); - if (!cg::is_valid(r)) - return {}; - return IntTupleType::get(ctx, std::move(r)); - }) - .Default([](Type) -> Type { return {}; }); - }); -} - -LogicalResult SliceOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, PropertyRef props, - RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto coord = llvm::dyn_cast(a.getCoord().getType()); - if (!coord) - return emitOptionalError(loc, "coord must be a coord"); - Type in = a.getInput().getType(); - return infer(loc, out, [&]() -> Type { - if (auto c = carrierOf(in)) - return sliceCarrier(ctx, *c, coord.getRef()); - return llvm::TypeSwitch(in) - .Case( - [&](auto ty) -> Type { - auto r = cg::slice(coord.getRef(), ty.getRef()); - if (!cg::is_valid(r)) - return {}; - return decltype(ty)::get(ctx, std::move(r)); - }) - .Default([](Type) -> Type { return {}; }); - }); -} - -LogicalResult TileToShapeOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto shape = llvm::dyn_cast(a.getTrgShape().getType()); - auto order = llvm::dyn_cast(a.getOrdShape().getType()); - if (!shape || !order) - return emitOptionalError(loc, "expects a shape and an order"); - return infer(loc, out, [&]() -> Type { - return mapLayout(ctx, a.getBlock().getType(), [&](const auto &l) { - return cg::tile_to_shape(l, shape.getRef(), order.getRef()); - }); - }); -} - -LogicalResult -ToIntTupleOp::inferReturnTypes(MLIRContext *ctx, std::optional loc, - ValueRange operands, DictionaryAttr, PropertyRef, - RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - return llvm::TypeSwitch(operands[0].getType()) - .Case( - [&](auto ty) -> Type { - return IntTupleType::get( - ctx, cg::rec_var_cast(ty.getRef())); - }) - .Default([](Type) -> Type { return {}; }); - }); -} - -namespace { -template -LogicalResult inferTupleArith(MLIRContext *ctx, std::optional loc, - ValueRange operands, SmallVectorImpl &out, - F &&f) { - return infer(loc, out, [&]() -> Type { - return llvm::TypeSwitch(operands[0].getType()) - .Case([&](auto lhs) -> Type { - using T = decltype(lhs); - auto rhs = llvm::dyn_cast(operands[1].getType()); - if (!rhs) - return {}; - auto r = f.template operator()(lhs.getRef(), - rhs.getRef()); - if (!cg::is_valid(r)) - return {}; - return T::get(ctx, std::move(r)); - }) - .Default([](Type) -> Type { return {}; }); - }); -} -} // namespace - -LogicalResult TupleAddOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - return inferTupleArith(ctx, loc, operands, out, - [](const auto &a, const auto &b) { - return cg::arith_tuple_sum(a, b); - }); -} - -LogicalResult TupleMulOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - return inferTupleArith(ctx, loc, operands, out, - [](const auto &a, const auto &b) { - return cg::arith_tuple_mul(a, b); - }); -} - -LogicalResult TupleDivOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - return inferTupleArith(ctx, loc, operands, out, - [](const auto &a, const auto &b) { - return cg::arith_tuple_div(a, b); - }); -} - -// Without a stride the layout is compact column-major, unit modes at 0. -Type mlir::cutlass_compiler::cute::makeLayoutResultType(Type shape, - Type stride) { - auto s = llvm::dyn_cast(shape); - if (!s) - return {}; - MLIRContext *ctx = shape.getContext(); - if (stride) { - auto st = llvm::dyn_cast(stride); - if (!st) - return {}; - return LayoutType::get(ctx, cg::layout(s.getRef(), st.getRef())); - } - cg::layout l(s.getRef(), - cg::compact_col_major(s.getRef())); - return LayoutType::get(ctx, zeroUnitModes(l)); -} - -namespace { -// `MxNxK` and `num_cta` of an SM100 block-scaled or plain mma atom. -struct MmaAtom { - int64_t m = 0, n = 0, k = 0, ctas = 1; - Type c; // accumulator element - bool aTmem = false; // frag_kind = ts -}; -std::optional mmaAtomOf(Type tiledMma) { - auto tiled = llvm::dyn_cast(tiledMma); - if (!tiled) - return std::nullopt; - // One atom, not permuted: other tilings are not covered here. - auto n = cg::size(tiled.getAtomLayout_MNK().getRef()); - TileAttr perm = tiled.getPermutation_MNK(); - if (!cg::holds_int(n) || n.as_int64() != 1 || - (perm && cg::to_string(perm.getRef()) != "[_;_;_]")) - return std::nullopt; - MmaAtom atom; - ShapeAttr shape; - if (auto bs = llvm::dyn_cast( - tiled.getMmaAtom())) { - shape = bs.getShapeMnk(); - atom.ctas = bs.getNumCta(); - atom.c = bs.getCType(); - atom.aTmem = bs.getAFragKind() == cute_nvgpu::MmaFragKind::tmem; - } else if (auto mma = llvm::dyn_cast( - tiled.getMmaAtom())) { - shape = mma.getShapeMnk(); - atom.ctas = mma.getNumCta(); - atom.c = mma.getCType(); - atom.aTmem = mma.getAFragKind() == cute_nvgpu::MmaFragKind::tmem; - } else { - return std::nullopt; - } - if (cg::rank(shape.getRef()) != 3) - return std::nullopt; - int64_t *mnk[] = {&atom.m, &atom.n, &atom.k}; - for (int i = 0; i < 3; ++i) { - auto e = cg::get(shape.getRef(), i); - if (!cg::holds_int(e)) - return std::nullopt; - *mnk[i] = e.as_int64(); - } - return atom; -} -} // namespace - -// partition_A/B/C of the trivial tiled mma: the leading two modes are divided -// by the per-CTA atom shape, the CTA's share of the first is taken at the -// thread coordinate, and what remains follows the atom modes. -LogicalResult TiledMmaPartitionOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto atom = mmaAtomOf(a.getTiledMma().getType()); - auto c = carrierOf(a.getInput().getType()); - auto coord = llvm::dyn_cast(a.getCoord().getType()); - auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); - if (!atom || !c || !coord || !which) - return emitOptionalError( - loc, "expects a tiled mma, a tensor, a coord and an operand"); - return infer(loc, out, [&]() -> Type { - auto layout = llvm::dyn_cast(c->layout); - if (!layout) - return {}; - int64_t a0, a1; - switch (which.getInt()) { - case 0: - a0 = atom->m / atom->ctas; - a1 = atom->k; - break; // A: (M, K) - case 1: - a0 = atom->n / atom->ctas; - a1 = atom->k; - break; // B: (N, K) - case 2: - a0 = atom->m / atom->ctas; - a1 = atom->n; - break; // C: (M, N) - default: - return {}; - } - const cg::layout &l = layout.getRef(); - if (cg::rank(l.shape()) < 2) - return {}; - auto tile = cg::from_string("[" + std::to_string(a0) + ":1;" + - std::to_string(a1) + ":1]"); - if (!tile) - return {}; - cg::layout two = - cg::make_layout(std::vector{cg::get(l, 0), cg::get(l, 1)}); - cg::layout divided = cg::zipped_divide(two, *tile); - if (!cg::is_valid(divided)) - return {}; - cg::layout atoms = cg::get(divided, 0), rest = cg::get(divided, 1); - // The first rest mode holds the CTA index innermost. - auto ctaTile = - cg::from_string("[" + std::to_string(atom->ctas) + ":1]"); - if (!ctaTile) - return {}; - cg::layout rest0 = cg::zipped_divide(cg::get(rest, 0), *ctaTile); - if (!cg::is_valid(rest0) || cg::rank(rest0.shape()) != 2) - return {}; - // A lone CTA has nothing to select: its coordinate is 0 whatever the - // thread index says. - std::string which_cta = - atom->ctas == 1 ? "0" : cg::to_string(coord.getRef()); - auto [rest0Sliced, offset] = cg::slice_and_offset( - *cg::from_string("(" + which_cta + ",_)"), rest0); - if (!cg::is_valid(rest0Sliced)) - return {}; - // What the slice leaves is one mode, not a tuple around it. - while (cg::holds_vector(rest0Sliced.shape()) && - cg::rank(rest0Sliced.shape()) == 1) - rest0Sliced = cg::get(rest0Sliced, 0); - std::vector modes{atoms, rest0Sliced, cg::get(rest, 1)}; - for (size_t i = 2; i < cg::rank(l.shape()); ++i) - modes.push_back(cg::get(l, i)); - // Modes beyond M and K (or N) are untouched. - Type result = layoutType(ctx, zeroUnitModesHead(cg::make_layout(modes), - modes.size() - 3, false)); - if (c->ptr) - return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); - if (c->iter) { - auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); - if (!cg::is_valid(sum)) - return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), - result); - } - return result; - }); -} - -// The fragment an mma reads: for A and B a descriptor view over the staged -// smem tile, atom modes collapsed and strides in 16-byte units; for C the -// accumulator in tensor memory, one lane per row and one column per element. -LogicalResult MmaMakeFragmentOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); - auto atom = mmaAtomOf(a.getAtom().getType()); - // A in tensor memory (frag_kind = ts) is not covered. - if (!which || !atom || (which.getInt() == 0 && atom->aTmem)) - return emitOptionalError(loc, - "expects an SM100 tiled mma and an operand id"); - Type in = a.getInput().getType(); - return infer(loc, out, [&]() -> Type { - if (which.getInt() == 2) { - auto shape = llvm::dyn_cast(in); - if (!shape) - return {}; - const cg::shape &s = shape.getRef(); - if (cg::rank(s) < 1 || !cg::holds_vector(cg::get(s, 0))) - return {}; - auto m = cg::size(cg::get(s, 0), 0), n = cg::size(cg::get(s, 0), 1); - if (!cg::holds_int(m) || !cg::holds_int(n)) - return {}; - // ((M,N), rest...): rows on lanes, columns contiguous, stages by N - // columns; unit modes at 0. A 64-row share of a 2-CTA atom puts the - // second half of the columns on lanes 64 and up. - int64_t cols = n.as_int64(); - // Mode 0 is flattened to its (M,N) extents. - std::string text = "((" + cg::to_string(m) + "," + cg::to_string(n) + ")", - stride = "((65536,1)"; - if (atom->m / atom->ctas == 64) { - if (atom->ctas != 2 || m.as_int64() != 64 || cols % 2) - return {}; - cols /= 2; - text = "((64,(" + std::to_string(cols) + ",2))"; - stride = "((65536,(1,4194304))"; - } else if (atom->m / atom->ctas != 128) { - return {}; - } - for (size_t i = 1; i < cg::rank(s); ++i) { - auto e = cg::size(s, i); - text += "," + cg::to_string(cg::get(s, i)); - stride += "," + ((cg::holds_int(e) && e.as_int64() == 1) - ? std::string("0") - : std::to_string(cols)); - } - auto l = cg::from_string(text + "):" + stride + ")"); - if (!l) - return {}; - auto ptr = PtrType::get(ctx, atom->c, AddressSpace::tmem, 1, {}, {}); - return CuteMemRefType::get(ctx, ptr, LayoutType::get(ctx, *l)); - } - auto m = llvm::dyn_cast(in); - if (!m) - return {}; - cg::layout l; - if (auto plain = llvm::dyn_cast(m.getLayout())) - l = plain.getRef(); - else if (auto composed = llvm::dyn_cast(m.getLayout())) - l = composed.getRef().layout_b(); - else - return {}; - // The atom mode collapses to one; the rest modes follow. - int64_t bits = bitsOfType(m.getValueType()); - std::string shape = "(1", stride = "(0"; - for (size_t i = 1; i < cg::rank(l.shape()); ++i) { - auto e = cg::size(l.shape(), i); - auto st = cg::get(l.stride(), i); - if (!cg::holds_int(e) || !cg::holds_int(st)) - return {}; - shape += "," + std::to_string(e.as_int64()); - stride += "," + std::to_string(st.as_int64() * bits / 128); - } - auto view = cg::from_string(shape + "):" + stride + ")"); - if (!view) - return {}; - return cute_nvgpu::SmemDescViewType::get( - ctx, cute_nvgpu::SmemDescType::get(ctx), LayoutAttr::get(ctx, *view)); - }); -} - -LogicalResult TupleSubOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - return inferTupleArith(ctx, loc, operands, out, - [](const auto &a, const auto &b) { - return cg::arith_tuple_sub(a, b); - }); -} - -LogicalResult TupleModOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - if (failed(inferTupleArith(ctx, loc, operands, out, - [](const auto &a, const auto &b) { - return cg::arith_tuple_mod(a, b); - }))) - return failure(); - // A dynamic leaf modulo a static one: 0 when its divisibility is a - // multiple, itself when a divisor, else unknown (cutegen drops this). - std::vector lhs, rhs, res; - auto leaves = [](Type t, std::vector &l) { - if (auto i = llvm::dyn_cast(t)) - leavesOf(i.getRef(), l); - if (auto s = llvm::dyn_cast(t)) - leavesOf(s.getRef(), l); - }; - leaves(operands[0].getType(), lhs); - leaves(operands[1].getType(), rhs); - leaves(out[0], res); - if (lhs.size() != rhs.size() || lhs.size() != res.size()) - return success(); - for (size_t i = 0; i < res.size(); ++i) { - int64_t n, div = 1; - if (lhs[i].front() != '?' || StringRef(rhs[i]).getAsInteger(10, n) || - n <= 0) - continue; - size_t d = lhs[i].find("div="); - if (d != std::string::npos) - StringRef(lhs[i]) - .substr(d + 4) - .take_while(llvm::isDigit) - .getAsInteger(10, div); - res[i] = div % n == 0 ? "0" - : n % div == 0 - ? lhs[i] - : dynamicInt(1, lhs[i].find("i64") != std::string::npos); - } - size_t i = 0; - Type t = out[0]; - std::string text; - if (auto it = llvm::dyn_cast(t)) - text = nestLike(it.getRef(), res, i); - else - text = nestLike(llvm::cast(t).getRef(), res, i); - if (llvm::isa(t)) { - if (auto r = cg::from_string(text)) - out[0] = IntTupleType::get(ctx, *r); - } else if (auto r = cg::from_string(text)) { - out[0] = ShapeType::get(ctx, *r); - } - return success(); -} - -// ---------------------------------------------------------------- more cute - -namespace { - -template -LogicalResult inferProduct(MLIRContext *ctx, std::optional loc, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, SmallVectorImpl &out, - F &&f) { - typename Op::Adaptor a(operands, attrs, props); - auto tiler = llvm::dyn_cast(a.getTiler().getType()); - if (!tiler || !cg::is_int_or_dynamic_int_only(tiler.getRef().stride())) - return emitOptionalError(loc, "expects an integer-strided layout tiler"); - Type in = a.getInput().getType(); - return infer(loc, out, [&]() -> Type { - auto outer = outerLayout(in); - if (!outer || !cg::is_static(*outer) || - (!cg::is_int_or_dynamic_int_only(outer->stride()) && - !cg::is_static(tiler.getRef()))) - return {}; - return mapLayout(ctx, in, - [&](const auto &l) { return f(l, tiler.getRef()); }); - }); -} - -// A coordinate per leaf of `shape`: 0 where the extent is 1, else dynamic. -std::optional dynamicCoordLike(const cg::shape &shape) { - std::vector leaves; - leavesOf(shape, leaves); - for (auto &leaf : leaves) - leaf = isOne(leaf) ? "0" : "?"; - size_t i = 0; - return cg::from_string(nestLike(shape, leaves, i)); -} - -// Moves a carrier's iterator by `offset` and gives it `layout`. -Type moved(MLIRContext *ctx, const Carrier &c, const cg::int_tuple &offset, - Type layout) { - if (c.ptr) - return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), layout); - if (c.iter) { - auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); - if (!cg::is_valid(sum)) - return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), - layout); - } - return rebuild(ctx, c, layout); -} - -} // namespace - -#define CUTE_PRODUCT(Op, fn) \ - LogicalResult Op::inferReturnTypes( \ - MLIRContext *ctx, std::optional loc, ValueRange operands, \ - DictionaryAttr attrs, PropertyRef props, RegionRange, \ - SmallVectorImpl &out) { \ - return inferProduct( \ - ctx, loc, operands, attrs, props, out, \ - [](const auto &l, const auto &t) { return cg::fn(l, t); }); \ - } -CUTE_PRODUCT(LogicalProductOp, logical_product) -CUTE_PRODUCT(ZippedProductOp, zipped_product) -CUTE_PRODUCT(TiledProductOp, tiled_product) -CUTE_PRODUCT(FlatProductOp, flat_product) -CUTE_PRODUCT(BlockedProductOp, blocked_product) -#undef CUTE_PRODUCT - -// Unlike the other products, the raked one leaves unit modes at stride 0. -LogicalResult RakedProductOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - return inferProduct( - ctx, loc, operands, attrs, props, out, [](const auto &l, const auto &t) { - return zeroUnitModes(cg::raked_product(l, t)); - }); -} - -// The pointer's type is kept: the compiler records no swizzle on it. -LogicalResult ApplySwizzleOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, SmallVectorImpl &out) { - out.push_back(operands[0].getType()); - return success(); -} - -LogicalResult ComplementOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto in = llvm::dyn_cast(a.getInput().getType()); - Type co = a.getCotarget().getType(); - return infer(loc, out, [&]() -> Type { - if (!in) - return {}; - const cg::layout &l = in.getRef(); - if (!cg::is_int_or_dynamic_int_only(l.stride()) || - (!cg::is_static(l.stride()) && cg::rank(l.shape()) > 1)) - return {}; - cg::shape shape; - if (auto s = llvm::dyn_cast(co)) - shape = s.getRef(); - else if (auto c = llvm::dyn_cast(co)) - shape = c.getRef().shape(); - else - return {}; - cg::layout r = cg::complement(l, shape); - return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); - }); -} - -LogicalResult ComposedGetInnerOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - auto c = llvm::dyn_cast(operands[0].getType()); - if (!c) - return {}; - if (c.getRef().is_a_swizzle()) - return SwizzleType::get(ctx, c.getRef().swizzle_a()); - return LayoutType::get(ctx, c.getRef().layout_a()); - }); -} - -LogicalResult ComposedGetOffsetOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - auto c = llvm::dyn_cast(operands[0].getType()); - return c ? IntTupleType::get(ctx, c.getRef().offset()) : Type(); - }); -} - -LogicalResult CosizeOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - ArrayRef mode = a.getMode(); - auto c = carrierOf(a.getInput().getType()); - return infer(loc, out, [&]() -> Type { - if (!c) - return {}; - cg::int_tuple r; - if (auto l = llvm::dyn_cast(c->layout)) { - r = cg::rec_var_cast( - cg::cosize(cg::get(l.getRef(), mode))); - } else { - // An affine inner layout has no cosize. - auto cl = llvm::cast(c->layout); - if (!cl.getRef().is_a_swizzle()) - return {}; - r = cg::rec_var_cast( - cg::cosize(cg::get(cl.getRef(), mode))); - } - return cg::is_valid(r) ? IntTupleType::get(ctx, std::move(r)) : Type(); - }); -} - -// The multiplier and the two shifts of the divisor's magic numbers. -LogicalResult FastDivmodGetAuxOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - Type i = fastDivmodComputeResultType(operands[0].getType()); - if (!i) - return emitOptionalError(loc, "expects a fast divmod divisor"); - out.append({i, IntegerType::get(ctx, 8), IntegerType::get(ctx, 8)}); - return success(); -} - -LogicalResult FastDivmodGetDivisorOp::inferReturnTypes( - MLIRContext *, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&] { - return fastDivmodComputeResultType(operands[0].getType()); - }); -} - -LogicalResult FilterOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - auto c = carrierOf(operands[0].getType()); - if (!c) - return emitOptionalError(loc, "input carries no layout"); - return infer(loc, out, [&]() -> Type { - Type layout; - if (auto l = llvm::dyn_cast(c->layout)) { - layout = layoutType(ctx, cg::filter(l.getRef())); - } else { - // The outer layout is filtered; the inner function is kept. - const cg::composed_layout &cl = - llvm::cast(c->layout).getRef(); - cg::layout f = cg::filter(cl.layout_b()); - if (!cg::is_valid(f)) - return {}; - layout = layoutType( - ctx, cl.is_a_swizzle() - ? cg::composed_layout(cl.swizzle_a(), cl.offset(), f) - : cg::composed_layout(cl.layout_a(), cl.offset(), f)); - } - return rebuild(ctx, *c, layout); - }); -} - -namespace { -// Layout::get_hier_coord: each leaf's coordinate is (index / stride) % shape. -// A constant index gives static coordinates where the leaf is static; any -// other index gives dynamic ones, 0 on leaves of extent 1. -std::optional> hierCoord(Value index, - const cg::layout &l) { - std::vector shapes, strides; - leavesOf(l.shape(), shapes); - leavesOf(l.stride(), strides); - if (shapes.size() != strides.size()) - return std::nullopt; - APInt value; - bool constant = matchPattern(index, m_ConstantInt(&value)); - std::vector crd; - for (size_t i = 0; i < shapes.size(); ++i) { - int64_t n = 0, d = 0; - bool staticShape = !StringRef(shapes[i]).getAsInteger(10, n); - if (staticShape && n == 1) { - crd.push_back("0"); - } else if (!constant) { - crd.push_back("?"); - } else if (StringRef(strides[i]).getAsInteger(10, d) || d <= 0) { - return std::nullopt; // not recovered for dynamic or zero strides - } else { - int64_t q = value.getSExtValue() / d; - crd.push_back(staticShape ? std::to_string(q % n) : "?"); - } - } - return crd; -} - -// Column-major index of static coordinates in `shape`, or "?". -std::string colMajor(const std::vector &crd, - const std::vector &shapes, size_t begin, - size_t end) { - int64_t idx = 0, scale = 1; - for (size_t i = begin; i < end; ++i) { - int64_t c, n; - if (StringRef(crd[i]).getAsInteger(10, c)) - return "?"; - idx += c * scale; - if (i + 1 < end && StringRef(shapes[i]).getAsInteger(10, n)) - return "?"; - scale *= n; - } - return std::to_string(idx); -} -} // namespace - -LogicalResult GetHierCoordOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto layout = llvm::dyn_cast(a.getInput().getType()); - return infer(loc, out, [&]() -> Type { - auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; - if (!crd) - return {}; - size_t i = 0; - auto c = - cg::from_string(nestLike(layout.getRef().shape(), *crd, i)); - return c ? CoordType::get(ctx, std::move(*c)) : Type(); - }); -} - -// One coordinate per mode: the mode's hierarchical coordinate, column-major. -LogicalResult GetFlatCoordOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto layout = llvm::dyn_cast(a.getInput().getType()); - return infer(loc, out, [&]() -> Type { - auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; - if (!crd) - return {}; - const cg::shape &shape = layout.getRef().shape(); - std::vector shapes, modes; - leavesOf(shape, shapes); - if (!cg::holds_vector(shape)) { - modes.push_back(colMajor(*crd, shapes, 0, shapes.size())); - } else { - size_t begin = 0; - for (size_t m = 0; m < cg::rank(shape); ++m) { - std::vector sub; - leavesOf(shape[m], sub); - modes.push_back(colMajor(*crd, shapes, begin, begin + sub.size())); - begin += sub.size(); - } - } - auto c = cg::from_string(cg::holds_vector(shape) - ? "(" + llvm::join(modes, ",") + ")" - : modes[0]); - return c ? CoordType::get(ctx, std::move(*c)) : Type(); - }); -} - -LogicalResult GetIntegralCoordOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto layout = llvm::dyn_cast(a.getInput().getType()); - return infer(loc, out, [&]() -> Type { - auto crd = layout ? hierCoord(a.getIndex(), layout.getRef()) : std::nullopt; - if (!crd) - return {}; - std::vector shapes; - leavesOf(layout.getRef().shape(), shapes); - auto c = - cg::from_string(colMajor(*crd, shapes, 0, shapes.size())); - return c ? CoordType::get(ctx, std::move(*c)) : Type(); - }); -} - -LogicalResult GetLayoutsFromTileOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - auto tile = llvm::dyn_cast(operands[0].getType()); - if (!tile) - return emitOptionalError(loc, "expects a tile"); - auto flat = cg::flatten(tile.getRef()); - for (size_t i = 0; i < cg::rank(flat); ++i) { - flat[i].visit([&](const auto &e) { - if constexpr (cg::is_layout>::value) - out.push_back(LayoutType::get(ctx, e)); - }); - } - if (out.empty()) - return emitOptionalError(loc, "the tile holds no layout"); - return success(); -} - -LogicalResult GetStrideOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - auto l = llvm::dyn_cast(operands[0].getType()); - return l ? StrideType::get(ctx, l.getRef().stride()) : Type(); - }); -} - -LogicalResult Idx2CrdOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto index = llvm::dyn_cast(a.getIndex().getType()); - auto shape = llvm::dyn_cast(a.getShape().getType()); - return infer(loc, out, [&]() -> Type { - if (!index || !shape) - return {}; - size_t r = cg::rank(index.getRef()); - if (r > 1 && r != cg::rank(shape.getRef())) - return {}; - if (auto split = cg::holds_dynamic_int(index.getRef()) - ? splitIndex(index.getRef(), shape.getRef()) - : std::nullopt) { - auto c = cg::from_string(*split); - return c ? CoordType::get(ctx, std::move(*c)) : Type(); - } - auto c = cg::rec_var_cast( - cg::idx2crd(index.getRef(), shape.getRef())); - return cg::is_valid(c) ? CoordType::get(ctx, std::move(c)) : Type(); - }); -} - -LogicalResult IncrementCoordOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto coord = llvm::dyn_cast(a.getCoord().getType()); - auto shape = llvm::dyn_cast(a.getShape().getType()); - return infer(loc, out, [&]() -> Type { - if (!coord || !shape || !cg::is_congruent(coord.getRef(), shape.getRef()) || - cg::has_underscore(coord.getRef())) - return {}; - auto c = cg::increment_coord(coord.getRef(), shape.getRef()); - return cg::is_valid(c) ? CoordType::get(ctx, std::move(c)) : Type(); - }); -} - -namespace { -template -LogicalResult inferInverse(MLIRContext *ctx, std::optional loc, - Type input, bool staticStride, - SmallVectorImpl &out, F &&f) { - return infer(loc, out, [&]() -> Type { - auto l = llvm::dyn_cast(input); - if (!l || !cg::is_static(l.getRef().shape()) || - (staticStride && !cg::is_static(l.getRef().stride()))) - return {}; - cg::layout r = f(l.getRef()); - return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); - }); -} -} // namespace - -LogicalResult LeftInverseOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return inferInverse(ctx, loc, operands[0].getType(), true, out, - [](const cg::layout &l) { return cg::left_inverse(l); }); -} - -LogicalResult RightInverseOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return inferInverse(ctx, loc, operands[0].getType(), false, out, - [](const cg::layout &l) { return cg::right_inverse(l); }); -} - -LogicalResult LoadScaledIndexOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, SmallVectorImpl &out) { - out.push_back(operands[0].getType()); - return success(); -} - -// CuTe's local_partition: the tensor divided by the tiler's shape, sliced at -// the thread's (dynamic) coordinate in the tile. -LogicalResult LocalPartitionOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto c = carrierOf(a.getInput().getType()); - auto tiler = llvm::dyn_cast(a.getTiler().getType()); - return infer(loc, out, [&]() -> Type { - auto layout = c ? llvm::dyn_cast(c->layout) : LayoutType(); - if (!layout || !tiler || !cg::is_static(tiler.getRef())) - return {}; - cg::shape tile = cg::product_each(tiler.getRef().shape()); - cg::layout zipped = cg::zipped_divide(layout.getRef(), tile); - auto thread = dynamicCoordLike(tile); - if (!cg::is_valid(zipped) || !thread) - return {}; - auto crd = cg::from_string("(" + cg::to_string(*thread) + ",_)"); - if (!crd) - return {}; - auto [sliced, offset] = cg::slice_and_offset(*crd, zipped); - if (!cg::is_valid(sliced)) - return {}; - // The slice keeps the rest mode's own modes, not a tuple around them. - if (cg::holds_vector(sliced.shape()) && cg::rank(sliced.shape()) == 1) - sliced = cg::get(sliced, 0); - return moved(ctx, *c, offset, LayoutType::get(ctx, zeroUnitModes(sliced))); - }); -} - -// CuTe's make_fragment_like: the first mode compact column-major with its -// stride-0 modes kept, the other modes ordered by their strides (dynamic -// ones last) after it; column-major when there is one mode or the first -// mode's shape is dynamic. -LogicalResult MakeFragmentLikeOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - auto src = swizzledOuter(operands[0].getType()); - return infer(loc, out, [&]() -> Type { - if (!src) - return {}; - const cg::shape &shape = src->shape(); - std::vector shapes, strides; - leavesOf(shape, shapes); - leavesOf(src->stride(), strides); - if (shapes.size() != strides.size()) - return {}; - size_t first = shapes.size(); // leaves of mode 0 - bool ordered = cg::holds_vector(shape) && cg::rank(shape) > 1 && - cg::is_static(shape[0]); - if (ordered) { - std::vector m0; - leavesOf(shape[0], m0); - first = m0.size(); - } - // The order the leaves are laid out in. - std::vector order(shapes.size()); - std::iota(order.begin(), order.end(), 0); - auto key = [&](size_t i) -> int64_t { - int64_t v; - return StringRef(strides[i]).getAsInteger(10, v) ? INT64_MAX : v; - }; - if (ordered) - std::stable_sort(order.begin() + first, order.end(), - [&](size_t x, size_t y) { return key(x) < key(y); }); - std::vector result(shapes.size()); - cg::int_tuple step = *cg::from_string("1"); - for (size_t i : order) { - if (isOne(shapes[i]) || (ordered && i < first && strides[i] == "0")) { - result[i] = "0"; - continue; - } - result[i] = cg::to_string(step); - auto n = cg::from_string(shapes[i]); - if (!n) - return {}; - step = cg::arith_tuple_mul(step, *n); - } - size_t i = 0; - auto l = cg::from_string(cg::to_string(shape) + ":" + - nestLike(shape, result, i)); - return l ? LayoutType::get(ctx, std::move(*l)) : Type(); - }); -} - -LogicalResult MakeIdentityTensorOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - auto s = llvm::dyn_cast(operands[0].getType()); - if (!s) - return {}; - std::vector leaves; - leavesOf(s.getRef(), leaves); - for (auto &leaf : leaves) - leaf = "0"; - size_t i = 0; - auto zero = cg::from_string(nestLike(s.getRef(), leaves, i)); - cg::layout l = cg::make_identity_layout(s.getRef()); - if (!zero || !cg::is_valid(l)) - return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, *zero), - LayoutType::get(ctx, l)); - }); -} - -LogicalResult MakeLayoutLikeOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - Type src = operands[0].getType(); - cg::layout r; - if (auto l = llvm::dyn_cast(src)) { - if (cg::has_scaled_basis(l.getRef().stride())) - return {}; - r = cg::make_layout_like(l.getRef()); - } else if (auto c = llvm::dyn_cast(src)) { - if (!c.getRef().is_a_swizzle() || - cg::has_scaled_basis(c.getRef().layout_b().stride())) - return {}; - r = cg::make_layout_like(c.getRef()); - } else { - return {}; - } - return cg::is_valid(r) ? LayoutType::get(ctx, std::move(r)) : Type(); - }); -} - -LogicalResult PtrLoadOp::inferReturnTypes(MLIRContext *, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - auto p = llvm::dyn_cast(operands[0].getType()); - return p ? p.getValueType() : Type(); - }); -} - -LogicalResult RecastLayoutOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - int newBits = a.getNewTypeBits(), oldBits = a.getOldTypeBits(); - return infer(loc, out, [&]() -> Type { - if (newBits <= 0 || oldBits <= 0) - return {}; - auto outer = outerLayout(a.getSrc().getType()); - if (!outer || !cg::is_int_or_dynamic_int_only(outer->stride())) - return {}; - return mapLayout(ctx, a.getSrc().getType(), [&](const auto &l) { - return cg::recast(newBits, oldBits, l); - }); - }); -} - -// Without a profile the whole vector reduces to a scalar; with one, the -// modes it marks reduce and the others (`_`) remain, as a vector and shape. -LogicalResult ReduceOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto vec = llvm::dyn_cast(a.getInput().getType()); - if (!vec) - return emitOptionalError(loc, "expects a vector input"); - if (!a.getReductionProfileAttr()) { - out.push_back(vec.getElementType()); - return success(); - } - auto shape = llvm::dyn_cast(a.getShape().getType()); - if (!shape) - return emitOptionalError(loc, "expects a shape"); - cg::shape rest = - cg::slice(a.getReductionProfileAttr().getRef(), shape.getRef()); - auto n = cg::size(rest); - if (!cg::is_valid(rest) || !cg::holds_int(n)) - return emitOptionalError(loc, "cannot reduce the shape"); - out.push_back(VectorType::get({n.as_int64()}, vec.getElementType())); - out.push_back(ShapeType::get(ctx, std::move(rest))); - return success(); -} - -LogicalResult SelectOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, - DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - ArrayRef mode = a.getMode(); - return infer(loc, out, [&]() -> Type { - return llvm::TypeSwitch(a.getInput().getType()) - .Case([&](auto ty) -> Type { - size_t r = cg::rank(ty.getRef()); - for (int32_t m : mode) - if (m < 0 || static_cast(m) >= r) - return {}; - auto res = cg::select(ty.getRef(), mode); - if (!cg::is_valid(res)) - return {}; - return decltype(ty)::get(ctx, std::move(res)); - }) - .Default([](Type) -> Type { return {}; }); - }); -} - -LogicalResult ShapeDivOp::inferReturnTypes(MLIRContext *ctx, - std::optional loc, - ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - auto x = llvm::dyn_cast(operands[0].getType()); - auto y = llvm::dyn_cast(operands[1].getType()); - if (!x || !y) - return {}; - cg::shape r = cg::shape_div(x.getRef(), y.getRef()); - return cg::is_valid(r) ? ShapeType::get(ctx, std::move(r)) : Type(); - }); -} - -// The compiler aborts on every input tried (bad_alloc, segfault): no rule. -LogicalResult StencilDivideOp::inferReturnTypes(MLIRContext *, - std::optional loc, - ValueRange, DictionaryAttr, - PropertyRef, RegionRange, - SmallVectorImpl &) { - return emitOptionalError(loc, - "cute.stencil_divide inference is not recovered"); -} - -// No type of the compiler's registry implements the descriptor iterator -// interface this needs, so the compiler rejects every input. -LogicalResult DereferenceDescriptorIteratorOp::inferReturnTypes( - MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, - PropertyRef, RegionRange, SmallVectorImpl &) { - return emitOptionalError(loc, "expects a descriptor iterator"); -} - -LogicalResult TupleProductOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - return llvm::TypeSwitch(operands[0].getType()) - .Case([&](auto ty) -> Type { - auto r = cg::product(ty.getRef()); - return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) - : Type(); - }) - .Default([](Type) -> Type { return {}; }); - }); -} - -LogicalResult TupleProductEachOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - return infer(loc, out, [&]() -> Type { - return llvm::TypeSwitch(operands[0].getType()) - .Case([&](auto ty) -> Type { - auto r = cg::product_each(ty.getRef()); - return cg::is_valid(r) ? decltype(ty)::get(ctx, std::move(r)) - : Type(); - }) - .Default([](Type) -> Type { return {}; }); - }); -} - -// A SIMT copy's fragment is the tensor itself; the rule for other atoms is -// not recovered. -LogicalResult -CopyMakeFragmentOp::inferReturnTypes(MLIRContext *, std::optional loc, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto tiled = llvm::dyn_cast(a.getAtom().getType()); - Type in = a.getInput().getType(); - if (!tiled || - !llvm::isa(tiled.getCopyAtom()) || - !llvm::isa(in)) - return emitOptionalError( - loc, "copy fragment inference covers tiled SIMT copies only"); - out.push_back(in); - return success(); -} - -// The atom's shape per CTA, then how often the atom repeats over the tile. -LogicalResult TiledMmaPartitionShapeOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto atom = mmaAtomOf(a.getTiledMma().getType()); - auto shape = llvm::dyn_cast(a.getInput().getType()); - auto which = llvm::dyn_cast_or_null(a.getOperandIdAttr()); - return infer(loc, out, [&]() -> Type { - if (!atom || !shape || !which || !cg::holds_vector(shape.getRef()) || - cg::rank(shape.getRef()) != 2) - return {}; - int64_t full[2], part[2]; - switch (which.getInt()) { - case 0: - full[0] = atom->m; - full[1] = atom->k; - part[0] = atom->m / atom->ctas; - break; - case 1: - full[0] = atom->n; - full[1] = atom->k; - part[0] = atom->n / atom->ctas; - break; - case 2: - full[0] = atom->m; - full[1] = atom->n; - part[0] = atom->m / atom->ctas; - break; - default: - return {}; - } - part[1] = full[1]; - std::string rest; - for (int i = 0; i < 2; ++i) { - auto n = cg::size(shape.getRef(), i); - if (!cg::holds_int(n) || n.as_int64() % full[i]) - return {}; - rest += "," + std::to_string(n.as_int64() / full[i]); - } - auto r = - cg::from_string("((" + std::to_string(part[0]) + "," + - std::to_string(part[1]) + ")" + rest + ")"); - return r ? ShapeType::get(ctx, std::move(*r)) : Type(); - }); -} - -// ---------------------------------------------------------------- copy atoms - -namespace { - -std::optional layoutOf(const std::string &text) { - return cg::from_string(text); -} - -template int64_t asInt(const V &v) { - return cg::holds_int(v) ? v.as_int64() : -1; -} - -// TMEM addresses count lanes in the upper half-word: 2^21 bits per lane. -constexpr int64_t kTmemLaneBits = int64_t(1) << 21; - -// A copy atom's traits as CuTe spells them (cute/atom/copy_traits_*.hpp): -// the thread count and the (thr,val) -> bit layouts, and for tensor memory -// atoms the bit -> address layout. -struct CopyTraits { - int64_t threads = 1; - int64_t bits = 8; - cg::layout src, dst, ref, valId; -}; - -std::optional copyTraitsOf(Type atom) { - CopyTraits t; - auto set = [&](std::string src, std::string dst, bool refIsSrc, - std::string valId = "") -> std::optional { - auto s = layoutOf(src), d = layoutOf(dst); - if (!s || !d) - return std::nullopt; - t.src = *s; - t.dst = *d; - t.ref = refIsSrc ? *s : *d; - if (!valId.empty()) { - auto v = layoutOf(valId); - if (!v) - return std::nullopt; - t.valId = *v; - } - return t; - }; - auto n = [](int64_t v) { return std::to_string(v); }; - if (auto u = llvm::dyn_cast(atom)) { - t.bits = bitsOfType(u.getValType()); - std::string l = - "(1," + n(u.getCopyBits() ? u.getCopyBits() : t.bits) + "):(0,1)"; - return set(l, l, true); - } - if (auto s = llvm::dyn_cast(atom)) { - // SM90_U32xN_STSM_N and SM90_U16x2N_STSM_T: the SM75 ldmatrix layouts - // with source and destination swapped. - t.bits = bitsOfType(s.getValType()); - t.threads = 32; - int64_t matrices = s.getNumMatrices(); - bool transposed = bool(s.getTranspose()); - std::string ldsmSrc, ldsmDst; - if (!transposed) { - switch (matrices) { - case 1: - ldsmSrc = "((8,4),128):((128,0),1)"; - ldsmDst = "(32,32):(32,1)"; - break; - case 2: - ldsmSrc = "((16,2),128):((128,0),1)"; - ldsmDst = "(32,(32,2)):(32,(1,1024))"; - break; - case 4: - ldsmSrc = "(32,128):(128,1)"; - ldsmDst = "(32,(32,4)):(32,(1,1024))"; - break; - default: - return std::nullopt; - } - } else { - switch (matrices) { - case 1: - ldsmSrc = "((8,4),128):((128,0),1)"; - ldsmDst = "((4,8),(16,2)):((256,16),(1,128))"; - break; - case 2: - ldsmSrc = "((16,2),128):((128,0),1)"; - ldsmDst = "((4,8),(16,2,2)):((256,16),(1,128,1024))"; - break; - case 4: - ldsmSrc = "(32,128):(128,1)"; - ldsmDst = "((4,8),(16,2,4)):((256,16),(1,128,1024))"; - break; - default: - return std::nullopt; - } - } - return set(ldsmDst, ldsmSrc, true); - } - // Tensor memory loads and stores: the tmem side is the reference; a store - // is a load with source and destination swapped. - auto tmem = [&](Type val, int64_t dp, int64_t bit, int64_t x, - bool load) -> std::optional { - t.bits = bitsOfType(val); - t.threads = 32; - std::string mem, reg, valId; - if (dp == 32 && bit == 32) { - mem = "(32," + n(1024 * x) + "):(0,1)"; - reg = "(32," + n(32 * x) + "):(" + n(32 * x) + ",1)"; - valId = "(" + n(32 * x) + ",32):(1," + n(kTmemLaneBits) + ")"; - } else if (dp == 16 && bit == 256) { - mem = "(32," + n(4096 * x) + "):(0,1)"; - reg = x == 1 ? "((4,8),(64,2)):((64,256),(1,2048))" - : "((4,8),(64,2," + n(x) + ")):((64," + n(256 * x) + - "),(1," + n(2048 * x) + ",256))"; - valId = "(" + n(256 * x) + ",16):(1," + n(kTmemLaneBits) + ")"; - } else { - return std::nullopt; - } - return load ? set(mem, reg, true, valId) : set(reg, mem, false, valId); - }; - // 16-bit packing and split reductions are not covered. - if (auto l = llvm::dyn_cast(atom)) - if (!l.getPack16b()) - return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), - true); - if (auto l = llvm::dyn_cast(atom)) - if (!l.getHalfSplitOff()) - return tmem(l.getValType(), l.getNumDp(), l.getNumBit(), l.getNumRep(), - true); - if (auto st = llvm::dyn_cast(atom)) - if (!st.getExpand16b()) - return tmem(st.getValType(), st.getNumDp(), st.getNumBit(), - st.getNumRep(), false); - if (auto s = llvm::dyn_cast(atom)) { - // : SM100_UTCCP_4x32dp128bit_{C}cta. - t.bits = bitsOfType(s.getValType()); - int64_t dp = s.getNumDp(), bit = s.getNumBit(); - t.threads = s.getNumCta(); - if (dp != 32 || bit != 128 || - s.getBroadcast() != cute_nvgpu::CopyS2TBroadcast::x4 || t.threads < 1) - return std::nullopt; - return set("(" + n(t.threads) + ",(32,128,4)):(0,(1,32,0))", - "(" + n(t.threads) + ",16384):(0,1)", false, - "(32,128,4):(" + n(kTmemLaneBits) + ",1," + - n(32 * kTmemLaneBits) + ")"); - } - return std::nullopt; -} - -// A tiled copy: its atom in element units, the (thr,val) -> tile layout and -// the tiler over MN. -struct TiledCopy { - Type atom; - CopyTraits traits; - cg::layout tv; - cg::tile tiler; - int64_t atomThreads = 0, atomVals = 0, tiledThreads = 0; -}; - -std::optional tiledCopyOf(Type t) { - auto tiled = llvm::dyn_cast(t); - if (!tiled) - return std::nullopt; - auto traits = copyTraitsOf(tiled.getCopyAtom()); - if (!traits) - return std::nullopt; - TiledCopy tc; - tc.atom = tiled.getCopyAtom(); - tc.traits = *traits; - // Copy_Atom recasts the bit layouts to the value type. - tc.traits.src = cg::upcast(traits->bits, traits->src); - tc.traits.dst = cg::upcast(traits->bits, traits->dst); - tc.traits.ref = cg::upcast(traits->bits, traits->ref); - if (!cg::is_valid(tc.traits.src) || !cg::is_valid(tc.traits.dst) || - !cg::is_valid(tc.traits.ref)) - return std::nullopt; - tc.tv = tiled.getLayoutCopyTv().getRef(); - tc.tiler = tiled.getTilerMn().getRef(); - tc.atomThreads = asInt(cg::size(tc.traits.ref, 0)); - tc.atomVals = asInt(cg::size(tc.traits.ref, 1)); - tc.tiledThreads = asInt(cg::size(tc.tv, 0)); - if (tc.atomThreads <= 0 || tc.atomVals <= 0 || tc.tiledThreads <= 0) - return std::nullopt; - return tc; -} - -// The modes of a layout, one leaf mode as itself. -std::vector modesOf(const cg::layout &l) { - std::vector out; - if (!cg::holds_vector(l.shape())) - return {l}; - for (size_t i = 0; i < cg::rank(l.shape()); ++i) - out.push_back(cg::get(l, i)); - return out; -} - -// `l` with its first mode composed with `f` and the others kept: CuTe's -// tensor.compose(f, _). -cg::layout composeFirst(const cg::layout &l, const cg::layout &f) { - std::vector modes = modesOf(l); - modes[0] = cg::composition(modes[0], f); - return cg::make_layout(modes); -} - -// ((a,b),(c,d)) -> ((a,c),(b,d)) -cg::layout zip2(const cg::layout &l) { - return cg::make_layout(std::vector{ - cg::make_layout( - std::vector{cg::get(l, 0, 0), cg::get(l, 1, 0)}), - cg::make_layout( - std::vector{cg::get(l, 0, 1), cg::get(l, 1, 1)})}); -} - -cg::coord profileOf(const char *text) { - return *cg::from_string(text); -} - -// TiledCopy::tile2thrfrg: ((Tile),(Rest...)) -> (Thr, (FrgV, FrgX), Rest...) -// through the atom's (src or dst) thread-value map `ref2trg`. -cg::layout tile2thrfrg(const TiledCopy &tc, const cg::layout &tiled, - const cg::layout &ref2trg, bool flatRest = true) { - auto atomShape = - cg::from_string("(" + std::to_string(tc.atomThreads) + "," + - std::to_string(tc.atomVals) + ")"); - if (!atomShape) - return cg::layout(cg::cg_error_t{}); - cg::layout atomTV = cg::zipped_divide(tc.tv, *atomShape); - if (!cg::is_valid(atomTV)) - return atomTV; - cg::layout trgTV = composeFirst(atomTV, ref2trg); - cg::layout thrval2mn = cg::coalesce(zip2(trgTV), profileOf("(1,(1,1))")); - if (!cg::is_valid(thrval2mn)) - return thrval2mn; - cg::layout tv = cg::composition(cg::get(tiled, 0), thrval2mn); - if (!cg::is_valid(tv)) - return tv; - std::vector modes{cg::get(tv, 0), cg::get(tv, 1)}; - // The rest modes follow; a rank-1 tensor keeps its one rest mode whole. - if (!flatRest) - modes.push_back(cg::get(tiled, 1)); - else - for (const cg::layout &m : modesOf(cg::get(tiled, 1))) - modes.push_back(m); - return cg::make_layout(modes); -} - -// TiledCopy::tidfrg_S/D: (M,N,...) -> (Thr, (FrgV, FrgX), (RestM, RestN, ...)). -cg::layout tidfrg(const TiledCopy &tc, const cg::layout &tensor, bool src) { - cg::layout tiled = cg::zipped_divide(tensor, tc.tiler); - if (!cg::is_valid(tiled)) - return tiled; - cg::layout ref2trg = cg::composition(cg::right_inverse(tc.traits.ref), - src ? tc.traits.src : tc.traits.dst); - bool flatRest = - cg::holds_vector(tensor.shape()) && cg::rank(tensor.shape()) > 1; - return tile2thrfrg(tc, tiled, ref2trg, flatRest); -} - -// ThrCopy::partition_S/D: the thread's slice of tidfrg. -Type partitionCopy(MLIRContext *ctx, const TiledCopy &tc, const Carrier &c, - const cg::coord &thread, bool src) { - auto layout = llvm::dyn_cast(c.layout); - if (!layout) - return {}; - cg::layout tv = tidfrg(tc, layout.getRef(), src); - if (!cg::is_valid(tv)) - return {}; - std::string coord = "(" + cg::to_string(thread); - for (size_t i = 1; i < cg::rank(tv.shape()); ++i) - coord += ",_"; - coord += ")"; - auto crd = cg::from_string(coord); - if (!crd) - return {}; - auto [sliced, offset] = cg::slice_and_offset(*crd, tv); - if (!cg::is_valid(sliced)) - return {}; - // The tensor's modes beyond the tiler are untouched and keep their strides. - size_t in = cg::rank(layout.getRef().shape()), r = cg::rank(tc.tiler); - size_t keep = - cg::holds_vector(layout.getRef().shape()) && in > r ? in - r : 0; - Type result = LayoutType::get(ctx, zeroUnitModesHead(sliced, keep, false)); - if (c.ptr) - return CuteMemRefType::get(ctx, movedPtr(c.ptr, offset), result); - if (c.iter) { - auto sum = cg::arith_tuple_sum(c.iter.getRef(), offset); - if (!cg::is_valid(sum)) - return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), - result); - } - return rebuild(ctx, c, result); -} - -// The shapes of a tiler's modes, as one shape. -std::optional tilerShape(const cg::tile &tiler) { - std::string text = cg::to_string(tiler); - if (text.size() < 2 || text.front() != '[') - return std::nullopt; - SmallVector parts; - StringRef(text).drop_front().drop_back().split(parts, ';'); - std::string shape = "("; - for (size_t i = 0; i < parts.size(); ++i) { - auto l = cg::from_string(parts[i].str()); - if (!l) - return std::nullopt; - shape += (i ? "," : "") + cg::to_string(l->shape()); - } - return cg::from_string(shape + ")"); -} - -// TiledCopy::retile: (V, RestM, RestN, ...) of one thread's values regrouped -// into the atom's values. -Type retileCopy(MLIRContext *ctx, const TiledCopy &tc, const Carrier &c) { - auto layout = llvm::dyn_cast(c.layout); - if (!layout) - return {}; - const cg::layout &l = layout.getRef(); - size_t R = cg::rank(l.shape()); - int64_t V = asInt(cg::size(l, 0)); - auto shape = tilerShape(tc.tiler); - if (V <= 0 || !shape) - return {}; - cg::layout inv = cg::right_inverse(tc.tv).with_shape(*shape); - cg::layout frgMN = cg::upcast(tc.tiledThreads * V, inv); - if (!cg::is_valid(frgMN)) - return {}; - auto vLayout = layoutOf(std::to_string(V) + ":1"); - auto atomVals = layoutOf(std::to_string(tc.atomVals) + ":1"); - if (!vLayout || !atomVals) - return {}; - cg::layout frgV = cg::zipped_divide( - cg::logical_product(*vLayout, cg::right_inverse(frgMN)), *atomVals); - if (!cg::is_valid(frgV)) - return {}; - std::string tiler = "(" + std::to_string(V); - auto each = cg::product_each(frgMN.shape()); - for (size_t i = 0; i < cg::rank(each); ++i) - tiler += "," + cg::to_string(each[i]); - auto tilerShape = cg::from_string(tiler + ")"); - if (!tilerShape) - return {}; - cg::layout tiled = cg::zipped_divide(l, *tilerShape); - if (!cg::is_valid(tiled)) - return {}; - cg::layout v = composeFirst(tiled, frgV); - if (!cg::is_valid(v)) - return {}; - // v(_, (0, _, _...)): the first rest mode is the one the tile added. - std::vector modes{cg::get(v, 0)}; - std::vector rest = modesOf(cg::get(v, 1)); - for (size_t i = 1; i < rest.size() && i < R; ++i) - modes.push_back(rest[i]); - return rebuild(ctx, c, - LayoutType::get(ctx, zeroUnitModes(cg::make_layout(modes)))); -} - -// make_cotiled_copy: the (thr,val) -> tile layout and the tiler of a copy -// whose atom addresses `data` directly. -std::optional> -cotiledCopy(const cg::layout &atomTV, const cg::layout &data) { - auto unit = layoutOf("1:0"); - cg::layout invData = - cg::make_layout(std::vector{cg::left_inverse(data), *unit}); - cg::layout tvData = cg::composition(invData, atomTV); - if (!cg::is_valid(tvData)) - return std::nullopt; - auto flat = cg::product_each(data.shape()); - size_t R = cg::rank(flat); - std::string flatText = cg::to_string(flat); - std::string tilerText = "["; - for (size_t i = 0; i < R; ++i) { - std::string stride = "("; - for (size_t j = 0; j < R; ++j) - stride += std::string(j ? "," : "") + (i == j ? "1" : "0"); - auto pick = layoutOf(flatText + ":" + stride + ")"); - if (!pick) - return std::nullopt; - cg::layout mode = cg::filter(cg::composition(*pick, tvData)); - if (!cg::is_valid(mode)) - return std::nullopt; - tilerText += (i ? ";" : "") + cg::to_string(mode); - } - auto tiler = cg::from_string(tilerText + "]"); - auto flatShape = cg::from_string(flatText); - if (!tiler || !flatShape) - return std::nullopt; - cg::layout tile2data = cg::composition(cg::layout(*flatShape), *tiler); - cg::layout tv = cg::composition(cg::left_inverse(tile2data), tvData); - if (!cg::is_valid(tv)) - return std::nullopt; - return std::make_pair(tv, *tiler); -} - -Type tiledCopyType(MLIRContext *ctx, Type atom, const cg::layout &tv, - const cg::tile &tiler) { - return TiledCopyType::get(ctx, atom, LayoutAttr::get(ctx, tv), - TileAttr::get(ctx, tiler)); -} - -// ---------------------------------------------------------------- TMA - -struct TmaFormat { - std::string name; - int64_t bits = 0; -}; - -// The descriptor's data format: the attribute when given, else the element's. -std::optional tmaFormatOf(Type elem, Attribute attr) { - TmaFormat f; - if (attr) { - auto format = llvm::dyn_cast(attr); - if (!format) - return std::nullopt; - f.name = cute_nvgpu::stringifyTmaDataFormat(format.getValue()).str(); - StringRef n = f.name; - if (n == "F16_RN" || n == "BF16_RN") - f.bits = 16; - else if (n == "F32_RN" || n == "F32_FTZ_RN" || n == "INT32" || n == "TF32") - f.bits = 32; - else if (n == "F64_RN" || n == "INT64") - f.bits = 64; - else if (!n.drop_until(llvm::isDigit) - .take_while(llvm::isDigit) - .getAsInteger(10, f.bits)) { - } else - return std::nullopt; - return f; - } - f.bits = bitsOfType(elem); - if (elem.isTF32()) - f.name = "TF32_RN"; - else if (elem.isF16()) - f.name = "F16_RN"; - else if (elem.isBF16()) - f.name = "BF16_RN"; - else if (elem.isF32()) - f.name = "F32_RN"; - else if (elem.isF64()) - f.name = "F64_RN"; - else - f.name = "U" + std::to_string(f.bits); - return f; -} - -// CuTe's coalesce_256: which adjacent modes of a flattened layout merge into -// one TMA box dimension, the merged extent staying within 256. -std::vector> coalesce256Groups(const cg::layout &l) { - auto fs = cg::flatten(l.shape()); - auto ft = cg::flatten(l.stride()); - if (!cg::holds_vector(fs)) - return {{0}}; - std::vector> groups; - std::vector cur{0}; - int64_t curS = asInt(fs[0]), curD = asInt(ft[0]); - for (size_t i = 1; i < cg::rank(fs); ++i) { - int64_t s = asInt(fs[i]), d = asInt(ft[i]); - if (curS == 1) { - cur = {i}; - curS = s; - curD = d; - } else if (s == 1) { - continue; - } else if (curS > 0 && curD >= 0 && s > 0 && d >= 0 && curS * curD == d && - s * curS <= 256) { - cur.push_back(i); - curS *= s; - } else { - groups.push_back(cur); - cur = {i}; - curS = s; - curD = d; - } - } - groups.push_back(cur); - return groups; -} - -// The layout with `shape`'s leaves regrouped by `groups`, over `stride`'s -// leaves: what composing with the box shape does, keeping every basis. -std::optional -regroup(const cg::layout &shape, const cg::layout &stride, - const std::vector> &groups) { - auto fs = cg::flatten(shape.shape()); - auto ft = cg::flatten(stride.stride()); - auto leaf = [&](const auto &flat, size_t i) { - return cg::holds_vector(flat) ? cg::to_string(flat[i]) - : cg::to_string(flat); - }; - std::string shapes, strides; - for (size_t g = 0; g < groups.size(); ++g) { - if (g) { - shapes += ","; - strides += ","; - } - if (groups[g].size() == 1) { - shapes += leaf(fs, groups[g][0]); - strides += leaf(ft, groups[g][0]); - continue; - } - shapes += "("; - strides += "("; - for (size_t k = 0; k < groups[g].size(); ++k) { - if (k) { - shapes += ","; - strides += ","; - } - shapes += leaf(fs, groups[g][k]); - strides += leaf(ft, groups[g][k]); - } - shapes += ")"; - strides += ")"; - } - return layoutOf("(" + shapes + "):(" + strides + ")"); -} - -// A layout composed with a coordinate layout: each mode of `rhs` selects, -// through its basis stride, the mode of `lhs` it indexes. -cg::layout composeCoord(const cg::layout &lhs, const cg::layout &rhs) { - std::vector modes; - for (const cg::layout &m : modesOf(rhs)) { - const auto &d = m.stride(); - if (cg::holds_int(d)) { - modes.push_back(m); - continue; - } - if (!cg::holds_scaled_basis(d)) - return cg::layout(cg::cg_error_t{}); - const auto &basis = std::get(d); - if (!basis.value_holds_int()) - return cg::layout(cg::cg_error_t{}); - auto scale = layoutOf(cg::to_string(m.shape()) + ":" + - std::to_string(basis.static_integral_value())); - if (!scale) - return cg::layout(cg::cg_error_t{}); - cg::layout sub = cg::get(lhs, basis.modes()); - cg::layout composed = cg::composition(sub, *scale); - if (!cg::is_valid(composed)) - return composed; - modes.push_back(composed); - } - return cg::make_layout(modes); -} - -// Leaves of congruent shape, stride and basis, with the path to each. -template -void walkLeaves(const cg::shape &s, const cg::stride &d, const cg::stride &e, - std::vector &path, F &&f) { - if (cg::holds_vector(s)) { - for (size_t i = 0; i < cg::rank(s); ++i) { - path.push_back(i); - walkLeaves(s[i], d[i], e[i], path, f); - path.pop_back(); - } - return; - } - f(s, d, e, path); -} - -bool basisAt(const cg::stride &leaf, const std::vector &path) { - return cg::holds_scaled_basis(leaf) && - std::get(leaf).modes() == path; -} - -// Which mode of a stride tuple carries the basis `path`, if any. -std::optional modeWithBasis(const cg::stride &strides, - const std::vector &path) { - for (size_t j = 0; j < cg::rank(strides); ++j) { - auto flat = cg::flatten(strides[j]); - if (!cg::holds_vector(flat)) { - if (basisAt(flat, path)) - return j; - continue; - } - for (size_t k = 0; k < cg::rank(flat); ++k) - if (basisAt(flat[k], path)) - return j; - } - return std::nullopt; -} - -// construct_tma_gbasis: the TMA box shape with, per mode, the gmem modes it -// runs over. -cg::layout tmaGbasis(const cg::layout &gmem, int64_t elemBits, int64_t tmaBits, - const cg::layout &smem, const cg::layout &ctaVMap) { - auto fail = [] { return cg::layout(cg::cg_error_t{}); }; - cg::layout inv = cg::right_inverse(smem); - cg::layout full = cg::coalesce(cg::composition(ctaVMap, inv)); - if (!cg::is_valid(full)) - return full; - std::vector fullModes = modesOf(full); - // Keep the leading modes that step through gmem one element at a time. - size_t smemRank = 0; - for (; smemRank < fullModes.size(); ++smemRank) { - const auto &d = fullModes[smemRank].stride(); - if (!cg::holds_scaled_basis(d)) - break; - const auto &b = std::get(d); - if (!b.value_holds_int() || b.static_integral_value() != 1) - break; - } - if (smemRank == 0) - return fail(); - fullModes.resize(smemRank); - cg::layout sidx2gmode = cg::make_layout(fullModes); - cg::layout tileGstride = - cg::recast(tmaBits, elemBits, composeCoord(gmem, sidx2gmode)); - if (!cg::is_valid(tileGstride)) - return tileGstride; - cg::layout gbasis = cg::make_identity_layout(gmem.shape()); - cg::layout tileGbasisTmp = composeCoord(gbasis, sidx2gmode); - if (!cg::is_valid(tileGbasisTmp)) - return tileGbasisTmp; - auto regrouped = - regroup(tileGstride, tileGbasisTmp, coalesce256Groups(tileGstride)); - if (!regrouped) - return fail(); - cg::layout tmaGbasisTile = *regrouped; - // Gmem modes the tile misses still shape the descriptor, as size-1 modes. - cg::layout gmemT = cg::recast(tmaBits, elemBits, gmem); - if (!cg::is_valid(gmemT)) - return gmemT; - std::vector modes = modesOf(tmaGbasisTile); - auto flatTile = cg::flatten(tmaGbasisTile.stride()); - std::vector path; - bool ok = true; - walkLeaves(gmemT.shape(), gmemT.stride(), gbasis.stride(), path, - [&](const auto &s, const auto &d, const auto &e, - const std::vector &p) { - if (asInt(s) == 1 || asInt(d) == 0) - return; - bool found = false; - if (!cg::holds_vector(flatTile)) { - found = basisAt(flatTile, p); - } else { - for (size_t k = 0; k < cg::rank(flatTile) && !found; ++k) - found = basisAt(flatTile[k], p); - } - if (found) - return; - auto extra = layoutOf("1:" + cg::to_string(e)); - if (!extra) - ok = false; - else - modes.push_back(*extra); - }); - if (!ok) - return fail(); - cg::layout result = cg::make_layout(modes); - if (modes.size() > 4) - result = cg::group(4, modes.size(), result); - return result; -} - -// The TMA tensor's layout: gmem shape over TMA coordinate bases, scaled to -// the descriptor's element. -std::optional tmaTensorLayout(const cg::layout &gmem, - int64_t elemBits, int64_t tmaBits, - const cg::layout &gbasis) { - cg::layout basis = cg::make_identity_layout(gmem.shape()); - const auto &tmaStrides = gbasis.stride(); - bool ok = true; - std::string text; - std::vector path; - // Written in the shape's own nesting. - std::function - write = [&](const cg::shape &s, const cg::stride &d, - const cg::stride &e) { - if (cg::holds_vector(s)) { - text += "("; - for (size_t i = 0; i < cg::rank(s); ++i) { - if (i) - text += ","; - path.push_back(i); - write(s[i], d[i], e[i]); - path.pop_back(); - } - text += ")"; - return; - } - if (asInt(s) == 1 || asInt(d) == 0) { - text += "0"; - return; - } - auto j = cg::holds_vector(tmaStrides) - ? modeWithBasis(tmaStrides, path) - : (basisAt(tmaStrides, path) ? std::optional(0) - : std::nullopt); - if (!j) { - text += "0"; - return; - } - if (*j == 0) { - int64_t stride = asInt(d); - if (stride <= 0) { - ok = false; - return; - } - int64_t num = stride * elemBits, den = tmaBits; - int64_t g = std::gcd(num, den); - num /= g; - den /= g; - text += std::to_string(num) + - (den == 1 ? "" : "/" + std::to_string(den)) + "@0"; - return; - } - auto mode = cg::holds_vector(tmaStrides) ? tmaStrides[*j] : tmaStrides; - auto flat = cg::flatten(mode); - if (cg::holds_vector(flat) && cg::rank(flat) != 1) { - ok = false; - return; - } - text += "1@" + std::to_string(*j); - }; - write(gmem.shape(), gmem.stride(), basis.stride()); - if (!ok) - return std::nullopt; - return layoutOf(cg::to_string(gmem.shape()) + ":" + text); -} - -// The smem tile layout without its swizzle. -std::optional plainLayoutOf(Type t) { - if (auto l = llvm::dyn_cast(t)) - return l.getRef(); - if (auto c = llvm::dyn_cast(t)) - return c.getRef().layout_b(); - return std::nullopt; -} - -struct TmaAtomInfo { - int64_t elemBits = 0, copyBits = 0; -}; - -std::optional tmaAtomInfoOf(Type atom) { - TmaAtomInfo info; - if (auto l = - llvm::dyn_cast(atom)) { - info.elemBits = bitsOfType(l.getValType()); - info.copyBits = l.getCopyBits(); - } else if (auto s = - llvm::dyn_cast( - atom)) { - info.elemBits = bitsOfType(s.getValType()); - info.copyBits = s.getCopyBits(); - } else if (auto r = - llvm::dyn_cast( - atom)) { - info.elemBits = bitsOfType(r.getValType()); - info.copyBits = r.getCopyBits(); - } else if (auto l = - llvm::dyn_cast( - atom)) { - info.elemBits = bitsOfType(l.getValType()); - info.copyBits = l.getCopyBits(); - } else if (auto s = - llvm::dyn_cast( - atom)) { - info.elemBits = bitsOfType(s.getValType()); - info.copyBits = s.getCopyBits(); - } else { - return std::nullopt; - } - if (info.elemBits <= 0 || info.copyBits <= 0) - return std::nullopt; - return info; -} - -// make_non_exec_tiled_tma_load/store: the atom and the TMA tensor. -LogicalResult inferTmaAtom(MLIRContext *ctx, std::optional loc, - Type gmemType, Type smemType, Type ctaVMapType, - Attribute kind, Attribute format, - SmallVectorImpl &out) { - auto gmem = llvm::dyn_cast(gmemType); - auto smem = plainLayoutOf(smemType); - auto ctaVMap = llvm::dyn_cast(ctaVMapType); - if (!gmem || !smem || !ctaVMap) - return emitOptionalError( - loc, "expects a gmem tensor, an smem layout and a CTA value map"); - auto glayout = llvm::dyn_cast(gmem.getLayout()); - if (!glayout) - return emitOptionalError(loc, "gmem tensor needs a plain layout"); - Type elem = gmem.getValueType(); - auto fmt = tmaFormatOf(elem, format); - if (!fmt || fmt->bits <= 0) - return emitOptionalError(loc, "unknown TMA data format"); - int64_t elemBits = bitsOfType(elem); - cg::layout gbasis = - tmaGbasis(glayout.getRef(), elemBits, fmt->bits, *smem, ctaVMap.getRef()); - if (!cg::is_valid(gbasis)) - return emitOptionalError(loc, "cannot build the TMA basis"); - auto tensor = tmaTensorLayout(glayout.getRef(), elemBits, fmt->bits, gbasis); - if (!tensor) - return emitOptionalError(loc, "cannot build the TMA tensor"); - int64_t copyBits = asInt(cg::size(gbasis)) * fmt->bits; - auto gbasisType = LayoutType::get(ctx, gbasis); - std::optional dataFormat = - cute_nvgpu::symbolizeTmaDataFormat(fmt->name); - Type atom; - if (auto k = llvm::dyn_cast_or_null(kind)) { - atom = cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::get( - ctx, k.getValue(), elem, copyBits, gbasisType, dataFormat); - } else if (auto k = - llvm::dyn_cast_or_null(kind)) { - atom = cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::get( - ctx, k.getValue(), elem, copyBits, gbasisType, dataFormat); - } else { - atom = cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::get( - ctx, elem, copyBits, gbasisType, dataFormat); - } - std::string zeros = "("; - for (size_t i = 0; i < cg::rank(gbasis.shape()); ++i) - zeros += i ? ",0" : "0"; - auto iter = cg::from_string(zeros + ")"); - if (!iter) - return emitOptionalError(loc, "cannot build the TMA coordinate"); - out.push_back(atom); - out.push_back(CoordTensorType::get(ctx, IntTupleType::get(ctx, *iter), - LayoutType::get(ctx, *tensor))); - return success(); -} - -} // namespace - -// ---------------------------------------------------------------- cute copy -// ops - -// TiledCopy::get_layoutD_TV over the reference tile, then the tiled copy of -// another atom with that layout (make_tiled_copy_D). -Type mlir::cutlass_compiler::cute::tiledCopyD(Type atom, TiledCopyType tiled) { - auto tc = tiledCopyOf(tiled); - if (!tc) - return {}; - auto shape = tilerShape(tc->tiler); - if (!shape) - return {}; - auto refShape = - cg::from_string("(" + cg::to_string(*shape) + ",1)"); - if (!refShape) - return {}; - cg::layout refD(*refShape); - cg::layout ref2trg = - cg::composition(cg::right_inverse(tc->traits.ref), tc->traits.dst); - cg::layout tv = tile2thrfrg(*tc, refD, ref2trg); - if (!cg::is_valid(tv) || cg::rank(tv.shape()) < 2) - return {}; - cg::layout layoutTV = zeroUnitModes( - cg::make_layout(std::vector{cg::get(tv, 0), cg::get(tv, 1)})); - MLIRContext *ctx = atom.getContext(); - return TiledCopyType::get(ctx, atom, LayoutAttr::get(ctx, layoutTV), - tiled.getTilerMn()); -} - -LogicalResult TiledCopyPartitionSOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto tc = tiledCopyOf(a.getTiledCopy().getType()); - auto c = carrierOf(a.getInput().getType()); - auto coord = llvm::dyn_cast(a.getCoord().getType()); - if (!tc || !c || !coord) - return emitOptionalError( - loc, "expects a tiled copy, a tensor and a thread coord"); - return infer(loc, out, [&] { - return partitionCopy(ctx, *tc, *c, coord.getRef(), true); - }); -} - -LogicalResult TiledCopyPartitionDOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto tc = tiledCopyOf(a.getTiledCopy().getType()); - auto c = carrierOf(a.getInput().getType()); - auto coord = llvm::dyn_cast(a.getCoord().getType()); - if (!tc || !c || !coord) - return emitOptionalError( - loc, "expects a tiled copy, a tensor and a thread coord"); - return infer(loc, out, [&] { - return partitionCopy(ctx, *tc, *c, coord.getRef(), false); - }); -} - -LogicalResult TiledCopyRetileOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto tc = tiledCopyOf(a.getTiledCopy().getType()); - auto c = carrierOf(a.getInput().getType()); - if (!tc || !c) - return emitOptionalError(loc, "expects a tiled copy and a tensor"); - return infer(loc, out, [&] { return retileCopy(ctx, *tc, *c); }); -} - -// ---------------------------------------------------------------- cute_nvgpu - -namespace mlir::cutlass_compiler::cute_nvgpu { - -LogicalResult AtomSetValueOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, DictionaryAttr, - PropertyRef, RegionRange, SmallVectorImpl &out) { - out.push_back(operands[0].getType()); - return success(); -} - -// The executable atom of a TMA descriptor: the element, copy size and basis -// carry over; the kind becomes the mode and, for loads, the CTA count. -LogicalResult AtomCopyMakeExecTmaOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - bool over = a.getOverride(), noOob = a.getNoFullyOobTile(); - Type in = a.getInput().getType(); - auto stride = cg::from_string("()"); - if (!stride) - return emitOptionalError(loc, "cannot build the TMA stride"); - auto gStride = StrideType::get(ctx, *stride); - if (auto load = llvm::dyn_cast(in)) { - TiledTmaLoad kind = load.getKind(); - bool twoSm = kind == TiledTmaLoad::sm_100_2sm || - kind == TiledTmaLoad::sm_100_2sm_multicast; - bool mcast = kind == TiledTmaLoad::sm_90_multicast || - kind == TiledTmaLoad::sm_100_2sm_multicast; - out.push_back(CopyAtomTmaLoadType::get( - ctx, load.getValType(), /*sparsity=*/1, load.getCopyBits(), - TmaLoadMode::tiled, twoSm ? 2 : 1, gStride, mcast, load.getTmaGbasis(), - over, noOob)); - return success(); - } - if (auto store = llvm::dyn_cast(in)) { - out.push_back(CopyAtomTmaStoreType::get( - ctx, store.getValType(), /*sparsity=*/1, store.getCopyBits(), - TmaStoreMode::tiled, gStride, store.getTmaGbasis(), over, noOob)); - return success(); - } - if (auto red = llvm::dyn_cast(in)) { - out.push_back(CopyAtomTmaReduceType::get( - ctx, red.getValType(), red.getCopyBits(), TmaStoreMode::tiled, - red.getKind(), gStride, red.getTmaGbasis())); - return success(); - } - if (auto load = llvm::dyn_cast(in)) { - Im2ColTmaLoad kind = load.getKind(); - bool twoSm = kind == Im2ColTmaLoad::sm_100_2sm || - kind == Im2ColTmaLoad::sm_100_2sm_multicast; - bool mcast = kind == Im2ColTmaLoad::sm_90_multicast || - kind == Im2ColTmaLoad::sm_100_2sm_multicast; - out.push_back(CopyAtomIm2ColTmaLoadType::get( - ctx, load.getValType(), load.getCopyBits(), twoSm ? 2 : 1, gStride, - mcast, load.getTmaGbasis())); - return success(); - } - if (auto store = llvm::dyn_cast(in)) { - out.push_back(CopyAtomIm2ColTmaStoreType::get(ctx, store.getValType(), - store.getCopyBits(), gStride, - store.getTmaGbasis())); - return success(); - } - // The gather/scatter atoms' executable forms are not recovered. - return emitOptionalError(loc, - "expects a non-executable tiled or im2col TMA atom"); -} - -LogicalResult AtomCopyMakeNonExecTiledTmaLoadOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), - a.getSmemLayout().getType(), a.getCtaVMap().getType(), - a.getKindAttr(), a.getTmaFormatAttr(), out); -} - -LogicalResult AtomCopyMakeNonExecTiledTmaStoreOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), - a.getSmemLayout().getType(), a.getCtaVMap().getType(), - /*kind=*/nullptr, a.getTmaFormatAttr(), out); -} - -LogicalResult AtomCopyMakeNonExecTiledTmaReduceOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - return inferTmaAtom(ctx, loc, a.getGmemTensor().getType(), - a.getSmemLayout().getType(), a.getCtaVMap().getType(), - a.getKindAttr(), a.getTmaFormatAttr(), out); -} - -// tma_partition: the smem tile's contiguous vectors, one instruction each, -// read through every tensor; a multicast CTA starts at its share. -LogicalResult AtomTmaPartitionOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto info = tmaAtomInfoOf(a.getTmaAtom().getType()); - auto coord = llvm::dyn_cast(a.getCtaCoord().getType()); - auto ctaLayout = llvm::dyn_cast(a.getCtaLayout().getType()); - auto smem = carrierOf(a.getSmemTensor().getType()); - if (!info || !coord || !ctaLayout || !smem) - return emitOptionalError( - loc, "expects a TMA atom, a CTA coord and layout, and tensors"); - auto smemLayout = llvm::dyn_cast_or_null(smem->layout); - if (!smemLayout || !cg::holds_vector(smemLayout.getRef().shape())) - return emitOptionalError(loc, - "smem tensor needs a plain layout of rank >= 1"); - int64_t numVal = info->copyBits / info->elemBits; - cg::layout inv = cg::right_inverse(cg::get(smemLayout.getRef(), 0)); - int64_t b = asInt(cg::size(inv)); - int64_t n = asInt(cg::size(smemLayout.getRef(), 0)); - if (b <= 0 || n <= 0 || n % b || numVal <= 0) - return emitOptionalError(loc, "cannot tile the smem vector"); - // tile_to_shape: the vector repeated over the tile, broadcast modes at 0. - cg::layout layoutV = inv; - if (n != b) { - auto rest = layoutOf(std::to_string(n / b) + ":1"); - layoutV = cg::logical_product(inv, *rest); - if (!cg::is_valid(layoutV)) - return emitOptionalError(loc, "cannot tile the smem vector"); - } - auto tmaV = layoutOf(std::to_string(numVal) + ":1"); - layoutV = cg::logical_divide(layoutV, *tmaV); - if (!cg::is_valid(layoutV)) - return emitOptionalError(loc, "cannot divide the smem vector"); - // The multicast offset inside the instruction's vector. - int64_t ctas = asInt(cg::cosize(ctaLayout.getRef())); - std::optional multicast; - if (ctas > 1) { - // The CTA layout scaled by the share: (cosize:share) o cta_layout. - auto share = - layoutOf(std::to_string(ctas) + ":" + std::to_string(numVal / ctas)); - cg::layout scaled = cg::composition(*share, ctaLayout.getRef()); - if (!cg::is_valid(scaled)) - return emitOptionalError(loc, "cannot scale the CTA layout"); - multicast = cg::layout_eval(coord.getRef(), scaled); - } - SmallVector results; - auto partition = [&](Type tensor) -> Type { - auto c = carrierOf(tensor); - if (!c) - return {}; - auto layout = llvm::dyn_cast(c->layout); - if (!layout) - return {}; - cg::layout v = cg::coalesce(composeFirst(layout.getRef(), layoutV), - profileOf("((1,1))")); - if (!cg::is_valid(v)) - return {}; - // Only the vector's unit modes get a zero stride; the rest keep theirs. - Type result = LayoutType::get( - ctx, zeroUnitModesHead(v, cg::rank(v.shape()) - 1, false)); - if (!multicast) - return rebuild(ctx, *c, result); - // The index split over the vector's modes, column-major, keeping what - // its divisibility says of each coordinate. - auto vec = cg::get(v.shape(), 0); - if (!cg::holds_vector(vec)) - return {}; - auto split = splitIndex(*multicast, vec[0]); - if (!split) - return {}; - std::string crd = "((" + *split + ",0)"; - for (size_t i = 1; i < cg::rank(v.shape()); ++i) - crd += ",0"; - auto at = cg::from_string(crd + ")"); - if (!at) - return {}; - cg::int_tuple offset = cg::layout_eval(*at, v); - if (c->ptr) - return CuteMemRefType::get(ctx, movedPtr(c->ptr, offset), result); - if (c->iter) { - auto sum = cg::arith_tuple_sum(c->iter.getRef(), offset); - if (!cg::is_valid(sum)) - return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, std::move(sum)), - result); - } - return rebuild(ctx, *c, result); - }; - results.push_back(partition(a.getSmemTensor().getType())); - for (Value target : a.getTargetTensors()) - results.push_back(partition(target.getType())); - for (Type t : results) - if (!t) - return emitOptionalError(loc, "cannot partition for TMA"); - out.append(results.begin(), results.end()); - return success(); -} - -namespace { -// make_cotiled_copy over a tensor memory tensor for an atom whose thread -// layout is `atomT`. -Type tmemTiledCopy(MLIRContext *ctx, std::optional loc, Type atom, - Type tmem, const std::string &atomT, - SmallVectorImpl &out) { - auto traits = copyTraitsOf(atom); - auto m = llvm::dyn_cast(tmem); - auto layout = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); - if (!traits || !layout || !cg::is_valid(traits->valId)) - return {}; - auto t = layoutOf(atomT); - if (!t) - return {}; - cg::layout v = cg::coalesce(cg::upcast(traits->bits, traits->valId)); - if (!cg::is_valid(v)) - return {}; - auto tiled = cotiledCopy(cg::make_layout(std::vector{*t, v}), - layout.getRef()); - if (!tiled) - return {}; - return tiledCopyType(ctx, atom, tiled->first, tiled->second); -} -} // namespace - -// make_tmem_copy: four warps, each at the same place in its own lane quarter. -LogicalResult AtomCopyMakeTmemCopyOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto traits = copyTraitsOf(a.getAtom().getType()); - if (!traits || traits->bits <= 0) - return emitOptionalError(loc, "expects a tmem atom"); - int64_t lane = kTmemLaneBits / traits->bits; - return infer(loc, out, [&] { - return tmemTiledCopy(ctx, loc, a.getAtom().getType(), - a.getTmemMemref().getType(), - "(32,4):(0," + std::to_string(32 * lane) + ")", out); - }); -} - -// make_s2t_copy (make_utccp_copy): one thread per CTA. -LogicalResult AtomCopyMakeS2TCopyOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - auto traits = copyTraitsOf(a.getCopyS2tAtom().getType()); - if (!traits) - return emitOptionalError(loc, "expects an s2t atom"); - return infer(loc, out, [&] { - return tmemTiledCopy(ctx, loc, a.getCopyS2tAtom().getType(), - a.getTmemMemref().getType(), - std::to_string(traits->threads) + ":0", out); - }); -} - -namespace { -// The smem side of an s2t copy as descriptor units: the tile recast to 128 -// bits, viewed through `desc`. -LogicalResult s2tSmemDescView(std::optional loc, Type atom, Type view, - Type desc, SmallVectorImpl &out) { - if (auto tiled = llvm::dyn_cast(atom)) - atom = tiled.getCopyAtom(); - auto m = llvm::dyn_cast(view); - auto layout = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); - if (!llvm::isa(atom) || !layout) - return emitOptionalError(loc, "expects an s2t atom and an smem tensor"); - // The compiler wants mode 0 as (values, 1). - const cg::layout &l = layout.getRef(); - auto m0 = cg::get(l.shape(), 0); - if (!cg::holds_vector(m0) || cg::rank(m0) != 2 || - !cg::holds_int(cg::size(m0, 1)) || cg::size(m0, 1).as_int64() != 1) - return emitOptionalError(loc, "expects mode 0 as (values, 1)"); - return infer(loc, out, [&]() -> Type { - cg::layout v = cg::recast(128, bitsOfType(m.getValueType()), l); - if (!cg::is_valid(v)) - return {}; - return SmemDescViewType::get(atom.getContext(), desc, - LayoutAttr::get(atom.getContext(), v)); - }); -} - -// An operation whose inference rule is not recovered: the compiler either -// aborts on the inputs tried or needs inputs no probe produced. -LogicalResult unrecovered(std::optional loc, StringRef op) { - return emitOptionalError(loc, op, " result inference is not recovered"); -} -} // namespace - -LogicalResult AtomGetCopyS2TSmemDescViewOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - return s2tSmemDescView(loc, a.getAtom().getType(), a.getView().getType(), - SmemDescType::get(ctx), out); -} - -LogicalResult AtomGetCopyS2TSmemDescViewSM107Op::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - return s2tSmemDescView(loc, a.getAtom().getType(), a.getView().getType(), - SmemDescSM107Type::get(ctx), out); -} - -// A generic-space pointer to the parameter, at its type's natural alignment. -LogicalResult GetGridConstantPointerOp::inferReturnTypes( - MLIRContext *ctx, std::optional, ValueRange operands, - DictionaryAttr, PropertyRef, RegionRange, SmallVectorImpl &out) { - Type t = operands[0].getType(); - bool desc = llvm::isa(t); - out.push_back(PtrType::get(ctx, t, AddressSpace::generic, - desc ? 64 : PtrType::getNaturalAlignment(t), {}, - {})); - return success(); -} - -// The residues of the TMA tensor's coordinates: one per box mode, starting -// at the extent of the gmem mode it runs over and counting down. -LogicalResult AtomMakeTmaResidueTensorOp::inferReturnTypes( - MLIRContext *ctx, std::optional loc, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange, - SmallVectorImpl &out) { - Adaptor a(operands, attrs, props); - Type atom = a.getTmaAtom().getType(), gbasis; - if (auto l = llvm::dyn_cast(atom)) - gbasis = l.getTmaGbasis(); - if (auto s = llvm::dyn_cast(atom)) - gbasis = s.getTmaGbasis(); - auto basis = llvm::dyn_cast_or_null(gbasis); - auto m = llvm::dyn_cast(a.getGmemTensor().getType()); - auto gmem = m ? llvm::dyn_cast(m.getLayout()) : LayoutType(); - return infer(loc, out, [&]() -> Type { - if (!basis || !gmem || !cg::holds_vector(basis.getRef().stride())) - return {}; - const cg::stride &modes = basis.getRef().stride(); - // Each box mode's one basis: the gmem leaf it runs over. - std::vector> paths; - for (size_t j = 0; j < cg::rank(modes); ++j) { - auto flat = cg::flatten(modes[j]); - const cg::stride &leaf = - cg::holds_vector(flat) && cg::rank(flat) == 1 ? flat[0] : flat; - if (!cg::holds_scaled_basis(leaf)) - return {}; - paths.push_back(std::get(leaf).modes()); - } - std::vector iter(paths.size()), strides; - std::vector path; - const cg::layout &g = gmem.getRef(); - walkLeaves(g.shape(), g.stride(), g.stride(), path, - [&](const auto &s, const auto &, const auto &, - const std::vector &p) { - auto j = - std::find(paths.begin(), paths.end(), p) - paths.begin(); - if (j == static_cast(paths.size())) - return strides.push_back("0"); - iter[j] = cg::to_string(s); - strides.push_back("-1@" + std::to_string(j)); - }); - for (const std::string &e : iter) - if (e.empty()) - return {}; - size_t i = 0; - auto layout = cg::from_string(cg::to_string(g.shape()) + ":" + - nestLike(g.shape(), strides, i)); - auto start = - cg::from_string("(" + llvm::join(iter, ",") + ")"); - if (!layout || !start) - return {}; - return CoordTensorType::get(ctx, IntTupleType::get(ctx, *start), - LayoutType::get(ctx, *layout)); - }); -} - -LogicalResult AtomGetCoordTensorOp::inferReturnTypes( - MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, - PropertyRef, RegionRange, SmallVectorImpl &) { - return unrecovered(loc, "cute_nvgpu.atom.get_coord_tensor"); -} - -// The im2col and gather/scatter TMA atoms and the circular descriptors -// validate inputs (canonical UMMA layouts, im2col corners) whose rules are -// not recovered. -#define CUTE_UNRECOVERED(Op, name) \ - LogicalResult Op::inferReturnTypes( \ - MLIRContext *, std::optional loc, ValueRange, DictionaryAttr, \ - PropertyRef, RegionRange, SmallVectorImpl &) { \ - return unrecovered(loc, name); \ - } -CUTE_UNRECOVERED(AtomCopyMakeNonExec2DGather4TmaLoadOp, - "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load") -CUTE_UNRECOVERED(AtomCopyMakeNonExec2DScatter4TmaStoreOp, - "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store") -CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaLoadOp, - "cute_nvgpu.atom.make_non_exec_im2col_tma_load") -CUTE_UNRECOVERED(AtomCopyMakeNonExecIm2ColTmaStoreOp, - "cute_nvgpu.atom.make_non_exec_im2col_tma_store") -CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM103Op, - "cute_nvgpu.sm103.make_umma_smem_desc_circular") -CUTE_UNRECOVERED(MakeUMMASmemDescCircularSM107Op, - "cute_nvgpu.sm107.make_umma_smem_desc_circular") -#undef CUTE_UNRECOVERED - -} // namespace mlir::cutlass_compiler::cute_nvgpu diff --git a/mlir/dialects/cute_ir/CuteInference.h b/mlir/dialects/cute_ir/CuteInference.h deleted file mode 100644 index c7837b20..00000000 --- a/mlir/dialects/cute_ir/CuteInference.h +++ /dev/null @@ -1,24 +0,0 @@ -//===- CuteInference.h - result types the compiler infers -------*- C++ -*-===// - -#ifndef MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H -#define MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H - -#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" - -namespace mlir::cutlass_compiler::cute { -// make_tiled_copy_D: `atom` tiled over `tiled`'s destination thread-value -// layout and tiler, or null when the atom is not one this tree knows. -Type tiledCopyD(Type atom, TiledCopyType tiled); - -// Result types of operations the compiler does not infer (their builders -// take the type): what the DSL passes, or null when not computable here. -Type makeLayoutResultType(Type shape, Type stride); // stride may be null -Type makeViewResultType(Type iter, Type layout); -Type getResultType(Type input, ArrayRef mode); -Type assumeResultType(Type src); -Type makeArithTupleIterResultType(Type tuple); -Type fastDivmodCreateDivisorResultType(Type divisor); -Type fastDivmodComputeResultType(Type divisor); // both results -} // namespace mlir::cutlass_compiler::cute - -#endif // MLIR_DIALECTS_CUTE_IR_CUTEINFERENCE_H diff --git a/mlir/dialects/cute_ir/CuteOps.cpp b/mlir/dialects/cute_ir/CuteOps.cpp deleted file mode 100644 index 0c8ddaf3..00000000 --- a/mlir/dialects/cute_ir/CuteOps.cpp +++ /dev/null @@ -1,9 +0,0 @@ -//===- CuteOps.cpp - the compiler's CuTe operations -----------------------===// - -#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" -// Operand constraints name cute_nvgpu and LLVM types. -#include "mlir/Dialect/LLVMIR/LLVMTypes.h" -#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" - -#define GET_OP_CLASSES -#include "mlir/dialects/cute_ir/CuteOps.cpp.inc" diff --git a/mlir/dialects/cute_ir/CuteOps.td b/mlir/dialects/cute_ir/CuteOps.td deleted file mode 100644 index d2e342df..00000000 --- a/mlir/dialects/cute_ir/CuteOps.td +++ /dev/null @@ -1,3795 +0,0 @@ -// The cute dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. -// -// Sources: the operation bindings (`_cute_ops_gen.py`: names, classes, -// docstrings, operand/result/attribute structure, inference) and the live -// compiler (`_cutlass_ir` loaded from Python): each operand and result -// constraint carries the verifier's wording and accepts every type class the -// verifier let through when probed with one instance of every type; traits -// and interfaces are the registered operation's; memory effects come from -// loop-invariant code motion (hoisted: Pure), canonicalize (dead: no write) -// and CSE (merged: at most reads). Operations whose effects include writes -// or could not be told apart carry none (unknown effects). -// -// Operations keep the generic assembly form; the compiler's custom syntax is -// shown in each description. -// -// Constraints not recovered: -// cute.deref_desc_iter operand #0: no probe type accepted; any type allowed -// cute.make_desc_iter operand #0: no probe type accepted; any type allowed - -#ifndef MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD -#define MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD - -include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" -include "mlir/dialects/cute_ir/CuteEnums.td" - -include "mlir/Interfaces/InferTypeOpInterface.td" -include "mlir/Interfaces/SideEffectInterfaces.td" - -// Operand and result constraints, in the compiler's words. -def Cute_32BitSignlessIntegerOr64BitSignless : Type, "32-bit signless integer or 64-bit signless integer">; -def Cute_AMemoryReferenceTypeThatRepresentsATensor : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "A memory reference type that represents a tensor view with an iterator and layout or CuTe Layout Type or Cute shape type">; -def Cute_AMemoryReferenceTypeThatRepresentsATensor2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self))">, "A memory reference type that represents a tensor view with an iterator and layout or Cute coord_tensor type">; -def Cute_AMemoryReferenceTypeThatRepresentsATensor3 : Type($_self))">, "A memory reference type that represents a tensor view with an iterator and layout">; -def Cute_ATupleIsAFixedLengthOrderedList : Type($_self))">, "A tuple is a fixed-length, ordered list consisting of multiple elements.">; -def Cute_ATypeInterfaceForDescriptorIteratorTypes : Type, "A type interface for descriptor iterator types.">; // not recovered: no pool type passed -def Cute_ATypeInterfaceForMemoryDescriptorsSuchAs : Type, "A type interface for memory descriptors such as smem desc etc.">; // not recovered: no pool type passed -def Cute_ArithmeticTupleType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "Arithmetic tuple type">; -def Cute_AtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "AtomTypeInterface instance">; -def Cute_CopyAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance">; -def Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance or CopyAtomTypeInterfaceV2 instance">; -def Cute_CopyAtomTypeInterfaceV2Instance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self))">, "CopyAtomTypeInterfaceV2 instance">; -def Cute_CuTeComposedLayoutType : Type($_self))">, "CuTe composed layout type">; -def Cute_CuTeIteratorTypeWithIntTupleAsIts : Type($_self))">, "CuTe iterator type with int_tuple as its underlying implementation">; -def Cute_CuTeLayoutType : Type($_self))">, "CuTe Layout Type">; -def Cute_CuTeLayoutTypeOrCuteSwizzleType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self))">, "CuTe Layout Type or Cute swizzle type">; -def Cute_CuTeLayoutTypeOrSwizzleWithValidParameters : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self))">, "CuTe Layout Type or swizzle with valid parameters that can be represented by 32-bit masks">; -def Cute_CuTePointerTo32BitSignlessInteger : Type($_self))">, "CuTe pointer to 32-bit signless integer">; -def Cute_CuTePointerType : Type($_self))">, " CuTe pointer type ">; -def Cute_CuTePointerTypeOrAMemoryReferenceType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::FastDivmodDivisorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::FloatType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ($_self.getDialect().getNamespace() == \"ptr\" || (::llvm::isa<::mlir::OpaqueType>($_self) && ::llvm::cast<::mlir::OpaqueType>($_self).getDialectNamespace() == \"ptr\")) || ::llvm::isa<::mlir::VectorType>($_self))">, " CuTe pointer type or A memory reference type that represents a tensor view with an iterator and layout or Integer type with constraints. or CuTe fast divmod divisor type or loadable maybe-static CuTe type with runtime storage or LLVM type with size">; -def Cute_CuTePointerTypeOrLLVMPointerType : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self))">, " CuTe pointer type or LLVM pointer type">; -def Cute_CuTeSparseElementType : Type($_self))">, " CuTe sparse element type">; -def Cute_CuteCoordTensorType : Type($_self))">, "Cute coord_tensor type">; -def Cute_CuteCoordType : Type($_self))">, "Cute coord type">; -def Cute_CuteShapeType : Type($_self))">, "Cute shape type">; -def Cute_CuteShapeTypeOrIntTupleAsIntegerOr : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "Cute shape type or IntTuple as integer or tuple of IntTuple(s) or Cute coord type">; -def Cute_CuteShapeTypeOrIntTupleAsIntegerOr2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "Cute shape type or IntTuple as integer or tuple of IntTuple(s) or Cute coord type or CuTe Layout Type or CuTe composed layout type or ViewTypeInterface instance">; -def Cute_CuteStrideType : Type($_self))">, "Cute stride type">; -def Cute_CuteTileType : Type($_self))">, "Cute tile type">; -def Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant : Type($_self))">, "Divisor for fast unsigned integer divmods by invariant integers.">; -def Cute_GeneralizedIteratorTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self))">, " Generalized iterator type of CuTe ">; -def Cute_GeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self))">, " Generalized layout type of CuTe ">; -def Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, " Generalized layout type of CuTe or ViewTypeInterface instance">; -def Cute_I32I64OrConstrainedInteger : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || $_self.isSignlessInteger(32) || $_self.isSignlessInteger(64))">, "i32, i64 or Constrained integer">; -def Cute_IntTupleAsIntegerOrTupleOfIntTupleS : Type($_self))">, "IntTuple as integer or tuple of IntTuple(s)">; -def Cute_IntTupleAsIntegerOrTupleOfIntTupleS2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self))">, "IntTuple as integer or tuple of IntTuple(s) or Cute shape type">; -def Cute_IntTupleTypeThatDepthIs0 : Type($_self))">, "Int tuple type that depth is 0">; -def Cute_Integer : Type($_self))">, "integer">; -def Cute_IntegerOrIntegerTypeWithConstraints : Type($_self) || ::llvm::isa<::mlir::IntegerType>($_self))">, "integer or Integer type with constraints. ">; -def Cute_IntegerTypeWithConstraints : Type($_self))">, " Integer type with constraints. ">; -def Cute_MaybeStaticTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "MaybeStaticTypeInterface instance">; -def Cute_MemrefInGenericOrRegisterMemOrSmem : Type($_self))">, "memref in generic or register mem or smem">; -def Cute_MemrefInSmemWithStaticAndNonComposed : Type($_self))">, "memref in smem with static and non-composed layout">; -def Cute_MmaAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "MmaAtomTypeInterface instance">; -def Cute_OfRanks1 : Type($_self))">, " of ranks 1">; -def Cute_OfRanks1OrFloatingPointOrInteger : Type($_self) || ::llvm::isa<::mlir::FloatType>($_self) || ::llvm::isa<::mlir::VectorType>($_self))">, " of ranks 1 or floating-point or integer">; -def Cute_PointerTypeInterfaceInstance : Type($_self))">, "PointerTypeInterface instance">; -def Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self))">, "PrefetchAtomTypeInterface or TiledCopyType with PrefetchAtomTypeInterface copy atom">; -def Cute_RefinableTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "RefinableTypeInterface instance">; -def Cute_RefinableTypeInterfaceInstanceOrInteger : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self) || ::llvm::isa<::mlir::IntegerType>($_self))">, "RefinableTypeInterface instance or integer">; -def Cute_StaticLayout : Type($_self))">, "static layout">; -def Cute_StaticShape : Type($_self))">, "static shape">; -def Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "Tile-like types, e.g. LayoutType, ShapeType, TileType">; -def Cute_TiledCopyAtomType : Type($_self))">, "Tiled copy atom type.">; -def Cute_TiledCopyAtomTypeOrTiledCopyV2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self))">, "Tiled copy atom type. or Tiled copy V2 atom type.">; -def Cute_TiledCopyV2AtomType : Type($_self))">, "Tiled copy V2 atom type.">; -def Cute_TiledMMAAtomType : Type($_self))">, "Tiled MMA atom type.">; -def Cute_TypeInterfaceForCuteTypesCanBePrinted : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ComposedLayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::PtrType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::FloatType>($_self))">, "Type interface for cute types can be printed by `cute.print`">; -def Cute_ValueTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "ValueTypeInterface instance">; -def Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraints : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "ValueTypeInterface instance or Integer type with constraints. ">; -def Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraintsOr : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ConstrainedIntType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::SwizzleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || $_self.isSignlessInteger(8) || $_self.isSignlessInteger(16) || $_self.isSignlessInteger(32) || $_self.isSignlessInteger(64) || ::llvm::isa<::mlir::IndexType>($_self))">, "ValueTypeInterface instance or Integer type with constraints. or 8/16/32/64-bit integer or index">; -def Cute_VectorOf1BitSignlessIntegerValuesOf : Type($_self))">, "vector of 1-bit signless integer values of ranks 1">; -def Cute_ViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance">; -def Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance or Generalized layout type of CuTe ">; -def Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::CoordTensorType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance or Generalized layout type of CuTe or Cute tile type">; -def Cute_XTupleLikeType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "XTuple-like type">; -def Cute_XTupleLikeTypeOrCuTeLayoutType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::IntTupleType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::ShapeType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::StrideType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TileType>($_self))">, "XTuple-like type or CuTe Layout Type">; - -// Operations whose result types are inferred in C++. Verification compares -// them with the inferred ones where the rule is recovered for the operands; -// elsewhere the types written stand, as the compiler accepted them. -class Cute_InferredOp traits = []> - : Cute_Op]> { - let extraClassDefinition = [{ - ::llvm::LogicalResult $cppClass::refineReturnTypes( - ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, - ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, - ::mlir::PropertyRef properties, ::mlir::RegionRange regions, - ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { - ::llvm::SmallVector<::mlir::Type, 4> inferred; - if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, - attributes, properties, regions, - inferred))) - return ::mlir::success(); - if (!isCompatibleReturnTypes(inferred, returnTypes)) - return ::mlir::emitOptionalError( - location, "'", getOperationName(), "' op inferred type(s) ", - inferred, " are incompatible with return type(s) of operation ", - returnTypes); - return ::mlir::success(); - } - }]; -} - -def Cute_AddOffsetOp : Cute_InferredOp<"add_offset", [Pure]> { - let description = [{ - The `cute.add_offset` operation accepts an input IteratorTypeInterface and - a corresponding offset type which must be an - arithmetic tuple. It returns a new SSA value as type IteratorTypeInterface. - - When src operand is a sparse pointer, the logical offset must be a multiple - of the sparsity factor. The actual offset added to the src operand is - equal to the logical offset divided by the sparsity factor. - - `add_offset(%it, N)` returns a new iterator pointing to an object with an - offset of`N` elements. It is similar to C++'s iterator arithmetic. - - Example: - - ```mlir - // Add offset to 1-D linear pointer - %ptr = cute.add_offset(%0, %1) : (!cute.ptr>, !cute.int_tuple<"?">) -> !cute.ptr> - - // Add offset to 1-D linear pointer with static offset - %off = cute.static : !cute.int_tuple<"1012"> - %ptr = cute.add_offset(%0, %off) : (!cute.ptr>) -> !cute.ptr> - - // Add offset as arithmetic tuple to N-D nested arithmetic tuple - %arith_tuple = cute.add_offset(%0, %1) : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(4,?))"> - - // Add offset as arithmetic tuple to N-D nested arithmetic tuple with static offset - %arith_tuple = cute.add_offset(%0, "(3,(1,3))") : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(4,?))"> - ``` - }]; - let arguments = (ins - Cute_GeneralizedIteratorTypeOfCuTe:$src, - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$offset - ); - let results = (outs - Cute_GeneralizedIteratorTypeOfCuTe:$dst - ); -} -def Cute_AlgorithmCopyOp : Cute_Op<"copy", [AttrSizedOperandSegments]> { - let description = [{ - CuTe's copy algorithm copies the elements of a source Tensor into the elements - of a destination Tensor. - - See the file cutlass/media/docs/cute/04_algorithms.md in the CUTLASS repository - for more information. - - A Tensor (cute.memref) encapsulates the data type, data location, and possibly - also the shape and stride of the tensor at compile time. As a result, copy can - and does dispatch, based on the types of its arguments, to use any of various - synchronous or asynchronous hardware copy instructions. - - An .copy accepts CuteCopyAtomTypeInterface as first parameter. To keep consistent - interface with CUTLASS C++, it could be either atom.tiled_copy or atom.universal_copy. - - Optional argument 'pred' is used to guard memory accesses performed in copy. - Predication works in granularity of atom_v. For src tensor whose shape is: - `((atom_v, rest_v), rest_m, rest_n, ...)`, the pred tensor shape should be: - `((rest_v), rest_m, rest_n, ...)`, we denoted this as the canonical format. - If the pred tensor's value equals 0, the corresponding copy will be skipped. - - When using atom.universal_copy instead of atom.tiled_copy, we allow the layout of - `pred` to deviate from the canonical format. Instead, the layout shape of `pred` - must match the layout shape of `src` or `dst`. Additionally, the `copy_bits` - parameter cannot be set for atom.universal_copy when predicate is present. - - The operands `src` and `dst` are variadic, each containing a variable number of tensors: - - For regular copy, `src` and `dst` each contain a single tensor. - - For copy with auxiliary operands, they contain the main tensor followed by - auxiliary tensors. For example: - - For load with row-wise reduction, `dst` = [data, stat]. - - For SPARSIFY (structured sparsity compression), `dst` = [data, metadata]. - - For TMA gather4, `src` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). - - For TMA scatter4, `dst` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). - - Example: - - ```mlir - cute.copy(%tiled_copy, %src, %dst) : ( - !cute.tiled_copy, - layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, - tiler_mn = <"[2048:1;1:1;4:1;1:1]">>, - !cute.coord_tensor<"(0,0)", "(((64, 128), 1), 1, 1, 2):(((1@0, 1@1), 0), 0, 0, 64@0)">, - !cute.memref, "(((64,128),1),1,1,2):(((1,64),0),0,0,8192)">) - - // atom.universal_copy with canonicalize layout format - cute.copy(%copy_atom, %src, %dst, %pred) : ( - !cute_nvgpu.atom.universal_copy, - !cute.memref, "((1,1),2,4):((0,0),1,2)">, - !cute.memref, "((1,1),2,4):((0,0),1,2)">, - !cute.memref, "((1),2,4):((0),1,2)">) - - // atom.universal_copy with relaxed layout format - cute.copy(%atom, %arg0, %arg1, %arg2) : ( - !cute_nvgpu.atom.universal_copy, - !cute.memref, "(4,4):(1,4)">, - !cute.memref, - !cute.memref, "(4,4):(1,4)">) - - ``` - }]; - let arguments = (ins - Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$copy_atom, - Variadic:$src, - Variadic:$dst, - Optional:$pred - ); - let results = (outs - ); -} -def Cute_AlgorithmGemmOp : Cute_Op<"gemm", [AttrSizedOperandSegments]> { - let description = [{ - CuTe's gemm algorithm takes 1 mma atom and 4 inputs(cute.memref/cute_nvgpu.smem_desc_view) - as input. - It performs the general matrix multiplication: - `D += A * B + C`. - Layout carried by operads determines the exact form of the gemm computation. - Generally speaking, there are 5 valid forms: - 1. `(V) x (V) => (V)`. - The element-wise product of vectors. Dispatches to FMA or MMA. - In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 1`. - 2. `(M) x (N) => (M,N)`. - The outer product of vectors. Dispatches to [3] with new mode K=(1). - In this case, `rank(A) = rank(B) = 1` and `rank(C) = rank(D) = 2`. - 3. `(M,K) x (N,K) => (M,N)`. - The product of matrices. Dispatches to [5] with MMA vector-mode V. - In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 2`. - 4. `(V,M) x (V,N) => (V,M,N)`. - The batched outer product of vectors. Accounts for register reuse and dispatches to [1] for each (m,n). - In this case, `rank(A) = rank(B) = 2` and `rank(C) = rank(D) = 3`. - 5. `(V,M,K) x (V,N,K) => (V,M,N)`. - The batched product of matrices. Dispatches to [4] for each (k). - In this case, `rank(A) = rank(B) = rank(C) = rank(D) = 3`. - - Currently, only the form in item 5 is supported. - - The operands `a` and `b` are variadic, each containing a variable number of tensors: - - For regular GEMM, `a` and `b` contain the GEMM A and B tensors respectively. - - For GEMM with auxiliary operands, `a` and `b` contain the GEMM A and B tensors followed by their respective auxiliary tensors. For example: - - For BlockScaledGemm, `a` = [A, SFA] and `b` = [B, SFB]. - - For SparseGemm, `a` = [A, E] and `b` = [B]. - - For BlockScaledSparseGemm, `a` = [A, SFA, E] and `b` = [B, SFB]. - See the file cutlass/media/docs/cute/04_algorithms.md and include/cute/algorithm/gemm.hpp in the CUTLASS repository for more information. - - Example (regular GEMM): - ```mlir - cute.gemm(%mma, %d, %a, %b, %c) { - } : (!cute.tiled_mma, - atom_layout_MNK = <"(1,1,1)">>, - !cute.memref, "((128,128),1,1):((131072,1),0,0)">, - !cute_nvgpu.smem_desc_view o 0 o ((128,16),1,4):((64,1),0,16)">, - !cute_nvgpu.smem_desc_view o 0 o ((128,16),1,4):((64,1),0,16)">, - !cute.memref, "((128,128),1,1):((131072,1),0,0)">) - ``` - - Example (block-scaled GEMM): - ```mlir - cute.gemm(%mma, %d, [%a, %sfa], [%b, %sfb], %c) : ( - !cute.tiled_mma, - atom_layout_MNK = <"(1,2,1):(0,1,0)">>, - !cute.memref, "((128,128),1,1):((65536,1),0,0)">, - [!cute_nvgpu.smem_desc_view, - !cute.memref, "((((32,4),4),(16,4)),1):((((262144,4),8388608),(0,1)),0)">], - [!cute_nvgpu.smem_desc_view, - !cute.memref, "((((32,4),4),(16,4)),1):((((262144,4),8388608),(0,1)),0)">], - !cute.memref, "((128,128),1,1):((65536,1),0,0)">) - ``` - }]; - let arguments = (ins - Cute_MmaAtomTypeInterfaceInstance:$mma_atom, - Cute_ViewTypeInterfaceInstance:$d, - Variadic:$a, - Variadic:$b, - Cute_ViewTypeInterfaceInstance:$c - ); - let results = (outs - ); -} -def Cute_AlgorithmPrefetchOp : Cute_Op<"prefetch"> { - let summary = "CuTe's prefetch algorithm prefetches the elements of source Tensor(s) into cache."; - let description = [{ - A Tensor encapsulates the data type, data location, and possibly - also the shape and stride of the tensor at compile time. As a result, prefetch can - and does dispatch, based on the types of its arguments, to use any of various - synchronous or asynchronous hardware prefetch instructions. - - An .prefetch accepts CutePrefetchAtomTypeinterface as first parameter. - - The `src` operand is variadic: - - For regular TMA prefetch, `src` is a single tensor. - - For 2D `tile::gather4` mode, `src` is `[data_coord_tensor, gmem_index_tensor]`, - mirroring `cute.copy` for the same mode. - - Example: - - ```mlir - cute.prefetch(%tiled_copy, %src) : ( - !cute.tiled_copy, - layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, - tiler_mn = <"[2048:1;1:1;4:1;1:1]">>, - !cute.coord_tensor<"(0,0)", "(((64, 128), 1), 1, 1, 2):(((1@0, 1@1), 0), 0, 0, 64@0)">) - - // gather4 mode: data tensor + gmem index tensor - cute.prefetch(%tiled_copy, %src_data, %src_idx) : ( - !cute.tiled_copy, ...>, - !cute.coord_tensor<...>, - !cute.coord_tensor<...>) - ``` - }]; - let arguments = (ins - Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom:$prefetch_atom, - Variadic:$src - ); - let results = (outs - ); -} -def Cute_AppendToRankOp : Cute_InferredOp<"append_to_rank", [Pure]> { - let summary = "Append element to input to rank N."; - let description = [{ - Example: - - ```mlir - %shp = cute.make_shape() : !cute.shape<"(5,4,3)"> - %elem = cute.make_shape() : !cute.shape<"(2,1)"> - %new_layout = cute.append_to_rank<5> (%shp, %elem) - : (..., ...) -> !cute.shape<"(5,4,3,(2,1),(2,1))"> - ``` - }]; - let arguments = (ins - Cute_ValueTypeInterfaceInstance:$input, - Cute_ValueTypeInterfaceInstance:$element, - SI32Attr:$rank - ); - let results = (outs - Cute_ValueTypeInterfaceInstance:$result - ); -} -def Cute_ApplySwizzleOp : Cute_InferredOp<"apply_swizzle", [Pure]> { - let description = [{ - The `apply_swizzle` operation applies a swizzle transformation to a pointer's - address, producing a new pointer with the swizzled address. This is necessary - for dereferencing pointers that use swizzled memory layouts, such as those used - for improved memory access patterns on GPUs. - - Two forms are supported: - - 1. **CutePtrType form**: The swizzle is encoded in the pointer type itself. - No explicit swizzle operand is required or allowed. - - ```mlir - %swizzled = cute.apply_swizzle(%ptr) : !cute.ptr, S<3,4,3>> - ``` - - 2. **llvm.ptr form**: The swizzle is passed as an optional attribute in the - attribute dictionary. If the swizzle attribute is absent, the operation - is a no-op (identity swizzle). - - ```mlir - // With explicit swizzle: - %swizzled = cute.apply_swizzle(%ptr) { S<3,4,3> } : !llvm.ptr<3> - // Identity (no-op): - %swizzled = cute.apply_swizzle(%ptr) : !llvm.ptr<3> - ``` - - If the input pointer type has a non-identity swizzle attribute, this operation - computes the swizzled address by applying the swizzle transformation to the - pointer's address bits. The transformation is typically an XOR-based permutation - of certain address bits, as defined by the swizzle parameters. - - Swizzle is parameterized by three values: - - MBase: The number of least-significant bits to keep constant (not affected by the swizzle) - - BBits: The number of bits in the mask (the width of the swizzle region) - - SShift: The distance to shift the mask (positive values shift right, negative values shift left) - - The swizzle transformation operates on the address as follows: - - 0bxxxxxxxxxxxxxxxYYYxxxxxxxZZZxxxx - ^--^ MBase: least-significant bits kept constant - ^-^ ^-^ BBits: width of the mask (YYY and ZZZ) - ^---------^ SShift: shift distance for the YYY mask - - The swizzled address is computed by XOR'ing the ZZZ and YYY bitfields: - - Example: - - ``` - 0bxxxxxxxxxxxxxxxxYYxxxxxxxxxZZxxx - - the result is - 0bxxxxxxxxxxxxxxxxYYxxxxxxxxxAAxxx where AA = ZZ `xor` YY - ``` - - This operation is typically used just before dereferencing a pointer, - ensuring that the correct physical memory location is accessed according - to the swizzle pattern. If the pointer type has an identity swizzle, - this operation is a no-op and returns the original pointer. - }]; - let arguments = (ins - Cute_CuTePointerTypeOrLLVMPointerType:$ptr, - OptionalAttr:$swizzle - ); - let results = (outs - Cute_CuTePointerTypeOrLLVMPointerType:$swizzled_ptr - ); -} -def Cute_AssumeOp : Cute_Op<"assume", [Pure]> { - let description = [{ - `cute.assume` represents assumptions such as divisibility for an SSA value. - unlike `llvm.assume`, `cute.assume` produces a strong typed value as output. - It is limited to support all constrains defined by `ConstrainedIntType`. - - Please see [constrained integer type](### ConstrainedIntType) for details. - - Example: - ```mlir - // Claim %1 is divisible by 8 - %1 = cute.assume(%0 : i32) : !cute.i32 - - ``` - - Note: - - // xxx: need to revisit - - ```mlir - %0 = ... -> !cute.i32 - if (%cond) { - %1 = cute.assume(%0 : !cute.i32) : !cute.i32 - ... // %1 has the assumption. - - // %0 is not assumed here - %data = load %ptr[%0] : ? - } - - // %0 here does not have any assumption. - ``` - }]; - let arguments = (ins - Cute_Integer:$src - ); - let results = (outs - Cute_IntegerTypeWithConstraints:$dst - ); -} -def Cute_BlockedProductOp : Cute_InferredOp<"blocked_product", [Pure]> { - let description = [{ - This operation performs "blocked_product" on the input and produce a blocked_product result, - and it automatically infer the "blocked_product" layout for the result type. - - Example: - - ```mlir - %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> - %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> - %bprod = cute.blocked_product(%input, %tiler) - : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) - -> !cute.layout<"((3,2),(4,5)):((4,12),(1,24))"> - ``` - - Example with dynamic composed_layout: - - ``` - %bprod = cute.blocked_product(%input, %tiler) - : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, !cute.layout<"(?,?):(?,?)">) - -> !cute.composed_layout<"(?):(?) o ? o ((5,?),(4,?)):((4,?{div=20}),(1,?{div=20}))"> - ``` - }]; - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTe:$input, - Cute_CuTeLayoutType:$tiler - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_CeilDivOp : Cute_InferredOp<"ceil_div", [Pure]> { - let arguments = (ins - Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input, - Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler - ); - let results = (outs - Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result - ); -} -def Cute_CoalesceOp : Cute_InferredOp<"coalesce", [Pure]> { - let summary = "`coalesce` op simplifies layout by combining modes that are possible to combine."; - let description = [{ - It does not respect the shape of the layout but preserve total size. - - ```mlir - // coalesce modes with contiguous/packed strides from left to right - %0 = cute.coalesce(%arg0) - : (!cute.layout<"((4,5),3):((1,4),20)">) -> !cute.layout<"60:1"> - - // coalesce from left to right but not right to left - %0 = cute.coalesce(%arg0) - : (!cute.layout<"(3,(4,5)):(20,(1,4))">) -> !cute.layout<"(3,20):(20,1)"> - %0 = cute.coalesce(%arg0, %arg1) - : (!cute.layout<"(3,(4,5)):(20,(1,4))">, !cute.coord<"(1,1)">) -> !cute.layout<"(3,20):(20,1)"> - ``` - }]; - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input, - Optional:$target_profile - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$result - ); -} -def Cute_ComplementOp : Cute_InferredOp<"complement", [Pure]> { - let summary = "Produces a disjoint layout."; - let description = [{ - The complement of a layout L is a layout L* such that (L,L*) achieves greater surjectivity. - That is, L* maps to elements in the codomain of L that are not in the image of L. - - It's worth to note that (L,L*) isn't alway surjective. - - Examples: - ``` - // trivial complement of L under it's domain is 1:0 - %res = cute.complement(%arg0, %arg1) - : (!cute.layout<"3:1">, !cute.shape<"3">) -> !cute.layout<"1:0"> - - // 3:2 is strided layout. complement of 3:2 under 6 is 2:1 filling stride - // (L,L*) => (3,2):(2,1) is a surjective layout - %res = cute.complement(%arg0, %arg1) - : (!cute.layout<"3:2">, !cute.shape<"6">) -> !cute.layout<"2:1"> - - %res = cute.complement(%arg0, %arg1) - : (!cute.layout<"(3,(5,2)):(1,(3,15))">, !cute.shape<"(6,20)">) -> !cute.layout<"4:30"> - ``` - }]; - let arguments = (ins - Cute_CuTeLayoutType:$input, - Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$cotarget - ); - let results = (outs - Cute_CuTeLayoutType:$res - ); -} -def Cute_ComposedGetInnerOp : Cute_InferredOp<"composed_get_inner", [Pure]> { - let summary = "Gets inner layout from a layout."; - let description = [{ - Example: - - ```mlir - %layout = cute.make_layout(.., ..): !cute.layout<"(32,64):(1,32)>"> - %layout_a = cute.composed_get_layout_a(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(64,32):(1,64)"> - %offset = cute.composed_get_offset(%layout) : (!cute.composed_layout<"S<3,4,3> o 7 o (32,64):(1,32)>">) -> !cute.shape<"7"> - %layout_b = cute.composed_get_layout_b(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> - ``` - }]; - let arguments = (ins - Cute_CuTeComposedLayoutType:$input - ); - let results = (outs - Cute_CuTeLayoutTypeOrCuteSwizzleType:$result - ); -} -def Cute_ComposedGetOffsetOp : Cute_InferredOp<"composed_get_offset", [Pure]> { - let summary = "Gets offset from a layout."; - let description = [{ - Example: - - ```mlir - %layout = cute.make_layout(.., ..): !cute.layout<"(32,64):(1,32)>"> - %layout_a = cute.composed_get_layout_a(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(64,32):(1,64)"> - %offset = cute.composed_get_offset(%layout) : (!cute.composed_layout<"S<3,4,3> o 7 o (32,64):(1,32)>">) -> !cute.shape<"7"> - %layout_b = cute.composed_get_layout_b(%layout) : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> - ``` - }]; - let arguments = (ins - Cute_CuTeComposedLayoutType:$input - ); - let results = (outs - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result - ); -} -def Cute_ComposedGetOuterOp : Cute_InferredOp<"composed_get_outer", [Pure]> { - let summary = "Gets layout outer from composed layout."; - let description = [{ - Example: - - ```mlir - %layout_outer = cute.composed_get_outer(%layout) - : (!cute.composed_layout<"(64,32):(1,64) o 0 o (32,64):(1,32)>">) -> !cute.layout<"(32,64):(1,32)"> - ``` - }]; - let arguments = (ins - Cute_CuTeComposedLayoutType:$input - ); - let results = (outs - Cute_CuTeLayoutType:$result - ); -} -def Cute_CompositionOp : Cute_InferredOp<"composition", [Pure]> { - let description = [{ - The `composition` operation computes the composition of two layouts - (`!cute.layout`), in the following called `lhs` and `rhs`. - - Examples: - ``` - // Composition of two static layouts. - %0 = cute.composition(%arg0, %arg1) - : (!cute.layout<"(20):(2)">, !cute.layout<"(5,4):(4,1)">) -> !cute.layout<"(5,4):(8,2)"> - - // Composition of a static and a dynamic layout. - %1 = cute.composition(%arg0, %arg1) - : (!cute.layout<"(20):(2)">, !cute.layout<"(?,?):(?,?)">) -> !cute.layout<"(?,?):(?{div=2},?{div=2})"> - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$lhs, - Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$rhs - ); - let results = (outs - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$res - ); -} -def Cute_CopyAtomCallOp : Cute_Op<"copy_atom_call", [AttrSizedOperandSegments]> { - let summary = "`cute.copy_atom_call` op executes a copy atom operation with the given operands."; - let description = [{ - Following src/dst layout of atom are valid: - * ((atom_v, rest_v)) - * ((atom_v)) - * (atom_v) - - Both `src` and `dst` operands are variadic, containing a variable number of tensors: - - - For regular copy, `src` and `dst` each contain a single tensor. - - For copy with auxiliary operands, they contain the main tensor followed by - auxiliary tensors. For example: - - - For load with row-wise reduction, `dst` = [data, stat]. - - For SPARSIFY (structured sparsity compression), `dst` = [data, metadata]. - - For TMA gather4, `src` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). - - For TMA scatter4, `dst` = [coord0, coord1, coord2, coord3] (four 2D coordinate tensors). - - Examples: - - ```mlir - // Regular copy atom operation - cute.copy_atom_call(%copy_load, %src_v, %dst_v) - : (!cute_nvgpu.atom.tmem_load, - !cute.memref, "(((2,32),1)):(((1,131072),0))">, - !cute.memref, "((2,1)):((1,0))">) - -> () - ``` - - ```mlir - // TMA gather4: combine four 2D coordinate tensors into single destination - cute.copy_atom_call(%tma_gather4_atom, [%coord0, %coord1, %coord2, %coord3], %dst_v) - : (!cute_nvgpu.atom.tma_gather4<...>, - [!cute.coord_tensor<...>, !cute.coord_tensor<...>, - !cute.coord_tensor<...>, !cute.coord_tensor<...>], - !cute.memref) - -> () - ``` - }]; - let arguments = (ins - Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$atom, - Variadic:$src, - Variadic:$dst, - Optional:$pred - ); - let results = (outs - ); -} -def Cute_CopyMakeFragmentOp : Cute_InferredOp<"copy.make_fragment", [Pure]> { - let description = [{ - Construct a copy fragment for the S or D side from either a partitioned view or an abstract TV - description. Intended for use between `cute.tiled.copy.partition_{S,D}` and `cute.copy`, but - also supports layout/shape inputs for RMEM/TMEM fragment construction similar to MMA. - - The op accepts: - 1. an operand id `S` or `D` (printed as a keyword); - 2. a copy atom value; - 3. either a partitioned view or a layout/shape to build the fragment. - - Returns a new ViewTypeInterface derived value to be consumed by `cute.copy`. - - Examples: - ```mlir - // Load from global memory (gmem -> rmem): build S and D fragments from S-partition - %atom_load = cute.make_atom() : () -> !cute_nvgpu.atom.universal_copy - %tiled_copy_load = cute.make_tiled_copy(%atom_load) - : !cute.tiled_copy, - layout_copy_tv = <"(128,8):(1,128)">, tiler_mn = <"[128;8]">> - %tGSrc_load = cute.tiled.copy.partition_S(%tiled_copy_load, %gSrc, %tid_coord) - : (!cute.tiled_copy, - layout_copy_tv = <"(128,8):(1,128)">, tiler_mn = <"[128;8]">>, - !cute.memref, "(128,128):(?{div=8},1)">, !cute.coord<"?">) - -> !cute.memref, "((8,1),1,16):((1,0),0,8)"> - - %frgS_load = cute.copy.make_fragment S (%atom_load, %tGSrc_load) : ( - !cute_nvgpu.atom.universal_copy, - !cute.memref, "((8,1),1,16):((1,0),0,8)"> - ) -> !cute.memref, "((8,1),1,16):((1,0),0,8)"> - - %rLoad = cute.copy.make_fragment D (%atom_load, %tGSrc_load) : ( - !cute_nvgpu.atom.universal_copy, - !cute.memref, "((8,1),1,16):((1,0),0,8)" - ) -> !cute.memref, "((8,1),1,16):((1,0),0,8)" - ``` - }]; - let arguments = (ins - Cute_CopyAtomTypeInterfaceInstanceOrCopyAtomTypeInterfaceV2Instance:$atom, - Cute_AMemoryReferenceTypeThatRepresentsATensor:$input, - Cute_CopyOperandEnum:$operand_id - ); - let results = (outs - Cute_ViewTypeInterfaceInstance:$result - ); -} -def Cute_CosizeOp : Cute_InferredOp<"cosize", [Pure]> { - let description = [{ - This operation calculates the cosize of a layout. - * For a layout `l`, cosize(l) = l(size(l) - 1) + 1. - * For a layout `l` with mode `mode`, cosize(l) = cosize(get(l)). - * For a composed layout 'l = a o b', cosize(l) = cosize(b). - - The `mode` attribute specifies the nested mode in the type for cosize calculation. - It indexes a sub-scope within the input type and returns the cosize of that sub-scope. - The rank of `mode` must be less than or equal to the depth of the type. - If `mode` is empty, the operation calculates the cosize of the entire type. - - ### Examples - - ```mlir - %input: !cute.composed_layout<"(100):(1) o 0 o (3,4):(4,1)"> - %0 = cute.cosize(%input) - : !cute.composed_layout<"(100):(1) o 0 o (3,4):(4,1)"> - // cosize(%input) = cosize((3,4):(4,1)), ret is 12 - ``` - - Example with mode: - ```mlir - %input: !cute.layout<"((4,3),((3,(2,2)),2)):((1,4),((12,(36,72)),144))"> - %0 = cute.cosize(%input) <{mode=[1,0,1]}> - : (!cute.layout<"((4,3),((3,(2,2)),2)):((1,4),((12,(36,72)),144))">) -> !cute.int_tuple<"109"> - // cosize(%input) = cosize((2,2):(36,72)), ret is 109 - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, - DefaultValuedAttr:$mode - ); - let results = (outs - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result - ); -} -def Cute_Crd2IdxOp : Cute_InferredOp<"crd2idx", [Pure]> { - let description = [{ - Compute the index of a coordinate with respect to the given layout. I.e., - compute the natural coordinate of `coord` wrt. to the shape - and then compute the dot product with the strides. - - Examples: - ```mlir - // %crd0 is already a natural coordinate for %shape. - %0 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,(?,?))">, !cute.layout<"(?,(?,?)):(?,(?,?))">) -> (i32) - // Compute the natural coordinate, then dot product. - %1 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,?)">, !cute.layout<"(?,(?,?)):(?,(?,?))">) -> (i32) - // Static and dynamic shape/strides are supported. - %3 = cute.crd2idx(%crd, %layout): (!cute.coord<"(?,?)">, !cute.layout<"(4,2):(1,2)">) -> (i32) - // Strides with scaled basis produces a tuple. - %5 = cute.crd2idx(%crd, %layout) : (!cute.coord<"(?,?,?)">, !cute.layout<"(10,20,30):(1@0,1@1,20@1)">) -> (!cute.int_tuple<"(?,?)">) - ``` - }]; - let arguments = (ins - Cute_CuteCoordType:$coord, - Cute_GeneralizedLayoutTypeOfCuTe:$layout - ); - let results = (outs - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$index - ); -} -def Cute_DereferenceArithTupleIteratorOp : Cute_InferredOp<"deref_arith_tuple_iter", [Pure]> { - let summary = "Dereferences an arithmetic tuple iterator to return the int_tuple value it holds."; - let description = [{ - Example: - - ```mlir - %0 = cute.deref_arith_tuple_iter(%arg0) : !cute.arith_tuple_iter<"(?,4)"> -> !cute.int_tuple<"(?,4)"> - ``` - }]; - let arguments = (ins - Cute_CuTeIteratorTypeWithIntTupleAsIts:$iter - ); - let results = (outs - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$arith_tuple - ); -} -def Cute_DereferenceDescriptorIteratorOp : Cute_InferredOp<"deref_desc_iter", [Pure]> { - let description = [{ - This Operation dereferences a descriptor iterator to return the descriptor - value it holds. - }]; - let arguments = (ins - Cute_ATypeInterfaceForDescriptorIteratorTypes:$iter - ); - let results = (outs - AnyType:$value - ); -} -def Cute_DerefineOp : Cute_Op<"derefine", [Pure]> { - let description = [{ - The `cute.derefine` operation discard static information from modes - of a given cute algebra type like shape, stride, layout, coord or tile into - modes with less static information. - - For each mode, it either cast it into dynamic mode or keep it unchanged. In future, - it can be extended to support removal of `divby N` from dynamic mode. - - Casting dynamic mode into static mode is illegal. - - E.g. - - ```mlir - %dyn_0 = cute.derefine(%0) : !cute.shape<"(3,5)"> to !cute.shape<"(?,5)"> - %dyn_1 = cute.derefine(%1) : !cute.stride<"(?,5)"> to !cute.stride<"(?,?)"> - %dyn_2 = cute.derefine(%2) : !cute.coord<"(8,_)"> to !cute.coord<"(?,_)"> - ``` - }]; - let arguments = (ins - Cute_RefinableTypeInterfaceInstance:$src - ); - let results = (outs - Cute_RefinableTypeInterfaceInstanceOrInteger:$dst - ); -} -def Cute_DiceOp : Cute_InferredOp<"dice", [Pure]> { - let description = [{ - This operation dices a type implementing the `CuteValueTypeInterface` (ShapeType, IntTupleType, - LayoutType,...) using a static coordinate attribute. The profile of said coordinate must be - weakly congruent with that of `input`. Modes associated with an underscore are discarded in the - result. - }]; - let arguments = (ins - Cute_ValueTypeInterfaceInstance:$input, - Cute_CoordAttr:$coord - ); - let results = (outs - Cute_ValueTypeInterfaceInstance:$result - ); -} -def Cute_ElemLessOp : Cute_Op<"elem_less", [Pure]> { - let description = [{ - The cute.elem_less operation compares every elements in Lhs to the - corresponding elements in Rhs. The result is 1 if every Lhs elements - are less than corresponding Rhs elements, otherwise result is 0. - - `ElemLessOp` is useful when we want to know if a coordinate is out of bound: - - ```mlir - %is_in_bound = cute.elem_less(%coord, %shape) : !cute.coord<"(?,?)">, !cute.shape<"(?,?)"> - ``` - }]; - let arguments = (ins - Cute_CuteShapeTypeOrIntTupleAsIntegerOr:$lhs, - Cute_CuteShapeTypeOrIntTupleAsIntegerOr:$rhs - ); - let results = (outs - I1:$result - ); -} -def Cute_EqualOp : Cute_Op<"equal", [Pure]> { - let description = [{ - Checks two dynamic cute values for equality. Full static values are considered - equal only when their types are the same. - - Example: - - ```mlir - %1 = cute.make_shape : !cute.shape<"(32, 64)"> - %2 = cute.make_shape : !cute.shape<"(32, 64)"> - %result = cute.equal %1, %2 : !cute.shape<"(32,64)">, !cute.shape<"(32,64)"> - ``` - - The generated code only checks profile positions in which at least one side has a dynamic value. - Static comparisons are performed at compile time by the operation folder. - - ```mlir - %1 = cute.make_shape : !cute.shape<"(?, 64)"> - %2 = cute.make_shape : !cute.shape<"(?, 64)"> - %result = cute.equal %1, %2 : !cute.shape<"(?,64)">, !cute.shape<"(?,64)"> - ``` - - This would result in the first element being checked for equality in the runtime representation of the shapes. - - Note that this is dynamic if either of the operands will force that operand to be checked. So, - - ```mlir - %1 = cute.make_shape : !cute.shape<"(?, 64)"> - %2 = cute.make_shape : !cute.shape<"(64, ?)"> - %result = cute.equal %1, %2 : !cute.shape<"(?,64)">, !cute.shape<"(64,?)"> - ``` - - will result in both dimensions being checked. - }]; - let arguments = (ins - Cute_ValueTypeInterfaceInstance:$lhs, - Cute_ValueTypeInterfaceInstance:$rhs - ); - let results = (outs - I1:$result - ); -} -def Cute_FastDivmodComputeOp : Cute_Op<"fast_divmod.compute", [Pure]> { - let description = [{ - This operation performs a fast divmod using a fast divmod divisor. It computes the quotient and - the remainder without emitting division instructions. The quotient and the remainder are of the - same width as the divisor. - }]; - let arguments = (ins - Cute_32BitSignlessIntegerOr64BitSignless:$dividend, - Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor - ); - let results = (outs - Cute_32BitSignlessIntegerOr64BitSignless:$quotient, - Cute_32BitSignlessIntegerOr64BitSignless:$remainder - ); -} -def Cute_FastDivmodCreateDivisorOp : Cute_Op<"fast_divmod.create_divisor", [Pure]> { - let description = [{ - This operation creates a fast divmod divisor from an integer. It is meant to be pre-computed, - ideally in host code, and then used throughout the lifetime of a device kernel for performing - divmods without emitting division instructions. - }]; - let arguments = (ins - Cute_32BitSignlessIntegerOr64BitSignless:$divisor - ); - let results = (outs - Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor - ); -} -def Cute_FastDivmodDivideOp : Cute_Op<"fast_divmod.divide", [Pure]> { - let description = [{ - This operation performs a fast divide using a fast divmod divisor. It computes the quotient - without emitting division instructions. The quotient is of the same width as the divisor. - }]; - let arguments = (ins - Cute_32BitSignlessIntegerOr64BitSignless:$dividend, - Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor - ); - let results = (outs - Cute_32BitSignlessIntegerOr64BitSignless:$quotient - ); -} -def Cute_FastDivmodGetAuxOp : Cute_InferredOp<"fast_divmod.get_aux", [Pure]> { - let arguments = (ins - Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$divisor - ); - let results = (outs - Cute_32BitSignlessIntegerOr64BitSignless:$multiplier, - I8:$shift1, - I8:$shift2 - ); -} -def Cute_FastDivmodGetDivisorOp : Cute_InferredOp<"fast_divmod.get_divisor", [Pure]> { - let arguments = (ins - Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor - ); - let results = (outs - Cute_32BitSignlessIntegerOr64BitSignless:$divisor - ); -} -def Cute_FastDivmodMakeDivisorOp : Cute_Op<"fast_divmod.make_divisor", [Pure]> { - let arguments = (ins - Cute_32BitSignlessIntegerOr64BitSignless:$divisor, - Cute_32BitSignlessIntegerOr64BitSignless:$multiplier, - I8:$sh1, - I8:$sh2 - ); - let results = (outs - Cute_DivisorForFastUnsignedIntegerDivmodsByInvariant:$fast_divmod_divisor - ); -} -def Cute_FilterOp : Cute_InferredOp<"filter", [Pure]> { - let summary = "This operation filters out dimensions with 0-strides and 1-sizes from a layout or view."; - let description = [{ - When working with tensor layouts, some dimensions may have stride 0 (indicating - broadcasting) or size 1 (singleton dimensions). This operation creates a new layout - or view with these dimensions removed, which can simplify subsequent operations. - - OptionalOperand `target_profile` is the user indicated filter control, with this operand, - the original layout's stride is ignored in the zero-stride selection. - But the `target_profile` will not take effect currently. This is a placeholder for future use. - - This operation accepts both layout and view types as valid input. - }]; - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$result - ); -} -def Cute_FilterZerosOp : Cute_InferredOp<"filter_zeros", [Pure]> { - let description = [{ - This operation produces a transformed layout or view by replacing the shape - of any mode with stride zero and non-unit extent with 1. This is commonly - applied prior to copy operations to prevent redundant or duplicated - data movement arising from broadcasted dimensions. - - The optional operand `target_profile` allows the user to specify a stride - filter profile; when provided, the selection of zero-stride dimensions is - determined by this stride profile rather than the original layout's stride - information. - - This operation accepts both layout and view types as input. - }]; - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$input, - Optional:$target_profile - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTeOrViewTypeInterfaceInstance:$res - ); -} -def Cute_FlatDivideOp : Cute_InferredOp<"flat_divide", [Pure]> { - let summary = "This operation performs a flat_divide on a view or a layout by a tiler."; - let arguments = (ins - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, - Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler - ); - let results = (outs - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_FlatProductOp : Cute_InferredOp<"flat_product", [Pure]> { - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTe:$input, - Cute_CuTeLayoutType:$tiler - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_GetFlatCoordOp : Cute_InferredOp<"get_flat_coord", [Pure]> { - let description = [{ - This OP accept a scalar index and a normal layout, - returns a flat coord. - - - The "flat" here means the reversed coordinate will be depth=1 coordinate. - This is useful when we just care about the rank semantic of the layout and don't care about the detail profile. - - - Example: - - ```mlir - %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> - %index = arith.constant 117 : i32 - %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> - %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> - %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> - ``` - }]; - let arguments = (ins - Cute_32BitSignlessIntegerOr64BitSignless:$index, - Cute_CuTeLayoutType:$input - ); - let results = (outs - Cute_CuteCoordType:$result - ); -} -def Cute_GetHierCoordOp : Cute_InferredOp<"get_hier_coord", [Pure]> { - let description = [{ - This OP accept a scalar index and a normal layout, - returns a hier coord. - - - The "hier" here means the reversed coordinate will have the same profile with the given layout. - This is useful when we care about both the rank semantic and the detail layout profile. - - - Example: - - ```mlir - %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> - %index = arith.constant 117 : i32 - %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> - %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> - %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> - ``` - }]; - let arguments = (ins - Cute_32BitSignlessIntegerOr64BitSignless:$index, - Cute_CuTeLayoutType:$input - ); - let results = (outs - Cute_CuteCoordType:$result - ); -} -def Cute_GetIntegralCoordOp : Cute_InferredOp<"get_integral_coord", [Pure]> { - let description = [{ - This OP accept a scalar index and a normal layout, - returns a integral coord. - - - The "integral" here means the reversed coordinate will be a "scalar". - This is useful when we don't care about the layout's semantic and profile and we just want to know the offset in "domain". - - - Example: - - ```mlir - %layout = cute.make_layout() : !cute.layout<"(5,(4,3),2):(1,(5,20),60)"> - %index = arith.constant 117 : i32 - %flat_coord = cute.get_flat_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,11,1)"> - %hier_coord = cute.get_hire_coord(%index, %layout) : (..., ...) -> !cute.coord<"(2,(3,2),1)"> - %integral_coord = cute.get_integral_coord(%index, %layout) : (..., ...) -> !cute.coord<"117"> - ``` - }]; - let arguments = (ins - Cute_32BitSignlessIntegerOr64BitSignless:$index, - Cute_CuTeLayoutType:$input - ); - let results = (outs - Cute_CuteCoordType:$result - ); -} -def Cute_GetIterOp : Cute_InferredOp<"get_iter", [Pure]> { - let description = [{ - The "cute.get_iter" gets the underlying iterator from memref or coord_tensor. - It is solely as step during lowering, it has no side effects. - - Example: - - ```mlir - %ptr = cute.get_iter(%0) : !cute.memref, "((64,128),2):((1,1),64)"> - - // Or for TMA tensor - - %arith_tuple = cute.get_iter(%0) : !cute.coord_tensor<"(128, 64)", "((64,64),1):((1@0,1@0),64@1)"> - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstance:$source - ); - let results = (outs - Cute_GeneralizedIteratorTypeOfCuTe:$ptr - ); -} -def Cute_GetLayoutOp : Cute_InferredOp<"get_layout", [Pure]> { - let summary = "Gets the layout out of a `CuteViewTypeInterface`."; - let description = [{ - Example: - - ```mlir - %0 = cute.get_layout(%arg0) : !cute.memref - - %0 = cute.get_layout(%arg0) : !cute.memref o 0 o (256,?):(?,1)"> - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstance:$input - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTe:$layout - ); -} -def Cute_GetLayoutsFromTileOp : Cute_InferredOp<"get_layouts_from_tile", [Pure]> { - let description = [{ - This operation extracts the layout elements out of a tile. A tile is a - layout, an underscore, or a tuple of tiles. - - Examples: - - - the type of the tile is !cute.tile<"[1:0;2:1]"> - 2 layouts are extracted of type !cute.layout<"1:0"> and !cute.layout<"2:1">, respectively - - the type of the tile is !cute.tile<[_;[1:0;2:1];3:1]> - 3 layouts are extracted of type !cute.layout<"1:0">, !cute.layout<"2:1">, and - !cute.layout<"3:1">, respectively - }]; - let arguments = (ins - Cute_CuteTileType:$tile - ); - let results = (outs - Variadic:$layouts - ); -} -def Cute_GetLeavesOp : Cute_InferredOp<"get_leaves", [Pure]> { - let description = [{ - The `cute.get_leaves` operation extracts all leaf values from a hierarchical tuple into - a flat sequence of results. Each result corresponds to a leaf value found in the input tuple, - preserving the order of a depth-first traversal. - - The operation handles nested tuples of any depth, including tuples containing other tuples. - The type of each result matches the type of the corresponding leaf value in the input. - - Example: - ```mlir - // Extract leaves from a hierarchical shape tuple - %0:3 = cute.get_leaves(%hier) : !cute.shape<"(3,?{div=4},5)"> - -> (!cute.shape<"3">, !cute.shape<"?{div=4}">, !cute.shape<"5">) - - // Extract leaves from a nested int tuple - %1:4 = cute.get_leaves(%t) : !cute.int_tuple<"((1,2),(3,4))"> - -> (!cute.int_tuple<"1">, !cute.int_tuple<"2">, - !cute.int_tuple<"3">, !cute.int_tuple<"4">) - ``` - }]; - let arguments = (ins - Cute_XTupleLikeType:$input - ); - let results = (outs - Variadic:$results_ - ); -} -def Cute_GetOp : Cute_Op<"get", [Pure]> { - let description = [{ - This operation returns a potentially nested mode out of a type implementing - the `CuteValueTypeInterface`, e.g. a shape, a tuple, a layout,... - - The optional `mode` attribute indices are used to select the portion of the - input type. The number of indices in the `mode` attribute should be less - than or equal to the depth of type. If `mode` is empty, a copy of the input - value is returned. - - Example: - - ```mlir - %shape = cute.make_shape() : () -> !cute.shape<"(1, (2, 4), 8)"> - %mode0 = cute.get(%shape) <{mode=[1]}> - : !cute.shape<"(1,(2,4),8)"> -> !cute.shape<"(2,4)"> - %mode1 = cute.get(%shape) <{mode=[1,1]}> - : !cute.shape<"(1,(2,4),8)"> -> !cute.shape<"4"> - ``` - }]; - let arguments = (ins - Cute_ValueTypeInterfaceInstance:$input, - OptionalAttr:$mode - ); - let results = (outs - Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraintsOr:$result - ); -} -def Cute_GetScalarsOp : Cute_InferredOp<"get_scalars", [Pure]> { - let summary = "Gets the scalars out of a value with CuteValueTypeInterface."; - let description = [{ - This operation returns only the dynamic scalars when `only_dynamic` is set. - - A "scalar" in this context refers to a single value used as part of the - state of a type. In general, each state value held by a type will be - considered a scalar if it is printed in the text format. This includes: - - static integers - - dynamic integers - - swizzle parameters - - scaled basis value (static integer, ratio (num + denom), dynamic integer) - - scaled basis modes - - indexed pointer address (always a dynamic value) - - indexed pointer stride (dynamic integer or dynamic) - - Note that underscore values in coordinate types are NOT considered scalars. - - Example: - - ```mlir - %1 = cute.get_shape(%0) : (!cute.layout<"(256,?):(?,1)">) -> !cute.shape<"(256,?)"> - %2 = cute.get_scalars(%1) <{only_dynamic}> : (!cute.shape<"(256,?)">) -> i32 - ``` - }]; - let arguments = (ins - Cute_ValueTypeInterfaceInstanceOrIntegerTypeWithConstraints:$cute_value, - UnitAttr:$only_dynamic - ); - let results = (outs - Variadic:$scalars - ); -} -def Cute_GetShapeOp : Cute_InferredOp<"get_shape", [Pure]> { - let summary = "get shape from view, layout or tile"; - let description = [{ - Example: - - ```mlir - %shape = cute.get_shape(%layout) : (!cute.layout<"(32,64):(1,32)>">) -> !cute.shape<"(32,64)"> - %shape = cute.get_shape(%tile) - : (!cute.tile<"[(32,64):(1,32);3:1]>">) -> !cute.shape<"((32,64),3)"> - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe2:$input - ); - let results = (outs - Cute_CuteShapeType:$result - ); -} -def Cute_GetStrideOp : Cute_InferredOp<"get_stride", [Pure]> { - let summary = "get stride from layout."; - let description = [{ - Please be noted that stride is not generalized for view and layout because composed layout - doesn't have generalized layout concept. - - Example: - - ```mlir - %stride = cute.get_stride(%layout) : (!cute.layout<"(32,64):(1,32)>">) -> !cute.stride<"(1, 32)"> - ``` - }]; - let arguments = (ins - Cute_CuTeLayoutType:$input - ); - let results = (outs - Cute_CuteStrideType:$result - ); -} -def Cute_GroupModesOp : Cute_InferredOp<"group_modes", [Pure]> { - let description = [{ - This operation will make the input view/layout's [B,E) modes - into one new mode with one-depth deeper. - - Example: - - ```mlir - %res = cute.group_modes(%arg0) <0,1> - : (!cute.coord_tensor<"(0,0)", "(512,64,2):(1@1,1@0,64@0)">) - -> !cute.coord_tensor<"(0,0)", "((512),64,2):((1@1),1@0,64@0)"> - %res = cute.group_modes(%arg0) <0,2> - : (!cute.composed_layout<"(3,4,5):(6,7,8) o (0,1,2) o (9,9,9):(1@1,1@0,1@2)">) - -> !cute.composed_layout<"(3,4,5):(6,7,8) o (0,1,2) o ((9,9),9):((1@1,1@0),1@2)"> - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, - I32Attr:$begin, - I32Attr:$end - ); - let results = (outs - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_Idx2CrdOp : Cute_InferredOp<"idx2crd", [Pure]> { - let description = [{ - compute the coordinate corresponding to an index with respect to the given shape. I.e., - compute the coordinate that maps to the given index using given shape and default stride. - Note: This only works for compact layout. - - Examples: - ```mlir - // Convert a linear index to coordinate using shape and default stride. - %0 = cute.idx2crd(%index, %shape): (!cute.int_tuple<"?">, !cute.shape<"(4,2)">) -> (!cute.coord<"(?,?)">) - // Static and dynamic shape are supported. - %1 = cute.idx2crd(%index, %shape): (!cute.int_tuple<"?">, !cute.shape<"(?,(?,?))">) -> (!cute.coord<"(?,(?,?))">) - ``` - }]; - let arguments = (ins - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$index, - Cute_CuteShapeType:$shape - ); - let results = (outs - Cute_CuteCoordType:$coord - ); -} -def Cute_IncrementCoordOp : Cute_InferredOp<"increment_coord", [Pure]> { - let summary = "This operation colexicographically increments a coord within a coord space defined by a shape."; - let description = [{ - Example: - - ```mlir - // With static coord and shape - %coord1 = cute.increment_coord(%coord0, %shape) : - (!cute.coord<"(0,0)">, !cute.shape<"(2,2)">) -> !cute.coord<"(1,0)"> - %coord2 = cute.increment_coord(%coord1, %shape) : - (!cute.coord<"(1,0)">, !cute.shape<"(2,2)">) -> !cute.coord<"(0,1)"> - - // With dynamic coord or shape - %coord1 = cute.increment_coord(%coord0, %shape) : - (!cute.coord<"(?,?)">, !cute.shape<"(?,8)">) -> !cute.coord<"(?,?)"> - ``` - }]; - let arguments = (ins - Cute_CuteCoordType:$coord, - Cute_CuteShapeType:$shape - ); - let results = (outs - Cute_CuteCoordType:$result - ); -} -def Cute_IntToPtrOp : Cute_Op<"inttoptr", [Pure]> { - let description = [{ - The `cute.inttoptr` operation accepts an integer value and cast to cute - pointer value with align=1 (byte) by default. - - To create ptr with align > 1, we must use constrained int type with divby > 1. - Alignment of result ptr is the same as `divby` of constrained int type - - Example: - - ```mlir - // Convert to cute.ptr with unrestricted integer type. Can only have align<1> - %ptr = cute.inttoptr(%0) : i64 to !cute.ptr> - - // Convert to cute.ptr with constrained integer type. - %aligned_ptr = cute.inttoptr(%0) : !cute.i64 to !cute.ptr> - ``` - }]; - let arguments = (ins - Cute_IntegerOrIntegerTypeWithConstraints:$src - ); - let results = (outs - Cute_PointerTypeInterfaceInstance:$dst - ); -} -def Cute_KernelSmemSizeOp : Cute_Op<"kernel_smem_size", [Pure]> { - let description = [{ - This host-side operation gets the static smem size from a kernel function - and returns the size as an i64 integer. - }]; - let arguments = (ins - SymbolRefAttr:$kernel_name - ); - let results = (outs - I64:$size - ); -} -def Cute_LeftInverseOp : Cute_InferredOp<"left_inverse", [Pure]> { - let arguments = (ins - Cute_CuTeLayoutType:$input - ); - let results = (outs - Cute_CuTeLayoutType:$result - ); -} -def Cute_LoadScaledIndexOp : Cute_InferredOp<"load_scaled_index", [MemoryEffects<[MemRead]>]> { - let description = [{ - The `cute.load_scaled_index` operation adds the offset (%ptr_b[%index] * %stride) to a - cute pointer %ptr_a. %ptr_a can be any cute pointer type, while %ptr_b must be an I32 cute pointer. - The behavior is undefined if %ptr_b[%index] results in an out-of-bound access. - The result is a cute pointer that has identical type as %ptr_a except that its alignment is derived - from ptr_a's alignment and the stride. - - Example: - - ```mlir - %ptr = cute.load_scaled_index(%src_ptr, %table_ptr, %index, %stride) - : (!cute.ptr, !cute.ptr, !cute.int_tuple<"?">, !cute.int_tuple<"?{div=16}">) - -> !cute.ptr - ``` - }]; - let arguments = (ins - Cute_CuTePointerType:$ptr_a, - Cute_CuTePointerTo32BitSignlessInteger:$ptr_b, - Cute_IntTupleTypeThatDepthIs0:$index, - Cute_IntTupleTypeThatDepthIs0:$stride - ); - let results = (outs - Cute_CuTePointerType:$result - ); -} -def Cute_LocalPartitionOp : Cute_InferredOp<"local_partition", [Pure]> { - let description = [{ - The `local_partition` operation tiles a tensor according to the flat shape of a layout - that provides the coordinate of the target index. - This is typical at the Thread level where data is partitioned across repeated patterns - of threads. - - Example: - - ```mlir - %local_partition = cute.local_partition(%input, %tile, %index) - : (!cute.memref, "(16,?):(1,16)">, !cute.layout<"(2,4)">, i32) - -> !cute.memref - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstance:$input, - Cute_StaticLayout:$tiler, - Cute_32BitSignlessIntegerOr64BitSignless:$index - ); - let results = (outs - Cute_ViewTypeInterfaceInstance:$result - ); -} -def Cute_LocalTileOp : Cute_InferredOp<"local_tile", [Pure]> { - let description = [{ - The `local_tile` operation tiles a tensor according to a tiler and use a coord to - index into the remainder, keeping the tile. - - This is typical at the CTA level where tiles of data are extracted: - - ```mlir - %data = // ( M, N) - %crd = cute.make_coord(blockIdx.x,blockIdx.y) : !cute.coord<"(?,?)"> - %cta_data = cute.local_tile(%data, %tile, %crd) : ... // (_32,_64) - ``` - - Example: - - ```mlir - %0 = cute.local_tile(%alloc, %tile, %coord) - : (!cute.memref, "(128,128):(65536,1)">, - !cute.tile<"[128:1;32:1]">, - !cute.coord<"(_,_)">) - -> !cute.memref, "(128,32,1,4):(65536,1,0,32)"> - - %1 = cute.local_tile(%arg0, %shape, %coord_1) - : (!cute.coord_tensor<"(0,0,0)", "(256,64,1):(1@1,1@0,1@2)">, - !cute.shape<"(256,64)">, - !cute.coord<"(0,_,_)">) - -> !cute.coord_tensor<"(0,0,0)", "(256,64,1,1):(1@1,1@0,0,1@2)"> - ``` - - Utilizing a flat coordinate '(_)' will flatten the resulting tile. - - ```mlir - %0 = cute.local_tile(%alloc, %tile, %coord) - : (!cute.memref, "(128,128):(65536,1)">, - !cute.tile<"[128:1;32:1]">, - !cute.coord<"(_)">) - -> !cute.memref, "(128,32,1,4):(65536,1,0,32)"> - - %1 = cute.local_tile(%arg0, %shape, %coord_1) - : (!cute.coord_tensor<"(0,0,0)", "(256,64,1):(1@1,1@0,1@2)">, - !cute.shape<"(256,64)">, - !cute.coord<"(_)">) - -> !cute.coord_tensor<"(0,0,0)", "(256,64,1,1,1):(1@1,1@0,0,0,1@2)"> - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstance:$input, - Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tile, - Cute_CuteCoordType:$coord, - OptionalAttr:$proj - ); - let results = (outs - Cute_ViewTypeInterfaceInstance:$result - ); -} -def Cute_LogicalDivideOp : Cute_InferredOp<"logical_divide", [Pure]> { - let description = [{ - This operation performs "logical_divide" on the input and produces a logical_divided result, - and it automatically infers the "logical_divided" layout for the result type. - - Example: - - ```mlir - %alloc = cute.memref.alloc() : !cute.memref - %shape = cute.make_shape() : () -> !cute.shape<"(4,8)"> - %zdiv = cute.logical_divide(%alloc, %shape) - : (!cute.memref, !cute.shape<"(4,8)">) - -> !cute.memref - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, - Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler - ); - let results = (outs - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_LogicalProductOp : Cute_InferredOp<"logical_product", [Pure]> { - let description = [{ - This operation performs "logical_product" on the input and produce a logical_product result, - and it automatically infer the "logical_product" layout for the result type. - - Example: - - ```mlir - %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> - %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> - %prod = cute.logical_product(%input, %tiler) - : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) - -> !cute.layout<"((3,4),(2,5)):((4,1),(12,24))"> - ``` - - Example with dynamic composed_layout: - - ``` - %prod = cute.logical_product(%input, %tiler) - : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, - !cute.layout<"(?,?):(?,?)">) - -> !cute.composed_layout<"(?):(?) o ? o ((5,4),(?,?)):((4,1),(?{div=20},?{div=20}))"> - ``` - }]; - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTe:$input, - Cute_CuTeLayoutType:$tiler - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_MakeArithTupleIteratorOp : Cute_Op<"make_arith_tuple_iter", [Pure]> { - let description = [{ - This Operation makes an arithmetic tuple iterator (`!cute.arith_tuple_iter<...>`) - from an int_tuple value (`!cute.int_tuple<...>`) if provided, or it is empty - when all the modes are static. - - Its verifier checks that the value type of the result iterator matches the - operand value type if provided. - }]; - let arguments = (ins - Optional:$value - ); - let results = (outs - Cute_CuTeIteratorTypeWithIntTupleAsIts:$iter - ); -} -def Cute_MakeAtomOp : Cute_Op<"make_atom", [Pure]> { - let summary = "`cute.make_atom` op is generic constructor of SSA value of copy or mma atom type."; - let description = [{ - It optionally takes variadic list of SSA values as well as named attribute to - initialize corresponding fields of copy or mma atom. - - Examples: - - ```mlir - // create copy atom which doesn't require initialization - %0 = cute.make_atom() : () - -> !cute_nvgpu.atom.tmem_load - - // create mma atom and initialize it with SSA values - %1 = cute.make_atom(%accum_c, %a_neg, %b_neg) : (i1, i1, i1) - -> !cute_nvgpu.sm100.mma<128x128x16, - num_cta = 1, - ab_major = (k,k), - elem_type = (f16,f16,f32), - frag_kind = ss, - c_scale_exp = 0> - ``` - }]; - let arguments = (ins - Variadic:$values - ); - let results = (outs - Cute_AtomTypeInterfaceInstance:$atom - ); -} -def Cute_MakeComposedLayoutOp : Cute_InferredOp<"make_composed_layout", [Pure]> { - let summary = "The `make_composed_layout` operation constructs composed layout with"; - let description = [{ - * (inner) affined layout or swizzle layout - * offset - * (outer) layout - - In CuTe algebra, composed layout represent composition of - - ```math - L_{inner} o offset o L_{outer} - ``` - - where outer layout defined the domain of the layout which is visible to user. For given input coordinate - in domain of outer layout, composed layout maps coordinate to codomain by - - ```math - L_{inner}(L_{outer}(coordinate) + offset) - ``` - - Composed Layout is useful in cases where composition() does not or cannot apply to LayoutA and LayoutB. - For example, when the "divisibility condition" in shape_div is violated in composition(LayoutA, LayoutB). - - This ComposedLayout provides similar functionality to Layout including tiling, partitioning, - coordinate-to-index mapping and layout manipulations, but is not considered a "normal" layout. - For example, this layout provides shape() and size() functions, but does not provide stride() functions. - Mostly, the similar functionality is accomplished by applying each operation to LayoutB only - as LayoutB defines the domain. - - ### Examples: - - ```mlir - // Swizzle composed layout - %inner : !cute.swizzle<"S<3,4,3>"> - %layout = cute.make_composed_layout(%inner, %offset, %outer) - : !cute.composed_layout<"S<3,4,3> o 5 o (?,?):(?,?)"> - - // Linearized composed layout - %inner : !cute.layout<"((?,?),(?,?)):((?,?),(?,?))"> - %layout = cute.make_composed_layout(%inner, %offset, %outer) - : !cute.composed_layout<"((?,?),(?,?)):((?,?),(?,?)) o ? o (1,2):(3,4)"> - ``` - }]; - let arguments = (ins - Cute_CuTeLayoutTypeOrSwizzleWithValidParameters:$inner, - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$offset, - Cute_CuTeLayoutType:$outer - ); - let results = (outs - Cute_CuTeComposedLayoutType:$result - ); -} -def Cute_MakeCoordOp : Cute_Op<"make_coord", [Pure]> { - let description = [{ - Constructs a coord. The operands correspond to the - coord's dynamic elements, ordered from left to right. - The number of operands should match the count of dynamic elements in the - resulting coord type. - - Example: - - ```mlir - // Definitions for %0, %1 come earlier - %shape = cute.make_coord(%0, %1) : (i32, i32) - -> !cute.coord<"((?,32),?)"> - ``` - }]; - let arguments = (ins - Variadic:$dynamicElements - ); - let results = (outs - Cute_CuteCoordType:$result - ); -} -def Cute_MakeDescriptorIteratorOp : Cute_Op<"make_desc_iter", [Pure]> { - let description = [{ - This Operation makes a descriptor iterator from a descriptor value. Its - verifier verifies that the value Type of the result Type implementing the - iterator Type Interface is the same as the Type as the input operand. - }]; - let arguments = (ins - Cute_ATypeInterfaceForMemoryDescriptorsSuchAs:$value - ); - let results = (outs - AnyType:$iter - ); -} -def Cute_MakeFragmentLikeOp : Cute_InferredOp<"make_fragment_like", [Pure]> { - let summary = "A deprecated operation but used by collective DSL (keep it here to avoid breaking changes)"; - let description = [{ - TODO: remove this operation in the future - }]; - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTe:$src - ); - let results = (outs - Cute_CuTeLayoutType:$dst - ); -} -def Cute_MakeIdentityLayoutOp : Cute_InferredOp<"make_identity_layout", [Pure]> { - let summary = "This operation makes an identity layout with the given shape."; - let description = [{ - Example: - ```mlir - %sha = cute.make_shape() : () -> !cute.shape<"(2,(3,4),5)"> - %lay = cute.make_identity_layout(%sha) : !cute.layout<"(2,(3,4),5):(1@0,(1@0@1,1@1@1),1@2)"> - ``` - }]; - let arguments = (ins - Cute_CuteShapeType:$shape - ); - let results = (outs - Cute_CuTeLayoutType:$result - ); -} -def Cute_MakeIdentityTensorOp : Cute_InferredOp<"make_identity_tensor", [Pure]> { - let description = [{ - The cute.make_identity_tensor operation construct an identity_tensor - based on the given shape. - - Example to make an identity_tensor from a static shape: - - ```mlir - %identity_tensor = cute.make_identity_tensor(%shape) - : !cute.coord_tensor<"(0,(0,0))", "(2,(3,4)):(1@0,(1@0@1,1@1@1))"> - ``` - - Example to make an identity_tensor from a dynamic shape: - - ```mlir - %identity_tensor = cute.make_identity_tensor(%shape) - : !cute.coord_tensor<"((0,0),(0,0))", "((?,?),(?,?)):((1@0@0,1@1@0),(1@0@1,1@1@1))"> - ``` - }]; - let arguments = (ins - Cute_CuteShapeType:$shape - ); - let results = (outs - Cute_CuteCoordTensorType:$result - ); -} -def Cute_MakeIntTupleOp : Cute_Op<"make_int_tuple", [Pure]> { - let description = [{ - Constructs a int_tuple. The operands correspond to the - int_tuple's dynamic elements, ordered from left to right. - The number of operands should match the count of dynamic elements in the - resulting int_tuple type. - - Example: - - ```mlir - // Definitions for %0, %1 come earlier - %shape = cute.make_int_tuple(%0, %1) : (i32, i32) - -> !cute.int_tuple<"((?,32),?)"> - ``` - }]; - let arguments = (ins - Variadic:$dynamicElements - ); - let results = (outs - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result - ); -} -def Cute_MakeLayoutLikeOp : Cute_InferredOp<"make_layout_like", [Pure]> { - let description = [{ - Creates a compact memory layout that: - 1. Has a congruent shape with the source layout - 2. Has strides in the same order of major-ness as the input layout - 3. Mode 0 strides are `LayoutLeft` (column major) with zero strides preserved from the input layout - - Mode-0 constraints are applied due to its common usage with MMA_Atoms and Copy_Atoms - that need to create RMEM fragments. - - Example: - ```mlir - %0 = cute.make_layout_like(%layout) - : !cute.layout<"(4,8):(1,1024)"> to !cute.layout<"(4,8):(1,4)"> - ``` - }]; - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTe:$src - ); - let results = (outs - Cute_CuTeLayoutType:$dst - ); -} -def Cute_MakeLayoutOp : Cute_Op<"make_layout", [AttrSizedOperandSegments, Pure]> { - let description = [{ - The `make_layout` operation generates a layout based on provided shape and stride. - If the stride component is omitted in the operation's result type - (e.g., `!cute.layout<"(?,(?,?))">`), the operation defaults to a column-major - layout, and missing stride details are automatically inferred to complete the type - of the resulting layout. - - The shape or stride operand must be supplied if they contain dynamic elements. - However, if the shape or stride is entirely static, corresponding operands can be - neglected. - - When the result type does not specify stride information (like `!cute.layout<"(?,?)">` - ), it is not necessary to provide a stride operand, even if the stride to be inferred - includes dynamic components (as in the inferred type `!cute.layout<"(?,?):(1,?)">`). - - ### Examples: - - ```mlir - // Shape and stride are both dynamic - %layout = cute.make_layout(%shape, %stride) : !cute.layout<"(?,64):(1,?)"> - - // Shape and stride are both static - %layout = cute.make_layout() : !cute.layout<"(32,64):(1,32)"> - - // Stride is omitted in the result type, implying it should be inferred - %layout = cute.make_layout(%shape) : !cute.layout<"(?,(?,?))"> - ``` - }]; - let arguments = (ins - Optional:$shape, - Optional:$stride - ); - let results = (outs - Cute_CuTeLayoutType:$layout - ); -} -def Cute_MakeOrderedLayoutOp : Cute_InferredOp<"make_ordered_layout", [Pure]> { - let description = [{ - The `make_ordered_layout` operation generates a compact layout based on provided `shape` - and stride is calculated by `shape` and `order`. - - Different from `make_layout`, `shape` operand is required, `order` is optional. When `order` - isn't set, default stride growth order is from left to right. Supports dynamic values in `shape`, - while currently `order` is restricted to static. Rank of `order` should be equal to rank of - shape but no need for congruent. - - Note that numbers in `order` are just used for relative order relationship, e.g. (67,46,23,32) - is equivalent to (4,3,1,2), (3,2,4) is equivalent to (2,1,3). - - ### Examples: - - ```mlir - %shape = cute.make_shape() : !cute.shape<"(32,64)"> - %shape_dyn = cute.make_shape() : !cute.shape<"(32,?)"> - %order = cute.make_shape() : !cute.shape<"(1,2)"> - %layout = cute.make_ordered_layout(%shape, %order) - : (!cute.shape<"(32,64)">, !cute.shape<"(1,2)">) -> !cute.layout<"(32,64):(1,32)"> - %layout_dyn = cute.make_ordered_layout(%shape_dyn, %order) - : (!cute.shape<"(32,?)">, !cute.shape<"(1,2)">) -> !cute.layout<"(32,?):(1,32)"> - - %order = cute.make_shape() : !cute.shape<"(2,1)"> - %layout = cute.make_ordered_layout(%shape, %order) - : (!cute.shape<"(32,64)">, !cute.shape<"(2,1)">) -> !cute.layout<"(32,64):(64,1)"> - %layout_dyn = cute.make_ordered_layout(%shape_dyn, %order) - : (!cute.shape<"(32,?)">, !cute.shape<"(2,1)">) -> !cute.layout<"(32,?):(?,1)"> - - %shape = cute.make_shape() : !cute.shape<"(2,(2,2),2)"> - %order = cute.make_shape() : !cute.shape<"(0,(2,3),1)"> - %layout = cute.make_ordered_layout(%shape, %order) : !cute.layout<"(2,(2,2),2):(1,(4,8),2)"> - - %order = cute.make_shape() : !cute.shape<"(2,1,3)"> - %layout = cute.make_ordered_layout(%shape, %order) : !cute.layout<"(2,(2,2),2):(4,(1,2),8)"> - }]; - let arguments = (ins - Cute_CuteShapeType:$shape, - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$order - ); - let results = (outs - Cute_CuTeLayoutType:$layout - ); -} -def Cute_MakeShapeOp : Cute_Op<"make_shape", [Pure]> { - let description = [{ - Constructs a shape. The operands correspond to the - shape's dynamic elements, ordered from left to right. - The number of operands should match the count of dynamic elements in the - resulting shape type. - - Example: - - ```mlir - // Definitions for %0, %1 come earlier - %shape = cute.make_shape(%0, %1) : (i32, i32) - -> !cute.shape<"((?,32),?)"> - ``` - }]; - let arguments = (ins - Variadic:$dynamicElements - ); - let results = (outs - Cute_CuteShapeType:$result - ); -} -def Cute_MakeSparseElemOp : Cute_Op<"make_sparse_elem", [Pure]> { - let summary = "Creates a sparse element from a physical storage."; - let description = [{ - Example: - - ```mlir - %v0 = arith.constant 8 : i8 - %v1 = arith.constant 31 : i8 - %physical_storage = vector.from_elements(vector<2xf16>, %v0, %v1) - %sparse_elem = cute.make_sparse_elem(%physical_storage) : (!cute.sparse_elem<4, vector<2xf16>>) - ``` - }]; - let arguments = (ins - AnyType:$physical_storage - ); - let results = (outs - Cute_CuTeSparseElementType:$result - ); -} -def Cute_MakeStrideOp : Cute_Op<"make_stride", [Pure]> { - let description = [{ - Constructs a stride. The operands correspond to the - stride's dynamic elements, ordered from left to right. - The number of operands should match the count of dynamic elements in the - resulting stride type. - - Example: - - ```mlir - // Definitions for %0, %1 come earlier - %shape = cute.make_stride(%0, %1) : (i32, i32) - -> !cute.stride<"((?,32),?)"> - ``` - }]; - let arguments = (ins - Variadic:$dynamicElements - ); - let results = (outs - Cute_CuteStrideType:$result - ); -} -def Cute_MakeTileOp : Cute_Op<"make_tile", [Pure]> { - let description = [{ - Constructs a tile. The operands correspond to the - tile's dynamic elements, ordered from left to right. - The number of operands should match the count of dynamic elements in the - resulting tile type. - - Example: - - ```mlir - // Definitions for %0, %1 come earlier - %shape = cute.make_tile(%0, %1) : (i32, i32) - -> !cute.tile<"((?,32),?)"> - ``` - }]; - let arguments = (ins - Variadic:$dynamicElements - ); - let results = (outs - Cute_CuteTileType:$result - ); -} -def Cute_MakeTiledCopyOp : Cute_Op<"make_tiled_copy", [Pure]> { - let description = [{ - This operation creates a new copy atom instance by tiling a "base" copy atom - instance. Copy atoms are used by the CuTe algorithm layer during lowering to - generate the appropriate `cute_nvgpu.arch` operations. - - Example: - The below values are origin from cutlass SM100_f16f16_UMMA kernel, the original code: - auto tAgA_mkl = cta_tma_a.partition_S(cta_mma.partition_A(gA_mkl))(_,_0{},_0{},_0{},_,_,_); - All the values are from the "cta_tma_a". - ```mlir - %0 = cute.make_atom(%tma_desc_ptr, %tma_bar) : (!cute.ptr>!cute.ptr>) - -> !cute_nvgpu.atom.tma_load - %1 = cute.make_tiled_copy(%0) - : !cute.tiled_copy, - layout_copy_tv = <"((4,2),(64,128)):((0,64),(128,1))">, - tiler_mn = <"[2048:1;1:1;4:1;1:1]">> - ``` - }]; - let arguments = (ins - Cute_CopyAtomTypeInterfaceInstance:$atom - ); - let results = (outs - Cute_TiledCopyAtomType:$result - ); -} -def Cute_MakeTiledCopyV2Op : Cute_Op<"make_tiled_copy_v2", [Pure]> { - let description = [{ - This operation creates a V2 tiled copy instance by tiling a V2 copy atom. - V2 copy atoms use fragment attributes for operand-specific behavior. - - Example: - ```mlir - %0 = cute.make_atom() : () -> !cute_nvgpu.atom.sm100_s2t_copy_v2 - %1 = cute.make_tiled_copy_v2(%0) - : !cute.tiled_copy_v2, - layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, - tiler_mn = <"[(4,32):(32,1);2:1]">> - ``` - }]; - let arguments = (ins - Cute_CopyAtomTypeInterfaceV2Instance:$atom - ); - let results = (outs - Cute_TiledCopyV2AtomType:$result - ); -} -def Cute_MakeTiledMmaOp : Cute_Op<"make_tiled_mma", [Pure]> { - let description = [{ - This operation creates a new MMA atom instance by tiling a "base" MMA atom - instance. MMA atoms are used by the CuTe algorithm layer during lowering to - generate the appropriate `cute.arch` operations. - - Example: - ```mlir - %0 = cute.make_atom(%accum, %a_neg, %b_neg) : (i1, i1, i1) - -> !cute_nvgpu.sm100.mma<64x64x16, num_cta = 1, ab_major = (k,mn), elem_type = (f16,f16,f16), frag_kind = ss, c_scale_exp = 0> - %1 = cute.make_tiled_mma(%0) - : !cute.tiled_mma, - atom_layout_MNK = <"(1,1,1)">> - ``` - }]; - let arguments = (ins - Cute_MmaAtomTypeInterfaceInstance:$atom - ); - let results = (outs - Cute_TiledMMAAtomType:$result - ); -} -def Cute_MakeTupleOp : Cute_Op<"make_tuple", [Pure]> { - let description = [{ - Creates a tuple from multiple values. The number and types of the - parameters must match the number and types of the result tuple. - }]; - let arguments = (ins - Variadic:$values - ); - let results = (outs - Cute_ATupleIsAFixedLengthOrderedList:$result - ); -} -def Cute_MakeViewOp : Cute_Op<"make_view", [Pure]> { - let description = [{ - The cute.make_view operation construct `CuteViewTypeInterface` - (i.e., cute.memref or cute.coord_tensor) with given `iter` and `layout`. - Internally, we infer the exact return type by checking `iter`'s type: - - if `iter` is cute.ptr, return type is `CuteMemRefType` - - if `iter` is cute.tuple, return type is `CuteCoordTensorType` - - `iter` and `layout` could contain both static and dynamic values. - - `MakeViewOp` usually is the result of a previous operation - (e.g., cute.slice, cute.zipped_divide), where creating a new - `CuteViewTypeInterface` is needed. - - `MakeViewOp` does not involve allocating buffers of any kind. - - Example to make cute.coord_tensor with an iter of cute.tuple type: - - ```mlir - %coord_tensor= cute.make_view(%iter, %layout) : !cute.coord_tensor<"(0,2)", "(30):(20@1)"> - ``` - - Example to make cute.memref with an iter of cute.ptr type: - - ```mlir - %memref= cute.make_view(%iter, %layout) : !cute.memref - ``` - - Static Layout - - ```mlir - %memref= cute.make_view(%iter) : !cute.memref - ``` - - Composed+Static Layout - - ```mlir - %memref= cute.make_view(%iter) : !cute.memref o 9 o (64):(16)"> - ``` - }]; - let arguments = (ins - Cute_GeneralizedIteratorTypeOfCuTe:$iter, - Optional:$layout - ); - let results = (outs - Cute_ViewTypeInterfaceInstance:$result - ); -} -def Cute_MemRefAllocSmemOp : Cute_Op<"memref.alloc_smem"> { - let description = [{ - The memref.alloc_smem operation is utilized for the allocation of static - shared memory. The cosize of the memref layout, which must be static, - determines the size of the memory allocated by this operation. - - ```mlir - %0 = cute.memref.alloc_smem() : !cute.memref - ``` - }]; - let arguments = (ins - ); - let results = (outs - Cute_MemrefInSmemWithStaticAndNonComposed:$memref - ); -} -def Cute_MemRefAllocaOp : Cute_Op<"memref.alloca"> { - let description = [{ - The memref.alloca operation is used to allocate shared/stack memory, which is automatically - released when control returns from the closest surrounding operation's region. This - operation only supports memref with the address space reg, and attempting to allocate - memory with other address spaces will result in an error. - - The size of memory allocated is determined by the layout operand's cosize, which can - be either static or dynamic. The layout operand can be omitted when the memref layout - is static. - - If the memref is sparse, the allocated memory size will be layout operand's cosize - divided by sparsity factor. - - ```mlir - // %dynamic_layout : !cute.layout<"(?,24)"> - %0 = cute.memref.alloca(%dynamic_layout) : !cute.memref - // static layout - %1 = cute.memref.alloca() : !cute.memref - ``` - }]; - let arguments = (ins - Optional:$layout - ); - let results = (outs - Cute_MemrefInGenericOrRegisterMemOrSmem:$result - ); -} -def Cute_MemRefLoadOp : Cute_InferredOp<"memref.load", [MemoryEffects<[MemRead]>]> { - let description = [{ - The `memref.load` operation loads one element at the specified logical coordinate from a memref - in gmem/rmem/smem/generic memory spaces to a register. This operation performs the following steps: - - 1. Converts the logical coordinate to a linear index using `cute.crd2idx` - 2. Retrieves the iterator from the memref using `cute.get_iter` - 3. Adds the offset to the iterator using `cute.add_offset` - 4. For CuteMemRefType, applies swizzling if needed, converts to LLVM pointer, and performs the load - 5. For CoordTensorType, returns the tuple directly - - The element type generally cannot be a sub-byte type (no f4, f6, i1, i2 etc.), with the exception - of i1 which has special handling. - - NOTE: For tf32 types, the operation treat it as f32 even bitwidth of MLIR type is 19. - - Example: - - ```mlir - // Load a fp32 - %coord = cute.make_coord(%c1_i32, %c2_i32) : (i32, i32) -> !cute.coord<"(?,?)"> - %0 = cute.memref.load(%memref, %coord) : (!cute.memref, !cute.coord<"(?,?)">) -> f32 - - // Load a tf32 value (will be handled with special bitwise casting) - %1 = cute.memref.load(%tf32_memref, %coord) : (!cute.memref, !cute.coord<"(?,?)">) -> tf32 - ``` - }]; - let arguments = (ins - Cute_AMemoryReferenceTypeThatRepresentsATensor2:$src, - Cute_CuteCoordType:$coord - ); - let results = (outs - AnyType:$result - ); -} -def Cute_MemRefLoadVecOp : Cute_InferredOp<"memref.load_vec", [AttrSizedOperandSegments, MemoryEffects<[MemRead]>]> { - let description = [{ - The `memref.load_vec` operation loads all elements in the logical coordinate - space from a memref generic or register memory to a 1D vector. Only memrefs with - static layouts are accepted. - - The resulting vector has the same element type as the memref, with a size - equal to the **size** (not **cosize**) of the memref layout. - - An optional mask vector (of the same size as the output vector) can be - provided to mask out elements. When the mask is present, an optional - pass_thru vector can also be provided to fill in values for elements that - are masked out, and thus the result is equivalent to a regular memref load - followed by a select between the loaded and the passthru values, predicated - on the same mask, except that the masked-off lanes are not accessed. - - Example: - - ```mlir - // Load vector - %0 = cute.memref.load_vec(%memref) : (!cute.memref) -> vector<32xf32> - - // Load vector from swizzled memory - %0 = cute.memref.load_vec %memref : !cute.memref, "((32,1),1,1):((1,0),0,0)"> - - // Load vector with mask and pass_thru - %0 = cute.memref.load_vec(%memref, %mask, %pass_thru) - : (!cute.memref, vector<32xi1>, vector<32xf32>) -> vector<32xf32> - ``` - }]; - let arguments = (ins - Cute_StaticShape:$src, - Optional:$mask, - Optional:$pass_thru - ); - let results = (outs - Cute_OfRanks1:$result - ); -} -def Cute_MemRefStoreOp : Cute_Op<"memref.store"> { - let description = [{ - The `memref.store` operation stores a register value to a memref in gmem/rmem/smem/generic at given logical coordinate. - This operation performs the following steps: - - 1. Converts the logical coordinate to a linear index using `cute.crd2idx` - 2. Retrieves the iterator from the memref using `cute.get_iter` - 3. Adds the offset to the iterator using `cute.add_offset` - 4. Converts the pointer to the appropriate LLVM pointer type and performs the store - - The element type generally cannot be a sub-byte type (no f4, f6, i1, i2 etc.), with the exception - of i1 which has special handling. - - NOTE: For tf32 types, the operation treats it as f32 even though the bitwidth of the MLIR type is 19. - This special handling allows for proper conversion between tf32 and f32 representations during store operations. - - Example: - - ```mlir - // Store a fp32 - %coord = cute.make_coord(%c1_i32, %c2_i32) : (i32, i32) -> !cute.coord<"(?,?)"> - cute.memref.store(%memref, %coord, %value) : (!cute.memref, !cute.coord<"(?,?)">, f32) - - // Store a tf32 value (will be handled with special bitwise casting) - cute.memref.store(%tf32_memref, %coord, %tf32_value) : (!cute.memref, !cute.coord<"(?,?)">, tf32) - ``` - }]; - let arguments = (ins - Cute_AMemoryReferenceTypeThatRepresentsATensor3:$dst, - Cute_CuteCoordType:$coord, - AnyType:$value - ); - let results = (outs - ); -} -def Cute_MemRefStoreVecOp : Cute_Op<"memref.store_vec"> { - let description = [{ - The `memref.store_vec` operation stores all elements of a 1D vector to a to the logical coordinate - space of memref in generic or register memory. Only memrefs with static layouts are accepted. - - The vector has the same element type as the memref, with a size equal to - the **size** (not **cosize**) of the memref layout. - - An optional mask vector (of the same size as the input vector) can be - provided to mask out elements. - - ```mlir - // Store vector - %0 = cute.memref.store_vec(%value, %memref) : (vector<32xf32>, !cute.memref) -> () - - // Store vector to swizzled memory - %0 = cute.memref.store_vec %value, %memref : !cute.memref, "((32,1),1,1):((1,0),0,0)"> - - // Store vector with mask - %0 = cute.memref.store_vec(%value, %memref, %mask) - : (vector<32xf32>, !cute.memref, vector<32xi1>) -> () - ``` - }]; - let arguments = (ins - Cute_OfRanks1:$value, - Cute_StaticShape:$dst, - Optional:$mask - ); - let results = (outs - ); -} -def Cute_MmaAtomCallOp : Cute_Op<"mma_atom_call", [AttrSizedOperandSegments]> { - let summary = "`cute.mma_atom_call` op executes a mma atom operation with the given operands."; - let description = [{ - The operands `a` and `b` are variadic, each containing a variable number of tensors: - - For regular MMA, `a` and `b` contain the MMA A and B tensors respectively. - - For MMA with auxiliary operands, `a` and `b` contain the MMA A and B tensors followed by their respective auxiliary tensors. For example: - - For BlockScaledMMA, `a` = [A, SFA] and `b` = [B, SFB]. - - For SparseMMA, `a` = [A, E] and `b` = [B]. - - For BlockScaledSparseMMA, `a` = [A, SFA, E] and `b` = [B, SFB]. - Example (regular MMA): - - ```mlir - // Call a mma atom operation - cute.mma_atom_call(%mma_atom, %d, %a, %b, %c) - : (!cute_nvgpu.sm100.mma<128x128x16, num_cta = 1, ab_major = (k,k), - elem_type = (f16,f16,f32), - frag_kind = ss, - c_scale_exp = 0>, - !cute.memref, "((128,128)):((131072,1))">, - !cute_nvgpu.smem_desc_view, - !cute_nvgpu.smem_desc_view, - !cute.memref, "((128,128)):((131072,1))">) - -> () - ``` - - Example (block-scaled MMA): - - ```mlir - cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) - : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), - elem_type = (f4E2M1FN, f4E2M1FN, f32), - sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, - !cute.memref, "((128,128)):((65536,1))">, - [!cute_nvgpu.smem_desc_view, - !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], - [!cute_nvgpu.smem_desc_view, - !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], - !cute.memref, "((128,128)):((65536,1))">) - -> () - ``` - - Example (block-scaled MMA): - - ```mlir - cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) - : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), - elem_type = (f4E2M1FN, f4E2M1FN, f32), - sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, - !cute.memref, "((128,128)):((65536,1))">, - [!cute_nvgpu.smem_desc_view, - !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], - [!cute_nvgpu.smem_desc_view, - !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], - !cute.memref, "((128,128)):((65536,1))">) - -> () - ``` - - Example (block-scaled MMA): - - ```mlir - cute.mma_atom_call(%mma_atom, %d, [%a, %sfa], [%b, %sfb], %c) - : (!cute_nvgpu.sm100.mma_bs<256x128x64, num_cta = 2, ab_major = (k,k), - elem_type = (f4E2M1FN, f4E2M1FN, f32), - sf_type = f8E4M3FN, frag_kind = ss, vec_size = 16>, - !cute.memref, "((128,128)):((65536,1))">, - [!cute_nvgpu.smem_desc_view, - !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], - [!cute_nvgpu.smem_desc_view, - !cute.memref, "((((32,4),4),(16,4))):((((262144,4),8388608),(0,1)))">], - !cute.memref, "((128,128)):((65536,1))">) - -> () - ``` - }]; - let arguments = (ins - Cute_MmaAtomTypeInterfaceInstance:$atom, - Cute_ViewTypeInterfaceInstance:$d, - Variadic:$a, - Variadic:$b, - Cute_ViewTypeInterfaceInstance:$c - ); - let results = (outs - ); -} -def Cute_MmaMakeFragmentOp : Cute_InferredOp<"mma.make_fragment", [Pure]> { - let description = [{ - Given a partitioned tensor, the `cute.mma.make_fragment` operation returns - a tensor with the correct memory space for the specified operand and dot - atom. This op is usually used between partition and gemm. It is the - "prepare" step before gemm. - - `cute.tiled.mma.partition` returns a view into the specified tensor, but it - does not copy the data to the correct memory space. This is done by - `cute.mma.make_fragment` and explicit memcpy operations. Whether a memcpy - is needed or not depend on the result memory space. - - * RMEM: The result tensor is a newly allocated tensor with unspecified - contents. A memcpy is needed. - * SMEM: The result tensor is a view into the input tensor in the form of a - CuteSmemDescViewType. A memcpy is not needed. - * TMEM: The result tensor uses a dummy TMEM engine (fake pointer 0x0) and - must be set correctly later, aligned with Cute C++. - - Instead of a tensor, this op also accepts cute.layout (for RMEM/TMEM) and - cute.shape (for RMEM/TMEM). - - Example: - - ```mlir - // Example 1: Full example for SM80. - // Partition a tensor for operand A, which is stored in SMEM. - %partitioned = cute.tiled.mma.partition A(%atom, %smem_A, %tid_coord) - : (!cute.tiled_mma, - atom_layout_MNK = <"(2,2,1)">>, - !cute.memref, S<2,4,3>, "(128,32):(32,1)">, - !cute.coord<"?">) - -> !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)"> - - // The tensor is still in SMEM, but sm80.mma expects A in registers. - // Prepare a tensor with the correct shape and strides in RMEM. - %fragment = cute.mma.make_fragment A (%3, %partitioned) - : (!cute.tiled_mma, - atom_layout_MNK = <"(2,2,1)">>, - !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)">) - -> !cute.memref, "((2,2,2),4,2):((1,2,4),16,8)"> - - // Copy the data from SMEM to RMEM. - cute.copy(..., %partitioned, %fragment) - - // Example 2: Make fragment for SM100 (SMEM). - %fragment = cute.mma.make_fragment A (%0, %1) - : (!cute.tiled_mma, - atom_layout_MNK = <"(1,1,1):(0,0,0)">>, - !cute.memref, "((128,16),2,32):((512,1),65536,16)">) - -> !cute_nvgpu.smem_desc_view - - // Example 3: Make fragment for SM100 (TMEM) from layout. - %fragment = cute.mma.make_fragment A (%0, %1) - : (!cute.tiled_mma, - atom_layout_MNK = <"(1,1,1):(0,0,0)">>, - !cute.layout<"((64,16),1,4):((1,64),0,1024)">) - -> !cute.memref, "((128,16),1,4):((131072,1),0,16)"> - - // Example 3: Make fragment for SM100 (TMEM) from shape. - %fragment = cute.mma.make_fragment A (%0, %1) - : (!cute.tiled_mma, - atom_layout_MNK = <"(1,1,1):(0,0,0)">>, - !cute.shape<"((64,16),1,4)">) - -> !cute.memref, "((128,16),1,4):((131072,1),0,16)"> - ``` - }]; - let arguments = (ins - Cute_MmaAtomTypeInterfaceInstance:$atom, - Cute_AMemoryReferenceTypeThatRepresentsATensor:$input, - Cute_MmaOperandEnum:$operand_id - ); - let results = (outs - Cute_ViewTypeInterfaceInstance:$result - ); -} -def Cute_PrefetchAtomCallOp : Cute_Op<"prefetch_atom_call", [MemoryEffects<[MemRead]>]> { - let summary = "`cute.exec_prefetch_atom` op executes a prefetch atom operation with the given operands."; - let description = [{ - Following src layout of atom are valid - * ((atom_v, rest_v)) - * ((atom_v)) - * (atom_v) - - The `src` operand is variadic: - - For regular TMA prefetch, `src` is a single tensor. - - For 2D `tile::gather4` mode, `src` is `[data_coord_tensor, gmem_index_tensor]`. - - Examples: - - ```mlir - // Call a prefetch atom operation - cute.prefetch_atom_call(%copy_tma_load, %src_v) - : (!cute_nvgpu.atom.tma_load, - %src: !cute.coord_tensor<"(0,0)", "(((64,128),1),1,1,2):(((1@0,1@1),0),0,0,64@0)">) - -> () - - // gather4 prefetch: data tensor + gmem index tensor - cute.prefetch_atom_call(%gather4_atom, %src_v, %idx_v) - : (!cute_nvgpu.atom.tma_load>, - !cute.coord_tensor<...>, - !cute.coord_tensor<...>) - -> () - ``` - }]; - let arguments = (ins - Cute_PrefetchAtomTypeInterfaceOrTiledCopyTypeWithPrefetchAtomTypeInterfaceCopyAtom:$atom, - Variadic:$src - ); - let results = (outs - ); -} -def Cute_PrependToRankOp : Cute_InferredOp<"prepend_to_rank", [Pure]> { - let summary = "Prepend element to input to rank N."; - let description = [{ - Example: - - ```mlir - %shp = cute.make_shape() : !cute.shape<"(3,4,5)"> - %elem = cute.make_shape() : !cute.shape<"(1,2)"> - %new_layout = cute.prepend_to_rank<5> (%shp, %elem) - : (!cute.shape<"(3,4,5)">, !cute.shape<"1">) -> !cute.shape<"((1,2),(1,2),3,4,5)"> - ``` - }]; - let arguments = (ins - Cute_ValueTypeInterfaceInstance:$input, - Cute_ValueTypeInterfaceInstance:$element, - SI32Attr:$rank - ); - let results = (outs - Cute_ValueTypeInterfaceInstance:$result - ); -} -def Cute_PrintOp : Cute_Op<"print"> { - let summary = "Debug operation to print out value of cute data type."; - let description = [{ - Supported following cute data types - * shape, stride, layout, tile - * View: memref, coord_tensor - * Iterator: ptr, tuple - - Examples: - - ```mlir - cute.print(%0) : !cute.shape<"(3,5)"> - - cute.print(%1) : !cute.layout<"(5,(3,?)):(?,(4,5))"> - - // variadic operands without format string - cute.print(%0, %1) : !cute.shape<"(3,5)">, !cute.layout<"(5,(3,?)):(?,(4,5))"> - - // with format string - cute.print("{} -> {}", %0, %1) : !cute.shape<"(3,5)">, !cute.layout<"(5,(3,?)):(?,(4,5))"> - ``` - - The optional `operand_signed` attribute carries the DSL-level signedness of - each operand, one entry per value. It exists because DSL integers lower to - signless MLIR integer types, so an unsigned value is indistinguishable from - a signed one by type alone. When an entry is false, a generic `{}` - placeholder for that integer operand is rendered with an unsigned specifier - (`%u`/`%llu`) instead of `%d`/`%lld`. When absent, all operands are treated - as signed, and an explicit `%u` in the format string still forces unsigned. - }]; - let arguments = (ins - Variadic:$values, - OptionalAttr:$fmt, - OptionalAttr:$operand_signed - ); - let results = (outs - ); -} -def Cute_PrintTMADescIm2ColOp : Cute_Op<"print_tma_desc_im2col"> { - let summary = "This operation interprets the input i64 as a pointer to a TmaDescriptorInternal."; - let description = [{ - Example: - ```mlir - %tma_desc_a_host = cute_nvgpu.make_tma_desc_im2col( - %tensor, - %slayout, - %cta_v_tile, - %lower_corner_whd, - %upper_corner_whd, - %lower_padding_whd, - %upper_padding_whd, - %stride_whd, - %lower_srt, - %stride_srt) - <{host = true}> : ( - !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, - !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, - !cute.layout<"(128,32):(1@0,1@1)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(1,1,1)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(1,1,1)">) -> !cute.ptr> - %tma_desc_a_host_ptr_i64 = cute_nvgpu.cast_tma_desc_to_integer(%tma_desc_a_host: !cute.ptr) -> i64 - %fd = arith.constant 1 : i32 // stdout - cute.print_tma_desc_im2col(%tma_desc_a_host_ptr_i64, %fd) - }]; - let arguments = (ins - I64:$input, - I32:$fd - ); - let results = (outs - ); -} -def Cute_PrintTMADescTiledOp : Cute_Op<"print_tma_desc_tiled"> { - let summary = "This operation interprets the input i64 as a pointer to a TmaDescriptorInternal."; - let description = [{ - Example: - ```mlir - %tma_desc_a_host = cute_nvgpu.make_tma_desc_tiled(%a_tensor, %smem_ablayout_128_64_k_major, %cta_v_tile_128_64_k_major) - <{mode = host = true}> : - ( - !cute.memref, "(?,?,?) : (?,1,?)">, - !cute.composed_layout<"S<3, 4, 3> o 0 o ((128,64),1,1):((64,1),0,0)">, - !cute.layout<"((128,64),1,1):((1@0,1@1),0,0)"> - ) -> !cute.ptr - %tma_desc_a_host_ptr_i64 = cute_nvgpu.cast_tma_desc_to_integer(%tma_desc_a_host: !cute.ptr) -> i64 - %fd = arith.constant 1 : i32 // stdout - cute.print_tma_desc_tiled(%tma_desc_a_host_ptr_i64, %fd) - ``` - }]; - let arguments = (ins - I64:$input, - I32:$fd - ); - let results = (outs - ); -} -def Cute_PrintViewOp : Cute_Op<"print_view"> { - let summary = "Debug operation to print cute view types."; - let description = [{ - Currently supports cute.memref and cute.tensor. MemRefs with both static layout - and dynamic layout are accepted. An optional coord parameter can be used to - specify the element to be printed. If no coord is provided, all elements will - be printed. - - The `verbose` attribute controls the output format: - - When `verbose` is true (default), it prints the full details of the view - including memory address, shape, and strides. - - When `verbose` is false, it prints in a more compact torch-style matrix format. - - The `signed` attribute is used for integer data types to control whether values - should be interpreted as signed or unsigned. This attribute is ignored for - floating point types and coord tensors. - - Examples: - - ```mlir - cute.print_view(%0) : !cute.memref - output: - Memref raw_ptr(0x0000152397fffc80: f32, rmem, align<4>) o (2,2):(2,1) = ( - (0,0)=1.000000 - (0,1)=2.000000 - (1,0)=3.000000 - (1,1)=4.000000 - ) - - cute.print_view(%1) : !cute.memref - output: - Memref raw_ptr(0x0000148eabfffbf0: f32, rmem, align<4>) o (2,(2,2)):(4,(2,1)) = ( - (0,(0,0))=1.000000 - (0,(0,1))=2.000000 - (0,(1,0))=3.000000 - (0,(1,1))=4.000000 - (1,(0,0))=5.000000 - (1,(0,1))=6.000000 - (1,(1,0))=7.000000 - (1,(1,1))=8.000000 - ) - - cute.print_view(%3,%coord) : !cute.memref,!cute.coord<"(?,(?,?))"> - output: - Memref raw_ptr(0x0000150ce3fffc90: f32, rmem, align<4>) o (2,(2,2)):(4,(2,1)), coord (1,(1,0)) = (7.000000) - - // Using verbose=false to print in torch-style format - cute.print_view(%0) {verbose = false} : !cute.memref - - output: - tensor(raw_ptr(0x000000003746a2e0: f64, generic, align<8>) o (8,5,3):(15,3,1), data= - [[[ 1.426091, -0.279034, -1.635684, -0.026360, -0.732545, ], - [-0.733555, 0.886134, -1.418841, -0.203192, -1.836270, ], - ... - [-1.172012, 1.678573, -0.004324, 0.441605, 1.484385, ], - [-0.550848, 1.455505, -0.968547, 1.233722, 0.812084, ]], - - [[ 0.762814, -0.641657, 0.843695, -1.380960, 0.261200, ], - [ 1.500137, -0.921287, 1.001725, 0.106540, -1.326231, ], - ... - [-0.878001, -0.465694, -1.204548, -0.407944, -0.092967, ], - [ 1.035407, 0.043012, -1.959517, 0.075913, 0.893392, ]], - - [[-0.129989, 0.750885, 0.219408, 0.442481, -0.508588, ], - [ 0.791268, -0.547978, 0.194177, 0.652729, 2.642380, ], - ... - [-0.674993, -1.017359, -1.272791, 0.829529, 0.852879, ], - [-0.256070, -0.287006, 1.900199, -0.492318, -0.658134, ]]]) - - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstance:$src, - Optional:$coord, - DefaultValuedAttr:$verbose, - DefaultValuedAttr:$is_signed - ); - let results = (outs - ); -} -def Cute_PtrLoadOp : Cute_InferredOp<"ptr.load", [MemoryEffects<[MemRead]>]> { - let summary = "Loads an LLVM or CuTe value from the given `!cute.ptr`."; - let arguments = (ins - Cute_CuTePointerType:$ptr - ); - let results = (outs - Cute_CuTePointerTypeOrAMemoryReferenceType:$value - ); -} -def Cute_PtrStoreOp : Cute_Op<"ptr.store"> { - let description = [{ - Stores the given CuTe value at the provided pointer. Handles any - LLVM-storable type, including those that can be stored with - `llvm.store` *and* CuTe types. - }]; - let arguments = (ins - Cute_CuTePointerTypeOrAMemoryReferenceType:$value, - Cute_CuTePointerType:$ptr - ); - let results = (outs - ); -} -def Cute_PtrToIntOp : Cute_Op<"ptrtoint", [Pure]> { - let description = [{ - The `cute.ptrtoint` operation accepts a cute pointer value and casts it to an - integer value. The pointer must have byte alignment, meaning subbyte types - must have explicit alignment specified (e.g., align<1> or higher). - - Example: - - ```mlir - %int_aligned = cute.ptrtoint(%ptr) : !cute.ptr> to i64 - ``` - }]; - let arguments = (ins - Cute_PointerTypeInterfaceInstance:$src - ); - let results = (outs - Cute_Integer:$dst - ); -} -def Cute_RakedProductOp : Cute_InferredOp<"raked_product", [Pure]> { - let description = [{ - This operation performs "raked_product" on the input and produce a raked_product result, - and it automatically infer the "raked_product" layout for the result type. - - Example: - - ```mlir - %input = cute.make_layout() : !cute.layout<"(3,4):(4,1)"> - %tiler = cute.make_layout() : !cute.layout<"(2,5):(1,2)"> - %prod = cute.raked_product(%arg0, %arg1) - : (!cute.layout<"(3,4):(4,1)">, !cute.layout<"(2,5):(1,2)">) - -> !cute.layout<"((2,3),(5,4)):((12,4),(24,1))"> - ``` - - Example with dynamic composed_layout: - - ``` - %prod = cute.raked_product(%input, %tiler) - : (!cute.composed_layout<"(?):(?) o ? o (5,4):(4,1)">, - !cute.layout<"(?,?):(?,?)">) - -> !cute.composed_layout<"(?):(?) o ? o ((?,5),(?,4)):((?{div=20},4),(?{div=20},1))"> - ``` - }]; - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTe:$input, - Cute_CuTeLayoutType:$tiler - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_RandomDelayOp : Cute_Op<"random_delay"> { - let arguments = (ins - ); - let results = (outs - ); -} -def Cute_RecastIterOp : Cute_Op<"recast_iter", [Pure]> { - let description = [{ - This `cute.recast_iter` is similar to CuTe C++'s recast_ptr, convert two iterators - in certain aspects like type/swizzle for convenience. - But this operation has some usage limitation for correctness, - like it's not allowed for TMA tensor's iterator and it's not allowed to convert align attribute. - In general, it's not allowed to convert two different iterator kinds (PtrType <-> IntTupleType) - - Example: - - ```mlir - %ptr = cute.recast_iter(%0) : !cute.ptr, S<3,4,3>> - to !cute.ptr, S<2,4,3>> - ``` - }]; - let arguments = (ins - Cute_CuTePointerType:$src - ); - let results = (outs - Cute_CuTePointerType:$dst - ); -} -def Cute_RecastLayoutOp : Cute_InferredOp<"recast_layout", [Pure]> { - let summary = "Example:"; - let description = [{ - ```mlir - # upcast<8> operation - %0 = cute.recast_layout<32, 4>(%arg0) - : !cute.layout<"(128,64):(1,128)"> to !cute.layout<"(16,64):(1,16)"> - # upcast<2> for composed layout - %0 = cute.recast_layout<32, 16>(%arg0) - : !cute.composed_layout<"((6,7),8):((10,12),1) o 6 o (8,2):(1,4)"> - to !cute.composed_layout<"((6,7),4):((5,6),1) o 3 o (4,2):(1,2)"> - # downcast<8> operation - %0 = cute.recast_layout<8, 32>(%arg0) - : !!cute.layout<"(16,64):(1,16)"> to cute.layout<"(128,64):(1,128)"> - ``` - }]; - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTe:$src, - I32Attr:$new_type_bits, - I32Attr:$old_type_bits - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTe:$dst - ); -} -def Cute_ReduceOp : Cute_InferredOp<"reduce", [Pure]> { - let description = [{ - The `cute.reduce` operation reduces elements of a vector according to a given - shape and reduction operator, optionally guided by a reduction profile and/or a - predication mask. - - The reduction is performed over the dimensions of the input vector as described - by the provided shape. By default, all dimensions are reduced to a scalar. If a - reduction profile is specified, it determines which dimensions are kept (marked - with '_') and which are reduced (marked with an integer or tuple structure). The - result is either a scalar or a lower-rank vector, along with the reduced shape - if applicable. - - An optional predication vector (of the same size as the input vector) can be - provided to mask out elements from the reduction, enabling conditional - reduction. - - The reduction operator (e.g., MIN, MAX, ADD, MUL) is specified via the `op` - attribute. - - Examples: - ```mlir - // Reduce all elements to a scalar (minimum) - %reduced = cute.reduce(%input, %shape, %init) - { op = #cute.reduction_op } - : (vector<20xf32>, !cute.shape<"(4,5)">, f32) -> f32 - - // Reduce all elements to a scalar (maximum) with predication - %reduced = cute.reduce(%input, %shape, %init, %pred) - { op = #cute.reduction_op } - : (vector<20xf32>, !cute.shape<"(4,5)">, f32, vector<20xi1>) -> f32 - - // Reduce all but the second dimension (profile keeps dimension 1) - %reduced, %red_shp = cute.reduce(%input, %shape, %init) - { op = #cute.reduction_op, profile = #cute.coord<"(1,_,1)"> } - : (vector<60xf32>, !cute.shape<"(5,4,3)">, f32) -> (vector<4xf32>, !cute.shape<"(4)>") - - // Reduce a nested dimension (profile keeps outer and first inner dimension) - %reduced, %red_shp = cute.reduce(%input, %shape, %init) - { op = #cute.reduction_op, profile = #cute.coord<"(_,(_,1))"> } - : (vector<30xf32>, !cute.shape<"(5,(3,2))">, f32) -> (vector<15xf32>, !cute.shape<"(5,3)>") - ``` - The operation supports both flat and nested shapes and profiles, enabling reductions over complex multidimensional data layouts. - }]; - let arguments = (ins - Cute_OfRanks1:$input, - AnyType:$shape, - AnyType:$init_val, - Optional:$predication, - Cute_ReductionOpAttr:$reduction_op, - OptionalAttr:$reduction_profile - ); - let results = (outs - Cute_OfRanks1OrFloatingPointOrInteger:$output, - Optional:$reduced_shape - ); -} -def Cute_RightInverseOp : Cute_InferredOp<"right_inverse", [Pure]> { - let description = [{ - This operation builds the right inverse of a layout. - The current implementation only supports static layout. - An error is returned when the layout contains dynamic values. - - Example: - - ```mlir - %layout = cute.make_layout(.., ..): !cute.layout<"(8, 4):(4, 1)"> - %inv_layout = cute.right_inverse %layout : (!cute.layout<"(8, 4):(4, 1)">) -> !cute.layout<"(4, 8):(8, 1)"> - ``` - }]; - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTe:$input - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_SelectOp : Cute_InferredOp<"select", [Pure]> { - let description = [{ - This operation extracts specific dimensions out of a type implementing - the `CuteValueTypeInterface`, e.g. a shape, a coordinate, a tuple, a layout, ... - - The `mode` attribute indices are used to select the portion of the input type. - The indices in the `mode` attribute can be permutated or repeated, but should - be less than the rank of type. - - Example: - - ```mlir - %shape = cute.make_shape() : () -> !cute.shape<"(1, (2, 4), 8)"> - %mode0 = cute.select<[1]>(%shape) - : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"((2, 4))"> - %mode1 = cute.select<[0,1]>(%shape) - : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(1, (2,4))"> - %mode2 = cute.select<[2,0]>(%shape) - : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(8, 1)"> - %mode3 = cute.select<[2,2]>(%shape) - : (!cute.shape<"(1,(2,4),8)">) -> !cute.shape<"(8, 8)"> - ``` - }]; - let arguments = (ins - Cute_ValueTypeInterfaceInstance:$input, - DenseI32ArrayAttr:$mode - ); - let results = (outs - Cute_ValueTypeInterfaceInstance:$result - ); -} -def Cute_ShapeDivOp : Cute_InferredOp<"shape_div", [Pure]> { - let description = [{ - Division for Shapes. - Case Tuple Tuple: - Perform shape_div element-wise - Case Tuple Int: - Fold the division of b across each element of a - Example: shape_div((4,5,6),40) -> shape_div((1,5,6),10) -> shape_div((1,1,6),2) -> (1,1,3) - Case Int Tuple: - Return shape_div(a, product(b)) - Case Int Int: - Enforce the divisibility condition a % b == 0 || b % a == 0 when possible - Return a / b with rounding away from 0 (that is, 1 or -1 when a < b) - - ```mlir - %res = cute.shape_div(%a, %b) - : (!cute.int_tuple<"(4,5)">, !cute.int_tuple<"(40)">) -> !cute.int_tuple<"(1,1)"> - ``` - }]; - let arguments = (ins - Cute_CuteShapeType:$a, - Cute_CuteShapeType:$b - ); - let results = (outs - Cute_CuteShapeType:$result - ); -} -def Cute_SizeOp : Cute_InferredOp<"size", [Pure]> { - let summary = "This operation computes the size of a cute value type by multiplying its dimensions."; - let description = [{ - The `mode` attribute specifies the nested mode in the type for size calculation. - It indexes a sub-scope within the input type and returns the size of that sub-scope. - The rank of `mode` must be less than or equal to the depth of the type. - If `mode` is empty, the operation calculates the size of the entire type. - - ### Examples - - ```mlir - %shape = cute.make_shape() : () -> !cute.shape<"(1,(2,4),9)"> - %size = cute.size(%shape) <{mode=[1]}> : (!cute.shape<"(1,(2,4),9)">) -> !cute.int_tuple<"8"> - ``` - - In this example, the mode attribute is set to [1], which selects the second dimension of the - shape (1,(2,4),9). The size of this dimension is calculated as 2 * 4 = 8. - - ```mlir - %shape = cute.make_shape() : () -> !cute.shape<"(1,(2,4),8)"> - %size = cute.size(%shape) : (!cute.shape<"(1,(2,4),8)">) -> !cute.int_tuple<"64"> - ``` - - In this example, the mode attribute is empty, so the operation calculates the size of the entire - shape (1,(2,4),8). The size is calculated as 1 * 2 * 4 * 8 = 64. - }]; - let arguments = (ins - Cute_CuteShapeTypeOrIntTupleAsIntegerOr2:$input, - DefaultValuedAttr:$mode - ); - let results = (outs - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$result - ); -} -def Cute_SliceOp : Cute_InferredOp<"slice", [Pure]> { - let description = [{ - This operation takes slices of a input memref according to the `coord`. Note that - `!coord<"(_,_,k)">` means "create a slice that views (i, j, k) for all valid i, - all valid j, and a specific value of k. - - The `coord` elements can be static, dynamic (represented by "?"), or underscore - (similar to the ":" (the colon punctuation) in Python or Fortran array slices). - However, the mode of `coord` should be less than or equal to the mode of `input` - memref layout; otherwise, a runtime error will occur. - - Example: - - ```mlir - %slice = cute.slice(%alloc, %coord) - : !cute.memref, !cute.coord<"(0, (_, _), 0)"> - ``` - - Example with dynamic coord: - - ```mlir - %slice = cute.slice(%alloc, %coord) - : !cute.memref, !cute.coord<"(_,(64,?),0)"> - ``` - - Example with scaled basis in layout: - - ```mlir - %slice = cute.slice(%coord_tensor, %coord) - : !cute.coord_tensor<"(0,0)", "(10,20,30):(1@0,1@1,20@1)">, !cute.coord<"(0,2,_)"> - ``` - Example with swizzled layout: - - ```mlir - %slice = cute.slice(%swizzle_layout, %coord) - : (!cute.composed_layout<"S<1,5,1> o 0 o ((2,4),(8,4)):((1,16),(2,64))">, - !cute.coord<"((?,?),(_,_))">) -> !cute.composed_layout<"(8,(2,2)):(2,(?,128))"> - ``` - - Example with PDSL memref slice (swizzle in units of bytes): - - ```mlir - %res_memref = !cute.slice(%memref, %crd) : !cute.memref, "(64,32):(32,1)">, !cute.coord - ``` - - Example with PISL memref slice (swizzle in units of elements): - - ```mlir - %res_memref = !cute.slice(%memref, %crd) : !cute.memref o 0 o (64,32):(32,1)">, !cute.coord - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, - Cute_CuteCoordType:$coord - ); - let results = (outs - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_SmemPartitionSizeOp : Cute_Op<"smem_partition_size", [Pure]> { - let description = [{ - This operation queries the static smem layout for one partition in the - current kernel and returns the partition size as an i64 integer. It is only - valid on the kernel side; host-side total smem queries should use - `cute.kernel_smem_size`. - }]; - let arguments = (ins - I32Attr:$partition_id, - UnitAttr:$cumulative - ); - let results = (outs - I64:$size - ); -} -def Cute_StaticOp : Cute_Op<"static", [Pure]> { - let description = [{ - Creates a compile-time constant value with a fully static CuTe type. This operation takes no - operands and produces a single result of one of the following CuTe types: - - - !cute.int_tuple: A tuple of static integers - - !cute.coord: A coordinate with static dimensions - - !cute.stride: A stride with static values - - !cute.shape: A shape with static dimensions - - !cute.tile: A tile with static dimensions and layout - - !cute.layout: A layout with static shape and strides - - !cute.composed_layout: A composed layout with static inner/outer layouts - - The type of the result must be fully static (contain no dynamic/unknown values). - - Some atom types can be fully static with no runtime states, e.g. `UniversalFma`. - - ### Examples: - - ```mlir - // Create a static shape with value 5 - %0 = cute.static : !cute.shape<"5"> - - // Create a static 2D layout with shape (32,64) and strides (64,1) - %1 = cute.static : !cute.layout<"(32,64):(64,1)"> - ``` - }]; - let arguments = (ins - ); - let results = (outs - Cute_MaybeStaticTypeInterfaceInstance:$result - ); -} -def Cute_StencilDivideOp : Cute_InferredOp<"stencil_divide", [AttrSizedOperandSegments, Pure]> { - let summary = "A convolution has the following input tensors:"; - let description = [{ - * Activation tensor (Act): `((N,(D,H,W,...)), (C,(1,1,1)))` - * Filter tensor (Flt): `(K,(C,(T,R,S, ...)))` - * Output tensor (Out): `((N,(Z,P,Q, ...)), K)` - - Where - - * `N` := number of samples in the signal - * `D`, `H`, `W`, ... := spatial dimensions of the activation tensor - * `C` := channel dimension of the activation tensor - * `K` := channel dimension of the filter and output tensor - * `T`,`R`,`S`,... := spoke dimensions of the filter tensor - * `Z`,`P`,`Q`,... := spatial dimensions of the output tensor - - The number of spatial dimensions must DHW... must match both TRS... and ZPQ... - - As is evident in the tensor shapes, these cannot be issued to a GEMM without transformation, since - there are no logical M, N, and K modes into which we can group the tensor modes. - - Every spoke of the filter tensor (TRS) will be applied to some (offset) - view of the activation tensor, thus expanding the logical size of the activation tensor. - Additionally, a similar logical transform of the spatial dimensions can be encoded as - a function of the padding, dilations, traversal strides, and filter spokes. - This gets us to our im2col transform: - - Im2col transform affects the component shapes/strides of the activation tensor in the following way: - - * `ZPQ` Shape : changes `DHW` domain with formula `(1 + (DHW + pad - (((TRS-1) * dilation) + 1)) / traversal_stride)` - * `TRS` Shape : `TRS` domain instead of `(1,1,1)` - * `ZPQ` Strides : Original `DHW` strides get scaled by traversal strides `DHW` - * `TRS` Strides : Original `DHW` strides get scaled by dilation `DHW`. - - With this transform applied, we end up with a set of input and output tensors that are logically - consistent in their `MNK` dimensions, thus allowing us to dispatch to a GEMM. - - * im2col activation layout: `((N,(Z,P,Q)), (C,(T,R,S)))` // logical (M,K) - * filter layout : `( K, (C,(T,R,S)))` // logical (N,K) - * output layout : `((N,(Z,P,Q)), K )` // logical (M,N) - - `stencil_divide` performs this operation given these parameters. - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, - Cute_CuteTileType:$stencil, - Optional:$padding_upper, - Optional:$padding_lower, - Optional:$traversal_stride - ); - let results = (outs - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_SymbolicOp : Cute_Op<"symbolic", [Pure]> { - let description = [{ - A symbol in CuTe IR represent an abstracted value which could be materialized - later in a pass. If it's not materialized, the behavior is undefined. - - Example: - - ```mlir - %0 = cute.symbolic("tile_m") : i32 - %1 = cute.symbolic("cluster_size") : !cute.shape<"(?,2)"> - ``` - }]; - let arguments = (ins - StrAttr:$name - ); - let results = (outs - AnyType:$symbolic - ); -} -def Cute_TileToShapeOp : Cute_InferredOp<"tile_to_shape", [Pure]> { - let description = [{ - The `tile_to_shape` operation is similar to blocked_product, but specifies the - result shape instead of the product shape, which is more convenient in certain - circumstances. - - The op has three operands: - 1. `block`: The layout to repeat. - 2. `trg_shape`: The target shape of the result - 3. `ord_shape`: The order of the modes of `trg_shape` to tile layout `block` with. - E.g. (_2,_1,_3) will cause @a layout to repeat across the second mode first, - the first mode second, and the third mode last. Currently only supports static order. - - Example: - - ```mlir - %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) - : (!cute.layout<"(128,128):(128,1)">, - !cute.shape<"(1024,512)">, - !cute.shape<"(2,1)">) - -> !cute.layout<"((128,8),(128,4)):((128,65536),(1,16384))"> - - %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) - : (!cute.layout<"((32,4),(16,4)):((16,4),(0,1))">, - !cute.shape<"(256,512,2)">, - !cute.shape<"(1,2,3)">) - -> !cute.layout<"((32,4,2),(16,4,8),2):((16,4,512),(0,1,1024),8192)"> - - %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) - : (!cute.layout<"((32,4),(16,4)):((16,4),(0,1))">, - !cute.shape<"(256,512,2)">, - !cute.shape<"(2,1,3)">) - -> !cute.layout<"((32,4,2),(16,4,8),2):((16,4,4096),(0,1,512),8192)"> - - %0 = cute.tile_to_shape(%block, %trg_shape, %ord_shape) - : (!cute.composed_layout<"S<3,4,3> o 0 o ((32,4),(?,4)):((16,4),(0,1))">, - !cute.shape<"(256,?,2)">, - !cute.shape<"(3,2,1)">) - -> !cute.composed_layout<"S<3,4,3> o 0 o ((32,4,2),(?,4,?),2):((16,4,?{div=1024}),(0,1,1024),512)"> - ``` - }]; - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTe:$block, - Cute_CuteShapeType:$trg_shape, - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$ord_shape - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_TiledCopyPartitionDOp : Cute_InferredOp<"tiled.copy.partition_D", [Pure]> { - let description = [{ - The `cute.tiled.copy.partition_X` operation accepts: - 1. cute.tiled_copy; - 2. A ViewTypeInterface; - 3. cute.coord. - returns a new ViewTypeInterface type (MemrefType/CoordTensorType) with partitioned layout - and sliced offset (sliced offset will be implemented in the lowering). - - The new partitioned layout has the following meaning: ((vector_mode), rest_m, rest_n). vector_mode - represents the number of instructions issued and the number of elements processed per - instruction; in our example below, we have (2, 1) : (1, 0), meaning that each - instruction is processing two data. Rest_m is the number of iterations - along the first mode, a single iteration in our example. Finally, rest_n is the - number of iterations along the second mode, 128. - - Example: - - ```mlir - %dst_partitioned = cute.tiled.copy.partition_D(...) - : (!cute.tiled_copy, - layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, - tiler_mn = <"[(4,32):(32,1);2:1]">>, - !cute.memref, "(128,256):(256,1)">, - !cute.coord<"?">) - -> !cute.memref, "((2,1),1,128):((1,0),0,2)"> - ``` - }]; - let arguments = (ins - Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, - Cute_ViewTypeInterfaceInstance:$input, - Cute_CuteCoordType:$coord - ); - let results = (outs - Cute_ViewTypeInterfaceInstance:$result - ); -} -def Cute_TiledCopyPartitionSOp : Cute_InferredOp<"tiled.copy.partition_S", [Pure]> { - let description = [{ - The `cute.tiled.copy.partition_X` operation accepts: - 1. cute.tiled_copy; - 2. A ViewTypeInterface; - 3. cute.coord. - returns a new ViewTypeInterface type (MemrefType/CoordTensorType) with partitioned layout - and sliced offset (sliced offset will be implemented in the lowering). - - The new partitioned layout has the following meaning: ((vector_mode), rest_m, rest_n). vector_mode - represents the number of instructions issued and the number of elements processed per - instruction; in our example below, we have (2, 1) : (1, 0), meaning that each - instruction is processing two data. Rest_m is the number of iterations - along the first mode, a single iteration in our example. Finally, rest_n is the - number of iterations along the second mode, 128. - - Example: - - ```mlir - %dst_partitioned = cute.tiled.copy.partition_D(...) - : (!cute.tiled_copy, - layout_copy_tv = <"((32,4),(2,32)):((0,1),(128,4))">, - tiler_mn = <"[(4,32):(32,1);2:1]">>, - !cute.memref, "(128,256):(256,1)">, - !cute.coord<"?">) - -> !cute.memref, "((2,1),1,128):((1,0),0,2)"> - ``` - }]; - let arguments = (ins - Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, - Cute_ViewTypeInterfaceInstance:$input, - Cute_CuteCoordType:$coord - ); - let results = (outs - Cute_ViewTypeInterfaceInstance:$result - ); -} -def Cute_TiledCopyRetileOp : Cute_InferredOp<"tiled.copy.retile", [Pure]> { - let description = [{ - The cute.tiled.copy.retile operation is used to reorganize the layout of a tensor - that has already been tiled or partitioned. This is typically necessary when - two TiledMma or TiledCopy instances are connected, accessing the same memory, - but require different logical views of that memory. - - ### Use Case - - Consider a scenario where a TiledMma instance is the consumer, and a TiledCopy - instance is the producer. The TiledMma instance partitions the memory, creating - a logical view that suits its needs. However, this view may not be compatible with - the TiledCopy instance. To resolve this, the cute.tiled.copy.retile operation is used - to retile the TiledCopy instance, creating a new logical view that is compatible with - the TiledMma instance. - - ``` - // consumer - partition_A(%tiled_mma, ...) - %mma_view = partition_A(%tiled_mma, %sa_tensor, %idx) - - // producer - %tiled_copy = make_tiled_copy_A(%copy_atom, %tiled_mma) - // retile %mma_view to meet needs of tiled_copy - %lds_view = retile(%tiled_copy, %mma_view) - ``` - - ### Operation - - The cute.tiled.copy.retile operation takes two inputs: - * A cute.tiled_copy instance, which represents the tensor to be retiled. - * A CuTeTypeWithLayout derived type, which represents the tensor that has already been tiled or partitioned. - - - Example: - - ```mlir - %retiled = cute.tiled.copy.retile(%tiled_copy, %src_tensor) - : (!cute.tiled_copy, num_matrices=4>, - layout_copy_tv = <"((4,8,4),((2,2,4),1)):((128,4,1),((64,32,512),0))">, - tiler_mn = <"[(4,16):(32,1);32:1]">>, - !cute.memref) - -> !cute.memref - ``` - }]; - let arguments = (ins - Cute_TiledCopyAtomTypeOrTiledCopyV2:$tiled_copy, - Cute_ViewTypeInterfaceInstance:$input - ); - let results = (outs - Cute_ViewTypeInterfaceInstance:$result - ); -} -def Cute_TiledDivideOp : Cute_InferredOp<"tiled_divide", [Pure]> { - let summary = "This operation performs a tiled_divide on a view or a layout by a tiler."; - let arguments = (ins - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, - Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler - ); - let results = (outs - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_TiledMmaPartitionOp : Cute_InferredOp<"tiled.mma.partition", [Pure]> { - let description = [{ - `cute.tiled.mma.partition` reorganizes a tensor's layout to match what the - MMA atom expects. The operation takes 4 arguments: - - 1. The type of operand. E.g., `CUTE_MMA_OPERAND_A` or `CUTE_MMA_OPERAND_B`. - 2. A `!cute.tiled_mma` value that specifies the kind of MMA. - 3. The unpartitioned operand tensor (ViewTypeInterface-derived SSA value). - 4. The `!cute.coord` value of the execution unit for which to partition the - the input tensor. - - The coordinate type depends on the MMA atom and the operand memory space. - Some examples: - * SM80: thread ID - * SM90: It depends... - - When partitioning A with "frag_kind = ss": warp-group ID - - When partitioning A with "frag_kind = rs": thread ID - - When partitioning B: warp-group ID - - When partitioning C/D: thread ID - * SM100: CTA ID within the cluster (always 0 for 1-CTA mode) - - The operation returns a new ViewTypeInterface-derived value - (CuteMemrefType/CuteCoordTensorType) with partitioned layout and sliced - offset (sliced offset will be implemented in the lowering). - - Example: - - ```mlir - // Partition an operand A tensor of shape 128x32 (residing in shared - // memory) for a !tiled_mma atom that operates on shape 32x16x16. (I.e., - // four SM80 MMA atoms with shape 16x8x16.) The result tensor is a view - // into the A tensor that represents the part that thread %tid_coord is - // operating on. - // - // The last two modes of the result tensor are 4,2. This is because the - // tiled atom is applied 4 times in M direction and 2 times in K direction. - // - // K=0:15 K=16:31 - // ┌────────────────────┬────────────────────┐ - // M=0:31 │ tiled atom (0,0) │ tiled atom (0,1) │ - // ├────────────────────┼────────────────────┤ - // M=32:63 │ tiled atom (1,0) │ tiled atom (1,1) │ - // ├────────────────────┼────────────────────┤ - // M=64:95 │ tiled atom (2,0) │ tiled atom (2,1) │ - // ├────────────────────┼────────────────────┤ - // M=96:127 │ tiled atom (3,0) │ tiled atom (3,1) │ - // └────────────────────┴────────────────────┘ - // - // For each tiled atom application of size 32x16x16, thread %tid_coord gets - // a subview of size 2,2,2 of operand A. (128 threads are cooperatively - // computing the !tiled_mma of shape 32x16x16, but an individual thread - // does not need access to the full 32x16 matrix A part.) - // - %partitioned = cute.tiled.mma.partition A(%0, %1, %tid_coord) - : (!cute.tiled_mma, - atom_layout_MNK = <"(2,2,1)">>, - !cute.memref, S<2,4,3>, "(128,32):(32,1)">, - !cute.coord<"?">) - -> !cute.memref, S<2,4,3>, "((2,2,2),4,2):((1,256,8),1024,16)"> - - // Partition an operand A tensor of shape 256x512 for an SM100 !tiled_mma. - // There are 256/128=2 tiled atom applications in M direction and - // 512/16=32 tiled atom applications in K direction. The atom specifies a - // single CTA, so that CTA get the entire A tensor. The coordinate is not - // really need in this case, but we pass it anyway to keep the API uniform. - %partitioned = cute.tiled.mma.partition A (%0, %1, %cta_coord) - : (!cute.tiled_mma, - atom_layout_MNK = <"(1,1,1):(0,0,0)">>, - !cute.memref, "(256,512):(512,1)">, - !cute.coord<"0">) - -> !cute.memref, "((128,16),2,32):((512,1),65536,16)"> - ``` - }]; - let arguments = (ins - Cute_TiledMMAAtomType:$tiled_mma, - Cute_ViewTypeInterfaceInstance:$input, - Cute_CuteCoordType:$coord, - Cute_MmaOperandEnum:$operand_id - ); - let results = (outs - Cute_ViewTypeInterfaceInstance:$result - ); -} -def Cute_TiledMmaPartitionShapeOp : Cute_InferredOp<"tiled.mma.partition_shape", [Pure]> { - let description = [{ - This operation partitions a shape into an MMA shape. For example, it takes a logical (M,K,...) - shape and tiles it into a shape like (MMA,MMA_M,MMA_K,...) where - - MMA is the MMA mode, for example (128,16) for an MMA instruction of shape 128x128x16 - - MMA_M is the number of times the MMA repeats in the M-mode - - MMA_K is the number of times the MMA repeats in the K-mode - - ... denotes the "rest" - }]; - let arguments = (ins - Cute_TiledMMAAtomType:$tiled_mma, - Cute_CuteShapeType:$input, - Cute_MmaOperandEnum:$operand_id - ); - let results = (outs - Cute_CuteShapeType:$result - ); -} -def Cute_TiledProductOp : Cute_InferredOp<"tiled_product", [Pure]> { - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTe:$input, - Cute_CuTeLayoutType:$tiler - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_ToIntTupleOp : Cute_InferredOp<"to_int_tuple", [Pure]> { - let description = [{ - The `cute.cast_to_int_tuple` operation casts from any x-tuple type (shape, stride, coord) to an int-tuple type. - The operation preserves the values but changes their type to int-tuple. - - cast will fail if input types contains modes not supported by int-tuple, e.g. _, scaled based, etc. - - Example: - - ```mlir - // Cast from shape to int_tuple - %0 = cute.to_int_tuple(%shape) : !cute.shape<"(8,4)"> to !cute.int_tuple<"(8,4)"> - - // Cast from stride to int_tuple - %1 = cute.to_int_tuple(%stride) : !cute.stride<"(8,4)"> to !cute.int_tuple<"(8,4)"> - - // Cast from coord to int_tuple - %2 = cute.to_int_tuple(%coord) : !cute.coord<"(8,4)"> to !cute.int_tuple<"(8,4)"> - - // Cast from stride to int_tuple preserving divisibility - %3 = cute.to_int_tuple(%stride) : !cute.stride<"(8,?{div=4})"> to !cute.int_tuple<"(8,?{div=4})"> - ``` - }]; - let arguments = (ins - Cute_XTupleLikeType:$src - ); - let results = (outs - Cute_IntTupleAsIntegerOrTupleOfIntTupleS:$dst - ); -} -def Cute_TupleAddOp : Cute_InferredOp<"tuple_add", [Pure]> { - let description = [{ - The `cute.tuple_add` operation performs element-wise addition between two CuTe x-tuple types. - It accepts two input operands and returns a result of the same kind, where each element in the result - is the sum of the corresponding elements in the input tuples. - - This operation applies subtraction recursively for nested tuples, matching corresponding elements - at each level of the tuple hierarchy. - - Example: - - ```mlir - // Element-wise add two arithmetic tuples - %result = cute.tuple_add(%a, %b) - : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(4,?))"> - - // Element-wise add with static values - %b = cute.static : !cute.int_tuple<"(3,(1,3))"> - %result = cute.tuple_add(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(4,?))"> - ``` - }]; - let arguments = (ins - Cute_ArithmeticTupleType:$lhs, - Cute_ArithmeticTupleType:$rhs - ); - let results = (outs - Cute_ArithmeticTupleType:$result - ); -} -def Cute_TupleDivOp : Cute_InferredOp<"tuple_div", [Pure]> { - let description = [{ - The `cute.tuple_div` operation performs element-wise division between two CuTe x-tuple types. - It accepts two input operands and returns a result of the same kind, where each element in the result - is the quotient of the corresponding elements in the input tuples. - - This operation applies division recursively for nested tuples, matching corresponding elements - at each level of the tuple hierarchy. - - Example: - - ```mlir - // Element-wise divide two arithmetic tuples - %result = cute.tuple_div(%a, %b) - : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(3,?))"> - - // Element-wise divide with static values - %b = cute.static : !cute.int_tuple<"(3,(1,3))"> - %result = cute.tuple_div(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(0,?))"> - ``` - }]; - let arguments = (ins - Cute_ArithmeticTupleType:$lhs, - Cute_ArithmeticTupleType:$rhs - ); - let results = (outs - Cute_ArithmeticTupleType:$result - ); -} -def Cute_TupleModOp : Cute_InferredOp<"tuple_mod", [Pure]> { - let description = [{ - The `cute.tuple_mod` operation computes the element-wise modulo (remainder) - between two CuTe x-tuple types. Both operands must be of the same tuple kind - (e.g., `!cute.int_tuple`), and the result is a tuple of the same structure - and type. - - For each corresponding element in the input tuples, the operation computes - the remainder of the division of the left-hand side element by the right-hand - side element. If the tuples are nested, the operation is applied recursively, - matching elements at each level of the tuple hierarchy. - - This operation supports both static and dynamic tuple elements. When static - values are provided, the modulo is computed at compile time; otherwise, it is - performed at runtime. - - **Examples:** - - ```mlir - // Element-wise modulo of two arithmetic tuples - %result = cute.tuple_mod(%a, %b) - : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) - -> !cute.int_tuple<"(?,(0,?))"> - - // Element-wise modulo with static values - %b = cute.static : !cute.int_tuple<"(3,(1,3))"> - %result = cute.tuple_mod(%a, %b) - : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(1,?))"> - ``` - - **Semantics:** - - The operation traverses the tuple structure recursively. - - For each pair of corresponding elements `(x, y)`, the result is `x % y`. - - The structure and rank of the result tuple matches that of the operands. - - If the tuple contains nested tuples, the modulo is applied to each - corresponding sub-tuple. - - This operation is useful for broadcasting, tiling, and other tensor algebra - applications where modular arithmetic over structured tuples is required. - }]; - let arguments = (ins - Cute_ArithmeticTupleType:$lhs, - Cute_ArithmeticTupleType:$rhs - ); - let results = (outs - Cute_ArithmeticTupleType:$result - ); -} -def Cute_TupleMulOp : Cute_InferredOp<"tuple_mul", [Pure]> { - let description = [{ - The `cute.tuple_mul` operation performs element-wise multiplication between two CuTe x-tuple types. - It accepts two input operands and returns a result of the same kind, where each element in the result - is the product of the corresponding elements in the input tuples. - - This operation applies multiplication recursively for nested tuples, matching corresponding elements - at each level of the tuple hierarchy. - - Example: - - ```mlir - // Element-wise multiply two arithmetic tuples - %result = cute.tuple_mul(%a, %b) - : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(3,?))"> - - // Element-wise multiply with static values - %b = cute.static : !cute.int_tuple<"(3,(1,3))"> - %result = cute.tuple_mul(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(3,?))"> - ``` - }]; - let arguments = (ins - Cute_ArithmeticTupleType:$lhs, - Cute_ArithmeticTupleType:$rhs - ); - let results = (outs - Cute_ArithmeticTupleType:$result - ); -} -def Cute_TupleProductEachOp : Cute_InferredOp<"tuple.product_each", [Pure]> { - let summary = "This operation performs \"product_each\" on each element in a tuple"; - let description = [{ - Example: - - ```mlir - %tuple = cute.make_int_tuple() : !cute.int_tuple<"(2,(5,4))"> - %prod = cute.tuple.product_each(%tuple) : (!cute.int_tuple<"(2,(5,4))">) -> !cute.int_tuple<"(2, 20)"> - ``` - }]; - let arguments = (ins - Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input - ); - let results = (outs - Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result - ); -} -def Cute_TupleProductOp : Cute_InferredOp<"tuple.product", [Pure]> { - let summary = "This operation performs \"product\" on all elements in a tuple"; - let description = [{ - Example: - - ```mlir - %tuple = cute.make_int_tuple() : !cute.int_tuple<"(2,(5,4))"> - %prod = cute.product(%tuple) : (!cute.int_tuple<"(2,(5,4))">) -> !cute.int_tuple<"40"> - ``` - }]; - let arguments = (ins - Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$input - ); - let results = (outs - Cute_IntTupleAsIntegerOrTupleOfIntTupleS2:$result - ); -} -def Cute_TupleSubOp : Cute_InferredOp<"tuple_sub", [Pure]> { - let description = [{ - The `cute.tuple_sub` operation performs element-wise subtraction between two CuTe x-tuple types. - It accepts two input operands and returns a result of the same kind, where each element in the result - is the difference of the corresponding elements in the input tuples. - - This operation applies subtraction recursively for nested tuples, matching corresponding elements - at each level of the tuple hierarchy. - - Example: - - ```mlir - // Element-wise subtract two arithmetic tuples - %result = cute.tuple_sub(%a, %b) - : (!cute.int_tuple<"(?,(3,?))">, !cute.int_tuple<"(3,(1,?))">) -> !cute.int_tuple<"(?,(2,?))"> - - // Element-wise subtract with static values - %b = cute.static : !cute.int_tuple<"(3,(1,3))"> - %result = cute.tuple_sub(%a, %b) : (!cute.int_tuple<"(?,(3,?))">) -> !cute.int_tuple<"(?,(2,?))"> - ``` - }]; - let arguments = (ins - Cute_ArithmeticTupleType:$lhs, - Cute_ArithmeticTupleType:$rhs - ); - let results = (outs - Cute_ArithmeticTupleType:$result - ); -} -def Cute_UnpackTupleOp : Cute_Op<"unpack_tuple", [Pure]> { - let description = [{ - Unpacks a tuple into its individual components. The number and types - of the components must match the number and types of the results. - }]; - let arguments = (ins - Cute_ATupleIsAFixedLengthOrderedList:$tuple - ); - let results = (outs - Variadic:$result - ); -} -def Cute_ZippedDivideOp : Cute_InferredOp<"zipped_divide", [Pure]> { - let description = [{ - This operation performs "zipped_divide" on the input and produce a zipped_divided result, - and it automatically infer the "zipped_divided" layout for the result type. - - Example: - - ```mlir - %alloc = cute.memref.alloc() : !cute.memref - %shape = cute.make_shape() : () -> !cute.shape<"(4,8)"> - %zdiv = cute.zipped_divide(%alloc, %shape) - : !cute.memref, !cute.shape<"(4,8)"> - ``` - }]; - let arguments = (ins - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$input, - Cute_TileLikeTypesEGLayoutTypeShapeTypeTileType:$tiler - ); - let results = (outs - Cute_ViewTypeInterfaceInstanceOrGeneralizedLayoutTypeOfCuTe:$result - ); -} -def Cute_ZippedProductOp : Cute_InferredOp<"zipped_product", [Pure]> { - let arguments = (ins - Cute_GeneralizedLayoutTypeOfCuTe:$input, - Cute_CuTeLayoutType:$tiler - ); - let results = (outs - Cute_GeneralizedLayoutTypeOfCuTe:$result - ); -} - -#endif // MLIR_DIALECTS_CUTE_IR_CUTEOPS_TD diff --git a/mlir/dialects/cute_ir/CuteTypes.cpp b/mlir/dialects/cute_ir/CuteTypes.cpp deleted file mode 100644 index 604dfbf2..00000000 --- a/mlir/dialects/cute_ir/CuteTypes.cpp +++ /dev/null @@ -1,1602 +0,0 @@ -//===- CuteTypes.cpp - parsers, printers and verifiers of cute types ------===// - -// clang-format off -/*************************************************************************************************** - * Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - * SPDX-License-Identifier: BSD-3-Clause - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * 3. Neither the name of the copyright holder nor the names of its - * contributors may be used to endorse or promote products derived from - * this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * - **************************************************************************************************/ -// clang-format on - -#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" -#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" - -#include "mlir/Dialect/LLVMIR/LLVMTypes.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/DialectImplementation.h" -#include "mlir/IR/OpImplementation.h" -#include "llvm/ADT/TypeSwitch.h" -#include "llvm/Support/FormatVariadic.h" -#include "llvm/Support/MathExtras.h" - -#include - -using namespace mlir; -using namespace mlir::cutlass_compiler::cute; - -namespace cg = cutegen; - -// The compiler's parser of the algebra types reads `<"text">` leniently: a -// missing or unreadable payload, including a bare mnemonic, gives the default -// value (`()`, `():()`, `[]`, `S<0,4,3>`) instead of an error. -template static T parseLenientPayload(mlir::AsmParser &parser) { - std::string str; - if (succeeded(parser.parseOptionalLess()) && - succeeded(parser.parseOptionalString(&str)) && - succeeded(parser.parseOptionalGreater())) - return cutegen::from_string(str).value_or(T{}); - return T{}; -} - -// The compiler's integer type is `!cute.i`, a mnemonic per width that -// the generated dispatch cannot match. The generated CuteDialect::parseType -// and printType pass the dialect parser and printer, so these overloads of -// the renamed dispatch functions catch them first (defined below). -static OptionalParseResult generatedCuteTypeParser(DialectAsmParser &parser, - StringRef *mnemonic, - Type &value); -static LogicalResult generatedCuteTypePrinter(Type type, - DialectAsmPrinter &printer); -#define generatedTypeParser generatedCuteTypeParser -#define generatedTypePrinter generatedCuteTypePrinter - -// Emit the tablegen-generated type storage / parsing machinery (also drives -// CuteDialect::parseType / printType via useDefaultTypePrinterParser). -#define GET_TYPEDEF_CLASSES -#include "mlir/dialects/cute_ir/CuteTypes.cpp.inc" -#undef generatedTypeParser -#undef generatedTypePrinter - -//===----------------------------------------------------------------------===// -// CuteDialect — type registration -//===----------------------------------------------------------------------===// - -namespace { -namespace cute = ::mlir::cutlass_compiler::cute; -namespace nv = ::mlir::cutlass_compiler::cute_nvgpu; - -std::string printed(Type type) { - std::string text; - llvm::raw_string_ostream os(text); - type.print(os); - return text; -} - -// The compiler's name for a tiled copy's atom; empty when it has none. -std::string copyAliasName(Type atom) { - return llvm::TypeSwitch(atom) - .Case([](auto) { return "copy_simt"; }) - .Case([](auto) { return "copy_ldgsts"; }) - .Case([](auto) { return "copy_dsmem_store"; }) - .Case([](auto) { return "copy_blkcp_g2s"; }) - .Case([](auto) { return "copy_blkcp_s2g"; }) - .Case([](auto) { return "copy_blkcp_s2s"; }) - .Case([](auto) { return "copy_g2r"; }) - .Case([](auto) { return "copy_r2g"; }) - .Case([](auto) { return "copy_r2s"; }) - .Case([](auto) { return "copy_s2r"; }) - .Case([](auto) { return "copy_utccp"; }) - .Case( - [](auto t) { return "copy_ldtm_" + std::to_string(t.getNumBit()); }) - .Case( - [](auto t) { return "copy_sttm_" + std::to_string(t.getNumBit()); }) - .Case([](auto t) { - return "copy_ldtm_red_" + std::to_string(t.getNumBit()); - }) - .Case([](auto t) { - return "copy_ldtm_spcompress_" + std::to_string(t.getNumBit()); - }) - .Case([](auto t) { - return "copy_ldsm_" + std::to_string(t.getNumMatrices()); - }) - .Case([](auto t) { - return "copy_stsm_" + std::to_string(t.getNumMatrices()); - }) - .Case([](auto t) { - return "copy_tmaldg_" + std::to_string(t.getCopyBits() / 8) + "b"; - }) - .Case([](auto t) { - return "copy_tmastg_" + std::to_string(t.getCopyBits() / 8) + "b"; - }) - .Case([](auto t) { - return "copy_tmaredg_" + std::to_string(t.getCopyBits() / 8) + "b"; - }) - .Default([](Type) { return "copy_unknown_type"; }); -} - -// `mma____xx`; empty for a type that is not an MMA atom. -std::string mmaAliasName(Type atom) { - auto name = [](auto t) { - std::string shape; - auto mnk = cutegen::flatten(t.getShapeMnk().getRef()); - for (size_t i = 0; i < cutegen::rank(mnk); ++i) - shape += (i ? "x" : "") + cutegen::to_string(cutegen::get(mnk, i)); - return "mma_" + printed(t.getAType()) + "_" + printed(t.getBType()) + "_" + - printed(t.getCType()) + "_" + shape; - }; - return llvm::TypeSwitch(atom) - .Case(name) - .Default([](Type) { return std::string(); }); -} - -// The compiler's type aliases: `!memref__` for a memref, -// `!copy_` for a tiled copy and `!mma____xx` for a -// tiled MMA (MLIR escapes and numbers them). Other types have none. -struct CuteOpAsmInterface : public OpAsmDialectInterface { - using OpAsmDialectInterface::OpAsmDialectInterface; - AliasResult getAlias(Type type, raw_ostream &os) const override { - std::string name; - if (auto memref = llvm::dyn_cast(type)) - name = "memref_" + - stringifyAddressSpace(memref.getPtr().getAddressSpace()).str() + - "_" + printed(memref.getPtr().getValueType()); - else if (auto copy = llvm::dyn_cast(type)) - name = copyAliasName(copy.getCopyAtom()); - else if (auto mma = llvm::dyn_cast(type)) - name = mmaAliasName(mma.getMmaAtom()); - if (name.empty()) - return AliasResult::NoAlias; - os << name; - return AliasResult::OverridableAlias; - } -}; -} // namespace - -void CuteDialect::registerCuteTypes(CuteDialect *dialect) { - dialect->addTypes< -#define GET_TYPEDEF_LIST -#include "mlir/dialects/cute_ir/CuteTypes.cpp.inc" - >(); - dialect->addInterfaces(); -} - -//===----------------------------------------------------------------------===// -// IntTupleType — custom assembly format -// - -// Grammar: < " cutegen-int-tuple-string " > (read leniently, see above) -//===----------------------------------------------------------------------===// - -mlir::Type IntTupleType::parse(mlir::AsmParser &parser) { - return get( - parser.getContext(), - IntTupleAttr::get(parser.getContext(), - parseLenientPayload(parser))); -} - -void IntTupleType::print(mlir::AsmPrinter &printer) const { - printer << '<'; - printer.printString(cutegen::to_string(getRef())); - printer << '>'; -} - -bool IntTupleType::isStatic() const { return cutegen::is_static(getRef()); } - -mlir::Attribute IntTupleType::getValueAttr() const { return getAttr(); } - -llvm::TypeSize -IntTupleType::getTypeSize(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); -} - -llvm::TypeSize -IntTupleType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); -} - -uint64_t -IntTupleType::getABIAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); -} - -uint64_t -IntTupleType::getPreferredAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypePreferredAlignment( - cg::get_llvm_type(builder, getRef())); -} - -std::optional -IntTupleType::getIndexBitwidth(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); -} - -bool IntTupleType::areCompatible( - mlir::DataLayoutEntryListRef oldLayout, - mlir::DataLayoutEntryListRef newLayout, - mlir::DataLayoutSpecInterface newSpec, - const mlir::DataLayoutIdentifiedEntryMap &identified) const { - OpBuilder builder(getContext()); - // The lowering can be a bare integer (scalar/depth-0 dynamic leaves), which - // does not implement DataLayoutTypeInterface; only delegate when it does and - // otherwise fall back to the interface default ("compatible"). - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); - } - return true; -} - -llvm::LogicalResult -IntTupleType::verifyEntries(mlir::DataLayoutEntryListRef entries, - mlir::Location loc) const { - OpBuilder builder(getContext()); - // See areCompatible: bare-integer lowerings cannot be queried, so only - // delegate when the lowered type implements the interface. - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.verifyEntries(entries, loc); - } - return mlir::success(); -} - -//===----------------------------------------------------------------------===// -// CoordType — custom assembly format -//===----------------------------------------------------------------------===// - -mlir::Type CoordType::parse(mlir::AsmParser &parser) { - return get(parser.getContext(), - CoordAttr::get(parser.getContext(), - parseLenientPayload(parser))); -} - -void CoordType::print(mlir::AsmPrinter &printer) const { - printer << '<'; - printer.printString(cutegen::to_string(getRef())); - printer << '>'; -} - -bool CoordType::isStatic() const { return cutegen::is_static(getRef()); } - -mlir::Attribute CoordType::getValueAttr() const { return getAttr(); } - -llvm::TypeSize -CoordType::getTypeSize(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); -} - -llvm::TypeSize -CoordType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); -} - -uint64_t CoordType::getABIAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); -} - -uint64_t -CoordType::getPreferredAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypePreferredAlignment( - cg::get_llvm_type(builder, getRef())); -} - -std::optional -CoordType::getIndexBitwidth(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); -} - -bool CoordType::areCompatible( - mlir::DataLayoutEntryListRef oldLayout, - mlir::DataLayoutEntryListRef newLayout, - mlir::DataLayoutSpecInterface newSpec, - const mlir::DataLayoutIdentifiedEntryMap &identified) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); - } - return true; -} - -llvm::LogicalResult -CoordType::verifyEntries(mlir::DataLayoutEntryListRef entries, - mlir::Location loc) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.verifyEntries(entries, loc); - } - return mlir::success(); -} - -//===----------------------------------------------------------------------===// -// ShapeType — custom assembly format -//===----------------------------------------------------------------------===// - -mlir::Type ShapeType::parse(mlir::AsmParser &parser) { - return get(parser.getContext(), - ShapeAttr::get(parser.getContext(), - parseLenientPayload(parser))); -} - -void ShapeType::print(mlir::AsmPrinter &printer) const { - printer << '<'; - printer.printString(cutegen::to_string(getRef())); - printer << '>'; -} - -bool ShapeType::isStatic() const { return cutegen::is_static(getRef()); } - -mlir::Attribute ShapeType::getValueAttr() const { return getAttr(); } - -llvm::TypeSize -ShapeType::getTypeSize(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); -} - -llvm::TypeSize -ShapeType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); -} - -uint64_t ShapeType::getABIAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); -} - -uint64_t -ShapeType::getPreferredAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypePreferredAlignment( - cg::get_llvm_type(builder, getRef())); -} - -std::optional -ShapeType::getIndexBitwidth(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); -} - -bool ShapeType::areCompatible( - mlir::DataLayoutEntryListRef oldLayout, - mlir::DataLayoutEntryListRef newLayout, - mlir::DataLayoutSpecInterface newSpec, - const mlir::DataLayoutIdentifiedEntryMap &identified) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); - } - return true; -} - -llvm::LogicalResult -ShapeType::verifyEntries(mlir::DataLayoutEntryListRef entries, - mlir::Location loc) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.verifyEntries(entries, loc); - } - return mlir::success(); -} - -//===----------------------------------------------------------------------===// -// StrideType — custom assembly format -//===----------------------------------------------------------------------===// - -mlir::Type StrideType::parse(mlir::AsmParser &parser) { - auto loc = parser.getCurrentLocation(); - auto attr = StrideAttr::get(parser.getContext(), - parseLenientPayload(parser)); - return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), - attr); -} - -void StrideType::print(mlir::AsmPrinter &printer) const { - printer << '<'; - printer.printString(cutegen::to_string(getRef())); - printer << '>'; -} - -bool StrideType::isStatic() const { return cutegen::is_static(getRef()); } - -mlir::Attribute StrideType::getValueAttr() const { return getAttr(); } - -// The compiler accepts every stride cutegen reads, mixed integer and -// scaled-basis leaves and mixed-depth basis paths included. -mlir::LogicalResult -StrideType::verify(llvm::function_ref emitError, - StrideAttr attr) { - return mlir::success(); -} - -llvm::TypeSize -StrideType::getTypeSize(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); -} - -llvm::TypeSize -StrideType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); -} - -uint64_t -StrideType::getABIAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); -} - -uint64_t -StrideType::getPreferredAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypePreferredAlignment( - cg::get_llvm_type(builder, getRef())); -} - -std::optional -StrideType::getIndexBitwidth(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); -} - -bool StrideType::areCompatible( - mlir::DataLayoutEntryListRef oldLayout, - mlir::DataLayoutEntryListRef newLayout, - mlir::DataLayoutSpecInterface newSpec, - const mlir::DataLayoutIdentifiedEntryMap &identified) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); - } - return true; -} - -llvm::LogicalResult -StrideType::verifyEntries(mlir::DataLayoutEntryListRef entries, - mlir::Location loc) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.verifyEntries(entries, loc); - } - return mlir::success(); -} - -//===----------------------------------------------------------------------===// -// LayoutType — custom assembly format -//===----------------------------------------------------------------------===// - -mlir::Type LayoutType::parse(mlir::AsmParser &parser) { - auto loc = parser.getCurrentLocation(); - auto attr = LayoutAttr::get(parser.getContext(), - parseLenientPayload(parser)); - return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), - attr); -} - -void LayoutType::print(mlir::AsmPrinter &printer) const { - printer << '<'; - printer.printString(cutegen::to_string(getRef())); - printer << '>'; -} - -bool LayoutType::isStatic() const { return cutegen::is_static(getRef()); } - -mlir::Attribute LayoutType::getValueAttr() const { return getAttr(); } - -/// Verify LayoutType storage invariants. cutegen leaves these -/// unchecked at construction time for performance; enforce them at the -/// IR boundary. -mlir::LogicalResult -LayoutType::verify(llvm::function_ref emitError, - LayoutAttr attr) { - auto const &layout = attr.getRef(); - if (cutegen::has_error(layout)) { - return emitError() << "expects no error(`x`) in layout, but got \"" - << cutegen::to_string(layout) << "\""; - } - if (cutegen::any_leaf_is(layout.shape(), [](auto const &e) { - return cutegen::holds_int(e) && cutegen::get_int(e) <= 0; - })) { - return emitError() << "expects positive shape mode, but got \"" - << cutegen::to_string(layout) << "\""; - } - if (!cutegen::is_congruent(layout.shape(), layout.stride())) { - return emitError() - << "expects shape and stride profile to match, but got \"" - << cutegen::to_string(layout) << "\""; - } - return mlir::success(); -} - -llvm::TypeSize -LayoutType::getTypeSize(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); -} - -llvm::TypeSize -LayoutType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); -} - -uint64_t -LayoutType::getABIAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); -} - -uint64_t -LayoutType::getPreferredAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypePreferredAlignment( - cg::get_llvm_type(builder, getRef())); -} - -std::optional -LayoutType::getIndexBitwidth(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); -} - -bool LayoutType::areCompatible( - mlir::DataLayoutEntryListRef oldLayout, - mlir::DataLayoutEntryListRef newLayout, - mlir::DataLayoutSpecInterface newSpec, - const mlir::DataLayoutIdentifiedEntryMap &identified) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); - } - return true; -} - -llvm::LogicalResult -LayoutType::verifyEntries(mlir::DataLayoutEntryListRef entries, - mlir::Location loc) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.verifyEntries(entries, loc); - } - return mlir::success(); -} - -//===----------------------------------------------------------------------===// -// TileType — custom assembly format -//===----------------------------------------------------------------------===// - -mlir::Type TileType::parse(mlir::AsmParser &parser) { - auto loc = parser.getCurrentLocation(); - auto attr = TileAttr::get(parser.getContext(), - parseLenientPayload(parser)); - return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), - attr); -} - -void TileType::print(mlir::AsmPrinter &printer) const { - printer << '<'; - printer.printString(cutegen::to_string(getRef())); - printer << '>'; -} - -bool TileType::isStatic() const { return cutegen::is_static(getRef()); } - -mlir::Attribute TileType::getValueAttr() const { return getAttr(); } - -// Like strides, the layouts of a tile are not restricted further. -mlir::LogicalResult -TileType::verify(llvm::function_ref emitError, - TileAttr attr) { - return mlir::success(); -} - -llvm::TypeSize -TileType::getTypeSize(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); -} - -llvm::TypeSize -TileType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); -} - -uint64_t TileType::getABIAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); -} - -uint64_t -TileType::getPreferredAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypePreferredAlignment( - cg::get_llvm_type(builder, getRef())); -} - -std::optional -TileType::getIndexBitwidth(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); -} - -bool TileType::areCompatible( - mlir::DataLayoutEntryListRef oldLayout, - mlir::DataLayoutEntryListRef newLayout, - mlir::DataLayoutSpecInterface newSpec, - const mlir::DataLayoutIdentifiedEntryMap &identified) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); - } - return true; -} - -llvm::LogicalResult -TileType::verifyEntries(mlir::DataLayoutEntryListRef entries, - mlir::Location loc) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.verifyEntries(entries, loc); - } - return mlir::success(); -} - -//===----------------------------------------------------------------------===// -// ComposedLayoutType — custom assembly format -//===----------------------------------------------------------------------===// - -mlir::Type ComposedLayoutType::parse(mlir::AsmParser &parser) { - // Capture the start location so verify() failures (positivity, - // congruence) are attributed to the type literal. - auto loc = parser.getCurrentLocation(); - if (parser.parseLess()) { - return {}; - } - std::string str; - if (parser.parseString(&str)) { - return {}; - } - auto opt = cutegen::from_string(str); - if (!opt) { - parser.emitError(parser.getCurrentLocation(), - "failed to parse composed_layout from \"" + str + "\""); - return {}; - } - if (parser.parseGreater()) { - return {}; - } - auto attr = ComposedLayoutAttr::get(parser.getContext(), std::move(*opt)); - return ComposedLayoutType::getChecked([&] { return parser.emitError(loc); }, - parser.getContext(), attr); -} - -void ComposedLayoutType::print(mlir::AsmPrinter &printer) const { - printer << '<'; - printer.printString(cutegen::to_string(getRef())); - printer << '>'; -} - -bool ComposedLayoutType::isStatic() const { - // No free cutegen::is_static(composed_layout_t) exists — check each part. - const auto &cl = getRef(); - bool b_ok = cutegen::is_static(cl.layout_b()); - bool off_ok = cutegen::is_static(cl.offset()); - bool a_ok = cl.is_a_swizzle() || cutegen::is_static(cl.layout_a()); - return b_ok && off_ok && a_ok; -} - -mlir::Attribute ComposedLayoutType::getValueAttr() const { return getAttr(); } - -/// Verify ComposedLayoutType storage invariants: no error sentinel, -/// positive static shape leaves, addable offset, and a sum that's -/// composable into the inner layout (scalar for swizzle, scalar or -/// weakly congruent for affine). -mlir::LogicalResult ComposedLayoutType::verify( - llvm::function_ref emitError, - ComposedLayoutAttr attr) { - auto const &composed = attr.getRef(); - if (cutegen::has_error(composed)) { - return emitError() << "expects no error(`x`) in layout, but got \"" - << cutegen::to_string(composed) << "\""; - } - // A trivial wrap (identity swizzle, no offset) is accepted and prints as - // its plain layout, as the compiler does. - auto hasNonPositiveShape = [](cutegen::layout const &lyt) { - return cutegen::any_leaf_is(lyt.shape(), [](auto const &e) { - return cutegen::holds_int(e) && cutegen::get_int(e) <= 0; - }); - }; - if (hasNonPositiveShape(composed.layout_b()) || - (composed.is_a_affine() && hasNonPositiveShape(composed.layout_a()))) { - return emitError() << "expects positive shape mode, but got \"" - << cutegen::to_string(composed) << "\""; - } - - // Addable: the offset and layout_eval(0, outer) must be compatible - // under arith-tuple addition (no scaled-basis-vs-integer mismatch - // or rank disagreement). - auto offset_b = cutegen::layout_eval(0, composed.layout_b()); - auto sum = - cutegen::arith_tuple_sum(composed.offset(), offset_b); - if (cutegen::has_error(sum)) { - return emitError() << "expects offset and `layout_eval(0, outer)` to be " - "addable, but got " - << cutegen::to_string(composed.offset()) << " and " - << cutegen::to_string(offset_b); - } - - // Swizzle inner: the arith-tuple sum must be a scalar (int or - // dynamic int) — swizzles can only consume scalar indices, so any - // scaled-basis structure surviving from the offset or outer is a - // composability error. - if (composed.is_a_swizzle() && !cutegen::holds_int_or_dynamic_int(sum)) { - return emitError() << "swizzle layout expects scalar `offset` and " - "`outer` without scaled basis, but got " - << cutegen::to_string(composed.offset()) << " and " - << cutegen::to_string(composed.layout_b()); - } - - // Affine inner: the sum either is a scalar (and layout_a indexes - // it directly) or matches layout_a's shape profile so each leaf - // dispatches to the corresponding mode of A. - if (composed.is_a_affine() && !cutegen::holds_int_or_dynamic_int(sum) && - !cutegen::weakly_congruent(sum, composed.layout_a().shape())) { - return emitError() - << "expects arith sum of `layout_eval(0, outer)` and `offset` to be " - "scalar or weakly congruent to inner shape, but got sum " - << cutegen::to_string(sum) << " and inner shape " - << cutegen::to_string(composed.layout_a().shape()); - } - - return mlir::success(); -} - -llvm::TypeSize -ComposedLayoutType::getTypeSize(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSize(cg::get_llvm_type(builder, getRef())); -} - -llvm::TypeSize ComposedLayoutType::getTypeSizeInBits( - const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeSizeInBits(cg::get_llvm_type(builder, getRef())); -} - -uint64_t -ComposedLayoutType::getABIAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeABIAlignment(cg::get_llvm_type(builder, getRef())); -} - -uint64_t ComposedLayoutType::getPreferredAlignment( - const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypePreferredAlignment( - cg::get_llvm_type(builder, getRef())); -} - -std::optional ComposedLayoutType::getIndexBitwidth( - const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - OpBuilder builder(getContext()); - return dataLayout.getTypeIndexBitwidth(cg::get_llvm_type(builder, getRef())); -} - -bool ComposedLayoutType::areCompatible( - mlir::DataLayoutEntryListRef oldLayout, - mlir::DataLayoutEntryListRef newLayout, - mlir::DataLayoutSpecInterface newSpec, - const mlir::DataLayoutIdentifiedEntryMap &identified) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); - } - return true; -} - -llvm::LogicalResult -ComposedLayoutType::verifyEntries(mlir::DataLayoutEntryListRef entries, - mlir::Location loc) const { - OpBuilder builder(getContext()); - if (auto dlTy = llvm::dyn_cast( - cg::get_llvm_type(builder, getRef()))) { - return dlTy.verifyEntries(entries, loc); - } - return mlir::success(); -} - -//===----------------------------------------------------------------------===// -// SwizzleType — custom assembly format -//===----------------------------------------------------------------------===// - -mlir::Type SwizzleType::parse(mlir::AsmParser &parser) { - auto loc = parser.getCurrentLocation(); - auto attr = SwizzleAttr::get(parser.getContext(), - parseLenientPayload(parser)); - return getChecked([&] { return parser.emitError(loc); }, parser.getContext(), - attr); -} - -void SwizzleType::print(mlir::AsmPrinter &printer) const { - printer << '<'; - printer.printString(cutegen::to_string(getRef())); - printer << '>'; -} - -bool SwizzleType::isStatic() const { return true; } - -mlir::Attribute SwizzleType::getValueAttr() const { return getAttr(); } - -/// Verify SwizzleType storage invariants. Defense-in-depth — the -/// string parser already rejects invalid swizzle parameters via -/// cutegen::from_string, but programmatic -/// construction goes around the parser, so re-check here. -/// -/// cutegen::swizzle::is_valid (swizzle.hpp) enforces two -/// invariants on S: -/// 1. |num_shift| + num_bits + num_base <= 32 — parameters -/// must fit within the swizzle's 32-bit working width. -/// 2. |num_shift| >= num_bits — the shift amount must cover -/// the mask width so the Y and Z masks do not overlap. -mlir::LogicalResult -SwizzleType::verify(llvm::function_ref emitError, - SwizzleAttr attr) { - auto const &sw = attr.getRef(); - uint32_t numBits = sw.num_bits(); - uint32_t numBase = sw.num_base(); - int32_t numShift = sw.num_shift(); - uint32_t absShift = static_cast(std::abs(numShift)); - constexpr uint32_t kBitWidth = 32; - - if (absShift + numBits + numBase > kBitWidth) { - return emitError() << "expects |num_shift| + num_bits + num_base <= " - << kBitWidth << ", but got " << absShift << " + " - << numBits << " + " << numBase << " = " - << (absShift + numBits + numBase) << " for swizzle " - << cutegen::to_string(sw); - } - if (absShift < numBits) { - return emitError() << "expects |num_shift| >= num_bits, but got " - << absShift << " < " << numBits << " for swizzle " - << cutegen::to_string(sw); - } - return mlir::success(); -} - -// A swizzle is a compile-time-only object whose runtime representation is an -// empty LLVM struct (mirroring convertSwizzleType in CuteTypeConverter.cpp), so -// the data-layout queries delegate to that struct rather than to -// cg::get_llvm_type. -static mlir::Type getSwizzleLLVMType(mlir::MLIRContext *ctx) { - return LLVM::LLVMStructType::getLiteral(ctx, SmallVector{}); -} - -llvm::TypeSize -SwizzleType::getTypeSize(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - return dataLayout.getTypeSize(getSwizzleLLVMType(getContext())); -} - -llvm::TypeSize -SwizzleType::getTypeSizeInBits(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - return dataLayout.getTypeSizeInBits(getSwizzleLLVMType(getContext())); -} - -uint64_t -SwizzleType::getABIAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - return dataLayout.getTypeABIAlignment(getSwizzleLLVMType(getContext())); -} - -uint64_t -SwizzleType::getPreferredAlignment(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - return dataLayout.getTypePreferredAlignment(getSwizzleLLVMType(getContext())); -} - -std::optional -SwizzleType::getIndexBitwidth(const mlir::DataLayout &dataLayout, - mlir::DataLayoutEntryListRef params) const { - return dataLayout.getTypeIndexBitwidth(getSwizzleLLVMType(getContext())); -} - -bool SwizzleType::areCompatible( - mlir::DataLayoutEntryListRef oldLayout, - mlir::DataLayoutEntryListRef newLayout, - mlir::DataLayoutSpecInterface newSpec, - const mlir::DataLayoutIdentifiedEntryMap &identified) const { - if (auto dlTy = llvm::dyn_cast( - getSwizzleLLVMType(getContext()))) { - return dlTy.areCompatible(oldLayout, newLayout, newSpec, identified); - } - return true; -} - -llvm::LogicalResult -SwizzleType::verifyEntries(mlir::DataLayoutEntryListRef entries, - mlir::Location loc) const { - if (auto dlTy = llvm::dyn_cast( - getSwizzleLLVMType(getContext()))) { - return dlTy.verifyEntries(entries, loc); - } - return mlir::success(); -} - -//===----------------------------------------------------------------------===// -// The DSL compiler's types and attributes (CuteTypesCompiler.td, -// CuteAttrsCompiler.td) -//===----------------------------------------------------------------------===// - -#include "mlir/dialects/cute_ir/CuteEnums.cpp.inc" - -static OptionalParseResult generatedCuteTypeParser(DialectAsmParser &parser, - StringRef *mnemonic, - Type &value) { - OptionalParseResult result = generatedCuteTypeParser( - static_cast(parser), mnemonic, value); - if (result.has_value()) - return result; - StringRef digits = *mnemonic; - unsigned width; - if (!digits.consume_front("i") || digits.empty() || - !llvm::all_of(digits, llvm::isDigit) || digits.getAsInteger(10, width)) - return std::nullopt; - value = ConstrainedIntType::parse(parser, width); - return success(!!value); -} - -static LogicalResult generatedCuteTypePrinter(Type type, - DialectAsmPrinter &printer) { - if (auto integer = llvm::dyn_cast(type)) { - integer.print(printer); - return success(); - } - return generatedCuteTypePrinter(type, static_cast(printer)); -} - -// Bits of an integer or float (tf32 is stored in 32), or of a vector of them. -static unsigned storageBits(Type type) { - if (auto integer = llvm::dyn_cast_or_null(type)) - return integer.getWidth(); - if (auto fp = llvm::dyn_cast_or_null(type)) - return fp.isTF32() ? 32 : fp.getWidth(); - if (auto vector = llvm::dyn_cast_or_null(type)) - return vector.getNumElements() * storageBits(vector.getElementType()); - return 0; -} - -static bool isIdentitySwizzle(SwizzleAttr swizzle) { - return !swizzle || cutegen::to_string(swizzle.getRef()) == "S<0,4,3>"; -} - -uint64_t PtrType::getNaturalAlignment(Type valueType) { - unsigned bits = 0; - if (auto sparse = llvm::dyn_cast_or_null(valueType)) - bits = storageBits(sparse.getPhysicalType()); - else if (llvm::isa_and_nonnull(valueType)) - bits = storageBits(valueType); - if (bits <= 1) - return 1; - return bits % 8 ? 0 : bits / 8; -} - -BitLayoutAttr PtrType::getDefaultBitLayout(Type valueType, - AddressSpace addressSpace) { - unsigned chunk = addressSpace == AddressSpace::tmem ? 32 : 8; - std::string text; - if (auto sparse = llvm::dyn_cast_or_null(valueType)) { - unsigned bits = storageBits(sparse.getPhysicalType()); - if (!bits) - return {}; - text = llvm::formatv("({0},({1},{2})):(1,(0,{0}))", bits, - sparse.getNumLogical(), chunk / std::gcd(bits, chunk)) - .str(); - } else if (llvm::isa_and_nonnull(valueType)) { - unsigned bits = storageBits(valueType); - unsigned stride = bits == 1 ? 8 : bits; - text = llvm::formatv("({0},{1}):(1,{2})", bits, - chunk / std::gcd(stride, chunk), stride) - .str(); - } else { - return {}; - } - auto layout = cutegen::from_string(text); - if (!layout) - return {}; - MLIRContext *ctx = valueType.getContext(); - return BitLayoutAttr::get(ctx, LayoutAttr::get(ctx, std::move(*layout))); -} - -// The bits of a chunk: the largest extent times stride of the element mode. -static std::optional chunkBits(const cutegen::layout &layout) { - auto elements = cutegen::get(layout, 1); - auto shape = cutegen::flatten(elements.shape()); - auto stride = cutegen::flatten(elements.stride()); - if (cutegen::holds_int(shape) && cutegen::holds_int(stride)) - return shape.as_int64() * stride.as_int64(); - int64_t bits = 0; - for (size_t i = 0; i < cutegen::rank(shape); ++i) { - auto extent = cutegen::get(shape, i); - auto step = cutegen::get(stride, i); - if (!cutegen::holds_int(extent) || !cutegen::holds_int(step)) - return std::nullopt; - bits = std::max(bits, extent.as_int64() * step.as_int64()); - } - return bits; -} - -LogicalResult -BitLayoutAttr::verify(function_ref emitError, - LayoutAttr layout) { - const cutegen::layout &l = layout.getRef(); - if (!cutegen::is_static(l)) - return emitError() << "bitlayout must be fully static but got " << layout; - if (cutegen::rank(l) != 2) - return emitError() << "bitlayout must have 2 modes corresponding to " - "(element bit, element index), but got " - << cutegen::rank(l); - auto elements = cutegen::size(l, 1); - if (cutegen::holds_int(elements) && elements.as_int64() == 0) - return emitError() - << "bitlayout must contain at least one logical element, but got 0"; - return success(); -} - -//===- ptr and memref -----------------------------------------------------===// - -namespace { -struct PtrFields { - Type valueType; - AddressSpace space = AddressSpace::generic; - std::optional alignment; - SwizzleAttr swizzle; - BitLayoutAttr bitlayout; -}; -} // namespace - -// `bitlayout<"layout">`, the keyword already read. -static BitLayoutAttr parseBitLayout(AsmParser &parser) { - std::string text; - if (failed(parser.parseLess()) || failed(parser.parseString(&text)) || - failed(parser.parseGreater())) - return {}; - auto layout = cutegen::from_string(text); - if (!layout) { - parser.emitError(parser.getCurrentLocation(), "expects cg::layout"); - return {}; - } - MLIRContext *ctx = parser.getContext(); - return BitLayoutAttr::getChecked( - [&] { return parser.emitError(parser.getCurrentLocation()); }, ctx, - LayoutAttr::get(ctx, std::move(*layout))); -} - -// `S`, the `S` already read. -static SwizzleAttr parseInlineSwizzle(AsmParser &parser) { - int64_t bits, base, shift; - if (failed(parser.parseLess()) || failed(parser.parseInteger(bits)) || - failed(parser.parseComma()) || failed(parser.parseInteger(base)) || - failed(parser.parseComma()) || failed(parser.parseInteger(shift)) || - failed(parser.parseGreater())) - return {}; - auto value = cutegen::from_string( - llvm::formatv("S<{0},{1},{2}>", bits, base, shift).str()); - if (!value) { - parser.emitError(parser.getCurrentLocation(), "invalid swizzle parameters"); - return {}; - } - return SwizzleAttr::get(parser.getContext(), std::move(*value)); -} - -// A layout or composed layout spelled as a string; a composed layout that is -// a plain one (identity swizzle, no offset) is read as that layout. -static Type parseLayoutPayload(AsmParser &parser) { - std::string text; - llvm::SMLoc loc = parser.getCurrentLocation(); - if (failed(parser.parseString(&text))) - return {}; - MLIRContext *ctx = parser.getContext(); - auto emitError = [&] { return parser.emitError(loc); }; - if (auto value = cutegen::from_string(text)) - return LayoutType::getChecked(emitError, ctx, - LayoutAttr::get(ctx, std::move(*value))); - if (auto value = cutegen::from_string(text)) { - if (value->is_normal_layout()) - return LayoutType::getChecked(emitError, ctx, - LayoutAttr::get(ctx, value->layout_b())); - return ComposedLayoutType::getChecked( - emitError, ctx, ComposedLayoutAttr::get(ctx, std::move(*value))); - } - parser.emitError(parser.getCurrentLocation(), "layout parse failure"); - return {}; -} - -static void printLayoutPayload(AsmPrinter &printer, Type layout) { - if (auto plain = llvm::dyn_cast(layout)) - printer.printString(cutegen::to_string(plain.getRef())); - else - printer.printString( - cutegen::to_string(llvm::cast(layout).getRef())); -} - -static LogicalResult memrefLayoutExpected(AsmParser &parser) { - return parser.emitError(parser.getCurrentLocation(), - "failed to parse CuteMemRefType parameter `,` " - "expected, layout cannot be omitted."); -} - -// `[type, ][space][, align][, S][, bitlayout<"L">]` in this order; -// a memref requires the type and ends with its layout string instead of the -// bit layout, which follows the layout. -static FailureOr parsePtrFields(AsmParser &parser, Type *layout) { - PtrFields fields; - OptionalParseResult typed = parser.parseOptionalType(fields.valueType); - if (typed.has_value() && failed(*typed)) - return failure(); - if (layout && !typed.has_value()) - return memrefLayoutExpected(parser); - enum { kSpace, kAlign, kSwizzle, kBitLayout, kDone } next = kSpace; - bool first = !typed.has_value(); - while (true) { - if (!first && failed(parser.parseOptionalComma())) - break; - StringRef keyword; - if (failed(parser.parseOptionalKeyword(&keyword))) { - if (layout) { - if (!(*layout = parseLayoutPayload(parser))) - return failure(); - return fields; - } - if (first) - break; - parser.emitError(parser.getCurrentLocation(), - "expected a pointer field after `,`"); - return failure(); - } - first = false; - if (auto space = symbolizeAddressSpace(keyword); space && next <= kSpace) { - fields.space = *space; - next = kAlign; - } else if (keyword == "align" && next <= kAlign) { - uint64_t alignment; - if (failed(parser.parseLess()) || - failed(parser.parseInteger(alignment)) || - failed(parser.parseGreater())) - return failure(); - fields.alignment = alignment; - next = kSwizzle; - } else if (keyword == "S" && next <= kSwizzle) { - if (!(fields.swizzle = parseInlineSwizzle(parser))) - return failure(); - next = kBitLayout; - } else if (keyword == "bitlayout" && !layout && next <= kBitLayout) { - if (!(fields.bitlayout = parseBitLayout(parser))) - return failure(); - next = kDone; - } else { - parser.emitError(parser.getCurrentLocation(), - "unexpected pointer field `") - << keyword << "`"; - return failure(); - } - } - if (layout) - return memrefLayoutExpected(parser); - return fields; -} - -// Defaults are left out: they are what the parser fills in. -static PtrType buildPtr(AsmParser &parser, const PtrFields &fields) { - SwizzleAttr swizzle = - isIdentitySwizzle(fields.swizzle) ? SwizzleAttr() : fields.swizzle; - BitLayoutAttr bitlayout = fields.bitlayout; - if (bitlayout && - bitlayout == PtrType::getDefaultBitLayout(fields.valueType, fields.space)) - bitlayout = {}; - return PtrType::getChecked( - [&] { return parser.emitError(parser.getCurrentLocation()); }, - parser.getContext(), fields.valueType, fields.space, - fields.alignment.value_or(PtrType::getNaturalAlignment(fields.valueType)), - swizzle, bitlayout); -} - -static void printPtrFields(AsmPrinter &printer, PtrType ptr) { - if (ptr.getValueType()) { - printer.printType(ptr.getValueType()); - printer << ", "; - } - printer << stringifyAddressSpace(ptr.getAddressSpace()); - if (ptr.getAlignment() != PtrType::getNaturalAlignment(ptr.getValueType())) - printer << ", align<" << ptr.getAlignment() << '>'; - if (!isIdentitySwizzle(ptr.getSwizzle())) - printer << ", " << cutegen::to_string(ptr.getSwizzle().getRef()); -} - -static void printBitLayout(AsmPrinter &printer, PtrType ptr) { - BitLayoutAttr bitlayout = ptr.getBitlayout(); - if (!bitlayout || bitlayout == PtrType::getDefaultBitLayout( - ptr.getValueType(), ptr.getAddressSpace())) - return; - printer << ", bitlayout<"; - printer.printString(cutegen::to_string(bitlayout.getLayout().getRef())); - printer << '>'; -} - -Type PtrType::parse(AsmParser &parser) { - if (failed(parser.parseLess())) - return {}; - FailureOr fields = parsePtrFields(parser, nullptr); - if (failed(fields) || failed(parser.parseGreater())) - return {}; - return buildPtr(parser, *fields); -} - -void PtrType::print(AsmPrinter &printer) const { - printer << '<'; - printPtrFields(printer, *this); - printBitLayout(printer, *this); - printer << '>'; -} - -LogicalResult PtrType::verify(function_ref emitError, - Type valueType, AddressSpace addressSpace, - uint64_t alignment, SwizzleAttr swizzle, - BitLayoutAttr bitlayout) { - if (alignment == 0 && getNaturalAlignment(valueType) != 0) - return emitError() << "expects alignment >= 1, got 0"; - bool tmem = addressSpace == AddressSpace::tmem; - if (tmem && swizzle && swizzle.getRef().num_bits() != 0) - return emitError() << "tmem pointer does not support non-identity swizzle"; - BitLayoutAttr defaultBitLayout = getDefaultBitLayout(valueType, addressSpace); - BitLayoutAttr effective = bitlayout ? bitlayout : defaultBitLayout; - if (!effective) - return success(); - if (auto bits = chunkBits(effective.getLayout().getRef())) { - if (*bits % 8 != 0) - return emitError() << "bitlayout chunk must be byte aligned, but size is " - << *bits << " bits"; - if (tmem && (*bits < 32 || !llvm::isPowerOf2_64(*bits))) - return emitError() - << "tmem pointer only supports 32 bit chunks that is a power of " - "2, got " - << *bits; - } - if (llvm::isa_and_nonnull(valueType) && - effective != defaultBitLayout) - return emitError() << "expects bitlayout to be " << defaultBitLayout - << " for valueType '" << valueType << "'but got " - << effective; - return success(); -} - -// Whether the offsets of `layout` are whole aligned blocks of `sparsity` -// elements: the contiguous run from offset 0 (stride-1 mode, then each mode -// whose stride is the run so far) is a multiple of it, and so is every other -// stride. -static bool isDenseForSparsity(const cutegen::layout &layout, - int64_t sparsity) { - if (!cutegen::is_static(layout)) - return false; - auto shape = cutegen::flatten(layout.shape()); - auto stride = cutegen::flatten(layout.stride()); - std::vector> modes; // (stride, extent) - auto add = [&](const auto &extent, const auto &step) { - if (extent.as_int64() != 1 && step.as_int64() != 0) - modes.emplace_back(step.as_int64(), extent.as_int64()); - }; - if (cutegen::holds_int(shape)) { - add(shape, stride); - } else { - for (size_t i = 0; i < cutegen::rank(shape); ++i) - add(cutegen::get(shape, i), cutegen::get(stride, i)); - } - llvm::sort(modes, [](auto a, auto b) { - return std::abs(a.first) < std::abs(b.first); - }); - int64_t run = 1; - bool contiguous = true; - for (auto [step, extent] : modes) { - if (contiguous && step == run) { - run *= extent; - continue; - } - contiguous = false; - if (step % sparsity != 0) - return false; - } - return run % sparsity == 0; -} - -Type mlir::cutlass_compiler::cute::MemRefType::parse(AsmParser &parser) { - if (failed(parser.parseLess())) - return {}; - Type layout; - FailureOr fields = parsePtrFields(parser, &layout); - if (failed(fields)) - return {}; - if (succeeded(parser.parseOptionalComma())) { - if (failed(parser.parseKeyword("bitlayout")) || - !(fields->bitlayout = parseBitLayout(parser))) - return {}; - } - if (failed(parser.parseGreater())) - return {}; - PtrType ptr = buildPtr(parser, *fields); - if (!ptr) - return {}; - return getChecked( - [&] { return parser.emitError(parser.getCurrentLocation()); }, - parser.getContext(), ptr, layout); -} - -void mlir::cutlass_compiler::cute::MemRefType::print( - AsmPrinter &printer) const { - printer << '<'; - printPtrFields(printer, getPtr()); - printer << ", "; - printLayoutPayload(printer, getLayout()); - printBitLayout(printer, getPtr()); - printer << '>'; -} - -LogicalResult mlir::cutlass_compiler::cute::MemRefType::verify( - function_ref emitError, PtrType ptr, Type layout) { - if (!ptr) - return emitError() << "expects a CuTe pointer"; - if (!llvm::isa(layout)) - return emitError() << "expects a CuTe layout or composed layout"; - Type valueType = ptr.getValueType(); - if (!valueType || !(valueType.isIntOrIndexOrFloat() || - llvm::isa(valueType))) - return emitError() << "expects value type to be int, float, ptr or " - "sparse_elem, but got '" - << valueType << "'"; - if (auto sparse = llvm::dyn_cast(valueType)) { - int64_t sparsity = sparse.getNumLogical(); - bool dense; - if (auto composed = llvm::dyn_cast(layout)) { - // A swizzle may not move elements within a block. - const auto &c = composed.getRef(); - dense = c.is_a_swizzle() && - (int64_t(1) << c.swizzle_a().num_base()) % sparsity == 0 && - cutegen::holds_int(c.offset()) && - c.offset().as_int64() % sparsity == 0 && - isDenseForSparsity(c.layout_b(), sparsity); - } else { - dense = - isDenseForSparsity(llvm::cast(layout).getRef(), sparsity); - } - if (!dense) - return emitError() - << "layout must be dense but cannot be divided by sparsity"; - } - if (auto composed = llvm::dyn_cast(layout)) { - if (!isIdentitySwizzle(ptr.getSwizzle()) && - composed.getRef().is_a_swizzle() && - !composed.getRef().swizzle_a().is_identity()) - return emitError() << "cannot have both a swizzle on the pointer and a " - "non-identity swizzle in the layout"; - } else if (cutegen::has_scaled_basis( - llvm::cast(layout).getRef().stride())) { - return emitError() << "expected iterator and layout codomain to have " - "congruent profiles, but got iterator '" - << ptr << "' and layout layout: " - << cutegen::to_string( - llvm::cast(layout).getRef()); - } - return success(); -} - -//===- arith_tuple_iter, coord_tensor, i, fast_divmod_divisor, ... -----===// - -Type ArithTupleIteratorType::parse(AsmParser &parser) { - auto tuple = IntTupleType::parse(parser); - return tuple ? get(parser.getContext(), llvm::cast(tuple)) - : Type(); -} - -void ArithTupleIteratorType::print(AsmPrinter &printer) const { - getArithTuple().print(printer); -} - -Type CoordTensorType::parse(AsmParser &parser) { - std::string text; - if (failed(parser.parseLess()) || failed(parser.parseString(&text))) - return {}; - auto tuple = cutegen::from_string(text); - if (!tuple) { - parser.emitError(parser.getCurrentLocation(), - "failed to parse CoordTensorType parameter 'arith_tuple' " - "which is to be a `cutegen::int_tuple`"); - return {}; - } - Type layout; - if (failed(parser.parseComma()) || !(layout = parseLayoutPayload(parser)) || - failed(parser.parseGreater())) - return {}; - return getChecked( - [&] { return parser.emitError(parser.getCurrentLocation()); }, - parser.getContext(), - IntTupleType::get(parser.getContext(), std::move(*tuple)), layout); -} - -void CoordTensorType::print(AsmPrinter &printer) const { - printer << '<'; - printer.printString(cutegen::to_string(getArithTuple().getRef())); - printer << ", "; - printLayoutPayload(printer, getLayout()); - printer << '>'; -} - -LogicalResult -CoordTensorType::verify(function_ref emitError, - IntTupleType arithTuple, Type layout) { - if (!arithTuple) - return emitError() << "expects an arithmetic tuple"; - if (!llvm::isa(layout)) - return emitError() << "expects a CuTe layout or composed layout"; - return success(); -} - -// `<[pow2, ]divby D>`, or nothing for a plain integer. -Type ConstrainedIntType::parse(AsmParser &parser, unsigned width) { - int64_t divisibility = 1; - bool isPow2 = false; - if (succeeded(parser.parseOptionalLess())) { - isPow2 = succeeded(parser.parseOptionalKeyword("pow2")); - bool divided = !isPow2 || succeeded(parser.parseOptionalComma()); - if (divided && (failed(parser.parseKeyword("divby")) || - failed(parser.parseInteger(divisibility)))) - return {}; - if (failed(parser.parseGreater())) - return {}; - } - return get(parser.getContext(), divisibility, width, isPow2); -} - -void ConstrainedIntType::print(AsmPrinter &printer) const { - printer << 'i' << getWidth(); - if (!getIsPow2() && getDivisibility() == 1) - return; - printer << '<'; - if (getIsPow2()) - printer << "pow2"; - if (getDivisibility() != 1) - printer << (getIsPow2() ? ", " : "") << "divby " << getDivisibility(); - printer << '>'; -} - -Type FastDivmodDivisorType::parse(AsmParser &parser) { - unsigned width; - if (failed(parser.parseLess()) || failed(parser.parseInteger(width))) - return {}; - bool isPow2 = false; - if (succeeded(parser.parseOptionalComma())) { - if (failed(parser.parseKeyword("is_pow_2"))) - return {}; - isPow2 = true; - } - if (failed(parser.parseGreater())) - return {}; - return get(parser.getContext(), width, isPow2); -} - -void FastDivmodDivisorType::print(AsmPrinter &printer) const { - printer << '<' << getWidth(); - if (getIsPow_2()) - printer << ", is_pow_2"; - printer << '>'; -} - -LogicalResult -SparseElemType::verify(function_ref emitError, - int numLogical, Type physicalType) { - if (numLogical <= 0) - return emitError() << "sparse element type logical count must be > 0"; - Type element = physicalType; - if (auto vector = llvm::dyn_cast_or_null(physicalType)) - element = vector.getElementType(); - if (!llvm::isa_and_nonnull(element)) - return emitError() << "unsupported physical element type '" << physicalType - << "'"; - return success(); -} - -LogicalResult TiledMmaType::verify(function_ref emitError, - Type mmaAtom, LayoutAttr atomLayoutMNK, - TileAttr permutationMNK) { - if (cutegen::rank(atomLayoutMNK.getRef()) != 3) - return emitError() << "TiledMMA requires rank-3 atomLayoutMNK"; - if (permutationMNK && cutegen::rank(permutationMNK.getRef()) != 3) - return emitError() << "TiledMMA requires rank-3 permutationMNK"; - return success(); -} diff --git a/mlir/dialects/cute_ir/CuteTypesAll.td b/mlir/dialects/cute_ir/CuteTypesAll.td deleted file mode 100644 index 99766dcf..00000000 --- a/mlir/dialects/cute_ir/CuteTypesAll.td +++ /dev/null @@ -1,12 +0,0 @@ -//===- CuteTypesAll.td - all cute types --------------------*- tablegen -*-===// -// -// The dialect has one generated type parser and printer, so the release's -// types and the compiler's are generated together from here. - -#ifndef MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD -#define MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD - -include "cute_ir/Dialect/Cute/IR/CuteTypes.td" -include "mlir/dialects/cute_ir/CuteTypesCompiler.td" - -#endif // MLIR_DIALECTS_CUTE_IR_CUTETYPESALL_TD diff --git a/mlir/dialects/cute_ir/CuteTypesCompiler.td b/mlir/dialects/cute_ir/CuteTypesCompiler.td deleted file mode 100644 index 6ca5588a..00000000 --- a/mlir/dialects/cute_ir/CuteTypesCompiler.td +++ /dev/null @@ -1,269 +0,0 @@ -// The cute dialect's types beyond the release, generated from the -// nvidia-cutlass-dsl 4.8.0 release; do not edit. -// -// Sources: the compiler library (`_cutlass_ir`): its generated parser -// messages give the classes with a declarative syntax, their parameters in -// declaration order and their C++ types; the typed Python bindings of the -// types (`get` signatures and getters) give the parameters of the others; the -// live compiler, loaded from Python, gives the syntax (what it prints for what -// it parses, including every default it leaves out) and the verifier -// messages. -// -// Differences from the compiler, where its C++ types are not defined here: -// - The atoms of tiled_copy / tiled_copy_v2 / tiled_mma are -// `CopyAtomTypeInterface` / `CopyAtomTypeInterfaceV2` / -// `MmaAtomTypeInterface` in the compiler; plain types here. -// - ptr, memref and i have bespoke parsers in the compiler, so their -// parameter names and order are not recorded; the order here follows the -// Python `get` of each. -// - coord_tensor and memref layouts are `cutegen::layout` in the compiler's -// messages but accept composed layouts; they are a layout or composed -// layout type here. -// Verifier rules not reproduced: the bitlayout's cosize and element-count -// rules and TiledMMA's "requires static permutations of MNK" (neither -// triggered by any probe). The memref's sparse density rule ("layout must be -// dense but cannot be divided by sparsity") is inferred from probes: every -// probe agrees with it. - -#ifndef MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD -#define MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD - -include "cute_ir/Dialect/Cute/IR/CuteTypes.td" -include "mlir/dialects/cute_ir/CuteAttrsCompiler.td" -include "mlir/dialects/cute_ir/CuteEnums.td" - -def Cute_PtrType : Cute_Type<"Ptr", "ptr"> { - let summary = "Pointer with address space, alignment, swizzle and bit layout"; - let description = [{ - `<[value_type, ][space][, align][, S][, bitlayout<"L">]>`, the - fields in this order. Defaults are left out when printing and filled in - when parsing: - - - the address space is `generic` (always printed); - - the alignment is the value type's natural one: its width in bytes for - an integer or float that is a whole number of bytes, 1 for `i1`, 0 - (none) for other sub-byte widths, the physical type's bytes for a - `!cute.sparse_elem`, and 1 for any other type or no type; - - the swizzle is the identity `S<0,4,3>`; - - the bit layout is `(b,n):(1,s)` for an integer or float of `b` bits - (`s` = 8 for `i1`, else `b`) with `n` the least count making a chunk of - whole bytes (32-bit chunks in `tmem`), and - `(pb,(L,n)):(1,(0,pb))` for a `!cute.sparse_elem` of `pb` bits. - - A sparse pointer (the Python `sparsity`) is a `!cute.sparse_elem` value - type. - - ```mlir - !cute.ptr, S<2,4,3>> - !cute.ptr - !cute.ptr> - ``` - }]; - let parameters = (ins - Cute_OptionalParam<"mlir::Type", "pointee type">:$value_type, - Cute_Param<"AddressSpace", "address space">:$address_space, - Cute_Param<"uint64_t", "alignment in bytes; 0 only as a sub-byte type's natural alignment">:$alignment, - Cute_OptionalParam<"SwizzleAttr", "swizzle; null for the identity">:$swizzle, - Cute_OptionalParam<"BitLayoutAttr", "bit layout; null for the default">:$bitlayout - ); - let hasCustomAssemblyFormat = 1; - let genVerifyDecl = 1; - let extraClassDeclaration = [{ - /// The alignment a pointer to `valueType` has when none is spelled. - static uint64_t getNaturalAlignment(mlir::Type valueType); - /// The bit layout a pointer has when none is spelled (null if none). - static BitLayoutAttr getDefaultBitLayout(mlir::Type valueType, - AddressSpace addressSpace); - }]; -} - -def Cute_MemRefType : Cute_Type<"MemRef", "memref"> { - let summary = "Tensor view: a pointer with a layout"; - let description = [{ - `][, S], "layout"[, bitlayout<"L">]>`: - the pointer fields as in `!cute.ptr` (the value type is required), then the - layout or composed layout, then a non-default bit layout. A composed - layout whose inner part is the identity swizzle is read as its outer - layout. - - ```mlir - !cute.memref, S<2,4,3>, "((8,16),(32,1),(1,3)):((32,256),(1,0),(0,4096))"> - !cute.memref> - ``` - }]; - let parameters = (ins - Cute_Param<"PtrType", "pointer">:$ptr, - Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout - ); - let hasCustomAssemblyFormat = 1; - let genVerifyDecl = 1; - let extraClassDeclaration = [{ - mlir::Type getValueType() const { return getPtr().getValueType(); } - AddressSpace getAddressSpace() const { return getPtr().getAddressSpace(); } - uint64_t getAlignment() const { return getPtr().getAlignment(); } - SwizzleAttr getSwizzle() const { return getPtr().getSwizzle(); } - }]; -} - -def Cute_CoordTensorType : Cute_Type<"CoordTensor", "coord_tensor"> { - let summary = "Coordinate tensor: an arithmetic tuple with a layout"; - let description = [{ - ```mlir - !cute.coord_tensor<"(?,?,?)", "((512,1),1):((1/2@0,0),0)"> - ``` - }]; - let parameters = (ins - Cute_Param<"IntTupleType", "origin arithmetic tuple">:$arith_tuple, - Cute_Param<"mlir::Type", "layout or composed-layout type">:$layout - ); - let hasCustomAssemblyFormat = 1; - let genVerifyDecl = 1; -} - -def Cute_ArithTupleIteratorType - : Cute_Type<"ArithTupleIterator", "arith_tuple_iter"> { - let summary = "Iterator over an arithmetic tuple"; - let description = [{ - ```mlir - !cute.arith_tuple_iter<"(?,?{div=16})"> - ``` - }]; - let parameters = (ins - Cute_Param<"IntTupleType", "current arithmetic tuple">:$arith_tuple - ); - // `$arith_tuple` alone would print a space after the mnemonic. - let hasCustomAssemblyFormat = 1; -} - -// `!cute.i`: one mnemonic per width, which the generated type dispatch -// cannot match, so the class has no mnemonic and CuteTypes.cpp routes -// `i` keywords to it. -def Cute_ConstrainedIntType : TypeDef { - let summary = "Integer of any width with known divisibility and power of two"; - let description = [{ - `!cute.i[<[pow2, ]divby D>]`; `divby 1` is left out (the default), - `divby 0` and negative divisors are kept. The compiler also reads - `!cute_nvgpu.i` as this type. - - ```mlir - !cute.i32 - !cute.i16 - !cute.i64 - ``` - }]; - // The compiler's type name for it. - let typeName = "cute.ConstrainedInt"; - let parameters = (ins - Cute_Param<"int64_t", "known divisor">:$divisibility, - Cute_Param<"unsigned", "integer width">:$width, - Cute_Param<"bool", "known to be a power of two">:$is_pow2 - ); - let extraClassDeclaration = [{ - static mlir::Type parse(mlir::AsmParser &parser, unsigned width); - void print(mlir::AsmPrinter &printer) const; - }]; -} - -def Cute_FastDivmodDivisorType - : Cute_Type<"FastDivmodDivisor", "fast_divmod_divisor"> { - let summary = "Precomputed divisor of the fast divmod operations"; - let description = [{ - ```mlir - !cute.fast_divmod_divisor<32> - !cute.fast_divmod_divisor<64, is_pow_2> - ``` - }]; - let parameters = (ins - Cute_Param<"unsigned", "integer width">:$width, - Cute_Param<"bool", "whether the divisor is a power of two">:$is_pow_2 - ); - let hasCustomAssemblyFormat = 1; -} - -def Cute_SparseElemType : Cute_Type<"SparseElem", "sparse_elem"> { - let summary = "Logical sparse element stored in a physical type"; - let description = [{ - `num_logical` logical elements stored as `physical_type` (an integer, - float or vector of them). - - ```mlir - !cute.sparse_elem<2, vector<1xf16>> - ``` - }]; - let parameters = (ins - Cute_Param<"int", "number of logical elements">:$num_logical, - Cute_Param<"mlir::Type", "physical storage type">:$physical_type - ); - let assemblyFormat = "`<` $num_logical `,` $physical_type `>`"; - let genVerifyDecl = 1; -} - -def Cute_TiledCopyType : Cute_Type<"TiledCopy", "tiled_copy"> { - let summary = "Copy atom tiled over threads and values"; - let description = [{ - ```mlir - !cute.tiled_copy, layout_copy_tv = <"(32,4):(4,1)">, tiler_mn = <"[4:1;32:1]">> - ``` - }]; - let parameters = (ins - Cute_Param<"mlir::Type", "copy atom">:$copy_atom, - Cute_Param<"LayoutAttr", "thread-value layout">:$layout_copy_tv, - Cute_Param<"TileAttr", "MN tiler">:$tiler_mn - ); - let assemblyFormat = [{ - `<` $copy_atom `,` `layout_copy_tv` `=` $layout_copy_tv `,` - `tiler_mn` `=` $tiler_mn `>` - }]; -} - -def Cute_TiledCopyV2Type : Cute_Type<"TiledCopyV2", "tiled_copy_v2"> { - let summary = "V2 copy atom tiled over threads and values"; - let description = [{ - ```mlir - !cute.tiled_copy_v2, layout_copy_tv = <"(1,8):(0,1)">, tiler_mn = <"[8:1]">> - ``` - }]; - let parameters = (ins - Cute_Param<"mlir::Type", "V2 copy atom">:$copy_atom, - Cute_Param<"LayoutAttr", "thread-value layout">:$layout_copy_tv, - Cute_Param<"TileAttr", "MN tiler">:$tiler_mn - ); - let assemblyFormat = [{ - `<` $copy_atom `,` `layout_copy_tv` `=` $layout_copy_tv `,` - `tiler_mn` `=` $tiler_mn `>` - }]; -} - -def Cute_TiledMmaType : Cute_Type<"TiledMma", "tiled_mma"> { - let summary = "MMA atom tiled over MNK"; - let description = [{ - The permutation is optional (absent, not `[_;_;_]`, when not given). - - ```mlir - !cute.tiled_mma f32 >, atom_layout_MNK = <"(16,16,1):(16,1,0)">, permutation_MNK = <"[(16,4):(4,1);(16,4):(4,1);_]">> - ``` - }]; - let parameters = (ins - Cute_Param<"mlir::Type", "mma atom">:$mma_atom, - Cute_Param<"LayoutAttr", "rank-3 atom layout over MNK">:$atom_layout_MNK, - Cute_OptionalParam<"TileAttr", "rank-3 permutation over MNK">:$permutation_MNK - ); - let assemblyFormat = [{ - `<` $mma_atom `,` `atom_layout_MNK` `=` $atom_layout_MNK - (`,` `permutation_MNK` `=` $permutation_MNK^)? `>` - }]; - let genVerifyDecl = 1; -} - -def Cute_TupleType : Cute_Type<"Tuple", "tuple"> { - let summary = "Non-empty tuple of types"; - let description = [{ - ```mlir - !cute.tuple - ``` - }]; - let parameters = (ins ArrayRefParameter<"mlir::Type", "element types">:$types); - let assemblyFormat = "`<` $types `>`"; -} - -#endif // MLIR_DIALECTS_CUTE_IR_CUTETYPESCOMPILER_TD diff --git a/mlir/dialects/cute_ir/README.md b/mlir/dialects/cute_ir/README.md deleted file mode 100644 index 6a667ce5..00000000 --- a/mlir/dialects/cute_ir/README.md +++ /dev/null @@ -1,106 +0,0 @@ -# CuTe dialects and Zig bindings - -`mlir/dialects/cute_ir` is the CuTe dialect of nvidia-cutlass-dsl 4.8.0. The -dialect, attributes, public types and `cutegen` layout algebra come verbatim -from the CUTLASS release's `cutlass_compiler/` tree (`@cute_ir`, -pinned in `third_party/cute_ir`, one patch giving cutegen the -`?{div=N}` leaf); this tree carries what differs from it: `CuteTypes.cpp` with -the compiler's remaining types, attributes and enums and ratio strides -allowed, the compiler's attributes, types and enums beyond the release -(`CuteAttrsCompiler.td`, `CuteTypesCompiler.td`, `CuteEnums.td` and the -`../cute_nvgpu` definitions), and every operation of the compiler's registry in -`CuteOps.td` and the companion `CuteNVGPUOps.td`. - -The companion dialects live in `../cute_nvgpu`, `../nvvm`, and `../cuda`, -with separate Bazel targets and Zig modules. NVIDIA's `nvvm` differs from -upstream's. The CuTe builder registers these dialects together with upstream -`llvm`. - -Updating to a new DSL release means refreshing the compiler entry-point -offsets and regenerating the `.td` files from the release's Python bindings -and compiler. The C API (except the algebra attribute functions in `capi.cc`) -and the Zig types, attributes, enums and operation builders are generated -from those `.td` files and follow them. - -## Zig - -Depend on `//mlir/dialects/cute_ir` and import `mlir/dialects/cute_ir`. -`cute.registerDialect(registry)` registers `cute`. - -Import each companion dialect explicitly and add its Bazel dependency: -`mlir/dialects/cute_nvgpu`, `mlir/dialects/nvvm`, and `mlir/dialects/cuda`. -CUDA and NVVM expose generated bindings in `.ops` and convenience builders -at the module root. NVIDIA's NVVM handle is `cute_nvvm`; its dialect cannot -share a registry with upstream NVVM. - -```zig -const cute = @import("mlir/dialects/cute_ir"); - -const pointer = try cute.PtrType.get(ctx, .{ - .valueType = .float(ctx, .f32), - .addressSpace = .smem, - .alignment = 16, -}); -const layout_attr = try cute.LayoutAttr.get(ctx, .{ - .shape = try .getStatic(ctx, 4), - .stride = try .getStatic(ctx, 1), -}); -const layout = try cute.LayoutType.get(ctx, .{ .attr = layout_attr.attribute() }); -const memref = try cute.MemRefType.get(ctx, .{ - .ptr = pointer.type_(), - .layout = layout.type_(), -}); -const alloc = cute.memref_alloc_smem(ctx, memref.type_(), loc).appendTo(block); -``` - -The bindings follow `mlir/dialects/README.md`. Algebra types take an algebra -attribute, `cute.LayoutType.get(ctx, .{ .attr = layout_attr.attribute() })`. -Algebra attributes are built bottom up with named constructors: integer -tuples, coordinates, shapes and strides from `getStatic`, `getDynamic`, -`getBasis` (strides) or `getUnderscore` (coordinates) leaves and `getTuple`; -layouts from a shape and a stride; tiles from `getLayout`, `getUnderscore` and -`getTuple`; swizzles from their three parameters and composed layouts from an -inner swizzle or layout, an offset and an outer layout. `value()` returns the -canonical text. NVGPU atom types take their typed parameters, such -as `.valType` and `.copyBits`. For reading existing IR use MLIR's generic -`Type.parse`. - -Operation builders take the operands, then the result types, then the -attributes, then the location, and call `mlir.Operation.make`. Parameters -follow ODS names, with a trailing underscore where needed. Result parameters -append `_type` or `_types`; attributes take `*const mlir.Attribute`. Optional -operands, attributes and result types go in a trailing `opts` struct whose -fields default to null; pass `.{}` to omit them, and `&.{}` for empty variadic -operands. Dots in -mnemonics become underscores. Variable-length operand and result groups go -through `Operation.make`'s `.variadic` API, which writes the segment-size -attributes. Builders verify the created operations; to defer verification, use -`mlir.Operation.make` directly with `.verify = false`. - -Builders of operations with regions (`cuda.kernel`) and of terminators -(`cuda.return`) do not verify on creation, since their blocks are completed -afterwards; the module is verified instead. - -Each dialect exposes a `:capi` target and a package-local `capi.h`, independently -of Zig: `//mlir/dialects/cute_ir:capi`, `//mlir/dialects/cute_nvgpu:capi`, -`//mlir/dialects/nvvm:capi`, and `//mlir/dialects/cuda:capi`. C API declarations -and definitions are grouped by registration, attributes, and types in -`capi.h` and `capi.cc`. Constructors return a null handle for invalid parameters. - -C++ implementations, headers, and TableGen sources live directly in each -dialect package. Generated `.inc` files are emitted into the same package -under Bazel's output tree. Virtual headers preserve upstream CuTe include -paths required by the release sources. - -```sh -bazel test //mlir/dialects/cute_ir:test //mlir/dialects/cute_nvgpu:test \ - //mlir/dialects/nvvm:test //mlir/dialects/cuda:test -``` - -These tests only check the bindings: every bound operation is registered, -every type and attribute rebuilds from its getters, and a sample of the nvvm -and cuda attributes and types print the compiler's own text byte for byte -(`#cuda` and `#cuda` print with a -space the compiler omits; it parses both). The dialect itself is -tested where it is maintained. The dialects here do not implement the -compiler's semantic verification or lowering. diff --git a/mlir/dialects/cute_ir/REGENERATING.md b/mlir/dialects/cute_ir/REGENERATING.md deleted file mode 100644 index 52b49e69..00000000 --- a/mlir/dialects/cute_ir/REGENERATING.md +++ /dev/null @@ -1,333 +0,0 @@ -# Regenerating the dialect definitions - -These are instructions for an agent. They describe what this directory must -contain after a CuTe DSL release bump, what is available to derive it, and how -to check the result. How to get there is left to you: combine the sources as -you see fit, write throwaway scripts, probe the compiler. Scripts and -intermediate tables are not committed; only the definitions are. - -Bare file names are in this directory, `../` are the sibling packages, -and other paths are relative to the repository root. - -## Goal - -The `cute` dialect in this directory and its companions in `../cute_nvgpu`, -`../nvvm`, and `../cuda` must describe the DSL compiler's -registry as completely and as precisely as the release allows: every -operation, type, attribute and enum the compiler registers, with every fact -that can be established about each of them. Nothing is opaque unless it cannot -be recovered, and each remaining gap is stated in the file where it occurs. - -The DSL compiler stays the authority for semantics and lowering. These -definitions exist so that IR can be built, parsed, printed and type-inferred -without it, and so that anything built here is accepted by it unchanged. - -## What lives here - -- **Taken from the release archive.** Its `cutlass_compiler/` tree provides the - cute dialect declaration, the public types and attributes and their `.td` - files, the type interfaces, `CuteDialect.h`, `CuteAttributes.cpp`, - `CuteDialect.cpp` and the `cutegen` layout algebra. The build reads them - from the archive; they are never copied or edited here. One patch on the - archive (`cutegen_dynamic_divisibility.patch`, next to the archive's - repository rule) teaches `cutegen` the `?{div=N}` leaf. Refresh it if it no - longer applies. The archive's `CuteOps.td` is not used: its operations - differ from the compiler's. -- **Maintained here.** `CuteTypes.cpp` started as the release's copy and holds - the parsers, printers and verifiers of every cute type, fixed wherever the - release's parsing diverges from the compiler's. `CuteInference.{h,cpp}` holds - result type inference; the other dialect `.cpp` files and the `BUILD.bazel` - files complete them. Edit them as the new definitions require. For `nvvm` - and `cuda` these are `NVVMDialect.{h,cpp}` (with the `#nvvm.target` - verifier) and `CudaDialect.{h,cpp}` in their dialect package roots. -- **Generated: what you produce.** Every package-local `.td` file that is not - a dialect declaration: the operations (`CuteOps.td`, `CuteNVGPUOps.td`, - `NVVMOps.td`, `CudaOps.td`), the compiler's types, attributes and enums - beyond the release (`CuteTypesCompiler.td`, `CuteAttrsCompiler.td`, - `CuteEnums.td`, `CuteNVGPUTypes.td`, `CuteNVGPUAttrs.td`, `CuteNVGPUEnums.td`, - `NVVMAttrs.td`, `NVVMEnums.td`, `CudaTypes.td`, `CudaAttrs.td`, - `CudaEnums.td`), the inference of `nvvm` and `cuda` operations whose results - are not one buildable type (`NVVMInference.cpp`, `CudaInference.cpp`), and - any new `.td` you need to split them sensibly. The dialect declarations (`*Dialect.td`) change only as far as the - generated files need. -- **Language bindings, if present.** When the tree carries bindings derived - from these definitions (a `capi.cc`/`capi.h` C API in each dialect package, builder files in each - directory), regenerate them from the records of the new definitions - (`llvm-tblgen --dump-json` gives them all; do not regex the `.td` files). - When a class changes shape (opaque to parametric, merged, split, renamed - accessors), change the bindings to match, and keep the existing - convenience helpers working where it costs little. Names, builder - parameters and `opts` follow `mlir/dialects/README.md`. - -Consumers outside this directory use the dialect classes and must keep -compiling: search the repository for the dialect namespaces -(`cutlass_compiler::cute`, `cute_nvgpu`, `nvvm`, `cuda`), for the Zig modules -(`mlir/dialects/cute_nvgpu`, `mlir/dialects/nvvm`, and -`mlir/dialects/cuda`) and for the headers -in the dialect packages. When a definition changes shape, update those users in the -same change. Tighter operand constraints also break consumers that built IR -with loose types, and so does an operation that stops inferring its results: -fix the consumer, not the constraint. - -## NVIDIA's `nvvm` and `cuda` dialects - -The kernels the CuTe builder emits (`dsl/cute`) use two more dialects of -the compiler: `nvvm` (special registers, mbarriers, tcgen05, ...) and `cuda` -(kernels, launch configurations, host runtime calls). Both have sibling packages -and must be regenerated with every release, like `cute` and `cute_nvgpu`. - -- **Neither is upstream's.** The compiler is built on NVIDIA's own LLVM - revision (the `.so` names it: `LLVM version 23.0.0git ()`, a hash - upstream does not have), and its `nvvm` differs from upstream NVVM: it has - operations, operands and enums upstream lacks (`nvvm.mbarrier.wait.parity`, - `#nvvm.mbar_wait`) and lacks some upstream has. `cuda` exists only in the - compiler. No `.td` ships with the release for either: rebuild both from the - bindings and the live compiler, as for the cute dialects. Upstream's NVVM - `.td` is neither a source nor a substitute. -- **Where they live.** `../nvvm` and `../cuda` contain the declarations, - generated `.td` files, dialect and inference `.cpp` files, and consolidated - `capi.h`/`capi.cc` C APIs. Each package's `ops.zig` contains the generated - bindings; `nvvm.zig` and `cuda.zig` provide the convenience builders. - -- **Names.** The C++ namespaces are `mlir::cutlass_compiler::nvvm` and - `mlir::cutlass_compiler::cuda`. The dialects are named `nvvm` and `cuda` as - in the compiler, so a registry holds this `nvvm` or upstream's, never both; - upstream's NVVM C++ is linked into the tree and must stay unregistered where - this one is. The C dialect handles are `cute_nvvm` and `cuda`, and the C API - of `nvvm` is prefixed `CuteNVVM` so that neither collides with upstream's - `mlirGetDialectHandle__nvvm__` or a future upstream NVVM C API; `cuda`'s is - prefixed `Cuda`. -- **`llvm` is upstream's.** NVVM operands are `!llvm.ptr` and LLVM - structs, and the builder emits `llvm.*` operations. The CuTe context - registers the upstream LLVM dialect, bound by `mlir/dialects/llvm/llvm.zig` - (its builders, and `mlirLLVMPointerTypeGet` for `!llvm.ptr`); the nvvm - and cuda constraints name `::mlir::LLVM` types. NVIDIA's `llvm` is not - vendored: its types print and parse the same, and the compiler reads the - `llvm.*` operations upstream prints (custom and generic forms, with the - properties upstream fills in) into the same module. This holds only while - the compatibility check below passes; when it stops holding, vendor NVIDIA's - `llvm` the same way (its bindings are `_llvm_ops_gen.py` and - `_llvm_enum_gen.py`) rather than patching around it. - -Sources specific to them, besides those in the table below: - -- `_nvvm_ops_gen.py`, `_cuda_ops_gen.py`: the operations. Operand kinds come - from how `__init__` appends them (and `_ODS_OPERAND_SEGMENTS`), result names - and kinds from the result properties, inference from whether `__init__` - builds a `results` list or passes its `results=` argument through. -- `_nvvm_enum_gen.py`, `_cuda_enum_gen.py`: the enums. A builder registered - under the enum's own name builds the plain `I32EnumAttr` (it follows its - enum class in the file); one registered as `nvvm.Attr` builds the - dialect attribute and gives its mnemonic and form: `#nvvm.` - or `#nvvm`. Some enums only have the latter. Parsing an - invalid case names the C++ enum (`expected ::mlir::NVVM:: to be one - of: ...`) or its summary, which maps each attribute to its enum. -- `cutlass._mlir._mlir_libs._cutlass_ir._mlirDialectsCuda`: the typed cuda - type and attribute classes and their `get()` parameters. The other cuda - types and attributes, and the three non-enum nvvm attributes (`shape`, - `ld_st_matrix_shape`, `target`), are found by trying the `.so`'s - identifiers as mnemonics (`!cuda.`, `#nvvm.`); `nvvm` has no - types. -- Attribute constraints the bindings name but no builder covers - (`nvvm.FPArithRoundingMode`, the `nvvm.Multimem*` orders, anonymous - `TypeAttr` constraints) are recovered by probing values: the verifier's - `failed to satisfy constraint: ...` names the rule, and the values it lets - through give the predicate. -- The compiler aborts on some malformed probes (`nvvm.mma.sync`, - `nvvm.mma.sp.sync`): run those checks one process each. -- Results the compiler infers but that are not one buildable type get an - inference rule only where varying the operand types shows the result is one - operand's type; elsewhere inference fails and the types written stand - (`_InferredOp`, as in cute). Vary the attributes too: a rule must - hold for every attribute value (`nvvm.mul` with `mode = wide` doubles the - width, `nvvm.shfl.sync` with `return_value_and_is_valid` returns a struct), - or inference must fail. - -The bindings follow the cute ones: the C API and the Zig enums, typed -attribute and type wrappers and operation builders are generated from -`llvm-tblgen --dump-json` of the generated `.td` files, following -`mlir/dialects/README.md` (names, wrappers, comments, formatting). Builders of -operations with regions, and of terminators, do not verify on creation (their -blocks are filled, or their block is completed, afterwards). `mlir/dialects/nvvm/nvvm.zig` and `mlir/dialects/cuda/cuda.zig` are the -hand-written convenience layer the builder calls; they fix the attributes the -kernels do not choose and must keep building no text. - -## Sources - -Pin everything to one release version, read from the repository rule that -fetches the release archive. The same version names the Python wheels. - -The packaging below is the one of the release these instructions were last -run against. Releases move things around: when a source is missing or has -changed shape, find where the same fact now lives, and update this file. - -| Source | Where | What it gives | -|---|---|---| -| Release archive | `https://github.com/NVIDIA/cutlass/archive/refs/tags/v.tar.gz`, `cutlass_compiler/` | The release's own `.td` files: what is already defined, their style, the parameter C++ types and helpers (`Cute_Param`, interfaces). Anything defined there is not redefined here. | -| Operation bindings | wheel `nvidia-cutlass-dsl-libs-base`, `cutlass/_mlir/dialects/__ops_gen.py` | Per operation: name, class name, docstring (summary, description, examples), operands and results in order with single, optional or variadic kind, segment attributes, attribute names with their ODS constraint (`AttrBuilder.get('')`), whether the builder infers result types, region count, successors. | -| Enum bindings | same wheel, `__enum_gen.py` | Every enum: cases, integer values, spellings, and for the ones that are attributes, the mnemonic they print under. | -| Compiler library | wheel `nvidia-cutlass-dsl-libs-cu13` (or `-cu12`), `cutlass/_mlir/_mlir_libs/_cutlass_ir.cu13.*.so` | Its strings: type and attribute class names, parameter names and C++ types (from the generated parser messages `failed to parse parameter '

' which is to be a `), mnemonics, verifier messages and constraint summaries. The strings are not in declaration order, and parameters read by custom directives have no message at all. | -| Typed Python classes | the same `.so`, as `cutlass._mlir.dialects.cute`, `cute_nvgpu`, `cuda` | For every compiler type and attribute: a `get(...)` with its parameters in order, and getters. Building a value and printing it shows the canonical syntax of each parameter value. | -| The examples | the release archive's `examples/python/CuTeDSL` | Real compiler-accepted modules, the round-trip corpus (see below). | -| The live compiler | the same `.so`, loaded from Python | Ground truth for everything above. | - -Loading the compiler. The `.so` only runs on Linux, on the architecture it was -built for. Two ways: - -- **Full install** (needed for the typed classes): a venv with - `nvidia-cutlass-dsl==` installed, then `import cutlass` before - anything else. -- **Bare wheels** (generic API only): unpack the two wheels into one directory, - put its `nvidia_cutlass_dsl/dsl_packages` on `sys.path`, `import - cutlass._mlir._mlir_libs`. The typed classes fail to import this way. - -Then use `cutlass._mlir.ir` with a fresh `ir.Context()`. The compiler -segfaults on some malformed inputs (some bitlayouts, some bad inference -inputs): run probes in a forked process. - -What the live compiler tells you: - -- `ir.Type.parse` / `ir.Attribute.parse` accept or reject a spelling, and on - rejection say where and what they expected; `str()` gives the canonical - printed form. Together with the typed classes this establishes each syntax: - the generated parser messages name each missing parameter, and the bespoke - custom directives (`128x128x32`, `32 DP`, `x32`, trailing flags such as - `mcast` or `t`) show up as `expected ...` errors. Every mnemonic a parser - knows can be found by trying every identifier-like string of the `.so`. -- `ir.Module.parse` on generic-form operations with wrong types yields the - verifier's constraint for each operand and result, in the compiler's words - (`operand #0 must be ... , but got ...`). Probe each operand with one - instance of every type class to learn exactly which classes it accepts. -- `op.has_trait(ir.IsTerminatorTrait)`, `ir.NoTerminatorTrait`; constructing - `ir.InferTypeOpInterface(op)` or `ir.MemoryEffectsOpInterface(op)` fails when - the operation lacks the interface. -- `ir.InferTypeOpInterface().inferReturnTypes(operands=..., - attributes=..., context=...)` is the inference oracle, with operand values - taken as block arguments of a dummy region. It cannot take properties: - for operations with attributes, parse and verify a module instead. -- Loop-invariant code motion hoists `Pure` operations, `canonicalize` removes - unused ones that do not write, and `cse` merges ones that at most read. -- Printing a parsed generic operation shows its custom assembly. - -The round-trip corpus: run every example of the archive's -`examples/python/CuTeDSL` with `CUTE_DSL_KEEP=ir-debug`, which keeps each -compiled module's IR; set `CUTE_DSL_ARCH` for the architecture-restricted -ones. This gives hundreds of real modules covering far more operations than -the kernels' fixtures in the repository, which are added to it. - -When sources disagree, the live compiler wins; record the disagreement in a -comment. - -## What each definition must carry - -Operations: - -- The operation name, and the binding's class name as the def name. -- `summary` and `description` from the docstring, kept whole, including its - examples. Reflow nothing that is code. Every example must parse and verify in - the compiler; fix the ones that do not and say so. -- Every operand and result, in order, with its name, its kind (single, - `Optional`, `Variadic`), and a type constraint that accepts exactly the type - classes the compiler's verifier accepts, with the verifier's wording as its - summary. Where a class or interface is not defined here or in the release, - or no probe type satisfies it, the predicate accepts any type and the file - says so. Constraints on types of unregistered dialects (such as `ptr`) must - also match the opaque form those types take here. A constraint must never - reject what the compiler accepts. -- Every attribute with its constraint: the ODS constraint the bindings name, - or the generated enum or dialect attribute it corresponds to; `OptionalAttr` - and `DefaultValuedAttr` as the bindings show. -- `AttrSizedOperandSegments` / `AttrSizedResultSegments`, regions (count and - kind), successors, terminator traits. -- `Pure` (or the memory effects) when established; none when unknown. -- `InferTypeOpInterface` exactly on the operations whose result types the - compiler infers. ODS generates the inference when all results are buildable - types; the others are written in `CuteInference.cpp`. The compiler's - inference is not a function of types alone: it folds constant operands and - keeps divisibility that `cutegen` drops. Where a rule is not recovered, - inference must fail rather than guess, and those operations override - `refineReturnTypes` so written result types stand: compiler-accepted IR is - never rejected for want of an inference rule. -- Operations keep the generic assembly form unless an `assemblyFormat` has been - checked to round-trip against the compiler (see below); the custom syntax - stays in the description. - -Types and attributes: - -- Every class the compiler has that the release lacks, with its mnemonic, - summary, parameters in declaration order (from the typed `get()`s and the - order parsing asks for them) with their C++ types and optional or defaulted - status, and a printed example in the description. C++ types of parameters - read by custom directives are not observable: choose the natural one. -- The syntax exactly as the compiler prints and parses it: a declarative - `assemblyFormat` when it can express it, otherwise a custom parser and - printer. Parametric beats opaque: a verbatim string payload is acceptable - only when the syntax cannot be reproduced, and the file says why. A hidden - parameter that never prints is left out: keeping it would make a built value - differ from its own printed text. -- The compiler's type aliases (`!memref_smem_f16`, `!copy_simt`, ...) are part - of its printed form: reproduce them through the dialect's - `OpAsmDialectInterface`. -- Verifiers (`genVerifyDecl`) where the compiler rejects parameter values, when - the rule is known. Large legality tables (instruction shapes and forms) may - be left to the compiler; the file says which. -- Enums as `I32EnumAttr` with the compiler's values and spellings, and their - `EnumAttr` with the compiler's mnemonic and `` form. - -Every generated file starts with a short header naming the release version and -the sources it came from, saying it is generated, and listing what it leaves -out or leaves unconstrained, and why. - -ODS pitfalls met so far: - -- Naming `InferTypeOpInterface` in the traits of an operation with buildable - results suppresses the `inferReturnTypes` ODS would generate. -- `mlir-tblgen -gen-enum-decls` emits every enum it sees, so a dialect's enums - file must not include another dialect's. -- The release's `CuteDialect.h` includes the attributes before the types, so - enums the attributes use must be emitted ahead of the attributes' include. -- A mnemonic that varies with a value (`!cute.i32`, `!cute.i7`) needs a hook - in front of the generated type parser. -- An attribute whose body is not in `<...>` prints in the opaque form - (`#dialect<...>`); its pretty form does not parse. - -## Checking the result - -A change is done when all of these hold: - -1. **Coverage.** Every operation in the bindings, every enum, and every - mnemonic the compiler's parsers know is either defined here, defined by the - release, or listed in the file's header as left out with the reason. -2. **Round trip against the compiler,** on the corpus and on sweeps of - spellings for every type and attribute (combinations of parameter values): - whatever the compiler accepts, this dialect accepts, and prints text the - compiler reads back to its own canonical form. Where the compiler's printed - form is itself reparseable, this dialect prints it byte-identical. Where - the compiler prints text its own parser rejects, or its parser drops data, - the reparseable, lossless form wins; list those cases. -3. **Inference agrees** with the compiler on every result the corpus and - crafted oracle cases produce. The inference test holds recorded compiler - results, since the compiler cannot be asked at test time: add a case for - every inferred operation the emitted modules do not already cover. -4. **The repository's tests pass**, including the dialect's own tests and the - tests of every consumer found above (kernel emitters, host tools, language - bindings), built with the tree's usual Bazel configuration. -5. **NVIDIA's compiler reads what the kernels emit.** Dump the IR of every - kernel the repository's tests emit through `dsl/cute`: the builder's - own tests and the CuTe kernels under `zml/kernels/moe/cute`, - `zml/kernels/flash_attention/cute` and `zml/kernels/paged_attention/cute`. - Dump both with the previous definitions and with the new ones. In the - release's venv, `import cutlass`, then parse and - verify each module with `cutlass._mlir.ir.Module.parse`, and compare the - generic print (`get_asm(print_generic_op_form=True)`) of each module before - and after: it must be identical, and a module the compiler rejected before - must be rejected the same way after. This is also what keeps upstream - `llvm` acceptable (see above). -6. **Both copies agree.** When the same dialect is carried by more than one - repository, the shared files are identical in each, and only - repository-specific files (`BUILD.bazel`, bindings) differ. - -Report, at the end: what changed in each file, the coverage figures, the -round-trip and inference figures, what is still opaque or unconstrained and -why, the test results, and anything in these instructions that was wrong, -missing or unclear. diff --git a/mlir/dialects/cute_ir/capi.cc b/mlir/dialects/cute_ir/capi.cc deleted file mode 100644 index 6e7c296d..00000000 --- a/mlir/dialects/cute_ir/capi.cc +++ /dev/null @@ -1,1454 +0,0 @@ -//===- capi.cc - C API for the cute dialect -------------------------------===// - -#include "capi.h" - -#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" -#include "mlir/CAPI/Support.h" -#include "llvm/ADT/STLExtras.h" - -#include - -// Registration - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Cute, cute, - mlir::cutlass_compiler::cute::CuteDialect) - -// Attributes - -namespace cute = mlir::cutlass_compiler::cute; - -// Unwraps `handle` into `out`: false when it is of another context or not a -// T, or when it is null and not optional. -template -static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { - auto value = unwrap(handle); - if (!value) - return optional; - if (value.getContext() != ctx) - return false; - out = llvm::dyn_cast(value); - return static_cast(out); -} - -template -static MlirAttribute getAlgebraAttr(MlirContext context, - typename Attr::algebra_t algebra) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(Attr::get(ctx, std::move(algebra))); -} - -template -static MlirAttribute getStaticAttr(MlirContext context, int64_t value) { - return getAlgebraAttr( - context, typename Attr::algebra_t(cutegen::static_int_t(value))); -} - -template -static MlirAttribute getDynamicAttr(MlirContext context, int32_t width, - int64_t divisibility) { - if (!llvm::is_contained({1, 8, 16, 32, 64}, width) || divisibility <= 0) - return {nullptr}; - return getAlgebraAttr( - context, typename Attr::algebra_t(cutegen::mlir_dynamic_t( - cutegen::dynamic_int_properties_t(width, divisibility)))); -} - -template -static MlirAttribute getUnderscoreAttr(MlirContext context) { - return getAlgebraAttr( - context, typename Attr::algebra_t(cutegen::underscore_t())); -} - -template -static MlirAttribute getTupleAttr(MlirContext context, intptr_t numElements, - MlirAttribute const *elements) { - auto *ctx = unwrap(context); - if (!ctx || numElements < 0 || (numElements > 0 && !elements)) - return {nullptr}; - typename Attr::algebra_t::vector_t children; - children.reserve(numElements); - for (intptr_t i = 0; i < numElements; ++i) { - Attr element; - if (!unwrapAs(ctx, elements[i], /*optional=*/false, element)) - return {nullptr}; - children.push_back(element.getRef()); - } - return getAlgebraAttr(context, - typename Attr::algebra_t(std::move(children))); -} - -template -static MlirStringRef getAlgebraValue(MlirAttribute attr) { - auto value = llvm::cast(unwrap(attr)); - // Intern the serialization so the returned view outlives this function. - auto text = mlir::StringAttr::get(value.getContext(), - cutegen::to_string(value.getRef())); - return wrap(text.getValue()); -} - -MlirAttribute mlirCuteIntTupleAttrStaticGet(MlirContext context, - int64_t value) { - return getStaticAttr(context, value); -} - -MlirAttribute mlirCuteIntTupleAttrDynamicGet(MlirContext context, int32_t width, - int64_t divisibility) { - return getDynamicAttr(context, width, divisibility); -} - -MlirAttribute mlirCuteIntTupleAttrTupleGet(MlirContext context, - intptr_t numElements, - MlirAttribute const *elements) { - return getTupleAttr(context, numElements, elements); -} - -bool mlirAttributeIsACuteIntTuple(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteIntTupleAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteCoordAttrStaticGet(MlirContext context, int64_t value) { - return getStaticAttr(context, value); -} - -MlirAttribute mlirCuteCoordAttrDynamicGet(MlirContext context, int32_t width, - int64_t divisibility) { - return getDynamicAttr(context, width, divisibility); -} - -MlirAttribute mlirCuteCoordAttrUnderscoreGet(MlirContext context) { - return getUnderscoreAttr(context); -} - -MlirAttribute mlirCuteCoordAttrTupleGet(MlirContext context, - intptr_t numElements, - MlirAttribute const *elements) { - return getTupleAttr(context, numElements, elements); -} - -bool mlirAttributeIsACuteCoord(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteShapeAttrStaticGet(MlirContext context, int64_t value) { - return getStaticAttr(context, value); -} - -MlirAttribute mlirCuteShapeAttrDynamicGet(MlirContext context, int32_t width, - int64_t divisibility) { - return getDynamicAttr(context, width, divisibility); -} - -MlirAttribute mlirCuteShapeAttrTupleGet(MlirContext context, - intptr_t numElements, - MlirAttribute const *elements) { - return getTupleAttr(context, numElements, elements); -} - -bool mlirAttributeIsACuteShape(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr) { - return cutegen::rank(llvm::cast(unwrap(attr)).getRef()); -} - -MlirAttribute mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value) { - return getStaticAttr(context, value); -} - -MlirAttribute mlirCuteStrideAttrDynamicGet(MlirContext context, int32_t width, - int64_t divisibility) { - return getDynamicAttr(context, width, divisibility); -} - -MlirAttribute mlirCuteStrideAttrBasisGet(MlirContext context, int64_t numerator, - int64_t denominator, intptr_t numModes, - int32_t const *modes) { - if (denominator <= 0 || numModes <= 0 || !modes) - return {nullptr}; - cutegen::scaled_basis::vector_t path(modes, modes + numModes); - auto basis = - denominator == 1 - ? cutegen::scaled_basis(std::move(path), numerator) - : cutegen::scaled_basis(std::move(path), - cutegen::ratio(numerator, denominator)); - return getAlgebraAttr(context, - cutegen::stride(std::move(basis))); -} - -MlirAttribute mlirCuteStrideAttrTupleGet(MlirContext context, - intptr_t numElements, - MlirAttribute const *elements) { - return getTupleAttr(context, numElements, elements); -} - -bool mlirAttributeIsACuteStride(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, MlirAttribute shape, - MlirAttribute stride) { - auto *ctx = unwrap(context); - cute::ShapeAttr shapeAttr; - cute::StrideAttr strideAttr; - if (!ctx || !unwrapAs(ctx, shape, /*optional=*/false, shapeAttr) || - !unwrapAs(ctx, stride, /*optional=*/false, strideAttr)) - return {nullptr}; - cutegen::layout layout(shapeAttr.getRef(), strideAttr.getRef()); - if (cutegen::has_error(layout)) - return {nullptr}; - return getAlgebraAttr(context, std::move(layout)); -} - -bool mlirAttributeIsACuteLayout(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteTileAttrLayoutGet(MlirContext context, - MlirAttribute layout) { - auto *ctx = unwrap(context); - cute::LayoutAttr layoutAttr; - if (!ctx || !unwrapAs(ctx, layout, /*optional=*/false, layoutAttr)) - return {nullptr}; - return getAlgebraAttr(context, - cutegen::tile(layoutAttr.getRef())); -} - -MlirAttribute mlirCuteTileAttrUnderscoreGet(MlirContext context) { - return getUnderscoreAttr(context); -} - -MlirAttribute mlirCuteTileAttrTupleGet(MlirContext context, - intptr_t numElements, - MlirAttribute const *elements) { - return getTupleAttr(context, numElements, elements); -} - -bool mlirAttributeIsACuteTile(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteComposedLayoutAttrGet(MlirContext context, - MlirAttribute inner, - MlirAttribute offset, - MlirAttribute outer) { - auto *ctx = unwrap(context); - cute::IntTupleAttr offsetAttr; - cute::LayoutAttr outerAttr; - if (!ctx || !unwrapAs(ctx, offset, /*optional=*/false, offsetAttr) || - !unwrapAs(ctx, outer, /*optional=*/false, outerAttr)) - return {nullptr}; - cute::SwizzleAttr swizzleAttr; - cute::LayoutAttr layoutAttr; - std::optional composed; - if (unwrapAs(ctx, inner, /*optional=*/false, swizzleAttr)) - composed.emplace(swizzleAttr.getRef(), offsetAttr.getRef(), - outerAttr.getRef()); - else if (unwrapAs(ctx, inner, /*optional=*/false, layoutAttr)) - composed.emplace(layoutAttr.getRef(), offsetAttr.getRef(), - outerAttr.getRef()); - if (!composed || cutegen::has_error(*composed)) - return {nullptr}; - return getAlgebraAttr(context, - std::move(*composed)); -} - -bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, uint32_t bits, - uint32_t base, int32_t shift) { - auto swizzle = cutegen::swizzle::get(bits, base, shift); - if (!swizzle) - return {nullptr}; - return getAlgebraAttr(context, *swizzle); -} - -bool mlirAttributeIsACuteSwizzle(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirStringRef mlirCuteSwizzleAttrGetValue(MlirAttribute attr) { - return getAlgebraValue(attr); -} - -// Compiler attributes - -bool mlirAttributeIsACuteBitLayout(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, - MlirAttribute layout) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::LayoutAttr layoutValue; - if (!unwrapAs(ctx, layout, false, layoutValue)) - return {nullptr}; - return wrap(cute::BitLayoutAttr::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - layoutValue)); -} -MlirAttribute mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayout())); -} - -bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteCopyAtomAttrGet(MlirContext context, MlirAttribute thrId, - MlirAttribute layoutSrc, - MlirAttribute layoutDst, - MlirAttribute layoutRef) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcValue; - if (!unwrapAs(ctx, layoutSrc, false, layoutSrcValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstValue; - if (!unwrapAs(ctx, layoutDst, false, layoutDstValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutRefValue; - if (!unwrapAs(ctx, layoutRef, false, layoutRefValue)) - return {nullptr}; - return wrap(cute::CopyAtomAttr::get(ctx, thrIdValue, layoutSrcValue, - layoutDstValue, layoutRefValue)); -} -MlirAttribute mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSrc())); -} -MlirAttribute mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutDst())); -} -MlirAttribute mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutRef())); -} - -bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute -mlirCuteCopyAtomV2AttrGet(MlirContext context, MlirAttribute thrId, - MlirAttribute layoutSrcTV, MlirAttribute layoutDstTV, - MlirAttribute frgSrc, MlirAttribute frgDst) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSrcTVValue; - if (!unwrapAs(ctx, layoutSrcTV, false, layoutSrcTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutDstTVValue; - if (!unwrapAs(ctx, layoutDstTV, false, layoutDstTVValue)) - return {nullptr}; - mlir::Attribute frgSrcValue; - if (!unwrapAs(ctx, frgSrc, false, frgSrcValue)) - return {nullptr}; - mlir::Attribute frgDstValue; - if (!unwrapAs(ctx, frgDst, false, frgDstValue)) - return {nullptr}; - return wrap(cute::CopyAtomV2Attr::get(ctx, thrIdValue, layoutSrcTVValue, - layoutDstTVValue, frgSrcValue, - frgDstValue)); -} -MlirAttribute mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSrc_TV())); -} -MlirAttribute mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutDst_TV())); -} -MlirAttribute mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgSrc())); -} -MlirAttribute mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgDst())); -} - -bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, - MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, - MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; - if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; - if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; - if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; - if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; - if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; - if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; - if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) - return {nullptr}; - mlir::Attribute frgAValue; - if (!unwrapAs(ctx, frgA, false, frgAValue)) - return {nullptr}; - mlir::Attribute frgBValue; - if (!unwrapAs(ctx, frgB, false, frgBValue)) - return {nullptr}; - mlir::Attribute frgCValue; - if (!unwrapAs(ctx, frgC, false, frgCValue)) - return {nullptr}; - return wrap(cute::MmaAtomAttr::get( - ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, - layoutCTVValue, shapeAMKValue, shapeBNKValue, shapeCMNValue, frgAValue, - frgBValue, frgCValue)); -} -MlirAttribute mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShape_MNK())); -} -MlirAttribute mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutA_TV())); -} -MlirAttribute mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutB_TV())); -} -MlirAttribute mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutC_TV())); -} -MlirAttribute mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeA_MK())); -} -MlirAttribute mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeB_NK())); -} -MlirAttribute mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeC_MN())); -} -MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgA())); -} -MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgB())); -} -MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgC())); -} - -bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteMxMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, - MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, - MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, - MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, - MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; - if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; - if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; - if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; - if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; - if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; - if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; - if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; - if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; - if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; - if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; - if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) - return {nullptr}; - mlir::Attribute frgAValue; - if (!unwrapAs(ctx, frgA, false, frgAValue)) - return {nullptr}; - mlir::Attribute frgSFAValue; - if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) - return {nullptr}; - mlir::Attribute frgBValue; - if (!unwrapAs(ctx, frgB, false, frgBValue)) - return {nullptr}; - mlir::Attribute frgSFBValue; - if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) - return {nullptr}; - mlir::Attribute frgCValue; - if (!unwrapAs(ctx, frgC, false, frgCValue)) - return {nullptr}; - return wrap(cute::MxMmaAtomAttr::get( - ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, - layoutBTVValue, layoutSFBTVValue, layoutCTVValue, shapeAMKValue, - shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, shapeCMNValue, frgAValue, - frgSFAValue, frgBValue, frgSFBValue, frgCValue)); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShape_MNK())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutA_TV())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSFA_TV())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutB_TV())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSFB_TV())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutC_TV())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeA_MK())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeSFA_MK())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeB_NK())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeSFB_NK())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeC_MN())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgA())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgSFA())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgB())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgSFB())); -} -MlirAttribute mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgC())); -} - -bool mlirAttributeIsACuteReductionOp(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeReductionOp(value); - if (!valueValue) - return {nullptr}; - return wrap(cute::ReductionOpAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, - MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, - MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, - MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; - if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; - if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; - if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; - if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; - if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; - if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; - if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; - if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; - if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) - return {nullptr}; - mlir::Attribute frgAValue; - if (!unwrapAs(ctx, frgA, false, frgAValue)) - return {nullptr}; - mlir::Attribute frgBValue; - if (!unwrapAs(ctx, frgB, false, frgBValue)) - return {nullptr}; - mlir::Attribute frgCValue; - if (!unwrapAs(ctx, frgC, false, frgCValue)) - return {nullptr}; - mlir::Attribute frgEValue; - if (!unwrapAs(ctx, frgE, false, frgEValue)) - return {nullptr}; - return wrap(cute::SparseMmaAtomAttr::get( - ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutBTVValue, - layoutCTVValue, layoutETVValue, shapeAMKValue, shapeBNKValue, - shapeCMNValue, shapeEMKValue, frgAValue, frgBValue, frgCValue, - frgEValue)); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShape_MNK())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutA_TV())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutB_TV())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutC_TV())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutE_TV())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeA_MK())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeB_NK())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeC_MN())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeE_MK())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgA())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgB())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgC())); -} -MlirAttribute mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgE())); -} - -bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, - MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, - MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, - MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, - MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, - MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMNKValue; - if (!unwrapAs(ctx, shapeMNK, false, shapeMNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr thrIdValue; - if (!unwrapAs(ctx, thrId, false, thrIdValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutATVValue; - if (!unwrapAs(ctx, layoutATV, false, layoutATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFATVValue; - if (!unwrapAs(ctx, layoutSFATV, false, layoutSFATVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutBTVValue; - if (!unwrapAs(ctx, layoutBTV, false, layoutBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutSFBTVValue; - if (!unwrapAs(ctx, layoutSFBTV, false, layoutSFBTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCTVValue; - if (!unwrapAs(ctx, layoutCTV, false, layoutCTVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutETVValue; - if (!unwrapAs(ctx, layoutETV, false, layoutETVValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeAMKValue; - if (!unwrapAs(ctx, shapeAMK, false, shapeAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFAMKValue; - if (!unwrapAs(ctx, shapeSFAMK, false, shapeSFAMKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeBNKValue; - if (!unwrapAs(ctx, shapeBNK, false, shapeBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeSFBNKValue; - if (!unwrapAs(ctx, shapeSFBNK, false, shapeSFBNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeCMNValue; - if (!unwrapAs(ctx, shapeCMN, false, shapeCMNValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr shapeEMKValue; - if (!unwrapAs(ctx, shapeEMK, false, shapeEMKValue)) - return {nullptr}; - mlir::Attribute frgAValue; - if (!unwrapAs(ctx, frgA, false, frgAValue)) - return {nullptr}; - mlir::Attribute frgSFAValue; - if (!unwrapAs(ctx, frgSFA, false, frgSFAValue)) - return {nullptr}; - mlir::Attribute frgBValue; - if (!unwrapAs(ctx, frgB, false, frgBValue)) - return {nullptr}; - mlir::Attribute frgSFBValue; - if (!unwrapAs(ctx, frgSFB, false, frgSFBValue)) - return {nullptr}; - mlir::Attribute frgCValue; - if (!unwrapAs(ctx, frgC, false, frgCValue)) - return {nullptr}; - mlir::Attribute frgEValue; - if (!unwrapAs(ctx, frgE, false, frgEValue)) - return {nullptr}; - return wrap(cute::SparseMxMmaAtomAttr::get( - ctx, shapeMNKValue, thrIdValue, layoutATVValue, layoutSFATVValue, - layoutBTVValue, layoutSFBTVValue, layoutCTVValue, layoutETVValue, - shapeAMKValue, shapeSFAMKValue, shapeBNKValue, shapeSFBNKValue, - shapeCMNValue, shapeEMKValue, frgAValue, frgSFAValue, frgBValue, - frgSFBValue, frgCValue, frgEValue)); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShape_MNK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getThrId())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutA_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSFA_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutB_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutSFB_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutC_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLayoutE_TV())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeA_MK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeSFA_MK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeB_NK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeSFB_NK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeC_MN())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getShapeE_MK())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgA())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgSFA())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgB())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgSFB())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgC())); -} -MlirAttribute mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFrgE())); -} - -// Types - -bool mlirTypeIsACuteType(MlirType type) { - mlir::Type cppType = unwrap(type); - return cppType && - cppType.getDialect().getNamespace() == - mlir::cutlass_compiler::cute::CuteDialect::getDialectNamespace(); -} - -bool mlirTypeIsACuteArithTupleIterator(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteArithTupleIteratorTypeGet(MlirContext context, - MlirType arithTuple) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; - if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) - return {nullptr}; - return wrap(cute::ArithTupleIteratorType::get(ctx, arithTupleValue)); -} -MlirType mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getArithTuple())); -} - -bool mlirTypeIsACuteComposedLayout(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, - MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ComposedLayoutAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::ComposedLayoutType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - attrValue)); -} -MlirAttribute mlirCuteComposedLayoutTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteConstrainedInt(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, - int64_t divisibility, uint32_t width, - bool isPow2) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute::ConstrainedIntType::get(ctx, divisibility, width, isPow2)); -} -int64_t mlirCuteConstrainedIntTypeGetDivisibility(MlirType type) { - return llvm::cast(unwrap(type)).getDivisibility(); -} -uint32_t mlirCuteConstrainedIntTypeGetWidth(MlirType type) { - return llvm::cast(unwrap(type)).getWidth(); -} -bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type) { - return llvm::cast(unwrap(type)).getIsPow2(); -} - -bool mlirTypeIsACuteCoordTensor(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteCoordTensorTypeGet(MlirContext context, MlirType arithTuple, - MlirType layout) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::IntTupleType arithTupleValue; - if (!unwrapAs(ctx, arithTuple, false, arithTupleValue)) - return {nullptr}; - mlir::Type layoutValue; - if (!unwrapAs(ctx, layout, false, layoutValue)) - return {nullptr}; - return wrap(cute::CoordTensorType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - arithTupleValue, layoutValue)); -} -MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getArithTuple())); -} -MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getLayout())); -} - -bool mlirTypeIsACuteCoord(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteCoordTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::CoordAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::CoordType::get(ctx, attrValue)); -} -MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteFastDivmodDivisor(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteFastDivmodDivisorTypeGet(MlirContext context, uint32_t width, - bool isPow2) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute::FastDivmodDivisorType::get(ctx, width, isPow2)); -} -uint32_t mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type) { - return llvm::cast(unwrap(type)).getWidth(); -} -bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type) { - return llvm::cast(unwrap(type)).getIsPow_2(); -} - -bool mlirTypeIsACuteIntTuple(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteIntTupleTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::IntTupleAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::IntTupleType::get(ctx, attrValue)); -} -MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteLayout(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteLayoutTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::LayoutAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::LayoutType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - attrValue)); -} -MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteMemRef(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteMemRefTypeGet(MlirContext context, MlirType ptr, - MlirType layout) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::PtrType ptrValue; - if (!unwrapAs(ctx, ptr, false, ptrValue)) - return {nullptr}; - mlir::Type layoutValue; - if (!unwrapAs(ctx, layout, false, layoutValue)) - return {nullptr}; - return wrap(cute::MemRefType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - ptrValue, layoutValue)); -} -MlirType mlirCuteMemRefTypeGetPtr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPtr())); -} -MlirType mlirCuteMemRefTypeGetLayout(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getLayout())); -} - -bool mlirTypeIsACutePtr(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCutePtrTypeGet(MlirContext context, MlirType valueType, - uint32_t addressSpace, uint64_t alignment, - MlirAttribute swizzle, MlirAttribute bitlayout) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valueTypeValue; - if (!unwrapAs(ctx, valueType, true, valueTypeValue)) - return {nullptr}; - auto addressSpaceValue = - ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(addressSpace); - if (!addressSpaceValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute::SwizzleAttr swizzleValue; - if (!unwrapAs(ctx, swizzle, true, swizzleValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::BitLayoutAttr bitlayoutValue; - if (!unwrapAs(ctx, bitlayout, true, bitlayoutValue)) - return {nullptr}; - return wrap(cute::PtrType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valueTypeValue, *addressSpaceValue, alignment, swizzleValue, - bitlayoutValue)); -} -MlirType mlirCutePtrTypeGetValueType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getValueType())); -} -uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getAddressSpace()); -} -uint64_t mlirCutePtrTypeGetAlignment(MlirType type) { - return llvm::cast(unwrap(type)).getAlignment(); -} -MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getSwizzle())); -} -MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getBitlayout())); -} - -bool mlirTypeIsACuteShape(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteShapeTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::ShapeType::get(ctx, attrValue)); -} -MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteSparseElem(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteSparseElemTypeGet(MlirContext context, int numLogical, - MlirType physicalType) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type physicalTypeValue; - if (!unwrapAs(ctx, physicalType, false, physicalTypeValue)) - return {nullptr}; - return wrap(cute::SparseElemType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - numLogical, physicalTypeValue)); -} -int mlirCuteSparseElemTypeGetNumLogical(MlirType type) { - return llvm::cast(unwrap(type)).getNumLogical(); -} -MlirType mlirCuteSparseElemTypeGetPhysicalType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPhysicalType())); -} - -bool mlirTypeIsACuteStride(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteStrideTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::StrideAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::StrideType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - attrValue)); -} -MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteSwizzle(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteSwizzleTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::SwizzleAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::SwizzleType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - attrValue)); -} -MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteTile(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteTileTypeGet(MlirContext context, MlirAttribute attr) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::TileAttr attrValue; - if (!unwrapAs(ctx, attr, false, attrValue)) - return {nullptr}; - return wrap(cute::TileType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - attrValue)); -} -MlirAttribute mlirCuteTileTypeGetAttr(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAttr())); -} - -bool mlirTypeIsACuteTiledCopy(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteTiledCopyTypeGet(MlirContext context, MlirType copyAtom, - MlirAttribute layoutCopyTv, - MlirAttribute tilerMn) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type copyAtomValue; - if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; - if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; - if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) - return {nullptr}; - return wrap(cute::TiledCopyType::get(ctx, copyAtomValue, layoutCopyTvValue, - tilerMnValue)); -} -MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getCopyAtom())); -} -MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type) { - return mlirCuteTiledCopyTypeGetCopyAtom(type); -} -MlirAttribute mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getLayoutCopyTv())); -} -MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getTilerMn())); -} - -bool mlirTypeIsACuteTiledCopyV2(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, - MlirAttribute layoutCopyTv, - MlirAttribute tilerMn) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type copyAtomValue; - if (!unwrapAs(ctx, copyAtom, false, copyAtomValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutCopyTvValue; - if (!unwrapAs(ctx, layoutCopyTv, false, layoutCopyTvValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::TileAttr tilerMnValue; - if (!unwrapAs(ctx, tilerMn, false, tilerMnValue)) - return {nullptr}; - return wrap(cute::TiledCopyV2Type::get(ctx, copyAtomValue, layoutCopyTvValue, - tilerMnValue)); -} -MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getCopyAtom())); -} -MlirAttribute mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getLayoutCopyTv())); -} -MlirAttribute mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getTilerMn())); -} - -bool mlirTypeIsACuteTiledMma(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteTiledMmaTypeGet(MlirContext context, MlirType mmaAtom, - MlirAttribute atomLayoutMNK, - MlirAttribute permutationMNK) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type mmaAtomValue; - if (!unwrapAs(ctx, mmaAtom, false, mmaAtomValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr atomLayoutMNKValue; - if (!unwrapAs(ctx, atomLayoutMNK, false, atomLayoutMNKValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::TileAttr permutationMNKValue; - if (!unwrapAs(ctx, permutationMNK, true, permutationMNKValue)) - return {nullptr}; - return wrap(cute::TiledMmaType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - mmaAtomValue, atomLayoutMNKValue, permutationMNKValue)); -} -MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getMmaAtom())); -} -MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type) { - return mlirCuteTiledMmaTypeGetMmaAtom(type); -} -MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAtomLayout_MNK())); -} -MlirAttribute mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type) { - return mlirCuteTiledMmaTypeGetAtomLayoutMNK(type); -} -MlirAttribute mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getPermutation_MNK())); -} -MlirAttribute mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type) { - return mlirCuteTiledMmaTypeGetPermutationMNK(type); -} - -bool mlirTypeIsACuteTuple(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteTupleTypeGet(MlirContext context, intptr_t numTypes, - MlirType const *types) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - llvm::SmallVector typesValue; - for (intptr_t i = 0; i < numTypes; ++i) { - mlir::Type element; - if (!unwrapAs(ctx, types[i], false, element)) - return {nullptr}; - typesValue.push_back(element); - } - return wrap(cute::TupleType::get(ctx, typesValue)); -} -intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getTypes().size()); -} -MlirType mlirCuteTupleTypeGetType(MlirType type, intptr_t pos) { - return wrap(llvm::cast(unwrap(type)).getTypes()[pos]); -} diff --git a/mlir/dialects/cute_ir/capi.h b/mlir/dialects/cute_ir/capi.h deleted file mode 100644 index 002f4885..00000000 --- a/mlir/dialects/cute_ir/capi.h +++ /dev/null @@ -1,492 +0,0 @@ -//===- capi.h - C API for the cute dialect ------------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_CUTE_IR_CAPI_H -#define MLIR_DIALECTS_CUTE_IR_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Registration - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Cute, cute); - -// Attributes - -// Algebra attributes are built bottom up. A leaf is a static integer, a -// dynamic integer of `width` bits (1, 8, 16, 32 or 64) known to be a multiple -// of `divisibility`, a scaled basis (strides) or an underscore (coordinates -// and tiles); a tuple groups attributes of the same kind. A basis `modes` is -// its mode path, outermost first, which CuTe prints innermost first: -// `1@1@0` is modes {0, 1}. Constructors return null for invalid parameters or -// handles of another context. GetValue returns canonical algebra text owned -// by the attribute's context; callers must not free it. Getters require an -// attribute accepted by the corresponding IsA predicate. - -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteIntTupleAttrStaticGet(MlirContext context, int64_t value); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleAttrDynamicGet( - MlirContext context, int32_t width, int64_t divisibility); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleAttrTupleGet( - MlirContext context, intptr_t numElements, MlirAttribute const *elements); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteIntTuple(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirCuteIntTupleAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrStaticGet(MlirContext context, - int64_t value); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrDynamicGet( - MlirContext context, int32_t width, int64_t divisibility); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCoordAttrUnderscoreGet(MlirContext context); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordAttrTupleGet( - MlirContext context, intptr_t numElements, MlirAttribute const *elements); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCoord(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteCoordAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrStaticGet(MlirContext context, - int64_t value); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrDynamicGet( - MlirContext context, int32_t width, int64_t divisibility); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeAttrTupleGet( - MlirContext context, intptr_t numElements, MlirAttribute const *elements); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteShapeAttrGetValue(MlirAttribute attr); -MLIR_CAPI_EXPORTED intptr_t mlirCuteShapeAttrGetRank(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteStrideAttrStaticGet(MlirContext context, int64_t value); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrDynamicGet( - MlirContext context, int32_t width, int64_t divisibility); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrBasisGet( - MlirContext context, int64_t numerator, int64_t denominator, - intptr_t numModes, int32_t const *modes); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideAttrTupleGet( - MlirContext context, intptr_t numElements, MlirAttribute const *elements); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteStride(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteStrideAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutAttrGet(MlirContext context, - MlirAttribute shape, - MlirAttribute stride); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteLayoutAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTileAttrLayoutGet(MlirContext context, MlirAttribute layout); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTileAttrUnderscoreGet(MlirContext context); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileAttrTupleGet( - MlirContext context, intptr_t numElements, MlirAttribute const *elements); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteTile(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirCuteTileAttrGetValue(MlirAttribute attr); - -// `inner` is a swizzle or a layout attribute. -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteComposedLayoutAttrGet(MlirContext context, MlirAttribute inner, - MlirAttribute offset, MlirAttribute outer); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteComposedLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirCuteComposedLayoutAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleAttrGet(MlirContext context, - uint32_t bits, - uint32_t base, - int32_t shift); -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSwizzle(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirCuteSwizzleAttrGetValue(MlirAttribute attr); - -// Compiler attributes - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles, and std::optional enums a -// negative value. Enums are their integer values (the dialect's *Enums.td). -// Getters require a value accepted by the matching IsA. - -// `#cute.bitlayout`: Bit layout of a pointer's elements within a storage chunk. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteBitLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteBitLayoutAttrGet(MlirContext context, - MlirAttribute layout); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteBitLayoutAttrGetLayout(MlirAttribute attr); - -// `#cute.copy_atom`: Thread and value layouts of a copy atom. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtom(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomAttrGet( - MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrc, - MlirAttribute layoutDst, MlirAttribute layoutRef); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomAttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomAttrGetLayoutSrc(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomAttrGetLayoutDst(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomAttrGetLayoutRef(MlirAttribute attr); - -// `#cute.copy_atom_v2`: Thread and value layouts and fragments of a V2 copy -// atom. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteCopyAtomV2(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCopyAtomV2AttrGet( - MlirContext context, MlirAttribute thrId, MlirAttribute layoutSrcTV, - MlirAttribute layoutDstTV, MlirAttribute frgSrc, MlirAttribute frgDst); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomV2AttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomV2AttrGetLayoutSrcTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomV2AttrGetLayoutDstTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomV2AttrGetFrgSrc(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteCopyAtomV2AttrGetFrgDst(MlirAttribute attr); - -// `#cute.mma_atom`: Shapes, thread-value layouts and fragments of an MMA atom. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMmaAtom(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, - MlirAttribute shapeAMK, MlirAttribute shapeBNK, MlirAttribute shapeCMN, - MlirAttribute frgA, MlirAttribute frgB, MlirAttribute frgC); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetShapeMNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetLayoutATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetLayoutBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetLayoutCTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetShapeAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetShapeBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMmaAtomAttrGetShapeCMN(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMmaAtomAttrGetFrgC(MlirAttribute attr); - -// `#cute.mx_mma_atom`: MMA atom attribute with scale-factor operands. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteMxMmaAtom(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteMxMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, - MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute shapeAMK, - MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, MlirAttribute shapeSFBNK, - MlirAttribute shapeCMN, MlirAttribute frgA, MlirAttribute frgSFA, - MlirAttribute frgB, MlirAttribute frgSFB, MlirAttribute frgC); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetFrgA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetFrgB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteMxMmaAtomAttrGetFrgC(MlirAttribute attr); - -// `#cute.reduction_op`: Op for cute reduce operations. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteReductionOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteReductionOpAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteReductionOpAttrGetValue(MlirAttribute attr); - -// `#cute.sparse_mma_atom`: MMA atom attribute with a sparsity-metadata operand. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMmaAtom(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutBTV, MlirAttribute layoutCTV, - MlirAttribute layoutETV, MlirAttribute shapeAMK, MlirAttribute shapeBNK, - MlirAttribute shapeCMN, MlirAttribute shapeEMK, MlirAttribute frgA, - MlirAttribute frgB, MlirAttribute frgC, MlirAttribute frgE); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetShapeMNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetLayoutATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetLayoutBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetLayoutCTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetLayoutETV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetShapeAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetShapeBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetShapeCMN(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetShapeEMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetFrgA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetFrgB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetFrgC(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMmaAtomAttrGetFrgE(MlirAttribute attr); - -// `#cute.sparse_mx_mma_atom`: MMA atom attribute with scale-factor and metadata -// operands. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteSparseMxMmaAtom(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSparseMxMmaAtomAttrGet( - MlirContext context, MlirAttribute shapeMNK, MlirAttribute thrId, - MlirAttribute layoutATV, MlirAttribute layoutSFATV, MlirAttribute layoutBTV, - MlirAttribute layoutSFBTV, MlirAttribute layoutCTV, MlirAttribute layoutETV, - MlirAttribute shapeAMK, MlirAttribute shapeSFAMK, MlirAttribute shapeBNK, - MlirAttribute shapeSFBNK, MlirAttribute shapeCMN, MlirAttribute shapeEMK, - MlirAttribute frgA, MlirAttribute frgSFA, MlirAttribute frgB, - MlirAttribute frgSFB, MlirAttribute frgC, MlirAttribute frgE); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeMNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetThrId(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetLayoutETV(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeCMN(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetShapeEMK(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgSFA(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgSFB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgC(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteSparseMxMmaAtomAttrGetFrgE(MlirAttribute attr); - -// Types - -/// Returns true when `type` belongs to the CuTe dialect. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteType(MlirType type); - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles, and std::optional enums a -// negative value. Enums are their integer values (the dialect's *Enums.td). -// Getters require a value accepted by the matching IsA. - -// `!cute.arith_tuple_iter`: Iterator over an arithmetic tuple. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteArithTupleIterator(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteArithTupleIteratorTypeGet(MlirContext context, MlirType arithTuple); -MLIR_CAPI_EXPORTED MlirType -mlirCuteArithTupleIteratorTypeGetArithTuple(MlirType type); - -// `!cute.composed_layout`: CuTe composed layout: A ∘ offset ∘ B. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteComposedLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteComposedLayoutTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteComposedLayoutTypeGetAttr(MlirType type); - -// `!cute.i`: Integer of any width with known divisibility and power of two. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteConstrainedInt(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteConstrainedIntTypeGet(MlirContext context, - int64_t divisibility, - uint32_t width, - bool isPow2); -MLIR_CAPI_EXPORTED int64_t -mlirCuteConstrainedIntTypeGetDivisibility(MlirType type); -MLIR_CAPI_EXPORTED uint32_t mlirCuteConstrainedIntTypeGetWidth(MlirType type); -MLIR_CAPI_EXPORTED bool mlirCuteConstrainedIntTypeGetIsPow2(MlirType type); - -// `!cute.coord_tensor`: Coordinate tensor: an arithmetic tuple with a layout. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoordTensor(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGet(MlirContext context, - MlirType arithTuple, - MlirType layout); -MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetArithTuple(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTensorTypeGetLayout(MlirType type); - -// `!cute.coord`: Scalar integer, underscore wildcard, or recursive tuple of -// coordinates. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteCoord(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteCoordTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteCoordTypeGetAttr(MlirType type); - -// `!cute.fast_divmod_divisor`: Precomputed divisor of the fast divmod -// operations. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteFastDivmodDivisor(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteFastDivmodDivisorTypeGet( - MlirContext context, uint32_t width, bool isPow2); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteFastDivmodDivisorTypeGetWidth(MlirType type); -MLIR_CAPI_EXPORTED bool mlirCuteFastDivmodDivisorTypeGetIsPow2(MlirType type); - -// `!cute.int_tuple`: Scalar integer or recursive tuple of integers. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteIntTuple(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteIntTupleTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteIntTupleTypeGetAttr(MlirType type); - -// `!cute.layout`: CuTe layout: a shape/stride pair. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteLayoutTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteLayoutTypeGetAttr(MlirType type); - -// `!cute.memref`: Tensor view: a pointer with a layout. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteMemRef(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGet(MlirContext context, - MlirType ptr, - MlirType layout); -MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetPtr(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteMemRefTypeGetLayout(MlirType type); - -// `!cute.ptr`: Pointer with address space, alignment, swizzle and bit layout. -MLIR_CAPI_EXPORTED bool mlirTypeIsACutePtr(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGet( - MlirContext context, MlirType valueType, uint32_t addressSpace, - uint64_t alignment, MlirAttribute swizzle, MlirAttribute bitlayout); -MLIR_CAPI_EXPORTED MlirType mlirCutePtrTypeGetValueType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t mlirCutePtrTypeGetAddressSpace(MlirType type); -MLIR_CAPI_EXPORTED uint64_t mlirCutePtrTypeGetAlignment(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetSwizzle(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirCutePtrTypeGetBitlayout(MlirType type); - -// `!cute.shape`: Scalar integer or recursive tuple of shape extents. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteShape(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteShapeTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteShapeTypeGetAttr(MlirType type); - -// `!cute.sparse_elem`: Logical sparse element stored in a physical type. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSparseElem(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteSparseElemTypeGet(MlirContext context, - int numLogical, - MlirType physicalType); -MLIR_CAPI_EXPORTED int mlirCuteSparseElemTypeGetNumLogical(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteSparseElemTypeGetPhysicalType(MlirType type); - -// `!cute.stride`: Scalar or recursive tuple of stride elements. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteStride(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteStrideTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteStrideTypeGetAttr(MlirType type); - -// `!cute.swizzle`: CuTe swizzle: S. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteSwizzle(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteSwizzleTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteSwizzleTypeGetAttr(MlirType type); - -// `!cute.tile`: CuTe tile: a recursive sequence of layouts and underscores. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTile(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteTileTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTileTypeGetAttr(MlirType type); - -// `!cute.tiled_copy`: Copy atom tiled over threads and values. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopy(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGet(MlirContext context, - MlirType copyAtom, - MlirAttribute layoutCopyTv, - MlirAttribute tilerMn); -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetCopyAtom(MlirType type); -// Former name of mlirCuteTiledCopyTypeGetCopyAtom. -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyTypeGetAtom(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledCopyTypeGetLayoutCopyTv(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteTiledCopyTypeGetTilerMn(MlirType type); - -// `!cute.tiled_copy_v2`: V2 copy atom tiled over threads and values. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledCopyV2(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteTiledCopyV2TypeGet(MlirContext context, MlirType copyAtom, - MlirAttribute layoutCopyTv, MlirAttribute tilerMn); -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledCopyV2TypeGetCopyAtom(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledCopyV2TypeGetLayoutCopyTv(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledCopyV2TypeGetTilerMn(MlirType type); - -// `!cute.tiled_mma`: MMA atom tiled over MNK. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTiledMma(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGet( - MlirContext context, MlirType mmaAtom, MlirAttribute atomLayoutMNK, - MlirAttribute permutationMNK); -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetMmaAtom(MlirType type); -// Former name of mlirCuteTiledMmaTypeGetMmaAtom. -MLIR_CAPI_EXPORTED MlirType mlirCuteTiledMmaTypeGetAtom(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledMmaTypeGetAtomLayoutMNK(MlirType type); -// Former name of mlirCuteTiledMmaTypeGetAtomLayoutMNK. -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledMmaTypeGetAtomLayoutMnk(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledMmaTypeGetPermutationMNK(MlirType type); -// Former name of mlirCuteTiledMmaTypeGetPermutationMNK. -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteTiledMmaTypeGetPermutationMnk(MlirType type); - -// `!cute.tuple`: Non-empty tuple of types. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteTuple(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGet(MlirContext context, - intptr_t numTypes, - MlirType const *types); -MLIR_CAPI_EXPORTED intptr_t mlirCuteTupleTypeGetNumTypes(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteTupleTypeGetType(MlirType type, - intptr_t pos); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_CUTE_IR_CAPI_H diff --git a/mlir/dialects/cute_ir/cute_ir.zig b/mlir/dialects/cute_ir/cute_ir.zig deleted file mode 100644 index b277d061..00000000 --- a/mlir/dialects/cute_ir/cute_ir.zig +++ /dev/null @@ -1,2844 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -pub const CuteType = opaque { - const M = mlir.Methods(CuteType, c.MlirType); - - pub const isAFn = c.mlirTypeIsACuteType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; -}; - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("cute"); -} - -/// `cute.kernel`: marks a function as a CuTe kernel. -pub fn kernel(ctx: *mlir.Context) mlir.NamedAttribute { - return .named(ctx, "cute.kernel", .unit(ctx)); -} - -/// The address space of a pointer or memref. -pub const MemorySpace = AddressSpace; - -pub fn pointerType( - ctx: *mlir.Context, - space: MemorySpace, - alignment: u64, - opts: struct { - element_type: ?*const mlir.Type = null, - }, -) !*const mlir.Type { - return (try PtrType.get(ctx, .{ - .valueType = opts.element_type, - .addressSpace = space, - .alignment = alignment, - })).type_(); -} - -pub fn constrainedIntType(ctx: *mlir.Context, width: enum { i32, i64 }, divisible_by: u64) !*const mlir.Type { - return (try ConstrainedIntType.get(ctx, .{ - .divisibility = std.math.cast(i64, divisible_by) orelse return error.InvalidMlir, - .width = switch (width) { - .i32 => 32, - .i64 => 64, - }, - .isPow2 = false, - })).type_(); -} - -/// Algebra attributes are built bottom up from leaves and tuples of the same -/// kind. A dynamic leaf has `width` bits (1, 8, 16, 32 or 64) and is known to -/// be a multiple of `divisibility`. Getter strings are owned by the MLIR -/// context and remain valid until that context is destroyed. -pub const IntTupleAttr = opaque { - const M = mlir.Methods(IntTupleAttr, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsACuteIntTuple; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const IntTupleAttr { - return @ptrCast(c.mlirCuteIntTupleAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); - } - - pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const IntTupleAttr { - return @ptrCast(c.mlirCuteIntTupleAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); - } - - pub fn getTuple(ctx: *mlir.Context, elements: []const *const IntTupleAttr) mlir.Error!*const IntTupleAttr { - return @ptrCast(c.mlirCuteIntTupleAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const IntTupleAttr) []const u8 { - return mlir.string(c.mlirCuteIntTupleAttrGetValue(self.ptr())); - } - - pub fn attribute(self: *const IntTupleAttr) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -pub const CoordAttr = opaque { - const M = mlir.Methods(CoordAttr, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsACuteCoord; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const CoordAttr { - return @ptrCast(c.mlirCuteCoordAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); - } - - pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const CoordAttr { - return @ptrCast(c.mlirCuteCoordAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); - } - - pub fn getUnderscore(ctx: *mlir.Context) mlir.Error!*const CoordAttr { - return @ptrCast(c.mlirCuteCoordAttrUnderscoreGet(ctx.ptr()).ptr orelse return error.InvalidMlir); - } - - pub fn getTuple(ctx: *mlir.Context, elements: []const *const CoordAttr) mlir.Error!*const CoordAttr { - return @ptrCast(c.mlirCuteCoordAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const CoordAttr) []const u8 { - return mlir.string(c.mlirCuteCoordAttrGetValue(self.ptr())); - } - - pub fn attribute(self: *const CoordAttr) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -pub const ShapeAttr = opaque { - const M = mlir.Methods(ShapeAttr, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsACuteShape; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const ShapeAttr { - return @ptrCast(c.mlirCuteShapeAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); - } - - pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const ShapeAttr { - return @ptrCast(c.mlirCuteShapeAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); - } - - pub fn getTuple(ctx: *mlir.Context, elements: []const *const ShapeAttr) mlir.Error!*const ShapeAttr { - return @ptrCast(c.mlirCuteShapeAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const ShapeAttr) []const u8 { - return mlir.string(c.mlirCuteShapeAttrGetValue(self.ptr())); - } - - pub fn rank(self: *const ShapeAttr) usize { - return @intCast(c.mlirCuteShapeAttrGetRank(self.ptr())); - } - - pub fn attribute(self: *const ShapeAttr) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -pub const StrideAttr = opaque { - const M = mlir.Methods(StrideAttr, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsACuteStride; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn getStatic(ctx: *mlir.Context, value_: i64) mlir.Error!*const StrideAttr { - return @ptrCast(c.mlirCuteStrideAttrStaticGet(ctx.ptr(), value_).ptr orelse return error.InvalidMlir); - } - - pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i64) mlir.Error!*const StrideAttr { - return @ptrCast(c.mlirCuteStrideAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); - } - - /// `numerator/denominator` along `modes`, the mode path outermost first; - /// CuTe prints it innermost first, so `1@1@0` is modes `{ 0, 1 }`. - pub fn getBasis(ctx: *mlir.Context, numerator: i64, denominator: i64, modes: []const i32) mlir.Error!*const StrideAttr { - return @ptrCast(c.mlirCuteStrideAttrBasisGet(ctx.ptr(), numerator, denominator, @intCast(modes.len), modes.ptr).ptr orelse return error.InvalidMlir); - } - - pub fn getTuple(ctx: *mlir.Context, elements: []const *const StrideAttr) mlir.Error!*const StrideAttr { - return @ptrCast(c.mlirCuteStrideAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const StrideAttr) []const u8 { - return mlir.string(c.mlirCuteStrideAttrGetValue(self.ptr())); - } - - pub fn attribute(self: *const StrideAttr) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -pub const LayoutAttr = opaque { - const M = mlir.Methods(LayoutAttr, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsACuteLayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - shape: *const ShapeAttr, - stride: *const StrideAttr, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const LayoutAttr { - const result = c.mlirCuteLayoutAttrGet(ctx.ptr(), args.shape.ptr(), args.stride.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const LayoutAttr) []const u8 { - return mlir.string(c.mlirCuteLayoutAttrGetValue(self.ptr())); - } - - pub fn attribute(self: *const LayoutAttr) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -pub const TileAttr = opaque { - const M = mlir.Methods(TileAttr, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsACuteTile; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn getLayout(ctx: *mlir.Context, layout: *const LayoutAttr) mlir.Error!*const TileAttr { - return @ptrCast(c.mlirCuteTileAttrLayoutGet(ctx.ptr(), layout.ptr()).ptr orelse return error.InvalidMlir); - } - - pub fn getUnderscore(ctx: *mlir.Context) mlir.Error!*const TileAttr { - return @ptrCast(c.mlirCuteTileAttrUnderscoreGet(ctx.ptr()).ptr orelse return error.InvalidMlir); - } - - pub fn getTuple(ctx: *mlir.Context, elements: []const *const TileAttr) mlir.Error!*const TileAttr { - return @ptrCast(c.mlirCuteTileAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const TileAttr) []const u8 { - return mlir.string(c.mlirCuteTileAttrGetValue(self.ptr())); - } - - pub fn attribute(self: *const TileAttr) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -pub const ComposedLayoutAttr = opaque { - const M = mlir.Methods(ComposedLayoutAttr, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsACuteComposedLayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - /// `inner ∘ offset ∘ outer`. - pub const InitArgs = struct { - inner: union(enum) { - swizzle: *const SwizzleAttr, - layout: *const LayoutAttr, - }, - offset: *const IntTupleAttr, - outer: *const LayoutAttr, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const ComposedLayoutAttr { - const inner = switch (args.inner) { - inline else => |attr| attr.ptr(), - }; - const result = c.mlirCuteComposedLayoutAttrGet(ctx.ptr(), inner, args.offset.ptr(), args.outer.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const ComposedLayoutAttr) []const u8 { - return mlir.string(c.mlirCuteComposedLayoutAttrGetValue(self.ptr())); - } - - pub fn attribute(self: *const ComposedLayoutAttr) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -pub const SwizzleAttr = opaque { - const M = mlir.Methods(SwizzleAttr, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsACuteSwizzle; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - /// `S`. - pub const InitArgs = struct { - bits: u32, - base: u32, - shift: i32, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const SwizzleAttr { - const result = c.mlirCuteSwizzleAttrGet(ctx.ptr(), args.bits, args.base, args.shift); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const SwizzleAttr) []const u8 { - return mlir.string(c.mlirCuteSwizzleAttrGetValue(self.ptr())); - } - - pub fn attribute(self: *const SwizzleAttr) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -// Enums, generated from the dialect's .td files; values are the compiler's. - -/// Address spaces for CuTe memrefs and pointers. -pub const AddressSpace = enum(u32) { - generic = 0, - gmem = 1, - smem = 3, - rmem = 5, - tmem = 6, - dsmem = 7, - cmem = 4, -}; - -/// Copy operand ID. -pub const CopyOperand = enum(u32) { - S = 0, - D = 1, -}; - -/// Major mode for MMA operations. -pub const MajorMode = enum(u32) { - k = 0, - mn = 1, -}; - -/// MMA operand ID. -pub const MmaOperand = enum(u32) { - A = 0, - B = 1, - C = 2, - D = 3, - SFA = 4, - SFB = 5, - E = 6, -}; - -/// Op for cute reduce operations. -pub const ReductionOp = enum(u32) { - ADD = 0, - MUL = 1, - MIN = 2, - MAX = 3, -}; - -// Types and attributes, generated from the dialect's .td files. - -/// `!cute.arith_tuple_iter`: Iterator over an arithmetic tuple. -pub const ArithTupleIteratorType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteArithTupleIterator; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "arith_tuple_iter"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - arithTuple: *const mlir.Type, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteArithTupleIteratorTypeGet(ctx.ptr(), args.arithTuple.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn arithTuple(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteArithTupleIteratorTypeGetArithTuple(self.ptr()).ptr.?); - } -}; - -/// `!cute.composed_layout`: CuTe composed layout: A ∘ offset ∘ B. -pub const ComposedLayoutType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteComposedLayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "composed_layout"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - attr: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteComposedLayoutTypeGet(ctx.ptr(), args.attr.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn attr(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteComposedLayoutTypeGetAttr(self.ptr()).ptr.?); - } -}; - -/// `!cute.i`: Integer of any width with known divisibility and power of two. -pub const ConstrainedIntType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteConstrainedInt; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - divisibility: i64, - width: u32, - isPow2: bool, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteConstrainedIntTypeGet(ctx.ptr(), args.divisibility, args.width, args.isPow2); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn divisibility(self: *const Self) i64 { - return c.mlirCuteConstrainedIntTypeGetDivisibility(self.ptr()); - } - pub fn width(self: *const Self) u32 { - return c.mlirCuteConstrainedIntTypeGetWidth(self.ptr()); - } - pub fn isPow2(self: *const Self) bool { - return c.mlirCuteConstrainedIntTypeGetIsPow2(self.ptr()); - } -}; - -/// `!cute.coord_tensor`: Coordinate tensor: an arithmetic tuple with a layout. -pub const CoordTensorType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteCoordTensor; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "coord_tensor"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - arithTuple: *const mlir.Type, - layout: *const mlir.Type, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteCoordTensorTypeGet(ctx.ptr(), args.arithTuple.ptr(), args.layout.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn arithTuple(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteCoordTensorTypeGetArithTuple(self.ptr()).ptr.?); - } - pub fn layout(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteCoordTensorTypeGetLayout(self.ptr()).ptr.?); - } -}; - -/// `!cute.coord`: Scalar integer, underscore wildcard, or recursive tuple of coordinates. -pub const CoordType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteCoord; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "coord"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - attr: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteCoordTypeGet(ctx.ptr(), args.attr.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn attr(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteCoordTypeGetAttr(self.ptr()).ptr.?); - } -}; - -/// `!cute.fast_divmod_divisor`: Precomputed divisor of the fast divmod operations. -pub const FastDivmodDivisorType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteFastDivmodDivisor; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "fast_divmod_divisor"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - width: u32, - isPow2: bool, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteFastDivmodDivisorTypeGet(ctx.ptr(), args.width, args.isPow2); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn width(self: *const Self) u32 { - return c.mlirCuteFastDivmodDivisorTypeGetWidth(self.ptr()); - } - pub fn isPow2(self: *const Self) bool { - return c.mlirCuteFastDivmodDivisorTypeGetIsPow2(self.ptr()); - } -}; - -/// `!cute.int_tuple`: Scalar integer or recursive tuple of integers. -pub const IntTupleType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteIntTuple; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "int_tuple"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - attr: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteIntTupleTypeGet(ctx.ptr(), args.attr.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn attr(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteIntTupleTypeGetAttr(self.ptr()).ptr.?); - } -}; - -/// `!cute.layout`: CuTe layout: a shape/stride pair. -pub const LayoutType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteLayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "layout"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - attr: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteLayoutTypeGet(ctx.ptr(), args.attr.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn attr(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteLayoutTypeGetAttr(self.ptr()).ptr.?); - } -}; - -/// `!cute.memref`: Tensor view: a pointer with a layout. -pub const MemRefType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteMemRef; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "memref"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - ptr: *const mlir.Type, - layout: *const mlir.Type, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteMemRefTypeGet(ctx.ptr(), args.ptr.ptr(), args.layout.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn ptr_(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteMemRefTypeGetPtr(self.ptr()).ptr.?); - } - pub fn layout(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteMemRefTypeGetLayout(self.ptr()).ptr.?); - } -}; - -/// `!cute.ptr`: Pointer with address space, alignment, swizzle and bit layout. -pub const PtrType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACutePtr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "ptr"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valueType: ?*const mlir.Type = null, - addressSpace: AddressSpace, - alignment: u64, - swizzle: ?*const mlir.Attribute = null, - bitlayout: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCutePtrTypeGet(ctx.ptr(), if (args.valueType) |v| v.ptr() else c.MlirType{ .ptr = null }, @backingInt(args.addressSpace), args.alignment, if (args.swizzle) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.bitlayout) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valueType(self: *const Self) ?*const mlir.Type { - return @ptrCast(c.mlirCutePtrTypeGetValueType(self.ptr()).ptr); - } - pub fn addressSpace(self: *const Self) AddressSpace { - return @fromBackingInt(@intCast(c.mlirCutePtrTypeGetAddressSpace(self.ptr()))); - } - pub fn alignment(self: *const Self) u64 { - return c.mlirCutePtrTypeGetAlignment(self.ptr()); - } - pub fn swizzle(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCutePtrTypeGetSwizzle(self.ptr()).ptr); - } - pub fn bitlayout(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCutePtrTypeGetBitlayout(self.ptr()).ptr); - } -}; - -/// `!cute.shape`: Scalar integer or recursive tuple of shape extents. -pub const ShapeType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteShape; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "shape"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - attr: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteShapeTypeGet(ctx.ptr(), args.attr.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn attr(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteShapeTypeGetAttr(self.ptr()).ptr.?); - } -}; - -/// `!cute.sparse_elem`: Logical sparse element stored in a physical type. -pub const SparseElemType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteSparseElem; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sparse_elem"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - numLogical: c_int, - physicalType: *const mlir.Type, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteSparseElemTypeGet(ctx.ptr(), args.numLogical, args.physicalType.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn numLogical(self: *const Self) c_int { - return c.mlirCuteSparseElemTypeGetNumLogical(self.ptr()); - } - pub fn physicalType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteSparseElemTypeGetPhysicalType(self.ptr()).ptr.?); - } -}; - -/// `!cute.stride`: Scalar or recursive tuple of stride elements. -pub const StrideType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteStride; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "stride"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - attr: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteStrideTypeGet(ctx.ptr(), args.attr.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn attr(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteStrideTypeGetAttr(self.ptr()).ptr.?); - } -}; - -/// `!cute.swizzle`: CuTe swizzle: S. -pub const SwizzleType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteSwizzle; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "swizzle"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - attr: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteSwizzleTypeGet(ctx.ptr(), args.attr.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn attr(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSwizzleTypeGetAttr(self.ptr()).ptr.?); - } -}; - -/// `!cute.tile`: CuTe tile: a recursive sequence of layouts and underscores. -pub const TileType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteTile; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "tile"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - attr: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteTileTypeGet(ctx.ptr(), args.attr.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn attr(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteTileTypeGetAttr(self.ptr()).ptr.?); - } -}; - -/// `!cute.tiled_copy`: Copy atom tiled over threads and values. -pub const TiledCopyType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteTiledCopy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "tiled_copy"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - copyAtom: *const mlir.Type, - layoutCopyTv: *const mlir.Attribute, - tilerMn: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteTiledCopyTypeGet(ctx.ptr(), args.copyAtom.ptr(), args.layoutCopyTv.ptr(), args.tilerMn.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn copyAtom(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteTiledCopyTypeGetCopyAtom(self.ptr()).ptr.?); - } - pub fn layoutCopyTv(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteTiledCopyTypeGetLayoutCopyTv(self.ptr()).ptr.?); - } - pub fn tilerMn(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteTiledCopyTypeGetTilerMn(self.ptr()).ptr.?); - } - pub const atom = copyAtom; -}; - -/// `!cute.tiled_copy_v2`: V2 copy atom tiled over threads and values. -pub const TiledCopyV2Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteTiledCopyV2; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "tiled_copy_v2"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - copyAtom: *const mlir.Type, - layoutCopyTv: *const mlir.Attribute, - tilerMn: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteTiledCopyV2TypeGet(ctx.ptr(), args.copyAtom.ptr(), args.layoutCopyTv.ptr(), args.tilerMn.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn copyAtom(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteTiledCopyV2TypeGetCopyAtom(self.ptr()).ptr.?); - } - pub fn layoutCopyTv(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteTiledCopyV2TypeGetLayoutCopyTv(self.ptr()).ptr.?); - } - pub fn tilerMn(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteTiledCopyV2TypeGetTilerMn(self.ptr()).ptr.?); - } -}; - -/// `!cute.tiled_mma`: MMA atom tiled over MNK. -pub const TiledMmaType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteTiledMma; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "tiled_mma"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - mmaAtom: *const mlir.Type, - atomLayoutMNK: *const mlir.Attribute, - permutationMNK: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteTiledMmaTypeGet(ctx.ptr(), args.mmaAtom.ptr(), args.atomLayoutMNK.ptr(), if (args.permutationMNK) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn mmaAtom(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteTiledMmaTypeGetMmaAtom(self.ptr()).ptr.?); - } - pub fn atomLayoutMNK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteTiledMmaTypeGetAtomLayoutMNK(self.ptr()).ptr.?); - } - pub fn permutationMNK(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteTiledMmaTypeGetPermutationMNK(self.ptr()).ptr); - } - pub const atom = mmaAtom; - pub const getAtomLayoutMnk = atomLayoutMNK; - pub const getPermutationMnk = permutationMNK; -}; - -/// `!cute.tuple`: Non-empty tuple of types. -pub const TupleType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteTuple; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "tuple"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - types: []const *const mlir.Type, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteTupleTypeGet(ctx.ptr(), @intCast(args.types.len), @ptrCast(args.types.ptr)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn numTypes(self: *const Self) usize { - return @intCast(c.mlirCuteTupleTypeGetNumTypes(self.ptr())); - } - pub fn @"type"(self: *const Self, pos: usize) *const mlir.Type { - return @ptrCast(c.mlirCuteTupleTypeGetType(self.ptr(), @intCast(pos)).ptr.?); - } -}; - -/// `#cute.bitlayout`: Bit layout of a pointer's elements within a storage chunk. -pub const BitLayoutAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteBitLayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "bitlayout"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - layout: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteBitLayoutAttrGet(ctx.ptr(), args.layout.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn layout(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteBitLayoutAttrGetLayout(self.ptr()).ptr.?); - } -}; - -/// `#cute.copy_atom`: Thread and value layouts of a copy atom. -pub const CopyAtomAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteCopyAtom; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "copy_atom"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - thrId: *const mlir.Attribute, - layoutSrc: *const mlir.Attribute, - layoutDst: *const mlir.Attribute, - layoutRef: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteCopyAtomAttrGet(ctx.ptr(), args.thrId.ptr(), args.layoutSrc.ptr(), args.layoutDst.ptr(), args.layoutRef.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn thrId(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteCopyAtomAttrGetThrId(self.ptr()).ptr.?); - } - pub fn layoutSrc(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutSrc(self.ptr()).ptr.?); - } - pub fn layoutDst(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutDst(self.ptr()).ptr.?); - } - pub fn layoutRef(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteCopyAtomAttrGetLayoutRef(self.ptr()).ptr.?); - } -}; - -/// `#cute.copy_atom_v2`: Thread and value layouts and fragments of a V2 copy atom. -pub const CopyAtomV2Attr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteCopyAtomV2; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "copy_atom_v2"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - thrId: *const mlir.Attribute, - layoutSrcTV: *const mlir.Attribute, - layoutDstTV: *const mlir.Attribute, - frgSrc: *const mlir.Attribute, - frgDst: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteCopyAtomV2AttrGet(ctx.ptr(), args.thrId.ptr(), args.layoutSrcTV.ptr(), args.layoutDstTV.ptr(), args.frgSrc.ptr(), args.frgDst.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn thrId(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteCopyAtomV2AttrGetThrId(self.ptr()).ptr.?); - } - pub fn layoutSrcTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteCopyAtomV2AttrGetLayoutSrcTV(self.ptr()).ptr.?); - } - pub fn layoutDstTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteCopyAtomV2AttrGetLayoutDstTV(self.ptr()).ptr.?); - } - pub fn frgSrc(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteCopyAtomV2AttrGetFrgSrc(self.ptr()).ptr.?); - } - pub fn frgDst(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteCopyAtomV2AttrGetFrgDst(self.ptr()).ptr.?); - } -}; - -/// `#cute.mma_atom`: Shapes, thread-value layouts and fragments of an MMA atom. -pub const MmaAtomAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteMmaAtom; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mma_atom"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMNK: *const mlir.Attribute, - thrId: *const mlir.Attribute, - layoutATV: *const mlir.Attribute, - layoutBTV: *const mlir.Attribute, - layoutCTV: *const mlir.Attribute, - shapeAMK: *const mlir.Attribute, - shapeBNK: *const mlir.Attribute, - shapeCMN: *const mlir.Attribute, - frgA: *const mlir.Attribute, - frgB: *const mlir.Attribute, - frgC: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutBTV.ptr(), args.layoutCTV.ptr(), args.shapeAMK.ptr(), args.shapeBNK.ptr(), args.shapeCMN.ptr(), args.frgA.ptr(), args.frgB.ptr(), args.frgC.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMNK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); - } - pub fn thrId(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMmaAtomAttrGetThrId(self.ptr()).ptr.?); - } - pub fn layoutATV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); - } - pub fn layoutBTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); - } - pub fn layoutCTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); - } - pub fn shapeAMK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); - } - pub fn shapeBNK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); - } - pub fn shapeCMN(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); - } - pub fn frgA(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgA(self.ptr()).ptr.?); - } - pub fn frgB(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgB(self.ptr()).ptr.?); - } - pub fn frgC(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMmaAtomAttrGetFrgC(self.ptr()).ptr.?); - } -}; - -/// `#cute.mx_mma_atom`: MMA atom attribute with scale-factor operands. -pub const MxMmaAtomAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteMxMmaAtom; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mx_mma_atom"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMNK: *const mlir.Attribute, - thrId: *const mlir.Attribute, - layoutATV: *const mlir.Attribute, - layoutSFATV: *const mlir.Attribute, - layoutBTV: *const mlir.Attribute, - layoutSFBTV: *const mlir.Attribute, - layoutCTV: *const mlir.Attribute, - shapeAMK: *const mlir.Attribute, - shapeSFAMK: *const mlir.Attribute, - shapeBNK: *const mlir.Attribute, - shapeSFBNK: *const mlir.Attribute, - shapeCMN: *const mlir.Attribute, - frgA: *const mlir.Attribute, - frgSFA: *const mlir.Attribute, - frgB: *const mlir.Attribute, - frgSFB: *const mlir.Attribute, - frgC: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteMxMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutSFATV.ptr(), args.layoutBTV.ptr(), args.layoutSFBTV.ptr(), args.layoutCTV.ptr(), args.shapeAMK.ptr(), args.shapeSFAMK.ptr(), args.shapeBNK.ptr(), args.shapeSFBNK.ptr(), args.shapeCMN.ptr(), args.frgA.ptr(), args.frgSFA.ptr(), args.frgB.ptr(), args.frgSFB.ptr(), args.frgC.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMNK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); - } - pub fn thrId(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetThrId(self.ptr()).ptr.?); - } - pub fn layoutATV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); - } - pub fn layoutSFATV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutSFATV(self.ptr()).ptr.?); - } - pub fn layoutBTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); - } - pub fn layoutSFBTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutSFBTV(self.ptr()).ptr.?); - } - pub fn layoutCTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); - } - pub fn shapeAMK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); - } - pub fn shapeSFAMK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeSFAMK(self.ptr()).ptr.?); - } - pub fn shapeBNK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); - } - pub fn shapeSFBNK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeSFBNK(self.ptr()).ptr.?); - } - pub fn shapeCMN(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); - } - pub fn frgA(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgA(self.ptr()).ptr.?); - } - pub fn frgSFA(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgSFA(self.ptr()).ptr.?); - } - pub fn frgB(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgB(self.ptr()).ptr.?); - } - pub fn frgSFB(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgSFB(self.ptr()).ptr.?); - } - pub fn frgC(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteMxMmaAtomAttrGetFrgC(self.ptr()).ptr.?); - } -}; - -/// `#cute.reduction_op`: Op for cute reduce operations. -pub const ReductionOpAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteReductionOp; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "reduction_op"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ReductionOp, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ReductionOp { - return @fromBackingInt(@intCast(c.mlirCuteReductionOpAttrGetValue(self.ptr()))); - } -}; - -/// `#cute.sparse_mma_atom`: MMA atom attribute with a sparsity-metadata operand. -pub const SparseMmaAtomAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteSparseMmaAtom; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "sparse_mma_atom"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMNK: *const mlir.Attribute, - thrId: *const mlir.Attribute, - layoutATV: *const mlir.Attribute, - layoutBTV: *const mlir.Attribute, - layoutCTV: *const mlir.Attribute, - layoutETV: *const mlir.Attribute, - shapeAMK: *const mlir.Attribute, - shapeBNK: *const mlir.Attribute, - shapeCMN: *const mlir.Attribute, - shapeEMK: *const mlir.Attribute, - frgA: *const mlir.Attribute, - frgB: *const mlir.Attribute, - frgC: *const mlir.Attribute, - frgE: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteSparseMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutBTV.ptr(), args.layoutCTV.ptr(), args.layoutETV.ptr(), args.shapeAMK.ptr(), args.shapeBNK.ptr(), args.shapeCMN.ptr(), args.shapeEMK.ptr(), args.frgA.ptr(), args.frgB.ptr(), args.frgC.ptr(), args.frgE.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMNK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); - } - pub fn thrId(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetThrId(self.ptr()).ptr.?); - } - pub fn layoutATV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); - } - pub fn layoutBTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); - } - pub fn layoutCTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); - } - pub fn layoutETV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetLayoutETV(self.ptr()).ptr.?); - } - pub fn shapeAMK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); - } - pub fn shapeBNK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); - } - pub fn shapeCMN(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); - } - pub fn shapeEMK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetShapeEMK(self.ptr()).ptr.?); - } - pub fn frgA(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgA(self.ptr()).ptr.?); - } - pub fn frgB(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgB(self.ptr()).ptr.?); - } - pub fn frgC(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgC(self.ptr()).ptr.?); - } - pub fn frgE(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMmaAtomAttrGetFrgE(self.ptr()).ptr.?); - } -}; - -/// `#cute.sparse_mx_mma_atom`: MMA atom attribute with scale-factor and metadata operands. -pub const SparseMxMmaAtomAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteSparseMxMmaAtom; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "sparse_mx_mma_atom"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMNK: *const mlir.Attribute, - thrId: *const mlir.Attribute, - layoutATV: *const mlir.Attribute, - layoutSFATV: *const mlir.Attribute, - layoutBTV: *const mlir.Attribute, - layoutSFBTV: *const mlir.Attribute, - layoutCTV: *const mlir.Attribute, - layoutETV: *const mlir.Attribute, - shapeAMK: *const mlir.Attribute, - shapeSFAMK: *const mlir.Attribute, - shapeBNK: *const mlir.Attribute, - shapeSFBNK: *const mlir.Attribute, - shapeCMN: *const mlir.Attribute, - shapeEMK: *const mlir.Attribute, - frgA: *const mlir.Attribute, - frgSFA: *const mlir.Attribute, - frgB: *const mlir.Attribute, - frgSFB: *const mlir.Attribute, - frgC: *const mlir.Attribute, - frgE: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteSparseMxMmaAtomAttrGet(ctx.ptr(), args.shapeMNK.ptr(), args.thrId.ptr(), args.layoutATV.ptr(), args.layoutSFATV.ptr(), args.layoutBTV.ptr(), args.layoutSFBTV.ptr(), args.layoutCTV.ptr(), args.layoutETV.ptr(), args.shapeAMK.ptr(), args.shapeSFAMK.ptr(), args.shapeBNK.ptr(), args.shapeSFBNK.ptr(), args.shapeCMN.ptr(), args.shapeEMK.ptr(), args.frgA.ptr(), args.frgSFA.ptr(), args.frgB.ptr(), args.frgSFB.ptr(), args.frgC.ptr(), args.frgE.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMNK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeMNK(self.ptr()).ptr.?); - } - pub fn thrId(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetThrId(self.ptr()).ptr.?); - } - pub fn layoutATV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutATV(self.ptr()).ptr.?); - } - pub fn layoutSFATV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutSFATV(self.ptr()).ptr.?); - } - pub fn layoutBTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutBTV(self.ptr()).ptr.?); - } - pub fn layoutSFBTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutSFBTV(self.ptr()).ptr.?); - } - pub fn layoutCTV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutCTV(self.ptr()).ptr.?); - } - pub fn layoutETV(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetLayoutETV(self.ptr()).ptr.?); - } - pub fn shapeAMK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeAMK(self.ptr()).ptr.?); - } - pub fn shapeSFAMK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeSFAMK(self.ptr()).ptr.?); - } - pub fn shapeBNK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeBNK(self.ptr()).ptr.?); - } - pub fn shapeSFBNK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeSFBNK(self.ptr()).ptr.?); - } - pub fn shapeCMN(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeCMN(self.ptr()).ptr.?); - } - pub fn shapeEMK(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetShapeEMK(self.ptr()).ptr.?); - } - pub fn frgA(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgA(self.ptr()).ptr.?); - } - pub fn frgSFA(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgSFA(self.ptr()).ptr.?); - } - pub fn frgB(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgB(self.ptr()).ptr.?); - } - pub fn frgSFB(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgSFB(self.ptr()).ptr.?); - } - pub fn frgC(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgC(self.ptr()).ptr.?); - } - pub fn frgE(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteSparseMxMmaAtomAttrGetFrgE(self.ptr()).ptr.?); - } -}; - -// Operation builders, generated from the operations' .td. -/// `cute.add_offset`. Result types are explicit; attributes use mlir.Attribute. -pub fn add_offset(ctx: *mlir.Context, src: *const mlir.Value, offset: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.add_offset", .{ - .operands = .{ .flat = &.{ src, offset } }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute.append_to_rank`. Result types are explicit; attributes use mlir.Attribute. -pub fn append_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *const mlir.Value, result_type: *const mlir.Type, rank: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.append_to_rank", .{ - .operands = .{ .flat = &.{ input, element } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "rank", rank), - }, - .location = location, - }); -} - -/// `cute.apply_swizzle`. Result types are explicit; attributes use mlir.Attribute. -pub fn apply_swizzle( - ctx: *mlir.Context, - ptr: *const mlir.Value, - swizzled_ptr_type: *const mlir.Type, - opts: struct { - swizzle: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.swizzle) |value| attributes.appendAssumeCapacity(.named(ctx, "swizzle", value)); - return mlir.Operation.make(ctx, "cute.apply_swizzle", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{swizzled_ptr_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.assume`. Result types are explicit; attributes use mlir.Attribute. -pub fn assume(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.assume", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute.blocked_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn blocked_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.blocked_product", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.ceil_div`. Result types are explicit; attributes use mlir.Attribute. -pub fn ceil_div(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.ceil_div", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.coalesce`. Result types are explicit; attributes use mlir.Attribute. -pub fn coalesce( - ctx: *mlir.Context, - input: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - target_profile: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.coalesce", .{ - .operands = .{ .variadic = &.{ - &.{input}, - if (opts.target_profile) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.complement`. Result types are explicit; attributes use mlir.Attribute. -pub fn complement(ctx: *mlir.Context, input: *const mlir.Value, cotarget: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.complement", .{ - .operands = .{ .flat = &.{ input, cotarget } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `cute.composed_get_inner`. Result types are explicit; attributes use mlir.Attribute. -pub fn composed_get_inner(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.composed_get_inner", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.composed_get_offset`. Result types are explicit; attributes use mlir.Attribute. -pub fn composed_get_offset(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.composed_get_offset", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.composed_get_outer`. Result types are explicit; attributes use mlir.Attribute. -pub fn composed_get_outer(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.composed_get_outer", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.composition`. Result types are explicit; attributes use mlir.Attribute. -pub fn composition(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.composition", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `cute.copy`. Result types are explicit; attributes use mlir.Attribute. -pub fn copy( - ctx: *mlir.Context, - copy_atom: *const mlir.Value, - src: []const *const mlir.Value, - dst: []const *const mlir.Value, - opts: struct { - pred: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.copy", .{ - .operands = .{ .variadic = &.{ - &.{copy_atom}, - src, - dst, - if (opts.pred) |value| &.{value} else &.{}, - } }, - .location = location, - }); -} - -/// `cute.copy.make_fragment`. Result types are explicit; attributes use mlir.Attribute. -pub fn copy_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.copy.make_fragment", .{ - .operands = .{ .flat = &.{ atom, input } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operand_id", operand_id), - }, - .location = location, - }); -} - -/// `cute.copy_atom_call`. Result types are explicit; attributes use mlir.Attribute. -pub fn copy_atom_call( - ctx: *mlir.Context, - atom: *const mlir.Value, - src: []const *const mlir.Value, - dst: []const *const mlir.Value, - opts: struct { - pred: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.copy_atom_call", .{ - .operands = .{ .variadic = &.{ - &.{atom}, - src, - dst, - if (opts.pred) |value| &.{value} else &.{}, - } }, - .location = location, - }); -} - -/// `cute.cosize`. Result types are explicit; attributes use mlir.Attribute. -pub fn cosize( - ctx: *mlir.Context, - input: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - mode: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "cute.cosize", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.crd2idx`. Result types are explicit; attributes use mlir.Attribute. -pub fn crd2idx(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, index_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.crd2idx", .{ - .operands = .{ .flat = &.{ coord, layout } }, - .results = .{ .flat = &.{index_type} }, - .location = location, - }); -} - -/// `cute.deref_arith_tuple_iter`. Result types are explicit; attributes use mlir.Attribute. -pub fn deref_arith_tuple_iter(ctx: *mlir.Context, iter: *const mlir.Value, arith_tuple_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.deref_arith_tuple_iter", .{ - .operands = .{ .flat = &.{iter} }, - .results = .{ .flat = &.{arith_tuple_type} }, - .location = location, - }); -} - -/// `cute.deref_desc_iter`. Result types are explicit; attributes use mlir.Attribute. -pub fn deref_desc_iter(ctx: *mlir.Context, iter: *const mlir.Value, value__type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.deref_desc_iter", .{ - .operands = .{ .flat = &.{iter} }, - .results = .{ .flat = &.{value__type} }, - .location = location, - }); -} - -/// `cute.derefine`. Result types are explicit; attributes use mlir.Attribute. -pub fn derefine(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.derefine", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute.dice`. Result types are explicit; attributes use mlir.Attribute. -pub fn dice(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, coord: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.dice", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "coord", coord), - }, - .location = location, - }); -} - -/// `cute.elem_less`. Result types are explicit; attributes use mlir.Attribute. -pub fn elem_less(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.elem_less", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.equal`. Result types are explicit; attributes use mlir.Attribute. -pub fn equal(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.equal", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.fast_divmod.compute`. Result types are explicit; attributes use mlir.Attribute. -pub fn fast_divmod_compute(ctx: *mlir.Context, dividend: *const mlir.Value, divisor: *const mlir.Value, quotient_type: *const mlir.Type, remainder_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.fast_divmod.compute", .{ - .operands = .{ .flat = &.{ dividend, divisor } }, - .results = .{ .flat = &.{ quotient_type, remainder_type } }, - .location = location, - }); -} - -/// `cute.fast_divmod.create_divisor`. Result types are explicit; attributes use mlir.Attribute. -pub fn fast_divmod_create_divisor(ctx: *mlir.Context, divisor: *const mlir.Value, fast_divmod_divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.fast_divmod.create_divisor", .{ - .operands = .{ .flat = &.{divisor} }, - .results = .{ .flat = &.{fast_divmod_divisor_type} }, - .location = location, - }); -} - -/// `cute.fast_divmod.divide`. Result types are explicit; attributes use mlir.Attribute. -pub fn fast_divmod_divide(ctx: *mlir.Context, dividend: *const mlir.Value, divisor: *const mlir.Value, quotient_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.fast_divmod.divide", .{ - .operands = .{ .flat = &.{ dividend, divisor } }, - .results = .{ .flat = &.{quotient_type} }, - .location = location, - }); -} - -/// `cute.fast_divmod.get_aux`. Result types are explicit; attributes use mlir.Attribute. -pub fn fast_divmod_get_aux(ctx: *mlir.Context, divisor: *const mlir.Value, multiplier_type: *const mlir.Type, shift1_type: *const mlir.Type, shift2_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.fast_divmod.get_aux", .{ - .operands = .{ .flat = &.{divisor} }, - .results = .{ .flat = &.{ multiplier_type, shift1_type, shift2_type } }, - .location = location, - }); -} - -/// `cute.fast_divmod.get_divisor`. Result types are explicit; attributes use mlir.Attribute. -pub fn fast_divmod_get_divisor(ctx: *mlir.Context, fast_divmod_divisor: *const mlir.Value, divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.fast_divmod.get_divisor", .{ - .operands = .{ .flat = &.{fast_divmod_divisor} }, - .results = .{ .flat = &.{divisor_type} }, - .location = location, - }); -} - -/// `cute.fast_divmod.make_divisor`. Result types are explicit; attributes use mlir.Attribute. -pub fn fast_divmod_make_divisor(ctx: *mlir.Context, divisor: *const mlir.Value, multiplier: *const mlir.Value, sh1: *const mlir.Value, sh2: *const mlir.Value, fast_divmod_divisor_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.fast_divmod.make_divisor", .{ - .operands = .{ .flat = &.{ divisor, multiplier, sh1, sh2 } }, - .results = .{ .flat = &.{fast_divmod_divisor_type} }, - .location = location, - }); -} - -/// `cute.filter`. Result types are explicit; attributes use mlir.Attribute. -pub fn filter(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.filter", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.filter_zeros`. Result types are explicit; attributes use mlir.Attribute. -pub fn filter_zeros( - ctx: *mlir.Context, - input: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - target_profile: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.filter_zeros", .{ - .operands = .{ .variadic = &.{ - &.{input}, - if (opts.target_profile) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `cute.flat_divide`. Result types are explicit; attributes use mlir.Attribute. -pub fn flat_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.flat_divide", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.flat_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn flat_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.flat_product", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.gemm`. Result types are explicit; attributes use mlir.Attribute. -pub fn gemm(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.gemm", .{ - .operands = .{ .variadic = &.{ - &.{mma_atom}, - &.{d}, - a, - b, - &.{c_}, - } }, - .location = location, - }); -} - -/// `cute.get`. Result types are explicit; attributes use mlir.Attribute. -pub fn get( - ctx: *mlir.Context, - input: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - mode: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "cute.get", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.get_flat_coord`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_flat_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_flat_coord", .{ - .operands = .{ .flat = &.{ index, input } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.get_hier_coord`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_hier_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_hier_coord", .{ - .operands = .{ .flat = &.{ index, input } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.get_integral_coord`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_integral_coord(ctx: *mlir.Context, index: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_integral_coord", .{ - .operands = .{ .flat = &.{ index, input } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.get_iter`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_iter(ctx: *mlir.Context, source: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_iter", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{ptr_type} }, - .location = location, - }); -} - -/// `cute.get_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_layout(ctx: *mlir.Context, input: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_layout", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{layout_type} }, - .location = location, - }); -} - -/// `cute.get_layouts_from_tile`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_layouts_from_tile(ctx: *mlir.Context, tile: *const mlir.Value, layouts_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_layouts_from_tile", .{ - .operands = .{ .flat = &.{tile} }, - .results = .{ .variadic = &.{ - layouts_types, - } }, - .location = location, - }); -} - -/// `cute.get_leaves`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_leaves(ctx: *mlir.Context, input: *const mlir.Value, results__types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_leaves", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .variadic = &.{ - results__types, - } }, - .location = location, - }); -} - -/// `cute.get_scalars`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_scalars( - ctx: *mlir.Context, - cute_value: *const mlir.Value, - scalars_types: []const *const mlir.Type, - opts: struct { - only_dynamic: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.only_dynamic) |value| attributes.appendAssumeCapacity(.named(ctx, "only_dynamic", value)); - return mlir.Operation.make(ctx, "cute.get_scalars", .{ - .operands = .{ .flat = &.{cute_value} }, - .results = .{ .variadic = &.{ - scalars_types, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.get_shape`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_shape(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_shape", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.get_stride`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_stride(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.get_stride", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.group_modes`. Result types are explicit; attributes use mlir.Attribute. -pub fn group_modes(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.group_modes", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "begin", begin), - .named(ctx, "end", end), - }, - .location = location, - }); -} - -/// `cute.idx2crd`. Result types are explicit; attributes use mlir.Attribute. -pub fn idx2crd(ctx: *mlir.Context, index: *const mlir.Value, shape: *const mlir.Value, coord_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.idx2crd", .{ - .operands = .{ .flat = &.{ index, shape } }, - .results = .{ .flat = &.{coord_type} }, - .location = location, - }); -} - -/// `cute.increment_coord`. Result types are explicit; attributes use mlir.Attribute. -pub fn increment_coord(ctx: *mlir.Context, coord: *const mlir.Value, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.increment_coord", .{ - .operands = .{ .flat = &.{ coord, shape } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.inttoptr`. Result types are explicit; attributes use mlir.Attribute. -pub fn inttoptr(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.inttoptr", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute.kernel_smem_size`. Result types are explicit; attributes use mlir.Attribute. -pub fn kernel_smem_size(ctx: *mlir.Context, size_type: *const mlir.Type, kernel_name: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.kernel_smem_size", .{ - .results = .{ .flat = &.{size_type} }, - .attributes = &.{ - .named(ctx, "kernel_name", kernel_name), - }, - .location = location, - }); -} - -/// `cute.left_inverse`. Result types are explicit; attributes use mlir.Attribute. -pub fn left_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.left_inverse", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.load_scaled_index`. Result types are explicit; attributes use mlir.Attribute. -pub fn load_scaled_index(ctx: *mlir.Context, ptr_a: *const mlir.Value, ptr_b: *const mlir.Value, index: *const mlir.Value, stride: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.load_scaled_index", .{ - .operands = .{ .flat = &.{ ptr_a, ptr_b, index, stride } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.local_partition`. Result types are explicit; attributes use mlir.Attribute. -pub fn local_partition(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, index: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.local_partition", .{ - .operands = .{ .flat = &.{ input, tiler, index } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.local_tile`. Result types are explicit; attributes use mlir.Attribute. -pub fn local_tile( - ctx: *mlir.Context, - input: *const mlir.Value, - tile: *const mlir.Value, - coord: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - proj: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.proj) |value| attributes.appendAssumeCapacity(.named(ctx, "proj", value)); - return mlir.Operation.make(ctx, "cute.local_tile", .{ - .operands = .{ .flat = &.{ input, tile, coord } }, - .results = .{ .flat = &.{result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.logical_divide`. Result types are explicit; attributes use mlir.Attribute. -pub fn logical_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.logical_divide", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.logical_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn logical_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.logical_product", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_arith_tuple_iter`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_arith_tuple_iter( - ctx: *mlir.Context, - iter_type: *const mlir.Type, - opts: struct { - value_: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_arith_tuple_iter", .{ - .operands = .{ .variadic = &.{ - if (opts.value_) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{iter_type} }, - .location = location, - }); -} - -/// `cute.make_atom`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_atom(ctx: *mlir.Context, values: []const *const mlir.Value, atom_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_atom", .{ - .operands = .{ .variadic = &.{ - values, - } }, - .results = .{ .flat = &.{atom_type} }, - .location = location, - }); -} - -/// `cute.make_composed_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_composed_layout(ctx: *mlir.Context, inner: *const mlir.Value, offset: *const mlir.Value, outer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_composed_layout", .{ - .operands = .{ .flat = &.{ inner, offset, outer } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_coord`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_coord(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_coord", .{ - .operands = .{ .variadic = &.{ - dynamicElements, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_desc_iter`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_desc_iter(ctx: *mlir.Context, value_: *const mlir.Value, iter_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_desc_iter", .{ - .operands = .{ .flat = &.{value_} }, - .results = .{ .flat = &.{iter_type} }, - .location = location, - }); -} - -/// `cute.make_fragment_like`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_fragment_like(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_fragment_like", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute.make_identity_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_identity_layout(ctx: *mlir.Context, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_identity_layout", .{ - .operands = .{ .flat = &.{shape} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_identity_tensor`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_identity_tensor(ctx: *mlir.Context, shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_identity_tensor", .{ - .operands = .{ .flat = &.{shape} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_int_tuple`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_int_tuple(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_int_tuple", .{ - .operands = .{ .variadic = &.{ - dynamicElements, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_layout( - ctx: *mlir.Context, - layout_type: *const mlir.Type, - opts: struct { - shape: ?*const mlir.Value = null, - stride: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_layout", .{ - .operands = .{ .variadic = &.{ - if (opts.shape) |value| &.{value} else &.{}, - if (opts.stride) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{layout_type} }, - .location = location, - }); -} - -/// `cute.make_layout_like`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_layout_like(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_layout_like", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute.make_ordered_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_ordered_layout(ctx: *mlir.Context, shape: *const mlir.Value, order: *const mlir.Value, layout_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_ordered_layout", .{ - .operands = .{ .flat = &.{ shape, order } }, - .results = .{ .flat = &.{layout_type} }, - .location = location, - }); -} - -/// `cute.make_shape`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_shape(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_shape", .{ - .operands = .{ .variadic = &.{ - dynamicElements, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_sparse_elem`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_sparse_elem(ctx: *mlir.Context, physical_storage: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_sparse_elem", .{ - .operands = .{ .flat = &.{physical_storage} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_stride`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_stride(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_stride", .{ - .operands = .{ .variadic = &.{ - dynamicElements, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_tile`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tile(ctx: *mlir.Context, dynamicElements: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_tile", .{ - .operands = .{ .variadic = &.{ - dynamicElements, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_tiled_copy`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tiled_copy(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_tiled_copy", .{ - .operands = .{ .flat = &.{atom} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_tiled_copy_v2`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tiled_copy_v2(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_tiled_copy_v2", .{ - .operands = .{ .flat = &.{atom} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_tiled_mma`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tiled_mma(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_tiled_mma", .{ - .operands = .{ .flat = &.{atom} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_tuple`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tuple(ctx: *mlir.Context, values: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_tuple", .{ - .operands = .{ .variadic = &.{ - values, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.make_view`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_view( - ctx: *mlir.Context, - iter: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - layout: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.make_view", .{ - .operands = .{ .variadic = &.{ - &.{iter}, - if (opts.layout) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.memref.alloc_smem`. Result types are explicit; attributes use mlir.Attribute. -pub fn memref_alloc_smem(ctx: *mlir.Context, memref_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.memref.alloc_smem", .{ - .results = .{ .flat = &.{memref_type} }, - .location = location, - }); -} - -/// `cute.memref.alloca`. Result types are explicit; attributes use mlir.Attribute. -pub fn memref_alloca( - ctx: *mlir.Context, - result_type: *const mlir.Type, - opts: struct { - layout: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.memref.alloca", .{ - .operands = .{ .variadic = &.{ - if (opts.layout) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.memref.load`. Result types are explicit; attributes use mlir.Attribute. -pub fn memref_load(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.memref.load", .{ - .operands = .{ .flat = &.{ src, coord } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.memref.load_vec`. Result types are explicit; attributes use mlir.Attribute. -pub fn memref_load_vec( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - mask: ?*const mlir.Value = null, - pass_thru: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.memref.load_vec", .{ - .operands = .{ .variadic = &.{ - &.{src}, - if (opts.mask) |value| &.{value} else &.{}, - if (opts.pass_thru) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.memref.store`. Result types are explicit; attributes use mlir.Attribute. -pub fn memref_store(ctx: *mlir.Context, dst: *const mlir.Value, coord: *const mlir.Value, value_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.memref.store", .{ - .operands = .{ .flat = &.{ dst, coord, value_ } }, - .location = location, - }); -} - -/// `cute.memref.store_vec`. Result types are explicit; attributes use mlir.Attribute. -pub fn memref_store_vec( - ctx: *mlir.Context, - value_: *const mlir.Value, - dst: *const mlir.Value, - opts: struct { - mask: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.memref.store_vec", .{ - .operands = .{ .variadic = &.{ - &.{value_}, - &.{dst}, - if (opts.mask) |value| &.{value} else &.{}, - } }, - .location = location, - }); -} - -/// `cute.mma.make_fragment`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_make_fragment(ctx: *mlir.Context, atom: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.mma.make_fragment", .{ - .operands = .{ .flat = &.{ atom, input } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operand_id", operand_id), - }, - .location = location, - }); -} - -/// `cute.mma_atom_call`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c_: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.mma_atom_call", .{ - .operands = .{ .variadic = &.{ - &.{atom}, - &.{d}, - a, - b, - &.{c_}, - } }, - .location = location, - }); -} - -/// `cute.prefetch`. Result types are explicit; attributes use mlir.Attribute. -pub fn prefetch(ctx: *mlir.Context, prefetch_atom: *const mlir.Value, src: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.prefetch", .{ - .operands = .{ .variadic = &.{ - &.{prefetch_atom}, - src, - } }, - .location = location, - }); -} - -/// `cute.prefetch_atom_call`. Result types are explicit; attributes use mlir.Attribute. -pub fn prefetch_atom_call(ctx: *mlir.Context, atom: *const mlir.Value, src: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.prefetch_atom_call", .{ - .operands = .{ .variadic = &.{ - &.{atom}, - src, - } }, - .location = location, - }); -} - -/// `cute.prepend_to_rank`. Result types are explicit; attributes use mlir.Attribute. -pub fn prepend_to_rank(ctx: *mlir.Context, input: *const mlir.Value, element: *const mlir.Value, result_type: *const mlir.Type, rank: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.prepend_to_rank", .{ - .operands = .{ .flat = &.{ input, element } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "rank", rank), - }, - .location = location, - }); -} - -/// `cute.print`. Result types are explicit; attributes use mlir.Attribute. -pub fn print( - ctx: *mlir.Context, - values: []const *const mlir.Value, - opts: struct { - fmt: ?*const mlir.Attribute = null, - operand_signed: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.fmt) |value| attributes.appendAssumeCapacity(.named(ctx, "fmt", value)); - if (opts.operand_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "operand_signed", value)); - return mlir.Operation.make(ctx, "cute.print", .{ - .operands = .{ .variadic = &.{ - values, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.print_tma_desc_im2col`. Result types are explicit; attributes use mlir.Attribute. -pub fn print_tma_desc_im2col(ctx: *mlir.Context, input: *const mlir.Value, fd: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.print_tma_desc_im2col", .{ - .operands = .{ .flat = &.{ input, fd } }, - .location = location, - }); -} - -/// `cute.print_tma_desc_tiled`. Result types are explicit; attributes use mlir.Attribute. -pub fn print_tma_desc_tiled(ctx: *mlir.Context, input: *const mlir.Value, fd: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.print_tma_desc_tiled", .{ - .operands = .{ .flat = &.{ input, fd } }, - .location = location, - }); -} - -/// `cute.print_view`. Result types are explicit; attributes use mlir.Attribute. -pub fn print_view( - ctx: *mlir.Context, - src: *const mlir.Value, - opts: struct { - coord: ?*const mlir.Value = null, - verbose: ?*const mlir.Attribute = null, - is_signed: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.verbose) |value| attributes.appendAssumeCapacity(.named(ctx, "verbose", value)); - if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); - return mlir.Operation.make(ctx, "cute.print_view", .{ - .operands = .{ .variadic = &.{ - &.{src}, - if (opts.coord) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.ptr.load`. Result types are explicit; attributes use mlir.Attribute. -pub fn ptr_load(ctx: *mlir.Context, ptr: *const mlir.Value, value__type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.ptr.load", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{value__type} }, - .location = location, - }); -} - -/// `cute.ptr.store`. Result types are explicit; attributes use mlir.Attribute. -pub fn ptr_store(ctx: *mlir.Context, value_: *const mlir.Value, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.ptr.store", .{ - .operands = .{ .flat = &.{ value_, ptr } }, - .location = location, - }); -} - -/// `cute.ptrtoint`. Result types are explicit; attributes use mlir.Attribute. -pub fn ptrtoint(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.ptrtoint", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute.raked_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn raked_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.raked_product", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.random_delay`. Result types are explicit; attributes use mlir.Attribute. -pub fn random_delay(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.random_delay", .{ - .location = location, - }); -} - -/// `cute.recast_iter`. Result types are explicit; attributes use mlir.Attribute. -pub fn recast_iter(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.recast_iter", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute.recast_layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn recast_layout(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, new_type_bits: *const mlir.Attribute, old_type_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.recast_layout", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = &.{ - .named(ctx, "new_type_bits", new_type_bits), - .named(ctx, "old_type_bits", old_type_bits), - }, - .location = location, - }); -} - -/// `cute.reduce`. Result types are explicit; attributes use mlir.Attribute. -pub fn reduce( - ctx: *mlir.Context, - input: *const mlir.Value, - shape: *const mlir.Value, - init_val: *const mlir.Value, - output_type: *const mlir.Type, - reduction_op: *const mlir.Attribute, - opts: struct { - predication: ?*const mlir.Value = null, - reduced_shape_type: ?*const mlir.Type = null, - reduction_profile: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "reduction_op", reduction_op)); - if (opts.reduction_profile) |value| attributes.appendAssumeCapacity(.named(ctx, "reduction_profile", value)); - return mlir.Operation.make(ctx, "cute.reduce", .{ - .operands = .{ .variadic = &.{ - &.{input}, - &.{shape}, - &.{init_val}, - if (opts.predication) |value| &.{value} else &.{}, - } }, - .results = .{ .variadic = &.{ - &.{output_type}, - if (opts.reduced_shape_type) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.right_inverse`. Result types are explicit; attributes use mlir.Attribute. -pub fn right_inverse(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.right_inverse", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.select`. Result types are explicit; attributes use mlir.Attribute. -pub fn select(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.select", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "mode", mode), - }, - .location = location, - }); -} - -/// `cute.shape_div`. Result types are explicit; attributes use mlir.Attribute. -pub fn shape_div(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.shape_div", .{ - .operands = .{ .flat = &.{ a, b } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.size`. Result types are explicit; attributes use mlir.Attribute. -pub fn size( - ctx: *mlir.Context, - input: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - mode: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "cute.size", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.slice`. Result types are explicit; attributes use mlir.Attribute. -pub fn slice(ctx: *mlir.Context, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.slice", .{ - .operands = .{ .flat = &.{ input, coord } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.smem_partition_size`. Result types are explicit; attributes use mlir.Attribute. -pub fn smem_partition_size( - ctx: *mlir.Context, - size_type: *const mlir.Type, - partition_id: *const mlir.Attribute, - opts: struct { - cumulative: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "partition_id", partition_id)); - if (opts.cumulative) |value| attributes.appendAssumeCapacity(.named(ctx, "cumulative", value)); - return mlir.Operation.make(ctx, "cute.smem_partition_size", .{ - .results = .{ .flat = &.{size_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute.static`. Result types are explicit; attributes use mlir.Attribute. -pub fn static(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.static", .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.stencil_divide`. Result types are explicit; attributes use mlir.Attribute. -pub fn stencil_divide( - ctx: *mlir.Context, - input: *const mlir.Value, - stencil: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - padding_upper: ?*const mlir.Value = null, - padding_lower: ?*const mlir.Value = null, - traversal_stride: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.stencil_divide", .{ - .operands = .{ .variadic = &.{ - &.{input}, - &.{stencil}, - if (opts.padding_upper) |value| &.{value} else &.{}, - if (opts.padding_lower) |value| &.{value} else &.{}, - if (opts.traversal_stride) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.symbolic`. Result types are explicit; attributes use mlir.Attribute. -pub fn symbolic(ctx: *mlir.Context, symbolic_type: *const mlir.Type, name: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.symbolic", .{ - .results = .{ .flat = &.{symbolic_type} }, - .attributes = &.{ - .named(ctx, "name", name), - }, - .location = location, - }); -} - -/// `cute.tile_to_shape`. Result types are explicit; attributes use mlir.Attribute. -pub fn tile_to_shape(ctx: *mlir.Context, block: *const mlir.Value, trg_shape: *const mlir.Value, ord_shape: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tile_to_shape", .{ - .operands = .{ .flat = &.{ block, trg_shape, ord_shape } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tiled.copy.partition_D`. Result types are explicit; attributes use mlir.Attribute. -pub fn tiled_copy_partition_D(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tiled.copy.partition_D", .{ - .operands = .{ .flat = &.{ tiled_copy, input, coord } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tiled.copy.partition_S`. Result types are explicit; attributes use mlir.Attribute. -pub fn tiled_copy_partition_S(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tiled.copy.partition_S", .{ - .operands = .{ .flat = &.{ tiled_copy, input, coord } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tiled.copy.retile`. Result types are explicit; attributes use mlir.Attribute. -pub fn tiled_copy_retile(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tiled.copy.retile", .{ - .operands = .{ .flat = &.{ tiled_copy, input } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tiled.mma.partition`. Result types are explicit; attributes use mlir.Attribute. -pub fn tiled_mma_partition(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tiled.mma.partition", .{ - .operands = .{ .flat = &.{ tiled_mma, input, coord } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operand_id", operand_id), - }, - .location = location, - }); -} - -/// `cute.tiled.mma.partition_shape`. Result types are explicit; attributes use mlir.Attribute. -pub fn tiled_mma_partition_shape(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, result_type: *const mlir.Type, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tiled.mma.partition_shape", .{ - .operands = .{ .flat = &.{ tiled_mma, input } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operand_id", operand_id), - }, - .location = location, - }); -} - -/// `cute.tiled_divide`. Result types are explicit; attributes use mlir.Attribute. -pub fn tiled_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tiled_divide", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tiled_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn tiled_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tiled_product", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.to_int_tuple`. Result types are explicit; attributes use mlir.Attribute. -pub fn to_int_tuple(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.to_int_tuple", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute.tuple.product`. Result types are explicit; attributes use mlir.Attribute. -pub fn tuple_product(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple.product", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tuple.product_each`. Result types are explicit; attributes use mlir.Attribute. -pub fn tuple_product_each(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple.product_each", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tuple_add`. Result types are explicit; attributes use mlir.Attribute. -pub fn tuple_add(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple_add", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tuple_div`. Result types are explicit; attributes use mlir.Attribute. -pub fn tuple_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple_div", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tuple_mod`. Result types are explicit; attributes use mlir.Attribute. -pub fn tuple_mod(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple_mod", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tuple_mul`. Result types are explicit; attributes use mlir.Attribute. -pub fn tuple_mul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple_mul", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.tuple_sub`. Result types are explicit; attributes use mlir.Attribute. -pub fn tuple_sub(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.tuple_sub", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.unpack_tuple`. Result types are explicit; attributes use mlir.Attribute. -pub fn unpack_tuple(ctx: *mlir.Context, tuple: *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.unpack_tuple", .{ - .operands = .{ .flat = &.{tuple} }, - .results = .{ .variadic = &.{ - result_types, - } }, - .location = location, - }); -} - -/// `cute.zipped_divide`. Result types are explicit; attributes use mlir.Attribute. -pub fn zipped_divide(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.zipped_divide", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute.zipped_product`. Result types are explicit; attributes use mlir.Attribute. -pub fn zipped_product(ctx: *mlir.Context, input: *const mlir.Value, tiler: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute.zipped_product", .{ - .operands = .{ .flat = &.{ input, tiler } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} diff --git a/mlir/dialects/cute_ir/test.zig b/mlir/dialects/cute_ir/test.zig deleted file mode 100644 index c6f44763..00000000 --- a/mlir/dialects/cute_ir/test.zig +++ /dev/null @@ -1,341 +0,0 @@ -const std = @import("std"); - -const cute_ir = @import("mlir/dialects/cute_ir"); -const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); -const mlir = @import("mlir"); - -const ArithTupleIteratorType = cute_ir.ArithTupleIteratorType; -const ComposedLayoutType = cute_ir.ComposedLayoutType; -const ConstrainedIntType = cute_ir.ConstrainedIntType; -const CoordTensorType = cute_ir.CoordTensorType; -const CoordType = cute_ir.CoordType; -const FastDivmodDivisorType = cute_ir.FastDivmodDivisorType; -const IntTupleType = cute_ir.IntTupleType; -const LayoutType = cute_ir.LayoutType; -const MemRefType = cute_ir.MemRefType; -const PtrType = cute_ir.PtrType; -const ShapeType = cute_ir.ShapeType; -const SparseElemType = cute_ir.SparseElemType; -const StrideType = cute_ir.StrideType; -const SwizzleType = cute_ir.SwizzleType; -const TileType = cute_ir.TileType; -const TiledCopyType = cute_ir.TiledCopyType; -const TiledCopyV2Type = cute_ir.TiledCopyV2Type; -const TiledMmaType = cute_ir.TiledMmaType; -const TupleType = cute_ir.TupleType; -const BitLayoutAttr = cute_ir.BitLayoutAttr; -const CopyAtomAttr = cute_ir.CopyAtomAttr; -const CopyAtomV2Attr = cute_ir.CopyAtomV2Attr; -const MmaAtomAttr = cute_ir.MmaAtomAttr; -const MxMmaAtomAttr = cute_ir.MxMmaAtomAttr; -const SparseMmaAtomAttr = cute_ir.SparseMmaAtomAttr; -const SparseMxMmaAtomAttr = cute_ir.SparseMxMmaAtomAttr; - -const operation_names: []const []const u8 = &.{ - "cute.add_offset", - "cute.append_to_rank", - "cute.apply_swizzle", - "cute.assume", - "cute.blocked_product", - "cute.ceil_div", - "cute.coalesce", - "cute.complement", - "cute.composed_get_inner", - "cute.composed_get_offset", - "cute.composed_get_outer", - "cute.composition", - "cute.copy", - "cute.copy.make_fragment", - "cute.copy_atom_call", - "cute.cosize", - "cute.crd2idx", - "cute.deref_arith_tuple_iter", - "cute.deref_desc_iter", - "cute.derefine", - "cute.dice", - "cute.elem_less", - "cute.equal", - "cute.fast_divmod.compute", - "cute.fast_divmod.create_divisor", - "cute.fast_divmod.divide", - "cute.fast_divmod.get_aux", - "cute.fast_divmod.get_divisor", - "cute.fast_divmod.make_divisor", - "cute.filter", - "cute.filter_zeros", - "cute.flat_divide", - "cute.flat_product", - "cute.gemm", - "cute.get", - "cute.get_flat_coord", - "cute.get_hier_coord", - "cute.get_integral_coord", - "cute.get_iter", - "cute.get_layout", - "cute.get_layouts_from_tile", - "cute.get_leaves", - "cute.get_scalars", - "cute.get_shape", - "cute.get_stride", - "cute.group_modes", - "cute.idx2crd", - "cute.increment_coord", - "cute.inttoptr", - "cute.kernel_smem_size", - "cute.left_inverse", - "cute.load_scaled_index", - "cute.local_partition", - "cute.local_tile", - "cute.logical_divide", - "cute.logical_product", - "cute.make_arith_tuple_iter", - "cute.make_atom", - "cute.make_composed_layout", - "cute.make_coord", - "cute.make_desc_iter", - "cute.make_fragment_like", - "cute.make_identity_layout", - "cute.make_identity_tensor", - "cute.make_int_tuple", - "cute.make_layout", - "cute.make_layout_like", - "cute.make_ordered_layout", - "cute.make_shape", - "cute.make_sparse_elem", - "cute.make_stride", - "cute.make_tile", - "cute.make_tiled_copy", - "cute.make_tiled_copy_v2", - "cute.make_tiled_mma", - "cute.make_tuple", - "cute.make_view", - "cute.memref.alloc_smem", - "cute.memref.alloca", - "cute.memref.load", - "cute.memref.load_vec", - "cute.memref.store", - "cute.memref.store_vec", - "cute.mma.make_fragment", - "cute.mma_atom_call", - "cute.prefetch", - "cute.prefetch_atom_call", - "cute.prepend_to_rank", - "cute.print", - "cute.print_tma_desc_im2col", - "cute.print_tma_desc_tiled", - "cute.print_view", - "cute.ptr.load", - "cute.ptr.store", - "cute.ptrtoint", - "cute.raked_product", - "cute.random_delay", - "cute.recast_iter", - "cute.recast_layout", - "cute.reduce", - "cute.right_inverse", - "cute.select", - "cute.shape_div", - "cute.size", - "cute.slice", - "cute.smem_partition_size", - "cute.static", - "cute.stencil_divide", - "cute.symbolic", - "cute.tile_to_shape", - "cute.tiled.copy.partition_D", - "cute.tiled.copy.partition_S", - "cute.tiled.copy.retile", - "cute.tiled.mma.partition", - "cute.tiled.mma.partition_shape", - "cute.tiled_divide", - "cute.tiled_product", - "cute.to_int_tuple", - "cute.tuple.product", - "cute.tuple.product_each", - "cute.tuple_add", - "cute.tuple_div", - "cute.tuple_mod", - "cute.tuple_mul", - "cute.tuple_sub", - "cute.unpack_tuple", - "cute.zipped_divide", - "cute.zipped_product", -}; - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - cute_ir.registerDialect(registry); - cute_nvgpu.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(cute_ir); - inline for (comptime std.meta.declarations(cute_ir)) |decl| { - const T = @field(cute_ir, decl); - if (@TypeOf(T) == type) std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const ctx = try testContext(); - defer ctx.deinit(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} - -test "kernel is a unit attribute of the cute dialect" { - const ctx = try testContext(); - defer ctx.deinit(); - var buf: [64]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{cute_ir.kernel(ctx)})}); - try std.testing.expectEqualStrings("{cute.kernel}", w.buffered()); -} - -test "generated types and attributes rebuild from their getters" { - @setEvalBranchQuota(100_000); - const ctx = try testContext(); - defer ctx.deinit(); - inline for (.{ - .{ ArithTupleIteratorType, "!cute.arith_tuple_iter<\"(?,?{div=16})\">" }, - .{ ComposedLayoutType, "!cute.composed_layout<\"(4,5):(1,4) o 2 o (2,3):(1,2)\">" }, - .{ ConstrainedIntType, "!cute.i16" }, - .{ CoordTensorType, "!cute.coord_tensor<\"(?,?,?)\", \"((512,1),1):((1/2@0,0),0)\">" }, - .{ CoordType, "!cute.coord<\"1\">" }, - .{ FastDivmodDivisorType, "!cute.fast_divmod_divisor<32>" }, - .{ IntTupleType, "!cute.int_tuple<\"1\">" }, - .{ LayoutType, "!cute.layout<\"(2,3):(1,2)\">" }, - .{ MemRefType, "!cute.memref, S<2,4,3>, \"((8,16),(32,1),(1,3)):((32,256),(1,0),(0,4096))\">" }, - .{ PtrType, "!cute.ptr, S<2,4,3>>" }, - .{ ShapeType, "!cute.shape<\"(2,3)\">" }, - .{ SparseElemType, "!cute.sparse_elem<2, vector<1xf16>>" }, - .{ StrideType, "!cute.stride<\"1\">" }, - .{ SwizzleType, "!cute.swizzle<\"S<0,4,3>\">" }, - .{ TileType, "!cute.tile<\"[(2,3):(1,2)]\">" }, - .{ TiledCopyType, "!cute.tiled_copy, layout_copy_tv = <\"(32,4):(4,1)\">, tiler_mn = <\"[4:1;32:1]\">>" }, - .{ TiledCopyV2Type, "!cute.tiled_copy_v2, layout_copy_tv = <\"(1,8):(0,1)\">, tiler_mn = <\"[8:1]\">>" }, - .{ TiledMmaType, "!cute.tiled_mma f32 >, atom_layout_MNK = <\"(16,16,1):(16,1,0)\">, permutation_MNK = <\"[(16,4):(4,1);(16,4):(4,1);_]\">>" }, - .{ TupleType, "!cute.tuple" }, - .{ BitLayoutAttr, "#cute.bitlayout<<\"(8,4):(1,8)\">>" }, - .{ CopyAtomAttr, "#cute.copy_atom, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">)>" }, - .{ CopyAtomV2Attr, "#cute.copy_atom_v2, layout_tv = (<\"(1,8):(0,1)\">, <\"(1,8):(0,1)\">), frg = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, - .{ MmaAtomAttr, "#cute.mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, - .{ MxMmaAtomAttr, "#cute.mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">, <\"2\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, - .{ SparseMmaAtomAttr, "#cute.sparse_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"1:0\">, <\"1:0\">, <\"1:0\">), shapes = (<\"(16,16)\">, <\"(8,16)\">, <\"(16,8)\">, <\"1\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, - .{ SparseMxMmaAtomAttr, "#cute.sparse_mx_mma_atom<<\"(16,8,16)\">, <\"32:1\">, layout_TV = (<\"1:0\">, <\"2:0\">, <\"3:0\">, <\"4:0\">, <\"5:0\">, <\"6:0\">), shapes = (<\"1\">, <\"2\">, <\"3\">, <\"4\">, <\"5\">, <\"6\">) frgs = (#cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>, #cute_nvgpu.not_implemented_frg<>)>" }, - }) |example| { - const T = example[0]; - const is_type = @hasDecl(T, "type_"); - const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); - const value = parsed.isA(T) orelse return error.TestUnexpectedResult; - try std.testing.expect((try rebuild(T, ctx, value)).eql(value)); - var text: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer text.deinit(); - try text.writer.print("{f}", .{value}); - try std.testing.expect(parsed.eql(if (is_type) try mlir.Type.parse(ctx, text.written()) else try mlir.Attribute.parse(ctx, text.written()))); - } - // Every case of every enum attribute. - inline for (.{cute_ir}) |namespace| { - inline for (comptime std.meta.declarations(namespace)) |decl| { - const T = @field(namespace, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { - const fields = @typeInfo(T.InitArgs).@"struct".field_names; - if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { - inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { - const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); - const attr = try T.get(ctx, .{ .value = case_value }); - try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.value())); - try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); - } - } - } - } - } -} - -test "algebra attributes build from their leaves and tuples" { - const ctx = try testContext(); - defer ctx.deinit(); - - const int_tuple = try cute_ir.IntTupleAttr.getTuple(ctx, &.{ - try .getStatic(ctx, 0), - try .getDynamic(ctx, 32, 1), - try .getDynamic(ctx, 32, 16), - try .getTuple(ctx, &.{ try .getStatic(ctx, 4), try .getDynamic(ctx, 64, 1) }), - }); - try expectType(ctx, "!cute.int_tuple<\"(0,?,?{div=16},(4,?{i64}))\">", (try IntTupleType.get(ctx, .{ .attr = int_tuple.attribute() })).type_()); - try expectType(ctx, "!cute.int_tuple<\"()\">", (try IntTupleType.get(ctx, .{ .attr = (try cute_ir.IntTupleAttr.getTuple(ctx, &.{})).attribute() })).type_()); - - const coord = try cute_ir.CoordAttr.getTuple(ctx, &.{ - try .getUnderscore(ctx), - try .getStatic(ctx, 1), - try .getTuple(ctx, &.{ try .getDynamic(ctx, 32, 1), try .getUnderscore(ctx) }), - }); - try expectType(ctx, "!cute.coord<\"(_,1,(?,_))\">", (try CoordType.get(ctx, .{ .attr = coord.attribute() })).type_()); - - const shape = try cute_ir.ShapeAttr.getTuple(ctx, &.{ - try .getTuple(ctx, &.{ try .getStatic(ctx, 512), try .getStatic(ctx, 1) }), - try .getStatic(ctx, 1), - }); - const stride = try cute_ir.StrideAttr.getTuple(ctx, &.{ - try .getTuple(ctx, &.{ try .getBasis(ctx, 1, 2, &.{0}), try .getStatic(ctx, 0) }), - try .getBasis(ctx, 3, 1, &.{ 0, 1 }), - }); - const layout = try cute_ir.LayoutAttr.get(ctx, .{ .shape = shape, .stride = stride }); - try expectType(ctx, "!cute.layout<\"((512,1),1):((1/2@0,0),3@1@0)\">", (try LayoutType.get(ctx, .{ .attr = layout.attribute() })).type_()); - try std.testing.expectError(error.InvalidMlir, cute_ir.LayoutAttr.get(ctx, .{ .shape = shape, .stride = try .getStatic(ctx, 1) })); - - const row = try cute_ir.LayoutAttr.get(ctx, .{ - .shape = try .getTuple(ctx, &.{ try .getStatic(ctx, 8), try .getStatic(ctx, 64) }), - .stride = try .getTuple(ctx, &.{ try .getStatic(ctx, 64), try .getDynamic(ctx, 32, 8) }), - }); - try expectType(ctx, "!cute.layout<\"(8,64):(64,?{div=8})\">", (try LayoutType.get(ctx, .{ .attr = row.attribute() })).type_()); - - const tile = try cute_ir.TileAttr.getTuple(ctx, &.{ try .getLayout(ctx, row), try .getUnderscore(ctx) }); - try expectType(ctx, "!cute.tile<\"[(8,64):(64,?{div=8});_]\">", (try TileType.get(ctx, .{ .attr = tile.attribute() })).type_()); - - const swizzle = try cute_ir.SwizzleAttr.get(ctx, .{ .bits = 3, .base = 4, .shift = 3 }); - try expectType(ctx, "!cute.swizzle<\"S<3,4,3>\">", (try SwizzleType.get(ctx, .{ .attr = swizzle.attribute() })).type_()); - - const outer = try cute_ir.LayoutAttr.get(ctx, .{ - .shape = try .getTuple(ctx, &.{ try .getStatic(ctx, 8), try .getStatic(ctx, 64) }), - .stride = try .getTuple(ctx, &.{ try .getStatic(ctx, 64), try .getStatic(ctx, 1) }), - }); - const composed = try cute_ir.ComposedLayoutAttr.get(ctx, .{ .inner = .{ .swizzle = swizzle }, .offset = try .getStatic(ctx, 0), .outer = outer }); - try expectType(ctx, "!cute.composed_layout<\"S<3,4,3> o 0 o (8,64):(64,1)\">", (try ComposedLayoutType.get(ctx, .{ .attr = composed.attribute() })).type_()); - - try std.testing.expectError(error.InvalidMlir, cute_ir.IntTupleAttr.getDynamic(ctx, 7, 1)); - try std.testing.expectError(error.InvalidMlir, cute_ir.StrideAttr.getBasis(ctx, 1, 1, &.{})); - try std.testing.expectError(error.InvalidMlir, cute_ir.SwizzleAttr.get(ctx, .{ .bits = 5, .base = 4, .shift = 3 })); -} - -fn expectType(ctx: *mlir.Context, expected: []const u8, actual: *const mlir.Type) !void { - const parsed = try mlir.Type.parse(ctx, expected); - if (!parsed.eql(actual)) { - std.debug.print("expected {s}, built {f}\n", .{ expected, actual }); - return error.TestUnexpectedResult; - } -} - -fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { - var args: T.InitArgs = undefined; - var elements: [8]*const mlir.Type = undefined; - inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { - const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; - const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; - const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; - if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { - const count = @field(T, "num" ++ name)(value); - for (elements[0..count], 0..) |*element, i| element.* = @field(T, noun[0 .. noun.len - 1])(value, i); - @field(args, field) = elements[0..count]; - } else { - @field(args, field) = @field(T, accessor)(value); - } - } - return T.get(ctx, args); -} diff --git a/mlir/dialects/cute_nvgpu/BUILD.bazel b/mlir/dialects/cute_nvgpu/BUILD.bazel deleted file mode 100644 index 07c40ea9..00000000 --- a/mlir/dialects/cute_nvgpu/BUILD.bazel +++ /dev/null @@ -1,192 +0,0 @@ -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - ":dialect", - ":headers", - "//mlir/dialects/cute_ir:dialect", - "@llvm-project//mlir:CAPIIR", - ], -) - -zig_library( - name = "cute_nvgpu", - import_name = "mlir/dialects/cute_nvgpu", - main = "cute_nvgpu.zig", - tags = ["manual"], - deps = [ - ":capi", - "//mlir", - "//mlir/dialects/cute_ir", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":cute_nvgpu", - "//mlir", - "//mlir/dialects/cute_ir", - ], -) - -td_library( - name = "td_files", - srcs = [ - "CuteNVGPUAttrs.td", - "CuteNVGPUDialect.td", - "CuteNVGPUEnums.td", - "CuteNVGPUOps.td", - "CuteNVGPUTypes.td", - ], - includes = ["/"], - deps = [ - "//mlir/dialects/cute_ir:td_files", - "@cute_ir//:cute_ir_dialect_td", - "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - ], -) - -gentbl_cc_library( - name = "dialect_inc_gen", - tbl_outs = [ - ( - [ - "-gen-dialect-decls", - "-dialect=cute_nvgpu", - ], - "CuteNVGPUDialect.h.inc", - ), - ( - [ - "-gen-dialect-defs", - "-dialect=cute_nvgpu", - ], - "CuteNVGPUDialect.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteNVGPUDialect.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "types_inc_gen", - tbl_outs = [ - ( - [ - "-gen-typedef-decls", - "-typedefs-dialect=cute_nvgpu", - ], - "CuteNVGPUTypes.h.inc", - ), - ( - [ - "-gen-typedef-defs", - "-typedefs-dialect=cute_nvgpu", - ], - "CuteNVGPUTypes.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteNVGPUTypes.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "enums_inc_gen", - tbl_outs = [ - ( - ["-gen-enum-decls"], - "CuteNVGPUEnums.h.inc", - ), - ( - ["-gen-enum-defs"], - "CuteNVGPUEnums.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteNVGPUEnums.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "attrs_inc_gen", - tbl_outs = [ - ( - [ - "-gen-attrdef-decls", - "-attrdefs-dialect=cute_nvgpu", - ], - "CuteNVGPUAttrs.h.inc", - ), - ( - [ - "-gen-attrdef-defs", - "-attrdefs-dialect=cute_nvgpu", - ], - "CuteNVGPUAttrs.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteNVGPUAttrs.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "ops_inc_gen", - tbl_outs = [ - ( - ["-gen-op-decls"], - "CuteNVGPUOps.h.inc", - ), - ( - ["-gen-op-defs"], - "CuteNVGPUOps.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "CuteNVGPUOps.td", - deps = [":td_files"], -) - -cc_library( - name = "headers", - hdrs = ["CuteNVGPUDialect.h"], - deps = [ - ":attrs_inc_gen", - ":dialect_inc_gen", - ":enums_inc_gen", - ":ops_inc_gen", - ":types_inc_gen", - "//mlir/dialects/cute_ir:headers", - "@llvm-project//mlir:BytecodeOpInterface", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:SideEffectInterfaces", - ], -) - -cc_library( - name = "dialect", - srcs = ["CuteNVGPUDialect.cpp"], - deps = [ - ":headers", - "//mlir/dialects/cute_ir:headers", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:AsmParser", - "@llvm-project//mlir:IR", - ], -) diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td b/mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td deleted file mode 100644 index b4230c61..00000000 --- a/mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td +++ /dev/null @@ -1,132 +0,0 @@ -//===- CuteNVGPUAttrs.td - CuTe NVIDIA GPU attributes ------*- tablegen -*-===// -// -// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler -// (`_cutlass_ir.cu13` parser messages and probing its attribute parser and -// printer) and its enum bindings; do not edit by hand. -// -// The enum attributes are CuteNVGPUEnums.td; `#cute_nvgpu.major`, over the -// cute dialect's MajorMode, is here so that the enum generator run over -// CuteNVGPUEnums.td emits the nvgpu enums only. The fragment attributes are -// `key = value` lists in a fixed order; a defaulted field is printed only -// when set. -// -//===----------------------------------------------------------------------===// - -#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD -#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD - -include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" -include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td" -include "mlir/dialects/cute_ir/CuteEnums.td" - -def CuteNVGPU_MajorModeAttr - : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -class CuteNVGPU_Attr - : AttrDef { - let mnemonic = attrMnemonic; -} - -// `true` / `false`. -class CuteNVGPU_BoolParam : AttrOrTypeParameter<"bool", ""> { - let parser = "::mlir::cutlass_compiler::cute_nvgpu::parseBool($_parser)"; - let printer = "$_printer << ($_self ? \"true\" : \"false\")"; -} -def CuteNVGPU_Bool : CuteNVGPU_BoolParam; -def CuteNVGPU_FalseByDefault : CuteNVGPU_BoolParam { - let defaultValue = "false"; -} - -def CuteNVGPU_RmemFrgAttr : CuteNVGPU_Attr<"RmemFrg", "rmem_frg"> { - let summary = "Register-memory fragment of an MMA operand"; - let description = [{ - `#cute_nvgpu.rmem_frg` - }]; - let parameters = (ins - "::mlir::Type":$value_type, - "::mlir::cutlass_compiler::cute::MmaOperand":$operand, - CuteNVGPU_FalseByDefault:$derive_elem_type - ); - let assemblyFormat = [{ - `<` `value_type` `=` $value_type `,` `operand` `=` $operand - (`,` `derive_elem_type` `=` $derive_elem_type^)? `>` - }]; -} - -class CuteNVGPU_SmemFrg - : CuteNVGPU_Attr { - let summary = arch # " shared-memory fragment"; - let description = "`#cute_nvgpu." # attrMnemonic # "`"; - let parameters = (ins "::mlir::cutlass_compiler::cute::MajorMode":$major_mode); - let assemblyFormat = "`<` `major` `=` $major_mode `>`"; -} - -def CuteNVGPU_SM90SmemFrgAttr - : CuteNVGPU_SmemFrg<"SM90SmemFrg", "arch.sm90.smem_frg", "SM90">; -def CuteNVGPU_SM100SmemFrgAttr - : CuteNVGPU_SmemFrg<"SM100SmemFrg", "arch.sm100.smem_frg", "SM100">; -def CuteNVGPU_SM100CircularSmemFrgAttr - : CuteNVGPU_SmemFrg<"SM100CircularSmemFrg", "arch.sm100.circular_smem_frg", - "SM100 circular">; -def CuteNVGPU_SM107SmemFrgAttr - : CuteNVGPU_SmemFrg<"SM107SmemFrg", "arch.sm107.smem_frg", "SM107">; - -def CuteNVGPU_SM100TmemFrgAttr - : CuteNVGPU_Attr<"SM100TmemFrg", "arch.sm100.tmem_frg"> { - let summary = "SM100 tensor-memory fragment"; - let description = [{ - `#cute_nvgpu.arch.sm100.tmem_frg` - }]; - let parameters = (ins - "::mlir::Type":$data_type, - "::mlir::Type":$storage_type, - "int":$cta_group, - "TmemAllocMode":$tmem_alloc_mode - ); - let assemblyFormat = [{ - `<` `data_type` `=` $data_type `,` `storage_type` `=` $storage_type `,` - `cta_group` `=` $cta_group `,` `tmem_alloc_mode` `=` $tmem_alloc_mode `>` - }]; -} - -def CuteNVGPU_SM100TmemSfFrgAttr - : CuteNVGPU_Attr<"SM100TmemSfFrg", "arch.sm100.tmem_sf_frg"> { - let summary = "SM100 tensor-memory scale-factor fragment"; - let description = [{ - `#cute_nvgpu.arch.sm100.tmem_sf_frg` - }]; - let parameters = (ins - "::mlir::Type":$sf_type, - "int":$sf_vec_size, - "int":$cta_group, - CuteNVGPU_Bool:$is_sfa, - "TmemAllocMode":$tmem_alloc_mode - ); - let assemblyFormat = [{ - `<` `sf_type` `=` $sf_type `,` `sf_vec_size` `=` $sf_vec_size `,` - `cta_group` `=` $cta_group `,` `is_sfa` `=` $is_sfa `,` - `tmem_alloc_mode` `=` $tmem_alloc_mode `>` - }]; -} - -def CuteNVGPU_SM100TmemEFrgAttr - : CuteNVGPU_Attr<"SM100TmemEFrg", "arch.sm100.tmem_e_frg"> { - let summary = "SM100 tensor-memory sparse-metadata fragment"; - let description = [{ - `#cute_nvgpu.arch.sm100.tmem_e_frg` - }]; - let parameters = (ins "::mlir::Type":$a_type, "::mlir::Type":$e_type); - let assemblyFormat = "`<` `a_type` `=` $a_type `,` `e_type` `=` $e_type `>`"; -} - -def CuteNVGPU_NotImplementedFrgAttr - : CuteNVGPU_Attr<"NotImplementedFrg", "not_implemented_frg"> { - let summary = "Fragment of an atom that has none"; - let description = "`#cute_nvgpu.not_implemented_frg<>`"; - let assemblyFormat = "`<` `>`"; -} - -#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUATTRS_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp deleted file mode 100644 index a912714e..00000000 --- a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp +++ /dev/null @@ -1,1921 +0,0 @@ -//===- CuteNVGPUDialect.cpp - CuTe NVIDIA GPU dialect ---------------------===// -// -// Parsers, printers and verifiers of the cute_nvgpu types that the declarative -// format cannot express. The syntax and the verifier messages are the DSL -// compiler's (see CuteNVGPUTypes.td). - -#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" - -#include "mlir/AsmParser/AsmParser.h" -#include "mlir/IR/Builders.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/DialectImplementation.h" -#include "llvm/ADT/StringExtras.h" -#include "llvm/ADT/TypeSwitch.h" - -#include -#include - -using namespace mlir; -using namespace mlir::cutlass_compiler::cute_nvgpu; -namespace cute = mlir::cutlass_compiler::cute; - -#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.cpp.inc" - -#include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.cpp.inc" - -#define GET_ATTRDEF_CLASSES -#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.cpp.inc" - -#define GET_TYPEDEF_CLASSES -#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.cpp.inc" - -#define GET_OP_CLASSES -#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.cpp.inc" - -FailureOr mlir::cutlass_compiler::cute_nvgpu::parseBool(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - StringRef kw; - if (p.parseKeyword(&kw)) - return failure(); - if (kw == "true" || kw == "false") - return kw == "true"; - return p.emitError(loc, "expected `true` or `false`"); -} - -//===----------------------------------------------------------------------===// -// Helpers -//===----------------------------------------------------------------------===// - -namespace { - -using EmitFn = llvm::function_ref; - -template ParseResult field(AsmParser &p, T &out) { - FailureOr v = FieldParser::parse(p); - if (failed(v)) - return failure(); - out = *v; - return success(); -} -ParseResult field(AsmParser &p, Type &out) { return p.parseType(out); } -ParseResult field(AsmParser &p, int &out) { return p.parseInteger(out); } -ParseResult field(AsmParser &p, bool &out) { - FailureOr v = parseBool(p); - if (failed(v)) - return failure(); - out = *v; - return success(); -} - -// `key = value`. -template -ParseResult keyField(AsmParser &p, StringRef key, T &out) { - return failure(p.parseKeyword(key) || p.parseEqual() || field(p, out)); -} -// `, key = value`. -template -ParseResult nextKeyField(AsmParser &p, StringRef key, T &out) { - return failure(p.parseComma() || keyField(p, key, out)); -} -// An optional ` key = value` group. -template -ParseResult optKeyField(AsmParser &p, StringRef key, T &out) { - if (failed(p.parseOptionalKeyword(key))) - return success(); - return failure(p.parseEqual() || field(p, out)); -} -// `N word`, as in `128 b` or `32 DP`. -ParseResult countOf(AsmParser &p, StringRef word, int &out) { - return failure(p.parseInteger(out) || p.parseKeyword(word)); -} -// `(a, b)`. -template ParseResult pairOf(AsmParser &p, T &a, T &b) { - return failure(p.parseLParen() || field(p, a) || p.parseComma() || - field(p, b) || p.parseRParen()); -} -// `elem_type = (A, B, C)`. -ParseResult elemTypes(AsmParser &p, Type &a, Type &b, Type &c) { - return failure(p.parseKeyword("elem_type") || p.parseEqual() || - p.parseLParen() || p.parseType(a) || p.parseComma() || - p.parseType(b) || p.parseComma() || p.parseType(c) || - p.parseRParen()); -} -void printElemTypes(AsmPrinter &p, Type a, Type b, Type c) { - p << "elem_type = (" << a << ", " << b << ", " << c << ")"; -} -// `xR`. -ParseResult repeat(AsmParser &p, unsigned &out) { - SMLoc loc = p.getCurrentLocation(); - StringRef kw; - if (p.parseKeyword(&kw)) - return failure(); - if (!kw.consume_front("x")) - return p.emitError( - loc, "failed to parse parameter 'num_rep' which is to start with `x`"); - if (kw.getAsInteger(10, out)) - return p.emitError( - loc, "failed to parse parameter 'num_rep' which is to be a `unsigned`"); - return success(); -} - -template -void printIfNot(AsmPrinter &p, StringRef key, E value, E dflt) { - if (value != dflt) - p << ' ' << key << " = " << stringifyEnum(value); -} -template -void printAttrField(AsmPrinter &p, StringRef key, A attr) { - if (!attr) - return; - p << ", " << key << " = "; - p.printStrippedAttrOrType(attr); -} - -// The width of an element: integers and floats, 64 for index and pointers, -// a sparse element's share of its physical storage. -int elementBits(Type t) { - if (auto s = llvm::dyn_cast(t)) - return s.getNumLogical() - ? elementBits(s.getPhysicalType()) / s.getNumLogical() - : 0; - if (t.isTF32()) - return 32; - if (t.isIntOrFloat()) - return t.getIntOrFloatBitWidth(); - if (llvm::isa(t)) - return 64; - return 0; -} -int orElementBits(int bits, Type t) { return bits ? bits : elementBits(t); } - -// The integers of a flat tuple `(a,b,c)`. -std::optional> flatInts(StringRef text) { - if (!text.consume_front("(") || !text.consume_back(")")) - return std::nullopt; - SmallVector out; - for (StringRef part : llvm::split(text, ',')) { - int64_t v; - if (part.getAsInteger(10, v)) - return std::nullopt; - out.push_back(v); - } - return out; -} - -// `MxNxK`. -ParseResult parseShapeMNK(AsmParser &p, cute::ShapeAttr &out) { - SMLoc loc = p.getCurrentLocation(); - SmallVector dims; - if (p.parseDimensionList(dims, /*allowDynamic=*/false, - /*withTrailingX=*/false)) - return p.emitError(loc, "can not parse shape_MNK"); - auto shape = cutegen::from_string( - "(" + - llvm::join( - llvm::map_range(dims, [](int64_t d) { return std::to_string(d); }), - ",") + - ")"); - if (!shape) - return p.emitError(loc, "can not parse shape_MNK"); - out = cute::ShapeAttr::get(p.getContext(), *shape); - return success(); -} -void printShapeMNK(AsmPrinter &p, cute::ShapeAttr shape) { - std::string text = cutegen::to_string(shape.getRef()); - StringRef body(text); - body.consume_front("("); - body.consume_back(")"); - for (char c : body) - p << (c == ',' ? 'x' : c); -} -LogicalResult verifyShapeMNK(EmitFn emitError, cute::ShapeAttr shape) { - std::string text = cutegen::to_string(shape.getRef()); - auto dims = flatInts(text); - if (!dims || dims->size() != 3) - return emitError() << "expects flatten rank-3 tuple of shape, but got " - << text; - return success(); -} - -// The A fragment kind: `ss` (shared memory), `rs` (registers) or `ts` -// (tensor memory). -ParseResult parseFragKind(AsmParser &p, MmaFragKind other, MmaFragKind &out) { - SMLoc loc = p.getCurrentLocation(); - StringRef kw; - bool ts = other == MmaFragKind::tmem; - if (p.parseKeyword("frag_kind") || p.parseEqual() || p.parseKeyword(&kw)) - return failure(); - if (kw == "ss") - out = MmaFragKind::smem_desc; - else if (kw == (ts ? "ts" : "rs")) - out = other; - else - return p.emitError(loc, - ts ? "expects `ts` or `ss`" : "expects `rs` or `ss`"); - return success(); -} -StringRef fragKindText(MmaFragKind k) { - switch (k) { - case MmaFragKind::smem_desc: - return "ss"; - case MmaFragKind::rmem: - return "rs"; - case MmaFragKind::tmem: - return "ts"; - default: - return stringifyEnum(k); - } -} - -// `t` or `n`. -ParseResult parseTranspose(AsmParser &p, UnitAttr &out) { - SMLoc loc = p.getCurrentLocation(); - StringRef kw; - if (p.parseKeyword(&kw)) - return failure(); - if (kw != "t" && kw != "n") - return p.emitError(loc, "expects `t` or `n`"); - out = kw == "t" ? UnitAttr::get(p.getContext()) : UnitAttr(); - return success(); -} - -bool isPow2AtLeast8(int v) { return v >= 8 && llvm::isPowerOf2_32(v); } - -} // namespace - -//===----------------------------------------------------------------------===// -// Descriptors -//===----------------------------------------------------------------------===// - -// sm103/sm107.smem_desc_circular: static `(B,128,N):(128,1,128*B)`, N >= 2. -static LogicalResult verifyCircular(EmitFn emitError, cute::LayoutAttr layout) { - std::string text = cutegen::to_string(layout.getRef()); - StringRef shapeText, strideText; - std::tie(shapeText, strideText) = StringRef(text).split(':'); - if (StringRef(text).contains('?')) - return emitError() << "block layout must be fully static but got " << text; - auto shape = flatInts(shapeText), stride = flatInts(strideText); - if (!shape || shape->size() != 3 || !stride || stride->size() != 3) - return emitError() << "expects block layout to have 3 modes, but got " - << text; - if ((*shape)[1] != 128) - return emitError() << "expects block layout to have 128B 1st mode, but got " - << text; - if ((*stride)[1] != 1 || (*stride)[0] != 128 || - (*stride)[2] != 128 * (*shape)[0]) - return emitError() << "expects mode-1 major compact layout, but got " - << text; - if ((*shape)[2] < 2) - return emitError() - << "Circular SMEM descriptor requires at least 2 128B blocks since " - "MMA may span 2 buffers, but got " - << text; - return success(); -} -LogicalResult SmemDescCircularSM103Type::verify(EmitFn emitError, - cute::LayoutAttr layout) { - return verifyCircular(emitError, layout); -} -LogicalResult SmemDescCircularSM107Type::verify(EmitFn emitError, - cute::LayoutAttr layout) { - return verifyCircular(emitError, layout); -} - -Type SmemDescViewType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type desc; - std::string text; - if (p.parseLess() || p.parseType(desc) || p.parseComma()) - return {}; - SMLoc layoutLoc = p.getCurrentLocation(); - if (p.parseString(&text) || p.parseGreater()) - return {}; - auto layout = cutegen::from_string(text); - if (!layout) { - p.emitError(layoutLoc, "failed to parse SmemDescViewType parameter " - "'layout' as `cutegen::layout`"); - return {}; - } - return getChecked([&] { return p.emitError(loc); }, p.getContext(), desc, - cute::LayoutAttr::get(p.getContext(), *layout)); -} -void SmemDescViewType::print(AsmPrinter &p) const { - p << '<' << getDesc() << ", "; - p.printString(cutegen::to_string(getLayout().getRef())); - p << '>'; -} -LogicalResult SmemDescViewType::verify(EmitFn emitError, Type desc, - cute::LayoutAttr) { - if (!llvm::isa(desc)) - return emitError() << "expected SmemDescType, SmemDescCircularSM103Type, " - "SmemDescCircularSM107Type or SmemDescSM107Type"; - return success(); -} - -//===----------------------------------------------------------------------===// -// SIMT copy atoms -//===----------------------------------------------------------------------===// - -// ``; the compiler prints -// `` for copy_bits 0, which is accepted too. -Type CopyAtomSIMTSyncCopyType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int bits = 0; - auto src = cute::AddressSpace::generic, dst = src; - if (p.parseLess() || p.parseType(t)) - return {}; - // `src_space` / `dst_space` groups; one alone is what the compiler prints - // when the other is generic (or copy_bits is 0), so it is read too. - auto spaces = [&]() -> ParseResult { - if (succeeded(p.parseOptionalKeyword("dst_space"))) - return failure(p.parseEqual() || field(p, dst)); - if (p.parseKeyword("src_space") || p.parseEqual() || field(p, src)) - return failure(); - if (succeeded(p.parseOptionalComma())) - return keyField(p, "dst_space", dst); - return success(); - }; - if (succeeded(p.parseOptionalComma())) { - OptionalParseResult n = p.parseOptionalInteger(bits); - if (n.has_value() && (failed(*n) || p.parseKeyword("b"))) - return {}; - if ((!n.has_value() || succeeded(p.parseOptionalComma())) && spaces()) - return {}; - } - if (p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, bits, - src, dst); -} -void CopyAtomSIMTSyncCopyType::print(AsmPrinter &p) const { - auto generic = cute::AddressSpace::generic; - bool spaces = getSrcSpace() != generic || getDstSpace() != generic; - p << '<' << getValType(); - if (getCopyBits() || spaces) - p << ", " << getCopyBits() << " b"; - if (spaces) - p << ", src_space = " << stringifyEnum(getSrcSpace()); - if (getDstSpace() != generic) - p << ", dst_space = " << stringifyEnum(getDstSpace()); - p << '>'; -} -LogicalResult CopyAtomSIMTSyncCopyType::verify(EmitFn emitError, Type, int bits, - cute::AddressSpace, - cute::AddressSpace) { - if (bits != 0 && !isPow2AtLeast8(bits)) - return emitError() << "expected copy_bits to be 0 (auto) or a power of 2 " - "(>= 8), but got " - << bits; - return success(); -} - -// ``. -Type CopyAtomSIMTAsyncCopyType::parse(AsmParser &p) { - Type t; - int bits = 0; - LoadCacheMode cache; - if (p.parseLess() || p.parseType(t) || nextKeyField(p, "cache", cache)) - return {}; - if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) - return {}; - if (p.parseGreater()) - return {}; - return get(p.getContext(), t, cache, orElementBits(bits, t)); -} -void CopyAtomSIMTAsyncCopyType::print(AsmPrinter &p) const { - p << '<' << getValType() << ", cache = " << stringifyEnum(getCache()) << ", " - << getCopyBits() << " b>"; -} - -// ``. -Type CopyAtomDsmemStoreType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int bits = 0; - if (p.parseLess() || p.parseType(t)) - return {}; - if (succeeded(p.parseOptionalComma()) && countOf(p, "b", bits)) - return {}; - if (p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, - orElementBits(bits, t)); -} -void CopyAtomDsmemStoreType::print(AsmPrinter &p) const { - p << '<' << getValType() << ", " << getCopyBits() << " b>"; -} -LogicalResult CopyAtomDsmemStoreType::verify(EmitFn emitError, Type, int bits) { - if (bits != 0 && bits != 32 && bits != 64 && bits != 128) - return emitError() - << "expects copy_bits to be one of (0, 32, 64, 128), but got " - << bits; - return success(); -} - -// Bulk copies: ``. -Type CopyAtomBulkCopyG2SType::parse(AsmParser &p) { - Type t; - int bits = 0; - bool mcast = false; - if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || - optKeyField(p, "mcast", mcast) || p.parseGreater()) - return {}; - return get(p.getContext(), t, bits, mcast); -} -void CopyAtomBulkCopyG2SType::print(AsmPrinter &p) const { - p << '<' << getValType(); - if (getCopyBits()) - p << " copy_bits = " << getCopyBits(); - if (getMcast()) - p << " mcast = true"; - p << '>'; -} -Type CopyAtomBulkCopyS2GType::parse(AsmParser &p) { - Type t; - int bits = 0; - bool mask = false; - if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || - optKeyField(p, "mask", mask) || p.parseGreater()) - return {}; - return get(p.getContext(), t, bits, mask); -} -void CopyAtomBulkCopyS2GType::print(AsmPrinter &p) const { - p << '<' << getValType(); - if (getCopyBits()) - p << " copy_bits = " << getCopyBits(); - if (getMask()) - p << " mask = true"; - p << '>'; -} -Type CopyAtomBulkCopyS2SType::parse(AsmParser &p) { - Type t; - int bits = 0; - if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || - p.parseGreater()) - return {}; - return get(p.getContext(), t, bits); -} -void CopyAtomBulkCopyS2SType::print(AsmPrinter &p) const { - p << '<' << getValType(); - if (getCopyBits()) - p << " copy_bits = " << getCopyBits(); - p << '>'; -} - -// g2r, r2g, r2s, s2r: ``. -static LogicalResult verifyCopyBits(EmitFn emitError, int bits) { - if (!isPow2AtLeast8(bits)) - return emitError() - << "expects copy_bits to be a power of 2 (>= 8), but got " << bits; - return success(); -} - -Type CopyAtomG2RType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int bits = 0; - auto order = MemOrderKind::WEAK; - auto scope = MemScopeKind::CTA; - auto prefetch = L2PrefetchSize::NONE; - auto evict = CacheEvictionPriority::EVICT_NORMAL; - auto cache = LoadCacheMode::always; - auto shared = SharedSpace::CTA; - bool invariant = false; - if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || - optKeyField(p, "mem_order", order) || - optKeyField(p, "mem_scope", scope) || - optKeyField(p, "l2_prefetch_size", prefetch) || - optKeyField(p, "l1_cache_evict_priority", evict) || - optKeyField(p, "load_cache_mode", cache) || - optKeyField(p, "shared_space", shared) || - optKeyField(p, "invariant", invariant) || p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, - orElementBits(bits, t), order, scope, prefetch, evict, - cache, shared, invariant); -} -void CopyAtomG2RType::print(AsmPrinter &p) const { - p << '<' << getValType() << " copy_bits = " << getCopyBits(); - printIfNot(p, "mem_order", getMemOrder(), MemOrderKind::WEAK); - printIfNot(p, "mem_scope", getMemScope(), MemScopeKind::CTA); - printIfNot(p, "l2_prefetch_size", getL2PrefetchSize(), L2PrefetchSize::NONE); - printIfNot(p, "l1_cache_evict_priority", getL1CacheEvictPriority(), - CacheEvictionPriority::EVICT_NORMAL); - printIfNot(p, "load_cache_mode", getLoadCacheMode(), LoadCacheMode::always); - printIfNot(p, "shared_space", getSharedSpace(), SharedSpace::CTA); - if (getInvariant()) - p << " invariant = true"; - p << '>'; -} -LogicalResult CopyAtomG2RType::verify(EmitFn emitError, Type, int bits, - MemOrderKind order, MemScopeKind scope, - L2PrefetchSize prefetch, - CacheEvictionPriority evict, - LoadCacheMode cache, SharedSpace, - bool invariant) { - if (failed(verifyCopyBits(emitError, bits))) - return failure(); - if (invariant && (order != MemOrderKind::WEAK || scope != MemScopeKind::CTA || - prefetch != L2PrefetchSize::NONE || - evict != CacheEvictionPriority::EVICT_NORMAL || - cache != LoadCacheMode::always)) - return emitError() - << "invariant_load is not allowed when the other non-default memory " - "features (e.g., mem_order) are set to non-default values"; - if (order != MemOrderKind::WEAK && cache != LoadCacheMode::always) - return emitError() - << "expects load_cache_mode to be 'always' when mem_order is not " - "'weak', but got '" - << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; - return success(); -} - -Type CopyAtomR2GType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int bits = 0; - auto order = MemOrderKind::WEAK; - auto scope = MemScopeKind::CTA; - auto evict = CacheEvictionPriority::EVICT_NORMAL; - auto cache = StoreCacheMode::write_back; - auto shared = SharedSpace::CTA; - if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || - optKeyField(p, "mem_order", order) || - optKeyField(p, "mem_scope", scope) || - optKeyField(p, "l1_cache_evict_priority", evict) || - optKeyField(p, "store_cache_mode", cache) || - optKeyField(p, "shared_space", shared) || p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, - orElementBits(bits, t), order, scope, evict, cache, shared); -} -void CopyAtomR2GType::print(AsmPrinter &p) const { - p << '<' << getValType() << " copy_bits = " << getCopyBits(); - printIfNot(p, "mem_order", getMemOrder(), MemOrderKind::WEAK); - printIfNot(p, "mem_scope", getMemScope(), MemScopeKind::CTA); - printIfNot(p, "l1_cache_evict_priority", getL1CacheEvictPriority(), - CacheEvictionPriority::EVICT_NORMAL); - printIfNot(p, "store_cache_mode", getStoreCacheMode(), - StoreCacheMode::write_back); - printIfNot(p, "shared_space", getSharedSpace(), SharedSpace::CTA); - p << '>'; -} -LogicalResult CopyAtomR2GType::verify(EmitFn emitError, Type, int bits, - MemOrderKind order, MemScopeKind, - CacheEvictionPriority, - StoreCacheMode cache, SharedSpace) { - if (failed(verifyCopyBits(emitError, bits))) - return failure(); - if (order != MemOrderKind::WEAK && cache != StoreCacheMode::write_back) - return emitError() - << "expects store_cache_mode to be 'write_back' when mem_order is " - "not 'weak', but got '" - << stringifyEnum(cache) << "' and '" << stringifyEnum(order) << "'"; - return success(); -} - -template static Type parseSharedCopy(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int bits = 0; - auto order = MemOrderKind::WEAK; - auto scope = MemScopeKind::CTA; - auto shared = SharedSpace::CTA; - if (p.parseLess() || p.parseType(t) || optKeyField(p, "copy_bits", bits) || - optKeyField(p, "mem_order", order) || - optKeyField(p, "mem_scope", scope) || - optKeyField(p, "shared_space", shared) || p.parseGreater()) - return {}; - return T::getChecked([&] { return p.emitError(loc); }, p.getContext(), t, - orElementBits(bits, t), order, scope, shared); -} -template static void printSharedCopy(AsmPrinter &p, T type) { - p << '<' << type.getValType() << " copy_bits = " << type.getCopyBits(); - printIfNot(p, "mem_order", type.getMemOrder(), MemOrderKind::WEAK); - printIfNot(p, "mem_scope", type.getMemScope(), MemScopeKind::CTA); - printIfNot(p, "shared_space", type.getSharedSpace(), SharedSpace::CTA); - p << '>'; -} -Type CopyAtomR2SType::parse(AsmParser &p) { - return parseSharedCopy(p); -} -void CopyAtomR2SType::print(AsmPrinter &p) const { printSharedCopy(p, *this); } -LogicalResult CopyAtomR2SType::verify(EmitFn emitError, Type, int bits, - MemOrderKind, MemScopeKind, SharedSpace) { - return verifyCopyBits(emitError, bits); -} -Type CopyAtomS2RType::parse(AsmParser &p) { - return parseSharedCopy(p); -} -void CopyAtomS2RType::print(AsmPrinter &p) const { printSharedCopy(p, *this); } -LogicalResult CopyAtomS2RType::verify(EmitFn emitError, Type, int bits, - MemOrderKind, MemScopeKind, SharedSpace) { - return verifyCopyBits(emitError, bits); -} - -//===----------------------------------------------------------------------===// -// Multimem atoms: `, ORDER, SCOPE>`, positional. Parsing -// takes each field by its kind, which also reads what the compiler prints -// when it drops a default field before a set one. -//===----------------------------------------------------------------------===// - -namespace { -struct Multimem { - int bits = 0; - ReductionKind red = ReductionKind::ADD; - LdReduceAccPrecisionKindAttr acc; - MemOrderKind order = MemOrderKind::RELAXED; - MemScopeKind scope = MemScopeKind::SYS; -}; - -ParseResult parseMultimem(AsmParser &p, Type &t, Multimem &m, bool hasRed, - bool hasAcc) { - if (p.parseLess() || p.parseType(t)) - return failure(); - while (succeeded(p.parseOptionalComma())) { - SMLoc loc = p.getCurrentLocation(); - StringRef kw; - OptionalParseResult bits = p.parseOptionalInteger(m.bits); - if (bits.has_value()) { - if (failed(*bits) || p.parseKeyword("b")) - return failure(); - continue; - } - if (hasAcc && succeeded(p.parseOptionalLess())) { - LdReduceAccPrecisionKind kind; - if (field(p, kind) || p.parseGreater()) - return failure(); - m.acc = LdReduceAccPrecisionKindAttr::get(p.getContext(), kind); - continue; - } - if (p.parseKeyword(&kw)) - return failure(); - if (auto r = symbolizeReductionKind(kw); r && hasRed) - m.red = *r; - else if (auto o = symbolizeMemOrderKind(kw)) - m.order = *o; - else if (auto s = symbolizeMemScopeKind(kw)) - m.scope = *s; - else - return p.emitError(loc, "unexpected `") << kw << "`"; - } - return p.parseGreater(); -} - -// Prints the fields up to the last one that is not default. -void printMultimem(AsmPrinter &p, Type t, const Multimem &m, bool bitsSet, - bool hasRed, bool hasAcc) { - int last = m.scope != MemScopeKind::SYS ? 4 - : m.order != MemOrderKind::RELAXED ? 3 - : m.acc ? 2 - : hasRed && m.red != ReductionKind::ADD ? 1 - : bitsSet ? 0 - : -1; - p << '<' << t; - if (last >= 0) - p << ", " << m.bits << " b"; - if (hasRed && last >= 1) - p << ", " << stringifyEnum(m.red); - if (hasAcc && last >= 2 && m.acc) { - p << ", "; - p.printStrippedAttrOrType(m.acc); - } - if (last >= 3) - p << ", " << stringifyEnum(m.order); - if (last >= 4) - p << ", " << stringifyEnum(m.scope); - p << '>'; -} -} // namespace - -Type CopyAtomSIMTMultimemStType::parse(AsmParser &p) { - Type t; - Multimem m; - if (parseMultimem(p, t, m, /*hasRed=*/false, /*hasAcc=*/false)) - return {}; - return get(p.getContext(), t, m.bits, m.order, m.scope); -} -void CopyAtomSIMTMultimemStType::print(AsmPrinter &p) const { - Multimem m{ - getCopyBits(), ReductionKind::ADD, {}, getMemOrder(), getMemScope()}; - printMultimem(p, getValType(), m, getCopyBits() != 0, false, false); -} - -Type CopyAtomSIMTMultimemRedType::parse(AsmParser &p) { - Type t; - Multimem m; - if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/false)) - return {}; - // An unset width is the element's, at least 32 bits (the f16x2 form). - int bits = m.bits ? m.bits : std::max(elementBits(t), 32); - return get(p.getContext(), t, bits, m.red, m.order, m.scope); -} -void CopyAtomSIMTMultimemRedType::print(AsmPrinter &p) const { - Multimem m{getCopyBits(), getReduction(), {}, getMemOrder(), getMemScope()}; - printMultimem(p, getValType(), m, /*bitsSet=*/true, true, false); -} - -Type CopyAtomSIMTMultimemLdReduceType::parse(AsmParser &p) { - Type t; - Multimem m; - if (parseMultimem(p, t, m, /*hasRed=*/true, /*hasAcc=*/true)) - return {}; - return get(p.getContext(), t, m.bits, m.red, m.acc, m.order, m.scope); -} -void CopyAtomSIMTMultimemLdReduceType::print(AsmPrinter &p) const { - Multimem m{getCopyBits(), getReduction(), getLdReduceAccPrecision(), - getMemOrder(), getMemScope()}; - printMultimem(p, getValType(), m, getCopyBits() != 0, true, true); -} - -//===----------------------------------------------------------------------===// -// Tensor-memory atoms: `` -//===----------------------------------------------------------------------===// - -static ParseResult parseTmemHead(AsmParser &p, Type &t, int &dp, int &bit, - unsigned &rep) { - return failure(p.parseLess() || p.parseType(t) || p.parseComma() || - countOf(p, "DP", dp) || p.parseComma() || - countOf(p, "bit", bit) || p.parseComma() || repeat(p, rep)); -} -static void printTmemHead(AsmPrinter &p, Type t, int dp, int bit, - unsigned rep) { - p << '<' << t << ", " << dp << " DP, " << bit << " bit, x" << rep; -} -// An optional `, word`. -static ParseResult parseFlag(AsmParser &p, StringRef word, UnitAttr &out) { - if (failed(p.parseOptionalComma())) - return success(); - if (p.parseKeyword(word)) - return failure(); - out = UnitAttr::get(p.getContext()); - return success(); -} - -static LogicalResult verifyTmemCopy(EmitFn emitError, int dp, int bit, - unsigned rep) { - if (dp != 16 && dp != 32) - return emitError() - << "expects the number of data paths to be one of [16,32], but got " - << dp; - if (bit != 32 && bit != 64 && bit != 128 && bit != 256) - return emitError() << "expects the number of bits to be one of " - "[32,64,128,256], but got " - << bit; - if (rep < 1 || rep > 128 || !llvm::isPowerOf2_32(rep)) - return emitError() << "expects the number of repetitions to be one of " - "[1,2,4,8,16,32,64,128], but got " - << rep; - if (dp == 32 && bit != 32) - return emitError() - << "only 32b patterns are supported when the number of data paths " - "is 32, but got " - << bit; - if (dp == 16 && bit == 256 && rep >= 64) - return emitError() - << "a repetition of 64 or 128 is not supported when the number of " - "data paths is 16 and the number of bits is 256"; - if (dp == 16 && bit == 128 && rep == 128) - return emitError() - << "a repetition of 128 is not supported when the number of data " - "paths is 16 and the number of bits is 128"; - return success(); -} - -Type CopyAtomSM100TmemLoadType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int dp, bit; - unsigned rep; - UnitAttr pack; - if (parseTmemHead(p, t, dp, bit, rep) || parseFlag(p, "pack16b", pack) || - p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, - bit, rep, pack); -} -void CopyAtomSM100TmemLoadType::print(AsmPrinter &p) const { - printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); - p << (getPack16b() ? ", pack16b>" : ">"); -} -LogicalResult CopyAtomSM100TmemLoadType::verify(EmitFn emitError, Type, int dp, - int bit, unsigned rep, - UnitAttr) { - return verifyTmemCopy(emitError, dp, bit, rep); -} - -Type CopyAtomSM100TmemStoreType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int dp, bit; - unsigned rep; - UnitAttr expand; - if (parseTmemHead(p, t, dp, bit, rep) || parseFlag(p, "expand16b", expand) || - p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, - bit, rep, expand); -} -void CopyAtomSM100TmemStoreType::print(AsmPrinter &p) const { - printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); - p << (getExpand16b() ? ", expand16b>" : ">"); -} -LogicalResult CopyAtomSM100TmemStoreType::verify(EmitFn emitError, Type, int dp, - int bit, unsigned rep, - UnitAttr) { - return verifyTmemCopy(emitError, dp, bit, rep); -} - -// `<..., OP[, nan][, half_split_off = N[ : T]]>`. -Type CopyAtomSM10xTmemLoadRedType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int dp, bit; - unsigned rep; - TmemLoadRedOp op; - UnitAttr nan; - IntegerAttr split; - if (parseTmemHead(p, t, dp, bit, rep) || p.parseComma() || field(p, op)) - return {}; - while (succeeded(p.parseOptionalComma())) { - if (!nan && !split && succeeded(p.parseOptionalKeyword("nan"))) { - nan = UnitAttr::get(p.getContext()); - continue; - } - if (p.parseKeyword("half_split_off") || p.parseEqual() || - p.parseAttribute(split)) - return {}; - } - if (p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, - bit, rep, op, nan, split); -} -void CopyAtomSM10xTmemLoadRedType::print(AsmPrinter &p) const { - printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); - p << ", " << stringifyEnum(getRedOp()); - if (getNan()) - p << ", nan"; - if (getHalfSplitOff()) - p << ", half_split_off=" << getHalfSplitOff(); - p << '>'; -} -LogicalResult -CopyAtomSM10xTmemLoadRedType::verify(EmitFn emitError, Type t, int dp, int bit, - unsigned rep, TmemLoadRedOp op, - UnitAttr nan, IntegerAttr split) { - if (!(t.isF32() || t.isInteger(32))) - return emitError() << "expect val_type to be u32, s32, or f32, but got '" - << t << "'"; - if (nan && !t.isF32()) - return emitError() << "expect val_type to be f32 when nan is specified"; - if ((op == TmemLoadRedOp::minabs || op == TmemLoadRedOp::maxabs) && - !t.isF32()) - return emitError() - << "expect val_type to be f32 when red_op is minabs or maxabs"; - if (bit != 32) - return emitError() << "only 32b patterns are supported, but got " << bit; - if (rep < 2 || rep > 128 || !llvm::isPowerOf2_32(rep)) - return emitError() << "expects the number of repetitions to be one of " - "[2,4,8,16,32,64,128], but got " - << rep; - if (split && dp != 16) - return emitError() - << "expect half_split_off to be set only when mode is 16dp 32bit"; - return success(); -} - -// `<..., OP[, red[, nan]]>`. -Type CopyAtomSM107TmemLoadSPCompressType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int dp, bit; - unsigned rep; - TmemLoadRedOp op; - UnitAttr red, nan; - if (parseTmemHead(p, t, dp, bit, rep) || p.parseComma() || field(p, op) || - parseFlag(p, "red", red) || (red && parseFlag(p, "nan", nan)) || - p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, - bit, rep, op, red, nan); -} -void CopyAtomSM107TmemLoadSPCompressType::print(AsmPrinter &p) const { - printTmemHead(p, getValType(), getNumDp(), getNumBit(), getNumRep()); - p << ", " << stringifyEnum(getRedOp()); - if (getRed()) - p << ", red"; - if (getNan()) - p << ", nan"; - p << '>'; -} -LogicalResult CopyAtomSM107TmemLoadSPCompressType::verify( - EmitFn emitError, Type t, int dp, int bit, unsigned rep, TmemLoadRedOp, - UnitAttr red, UnitAttr nan) { - if (dp != 32 || bit != 32) - return emitError() << "expect `num_dp` and `num_bit` to be 32, but got " - << dp << " and " << bit; - if (!t.isF32()) - return emitError() << "expect val_type to be f32, but got '" << t << "'"; - if (rep < 4 || rep > 128 || !llvm::isPowerOf2_32(rep)) - return emitError() << "expect the number of repetitions to be one of " - "[4,8,16,32,64,128], but got " - << rep; - if (nan && !red) - return emitError() << "expect `red` to be true when `nan` is true"; - return success(); -} - -// The UTCCP shapes of tcgen05.cp. -static bool isUtccp(int dp, int bit, CopyS2TBroadcast b) { - using B = CopyS2TBroadcast; - return (b == B::none && (dp == 4 || dp == 128) && - (bit == 128 || bit == 256)) || - (b == B::x4 && dp == 32 && bit == 128) || - ((b == B::lw_0213 || b == B::lw_0123) && dp == 64 && bit == 128); -} - -// ``. -Type CopyAtomSM100CopyS2TType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int dp, bit, cta; - auto bcast = CopyS2TBroadcast::none; - if (p.parseLess() || p.parseType(t) || p.parseComma() || - countOf(p, "DP", dp) || p.parseComma() || countOf(p, "bit", bit) || - p.parseComma() || countOf(p, "cta", cta)) - return {}; - if (succeeded(p.parseOptionalComma()) && field(p, bcast)) - return {}; - if (p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, - bit, cta, bcast); -} -void CopyAtomSM100CopyS2TType::print(AsmPrinter &p) const { - p << '<' << getValType() << ", " << getNumDp() << " DP, " << getNumBit() - << " bit, " << getNumCta() << " cta"; - if (getBroadcast() != CopyS2TBroadcast::none) - p << ", " << stringifyEnum(getBroadcast()); - p << '>'; -} -LogicalResult CopyAtomSM100CopyS2TType::verify(EmitFn emitError, Type, int dp, - int bit, int cta, - CopyS2TBroadcast b) { - if (cta != 1 && cta != 2) - return emitError() << "Expect cta is 1/2 but got: " << cta; - if (!isUtccp(dp, bit, b)) - return emitError() << "The given UTCCP params: dp " << dp << ", bits " - << bit << ", broadcast " << stringifyEnum(b) - << " is unsupported or illegal."; - return success(); -} - -// ``. -Type CopyAtomSM100S2TCopyV2Type::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int dp, bit, cta; - cute::MajorMode major; - auto bcast = CopyS2TBroadcast::none; - if (p.parseLess() || p.parseType(t) || nextKeyField(p, "num_dp", dp) || - nextKeyField(p, "num_bit", bit) || nextKeyField(p, "num_cta", cta) || - nextKeyField(p, "smem_major", major)) - return {}; - if (succeeded(p.parseOptionalComma()) && keyField(p, "broadcast", bcast)) - return {}; - if (p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, dp, - bit, cta, major, bcast); -} -void CopyAtomSM100S2TCopyV2Type::print(AsmPrinter &p) const { - p << '<' << getValType() << ", num_dp = " << getNumDp() - << ", num_bit = " << getNumBit() << ", num_cta = " << getNumCta() - << ", smem_major = " << stringifyEnum(getSmemMajor()); - if (getBroadcast() != CopyS2TBroadcast::none) - p << ", broadcast = " << stringifyEnum(getBroadcast()); - p << '>'; -} -LogicalResult CopyAtomSM100S2TCopyV2Type::verify(EmitFn emitError, Type, int dp, - int bit, int cta, - cute::MajorMode, - CopyS2TBroadcast b) { - if (cta != 1 && cta != 2) - return emitError() << "expects cta to be 1 or 2, but got " << cta; - if (!isUtccp(dp, bit, b)) - return emitError() << "expects valid UTCCP params, but got dp " << dp - << ", bits " << bit << ", broadcast " - << stringifyEnum(b); - return success(); -} - -//===----------------------------------------------------------------------===// -// ldmatrix / stmatrix -//===----------------------------------------------------------------------===// - -Type CopyAtomLdsmType::parse(AsmParser &p) { - Type t; - cute::ShapeAttr mode; - LdsmSzPattern sz; - int n; - UnitAttr trans; - if (p.parseLess() || keyField(p, "val_type", t) || - nextKeyField(p, "mode", mode) || nextKeyField(p, "sz_pattern", sz) || - nextKeyField(p, "num_matrices", n) || p.parseComma() || - parseTranspose(p, trans) || p.parseGreater()) - return {}; - return get(p.getContext(), t, mode, sz, n, trans); -} -void CopyAtomLdsmType::print(AsmPrinter &p) const { - p << "" : ", n>"); -} - -Type CopyAtomStsmType::parse(AsmParser &p) { - Type t; - cute::ShapeAttr mode; - int n; - UnitAttr trans; - if (p.parseLess() || p.parseType(t) || nextKeyField(p, "mode", mode) || - nextKeyField(p, "num_matrices", n) || p.parseComma() || - parseTranspose(p, trans) || p.parseGreater()) - return {}; - return get(p.getContext(), t, mode, n, trans); -} -void CopyAtomStsmType::print(AsmPrinter &p) const { - p << '<' << getValType() << ", mode = "; - p.printStrippedAttrOrType(getMode()); - p << ", num_matrices = " << getNumMatrices() - << (getTranspose() ? ", t>" : ", n>"); -} - -//===----------------------------------------------------------------------===// -// TMA atoms -//===----------------------------------------------------------------------===// - -static LogicalResult verifySparsity(EmitFn emitError, int sparsity) { - if (sparsity != 1 && sparsity != 2 && sparsity != 4 && sparsity != 8 && - sparsity != 16) - return emitError() - << "expects sparsity to be one of (1, 2, 4, 8, 16), but got " - << sparsity; - return success(); -} -// `'; -} -LogicalResult CopyAtomTmaLoadType::verify(EmitFn emitError, Type, int sparsity, - int, TmaLoadMode, int, - cute::StrideType, bool, - cute::LayoutType, bool, bool) { - return verifySparsity(emitError, sparsity); -} - -Type CopyAtomTmaStoreType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Type t; - int sparsity = 1, bits; - TmaStoreMode mode; - cute::StrideType stride; - cute::LayoutType gbasis; - bool override = false, noOob = false; - if (parseTmaHead(p, t, &sparsity, bits) || nextKeyField(p, "mode", mode) || - nextKeyField(p, "g_stride", stride) || - optKeyField(p, "tma_gbasis", gbasis) || - optKeyField(p, "override", override) || - optKeyField(p, "no_fully_oob_tile", noOob) || p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, - sparsity, bits, mode, stride, gbasis, override, noOob); -} -void CopyAtomTmaStoreType::print(AsmPrinter &p) const { - printTmaHead(p, getValType(), getSparsity(), getCopyBits()); - p << ", mode = " << stringifyEnum(getMode()); - printTmaTail(p, getGStride(), getTmaGbasis()); - printTrue(p, "override", getOverride()); - printTrue(p, "no_fully_oob_tile", getNoFullyOobTile()); - p << '>'; -} -LogicalResult CopyAtomTmaStoreType::verify(EmitFn emitError, Type, int sparsity, - int, TmaStoreMode, cute::StrideType, - cute::LayoutType, bool, bool) { - return verifySparsity(emitError, sparsity); -} - -Type CopyAtomTmaReduceType::parse(AsmParser &p) { - Type t; - int bits; - TmaStoreMode mode; - ReductionKind kind; - cute::StrideType stride; - cute::LayoutType gbasis; - if (parseTmaHead(p, t, nullptr, bits) || nextKeyField(p, "mode", mode) || - nextKeyField(p, "kind", kind) || nextKeyField(p, "g_stride", stride) || - optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) - return {}; - return get(p.getContext(), t, bits, mode, kind, stride, gbasis); -} -void CopyAtomTmaReduceType::print(AsmPrinter &p) const { - printTmaHead(p, getValType(), 1, getCopyBits()); - p << ", mode = " << stringifyEnum(getMode()) - << ", kind = " << stringifyEnum(getKind()); - printTmaTail(p, getGStride(), getTmaGbasis()); - p << '>'; -} - -Type CopyAtomIm2ColTmaLoadType::parse(AsmParser &p) { - Type t; - int bits, ctas; - cute::StrideType stride; - cute::LayoutType gbasis; - bool mcast = false; - if (parseTmaHead(p, t, nullptr, bits) || nextKeyField(p, "num_cta", ctas) || - nextKeyField(p, "g_stride", stride) || optKeyField(p, "mcast", mcast) || - optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) - return {}; - return get(p.getContext(), t, bits, ctas, stride, mcast, gbasis); -} -void CopyAtomIm2ColTmaLoadType::print(AsmPrinter &p) const { - printTmaHead(p, getValType(), 1, getCopyBits()); - p << ", num_cta = " << getNumCta() << ", g_stride = "; - p.printStrippedAttrOrType(getGStride()); - printTrue(p, "mcast", getMcast()); - if (getTmaGbasis()) { - p << " tma_gbasis = "; - p.printStrippedAttrOrType(getTmaGbasis()); - } - p << '>'; -} - -Type CopyAtomIm2ColTmaStoreType::parse(AsmParser &p) { - Type t; - int bits; - cute::StrideType stride; - cute::LayoutType gbasis; - if (parseTmaHead(p, t, nullptr, bits) || - nextKeyField(p, "g_stride", stride) || - optKeyField(p, "tma_gbasis", gbasis) || p.parseGreater()) - return {}; - return get(p.getContext(), t, bits, stride, gbasis); -} -void CopyAtomIm2ColTmaStoreType::print(AsmPrinter &p) const { - printTmaHead(p, getValType(), 1, getCopyBits()); - printTmaTail(p, getGStride(), getTmaGbasis()); - p << '>'; -} - -// The TMA data format of an element type when none is given. -std::optional -mlir::cutlass_compiler::cute_nvgpu::defaultTmaDataFormat(Type t) { - if (auto s = llvm::dyn_cast(t)) - t = s.getPhysicalType(); - if (t.isF16()) - return TmaDataFormat::F16_RN; - if (t.isBF16()) - return TmaDataFormat::BF16_RN; - if (t.isF32()) - return TmaDataFormat::F32_RN; - if (t.isTF32()) - return TmaDataFormat::TF32_RN; - if (t.isF64()) - return TmaDataFormat::F64_RN; - if (!t.isIntOrFloat()) - return std::nullopt; - unsigned bits = t.getIntOrFloatBitWidth(); - bool isSigned = t.isSignedInteger(); - if (bits == 4) - return TmaDataFormat::U4; - if (bits <= 8) - return TmaDataFormat::U8; - if (bits == 16) - return TmaDataFormat::U16; - if (bits == 32) - return isSigned ? TmaDataFormat::S32 : TmaDataFormat::U32; - if (bits == 64) - return isSigned ? TmaDataFormat::S64 : TmaDataFormat::U64; - return std::nullopt; -} - -namespace { -// `<[kind, ]T, copy_bits = N, tma_gbasis = L[, tma_format = F]>`. -template -ParseResult parseNonExecTma(AsmParser &p, Kind *kind, Type &t, int &bits, - cute::LayoutType &gbasis, - std::optional &format) { - if (p.parseLess()) - return failure(); - if (kind && (field(p, *kind) || p.parseComma())) - return failure(); - if (p.parseType(t) || nextKeyField(p, "copy_bits", bits) || - nextKeyField(p, "tma_gbasis", gbasis)) - return failure(); - format = defaultTmaDataFormat(t); - if (succeeded(p.parseOptionalComma())) { - // Empty for an element type without a format, as the compiler prints it. - if (p.parseKeyword("tma_format") || p.parseEqual()) - return failure(); - if (succeeded(p.parseOptionalGreater())) { - format = std::nullopt; - return success(); - } - TmaDataFormat f; - if (field(p, f)) - return failure(); - format = f; - } - return p.parseGreater(); -} -template void printNonExecTma(AsmPrinter &p, T type) { - p << type.getValType() << ", copy_bits = " << type.getCopyBits() - << ", tma_gbasis = "; - p.printStrippedAttrOrType(type.getTmaGbasis()); - p << ", tma_format = "; - if (type.getTmaFormat()) - p << stringifyEnum(*type.getTmaFormat()); - p << '>'; -} -LogicalResult verifyNonExecTma(EmitFn emitError, int bits) { - if (bits == 0) - return emitError() << "invalid copy_bits, got 0"; - return success(); -} -} // namespace - -#define NON_EXEC_TMA_WITH_KIND(TYPE, KIND) \ - Type TYPE::parse(AsmParser &p) { \ - SMLoc loc = p.getCurrentLocation(); \ - KIND kind; \ - Type t; \ - int bits; \ - cute::LayoutType gbasis; \ - std::optional format; \ - if (parseNonExecTma(p, &kind, t, bits, gbasis, format)) \ - return {}; \ - return getChecked([&] { return p.emitError(loc); }, p.getContext(), kind, \ - t, bits, gbasis, format); \ - } \ - void TYPE::print(AsmPrinter &p) const { \ - p << '<' << stringifyEnum(getKind()) << ", "; \ - printNonExecTma(p, *this); \ - } \ - LogicalResult TYPE::verify(EmitFn emitError, KIND, Type, int bits, \ - cute::LayoutType, std::optional) { \ - return verifyNonExecTma(emitError, bits); \ - } -#define NON_EXEC_TMA(TYPE) \ - Type TYPE::parse(AsmParser &p) { \ - SMLoc loc = p.getCurrentLocation(); \ - Type t; \ - int bits; \ - cute::LayoutType gbasis; \ - std::optional format; \ - if (parseNonExecTma(p, nullptr, t, bits, gbasis, format)) \ - return {}; \ - return getChecked([&] { return p.emitError(loc); }, p.getContext(), t, \ - bits, gbasis, format); \ - } \ - void TYPE::print(AsmPrinter &p) const { \ - p << '<'; \ - printNonExecTma(p, *this); \ - } \ - LogicalResult TYPE::verify(EmitFn emitError, Type, int bits, \ - cute::LayoutType, std::optional) { \ - return verifyNonExecTma(emitError, bits); \ - } -NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecTiledTmaLoadType, TiledTmaLoad) -NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecTiledTmaReduceType, ReductionKind) -NON_EXEC_TMA_WITH_KIND(CopyAtomNonExecIm2ColTmaLoadType, Im2ColTmaLoad) -NON_EXEC_TMA_WITH_KIND(CopyAtomNonExec2DGather4TmaLoadType, - GatherScatterTmaLoad) -NON_EXEC_TMA(CopyAtomNonExecTiledTmaStoreType) -NON_EXEC_TMA(CopyAtomNonExecIm2ColTmaStoreType) -NON_EXEC_TMA(CopyAtomNonExec2DScatter4TmaStoreType) -#undef NON_EXEC_TMA_WITH_KIND -#undef NON_EXEC_TMA - -//===----------------------------------------------------------------------===// -// MMA atoms -//===----------------------------------------------------------------------===// - -Type UniversalFmaAtomType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - cute::ShapeAttr shape; - Type a, b, c; - if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || - p.parseLParen() || p.parseType(a) || p.parseComma() || p.parseType(b) || - p.parseRParen() || p.parseArrow() || p.parseType(c) || p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, - b, c); -} -void UniversalFmaAtomType::print(AsmPrinter &p) const { - p << '<'; - printShapeMNK(p, getShapeMnk()); - p << ", (" << getAType() << ", " << getBType() << ") -> " << getCType() - << " >"; -} -LogicalResult UniversalFmaAtomType::verify(EmitFn emitError, - cute::ShapeAttr shape, Type, Type, - Type) { - return verifyShapeMNK(emitError, shape); -} - -// ``. -Type MmaAtomSM80Type::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - cute::ShapeAttr shape; - Type a, b, c; - MMAIntOverflowAttr overflow; - BinaryOpAttr op; - if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || - elemTypes(p, a, b, c)) - return {}; - if (succeeded(p.parseOptionalComma()) && - (keyField(p, "int_overflow", overflow) || - (succeeded(p.parseOptionalComma()) && keyField(p, "binary_op", op)))) - return {}; - if (p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, - b, c, overflow, op); -} -void MmaAtomSM80Type::print(AsmPrinter &p) const { - p << '<'; - printShapeMNK(p, getShapeMnk()); - p << ", "; - printElemTypes(p, getAType(), getBType(), getCType()); - printAttrField(p, "int_overflow", getIntOverflow()); - printAttrField(p, "binary_op", getBinaryOp()); - p << " >"; -} -LogicalResult MmaAtomSM80Type::verify(EmitFn emitError, cute::ShapeAttr shape, - Type, Type, Type, MMAIntOverflowAttr, - BinaryOpAttr) { - return verifyShapeMNK(emitError, shape); -} - -// ``. -Type MmaAtomSM80SparseType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - cute::ShapeAttr shape; - Type a, b, c; - SparseMetadataFormat format; - MMAIntOverflowAttr overflow; - if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || - elemTypes(p, a, b, c) || nextKeyField(p, "md_format", format)) - return {}; - if (succeeded(p.parseOptionalComma()) && - keyField(p, "int_overflow", overflow)) - return {}; - if (p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, - b, c, format, overflow); -} -void MmaAtomSM80SparseType::print(AsmPrinter &p) const { - p << '<'; - printShapeMNK(p, getShapeMnk()); - p << ", "; - printElemTypes(p, getAType(), getBType(), getCType()); - p << ", md_format = " << stringifyEnum(getSparseMetadataFormat()); - printAttrField(p, "int_overflow", getIntOverflow()); - p << " >"; -} -LogicalResult MmaAtomSM80SparseType::verify(EmitFn emitError, - cute::ShapeAttr shape, Type, Type, - Type, SparseMetadataFormat, - MMAIntOverflowAttr) { - return verifyShapeMNK(emitError, shape); -} - -Type MmaAtomSM89Type::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - cute::ShapeAttr shape; - Type a, b, c; - if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || - elemTypes(p, a, b, c) || p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, a, - b, c); -} -void MmaAtomSM89Type::print(AsmPrinter &p) const { - p << '<'; - printShapeMNK(p, getShapeMnk()); - p << ", "; - printElemTypes(p, getAType(), getBType(), getCType()); - p << " >"; -} -LogicalResult MmaAtomSM89Type::verify(EmitFn emitError, cute::ShapeAttr shape, - Type, Type, Type) { - return verifyShapeMNK(emitError, shape); -} - -// ``. -Type MmaAtomSM90Type::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - cute::ShapeAttr shape; - cute::MajorMode aMajor, bMajor; - Type a, b, c; - MmaFragKind frag; - MMAIntOverflowAttr overflow; - UnitAttr aNeg, bNeg; - if (p.parseLess() || parseShapeMNK(p, shape) || p.parseComma() || - p.parseKeyword("ab_major") || p.parseEqual() || - pairOf(p, aMajor, bMajor) || p.parseComma() || elemTypes(p, a, b, c) || - p.parseComma() || parseFragKind(p, MmaFragKind::rmem, frag)) - return {}; - while (succeeded(p.parseOptionalComma())) { - SMLoc at = p.getCurrentLocation(); - if (succeeded(p.parseOptionalKeyword("a_neg"))) - aNeg = UnitAttr::get(p.getContext()); - else if (succeeded(p.parseOptionalKeyword("b_neg"))) - bNeg = UnitAttr::get(p.getContext()); - else if (succeeded(p.parseOptionalKeyword("int_overflow"))) { - if (p.parseEqual() || field(p, overflow)) - return {}; - } else { - p.emitError(at, "expected `int_overflow`, `a_neg`, or `b_neg`"); - return {}; - } - } - if (p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, - aMajor, bMajor, a, b, c, frag, overflow, aNeg, bNeg); -} -void MmaAtomSM90Type::print(AsmPrinter &p) const { - p << '<'; - printShapeMNK(p, getShapeMnk()); - p << ", ab_major = (" << stringifyEnum(getAMajor()) << ", " - << stringifyEnum(getBMajor()) << "), "; - printElemTypes(p, getAType(), getBType(), getCType()); - p << ", frag_kind = " << fragKindText(getAFragKind()); - printAttrField(p, "int_overflow", getIntOverflow()); - if (getANeg()) - p << ", a_neg"; - if (getBNeg()) - p << ", b_neg"; - p << '>'; -} -LogicalResult MmaAtomSM90Type::verify(EmitFn emitError, cute::ShapeAttr shape, - cute::MajorMode, cute::MajorMode, Type, - Type, Type, MmaFragKind, - MMAIntOverflowAttr, UnitAttr, UnitAttr) { - return verifyShapeMNK(emitError, shape); -} - -// ``. -Type MmaAtomSM120BlockScaledType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - cute::ShapeAttr shape; - int vec; - Type a, b, c, sf; - bool useTV; - if (p.parseLess() || parseShapeMNK(p, shape) || - nextKeyField(p, "vec_size", vec) || p.parseComma() || - elemTypes(p, a, b, c) || nextKeyField(p, "sf_type", sf) || - nextKeyField(p, "use_sf_layout_TV", useTV) || p.parseGreater()) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), shape, - vec, a, b, c, sf, useTV); -} -void MmaAtomSM120BlockScaledType::print(AsmPrinter &p) const { - p << '<'; - printShapeMNK(p, getShapeMnk()); - p << ", vec_size = " << getVecSize() << ", "; - printElemTypes(p, getAType(), getBType(), getCType()); - p << ", sf_type = " << getSfType() - << ", use_sf_layout_TV = " << (getUseSfLayout_TV() ? "true" : "false") - << '>'; -} -LogicalResult MmaAtomSM120BlockScaledType::verify(EmitFn emitError, - cute::ShapeAttr shape, int, - Type, Type, Type, Type, - bool) { - return verifyShapeMNK(emitError, shape); -} - -// UMMA atoms: one grammar, fields present by kind (see CuteNVGPUTypes.td). -namespace { -struct Umma { - bool sf, sp, bs, sm107; - cute::ShapeAttr shape; - int numCta = 1; - cute::MajorMode aMajor, bMajor; - Type a, b, c, sfType, eType; - SparseMetadataFormat metadata = SparseMetadataFormat::tid; - MmaFragKind frag = MmaFragKind::smem_desc; - int scaleOrVec = 0; - Arch arch = Arch::sm_100; - MmaCollectorOp aCollector = MmaCollectorOp::discard, - bCollector = MmaCollectorOp::discard; - MMAIntOverflowAttr overflow; -}; - -ParseResult parseUmma(AsmParser &p, Umma &u) { - if (p.parseLess() || parseShapeMNK(p, u.shape) || - nextKeyField(p, "num_cta", u.numCta) || p.parseComma() || - p.parseKeyword("ab_major") || p.parseEqual() || - pairOf(p, u.aMajor, u.bMajor) || p.parseComma() || - elemTypes(p, u.a, u.b, u.c)) - return failure(); - if (u.sf && nextKeyField(p, "sf_type", u.sfType)) - return failure(); - if (u.sp && !u.sf && nextKeyField(p, "e_type", u.eType)) - return failure(); - if (u.sp && nextKeyField(p, "sparse_metadata_format", u.metadata)) - return failure(); - if (p.parseComma() || parseFragKind(p, MmaFragKind::tmem, u.frag) || - nextKeyField(p, u.bs ? "vec_size" : "c_scale_exp", u.scaleOrVec)) - return failure(); - bool comma = succeeded(p.parseOptionalComma()); - if (comma && u.bs && !u.sm107 && - succeeded(p.parseOptionalKeyword("arch_promote"))) { - if (p.parseEqual() || field(p, u.arch)) - return failure(); - comma = succeeded(p.parseOptionalComma()); - } - if (u.sm107) { - if (!comma) - return p.parseComma(); - if (p.parseKeyword("ab_collector_op") || p.parseEqual() || - pairOf(p, u.aCollector, u.bCollector)) - return failure(); - comma = succeeded(p.parseOptionalComma()); - } - if (comma && keyField(p, "int_overflow", u.overflow)) - return failure(); - return p.parseGreater(); -} - -void printUmma(AsmPrinter &p, const Umma &u) { - p << '<'; - printShapeMNK(p, u.shape); - p << ", num_cta = " << u.numCta << ", ab_major = (" << stringifyEnum(u.aMajor) - << ", " << stringifyEnum(u.bMajor) << "), "; - printElemTypes(p, u.a, u.b, u.c); - if (u.sf) - p << ", sf_type = " << u.sfType; - if (u.sp && !u.sf) - p << ", e_type = " << u.eType; - if (u.sp) - p << ", sparse_metadata_format = " << stringifyEnum(u.metadata); - p << ", frag_kind = " << fragKindText(u.frag) - << (u.bs ? ", vec_size = " : ", c_scale_exp = ") << u.scaleOrVec; - if (u.bs && !u.sm107 && (u.arch != Arch::sm_100 || u.overflow)) - p << ", arch_promote = " << stringifyEnum(u.arch); - if (u.sm107) - p << ", ab_collector_op = (" << stringifyEnum(u.aCollector) << ", " - << stringifyEnum(u.bCollector) << ")"; - printAttrField(p, "int_overflow", u.overflow); - p << '>'; -} - -// `sourced` picks the atom's wording of the tensor-memory A rule. -LogicalResult verifyUmma(EmitFn emitError, cute::ShapeAttr shape, int numCta, - MmaFragKind frag, cute::MajorMode aMajor, - int cScaleExp = 0, bool sourced = false) { - if (failed(verifyShapeMNK(emitError, shape))) - return failure(); - if (numCta != 1 && numCta != 2) - return emitError() << "expects `num_cta` equals to 1 or 2, but got " - << numCta; - if (cScaleExp > 15) - return emitError() << "expects `c_scale_exp` in range [0, 15], but got " - << cScaleExp; - if (frag == MmaFragKind::tmem && aMajor != cute::MajorMode::k) - return emitError() - << (sourced - ? "a operand must be k-major when sourced from tmem" - : "a operand must be of major-mode k when coming from tmem"); - return success(); -} -} // namespace - -Type MmaAtomSM100UMMAType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Umma u{false, false, false, false}; - if (parseUmma(p, u)) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, - u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, - u.scaleOrVec, u.overflow); -} -void MmaAtomSM100UMMAType::print(AsmPrinter &p) const { - Umma u{false, false, false, false}; - u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), - u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), - u.overflow = getIntOverflow(); - printUmma(p, u); -} -LogicalResult MmaAtomSM100UMMAType::verify(EmitFn emitError, - cute::ShapeAttr shape, int numCta, - cute::MajorMode aMajor, - cute::MajorMode, Type, Type, Type, - MmaFragKind frag, int cScaleExp, - MMAIntOverflowAttr) { - return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); -} - -Type MmaAtomSM100UMMASparseType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Umma u{false, true, false, false}; - if (parseUmma(p, u)) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, - u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, - u.metadata, u.frag, u.scaleOrVec, u.overflow); -} -void MmaAtomSM100UMMASparseType::print(AsmPrinter &p) const { - Umma u{false, true, false, false}; - u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), - u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.eType = getEType(), u.metadata = getSparseMetadataFormat(), - u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), - u.overflow = getIntOverflow(); - printUmma(p, u); -} -LogicalResult MmaAtomSM100UMMASparseType::verify( - EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, - cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, - MmaFragKind frag, int cScaleExp, MMAIntOverflowAttr) { - return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); -} - -Type MmaAtomSM100UMMABlockScaledType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Umma u{true, false, true, false}; - if (parseUmma(p, u)) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, - u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, - u.frag, u.scaleOrVec, u.arch, u.overflow); -} -void MmaAtomSM100UMMABlockScaledType::print(AsmPrinter &p) const { - Umma u{true, false, true, false}; - u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), - u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.sfType = getSfType(), u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), - u.arch = getArchPromote(), u.overflow = getIntOverflow(); - printUmma(p, u); -} -LogicalResult MmaAtomSM100UMMABlockScaledType::verify( - EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, - cute::MajorMode, Type, Type, Type, Type, MmaFragKind frag, int, Arch, - MMAIntOverflowAttr) { - return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, - /*sourced=*/true); -} - -Type MmaAtomSM100UMMABlockScaledSparseType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Umma u{true, true, true, false}; - if (parseUmma(p, u)) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, - u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, - u.metadata, u.frag, u.scaleOrVec, u.arch, u.overflow); -} -void MmaAtomSM100UMMABlockScaledSparseType::print(AsmPrinter &p) const { - Umma u{true, true, true, false}; - u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), - u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), - u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), - u.arch = getArchPromote(), u.overflow = getIntOverflow(); - printUmma(p, u); -} -LogicalResult MmaAtomSM100UMMABlockScaledSparseType::verify( - EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, - cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, - MmaFragKind frag, int, Arch, MMAIntOverflowAttr) { - return verifyUmma(emitError, shape, numCta, frag, aMajor); -} - -Type MmaAtomSM107UMMAType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Umma u{false, false, false, true}; - if (parseUmma(p, u)) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, - u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.frag, - u.scaleOrVec, u.aCollector, u.bCollector, u.overflow); -} -void MmaAtomSM107UMMAType::print(AsmPrinter &p) const { - Umma u{false, false, false, true}; - u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), - u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), - u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), - u.overflow = getIntOverflow(); - printUmma(p, u); -} -LogicalResult MmaAtomSM107UMMAType::verify( - EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, - cute::MajorMode, Type, Type, Type, MmaFragKind frag, int cScaleExp, - MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { - return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); -} - -Type MmaAtomSM107UMMASparseType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Umma u{false, true, false, true}; - if (parseUmma(p, u)) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, - u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.eType, - u.metadata, u.frag, u.scaleOrVec, u.aCollector, - u.bCollector, u.overflow); -} -void MmaAtomSM107UMMASparseType::print(AsmPrinter &p) const { - Umma u{false, true, false, true}; - u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), - u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.eType = getEType(), u.metadata = getSparseMetadataFormat(), - u.frag = getAFragKind(), u.scaleOrVec = getCScaleExp(), - u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), - u.overflow = getIntOverflow(); - printUmma(p, u); -} -LogicalResult MmaAtomSM107UMMASparseType::verify( - EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, - cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, - MmaFragKind frag, int cScaleExp, MmaCollectorOp, MmaCollectorOp, - MMAIntOverflowAttr) { - return verifyUmma(emitError, shape, numCta, frag, aMajor, cScaleExp); -} - -Type MmaAtomSM107UMMABlockScaledType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Umma u{true, false, true, true}; - if (parseUmma(p, u)) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, - u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, - u.frag, u.scaleOrVec, u.aCollector, u.bCollector, - u.overflow); -} -void MmaAtomSM107UMMABlockScaledType::print(AsmPrinter &p) const { - Umma u{true, false, true, true}; - u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), - u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.sfType = getSfType(), u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), - u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), - u.overflow = getIntOverflow(); - printUmma(p, u); -} -LogicalResult MmaAtomSM107UMMABlockScaledType::verify( - EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, - cute::MajorMode, Type, Type, Type, Type, MmaFragKind frag, int, - MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { - return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, - /*sourced=*/true); -} - -Type MmaAtomSM107UMMABlockScaledSparseType::parse(AsmParser &p) { - SMLoc loc = p.getCurrentLocation(); - Umma u{true, true, true, true}; - if (parseUmma(p, u)) - return {}; - return getChecked([&] { return p.emitError(loc); }, p.getContext(), u.shape, - u.numCta, u.aMajor, u.bMajor, u.a, u.b, u.c, u.sfType, - u.metadata, u.frag, u.scaleOrVec, u.aCollector, - u.bCollector, u.overflow); -} -void MmaAtomSM107UMMABlockScaledSparseType::print(AsmPrinter &p) const { - Umma u{true, true, true, true}; - u.shape = getShapeMnk(), u.numCta = getNumCta(), u.aMajor = getAMajor(), - u.bMajor = getBMajor(), u.a = getAType(), u.b = getBType(), u.c = getCType(), - u.sfType = getSfType(), u.metadata = getSparseMetadataFormat(), - u.frag = getAFragKind(), u.scaleOrVec = getVecSize(), - u.aCollector = getACollectorOp(), u.bCollector = getBCollectorOp(), - u.overflow = getIntOverflow(); - printUmma(p, u); -} -LogicalResult MmaAtomSM107UMMABlockScaledSparseType::verify( - EmitFn emitError, cute::ShapeAttr shape, int numCta, cute::MajorMode aMajor, - cute::MajorMode, Type, Type, Type, Type, SparseMetadataFormat, - MmaFragKind frag, int, MmaCollectorOp, MmaCollectorOp, MMAIntOverflowAttr) { - return verifyUmma(emitError, shape, numCta, frag, aMajor, 0, - /*sourced=*/true); -} - -//===----------------------------------------------------------------------===// -// Dialect -//===----------------------------------------------------------------------===// - -// `!cute_nvgpu.iN[]` is the compiler's alias of `!cute.iN[]`. -Type CuteNVGPUDialect::parseType(DialectAsmParser &parser) const { - SMLoc loc = parser.getCurrentLocation(); - StringRef mnemonic; - Type type; - OptionalParseResult result = generatedTypeParser(parser, &mnemonic, type); - if (result.has_value()) - return succeeded(*result) ? type : Type(); - StringRef width = mnemonic; - if (width.consume_front("i") && !width.empty() && - llvm::all_of(width, llvm::isDigit)) { - std::string spec = ("!cute." + mnemonic).str(); - int64_t divisor; - if (succeeded(parser.parseOptionalLess())) { - if (parser.parseKeyword("divby") || parser.parseInteger(divisor) || - parser.parseGreater()) - return {}; - spec += ""; - } - return mlir::parseType(spec, getContext()); - } - parser.emitError(loc) << "unknown type `" << mnemonic << "` in dialect `" - << getNamespace() << "`"; - return {}; -} - -void CuteNVGPUDialect::printType(Type type, DialectAsmPrinter &printer) const { - if (succeeded(generatedTypePrinter(type, printer))) - return; - llvm_unreachable("unexpected non-CuteNVGPU type"); -} - -void CuteNVGPUDialect::initialize() { - addAttributes< -#define GET_ATTRDEF_LIST -#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.cpp.inc" - >(); - addTypes< -#define GET_TYPEDEF_LIST -#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.cpp.inc" - >(); - addOperations< -#define GET_OP_LIST -#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.cpp.inc" - >(); -} diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h deleted file mode 100644 index 19d8cb14..00000000 --- a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h +++ /dev/null @@ -1,38 +0,0 @@ -//===- CuteNVGPUDialect.h - CuTe NVIDIA GPU dialect -------------*- C++ -*-===// - -#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H -#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H - -#include "cute_ir/Dialect/Cute/IR/CuteDialect.h" - -#include "mlir/Bytecode/BytecodeOpInterface.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/DialectImplementation.h" -#include "mlir/IR/OpImplementation.h" -#include "mlir/Interfaces/InferTypeOpInterface.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" - -#include - -#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h.inc" - -#include "mlir/dialects/cute_nvgpu/CuteNVGPUEnums.h.inc" - -namespace mlir::cutlass_compiler::cute_nvgpu { -/// Parses `true` or `false`. -FailureOr parseBool(AsmParser &parser); -/// The TMA data format a non-executable TMA atom takes for an element type -/// when none is given, if there is one. -std::optional defaultTmaDataFormat(Type elementType); -} // namespace mlir::cutlass_compiler::cute_nvgpu - -#define GET_ATTRDEF_CLASSES -#include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.h.inc" - -#define GET_TYPEDEF_CLASSES -#include "mlir/dialects/cute_nvgpu/CuteNVGPUTypes.h.inc" - -#define GET_OP_CLASSES -#include "mlir/dialects/cute_nvgpu/CuteNVGPUOps.h.inc" - -#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_H diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td b/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td deleted file mode 100644 index 5f6c6680..00000000 --- a/mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td +++ /dev/null @@ -1,36 +0,0 @@ -//===- CuteNVGPUDialect.td - CuTe NVIDIA GPU dialect -------*- tablegen -*-===// - -#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD -#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD - -include "mlir/IR/AttrTypeBase.td" -include "mlir/IR/DialectBase.td" -include "mlir/IR/OpBase.td" - -def CuteNVGPU_Dialect : Dialect { - let name = "cute_nvgpu"; - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let summary = "NVIDIA GPU operations and atom types used by CuTe-DSL"; - let description = [{ - This dialect mirrors the operation, type and attribute registry of - nvidia-cutlass-dsl 4.8.0, embedded in the proprietary CuTe-DSL compiler. - The type parser is hand-written only to accept `!cute_nvgpu.iN`, the - compiler's alias of `!cute.iN`. - }]; - let extraClassDeclaration = [{ - ::mlir::Type parseType(::mlir::DialectAsmParser &parser) const override; - void printType(::mlir::Type type, - ::mlir::DialectAsmPrinter &printer) const override; - }]; - let useDefaultAttributePrinterParser = 1; -} - -class CuteNVGPU_Op traits = []> - : Op; - -class CuteNVGPU_Type - : TypeDef { - let mnemonic = typeMnemonic; -} - -#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUDIALECT_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td b/mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td deleted file mode 100644 index a583a57e..00000000 --- a/mlir/dialects/cute_nvgpu/CuteNVGPUEnums.td +++ /dev/null @@ -1,683 +0,0 @@ -// The cute_nvgpu dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 -// enum bindings (`_cute_nvgpu_enum_gen.py`); do not edit. -// -// Case values and spellings are the compiler's. An enum the bindings build -// as a dialect attribute also has its EnumAttr, printed `#`. -// The cute enums the nvgpu bindings repeat are the cute dialect's; their -// nvgpu attributes are in CuteNVGPUAttrs.td, so that -gen-enum-decls -// over this file emits the nvgpu enums only. - -#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUENUMS_TD -#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUENUMS_TD - -include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" -include "mlir/IR/EnumAttr.td" - -def CuteNVGPU_BinaryOpEnum : I32EnumAttr<"BinaryOp", "Binary operation for single-bit MMA operations", [ - I32EnumAttrCase<"none", 0, "none">, - I32EnumAttrCase<"xor_popc", 1, "xor_popc">, - I32EnumAttrCase<"and_popc", 2, "and_popc"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldBulkCopyG2SEnum : I32EnumAttr<"AtomCopyFieldBulkCopyG2S", "Fields stored in Bulk Load Copy Atom type", [ - I32EnumAttrCase<"tma_bar", 1, "tma_bar">, - I32EnumAttrCase<"mcast_mask", 2, "mcast_mask">, - I32EnumAttrCase<"cache_policy", 7, "cache_policy"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldBulkCopyS2GEnum : I32EnumAttr<"AtomCopyFieldBulkCopyS2G", "Fields stored in Bulk Store Copy Atom type", [ - I32EnumAttrCase<"byte_mask", 4, "byte_mask">, - I32EnumAttrCase<"cache_policy", 7, "cache_policy"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldBulkCopyS2SEnum : I32EnumAttr<"AtomCopyFieldBulkCopyS2S", "Fields stored in Bulk CTA to Cluster Copy Atom type", [ - I32EnumAttrCase<"tma_bar", 1, "tma_bar">, - I32EnumAttrCase<"cta_rank", 5, "cta_rank"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldDsmemStoreEnum : I32EnumAttr<"AtomCopyFieldDsmemStore", "Fields stored in CopyAtomDsmemStoreType", [ - I32EnumAttrCase<"mbar_ptr", 8, "mbar_ptr"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldLoadGlobalEnum : I32EnumAttr<"AtomCopyFieldLoadGlobal", "Fields stored in CopyAtomG2RType", [ - I32EnumAttrCase<"cache_policy", 7, "cache_policy"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldNonExec2DGather4TmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExec2DGather4TmaLoad", "Fields stored in CopyAtomNonExec2DGather4TmaLoadType", [ - I32EnumAttrCase<"tma_desc", 3, "tma_desc"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldNonExec2DScatter4TmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExec2DScatter4TmaStore", "Fields stored in CopyAtomNonExec2DScatter4TmaStoreType", [ - I32EnumAttrCase<"tma_desc", 3, "tma_desc"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExecIm2ColTmaLoad", "Fields stored in CopyAtomNonExecIm2ColTmaLoadType", [ - I32EnumAttrCase<"im2col_tma_desc", 6, "im2col_tma_desc"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExecIm2ColTmaStore", "Fields stored in CopyAtomNonExecIm2ColTmaStoreType", [ - I32EnumAttrCase<"im2col_tma_desc", 6, "im2col_tma_desc"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldNonExecTiledTmaLoadEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaLoad", "Fields stored in CopyAtomNonExecTiledTmaLoadType", [ - I32EnumAttrCase<"tma_desc", 3, "tma_desc"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldNonExecTiledTmaReduceEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaReduce", "Fields stored in CopyAtomNonExecTiledTmaReduceType", [ - I32EnumAttrCase<"tma_desc", 3, "tma_desc"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldNonExecTiledTmaStoreEnum : I32EnumAttr<"AtomCopyFieldNonExecTiledTmaStore", "Fields stored in CopyAtomNonExecTiledTmaStoreType", [ - I32EnumAttrCase<"tma_desc", 3, "tma_desc"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldStoreGlobalEnum : I32EnumAttr<"AtomCopyFieldStoreGlobal", "Fields stored in CopyAtomR2GType", [ - I32EnumAttrCase<"cache_policy", 7, "cache_policy"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldTmaLoadEnum : I32EnumAttr<"AtomCopyFieldTmaLoad", "Fields stored in Tma Load Copy Atom type", [ - I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, - I32EnumAttrCase<"tma_bar", 1, "tma_bar">, - I32EnumAttrCase<"mcast_mask", 2, "mcast_mask">, - I32EnumAttrCase<"cache_policy", 7, "cache_policy">, - I32EnumAttrCase<"g_stride", 9, "g_stride"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldTmaReduceEnum : I32EnumAttr<"AtomCopyFieldTmaReduce", "Fields stored in Tma Reduce Copy Atom type", [ - I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, - I32EnumAttrCase<"cache_policy", 7, "cache_policy">, - I32EnumAttrCase<"g_stride", 9, "g_stride"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomCopyFieldTmaStoreEnum : I32EnumAttr<"AtomCopyFieldTmaStore", "Fields stored in Tma Store Copy Atom type", [ - I32EnumAttrCase<"tma_descriptor_ptr", 0, "tma_descriptor_ptr">, - I32EnumAttrCase<"cache_policy", 7, "cache_policy">, - I32EnumAttrCase<"g_stride", 9, "g_stride"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomMmaFieldSM80SparseEnum : I32EnumAttr<"AtomMmaFieldSM80Sparse", "Fields stored in MMA Atom types for SM80", [ - I32EnumAttrCase<"e", 0, "e"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomMmaFieldSM90Enum : I32EnumAttr<"AtomMmaFieldSM90", "Fields stored in MMA Atom types for SM90", [ - I32EnumAttrCase<"accum_c", 0, "accum_c"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomMmaFieldSM100BlockScaledEnum : I32EnumAttr<"AtomMmaFieldSM100BlockScaled", "Fields stored in MMA Atom types for SM100 (block scaled)", [ - I32EnumAttrCase<"accum_c", 0, "accum_c">, - I32EnumAttrCase<"neg_a", 1, "neg_a">, - I32EnumAttrCase<"neg_b", 2, "neg_b">, - I32EnumAttrCase<"sf_a", 3, "sf_a">, - I32EnumAttrCase<"sf_b", 4, "sf_b"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomMmaFieldSM100BlockScaledSparseEnum : I32EnumAttr<"AtomMmaFieldSM100BlockScaledSparse", "Fields stored in MMA Atom types for SM100 (block scaled)", [ - I32EnumAttrCase<"accum_c", 0, "accum_c">, - I32EnumAttrCase<"neg_a", 1, "neg_a">, - I32EnumAttrCase<"neg_b", 2, "neg_b">, - I32EnumAttrCase<"sf_a", 3, "sf_a">, - I32EnumAttrCase<"sf_b", 4, "sf_b">, - I32EnumAttrCase<"e", 5, "e"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomMmaFieldSM100Enum : I32EnumAttr<"AtomMmaFieldSM100", "Fields stored in MMA Atom types for SM100", [ - I32EnumAttrCase<"accum_c", 0, "accum_c">, - I32EnumAttrCase<"neg_a", 1, "neg_a">, - I32EnumAttrCase<"neg_b", 2, "neg_b">, - I32EnumAttrCase<"disable_output_lane", 6, "disable_output_lane"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomMmaFieldSM100SparseEnum : I32EnumAttr<"AtomMmaFieldSM100Sparse", "Fields stored in sparse MMA Atom types for SM100", [ - I32EnumAttrCase<"accum_c", 0, "accum_c">, - I32EnumAttrCase<"neg_a", 1, "neg_a">, - I32EnumAttrCase<"neg_b", 2, "neg_b">, - I32EnumAttrCase<"e", 5, "e"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_AtomMmaFieldSM120BlockScaledEnum : I32EnumAttr<"AtomMmaFieldSM120BlockScaled", "Fields stored in MMA Atom types for SM120 (block scaled)", [ - I32EnumAttrCase<"sf_a", 0, "sf_a">, - I32EnumAttrCase<"sf_b", 1, "sf_b"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_CacheEvictionPriorityEnum : I32EnumAttr<"CacheEvictionPriority", "Cute Cache Eviction Priority kind", [ - I32EnumAttrCase<"EVICT_NORMAL", 0, "evict_normal">, - I32EnumAttrCase<"EVICT_FIRST", 1, "evict_first">, - I32EnumAttrCase<"EVICT_LAST", 2, "evict_last">, - I32EnumAttrCase<"EVICT_UNCHANGED", 3, "evict_unchanged">, - I32EnumAttrCase<"NO_ALLOCATE", 4, "no_allocate"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_CopyS2TBroadcastEnum : I32EnumAttr<"CopyS2TBroadcast", "Broadcast modes for the different utccp instructions", [ - I32EnumAttrCase<"none", 0, "none">, - I32EnumAttrCase<"lw_0213", 1, "lw_0213">, - I32EnumAttrCase<"lw_0123", 2, "lw_0123">, - I32EnumAttrCase<"x4", 3, "x4"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_GatherScatterTmaLoadEnum : I32EnumAttr<"GatherScatterTmaLoad", "The various kinds of TMA loads in gather/scatter mode", [ - I32EnumAttrCase<"sm_100", 0, "sm_100">, - I32EnumAttrCase<"sm_100_multicast", 1, "sm_100_multicast">, - I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, - I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_ArchEnum : I32EnumAttr<"Arch", "NVGpu arch number like sm_103", [ - I32EnumAttrCase<"sm_52", 520, "sm_52">, - I32EnumAttrCase<"sm_70", 700, "sm_70">, - I32EnumAttrCase<"sm_75", 750, "sm_75">, - I32EnumAttrCase<"sm_80", 800, "sm_80">, - I32EnumAttrCase<"sm_86", 860, "sm_86">, - I32EnumAttrCase<"sm_90", 900, "sm_90">, - I32EnumAttrCase<"sm_100", 1000, "sm_100">, - I32EnumAttrCase<"sm_103", 1030, "sm_103">, - I32EnumAttrCase<"sm_107", 1070, "sm_107"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_Im2ColTmaLoadEnum : I32EnumAttr<"Im2ColTmaLoad", "The various kinds of TMA loads in im2col mode", [ - I32EnumAttrCase<"sm_90", 0, "sm_90">, - I32EnumAttrCase<"sm_90_multicast", 1, "sm_90_multicast">, - I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, - I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_L2PrefetchSizeEnum : I32EnumAttr<"L2PrefetchSize", "CuTe L2 prefetch size options", [ - I32EnumAttrCase<"NONE", 0, "none">, - I32EnumAttrCase<"RESERVED", 1, "reserved">, - I32EnumAttrCase<"SIZE_64B", 2, "size_64b">, - I32EnumAttrCase<"SIZE_128B", 3, "size_128b">, - I32EnumAttrCase<"SIZE_256B", 4, "size_256b"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_LdReduceAccPrecisionKindEnum : I32EnumAttr<"LdReduceAccPrecisionKind", "multimem ld_reduce accumulation precision kind", [ - I32EnumAttrCase<"F16", 0, "F16">, - I32EnumAttrCase<"F32", 1, "F32"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_LdsmSzPatternEnum : I32EnumAttr<"LdsmSzPattern", "LDSM's sz pattern, describing the bit size", [ - I32EnumAttrCase<"u16", 0, "u16">, - I32EnumAttrCase<"u4to8", 1, "u4to8">, - I32EnumAttrCase<"s4to8", 2, "s4to8">, - I32EnumAttrCase<"u2to4", 3, "u2to4">, - I32EnumAttrCase<"s2to4", 4, "s2to4">, - I32EnumAttrCase<"u4x16p64to8", 5, "u4x16p64to8">, - I32EnumAttrCase<"u6x16p32to8", 6, "u6x16p32to8">, - I32EnumAttrCase<"u8", 7, "u8"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_LoadCacheModeEnum : I32EnumAttr<"LoadCacheMode", "Cache modes for the load instructions", [ - I32EnumAttrCase<"always", 0, "always">, - I32EnumAttrCase<"global", 1, "global">, - I32EnumAttrCase<"streaming", 2, "streaming">, - I32EnumAttrCase<"last_use", 3, "last_use">, - I32EnumAttrCase<"none", 4, "none"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_MMAIntOverflowEnum : I32EnumAttr<"MMAIntOverflow", "MMA overflow options", [ - I32EnumAttrCase<"satfinite", 1, "satfinite">, - I32EnumAttrCase<"wrapped", 0, "wrapped"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_MemOrderKindEnum : I32EnumAttr<"MemOrderKind", "Memory Ordering kind", [ - I32EnumAttrCase<"WEAK", 0, "weak">, - I32EnumAttrCase<"RELAXED", 1, "relaxed">, - I32EnumAttrCase<"ACQUIRE", 2, "acquire">, - I32EnumAttrCase<"RELEASE", 3, "release">, - I32EnumAttrCase<"ACQ_REL", 4, "acq_rel">, - I32EnumAttrCase<"SC", 5, "sc">, - I32EnumAttrCase<"MMIO", 6, "mmio">, - I32EnumAttrCase<"CONSTANT", 7, "constant">, - I32EnumAttrCase<"VOLATILE", 8, "volatile"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_MemScopeKindEnum : I32EnumAttr<"MemScopeKind", "Cute Memory Scope kind", [ - I32EnumAttrCase<"CTA", 0, "cta">, - I32EnumAttrCase<"CLUSTER", 1, "cluster">, - I32EnumAttrCase<"GPU", 2, "gpu">, - I32EnumAttrCase<"SYS", 3, "sys"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_MmaCollectorOpEnum : I32EnumAttr<"MmaCollectorOp", "Enums for the mma collector op", [ - I32EnumAttrCase<"discard", 0, "discard">, - I32EnumAttrCase<"lastuse", 1, "lastuse">, - I32EnumAttrCase<"fill", 2, "fill">, - I32EnumAttrCase<"use", 3, "use"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_MmaFragKindEnum : I32EnumAttr<"MmaFragKind", "Enums for the mma frag type", [ - I32EnumAttrCase<"smem_desc", 0, "smem_desc">, - I32EnumAttrCase<"sparse_smem_desc", 1, "sparse_smem_desc">, - I32EnumAttrCase<"tmem", 2, "tmem">, - I32EnumAttrCase<"tmem_ws", 3, "tmem_ws">, - I32EnumAttrCase<"tmem_e", 4, "tmem_e">, - I32EnumAttrCase<"tmem_sf", 5, "tmem_sf">, - I32EnumAttrCase<"rmem", 6, "rmem"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_ReductionKindEnum : I32EnumAttr<"ReductionKind", "Op for the TMASTORE instruction", [ - I32EnumAttrCase<"ADD", 0, "ADD">, - I32EnumAttrCase<"MIN", 1, "MIN">, - I32EnumAttrCase<"MAX", 2, "MAX">, - I32EnumAttrCase<"INC", 3, "INC">, - I32EnumAttrCase<"DEC", 4, "DEC">, - I32EnumAttrCase<"AND", 5, "AND">, - I32EnumAttrCase<"OR", 6, "OR">, - I32EnumAttrCase<"XOR", 7, "XOR"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_SharedSpaceEnum : I32EnumAttr<"SharedSpace", "CuTe Shared Space kind", [ - I32EnumAttrCase<"CTA", 0, "cta">, - I32EnumAttrCase<"CLUSTER", 1, "cluster"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_SparseMetadataFormatEnum : I32EnumAttr<"SparseMetadataFormat", "Metadata format used in sparse MMA", [ - I32EnumAttrCase<"tid", 0, "tid"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_StoreCacheModeEnum : I32EnumAttr<"StoreCacheMode", "Cache modes for the store instructions", [ - I32EnumAttrCase<"write_back", 0, "write_back">, - I32EnumAttrCase<"global", 1, "global">, - I32EnumAttrCase<"streaming", 2, "streaming">, - I32EnumAttrCase<"write_through", 3, "write_through">, - I32EnumAttrCase<"none", 4, "none"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_TiledTmaLoadEnum : I32EnumAttr<"TiledTmaLoad", "The various kinds of TMA loads in tiled mode", [ - I32EnumAttrCase<"sm_90", 0, "sm_90">, - I32EnumAttrCase<"sm_90_multicast", 1, "sm_90_multicast">, - I32EnumAttrCase<"sm_100_2sm", 2, "sm_100_2sm">, - I32EnumAttrCase<"sm_100_2sm_multicast", 3, "sm_100_2sm_multicast"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_TmaDataFormatEnum : I32EnumAttr<"TmaDataFormat", "Bits 74-71 of the TMA descriptor", [ - I32EnumAttrCase<"U8", 0, "U8">, - I32EnumAttrCase<"U16", 1, "U16">, - I32EnumAttrCase<"U32", 2, "U32">, - I32EnumAttrCase<"S32", 3, "S32">, - I32EnumAttrCase<"U64", 4, "U64">, - I32EnumAttrCase<"S64", 5, "S64">, - I32EnumAttrCase<"F16_RN", 6, "F16_RN">, - I32EnumAttrCase<"F32_RN", 7, "F32_RN">, - I32EnumAttrCase<"F32_FTZ_RN", 8, "F32_FTZ_RN">, - I32EnumAttrCase<"F64_RN", 9, "F64_RN">, - I32EnumAttrCase<"BF16_RN", 10, "BF16_RN">, - I32EnumAttrCase<"U4", 11, "U4">, - I32EnumAttrCase<"U4_UNPACK_U8", 12, "U4_UNPACK_U8">, - I32EnumAttrCase<"U6_UNPACK_U8", 13, "U6_UNPACK_U8">, - I32EnumAttrCase<"TF32_RN", 14, "TF32_RN">, - I32EnumAttrCase<"TF32_FTZ_RN", 15, "TF32_FTZ_RN"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_TmaLoadModeEnum : I32EnumAttr<"TmaLoadMode", "Modes for the TMA load instruction", [ - I32EnumAttrCase<"tiled", 0, "tiled">, - I32EnumAttrCase<"im2col", 1, "im2col">, - I32EnumAttrCase<"w128", 2, "w128">, - I32EnumAttrCase<"w", 3, "w">, - I32EnumAttrCase<"gather4", 4, "gather4"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_TmaStoreModeEnum : I32EnumAttr<"TmaStoreMode", "Modes for the TMA store instruction", [ - I32EnumAttrCase<"tiled", 0, "tiled">, - I32EnumAttrCase<"im2col", 1, "im2col">, - I32EnumAttrCase<"scatter4", 2, "scatter4"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_TmemAllocModeEnum : I32EnumAttr<"TmemAllocMode", "Modes for the Tmem allocation", [ - I32EnumAttrCase<"Interleaved", 0, "Interleaved">, - I32EnumAttrCase<"NonInterleaved", 1, "NonInterleaved">, - I32EnumAttrCase<"Duplicated", 2, "Duplicated">, - I32EnumAttrCase<"SfDuplicated_4x1", 3, "SfDuplicated_4x1">, - I32EnumAttrCase<"SfDuplicated_2x2", 4, "SfDuplicated_2x2"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_TmemLoadRedOpEnum : I32EnumAttr<"TmemLoadRedOp", "Reduce operation for the Tmem load instruction", [ - I32EnumAttrCase<"max", 0, "max">, - I32EnumAttrCase<"maxabs", 1, "maxabs">, - I32EnumAttrCase<"min", 2, "min">, - I32EnumAttrCase<"minabs", 3, "minabs"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let genSpecializedAttr = 0; -} - -def CuteNVGPU_CopyS2TBroadcastAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldBulkCopyG2SAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldBulkCopyS2GAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldBulkCopyS2SAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldDsmemStoreAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldLoadGlobalAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldNonExec2DGather4TmaLoadAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldNonExec2DScatter4TmaStoreAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaLoadAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldNonExecIm2ColTmaStoreAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldNonExecTiledTmaLoadAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldNonExecTiledTmaReduceAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldNonExecTiledTmaStoreAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldStoreGlobalAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldTmaLoadAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldTmaReduceAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomCopyFieldTmaStoreAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomMmaFieldSM80SparseAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomMmaFieldSM90Attr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomMmaFieldSM100Attr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomMmaFieldSM100BlockScaledAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomMmaFieldSM100BlockScaledSparseAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomMmaFieldSM100SparseAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_AtomMmaFieldSM120BlockScaledAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_BinaryOpAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_GatherScatterTmaLoadAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_Im2ColTmaLoadAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_LdReduceAccPrecisionKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_LdsmSzPatternAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_LoadCacheModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_MMAIntOverflowAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_MmaCollectorOpAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_MmaFragKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_ReductionKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_TiledTmaLoadAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_TmaDataFormatAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_TmaLoadModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_TmaStoreModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_TmemAllocModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -def CuteNVGPU_TmemLoadRedOpAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::cute_nvgpu"; - let assemblyFormat = "`<` $value `>`"; -} - -#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUENUMS_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUOps.td b/mlir/dialects/cute_nvgpu/CuteNVGPUOps.td deleted file mode 100644 index f4c09d12..00000000 --- a/mlir/dialects/cute_nvgpu/CuteNVGPUOps.td +++ /dev/null @@ -1,2012 +0,0 @@ -// The cute_nvgpu dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. -// -// Sources: the operation bindings (`_cute_nvgpu_ops_gen.py`: names, classes, -// docstrings, operand/result/attribute structure, inference) and the live -// compiler (`_cutlass_ir` loaded from Python): each operand and result -// constraint carries the verifier's wording and accepts every type class the -// verifier let through when probed with one instance of every type; traits -// and interfaces are the registered operation's; memory effects come from -// loop-invariant code motion (hoisted: Pure), canonicalize (dead: no write) -// and CSE (merged: at most reads). Operations whose effects include writes -// or could not be told apart carry none (unknown effects). -// -// Operations keep the generic assembly form; the compiler's custom syntax is -// shown in each description. -// -// Constraints not recovered: -// cute_nvgpu.arch.mma.SM107.umma_block_scaled operand #3: no probe type accepted; any type allowed -// cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse operand #3: no probe type accepted; any type allowed -// cute_nvgpu.arch.mma.SM107.umma_sparse operand #3: no probe type accepted; any type allowed - -#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUOPS_TD -#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUOPS_TD - -include "mlir/dialects/cute_nvgpu/CuteNVGPUAttrs.td" -include "cute_ir/Dialect/Cute/IR/CuteAttrs.td" -include "mlir/dialects/cute_ir/CuteEnums.td" - -include "mlir/Interfaces/InferTypeOpInterface.td" -include "mlir/Interfaces/SideEffectInterfaces.td" - -// Operand and result constraints, in the compiler's words. -def CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor : Type($_self))">, "A memory reference type that represents a tensor view with an iterator and layout">; -def CuteNVGPU_AllowableDescriptorTypesForS2TSrcOperand : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescType>($_self))">, "allowable descriptor types for S2T src operand">; -def CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA : Type($_self))">, "allowable descriptor types for SM107 Blockscaled UMMA A operand">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; -def CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand : Type($_self))">, "allowable descriptor types for SM107 UMMA A operand">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; -def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA : Type, "allowable register types for SM120 block scaled MMA operations">; -def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA2 : Type($_self))">, "allowable register types for SM120 block scaled MMA operations">; -def CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3 : Type, "allowable register types for SM120 block scaled MMA operations">; -def CuteNVGPU_AtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::TiledMmaType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM120BlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100S2TCopyV2Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::UniversalFmaAtomType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM100UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMAType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM107UMMASparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM80SparseType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM89Type>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::MmaAtomSM90Type>($_self))">, "AtomTypeInterface instance">; -def CuteNVGPU_CopyAtomTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyG2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomBulkCopyS2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomDsmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomG2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomLdsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2GType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomR2SType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomS2RType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTAsyncCopyType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemLdReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTMultimemStType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomStsmType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM10xTmemLoadRedType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100TmemStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSIMTSyncCopyType>($_self))">, "CopyAtomTypeInterface instance">; -def CuteNVGPU_CuTeComposedLayoutType : Type($_self))">, "CuTe composed layout type">; -def CuteNVGPU_CuTeLayoutType : Type($_self))">, "CuTe Layout Type">; -def CuteNVGPU_CuTeMemrefToMemorySpacesSmem : Type($_self))">, "CuTe memref to memory spaces (smem)">; -def CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA : Type($_self))">, "CuTe pointer to allowable types for SM100 UMMA C/D operands in spaces (tmem)">; -def CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA2 : Type, "CuTe pointer to allowable types for SM100 UMMA E operand in spaces (tmem)">; // not recovered: no pool type passed -def CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA : Type, "CuTe pointer to allowable types for SM107 UMMA SF A/B operands in spaces (tmem)">; // not recovered: no pool type passed -def CuteNVGPU_CuTePointerToAllowableTypesForSM90MMA : Type($_self))">, "CuTe pointer to allowable types for SM90 MMA A/B operands in spaces (smem) with byte alignment 1">; -def CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte : Type($_self))">, "CuTe pointer to global memory with 64-byte alignment">; -def CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment : Type($_self))">, "CuTe pointer to global memory with byte alignment 1">; -def CuteNVGPU_CuTePointerToMemorySpacesRmem : Type($_self))">, "CuTe pointer to memory spaces (rmem)">; -def CuteNVGPU_CuTePointerToMemorySpacesSmem : Type($_self))">, "CuTe pointer to memory spaces (smem)">; -def CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe : Type($_self))">, "CuTe pointer to memory spaces (smem) or CuTe pointer to memory spaces (gmem)">; -def CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte : Type($_self))">, "CuTe pointer to memory spaces (smem) with byte alignment 1">; -def CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte2 : Type($_self))">, "CuTe pointer to memory spaces (smem) with byte alignment 128">; -def CuteNVGPU_CuTePointerToMemorySpacesTmem : Type($_self))">, "CuTe pointer to memory spaces (tmem)">; -def CuteNVGPU_CuTePointerToSharedMemoryWith8Byte : Type($_self))">, "CuTe pointer to shared memory with 8-byte alignment">; -def CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase : Type($_self))">, "CuTe pointer to shared memory with swizzle base atom compatible alignment with byte alignment 1">; -def CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2 : Type($_self))">, "CuTe pointer to shared memory with swizzle base atom compatible alignment">; -def CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory : Type($_self))">, "CuTe pointer to shared or distributed shared memory with 8-byte alignment">; -def CuteNVGPU_CuTePointerType : Type($_self))">, " CuTe pointer type ">; -def CuteNVGPU_CuteCoordTensorType : Type($_self))">, "Cute coord_tensor type">; -def CuteNVGPU_CuteCoordType : Type($_self))">, "Cute coord type">; -def CuteNVGPU_CuteShapeType : Type($_self))">, "Cute shape type">; -def CuteNVGPU_CuteSm107SmemDescType : Type($_self))">, "Cute sm107.smem_desc type">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescSM107Type::get($_builder.getContext())">; -def CuteNVGPU_CuteSmemDescType : Type($_self))">, "Cute smem_desc type">, BuildableType<"::mlir::cutlass_compiler::cute_nvgpu::SmemDescType::get($_builder.getContext())">; -def CuteNVGPU_CuteSmemDescViewType : Type($_self))">, "Cute smem_desc_view type">; -def CuteNVGPU_GeneralizedLayoutTypeOfCuTe : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::LayoutType>($_self))">, " Generalized layout type of CuTe ">; -def CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS : Type($_self))">, "IntTuple as integer or tuple of IntTuple(s)">; -def CuteNVGPU_Integer : Type($_self))">, "integer">; -def CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom : Type($_self))">, "SM100 (Blackwell) shared-to-tensor memory copy atom type via tcgen05.cp.">; -def CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomSM100CopyS2TType>($_self))">, "SM100 (Blackwell) shared-to-tensor memory copy atom type via tcgen05.cp. or Tiled copy atom type.">; -def CuteNVGPU_SM90PointerToTMATensorDescriptor : Type($_self))">, "SM90+ pointer to TMA tensor descriptor">; -def CuteNVGPU_SM90TMATensorMemoryAcceleratorLoadAtomType : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomTmaStoreType>($_self))">, "SM90+ TMA (Tensor Memory Accelerator) load atom type via cp.async.bulk. or SM90+ TMA im2col mode load atom type via cp.async.bulk.tensor. or SM90+ TMA (Tensor Memory Accelerator) store atom type via cp.async.bulk. or SM90+ TMA im2col mode store atom type via cp.async.bulk.tensor. or SM90+ TMA store with atomic reduction atom type via cp.reduce.async.bulk. or Type interface for non-executable TMA atom types">; -def CuteNVGPU_SignlessIntegerOrIndexOrCuTeLayoutType : Type($_self) || ::llvm::isa<::mlir::IntegerType>($_self) || ::llvm::isa<::mlir::IndexType>($_self))">, "signless integer or index or CuTe Layout Type">; -def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA : Type($_self))">, "The non-executable copy atom type for TMA loads in 2D gather4 mode">; -def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA2 : Type($_self))">, "The non-executable copy atom type for TMA stores in 2D scatter4 mode">; -def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA3 : Type($_self))">, "The non-executable copy atom type for TMA loads in im2col mode">; -def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA4 : Type($_self))">, "The non-executable copy atom type for TMA stores in im2col mode">; -def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA5 : Type($_self))">, "The non-executable copy atom type for TMA loads in tiled mode">; -def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA6 : Type($_self))">, "The non-executable copy atom type for TMA reduce in tiled mode">; -def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA7 : Type($_self))">, "The non-executable copy atom type for TMA stores in tiled mode">; -def CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA8 : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self))">, "The non-executable copy atom type for TMA loads in tiled mode or The non-executable copy atom type for TMA stores in tiled mode">; -def CuteNVGPU_TiledCopyAtomType : Type($_self))">, "Tiled copy atom type.">; -def CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaLoadType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaReduceType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::CopyAtomNonExecTiledTmaStoreType>($_self))">, "Type interface for non-executable TMA atom types">; -def CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf : Type($_self))">, "vector of 32-bit signless integer values of length 1/2/4/8/16/32/64/128">; -def CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf2 : Type($_self))">, "vector of 32-bit signless integer values of length 4/8">; -def CuteNVGPU_ViewTypeInterfaceInstance : Type($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute::MemRefType>($_self) || ::llvm::isa<::mlir::cutlass_compiler::cute_nvgpu::SmemDescViewType>($_self))">, "ViewTypeInterface instance">; - -// Attribute constraints the bindings name but do not describe. -def CuteNVGPU_ArchCopySM100CopyS2TBitAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{128, 256}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { - let storageType = "::mlir::IntegerAttr"; - let returnType = "::mlir::IntegerAttr"; - let convertFromStorage = "$_self"; -} -def CuteNVGPU_ArchCopySM100CopyS2TCtaAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { - let storageType = "::mlir::IntegerAttr"; - let returnType = "::mlir::IntegerAttr"; - let convertFromStorage = "$_self"; -} -def CuteNVGPU_ArchCopySM100CopyS2TDpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{4, 32, 64, 128}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { - let storageType = "::mlir::IntegerAttr"; - let returnType = "::mlir::IntegerAttr"; - let convertFromStorage = "$_self"; -} -def CuteNVGPU_ArchCopySM100TMEMNumBAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{32, 64, 128, 256}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { - let storageType = "::mlir::IntegerAttr"; - let returnType = "::mlir::IntegerAttr"; - let convertFromStorage = "$_self"; -} -def CuteNVGPU_ArchCopySM100TMEMNumDpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{16, 32}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { - let storageType = "::mlir::IntegerAttr"; - let returnType = "::mlir::IntegerAttr"; - let convertFromStorage = "$_self"; -} -def CuteNVGPU_ArchCopySM100TMEMNumRepAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2, 4, 8, 16, 32, 64, 128}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { - let storageType = "::mlir::IntegerAttr"; - let returnType = "::mlir::IntegerAttr"; - let convertFromStorage = "$_self"; -} -def CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef{1, 2}, ::llvm::cast<::mlir::IntegerAttr>($_self).getInt())">, "arbitrary integer attribute"> { - let storageType = "::mlir::IntegerAttr"; - let returnType = "::mlir::IntegerAttr"; - let convertFromStorage = "$_self"; -} -def CuteNVGPU_SM120BlockScaledABTypeAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E5M2Type>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of allowable types for SM120 block scaled MMA A/B operands"> { - let storageType = "::mlir::TypeAttr"; - let returnType = "::mlir::TypeAttr"; - let convertFromStorage = "$_self"; -} -def CuteNVGPU_SM120BlockScaledSFTypeAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E8M0FNUType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of allowable types for SM120 block scaled MMA A/B scale factors"> { - let storageType = "::mlir::TypeAttr"; - let returnType = "::mlir::TypeAttr"; - let convertFromStorage = "$_self"; -} -def CuteNVGPU_TmaLoadOrStoreModeAttr : Attr($_self)">, "TMA load or store mode attribute"> { - let storageType = "::mlir::Attribute"; - let returnType = "::mlir::Attribute"; - let convertFromStorage = "$_self"; -} - -// Operations whose result types are inferred in C++. Verification compares -// them with the inferred ones where the rule is recovered for the operands; -// elsewhere the types written stand, as the compiler accepted them. -class CuteNVGPU_InferredOp traits = []> - : CuteNVGPU_Op]> { - let extraClassDefinition = [{ - ::llvm::LogicalResult $cppClass::refineReturnTypes( - ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, - ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, - ::mlir::PropertyRef properties, ::mlir::RegionRange regions, - ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { - ::llvm::SmallVector<::mlir::Type, 4> inferred; - if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, - attributes, properties, regions, - inferred))) - return ::mlir::success(); - if (!isCompatibleReturnTypes(inferred, returnTypes)) - return ::mlir::emitOptionalError( - location, "'", getOperationName(), "' op inferred type(s) ", - inferred, " are incompatible with return type(s) of operation ", - returnTypes); - return ::mlir::success(); - } - }]; -} - -def CuteNVGPU_ArchAllocRmemOp : CuteNVGPU_Op<"arch.alloc_rmem", [MemoryEffects<[MemAlloc]>]> { - let description = [{ - This operation allocate a per-thread private buffer on the stack, with its size - determined by either the provided element count or the static layout. - - Examples: - - ```mlir - %c32_i32 = arith.constant 32 : i32 - %0 = cute_nvgpu.arch.alloc_rmem(%c32_i32) : i32 -> !cute.ptr - - // %static_layout : !cute.layout<"(8,4):(1,8)"> - %1 = cute_nvgpu.arch.alloc_rmem(%static_layout) : !cute.layout<"(8,4):(1,8)"> -> !cute.ptr - - // %dynamic_layout : !cute.layout<"(?,4):(1,?)"> - %2 = cute_nvgpu.arch.alloc_rmem(%dynamic_layout) : !cute.layout<"(?,4):(1,?)"> -> !cute.ptr - ``` - }]; - let arguments = (ins - CuteNVGPU_SignlessIntegerOrIndexOrCuTeLayoutType:$input - ); - let results = (outs - CuteNVGPU_CuTePointerToMemorySpacesRmem:$ptr - ); -} -def CuteNVGPU_ArchAllocSmemOp : CuteNVGPU_Op<"arch.alloc_smem", [MemoryEffects<[MemAlloc]>]> { - let description = [{ - This operation allocate static shared memory based on either a specified - element count or a static layout. - - Examples: - - ```mlir - // size provided by element count - %0 = cute_nvgpu.arch.alloc_smem(1024) : !cute.ptr> - - // size provided by static layout - %1 = cute_nvgpu.arch.alloc_smem("(8,4):(1,8)") : !cute.ptr> - ``` - }]; - let arguments = (ins - I32Attr:$input - ); - let results = (outs - CuteNVGPU_CuTePointerToMemorySpacesSmem:$ptr - ); -} -def CuteNVGPU_ArchCopySM90BulkCopyG2SOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_g2s", [AttrSizedOperandSegments]> { - let description = [{ - The `arch.copy.SM90.bulk_copy_g2s` operation performs a bulk load from global - to shared memory. - - The `gmem_data_addr` operand is a 16-byte aligned pointer to the source data - in global memory. - - The `dsmem_data_addr` operand is a 16-byte aligned pointer to the destination data - in shared cluster memory. - - The `dsmem_bar_addr` operand is a 8-byte aligned pointer to the barrier in shared - memory. - - The `size` operand specifies the bytes copied by BLKCP, aligned to 16 bytes. - - The `multicast_mask` operand is used when multicasting. - - The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. - - Examples: - ```mlir - %gmem = ... : !cute.ptr> - %smem = ... : !cute.ptr> - %bar = ... : !cute.ptr> - - // Without multicast mask - %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_g2s( - %gmem : !cute.ptr>, - %smem : !cute.ptr>, - %bar : !cute.ptr> - ) size = 128 - - // With multicast mask - %1 = cute_nvgpu.arch.copy.SM90.bulk_copy_g2s( - %gmem : !cute.ptr>, - %smem : !cute.ptr>, - %bar : !cute.ptr> - ) size = 128 mask = %mask - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment:$gmem_data_addr, - CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, - CuteNVGPU_CuTePointerToSharedMemoryWith8Byte:$dsmem_bar_addr, - Optional:$multicast_mask, - Optional:$cache_policy, - I32Attr:$size - ); - let results = (outs - ); -} -def CuteNVGPU_ArchCopySM90BulkCopyS2GOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_s2g", [AttrSizedOperandSegments]> { - let description = [{ - The `arch.copy.SM90.bulk_copy_s2g` operation performs a bulk load from shared - to global memory. - - The `smem_data_addr` operand is a 16-byte aligned pointer to the sourced data - in shared memory. - - The `gmem_data_addr` operand is a 16-byte aligned pointer to the estination data - in global memory. - - The `size` operand specifies the bytes copied by BLKCP, aligned to 16 bytes. - - The `byte_mask` operand is specifies whether the i-th byte of each 16-byte wide - chunk of source data is copied to the destination. - - The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. - - Examples: - ```mlir - %smem = ... : !cute.ptr> - %gmem = ... : !cute.ptr> - - // Without byte mask - %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2g( - %smem : !cute.ptr>, - %gmem : !cute.ptr>, - ) size = 128 - - // With byte mask - %1 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2g( - %smem : !cute.ptr>, - %gmem : !cute.ptr>, - ) size = 128 byte_mask = %mask - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$smem_data_addr, - CuteNVGPU_CuTePointerToGlobalMemoryWithByteAlignment:$gmem_data_addr, - Optional:$byte_mask, - Optional:$cache_policy, - I32Attr:$size - ); - let results = (outs - ); -} -def CuteNVGPU_ArchCopySM90BulkCopyS2SOp : CuteNVGPU_Op<"arch.copy.SM90.bulk_copy_s2s"> { - let description = [{ - The `arch.copy.SM90.bulk_copy_s2s` operation performs a bulk load from shared - cta to shared cluster. - - The `smem_data_addr` operand is a 16-byte aligned pointer to the source data - in shared memory. - - The `dsmem_data_addr` operand is a 16-byte aligned pointer to the destination data - in shared cluster memory. - - The `dsmem_bar_addr` operand is a 8-byte aligned pointer to the barrier in shared - memory. - - The `cta_rank` operand specifies the dst cta rank in cluster for BLKCP. - - The `size` attr specifies the bytes copied by BLKCP, aligned to 16 bytes. - - Examples: - ```mlir - %smem = ... : !cute.ptr> - %dsmem = ... : !cute.ptr> - %bar = ... : !cute.ptr> - %cta = ... : i32 - - %0 = cute_nvgpu.arch.copy.SM90.bulk_copy_s2s( - %smem : !cute.ptr>, - %dsmem : !cute.ptr>, - %bar : !cute.ptr>, - %cta : i32 - ) size = 128 - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$smem_data_addr, - CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, - CuteNVGPU_CuTePointerToSharedMemoryWith8Byte:$dsmem_bar_addr, - I32:$cta_rank, - I32Attr:$size - ); - let results = (outs - ); -} -def CuteNVGPU_ArchCopySM100CopyS2TOp : CuteNVGPU_Op<"arch.copy.SM100.copy_s2t"> { - let description = [{ - See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS - repository for more information. - }]; - let arguments = (ins - CuteNVGPU_AllowableDescriptorTypesForS2TSrcOperand:$src, - CuteNVGPU_CuTePointerToMemorySpacesTmem:$dst, - CuteNVGPU_ArchCopySM100CopyS2TDpAttr:$dp, - CuteNVGPU_ArchCopySM100CopyS2TBitAttr:$bits, - CuteNVGPU_ArchCopySM100CopyS2TCtaAttr:$cta, - CuteNVGPU_CopyS2TBroadcastAttr:$broadcast - ); - let results = (outs - ); -} -def CuteNVGPU_ArchCopySM100TmaLoadOp : CuteNVGPU_Op<"arch.copy.SM100.tma_load", [AttrSizedOperandSegments]> { - let description = [{ - The `arch.copy.SM100.tma_load` operation performs a tensor load from global - to shared memory. - - The attribute `mode` indicates which mode is selected. It can be: - - - tiled - - im2col for image-to-column loads - - w128 for activation-stationary convolution MMA mode (requires multicast) - - w for weight-stationary convolution MMA mode (requires multicast) - - gather4 for gather mode - - The attribute `num_cta` specifies the scope of the instruction. - - The `src_desc` operand is a 64-byte aligned pointer to the source tensor in - global memory. - - The `dsmem_data_addr` is a 128-byte aligned pointer to the destination data - address in shared memory. - - The `dsmem_bar_addr` is an 8-byte aligned pointer to the barrier in shared - or distributed shared memory. - - The `coord` operand holds the tensor coordinates (up to 5 dimensions). - - The `multicast_mask` operand is used when multicasting. - - The `offsets` operand holds offsets for im2col, w128, or w modes. - - The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$src_desc, - CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, - CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory:$dsmem_bar_addr, - Variadic:$coord, - Optional:$multicast_mask, - Variadic:$offsets, - Optional:$cache_policy, - CuteNVGPU_TmaLoadModeAttr:$mode, - CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr:$num_cta - ); - let results = (outs - ); -} -def CuteNVGPU_ArchCopySM100TmaReduceOp : CuteNVGPU_Op<"arch.copy.SM100.tma_reduce", [AttrSizedOperandSegments]> { - let description = [{ - The `arch.copy.SM100.tma_reduce` operation performs a tensor store from shared memory - to global with reduction. - - The attribute `mode` indicates which mode is selected. It can be: - - - tiled - - im2col for image-to-column loads - - The attribute `Kind` indicates which kind of reduce op is selected. It can be: - - - ADD - - MIN - - MAX - - INC - - DEC - - AND - - OR - - XOR - - The `dst_desc` is a 64-byte aligned pointer to the destination data - address in global memory. - - The `src_smem_data_addr` operand is a 128-byte aligned pointer to the source address in - shared memory. - - The `coord` operand holds the tensor coordinates (up to 5 dimensions). - - The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, - CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, - Variadic:$coord, - Optional:$cache_policy, - CuteNVGPU_TmaStoreModeAttr:$mode, - CuteNVGPU_ReductionKindAttr:$kind - ); - let results = (outs - ); -} -def CuteNVGPU_ArchCopySM100TmaStoreOp : CuteNVGPU_Op<"arch.copy.SM100.tma_store", [AttrSizedOperandSegments]> { - let description = [{ - The `arch.copy.SM100.tma_store` operation performs a tensor store from shared memory - to global. - - The attribute `mode` indicates which mode is selected. It can be: - - - tiled - - im2col for image-to-column loads - - scatter4 for scatter mode - - The `dst_desc` is a 64-byte aligned pointer to the destination data - address in global memory. - - The `src_smem_data_addr` operand is a 128-byte aligned pointer to the source address in - shared memory. - - The `coord` operand holds the tensor coordinates (up to 5 dimensions). - - The `cache_policy` operand is used to indicate the eviction priority of the L2 cache. - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, - CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, - Variadic:$coord, - Optional:$cache_policy, - CuteNVGPU_TmaStoreModeAttr:$mode - ); - let results = (outs - ); -} -def CuteNVGPU_ArchCopySM100TmemLoadOp : CuteNVGPU_Op<"arch.copy.SM100.tmem_load", [MemoryEffects<[MemRead]>]> { - let description = [{ - See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS - repository for more information. - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToMemorySpacesTmem:$src, - CuteNVGPU_ArchCopySM100TMEMNumDpAttr:$num_dp, - CuteNVGPU_ArchCopySM100TMEMNumBAttr:$num_b, - CuteNVGPU_ArchCopySM100TMEMNumRepAttr:$num_rep, - UnitAttr:$pack_16 - ); - let results = (outs - CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf:$dst - ); -} -def CuteNVGPU_ArchCopySM100TmemStoreOp : CuteNVGPU_Op<"arch.copy.SM100.tmem_store"> { - let description = [{ - See the file cutlass/include/cute/arch/copy_sm100.hpp in the CUTLASS - repository for more information. - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToMemorySpacesTmem:$dst, - CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf:$src, - CuteNVGPU_ArchCopySM100TMEMNumDpAttr:$num_dp, - CuteNVGPU_ArchCopySM100TMEMNumBAttr:$num_b, - CuteNVGPU_ArchCopySM100TMEMNumRepAttr:$num_rep, - UnitAttr:$expand_16 - ); - let results = (outs - ); -} -def CuteNVGPU_ArchCopySM107TmaLoadOverrideOp : CuteNVGPU_Op<"arch.copy.SM107.tma_load_override", [AttrSizedOperandSegments]> { - let description = [{ - The `arch.copy.SM107.tma_load_override` operation performs a TMA tensor load - from global to shared memory with runtime override of address, dimensions, - and strides. - - In addition to the normal TMA load operands, this op carries: - - `override_addr`: global pointer overriding the descriptor's base address - - `tensor_size`: per-mode sizes (encoded as size-1, i16) - - `lower_stride`: per-mode strides in 16B units (i32, modes 1..N-1) - - `upper_stride`: packed upper 4 bits of up to 4 strides (i16) - - Coordinates are passed as zeros (override form ignores them). - - Example: - - Suppose we want to load a 2D tile from a tensor with logical sizes - `{S0, S1}` and per-mode element strides `{1, ES1}` (mode 0 is contiguous, - so its stride is not passed). Let `EBITS` be the value type's bit-width - and `elemsPer16B = 128 / EBITS`. Mode 0's size is in `tensor_size[0]`, - mode 1's size is in `tensor_size[1]`, and mode 1's stride is split between - `lower_stride[0]` and the nibble at bits `[0:4]` of `upper_stride`. - - The encoding (matching `CopyAtomTmaLoadType::lowerOverrideToArch`) is: - - ```mlir - // ---- tensor_size: trunc(S_i - 1), one entry per mode ---- - %s0_m1 = arith.subi %S0_i32, %c1_i32 : i32 // S0 - 1 - %s1_m1 = arith.subi %S1_i32, %c1_i32 : i32 // S1 - 1 - %ts0 = arith.trunci %s0_m1 : i32 to i16 - %ts1 = arith.trunci %s1_m1 : i32 to i16 - - // ---- stride in 16B units: ES_i (element stride, i64) >> log2(elemsPer16B) - %sh = arith.constant log2(elemsPer16B) : i64 - %s1_16b = arith.shrui %ES1_i64, %sh : i64 // mode 1, stride in 16B - - // ---- lower_stride[idx]: low 32 bits of stride_in_16B (idx = mode - 1) ---- - %ls1 = arith.trunci %s1_16b : i64 to i32 // idx = 0 (mode 1) - - // ---- upper_stride: packed nibbles, OR-accumulated, truncated to i16 ---- - // nibble_i = (stride_in_16B[mode i] >> 32) & 0xF - // upper |= nibble_i << (4 * (i - 1)) - %c32 = arith.constant 32 : i64 - %c0xF = arith.constant 15 : i64 - %hi_1 = arith.shrui %s1_16b, %c32 : i64 // upper bits of mode 1 - %nib_1 = arith.andi %hi_1, %c0xF : i64 - %sh_1 = arith.constant 0 : i64 // 4 * (1 - 1) = 0 - %shf_1 = arith.shli %nib_1, %sh_1 : i64 - %acc = arith.ori %c0_i64, %shf_1 : i64 // accumulate over modes - %us = arith.trunci %acc : i64 to i16 - - // ---- the override TMA load itself ---- - cute_nvgpu.arch.copy.SM107.tma_load_override( - %tma_desc : !cute.ptr>, // src TMA descriptor - %dst_smem : !cute.ptr>, // dst data buffer - %dst_bar : !cute.ptr>, // mbarrier - %override_addr : !cute.ptr>, // runtime base addr override - [%c0_i32, %c0_i32] : i32, i32) // coords (ignored: zeros) - tensor_size = %ts0, %ts1 // (size - 1) per mode, i16 - lower_stride = %ls1 // strides for modes 1..N-1, in 16B units, low 32b (i32) - upper_stride = %us // packed upper 4 bits of up to 4 strides (i16) - cache_policy = %policy // L2 eviction policy (i64) - mode = num_cta = 1 - ``` - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$src_desc, - CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase:$dsmem_data_addr, - CuteNVGPU_CuTePointerToSharedOrDistributedSharedMemory:$dsmem_bar_addr, - CuteNVGPU_CuTePointerType:$override_addr, - Variadic:$tensor_size, - Variadic:$lower_stride, - Optional:$upper_stride, - Variadic:$coord, - Optional:$multicast_mask, - Optional:$cache_policy, - CuteNVGPU_TmaLoadModeAttr:$mode, - CuteNVGPU_ArchCopySM100TmaLoadOpNumCTAAttr:$num_cta - ); - let results = (outs - ); -} -def CuteNVGPU_ArchCopySM107TmaStoreOverrideOp : CuteNVGPU_Op<"arch.copy.SM107.tma_store_override", [AttrSizedOperandSegments]> { - let description = [{ - The `arch.copy.SM107.tma_store_override` operation performs a TMA tensor store - from shared memory to global with runtime override of address, dimensions, - and strides. - - In addition to the normal TMA store operands, this op carries: - - `override_addr`: global pointer overriding the descriptor's base address - - `tensor_size`: per-mode sizes (encoded as size-1, i16) - - `lower_stride`: per-mode strides in 16B units (i32, modes 1..N-1) - - `upper_stride`: packed upper 4 bits of up to 4 strides (i16) - - Example - The example of tma_load_override applies here except that tma_store_override - doesn't have the mbar argument. - - Coordinates are passed as zeros (override form ignores them). - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$dst_desc, - CuteNVGPU_CuTePointerToSharedMemoryWithSwizzleBase2:$src_smem_data_addr, - CuteNVGPU_CuTePointerType:$override_addr, - Variadic:$tensor_size, - Variadic:$lower_stride, - Optional:$upper_stride, - Variadic:$coord, - Optional:$cache_policy, - CuteNVGPU_TmaStoreModeAttr:$mode - ); - let results = (outs - ); -} -def CuteNVGPU_ArchGetDynamicSmemOp : CuteNVGPU_Op<"arch.get_dyn_smem", [MemoryEffects<[MemAlloc]>]> { - let description = [{ - This operation acquires the pointer to the dynamic shared memory, whose size - is determined at the kernel launch time. - }]; - let arguments = (ins - ); - let results = (outs - CuteNVGPU_CuTePointerToMemorySpacesSmem:$ptr - ); -} -def CuteNVGPU_ArchGetDynamicSmemSizeOp : CuteNVGPU_Op<"arch.get_dyn_smem_size", [Pure]> { - let description = [{ - This operation gets the size in bytes of the dynamic shared memory that was - specified at kernel launch time. This can be used for bounds checking during - shared memory allocation. - }]; - let arguments = (ins - ); - let results = (outs - I32:$size - ); -} -def CuteNVGPU_ArchMakeWarpUniformOp : CuteNVGPU_Op<"arch.make_warp_uniform", [Pure]> { - let description = [{ - This operation is to reach a similar goal as canonical_warp_idx_sync() in cutlass C++, which is - typically used to hint to ptxas to use uniform register to store current value so as to avoid - redundant R2URs in mainloop. It's actually a shortcut usage of `nvvm.shfl.sync idx 31, val, 0, -1`. - - Example: - - ```mlir - %thread_id = nvvm.read.ptx.sreg.tid.x : i32 - %warp_size_log2 = arith.constant 5 : i32 - %warp_id = arith.shrsi %thread_id, %warp_size_log2 : i32 - %warp_id_uniform = cute_nvgpu.arch.make_warp_uniform %warp_id : i32 - ``` - And this IR will be lowered to: - ```mlir - %thread_id = nvvm.read.ptx.sreg.tid.x : i32 - %warp_size_log2 = arith.constant 5 : i32 - %warp_id = arith.shrsi %thread_id, %warp_size_log2 : i32 - %warp_full_mask = arith.constant -1 : i32 - %c0_i32 = arith.constant 0 : i32 - %warp_id_uniform = nvvm.shfl.sync idx %c0_i32, %warp_id, %c0_i32, %warp_full_mask : i32 -> i32 - ``` - }]; - let arguments = (ins - I32:$val - ); - let results = (outs - I32:$res - ); -} -def CuteNVGPU_ArchMmaSM107UMMABlockScaledOp : CuteNVGPU_Op<"arch.mma.SM107.umma_block_scaled"> { - let description = [{ - The `cute_nvgpu.arch.mma.SM107.umma_block_scaled` operation performs a matrix - multiply-accumulate operation. This arch op contains scaling factor for A/B. - Matrix elements for operand A can reside in tensor memory or shared memory. - Operand B always reside in shared memory. Elements for the accumulator input - and output (CD) always reside in tensor memory. The `accum` operand controls - whether the matrix multiplication result is added to data in `C` (`accum` = `true`) - or overwrites the data in `C` (`accum` = `false`). The `sf_a/b` operands indicate - the scaling factor of A/B and they always reside in tensor memory. The `vec_size` - attribute indicates the scaling vector size. - See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS - repository for more information. - Examples: - ```mlir - cute_nvgpu.arch.mma.SM107.umma_block_scaled(%A, %B, %C, %sf_a, %sf_b, %idesc, %accum) - { num_cta = 1, vec_size = 32, a_type = f8E5M2, shape_k = 32 } - : (!cute_nvgpu.smem_desc, !cute_nvgpu.smem_desc, !cute.ptr, !cute.ptr, - !cute.ptr, i32, i1) - ``` - }]; - let arguments = (ins - CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA:$a, - CuteNVGPU_CuteSm107SmemDescType:$b, - CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, - CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA:$sf_a, - AnyType:$sf_b, - AnyType:$idesc, - AnyType:$accum, - APIntAttr:$num_cta, - I32Attr:$vec_size, - TypeAttr:$a_type, - TypeAttr:$b_type, - I32Attr:$shape_k, - CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, - CuteNVGPU_MmaCollectorOpAttr:$b_collector_op - ); - let results = (outs - ); -} -def CuteNVGPU_ArchMmaSM107UMMABlockScaledSparseOp : CuteNVGPU_Op<"arch.mma.SM107.umma_block_scaled_sparse"> { - let description = [{ - The `cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse` operation performs a matrix - multiply-accumulate operation. This arch op contains scaling factor for A/B and supports - structured sparsity. Matrix elements for operand A can reside in tensor memory or shared memory. - Operand B always reside in shared memory. Elements for the accumulator input - and output (CD) always reside in tensor memory. The `accum` operand controls - whether the matrix multiplication result is added to data in `C` (`accum` = `true`) - or overwrites the data in `C` (`accum` = `false`). The `sf_a/b` operands indicate - the scaling factor of A/B and they always reside in tensor memory. The `e_op` operand - provides the sparse metadata that describes the sparsity positional information of the matrix, - enabling efficient computation with structured sparse matrices. The `vec_size` - attribute indicates the scaling vector size. - See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS - repository for more information. - Examples: - ```mlir - cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse(%A, %B, %C, %sf_a, %sf_b, %E, %idesc, %accum) - { num_cta = 1, vec_size = 32, a_type = f8E5M2, shape_k = 32 } - : (!cute_nvgpu.sm107.smem_desc, !cute_nvgpu.sm107.smem_desc, !cute.ptr, !cute.ptr, - !cute.ptr, !cute.ptr>, i32, i1) - ``` - }]; - let arguments = (ins - CuteNVGPU_AllowableDescriptorTypesForSM107BlockscaledUMMAA:$a, - CuteNVGPU_CuteSm107SmemDescType:$b, - CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, - CuteNVGPU_CuTePointerToAllowableTypesForSM107UMMA:$sf_a, - AnyType:$sf_b, - AnyType:$e_op, - AnyType:$idesc, - AnyType:$accum, - APIntAttr:$num_cta, - I32Attr:$vec_size, - TypeAttr:$a_type, - TypeAttr:$b_type, - I32Attr:$shape_k, - CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, - CuteNVGPU_MmaCollectorOpAttr:$b_collector_op - ); - let results = (outs - ); -} -def CuteNVGPU_ArchMmaSM107UMMAOp : CuteNVGPU_Op<"arch.mma.SM107.umma"> { - let description = [{ - The `cute_nvgpu.arch.mma.SM107.umma` operation performs a matrix multiply- - accumulate operation. - Matrix elements for operand A can reside in either shared memory or tensor - memory (tmem). As such, the argument for operand A can be either a shared - memory descriptor or a pointer to tensor memory. - Elements for operand B always reside in shared memory. Elements for the - accumulator input and output (CD) always reside in tensor memory. The - `accum` operand controls whether the matrix multiplication result is added - to data in `C` (`accum` = `true`) or overwrites the data in `C` (`accum` = - `false`). Needs instruction desc according to nvvm backend and an additional - `a_type` for mmaKind inference. - See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS - repository for more information. - Examples: - ```mlir - cute_nvgpu.arch.mma.SM107.umma(%A, %B, %C, %idesc, %accum) - { num_cta = 1, ab_type = (f8E5M2, f8E5M2), shape_k = 64, ab_collector_op = (discard, discard) } - : (!cute_nvgpu.smem_desc, !cute_nvgpu.smem_desc, !cute.ptr, i32, i1) -> () - ``` - }]; - let arguments = (ins - CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand:$a, - CuteNVGPU_CuteSm107SmemDescType:$b, - CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, - I32:$idesc, - I1:$accum, - CuteNVGPU_VectorOf32BitSignlessIntegerValuesOf2:$disable_output_lane, - APIntAttr:$num_cta, - APIntAttr:$scale, - TypeAttr:$a_type, - TypeAttr:$b_type, - I32Attr:$shape_k, - CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, - CuteNVGPU_MmaCollectorOpAttr:$b_collector_op - ); - let results = (outs - ); -} -def CuteNVGPU_ArchMmaSM107UMMASparseOp : CuteNVGPU_Op<"arch.mma.SM107.umma_sparse"> { - let description = [{ - The `cute_nvgpu.arch.mma.SM107.umma_sparse` operation performs a matrix - multiply-accumulate operation where matrix A is structured sparse. - Matrix elements for operand A can reside in either shared memory or tensor - memory (tmem). As such, the argument for operand A can be either a shared - memory descriptor or a pointer to tensor memory. Operand E holds sparse - metadata that reside in tensor memory. - Elements for operand B always reside in shared memory. Elements for the - accumulator input and output (CD) always reside in tensor memory. The - `accum` operand controls whether the matrix multiplication result is added - to data in `C` (`accum` = `true`) or overwrites the data in `C` (`accum` = - `false`). Needs instruction desc according to nvvm backend and an additional - `a_type` for mmaKind inference. - See the file cutlass/include/cute/arch/mma_sm100_umma.hpp in the CUTLASS - repository for more information. - Examples: - ```mlir - cute_nvgpu.arch.mma.SM107.umma_sparse(%A, %B, %C, %E, %idesc, %accum) - {num_cta = 1 : i32, scale = 0 : i32, ab_type = (f16, f16), shape_k = 32 : i32, ab_collector_op = (discard, discard)} - : (!cute_nvgpu.sm107.smem_desc, !cute_nvgpu.sm107.smem_desc, !cute.ptr, - !cute.ptr, i32, i1) -> () - ``` - }]; - let arguments = (ins - CuteNVGPU_AllowableDescriptorTypesForSM107UMMAAOperand:$a, - CuteNVGPU_CuteSm107SmemDescType:$b, - CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA:$cd, - CuteNVGPU_CuTePointerToAllowableTypesForSM100UMMA2:$e, - AnyType:$idesc, - AnyType:$accum, - APIntAttr:$num_cta, - APIntAttr:$scale, - TypeAttr:$a_type, - TypeAttr:$b_type, - I32Attr:$shape_k, - CuteNVGPU_MmaCollectorOpAttr:$a_collector_op, - CuteNVGPU_MmaCollectorOpAttr:$b_collector_op - ); - let results = (outs - ); -} -def CuteNVGPU_ArchMmaSM120BlockScaledOp : CuteNVGPU_Op<"arch.mma.SM120.block_scaled", [AttrSizedOperandSegments]> { - let description = [{ - The `cute_nvgpu.arch.mma.SM120.block_scaled` operation performs a matrix - multiply-accumulate operation. This arch op contains scaling factor for A/B. - Matrix elements for A/B/C/D/SFA/SFB and accumulator are always in registers. - - C is always F32 under current SM120 arch, so there is no `c_type` here. - - See the file cutlass/include/cute/arch/mma_sm120.hpp in the CUTLASS repository - for more information. - - Examples: - ```mlir - cute_nvgpu.arch.mma.SM120.block_scaled - A[%a0, %a1, %a2, %a3] B[%b0, %b1] C[%c0, %c1, %c2, %c3] - sfA(%scaleAData) sfB(%scaleBData) - {shape_MNK=<"(16,8,32)">, vec_size = 32, a_type = f4E2M1FN, b_type = f4E2M1FN, sf_type = f8E8M0FNU} - : A[i32, i32, i32, i32] B[i32, i32] C[f32, f32, f32, f32] - sfA(i8) sfB(i8) -> (f32, f32, f32, f32) - ``` - }]; - let arguments = (ins - Variadic:$opA, - Variadic:$opB, - Variadic:$opC, - CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3:$sfA, - CuteNVGPU_AllowableRegisterTypesForSM120BlockScaledMMA3:$sfB, - Optional:$byte_id_a, - Optional:$byte_id_b, - Cute_ShapeAttr:$shape_MNK, - I32Attr:$vec_size, - DefaultValuedAttr:$thread_id_a, - DefaultValuedAttr:$thread_id_b, - CuteNVGPU_SM120BlockScaledABTypeAttr:$a_type, - CuteNVGPU_SM120BlockScaledABTypeAttr:$b_type, - CuteNVGPU_SM120BlockScaledSFTypeAttr:$sf_type - ); - let results = (outs - Variadic:$res - ); -} -def CuteNVGPU_ArchPrefetchTmaDescOp : CuteNVGPU_Op<"arch.prefetch_tma_desc"> { - let description = [{ - The `arch.prefetch_tma_desc` operation does prefetch TMA descriptor using generic addressing - (i.e. no specific state space: const or param). - - Example: - - ```mlir - cute_nvgpu.arch.prefetch_tma_desc(%tma_a) : (!cute.ptr>) -> () - ``` - }]; - let arguments = (ins - CuteNVGPU_SM90PointerToTMATensorDescriptor:$tma_descriptor_ptr - ); - let results = (outs - ); -} -def CuteNVGPU_ArchSM100AllocTmemOp : CuteNVGPU_Op<"arch.sm100.alloc_tmem"> { - let summary = "The `arch.sm100.alloc_tmem` operation performs dynamic allocation of TMEM."; - let description = [{ - It dynamically allocates storage on TMEM with requested `n_cols` and stores - the allocated TMEM pointer into the shared memory buffer at the shared memory - location specified by `dst_ptr`. - - This operation is blocking, meaning it will wait until enough memory is available. - The action writting pointer value into shared memory is not synchronized. It's - user's responsibility to synchronize to make sure it's visible to all threads in the CTA. - - The allocation may fail if one of following conditions is true: - 1. The requested allocation size is not a multiple of 32 columns or greater - than 512 columns or not a power of 2, or - 2. If an attempt is made to allocate after executing the - `cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit` operation. - - This operation must be issued by a single fully active warp of the CTA and - must not be issued by more than one warp simultaneously. For repeated - allocations, the same warp must be used to issue all allocations. All - `alloc_tmem` operations must be issued by the same warp. - - For the allocation of 2 CTAs, one warp from each of the peer CTAs must - collectively perform the allocation, and the issuing warp must make sure that - peer CTA is launched and is still active. The participating warps from both - CTAs should have the same logical warp ID. There is no explicit need for - synchronization between these 2 CTAs before executing this operation. - - The behavior of this operation is undefined if all the threads in the warp do - not use the same values of `n_cols`, or if any thread in the warp has exited. - - The meaning of each argument is as follows: - - `n_cols`: The number of columns to be allocated. The unit of allocation is - 32 columns and the number of columns must be a power of 2. It must be within - the range [32, 512]. The number of columns allocated should not increase - between any two allocations in the execution order within the CTA. Failing to - meet these conditions may lead to undefined behavior. - - `dst_ptr`: Pointer to shared memory where the result TMEM pointer will be - written. It should have a natural alignment of 32-bits. - - `is_two_cta`: Specifies the number of CTAs involved in the allocation. - - Examples: - - ```mlir - cute_nvgpu.arch.sm100.alloc_tmem(%n_cols, %res_shm_ptr) [cta_1] - : i32, !cute.ptr> - - cute_nvgpu.arch.sm100.alloc_tmem(%n_cols, %res_shm_ptr) [cta_2] - : i32, !cute.ptr> - ``` - }]; - let arguments = (ins - I32:$n_cols, - CuteNVGPU_CuTePointerToMemorySpacesSmem:$dst_ptr, - UnitAttr:$is_two_cta, - UnitAttr:$exclusive - ); - let results = (outs - ); -} -def CuteNVGPU_ArchSM100DeallocTmemOp : CuteNVGPU_Op<"arch.sm100.dealloc_tmem"> { - let description = [{ - The `arch.sm100.dealloc_tmem` operation deallocates the Tensor Memory - specified by the TMEM address `tmem_ptr` and slice count. The address - `tmem_ptr` must point to a previously allocated TMEM. If the - deallocation fails, a runtime error occurs. - - This operation should only be performed by a single fully active warp within - the CTA. It should not be carried out by multiple warps simultaneously. The - warp ID that does alloc and dealloc can be different, but all `dealloc_tmem` - operations must be issued by the same warp. - - For the 2 CTAs deallocation, one warp from each of the peer CTAs must - collectively perform the de-allocation, and the issuing warp must make sure - that peer CTA is launched and is still active. The warps from both CTAs should - have the same logical warp ID. Prior explicit synchronization between the two - CTAs is not required before executing this operation. - - The behavior of this operation is undefined if all the threads in the warp do - not use the same values of `n_cols`, or if any thread in the warp has exited. - - The meaning of each argument is as follows: - - `tmem_ptr`: The base address of the TMEM buffer being deallocated. - - `n_cols`: The number of columns to be deallocated. The unit of deallocation - is 32 columns and the number of columns must be a power of 2. It must be within - the range [32, 512]. Failing to meet these conditions may lead to undefined - behavior. - - `is_two_cta`: Specifies the number of CTAs involved in the deallocation. - - Examples: - - ```mlir - cute_nvgpu.arch.sm100.dealloc_tmem(%tmem_ptr, %n_cols) [cta_1] - : !cute.ptr>, i32 - - cute_nvgpu.arch.sm100.dealloc_tmem(%tmem_ptr, %n_cols) [cta_2] - : !cute.ptr>, i32 - ``` - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToMemorySpacesTmem:$tmem_ptr, - I32:$n_cols, - UnitAttr:$is_two_cta, - UnitAttr:$exclusive - ); - let results = (outs - ); -} -def CuteNVGPU_ArchSM100RelinquishTmemAllocPermitOp : CuteNVGPU_Op<"arch.sm100.relinquish_tmem_alloc_permit"> { - let description = [{ - This `arch.sm100.relinquish_tmem_alloc_permit` specifies that the CTA of the - executing thread is relinquishing the right to allocate Tensor Memory so that - other CTAs can allocate. - - Once this operation is executed, any subsequent attempts by the same CTA to - allocate TMEM are considered illegal. - - Example: - - ```mlir - cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit [cta_1] - ``` - }]; - let arguments = (ins - UnitAttr:$is_two_cta - ); - let results = (outs - ); -} -def CuteNVGPU_ArchSM100RetrieveTmemPtrOp : CuteNVGPU_Op<"arch.sm100.retrieve_tmem_ptr", [MemoryEffects<[MemRead]>]> { - let description = [{ - The `arch.sm100.retrieve_tmem_ptr` operation is a convenient shorthand for - loading the allocated TMEM pointer from shared memory and casting it into a - TMEM pointer. It includes a correctness check for the TMEM pointer and - returns a null pointer if the TMEM pointer is invalid. - - Example: - - ```mlir - %tmem_ptr = cute_nvgpu.arch.sm100.retrieve_tmem_ptr(%res_shm_ptr) - : !cute.ptr> -> !cute.ptr> - ``` - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToMemorySpacesSmem:$smem_ptr - ); - let results = (outs - CuteNVGPU_CuTePointerToMemorySpacesTmem:$tmem_ptr - ); -} -def CuteNVGPU_AtomCopyMakeExecTmaOp : CuteNVGPU_InferredOp<"atom.make_exec_tma", [Pure]> { - let arguments = (ins - CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$input, - UnitAttr:$override, - UnitAttr:$no_fully_oob_tile - ); - let results = (outs - CuteNVGPU_CopyAtomTypeInterfaceInstance:$result - ); -} -def CuteNVGPU_AtomCopyMakeNonExec2DGather4TmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_2d_gather4_tma_load", [Pure]> { - let summary = "This operation creates"; - let description = [{ - 1. a non-executable atom for TMA loads in tiled gather4 mode (.tile::gather4) - 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets - - The non-executable atom, which wraps a TMA descriptor and attaches type information to it for - safety, can then be used to create an executable one using - cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based - partitioning of source and destination tensors. - }]; - let arguments = (ins - CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, - CuteNVGPU_CuTeLayoutType:$gmem_coord_layout, - CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, - CuteNVGPU_CuTeLayoutType:$cta_v_map, - CuteNVGPU_GatherScatterTmaLoadAttr:$kind, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format - ); - let results = (outs - CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA:$non_exec_atom, - CuteNVGPU_CuteCoordTensorType:$tma_tensor - ); -} -def CuteNVGPU_AtomCopyMakeNonExec2DScatter4TmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_2d_scatter4_tma_store", [Pure]> { - let summary = "This operation creates"; - let description = [{ - 1. a non-executable atom for TMA stores in tiled scatter4 mode (.tile::scatter4) - 2. the TMA tensor mapping the logical coordinates of the scatter index layout to TMA coordinate offsets - - The non-executable atom, which wraps a TMA descriptor and attaches type information to it for - safety, can then be used to create an executable one using - cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based - partitioning of source and destination tensors. - }]; - let arguments = (ins - CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, - CuteNVGPU_CuTeLayoutType:$gmem_index_layout, - CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, - CuteNVGPU_CuTeLayoutType:$cta_v_map, - OptionalAttr:$tma_format - ); - let results = (outs - CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA2:$non_exec_atom, - CuteNVGPU_CuteCoordTensorType:$tma_tensor - ); -} -def CuteNVGPU_AtomCopyMakeNonExecIm2ColTmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_im2col_tma_load", [Pure]> { - let summary = "This operation creates"; - let description = [{ - 1. a non-executable atom for TMA loads in im2col mode - 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets - - The non-executable atom, which wraps a TMA descriptor and attaches type information to it for - safety, can then be used to create an executable one using - cute_nvgpu.atom.copy.make_exec_tma_load. It can also be used directly for TiledCopy-based - partitioning of source and destination tensors. - }]; - let arguments = (ins - CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, - CuteNVGPU_CuTeComposedLayoutType:$smem_layout, - CuteNVGPU_CuTeLayoutType:$cta_v_map, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, - CuteNVGPU_Im2ColTmaLoadAttr:$kind, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format - ); - let results = (outs - CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA3:$non_exec_atom, - CuteNVGPU_CuteCoordTensorType:$tma_tensor - ); -} -def CuteNVGPU_AtomCopyMakeNonExecIm2ColTmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_im2col_tma_store", [Pure]> { - let summary = "This operation creates"; - let description = [{ - 1. a non-executable atom for TMA stores in im2col mode - 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets - - The non-executable atom, which wraps a TMA descriptor and attaches type information to it for - safety, can then be used to create an executable one using - cute_nvgpu.atom.copy.make_exec_tma_store. It can also be used directly for TiledCopy-based - partitioning of source and destination tensors. - }]; - let arguments = (ins - CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, - CuteNVGPU_CuTeComposedLayoutType:$smem_layout, - CuteNVGPU_CuTeLayoutType:$cta_v_map, - OptionalAttr:$tma_format - ); - let results = (outs - CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA4:$non_exec_atom, - CuteNVGPU_CuteCoordTensorType:$tma_tensor - ); -} -def CuteNVGPU_AtomCopyMakeNonExecTiledTmaLoadOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_load", [Pure]> { - let summary = "This operation creates"; - let description = [{ - 1. a non-executable atom for TMA loads in tiled mode - 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets - - The non-executable atom, which wraps a TMA descriptor and attaches type information to it for - safety, can then be used to create an executable one using - cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based - partitioning of source and destination tensors. - }]; - let arguments = (ins - CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, - CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, - CuteNVGPU_CuTeLayoutType:$cta_v_map, - CuteNVGPU_TiledTmaLoadAttr:$kind, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format - ); - let results = (outs - CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA5:$non_exec_atom, - CuteNVGPU_CuteCoordTensorType:$tma_tensor - ); -} -def CuteNVGPU_AtomCopyMakeNonExecTiledTmaReduceOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_reduce", [Pure]> { - let summary = "This operation creates"; - let description = [{ - 1. a non-executable atom for TMA reduce in tiled mode - 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets - - The non-executable atom, which wraps a TMA descriptor and attaches type information to it for safety, can then be used to create an executable one using - cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based partitioning of source and destination tensors. - }]; - let arguments = (ins - CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, - CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, - CuteNVGPU_CuTeLayoutType:$cta_v_map, - CuteNVGPU_ReductionKindAttr:$kind, - OptionalAttr:$tma_format - ); - let results = (outs - CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA6:$non_exec_atom, - CuteNVGPU_CuteCoordTensorType:$tma_tensor - ); -} -def CuteNVGPU_AtomCopyMakeNonExecTiledTmaStoreOp : CuteNVGPU_InferredOp<"atom.make_non_exec_tiled_tma_store", [Pure]> { - let summary = "This operation creates"; - let description = [{ - 1. a non-executable atom for TMA store in tiled mode - 2. the TMA tensor mapping the logical coordinates of gmem_tensor to TMA coordinate offsets - - The non-executable atom, which wraps a TMA descriptor and attaches type information to it for - safety, can then be used to create an executable one using - cute_nvgpu.atom.copy.make_exec_tma. It can also be used directly for TiledCopy-based - partitioning of source and destination tensors. - }]; - let arguments = (ins - CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, - CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, - CuteNVGPU_CuTeLayoutType:$cta_v_map, - OptionalAttr:$tma_format - ); - let results = (outs - CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA7:$non_exec_atom, - CuteNVGPU_CuteCoordTensorType:$tma_tensor - ); -} -def CuteNVGPU_AtomCopyMakeS2TCopyOp : CuteNVGPU_InferredOp<"atom.make_s2t_copy", [Pure]> { - let description = [{ - This operation creates a new copy atom instance by tiling an utccp copy atom - instance with a tmem memref. - - Example: - ```mlir - %copy_s2t_atom = cute_nvgpu.atom.make_s2t_copy() : !cute_nvgpu.atom.s2t_copy - %tmem_memref = cute.memref.alloca() : !cute.memref, "(128,32):(65536,1)"> - %0 = cute_nvgpu.atom.make_s2t_copy(%copy_s2t_atom, %tmem_memref) - : (!cute_nvgpu.atom.s2t_copy, - !cute.memref, "(128,32):(65536,1)">) - -> !cute.tiled_copy, - layout_copy_tv = <"(1,(128,8)):(0,(1,128))">, - tiler_mn = <"[128:1;8:1]">> - ``` - }]; - let arguments = (ins - CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom:$copy_s2t_atom, - CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$tmem_memref - ); - let results = (outs - CuteNVGPU_TiledCopyAtomType:$result - ); -} -def CuteNVGPU_AtomCopyMakeTmemCopyOp : CuteNVGPU_InferredOp<"atom.make_tmem_copy", [Pure]> { - let description = [{ - This operation creates a new copy atom instance by tiling a load tmem copy atom - instance with a tmem memref. - - Example: - ```mlir - // %tmem_memref : !cute.memref, "(128,32):(65536,1)"> - %load_atom = cute.make_atom() : !cute_nvgpu.atom.tmem_load - %0 = cute_nvgpu.atom.make_tmem_copy(%load_atom, %tmem_memref) - : (!cute_nvgpu.atom.tmem_load, - !cute.memref, "(128,32):(65536,1)">) - -> !cute.tiled_copy, - layout_copy_tv = <"((32,4),(32,32)):((0,1),(128,4))">, - tiler_mn = <"[(4,32):(32,1);32:1]">> - ``` - }]; - let arguments = (ins - CuteNVGPU_CopyAtomTypeInterfaceInstance:$atom, - CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$tmem_memref - ); - let results = (outs - CuteNVGPU_TiledCopyAtomType:$result - ); -} -def CuteNVGPU_AtomGetCoordTensorOp : CuteNVGPU_InferredOp<"atom.get_coord_tensor", [SameVariadicOperandSize, Pure]> { - let description = [{ - This operation creates a get tma coord atom instance from copy atom and shapes. - The atom will be called before algo copy(tma copy) to provide tma coord for algo copy. - In Cute C++, copy_atom cache stride info during make_tiled_copy and the stride - info is associated with shapes. shapes in argument will make a congruent comparison - with cache first and then construct tma coord with stride. - - Example: - ```mlir - %5 = cute_nvgpu.atom.get_coord_tensor(%4, %5) - : (!cute_nvgpu.atom.tma_load>, - !cute.shape<128, 64>) - -> !cute.coord_tensor<"(0,0)", "(512,64,2):(1@1,1@0,64@0)"> - ``` - - ```mlir - %6 = cute_nvgpu.atom.get_coord_tensor(%4, %5, %2, %3) - : (!cute_nvgpu.atom.tma_load>, - !cute.shape<"(500, 128)">, - !cute.int_tuple<"(0, 0)">, - !cute.layout<"(128, 256): (1@0, 1@1) - -> !cute.coord_tensor<"(0,0)", "(500,128):(1@1,1@0) o (0,0) o (128,256):(1@0,1@1)"> - ``` - }]; - let arguments = (ins - CuteNVGPU_AtomTypeInterfaceInstance:$atom, - CuteNVGPU_CuteShapeType:$shape, - Optional:$offset_b, - Optional:$layout_b - ); - let results = (outs - CuteNVGPU_CuteCoordTensorType:$result - ); -} -def CuteNVGPU_AtomGetCopyS2TSmemDescViewOp : CuteNVGPU_InferredOp<"atom.get_copy_s2t_smem_desc_view", [Pure]> { - let description = [{ - This operation creates a SmemDescView instance from the given UtccpAtomType and - the SmemView. The given SmemView must match with the Utccp's TiledCopy src partitioned layout. - The result will be used as the src of algo.copy. - - Example: - ```mlir - %res_0 = cute_nvgpu.atom.get_copy_s2t_smem_desc_view(%atom_0, %src_0) - : (!cute_nvgpu.atom.s2t_copy, - !cute.memref, S<2,4,3>, "(((128,4),1),2,(4,2)):(((16,1),0),2048,(4,4096))">) - -> !cute_nvgpu.smem_desc_view - ``` - }]; - let arguments = (ins - CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2:$atom, - CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$view - ); - let results = (outs - CuteNVGPU_CuteSmemDescViewType:$result - ); -} -def CuteNVGPU_AtomGetCopyS2TSmemDescViewSM107Op : CuteNVGPU_InferredOp<"atom.sm107.get_copy_s2t_smem_desc_view", [Pure]> { - let description = [{ - This operation creates a SmemDescView instance from the given UtccpAtomType and - the SmemView. The given SmemView must match with the Utccp's TiledCopy src partitioned layout. - The result will be used as the src of algo.copy. - - Example: - ```mlir - %res_0 = cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view(%atom_0, %src_0) - : (!cute_nvgpu.atom.s2t_copy, - !cute.memref, S<2,4,3>, "(((128,4),1),2,(4,2)):(((16,1),0),2048,(4,4096))">) - -> !cute_nvgpu.smem_desc_view - ``` - }]; - let arguments = (ins - CuteNVGPU_SM100BlackwellSharedToTensorMemoryCopyAtom2:$atom, - CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$view - ); - let results = (outs - CuteNVGPU_CuteSmemDescViewType:$result - ); -} -def CuteNVGPU_AtomGetValueOp : CuteNVGPU_Op<"atom.get_value", [Pure]> { - let description = [{ - CuTe atoms represent both compile-time and runtime information for an - architecture-specific `copy` or `MMA` operation. For runtime information, - an atom can be thought of as similar to a C/C++ `struct` with predefined - member variables. Instead of defining operations to get and set the member - variables of each architecture atom, the CuTe dialect uses `get_value` and - `set_value` operations to uniformly handle retrieving and updating atom - member variables (hereafter referred to as "fields"). - The fields of each architecture-specific atom are identified by an enum - attribute. See the documentation for the operations that create an atom - or the attribute associated with the atom type for the expected enum values - and associated operand types. - The `field` attribute must correspond to the type of the `atom` operand. - For example, if the `atom` type is `!cute_nvgpu.sm100.mma`, the enum - attribute must be `#cute_nvgpu.atom_mma_field_sm100`. The verifier for this - operation will also compare the value `Type` to that expected by the - specific field. - - Example: - ```mlir - %5 = cute_nvgpu.atom.get_value(%4) : (!cute_nvgpu.sm100.mma<64x64x16, - num_cta = 1, - ab_major = (k,mn), - elem_type = (f16,f16,f16), - frag_kind = ss, - c_scale_exp = 0>) -> i1 - ``` - }]; - let arguments = (ins - CuteNVGPU_AtomTypeInterfaceInstance:$atom, - AnyAttr:$field - ); - let results = (outs - AnyType:$result - ); -} -def CuteNVGPU_AtomMakeTmaResidueTensorOp : CuteNVGPU_InferredOp<"atom.make_tma_residue_tensor", [Pure]> { - let description = [{ - Builds the per-TMA-mode "remaining elements" coord tensor that - `cute.copy` consumes on the override path. The op takes a non-executable - TMA atom (whose type carries `tma_gbasis`) and the corresponding gmem - tensor, and produces a coord tensor whose - - - iter is gmem tensor's shape combined and ordered according to tma gbasis. - - layout has the same shape as the tma tensor while the stride is negated. - - The residue tensor tracks the remaining shape to copy during TMA override, - e.g, slicing the residue tensor with a cta coordinate gets the remaining - shape for it given the negated stride. - - Example: - ```mlir - %residue = cute_nvgpu.atom.make_tma_residue_tensor(%non_exec_atom, %gmem) - : (!cute_nvgpu.atom.non_exec_tiled_tma_load>, - !cute.memref, "(?,?):(?,1)">) - -> !cute.coord_tensor<"(?,?)", "(?,?):(-1@1,-1@0)"> - ``` - }]; - let arguments = (ins - CuteNVGPU_TheNonExecutableCopyAtomTypeForTMA8:$tma_atom, - CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor - ); - let results = (outs - CuteNVGPU_CuteCoordTensorType:$residue - ); -} -def CuteNVGPU_AtomSetValueOp : CuteNVGPU_InferredOp<"atom.set_value", [Pure]> { - let description = [{ - CuTe atoms represent both compile-time and runtime information for an - architecture-specific `copy` or `MMA` operation. For runtime information, - an atom can be thought of as similar to a C/C++ `struct` with predefined - member variables. Instead of defining operations to get and set the member - variables of each architecture atom, the CuTe dialect uses `get_value` and - `set_value` operations to uniformly handle retrieving and updating atom - member variables (hereafter referred to as "fields"). - The fields of each architecture-specific atom are identified by an enum - attribute. See the documentation for the operations that create an atom - or the attribute associated with the atom type for the expected enum values - and associated operand types. - The `field` attribute must correspond to the type of the `atom` operand. - For example, if the `atom` type is `!cute_nvgpu.sm100.mma`, the enum - attribute must be `#cute_nvgpu.atom_mma_field_sm100`. The verifier for this - operation will also compare the value `Type` to that expected by the - specific field. - - Example: - ```mlir - %2 = cute_nvgpu.atom.set_value(%0, %no_accum) : - (!cute_nvgpu.sm100.mma<64x64x16, - num_cta = 1, - ab_major = (k,mn), - elem_type = (f16,f16,f16), - frag_kind = ss, - c_scale_exp = 0>, i1) - ``` - }]; - let arguments = (ins - CuteNVGPU_AtomTypeInterfaceInstance:$atom, - AnyType:$value, - AnyAttr:$field - ); - let results = (outs - CuteNVGPU_AtomTypeInterfaceInstance:$result - ); -} -def CuteNVGPU_AtomTmaPartitionOp : CuteNVGPU_InferredOp<"atom.tma_partition", [Pure]> { - let description = [{ - tma_partition is the new TMA routine for the dynamic and preferred cluster feature. - tma_partition op requires below arguments: - 1. tma_atom; - 2. cta_coord in the multicast dimension; - 3. cta_layout in the multicast dimension; - 4. smem_memref; - 5. variadic CuteViewTypeInterface tensors (one or more coord_tensor / memref), - covering the following cases: - 5.a: for regular TMA load/store, a single TMA tensor as input - 5.b: for tile::gather4 mode, a 2D TMA coord tensor plus the corresponding - gmem index tensor - and it produces: - 1. partitioned_smem_memref; - 2. one partitioned coord_tensor / memref per input, with the following cases: - 2.a: for regular TMA load/store, one partitioned TMA tensor - 2.b: for tile::gather4 mode, the partitioned TMA coord tensor plus the - partitioned gmem index tensor - - The return values are the src and dst arguments of algo.copy(atom, src, dst). - The second and the third input arguments can be runtime values, so the multicast - can be runtime determined. - - This operation is also compatible for the old static cluster TMA routines. - You only need to feed the cta_coord and cta_layout with static values, - then you can get compile-time multicast computation. - - The target_tensors operand is variadic and accepts any CuteViewTypeInterface - type (coord_tensor or memref). When a single tensor is provided, brackets - are omitted for backward compatibility. When multiple tensors are provided, - they are grouped in brackets. - - Example (single tensor): - ```mlir - %res_smem, %res_gmem = cute_nvgpu.atom.tma_partition( - %tma_atom, %cta_coord, %cta_layout, %stensor, %gtensor - ) : (!cute_nvgpu.atom.tma_load, - !cute.coord<"?">, - !cute.layout<"?:1">, - !cute.memref, S<3,4,3>, "(((128,16),1,4),8):(((64,1),0,16),8192)">, - !cute.coord_tensor<"(0,?,0)", "(((128,16),1,4),?,?,?):(((1@1,1@0),0,16@0),256@1,64@0,1@2)">) - ``` - - Example (multiple tensors): - ```mlir - %res_smem, %res_gmem0, %res_gmem1 = cute_nvgpu.atom.tma_partition( - %tma_atom, %cta_coord, %cta_layout, %stensor, [%gtensor0, %gtensor1] - ) : (!cute_nvgpu.atom.tma_load<...>, - !cute.coord<"?">, - !cute.layout<"?:1">, - !cute.memref, - [!cute.coord_tensor<...>, !cute.memref]) - -> (!cute.memref, - [!cute.coord_tensor<...>, !cute.memref]) - ``` - }]; - let arguments = (ins - CuteNVGPU_SM90TMATensorMemoryAcceleratorLoadAtomType:$tma_atom, - CuteNVGPU_CuteCoordType:$cta_coord, - CuteNVGPU_CuTeLayoutType:$cta_layout, - CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$smem_tensor, - Variadic:$target_tensors - ); - let results = (outs - CuteNVGPU_CuTeMemrefToMemorySpacesSmem:$res_smem_tensor, - Variadic:$res_target_tensors - ); -} -def CuteNVGPU_CastTMADescToIntegerOp : CuteNVGPU_Op<"cast_tma_desc_to_integer", [Pure]> { - let summary = "cast_tma_desc_to_integer cast a tma_desc into an integer."; - let description = [{ - Example: - ```mlir - %23 = cute_nvgpu.cast_tma_desc_to_integer(%11 : !cute.ptr>) -> i64 - ``` - }]; - let arguments = (ins - CuteNVGPU_SM90PointerToTMATensorDescriptor:$src - ); - let results = (outs - CuteNVGPU_Integer:$dst - ); -} -def CuteNVGPU_CopyTMADescriptorOp : CuteNVGPU_Op<"copy_tma_desc"> { - let description = [{ - This operation copies the TMA descriptor from a non-executable TMA atom to the pointer. - The pointer could be a smem or gmem pointer and should be aligned to at least the size of TMA descriptor. - }]; - let arguments = (ins - CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$atom, - CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe:$tma_desc_ptr - ); - let results = (outs - ); -} -def CuteNVGPU_GetGridConstantPointerOp : CuteNVGPU_InferredOp<"get_grid_constant_pointer", [Pure]> { - let description = [{ - 'cute_nvgpu.get_grid_constant_pointer' takes a grid constant value argument - from a kernel and returns a `cute.ptr` whose pointee type matches the - input. - - Before device ABI conversion, the input may be an intermediate result that - function finalization will promote to a grid-constant kernel argument. - Direct kernel arguments are verified immediately; intermediate provenance - is validated when the physical kernel signature is finalized. The result - type and alignment are fully inferred from the input type by - `InferTypeOpInterface`. - Example (TMA descriptor): - func.func @kernel( - %tma_desc_a : !cute_nvgpu.tma_descriptor_tiled {cute_nvgpu.grid_constant}) - attributes {cute.kernel} { - %tma_desc_a_ptr = cute_nvgpu.get_grid_constant_pointer (%tma_desc_a) : - !cute.ptr>; - ... - } - - Example (arbitrary value type): - func.func @kernel( - %params : !llvm.struct<(i64, i64)> {cute_nvgpu.grid_constant}) - attributes {cute.kernel} { - %params_ptr = cute_nvgpu.get_grid_constant_pointer (%params) : - !cute.ptr, generic>; - ... - } - }]; - let arguments = (ins - AnyType:$input - ); - let results = (outs - CuteNVGPU_CuTePointerType:$result - ); -} -def CuteNVGPU_GetTmaDescAddrOp : CuteNVGPU_Op<"get_tma_desc_addr", [Pure]> { - let description = [{ - The `cute_nvgpu.get_tma_desc_addr` operation accepts an input copy atom, - extracts the address within copy atom, then returns it as the address - of current copy atom's TMA desc in a single SSA value. The address within - input copy atom should have global/generic address space, and should be aligned to 64B. - Example: - - ```mlir - %1 = cute_nvgpu.get_tma_desc_addr(%cpy_atom : !cute_nvgpu.atom.tma_load) -> !cute.ptr> - ``` - }]; - let arguments = (ins - CuteNVGPU_CopyAtomTypeInterfaceInstance:$src - ); - let results = (outs - CuteNVGPU_CuTePointerToGlobalMemoryWith64Byte:$ptr - ); -} -def CuteNVGPU_MakeGMMASmemDescOp : CuteNVGPU_Op<"make_gmma_smem_desc", [Pure]> { - let description = [{ - The `cute_nvgpu.make_gmma_smem_desc` operation accepts an input cute.ptr alongside a cute.layout and a major attr, - then constructs the shared memory descriptor and returns it. The layout must be describing - the buffer pointed to by the input pointer and must carry valid swizzle information. - For GMMA, there are 4 supported swizzle variants: - - ``` - swizzle | cooresponding enum in gmma smem desc - S<0, 4, 3> | SWIZZLE_NONE - S<1, 4, 3> | SWIZZLE_32B - S<2, 4, 3> | SWIZZLE_64B - S<3, 4, 3> | SWIZZLE_128B - ``` - The cute.ptr must carry shared address space and must be aligned to 16B. - - Example: - - ```mlir - %1 = cute_nvgpu.make_gmma_smem_desc(%arg1 : !cute.ptr, S<3,4,3>>) - layout = <"((64,16),1):((64,1),0)">, major = #cute_nvgpu.major - -> !cute_nvgpu.smem_desc - ``` - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToAllowableTypesForSM90MMA:$src, - Cute_LayoutAttr:$layout, - CuteNVGPU_MajorModeAttr:$major - ); - let results = (outs - CuteNVGPU_CuteSmemDescType:$res - ); -} -def CuteNVGPU_MakeTmaDescIm2ColAtOp : CuteNVGPU_Op<"make_tma_desc_im2col_at"> { - let description = [{ - In-place variant of Cute_MakeTmaDescIm2ColOp. The descriptor is stored - into the address passed as the last operand. - - Operands (identical semantics to the im2col op) - - All operands of MakeTmaDescIm2ColOp (see above). - - Value tma_desc_addr – destination address for the descriptor. - - Optional attributes (identical semantics to the im2col op) - - num_multicast, tma_format - - Results: none – descriptor is written to the supplied address. - - Example: - ```mlir - cute_nvgpu.make_tma_desc_im2col_at( - %tensor, - %slayout, - %cta_v_tile, - %lower_corner_whd, - %upper_corner_whd, - %lower_padding_whd, - %upper_padding_whd, - %stride_whd, - %lower_srt, - %stride_srt, - %tma_desc_addr) - <{num_multicast = 1 tma_format = #cute_nvgpu.tma_data_format}> : ( - !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, - !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, - !cute.layout<"(128,32):(1@0,1@1)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(1,1,1)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(1,1,1)">, - !cute.ptr>) - ``` - }]; - let arguments = (ins - CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, - CuteNVGPU_CuTeComposedLayoutType:$smem_layout, - CuteNVGPU_CuTeLayoutType:$cta_value_tile, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, - CuteNVGPU_CuTePointerType:$tma_desc_addr, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format - ); - let results = (outs - ); -} -def CuteNVGPU_MakeTmaDescIm2ColOp : CuteNVGPU_Op<"make_tma_desc_im2col", [Pure]> { - let summary = "Create a TMA descriptor in im2col mode, suitable for convolutional forward passes."; - let description = [{ - Operands - - Value gmem_view, smem_layout, cta_value_tile – as in the tiled op. - - Value lower_corner_whd / upper_corner_whd – start & end (W,H,D) - positions of the - receptive-field window. - - Value lower_padding_whd / upper_padding_whd – padding applied - before/after each dim. - - Value stride_whd – traversal stride in WHD. - - Value lower_srt / stride_srt – split-K traversal - parameters (SRT). - - Optional attributes - - num_multicast : i32 - - tma_format : CuteTmaDataFormatAttr - - Result - - Cute pointer to the generated descriptor. - - Example: - ```mlir - %desc = cute_nvgpu.make_tma_desc_im2col( - %tensor, - %slayout, - %cta_v_tile, - %lower_corner_whd, - %upper_corner_whd, - %lower_padding_whd, - %upper_padding_whd, - %stride_whd, - %lower_srt, - %stride_srt) - <{num_multicast = 1 tma_format = #cute_nvgpu.tma_data_format}> : ( - !cute.memref, "((8,8,1,4),64,1):((64,512,4096,4096),1,0)">, - !cute.composed_layout<"S<2,4,3> o 0 o ((8,16),(32,1)):((32,256),(1,0))">, - !cute.layout<"(128,32):(1@0,1@1)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(1,1,1)">, - !cute.int_tuple<"(0,0,0)">, - !cute.int_tuple<"(1,1,1)">) -> !cute.ptr> - ``` - }]; - let arguments = (ins - CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, - CuteNVGPU_CuTeComposedLayoutType:$smem_layout, - CuteNVGPU_CuTeLayoutType:$cta_value_tile, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_corner_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_corner_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_padding_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$upper_padding_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_whd, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$lower_srt, - CuteNVGPU_IntTupleAsIntegerOrTupleOfIntTupleS:$stride_srt, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format - ); - let results = (outs - CuteNVGPU_SM90PointerToTMATensorDescriptor:$result - ); -} -def CuteNVGPU_MakeTmaDescTiledAtOp : CuteNVGPU_Op<"make_tma_desc_tiled_at"> { - let summary = "In-place version of Cute_MakeTmaDescTiledOp."; - let description = [{ - Operands (identical semantics to the Tiled op) - - Value gmem_view – Global-memory tensor view. - - Value smem_layout – SMEM box layout. - - Value cta_value_tile – CTA-value-tile mapping. - - Value tma_desc_addr – Cute pointer that names the memory location - where the TMA descriptor should be written. - - Value traversal_stride – Cute int tuple that defines the traversal stride. - If omitted, the traversal stride is set to 1 for all dimensions. - - Optional attributes (identical semantics to the Tiled op) - - mode, num_multicast, tma_format - - Results: none – descriptor is written to the supplied address. - - Example: - ```mlir - cute_nvgpu.make_tma_desc_tiled_at(%a_tensor, %a_smem_clayout, %a_cta_v_tile, %tma_desc_addr) - <{mode = }> : (!cute.memref, "(?,?,?) : (1,?,?)">, - !cute.composed_layout<"S<3,4,3> o 0 o (((64,64),2)):(((1,64,4096))">, - !cute.layout<"(((64,64),2)):(((1@0,1@1),64@0))">, - !cute.ptr>) - - cute_nvgpu.make_tma_desc_tiled_at(%sfa_tensor, %sfa_smem_clayout, %sfa_cta_v_tile, %existing_tma_desc) - <{mode = num_multicast = 2 tma_format = #cute_nvgpu.tma_data_format}> : ( - !cute.memref, "((32,4,?),(32,4,?)):((16,4,?),(0,1,512))">, - !cute.composed_layout<"S<0,4,3> o 0 o ((((32,4),1),(32,1)),1,(4,1))">, - !cute.layout<"(((32,4),32),1,4):(((1@0@0,1@1@0),1@0@1),0,1@1@1)">, - !cute.ptr>) - - cute_nvgpu.make_tma_desc_tiled(%view, %slayout, %cta_v_tile, %tma_desc_addr %traversal_stride) - <{mode = }> : ( - !cute.memref, "((?,?,?,?),?):((?,?,?,?),?)">, - !cute.composed_layout<"S<2,4,3> o 0 o (128,32):(32,1)">, - !cute.layout<"((128),32):((1@0@0),1@1)">, - !cute.ptr>, - !cute.int_tuple<"(1,?,?,?)">) - ``` - }]; - let arguments = (ins - CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, - CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, - CuteNVGPU_CuTeLayoutType:$cta_value_tile, - CuteNVGPU_CuTePointerType:$tma_desc_addr, - Optional:$traversal_stride, - CuteNVGPU_TmaLoadModeAttr:$mode, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format - ); - let results = (outs - ); -} -def CuteNVGPU_MakeTmaDescTiledOp : CuteNVGPU_Op<"make_tma_desc_tiled", [Pure]> { - let summary = "Create a TMA descriptor in tiled load mode."; - let description = [{ - Operands - - Value gmem_view – Cute view that describes the tensor in global memory. - - Value smem_layout – Composed/regular Cute layout that defines the - rectangular box in shared memory (SMEM tile). - - Value cta_value_tile – Cute layout that maps CTA threads to elements - within the SMEM tile. - - Value traversal_stride – Cute int tuple that defines the traversal stride. - If omitted, the traversal stride is set to 1 for all dimensions. - - Optional attributes - - mode : CuteTmaLoadModeAttr or CuteTmaStoreModeAttr (defaults to #cute_nvgpu.tma_load_mode) - - num_multicast : i32 (defaults to 1; 1 ⇒ no multicast) - - tma_format : CuteTmaDataFormatAttr (element format written to - bits 74–71 of the HW TMA - descriptor; if omitted an - appropriate default is deduced) - - Result - - Cute pointer to the freshly created TMA-descriptor object. - - Example: - ```mlir - %tma_desc_a = cute_nvgpu.make_tma_desc_tiled(%a_tensor, %a_smem_clayout, %a_cta_v_tile) - <{mode = #cute_nvgpu.tma_load_mode}> : (!cute.memref, "(?,?,?) : (1,?,?)">, - !cute.composed_layout<"S<3,4,3> o 0 o (((64,64),2)):(((1,64),4096))">, - !cute.layout<"(((64,64),2)):(((1@0,1@1),64@0))">) -> !cute.ptr> - - %tma_desc_sfa = cute_nvgpu.make_tma_desc_tiled(%sfa_tensor, %sfa_smem_clayout, %sfa_cta_v_tile) - <{mode = #cute_nvgpu.tma_load_mode num_multicast = 2 tma_format = #cute_nvgpu.tma_data_format}> : ( - !cute.memref, "((32,4,?),(32,4,?)):((16,4,?),(0,1,512))">, - !cute.composed_layout<"S<0,4,3> o 0 o ((((32,4),1),(32,1)),1,(4,1)):((((16,4),512),(0,1)),0,(1,512))">, - !cute.layout<"(((32,4),32),1,4):(((1@0@0,1@1@0),1@0@1),0,1@1@1)">) -> !cute.ptr> - - %tma_desc_strided_c = cute_nvgpu.make_tma_desc_tiled(%view, %slayout, %cta_v_tile, %traversal_stride) - <{mode = #cute_nvgpu.tma_load_mode}> : ( - !cute.memref, "((?,?,?,?),?):((?,?,?,?),?)">, - !cute.composed_layout<"S<2,4,3> o 0 o (128,32):(32,1)">, - !cute.layout<"((128),32):((1@0@0),1@1)">, - !cute.int_tuple<"(1,?,?,?)">) -> !cute.ptr> - ``` - }]; - let arguments = (ins - CuteNVGPU_ViewTypeInterfaceInstance:$gmem_view, - CuteNVGPU_GeneralizedLayoutTypeOfCuTe:$smem_layout, - CuteNVGPU_CuTeLayoutType:$cta_value_tile, - Optional:$traversal_stride, - CuteNVGPU_TmaLoadOrStoreModeAttr:$mode, - OptionalAttr:$num_multicast, - OptionalAttr:$tma_format - ); - let results = (outs - CuteNVGPU_SM90PointerToTMATensorDescriptor:$result - ); -} -def CuteNVGPU_MakeUMMASM107SmemDescOp : CuteNVGPU_Op<"sm107.make_umma_smem_desc", [Pure]> { - let summary = "Example:"; - let description = [{ - ```mlir - %1 = cute_nvgpu.sm107.make_umma_smem_desc(%arg1 : !cute.ptr) - layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major - -> !cute_nvgpu.sm107.smem_desc - ``` - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToMemorySpacesSmem:$src, - Optional:$nextSrc, - Cute_LayoutAttr:$layout, - CuteNVGPU_MajorModeAttr:$major - ); - let results = (outs - CuteNVGPU_CuteSm107SmemDescType:$res - ); -} -def CuteNVGPU_MakeUMMASmemDescCircularSM103Op : CuteNVGPU_InferredOp<"sm103.make_umma_smem_desc_circular", [Pure]> { - let summary = "Example:"; - let description = [{ - ```mlir - %1 = cute_nvgpu.sm103.make_umma_smem_desc_circular(%arg1 : !cute.ptr) - layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major - -> !cute_nvgpu.smem_desc_circular - ``` - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, - Optional:$nextSrc, - Cute_LayoutAttr:$layout, - CuteNVGPU_MajorModeAttr:$major - ); - let results = (outs - AnyType:$res - ); -} -def CuteNVGPU_MakeUMMASmemDescCircularSM107Op : CuteNVGPU_InferredOp<"sm107.make_umma_smem_desc_circular", [Pure]> { - let summary = "Example:"; - let description = [{ - ```mlir - %1 = cute_nvgpu.sm103.make_umma_smem_desc_circular(%arg1 : !cute.ptr) - layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major - -> !cute_nvgpu.smem_desc_circular - ``` - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, - Optional:$nextSrc, - Cute_LayoutAttr:$layout, - CuteNVGPU_MajorModeAttr:$major - ); - let results = (outs - AnyType:$res - ); -} -def CuteNVGPU_MakeUMMASmemDescOp : CuteNVGPU_Op<"make_umma_smem_desc", [Pure]> { - let summary = "Example:"; - let description = [{ - ```mlir - %1 = cute_nvgpu.make_umma_smem_desc(%arg1 : !cute.ptr) - layout = <"S<3, 4, 3> o 0 o ((64,16),1):((64,1),0)">, major = #cute_nvgpu.major - -> !cute_nvgpu.smem_desc - ``` - }]; - let arguments = (ins - CuteNVGPU_CuTePointerToMemorySpacesSmemWithByte:$src, - Optional:$nextSrc, - Cute_LayoutAttr:$layout, - CuteNVGPU_MajorModeAttr:$major - ); - let results = (outs - CuteNVGPU_CuteSmemDescType:$res - ); -} -def CuteNVGPU_PrefetchTmaDescriptorOp : CuteNVGPU_Op<"prefetch_tma_desc"> { - let arguments = (ins - CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$atom - ); - let results = (outs - ); -} -def CuteNVGPU_UpdateTMADescOp : CuteNVGPU_Op<"update_tma_desc"> { - let description = [{ - This operation updates the TMA descriptor in smem or gmem with the information from given gmem tensor. - Specifically, the following fields of the TMA descriptor will be updated: - 1. the global tensor base address will be updated with the base address of the global memory tensor. - 2. the tensor shapes will be updated with the shape of the global memory tensor. - 3. the tensor strides will be updated with the stride of the global memory tensor. - Other fields of the TMA descriptor will be left unchanged. - The TMA descriptor pointer specifies the address of the tma descriptor to be updated. - It could be a smem or gmem pointer. The pointer should be aligned to at least 64B. - Example: - ```mlir - cute_nvgpu.update_tma_desc(%tma_atom, %gmem_tensor, %smem_ptr) : ( - !cute_nvgpu.atom.non_exec_tiled_tma_load>, - !cute.memref, "(?,?,?) : (1,?,?)">, - !cute.ptr>) - -> () - - cute_nvgpu.update_tma_desc(%tma_atom, %gmem_tensor, %gmem_ptr) : ( - !cute_nvgpu.atom.non_exec_tiled_tma_load>, - !cute.memref, "(?,?,?) : (?,1,?)">, - !cute.ptr>) - -> () - ``` - }]; - let arguments = (ins - CuteNVGPU_TypeInterfaceForNonExecutableTMAAtomTypes:$tma_atom, - CuteNVGPU_AMemoryReferenceTypeThatRepresentsATensor:$gmem_tensor, - CuteNVGPU_CuTePointerToMemorySpacesSmemOrCuTe:$tma_desc_ptr - ); - let results = (outs - ); -} - -#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUOPS_TD diff --git a/mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td b/mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td deleted file mode 100644 index 817af564..00000000 --- a/mlir/dialects/cute_nvgpu/CuteNVGPUTypes.td +++ /dev/null @@ -1,750 +0,0 @@ -//===- CuteNVGPUTypes.td - CuTe NVIDIA GPU types -----------*- tablegen -*-===// -// -// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler -// (`_cutlass_ir.cu13` parser messages and verifier strings, its typed Python -// type classes, and probing its type parser and printer); do not edit by hand. -// -// Every cute_nvgpu type of the compiler is here, with its parameters in the -// compiler's order and C++ types. Parameters the compiler parses with custom -// directives (the `MxNxK` shape, `frag_kind`, `D DP`, `xR`, `t|n`, the sm90 -// negation flags) are named after the typed getters or the directive. -// -// Syntax is the compiler's, byte for byte, except where its printer emits text -// its own parser rejects; the parsers here also accept that text, and the -// printers here then emit the form the compiler parses back to the same type: -// - atom.universal_copy with copy_bits 0 and an address space prints `0 b`; -// - atom.simt_multimem_* print a default field when a later one is not; -// - sm100.mma_bs(_sp) print `arch_promote` when `int_overflow` is present. -// -// Verifiers carry the compiler's rules on parameter values where they are -// known, with its messages; instruction legality (the MMA shape and type -// tables, multimem PTX forms, ldmatrix/stmatrix configurations) is left to the -// compiler. -// -// Left out: the hidden `atom` attribute of the mma atoms' typed get()s. It -// overrides the atom's layouts, is never printed, and so cannot survive text; -// every mma atom parsed from text has it unset, which is what these types -// model. -// -//===----------------------------------------------------------------------===// - -#ifndef MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD -#define MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD - -include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.td" - -class CuteNVGPU_CustomType - : CuteNVGPU_Type { - let hasCustomAssemblyFormat = 1; -} - -defvar CuteNVGPU_Ty = "::mlir::Type"; -defvar CuteNVGPU_Shape = "::mlir::cutlass_compiler::cute::ShapeAttr"; -defvar CuteNVGPU_Layout = "::mlir::cutlass_compiler::cute::LayoutAttr"; -defvar CuteNVGPU_LayoutTy = "::mlir::cutlass_compiler::cute::LayoutType"; -defvar CuteNVGPU_StrideTy = "::mlir::cutlass_compiler::cute::StrideType"; -defvar CuteNVGPU_Major = "::mlir::cutlass_compiler::cute::MajorMode"; -defvar CuteNVGPU_Space = "::mlir::cutlass_compiler::cute::AddressSpace"; - -//===----------------------------------------------------------------------===// -// Descriptors -//===----------------------------------------------------------------------===// - -def CuteNVGPU_SmemDescType : CuteNVGPU_Type<"SmemDesc", "smem_desc"> { - let summary = "UMMA/GMMA shared-memory descriptor"; - let description = "`!cute_nvgpu.smem_desc`"; - let assemblyFormat = ""; -} - -def CuteNVGPU_SmemDescSM107Type - : CuteNVGPU_Type<"SmemDescSM107", "sm107.smem_desc"> { - let summary = "SM107 shared-memory descriptor"; - let description = "`!cute_nvgpu.sm107.smem_desc`"; - let assemblyFormat = ""; -} - -class CuteNVGPU_SmemDescCircular - : CuteNVGPU_Type { - let summary = "Circular shared-memory descriptor over a block layout in bytes"; - let description = [{ - The block layout is static, has three modes, a 128-byte mode 1 and is - mode-1-major compact, with at least two blocks in mode 2: - - `!cute_nvgpu.}] # typeMnemonic # [{<<"(8,128,2):(128,1,1024)">>` - }]; - let parameters = (ins CuteNVGPU_Layout:$block_layout_bytes); - let assemblyFormat = "`<` $block_layout_bytes `>`"; - let genVerifyDecl = 1; -} - -def CuteNVGPU_SmemDescCircularSM103Type - : CuteNVGPU_SmemDescCircular<"SmemDescCircularSM103", - "sm103.smem_desc_circular">; -def CuteNVGPU_SmemDescCircularSM107Type - : CuteNVGPU_SmemDescCircular<"SmemDescCircularSM107", - "sm107.smem_desc_circular">; - -def CuteNVGPU_SmemDescViewType - : CuteNVGPU_CustomType<"SmemDescView", "smem_desc_view"> { - let summary = "Shared-memory descriptor iterator with a layout"; - let description = [{ - The descriptor is a smem_desc, sm107.smem_desc or smem_desc_circular type; - the layout is a plain layout, as a string. - - `!cute_nvgpu.smem_desc_view` - }]; - let parameters = (ins CuteNVGPU_Ty:$desc, CuteNVGPU_Layout:$layout); - let genVerifyDecl = 1; -} - -def CuteNVGPU_TmaDescriptorTiledType - : CuteNVGPU_Type<"TmaDescriptorTiled", "tma_descriptor_tiled"> { - let summary = "Tiled TMA descriptor"; - let description = "`!cute_nvgpu.tma_descriptor_tiled`"; - let assemblyFormat = ""; -} - -def CuteNVGPU_TmaDescriptorIm2ColType - : CuteNVGPU_Type<"TmaDescriptorIm2Col", "tma_descriptor_im2col"> { - let summary = "Im2col TMA descriptor"; - let description = "`!cute_nvgpu.tma_descriptor_im2col`"; - let assemblyFormat = ""; -} - -def CuteNVGPU_WorkIdResponseType - : CuteNVGPU_Type<"WorkIdResponse", "workid_response"> { - let summary = "Cluster launch control work-id response"; - let description = "`!cute_nvgpu.workid_response`"; - let assemblyFormat = ""; -} - -//===----------------------------------------------------------------------===// -// SIMT copy atoms -//===----------------------------------------------------------------------===// - -def CuteNVGPU_CopyAtomSIMTSyncCopyType - : CuteNVGPU_CustomType<"CopyAtomSIMTSyncCopy", "atom.universal_copy"> { - let summary = "Universal (SIMT) copy atom"; - let description = [{ - `copy_bits` 0 means the element width and is kept as 0; the address - spaces default to generic. Each field needs the previous ones. - - `!cute_nvgpu.atom.universal_copy` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - DefaultValuedParameter<"int", "0">:$copy_bits, - DefaultValuedParameter:$src_space, - DefaultValuedParameter:$dst_space - ); - let genVerifyDecl = 1; -} - -def CuteNVGPU_CopyAtomSIMTAsyncCopyType - : CuteNVGPU_CustomType<"CopyAtomSIMTAsyncCopy", "atom.simt_async_copy"> { - let summary = "cp.async copy atom"; - let description = [{ - `copy_bits` absent or 0 parses as the element width and is always printed. - - `!cute_nvgpu.atom.simt_async_copy` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "LoadCacheMode":$cache, - "int":$copy_bits - ); -} - -def CuteNVGPU_CopyAtomDsmemStoreType - : CuteNVGPU_CustomType<"CopyAtomDsmemStore", "atom.dsmem_store"> { - let summary = "Distributed shared-memory store atom"; - let description = [{ - `copy_bits` absent or 0 parses as the element width and is always printed. - - `!cute_nvgpu.atom.dsmem_store` - }]; - let parameters = (ins CuteNVGPU_Ty:$val_type, "int":$copy_bits); - let genVerifyDecl = 1; -} - -class CuteNVGPU_BulkCopy - : CuteNVGPU_CustomType { - let summary = "Bulk (TMA non-tensor) copy atom"; - let description = [{ - Space-separated groups, printed when not default (`copy_bits` 0). - - `!cute_nvgpu.}] # typeMnemonic # "<" # example # ">`"; - let parameters = !con( - (ins CuteNVGPU_Ty:$val_type, DefaultValuedParameter<"int", "0">:$copy_bits), - !if(!empty(flag), (ins), - !dag(ins, [DefaultValuedParameter<"bool", "false">], [flag]))); -} - -def CuteNVGPU_CopyAtomBulkCopyG2SType - : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyG2S", "atom.bulk_copy_g2s", "mcast", - "f32 copy_bits = 128 mcast = true">; -def CuteNVGPU_CopyAtomBulkCopyS2GType - : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyS2G", "atom.bulk_copy_s2g", "mask", - "f32 copy_bits = 128 mask = true">; -def CuteNVGPU_CopyAtomBulkCopyS2SType - : CuteNVGPU_BulkCopy<"CopyAtomBulkCopyS2S", "atom.bulk_copy_s2s", "", - "f32 copy_bits = 128">; - -// g2r, r2g, r2s, s2r: ``, space separated, -// fields in parameter order and printed when not default; copy_bits absent or -// 0 parses as the element width and is always printed. - -def CuteNVGPU_CopyAtomG2RType - : CuteNVGPU_CustomType<"CopyAtomG2R", "atom.g2r"> { - let summary = "Global-to-register load atom"; - let description = [{ - `!cute_nvgpu.atom.g2r`, - `!cute_nvgpu.atom.g2r` (invariant only with - the default memory features) - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "int":$copy_bits, - DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, - DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, - DefaultValuedParameter<"L2PrefetchSize", "L2PrefetchSize::NONE">:$l2_prefetch_size, - DefaultValuedParameter<"CacheEvictionPriority", "CacheEvictionPriority::EVICT_NORMAL">:$l1_cache_evict_priority, - DefaultValuedParameter<"LoadCacheMode", "LoadCacheMode::always">:$load_cache_mode, - DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space, - DefaultValuedParameter<"bool", "false">:$invariant - ); - let genVerifyDecl = 1; -} - -def CuteNVGPU_CopyAtomR2GType - : CuteNVGPU_CustomType<"CopyAtomR2G", "atom.r2g"> { - let summary = "Register-to-global store atom"; - let description = [{ - `!cute_nvgpu.atom.r2g` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "int":$copy_bits, - DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, - DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, - DefaultValuedParameter<"CacheEvictionPriority", "CacheEvictionPriority::EVICT_NORMAL">:$l1_cache_evict_priority, - DefaultValuedParameter<"StoreCacheMode", "StoreCacheMode::write_back">:$store_cache_mode, - DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space - ); - let genVerifyDecl = 1; -} - -class CuteNVGPU_SharedCopy - : CuteNVGPU_CustomType { - let summary = what # " atom"; - let description = [{ - `!cute_nvgpu.}] # typeMnemonic # [{` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "int":$copy_bits, - DefaultValuedParameter<"MemOrderKind", "MemOrderKind::WEAK">:$mem_order, - DefaultValuedParameter<"MemScopeKind", "MemScopeKind::CTA">:$mem_scope, - DefaultValuedParameter<"SharedSpace", "SharedSpace::CTA">:$shared_space - ); - let genVerifyDecl = 1; -} - -def CuteNVGPU_CopyAtomR2SType - : CuteNVGPU_SharedCopy<"CopyAtomR2S", "atom.r2s", "Register-to-shared store">; -def CuteNVGPU_CopyAtomS2RType - : CuteNVGPU_SharedCopy<"CopyAtomS2R", "atom.s2r", "Shared-to-register load">; - -// Multimem atoms: positional `, ORDER, SCOPE>`; a trailing -// run of defaults is omitted (copy_bits 0, ADD, no accumulation precision, -// relaxed, sys). - -def CuteNVGPU_CopyAtomSIMTMultimemStType - : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemSt", "atom.simt_multimem_st"> { - let summary = "multimem.st atom"; - let description = [{ - `copy_bits` 0 (the element width) is kept as 0. - - `!cute_nvgpu.atom.simt_multimem_st` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - DefaultValuedParameter<"int", "0">:$copy_bits, - DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, - DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope - ); -} - -def CuteNVGPU_CopyAtomSIMTMultimemRedType - : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemRed", "atom.simt_multimem_red"> { - let summary = "multimem.red atom"; - let description = [{ - `copy_bits` absent or 0 parses as the element width and is always printed. - - `!cute_nvgpu.atom.simt_multimem_red` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "int":$copy_bits, - DefaultValuedParameter<"ReductionKind", "ReductionKind::ADD">:$reduction, - DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, - DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope - ); -} - -def CuteNVGPU_CopyAtomSIMTMultimemLdReduceType - : CuteNVGPU_CustomType<"CopyAtomSIMTMultimemLdReduce", - "atom.simt_multimem_ld_reduce"> { - let summary = "multimem.ld_reduce atom"; - let description = [{ - `copy_bits` 0 (the element width) is kept as 0. The accumulation - precision must be given to give a memory order or scope. - - `!cute_nvgpu.atom.simt_multimem_ld_reduce, acquire, gpu>` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - DefaultValuedParameter<"int", "0">:$copy_bits, - DefaultValuedParameter<"ReductionKind", "ReductionKind::ADD">:$reduction, - OptionalParameter<"LdReduceAccPrecisionKindAttr">:$ld_reduce_acc_precision, - DefaultValuedParameter<"MemOrderKind", "MemOrderKind::RELAXED">:$mem_order, - DefaultValuedParameter<"MemScopeKind", "MemScopeKind::SYS">:$mem_scope - ); -} - -//===----------------------------------------------------------------------===// -// Tensor-memory copy atoms: `` -//===----------------------------------------------------------------------===// - -class CuteNVGPU_TmemCopy - : CuteNVGPU_CustomType { - let summary = what; - let description = [{ - `!cute_nvgpu.}] # typeMnemonic # "`"; - let parameters = !con( - (ins CuteNVGPU_Ty:$val_type, "int":$num_dp, "int":$num_bit, - "unsigned":$num_rep), - !dag(ins, [OptionalParameter<"::mlir::UnitAttr">], [flag])); - let genVerifyDecl = 1; -} - -def CuteNVGPU_CopyAtomSM100TmemLoadType - : CuteNVGPU_TmemCopy<"CopyAtomSM100TmemLoad", "atom.tmem_load", - "tcgen05.ld atom", "pack16b">; -def CuteNVGPU_CopyAtomSM100TmemStoreType - : CuteNVGPU_TmemCopy<"CopyAtomSM100TmemStore", "atom.tmem_store", - "tcgen05.st atom", "expand16b">; - -def CuteNVGPU_CopyAtomSM10xTmemLoadRedType - : CuteNVGPU_CustomType<"CopyAtomSM10xTmemLoadRed", "atom.tmem_load_red"> { - let summary = "tcgen05.ld.red atom"; - let description = [{ - `half_split_off` prints as an integer attribute without spaces around `=`. - - `!cute_nvgpu.atom.tmem_load_red` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "int":$num_dp, - "int":$num_bit, - "unsigned":$num_rep, - "TmemLoadRedOp":$red_op, - OptionalParameter<"::mlir::UnitAttr">:$nan, - OptionalParameter<"::mlir::IntegerAttr">:$half_split_off - ); - let genVerifyDecl = 1; -} - -def CuteNVGPU_CopyAtomSM107TmemLoadSPCompressType - : CuteNVGPU_CustomType<"CopyAtomSM107TmemLoadSPCompress", - "atom.tmem_load_spcompress"> { - let summary = "SM107 tensor-memory load with sparse compression"; - let description = [{ - `!cute_nvgpu.atom.tmem_load_spcompress` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "int":$num_dp, - "int":$num_bit, - "unsigned":$num_rep, - "TmemLoadRedOp":$red_op, - OptionalParameter<"::mlir::UnitAttr">:$red, - OptionalParameter<"::mlir::UnitAttr">:$nan - ); - let genVerifyDecl = 1; -} - -def CuteNVGPU_CopyAtomSM100CopyS2TType - : CuteNVGPU_CustomType<"CopyAtomSM100CopyS2T", "atom.s2t_copy"> { - let summary = "tcgen05.cp (shared to tensor memory) atom"; - let description = [{ - The broadcast mode `none` is omitted. - - `!cute_nvgpu.atom.s2t_copy` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "int":$num_dp, - "int":$num_bit, - "int":$num_cta, - DefaultValuedParameter<"CopyS2TBroadcast", "CopyS2TBroadcast::none">:$broadcast - ); - let genVerifyDecl = 1; -} - -def CuteNVGPU_CopyAtomSM100S2TCopyV2Type - : CuteNVGPU_CustomType<"CopyAtomSM100S2TCopyV2", "atom.sm100_s2t_copy_v2"> { - let summary = "tcgen05.cp atom with the shared-memory major mode"; - let description = [{ - The broadcast mode `none` is omitted. - - `!cute_nvgpu.atom.sm100_s2t_copy_v2` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "int":$num_dp, - "int":$num_bit, - "int":$num_cta, - CuteNVGPU_Major:$smem_major, - DefaultValuedParameter<"CopyS2TBroadcast", "CopyS2TBroadcast::none">:$broadcast - ); - let genVerifyDecl = 1; -} - -//===----------------------------------------------------------------------===// -// ldmatrix / stmatrix -//===----------------------------------------------------------------------===// - -def CuteNVGPU_CopyAtomLdsmType - : CuteNVGPU_CustomType<"CopyAtomLdsm", "atom.ldsm"> { - let summary = "ldmatrix atom"; - let description = [{ - The last field is `t` (transposed) or `n`. - - `!cute_nvgpu.atom.ldsm, sz_pattern = u16, num_matrices = 4, t>` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - CuteNVGPU_Shape:$mode, - "LdsmSzPattern":$sz_pattern, - "int":$num_matrices, - OptionalParameter<"::mlir::UnitAttr">:$transpose - ); -} - -def CuteNVGPU_CopyAtomStsmType - : CuteNVGPU_CustomType<"CopyAtomStsm", "atom.stsm"> { - let summary = "stmatrix atom"; - let description = [{ - The last field is `t` (transposed) or `n`. - - `!cute_nvgpu.atom.stsm, num_matrices = 4, n>` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - CuteNVGPU_Shape:$mode, - "int":$num_matrices, - OptionalParameter<"::mlir::UnitAttr">:$transpose - ); -} - -//===----------------------------------------------------------------------===// -// TMA atoms -//===----------------------------------------------------------------------===// - -// Executable: comma-separated fields up to g_stride, then space-separated -// groups printed when set (booleans when true, sparsity when not 1). - -def CuteNVGPU_CopyAtomTmaLoadType - : CuteNVGPU_CustomType<"CopyAtomTmaLoad", "atom.tma_load"> { - let summary = "Executable TMA load atom"; - let description = [{ - `!cute_nvgpu.atom.tma_load mcast = true tma_gbasis = <"(64,1):(1@0,1@1)">>` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - DefaultValuedParameter<"int", "1">:$sparsity, - "int":$copy_bits, - "TmaLoadMode":$mode, - "int":$num_cta, - CuteNVGPU_StrideTy:$g_stride, - DefaultValuedParameter<"bool", "false">:$mcast, - OptionalParameter:$tma_gbasis, - DefaultValuedParameter<"bool", "false">:$override, - DefaultValuedParameter<"bool", "false">:$no_fully_oob_tile - ); - let genVerifyDecl = 1; -} - -def CuteNVGPU_CopyAtomTmaStoreType - : CuteNVGPU_CustomType<"CopyAtomTmaStore", "atom.tma_store"> { - let summary = "Executable TMA store atom"; - let description = [{ - `!cute_nvgpu.atom.tma_store tma_gbasis = <"(64,16):(1@0,1@1)">>` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - DefaultValuedParameter<"int", "1">:$sparsity, - "int":$copy_bits, - "TmaStoreMode":$mode, - CuteNVGPU_StrideTy:$g_stride, - OptionalParameter:$tma_gbasis, - DefaultValuedParameter<"bool", "false">:$override, - DefaultValuedParameter<"bool", "false">:$no_fully_oob_tile - ); - let genVerifyDecl = 1; -} - -def CuteNVGPU_CopyAtomTmaReduceType - : CuteNVGPU_CustomType<"CopyAtomTmaReduce", "atom.tma_reduce"> { - let summary = "Executable TMA reduce atom"; - let description = [{ - `!cute_nvgpu.atom.tma_reduce tma_gbasis = <"(32,128):(1@1,1@0)">>` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "int":$copy_bits, - "TmaStoreMode":$mode, - "ReductionKind":$kind, - CuteNVGPU_StrideTy:$g_stride, - OptionalParameter:$tma_gbasis - ); -} - -def CuteNVGPU_CopyAtomIm2ColTmaLoadType - : CuteNVGPU_CustomType<"CopyAtomIm2ColTmaLoad", "atom.im2col_tma_load"> { - let summary = "Executable im2col TMA load atom"; - let description = [{ - `!cute_nvgpu.atom.im2col_tma_load mcast = true tma_gbasis = <"(64,128):(1@1,1@0)">>` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "int":$copy_bits, - "int":$num_cta, - CuteNVGPU_StrideTy:$g_stride, - DefaultValuedParameter<"bool", "false">:$mcast, - OptionalParameter:$tma_gbasis - ); -} - -def CuteNVGPU_CopyAtomIm2ColTmaStoreType - : CuteNVGPU_CustomType<"CopyAtomIm2ColTmaStore", "atom.im2col_tma_store"> { - let summary = "Executable im2col TMA store atom"; - let description = [{ - `!cute_nvgpu.atom.im2col_tma_store tma_gbasis = <"(32,128):(1@1,1@0)">>` - }]; - let parameters = (ins - CuteNVGPU_Ty:$val_type, - "int":$copy_bits, - CuteNVGPU_StrideTy:$g_stride, - OptionalParameter:$tma_gbasis - ); -} - -// Non-executable: `<[kind, ]T, copy_bits = N, tma_gbasis = <"L">[, tma_format = F]>`. -// An omitted format is the element type's (F16_RN, BF16_RN, F32_RN, TF32_RN, -// F64_RN, U4 for 4-bit, U8 up to 8 bits, U16, U32/S32, U64/S64); it is always -// printed, and empty when the element type has none. - -class CuteNVGPU_NonExecTma - : CuteNVGPU_CustomType { - let summary = "Non-executable " # what # " atom"; - let description = "`!cute_nvgpu." # typeMnemonic # "<" # example # ">`"; - defvar common = (ins - CuteNVGPU_Ty:$val_type, - "int":$copy_bits, - CuteNVGPU_LayoutTy:$tma_gbasis, - "std::optional":$tma_format - ); - let parameters = !if(!empty(kindType), common, - !con((ins kindType:$kind), common)); - let genVerifyDecl = 1; -} - -def CuteNVGPU_CopyAtomNonExecTiledTmaLoadType - : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaLoad", - "atom.non_exec_tiled_tma_load", "tiled TMA load", "TiledTmaLoad", - "sm_90, bf16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = BF16_RN">; -def CuteNVGPU_CopyAtomNonExecTiledTmaStoreType - : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaStore", - "atom.non_exec_tiled_tma_store", "tiled TMA store", "", - "bf16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = BF16_RN">; -def CuteNVGPU_CopyAtomNonExecTiledTmaReduceType - : CuteNVGPU_NonExecTma<"CopyAtomNonExecTiledTmaReduce", - "atom.non_exec_tiled_tma_reduce", "tiled TMA reduce", "ReductionKind", - "ADD, f32, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F32_RN">; -def CuteNVGPU_CopyAtomNonExecIm2ColTmaLoadType - : CuteNVGPU_NonExecTma<"CopyAtomNonExecIm2ColTmaLoad", - "atom.non_exec_im2col_tma_load", "im2col TMA load", "Im2ColTmaLoad", - "sm_90, f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; -def CuteNVGPU_CopyAtomNonExecIm2ColTmaStoreType - : CuteNVGPU_NonExecTma<"CopyAtomNonExecIm2ColTmaStore", - "atom.non_exec_im2col_tma_store", "im2col TMA store", "", - "f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; -def CuteNVGPU_CopyAtomNonExec2DGather4TmaLoadType - : CuteNVGPU_NonExecTma<"CopyAtomNonExec2DGather4TmaLoad", - "atom.non_exec_2d_gather4_tma_load", "2-D gather4 TMA load", - "GatherScatterTmaLoad", - "sm_100, f16, copy_bits = 1024, tma_gbasis = <\"(64,1):(1@0,1@1)\">, tma_format = F16_RN">; -def CuteNVGPU_CopyAtomNonExec2DScatter4TmaStoreType - : CuteNVGPU_NonExecTma<"CopyAtomNonExec2DScatter4TmaStore", - "atom.non_exec_2d_scatter4_tma_store", "2-D scatter4 TMA store", "", - "f16, copy_bits = 1024, tma_gbasis = <\"(4,64):(1@0,1@1)\">, tma_format = F16_RN">; - -//===----------------------------------------------------------------------===// -// MMA atoms. The shape is `MxNxK` (a static rank-3 shape); `elem_type` lists -// A, B and C; `frag_kind` is the A fragment kind: `ss` (smem_desc), `rs` -// (rmem, sm90) or `ts` (tmem, sm100 and sm107). -//===----------------------------------------------------------------------===// - -class CuteNVGPU_MmaAtom - : CuteNVGPU_CustomType { - let summary = what # " MMA atom"; - let description = "`!cute_nvgpu." # typeMnemonic # "<" # example # ">`"; - let genVerifyDecl = 1; -} - -def CuteNVGPU_UniversalFmaAtomType - : CuteNVGPU_MmaAtom<"UniversalFmaAtom", "atom.universal_fma", "Universal FMA", - "1x1x1, (f32, f32) -> f32 "> { - let parameters = (ins CuteNVGPU_Shape:$shape_mnk, CuteNVGPU_Ty:$a_type, - CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type); -} - -def CuteNVGPU_MmaAtomSM80Type - : CuteNVGPU_MmaAtom<"MmaAtomSM80", "sm80.mma", "SM80 mma.sync", - "16x8x32, elem_type = (i8, i8, i32), int_overflow = "> { - let description = [{ - `binary_op` needs `int_overflow` before it. - - `!cute_nvgpu.sm80.mma<16x8x32, elem_type = (i8, i8, i32), int_overflow = , binary_op = >` - }]; - let parameters = (ins - CuteNVGPU_Shape:$shape_mnk, - CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, - OptionalParameter<"MMAIntOverflowAttr">:$int_overflow, - OptionalParameter<"BinaryOpAttr">:$binary_op - ); -} - -def CuteNVGPU_MmaAtomSM80SparseType - : CuteNVGPU_MmaAtom<"MmaAtomSM80Sparse", "sm80.sparse_mma", - "SM80 sparse mma.sync", - "16x8x64, elem_type = (i8, i8, i32), md_format = tid, int_overflow = "> { - let parameters = (ins - CuteNVGPU_Shape:$shape_mnk, - CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, - "SparseMetadataFormat":$sparse_metadata_format, - OptionalParameter<"MMAIntOverflowAttr">:$int_overflow - ); -} - -def CuteNVGPU_MmaAtomSM89Type - : CuteNVGPU_MmaAtom<"MmaAtomSM89", "sm89.mma", "SM89 fp8 mma.sync", - "16x8x32, elem_type = (f8E4M3FN, f8E4M3FN, f32) "> { - let parameters = (ins CuteNVGPU_Shape:$shape_mnk, CuteNVGPU_Ty:$a_type, - CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type); -} - -def CuteNVGPU_MmaAtomSM90Type - : CuteNVGPU_MmaAtom<"MmaAtomSM90", "sm90.mma", "SM90 wgmma", - "64x128x16, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = rs, a_neg, b_neg"> { - let description = [{ - The trailing `int_overflow`, `a_neg` and `b_neg` parse in any order and - print in this one. - - `!cute_nvgpu.sm90.mma<64x128x32, ab_major = (k, k), elem_type = (i8, i8, i32), frag_kind = ss, int_overflow = , a_neg, b_neg>` - }]; - let parameters = (ins - CuteNVGPU_Shape:$shape_mnk, - CuteNVGPU_Major:$a_major, CuteNVGPU_Major:$b_major, - CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, - "MmaFragKind":$a_frag_kind, - OptionalParameter<"MMAIntOverflowAttr">:$int_overflow, - OptionalParameter<"::mlir::UnitAttr">:$a_neg, - OptionalParameter<"::mlir::UnitAttr">:$b_neg - ); -} - -def CuteNVGPU_MmaAtomSM120BlockScaledType - : CuteNVGPU_MmaAtom<"MmaAtomSM120BlockScaled", "SM120.mma_bs", - "SM120 block-scaled mma.sync", - "16x8x64, vec_size = 32, elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, use_sf_layout_TV = true"> { - let parameters = (ins - CuteNVGPU_Shape:$shape_mnk, - "int":$vec_size, - CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type, - CuteNVGPU_Ty:$sf_type, - "bool":$use_sf_layout_TV - ); -} - -// UMMA (tcgen05.mma): `]>`. arch_promote (sm100 -// block-scaled only) defaults to sm_100 and is omitted then; the sm107 atoms -// require ab_collector_op. - -class CuteNVGPU_Umma - : CuteNVGPU_MmaAtom { - let parameters = !con( - (ins CuteNVGPU_Shape:$shape_mnk, "int":$num_cta, - CuteNVGPU_Major:$a_major, CuteNVGPU_Major:$b_major, - CuteNVGPU_Ty:$a_type, CuteNVGPU_Ty:$b_type, CuteNVGPU_Ty:$c_type), - !if(sf, (ins CuteNVGPU_Ty:$sf_type), (ins)), - !if(!and(sp, !not(sf)), (ins CuteNVGPU_Ty:$e_type), (ins)), - !if(sp, (ins "SparseMetadataFormat":$sparse_metadata_format), (ins)), - (ins "MmaFragKind":$a_frag_kind), - !if(bs, (ins "int":$vec_size), (ins "int":$c_scale_exp)), - !if(!and(bs, !not(sm107)), - (ins DefaultValuedParameter<"Arch", "Arch::sm_100">:$arch_promote), (ins)), - !if(sm107, (ins "MmaCollectorOp":$a_collector_op, - "MmaCollectorOp":$b_collector_op), (ins)), - (ins OptionalParameter<"MMAIntOverflowAttr">:$int_overflow)); -} - -def CuteNVGPU_MmaAtomSM100UMMAType - : CuteNVGPU_Umma<"MmaAtomSM100UMMA", "sm100.mma", "SM100 UMMA", - "128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0", - 0, 0, 0, 0>; -def CuteNVGPU_MmaAtomSM100UMMASparseType - : CuteNVGPU_Umma<"MmaAtomSM100UMMASparse", "sm100.mma_sp", "SM100 sparse UMMA", - "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0", - 0, 1, 0, 0>; -def CuteNVGPU_MmaAtomSM100UMMABlockScaledType - : CuteNVGPU_Umma<"MmaAtomSM100UMMABlockScaled", "sm100.mma_bs", - "SM100 block-scaled UMMA", - "128x128x96, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 16, arch_promote = sm_103", - 1, 0, 1, 0>; -def CuteNVGPU_MmaAtomSM100UMMABlockScaledSparseType - : CuteNVGPU_Umma<"MmaAtomSM100UMMABlockScaledSparse", "sm100.mma_bs_sp", - "SM100 block-scaled sparse UMMA", - "128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32", - 1, 1, 1, 0>; -def CuteNVGPU_MmaAtomSM107UMMAType - : CuteNVGPU_Umma<"MmaAtomSM107UMMA", "sm107.mma", "SM107 UMMA", - "128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0, ab_collector_op = (lastuse, fill)", - 0, 0, 0, 1>; -def CuteNVGPU_MmaAtomSM107UMMASparseType - : CuteNVGPU_Umma<"MmaAtomSM107UMMASparse", "sm107.mma_sp", "SM107 sparse UMMA", - "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0, ab_collector_op = (discard, discard)", - 0, 1, 0, 1>; -def CuteNVGPU_MmaAtomSM107UMMABlockScaledType - : CuteNVGPU_Umma<"MmaAtomSM107UMMABlockScaled", "sm107.mma_bs", - "SM107 block-scaled UMMA", - "128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, use), int_overflow = ", - 1, 0, 1, 1>; -def CuteNVGPU_MmaAtomSM107UMMABlockScaledSparseType - : CuteNVGPU_Umma<"MmaAtomSM107UMMABlockScaledSparse", "sm107.mma_bs_sp", - "SM107 block-scaled sparse UMMA", - "128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, discard)", - 1, 1, 1, 1>; - -#endif // MLIR_DIALECTS_CUTE_NVGPU_CUTENVGPUTYPES_TD diff --git a/mlir/dialects/cute_nvgpu/capi.cc b/mlir/dialects/cute_nvgpu/capi.cc deleted file mode 100644 index 73d55bc3..00000000 --- a/mlir/dialects/cute_nvgpu/capi.cc +++ /dev/null @@ -1,4429 +0,0 @@ -//===- capi.cc - C API for the cute_nvgpu dialect -------------------------===// - -#include "capi.h" - -#include "mlir/dialects/cute_nvgpu/CuteNVGPUDialect.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" - -// Registration - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION( - CuteNVGPU, cute_nvgpu, mlir::cutlass_compiler::cute_nvgpu::CuteNVGPUDialect) - -// Attributes - -namespace cute_nvgpu = mlir::cutlass_compiler::cute_nvgpu; - -// Unwraps `handle` into `out`: false when it is of another context or not a -// T, or when it is null and not optional. -template -static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { - auto value = unwrap(handle); - if (!value) - return optional; - if (value.getContext() != ctx) - return false; - out = llvm::dyn_cast(value); - return static_cast(out); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyG2S( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldBulkCopyG2SAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2G( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2GAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldBulkCopyS2S( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldBulkCopyS2SAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldDsmemStore( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldDsmemStoreAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldLoadGlobal( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldLoadGlobalAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExec2DGather4TmaLoad(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldNonExec2DGather4TmaLoadAttr::get( - ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( - MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExec2DScatter4TmaStore(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldNonExec2DScatter4TmaStoreAttr::get( - ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( - MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExecIm2ColTmaLoad(value); - if (!valueValue) - return {nullptr}; - return wrap( - cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExecIm2ColTmaStore(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldNonExecIm2ColTmaStoreAttr::get( - ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue( - MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExecTiledTmaLoad(value); - if (!valueValue) - return {nullptr}; - return wrap( - cute_nvgpu::AtomCopyFieldNonExecTiledTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExecTiledTmaReduce(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldNonExecTiledTmaReduceAttr::get( - ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue( - MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomCopyFieldNonExecTiledTmaStore(value); - if (!valueValue) - return {nullptr}; - return wrap( - cute_nvgpu::AtomCopyFieldNonExecTiledTmaStoreAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldStoreGlobal( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldStoreGlobalAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaLoad( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaReduce( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldTmaReduceAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomCopyFieldTmaStore( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomCopyFieldTmaStoreAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomMmaFieldSM100Attr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomMmaFieldSM100BlockScaled(value); - if (!valueValue) - return {nullptr}; - return wrap( - cute_nvgpu::AtomMmaFieldSM100BlockScaledAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse( - MlirAttribute attr) { - return llvm::isa_and_nonnull< - cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr>(unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomMmaFieldSM100BlockScaledSparse(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomMmaFieldSM100BlockScaledSparseAttr::get( - ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue( - MlirAttribute attr) { - return static_cast( - llvm::cast( - unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM100Sparse( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomMmaFieldSM100SparseAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute_nvgpu:: - symbolizeAtomMmaFieldSM120BlockScaled(value); - if (!valueValue) - return {nullptr}; - return wrap( - cute_nvgpu::AtomMmaFieldSM120BlockScaledAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM80Sparse( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomMmaFieldSM80SparseAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeAtomMmaFieldSM90(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::AtomMmaFieldSM90Attr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeBinaryOp(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::BinaryOpAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyS2TBroadcastAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeGatherScatterTmaLoad( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::GatherScatterTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeIm2ColTmaLoad(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::Im2ColTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdReduceAccPrecisionKind( - value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::LdReduceAccPrecisionKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdsmSzPattern(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::LdsmSzPatternAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::LoadCacheModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMMAIntOverflow(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::MMAIntOverflowAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUMajorModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::MajorModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::MmaCollectorOpAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::MmaFragKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute_nvgpu::NotImplementedFrgAttr::get(ctx)); -} - -bool mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUReductionKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::ReductionKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPURmemFrgAttrGet(MlirContext context, - MlirType valueType, uint32_t operand, - bool deriveElemType) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valueTypeValue; - if (!unwrapAs(ctx, valueType, false, valueTypeValue)) - return {nullptr}; - auto operandValue = - ::mlir::cutlass_compiler::cute::symbolizeMmaOperand(operand); - if (!operandValue) - return {nullptr}; - return wrap(cute_nvgpu::RmemFrgAttr::get(ctx, valueTypeValue, *operandValue, - deriveElemType)); -} -MlirType mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getValueType())); -} -uint32_t mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getOperand()); -} -bool mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getDeriveElemType(); -} - -bool mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet(MlirContext context, - uint32_t majorMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto majorModeValue = - ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); - if (!majorModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM100CircularSmemFrgAttr::get(ctx, *majorModeValue)); -} -uint32_t mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getMajorMode()); -} - -bool mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, - uint32_t majorMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto majorModeValue = - ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); - if (!majorModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM100SmemFrgAttr::get(ctx, *majorModeValue)); -} -uint32_t mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getMajorMode()); -} - -bool mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet(MlirContext context, - MlirType aType, - MlirType eType) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type eTypeValue; - if (!unwrapAs(ctx, eType, false, eTypeValue)) - return {nullptr}; - return wrap(cute_nvgpu::SM100TmemEFrgAttr::get(ctx, aTypeValue, eTypeValue)); -} -MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getAType())); -} -MlirType mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getEType())); -} - -bool mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet(MlirContext context, - MlirType dataType, - MlirType storageType, - int ctaGroup, - uint32_t tmemAllocMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type dataTypeValue; - if (!unwrapAs(ctx, dataType, false, dataTypeValue)) - return {nullptr}; - mlir::Type storageTypeValue; - if (!unwrapAs(ctx, storageType, false, storageTypeValue)) - return {nullptr}; - auto tmemAllocModeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( - tmemAllocMode); - if (!tmemAllocModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM100TmemFrgAttr::get( - ctx, dataTypeValue, storageTypeValue, ctaGroup, *tmemAllocModeValue)); -} -MlirType mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getDataType())); -} -MlirType mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getStorageType())); -} -int mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getCtaGroup(); -} -uint32_t mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getTmemAllocMode()); -} - -bool mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet(MlirContext context, - MlirType sfType, int sfVecSize, - int ctaGroup, bool isSfa, - uint32_t tmemAllocMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type sfTypeValue; - if (!unwrapAs(ctx, sfType, false, sfTypeValue)) - return {nullptr}; - auto tmemAllocModeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode( - tmemAllocMode); - if (!tmemAllocModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM100TmemSfFrgAttr::get( - ctx, sfTypeValue, sfVecSize, ctaGroup, isSfa, *tmemAllocModeValue)); -} -MlirType mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getSfType())); -} -int mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr) { - return llvm::cast(unwrap(attr)) - .getSfVecSize(); -} -int mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getCtaGroup(); -} -bool mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getIsSfa(); -} -uint32_t mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getTmemAllocMode()); -} - -bool mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, - uint32_t majorMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto majorModeValue = - ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); - if (!majorModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM107SmemFrgAttr::get(ctx, *majorModeValue)); -} -uint32_t mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getMajorMode()); -} - -bool mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, - uint32_t majorMode) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto majorModeValue = - ::mlir::cutlass_compiler::cute::symbolizeMajorMode(majorMode); - if (!majorModeValue) - return {nullptr}; - return wrap(cute_nvgpu::SM90SmemFrgAttr::get(ctx, *majorModeValue)); -} -uint32_t mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getMajorMode()); -} - -bool mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTiledTmaLoad(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TiledTmaLoadAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TmaDataFormatAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaLoadMode(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TmaLoadModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TmaStoreModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemAllocMode(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TmemAllocModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(value); - if (!valueValue) - return {nullptr}; - return wrap(cute_nvgpu::TmemLoadRedOpAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -// Types - -bool mlirTypeIsACuteNVGPUType(MlirType type) { - mlir::Type cppType = unwrap(type); - if (!cppType) - return false; - if (auto opaque = mlir::dyn_cast(cppType)) - return opaque.getDialectNamespace() == "cute_nvgpu"; - return cppType.getDialect().getNamespace() == "cute_nvgpu"; -} - -bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(MlirContext context, - MlirType valType, int copyBits, - bool mcast) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomBulkCopyG2SType::get(ctx, valTypeValue, - copyBits, mcast)); -} -MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -bool mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(MlirType type) { - return llvm::cast(unwrap(type)) - .getMcast(); -} - -bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(MlirContext context, - MlirType valType, int copyBits, - bool mask) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomBulkCopyS2GType::get(ctx, valTypeValue, - copyBits, mask)); -} -MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -bool mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(MlirType type) { - return llvm::cast(unwrap(type)) - .getMask(); -} - -bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(MlirContext context, - MlirType valType, - int copyBits) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - return wrap( - cute_nvgpu::CopyAtomBulkCopyS2SType::get(ctx, valTypeValue, copyBits)); -} -MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} - -bool mlirTypeIsACuteNVGPUCopyAtomDsmemStore(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(MlirContext context, - MlirType valType, - int copyBits) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomDsmemStoreType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, copyBits)); -} -MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} - -bool mlirTypeIsACuteNVGPUCopyAtomG2R(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomG2RTypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, - uint32_t memScope, uint32_t l2PrefetchSize, uint32_t l1CacheEvictPriority, - uint32_t loadCacheMode, uint32_t sharedSpace, bool invariant) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto memOrderValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); - if (!memOrderValue) - return {nullptr}; - auto memScopeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); - if (!memScopeValue) - return {nullptr}; - auto l2PrefetchSizeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeL2PrefetchSize( - l2PrefetchSize); - if (!l2PrefetchSizeValue) - return {nullptr}; - auto l1CacheEvictPriorityValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCacheEvictionPriority( - l1CacheEvictPriority); - if (!l1CacheEvictPriorityValue) - return {nullptr}; - auto loadCacheModeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode( - loadCacheMode); - if (!loadCacheModeValue) - return {nullptr}; - auto sharedSpaceValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); - if (!sharedSpaceValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomG2RType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, copyBits, *memOrderValue, *memScopeValue, - *l2PrefetchSizeValue, *l1CacheEvictPriorityValue, *loadCacheModeValue, - *sharedSpaceValue, invariant)); -} -MlirType mlirCuteNVGPUCopyAtomG2RTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getValType())); -} -int mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)).getCopyBits(); -} -uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getMemOrder()); -} -uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getMemScope()); -} -uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getL2PrefetchSize()); -} -uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getL1CacheEvictPriority()); -} -uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getLoadCacheMode()); -} -uint32_t mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getSharedSpace()); -} -bool mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(MlirType type) { - return llvm::cast(unwrap(type)).getInvariant(); -} - -bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(MlirContext context, - MlirType valType, - int copyBits, int numCta, - MlirType gStride, bool mcast, - MlirType tmaGbasis) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::StrideType gStrideValue; - if (!unwrapAs(ctx, gStride, false, gStrideValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomIm2ColTmaLoadType::get( - ctx, valTypeValue, copyBits, numCta, gStrideValue, mcast, - tmaGbasisValue)); -} -MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -int mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumCta(); -} -MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getGStride())); -} -bool mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(MlirType type) { - return llvm::cast(unwrap(type)) - .getMcast(); -} -MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getTmaGbasis())); -} - -bool mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(MlirContext context, - MlirType valType, - int copyBits, - MlirType gStride, - MlirType tmaGbasis) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::StrideType gStrideValue; - if (!unwrapAs(ctx, gStride, false, gStrideValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomIm2ColTmaStoreType::get( - ctx, valTypeValue, copyBits, gStrideValue, tmaGbasisValue)); -} -MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getGStride())); -} -MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getTmaGbasis())); -} - -bool mlirTypeIsACuteNVGPUCopyAtomLdsm(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomLdsmTypeGet(MlirContext context, MlirType valType, - MlirAttribute mode, - uint32_t szPattern, int numMatrices, - bool transpose) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr modeValue; - if (!unwrapAs(ctx, mode, false, modeValue)) - return {nullptr}; - auto szPatternValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLdsmSzPattern(szPattern); - if (!szPatternValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomLdsmType::get( - ctx, valTypeValue, modeValue, *szPatternValue, numMatrices, - transpose ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); -} -MlirType mlirCuteNVGPUCopyAtomLdsmTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getValType())); -} -MlirAttribute mlirCuteNVGPUCopyAtomLdsmTypeGetMode(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getMode())); -} -uint32_t mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getSzPattern()); -} -int mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumMatrices(); -} -bool mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getTranspose()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet( - MlirContext context, uint32_t kind, MlirType valType, int copyBits, - MlirType tmaGbasis, int64_t tmaFormat) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto kindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeGatherScatterTmaLoad(kind); - if (!kindValue) - return {nullptr}; - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) - return {nullptr}; - std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> - tmaFormatValue; - if (tmaFormat >= 0) { - tmaFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( - static_cast(tmaFormat)); - if (!tmaFormatValue) - return {nullptr}; - } - return wrap(cute_nvgpu::CopyAtomNonExec2DGather4TmaLoadType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); -} -uint32_t -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getKind()); -} -MlirType -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(MlirType type) { - return llvm::cast( - unwrap(type)) - .getCopyBits(); -} -MlirType -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getTmaGbasis())); -} -int64_t -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(MlirType type) { - auto value = - llvm::cast(unwrap(type)) - .getTmaFormat(); - return value ? static_cast(*value) : -1; -} - -bool mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore(MlirType type) { - return llvm::isa_and_nonnull< - cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType>(unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet( - MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, - int64_t tmaFormat) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) - return {nullptr}; - std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> - tmaFormatValue; - if (tmaFormat >= 0) { - tmaFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( - static_cast(tmaFormat)); - if (!tmaFormatValue) - return {nullptr}; - } - return wrap(cute_nvgpu::CopyAtomNonExec2DScatter4TmaStoreType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); -} -MlirType -mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits( - MlirType type) { - return llvm::cast( - unwrap(type)) - .getCopyBits(); -} -MlirType -mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getTmaGbasis())); -} -int64_t -mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(MlirType type) { - auto value = llvm::cast( - unwrap(type)) - .getTmaFormat(); - return value ? static_cast(*value) : -1; -} - -bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet( - MlirContext context, uint32_t kind, MlirType valType, int copyBits, - MlirType tmaGbasis, int64_t tmaFormat) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto kindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeIm2ColTmaLoad(kind); - if (!kindValue) - return {nullptr}; - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) - return {nullptr}; - std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> - tmaFormatValue; - if (tmaFormat >= 0) { - tmaFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( - static_cast(tmaFormat)); - if (!tmaFormatValue) - return {nullptr}; - } - return wrap(cute_nvgpu::CopyAtomNonExecIm2ColTmaLoadType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); -} -uint32_t mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getKind()); -} -MlirType -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -MlirType -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getTmaGbasis())); -} -int64_t -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(MlirType type) { - auto value = - llvm::cast(unwrap(type)) - .getTmaFormat(); - return value ? static_cast(*value) : -1; -} - -bool mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(MlirContext context, - MlirType valType, - int copyBits, - MlirType tmaGbasis, - int64_t tmaFormat) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) - return {nullptr}; - std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> - tmaFormatValue; - if (tmaFormat >= 0) { - tmaFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( - static_cast(tmaFormat)); - if (!tmaFormatValue) - return {nullptr}; - } - return wrap(cute_nvgpu::CopyAtomNonExecIm2ColTmaStoreType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); -} -MlirType -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -MlirType -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getTmaGbasis())); -} -int64_t -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(MlirType type) { - auto value = - llvm::cast(unwrap(type)) - .getTmaFormat(); - return value ? static_cast(*value) : -1; -} - -bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet( - MlirContext context, uint32_t kind, MlirType valType, int copyBits, - MlirType tmaGbasis, int64_t tmaFormat) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto kindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTiledTmaLoad(kind); - if (!kindValue) - return {nullptr}; - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) - return {nullptr}; - std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> - tmaFormatValue; - if (tmaFormat >= 0) { - tmaFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( - static_cast(tmaFormat)); - if (!tmaFormatValue) - return {nullptr}; - } - return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaLoadType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); -} -uint32_t mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getKind()); -} -MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getTmaGbasis())); -} -int64_t -mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(MlirType type) { - auto value = - llvm::cast(unwrap(type)) - .getTmaFormat(); - return value ? static_cast(*value) : -1; -} - -bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet( - MlirContext context, uint32_t kind, MlirType valType, int copyBits, - MlirType tmaGbasis, int64_t tmaFormat) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto kindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(kind); - if (!kindValue) - return {nullptr}; - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) - return {nullptr}; - std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> - tmaFormatValue; - if (tmaFormat >= 0) { - tmaFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( - static_cast(tmaFormat)); - if (!tmaFormatValue) - return {nullptr}; - } - return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaReduceType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - *kindValue, valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); -} -uint32_t mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getKind()); -} -MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getTmaGbasis())); -} -int64_t -mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(MlirType type) { - auto value = - llvm::cast(unwrap(type)) - .getTmaFormat(); - return value ? static_cast(*value) : -1; -} - -bool mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(MlirContext context, - MlirType valType, - int copyBits, - MlirType tmaGbasis, - int64_t tmaFormat) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, false, tmaGbasisValue)) - return {nullptr}; - std::optional<::mlir::cutlass_compiler::cute_nvgpu::TmaDataFormat> - tmaFormatValue; - if (tmaFormat >= 0) { - tmaFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaDataFormat( - static_cast(tmaFormat)); - if (!tmaFormatValue) - return {nullptr}; - } - return wrap(cute_nvgpu::CopyAtomNonExecTiledTmaStoreType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, copyBits, tmaGbasisValue, tmaFormatValue)); -} -MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getTmaGbasis())); -} -int64_t -mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(MlirType type) { - auto value = - llvm::cast(unwrap(type)) - .getTmaFormat(); - return value ? static_cast(*value) : -1; -} - -bool mlirTypeIsACuteNVGPUCopyAtomR2G(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomR2GTypeGet(MlirContext context, MlirType valType, - int copyBits, uint32_t memOrder, - uint32_t memScope, - uint32_t l1CacheEvictPriority, - uint32_t storeCacheMode, - uint32_t sharedSpace) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto memOrderValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); - if (!memOrderValue) - return {nullptr}; - auto memScopeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); - if (!memScopeValue) - return {nullptr}; - auto l1CacheEvictPriorityValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCacheEvictionPriority( - l1CacheEvictPriority); - if (!l1CacheEvictPriorityValue) - return {nullptr}; - auto storeCacheModeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeStoreCacheMode( - storeCacheMode); - if (!storeCacheModeValue) - return {nullptr}; - auto sharedSpaceValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); - if (!sharedSpaceValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomR2GType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, copyBits, *memOrderValue, *memScopeValue, - *l1CacheEvictPriorityValue, *storeCacheModeValue, *sharedSpaceValue)); -} -MlirType mlirCuteNVGPUCopyAtomR2GTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getValType())); -} -int mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)).getCopyBits(); -} -uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getMemOrder()); -} -uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getMemScope()); -} -uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getL1CacheEvictPriority()); -} -uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getStoreCacheMode()); -} -uint32_t mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getSharedSpace()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomR2S(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomR2STypeGet(MlirContext context, MlirType valType, - int copyBits, uint32_t memOrder, - uint32_t memScope, - uint32_t sharedSpace) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto memOrderValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); - if (!memOrderValue) - return {nullptr}; - auto memScopeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); - if (!memScopeValue) - return {nullptr}; - auto sharedSpaceValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); - if (!sharedSpaceValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomR2SType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, copyBits, *memOrderValue, *memScopeValue, - *sharedSpaceValue)); -} -MlirType mlirCuteNVGPUCopyAtomR2STypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getValType())); -} -int mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)).getCopyBits(); -} -uint32_t mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getMemOrder()); -} -uint32_t mlirCuteNVGPUCopyAtomR2STypeGetMemScope(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getMemScope()); -} -uint32_t mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getSharedSpace()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomS2R(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomS2RTypeGet(MlirContext context, MlirType valType, - int copyBits, uint32_t memOrder, - uint32_t memScope, - uint32_t sharedSpace) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto memOrderValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); - if (!memOrderValue) - return {nullptr}; - auto memScopeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); - if (!memScopeValue) - return {nullptr}; - auto sharedSpaceValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSharedSpace(sharedSpace); - if (!sharedSpaceValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomS2RType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, copyBits, *memOrderValue, *memScopeValue, - *sharedSpaceValue)); -} -MlirType mlirCuteNVGPUCopyAtomS2RTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getValType())); -} -int mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)).getCopyBits(); -} -uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getMemOrder()); -} -uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getMemScope()); -} -uint32_t mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getSharedSpace()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(MlirContext context, - MlirType valType, - uint32_t cache, - int copyBits) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto cacheValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeLoadCacheMode(cache); - if (!cacheValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSIMTAsyncCopyType::get( - ctx, valTypeValue, *cacheValue, copyBits)); -} -MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -uint32_t mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getCache()); -} -int mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} - -bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t reduction, - MlirAttribute ldReduceAccPrecision, uint32_t memOrder, uint32_t memScope) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto reductionValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(reduction); - if (!reductionValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::LdReduceAccPrecisionKindAttr - ldReduceAccPrecisionValue; - if (!unwrapAs(ctx, ldReduceAccPrecision, true, ldReduceAccPrecisionValue)) - return {nullptr}; - auto memOrderValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); - if (!memOrderValue) - return {nullptr}; - auto memScopeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); - if (!memScopeValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSIMTMultimemLdReduceType::get( - ctx, valTypeValue, copyBits, *reductionValue, ldReduceAccPrecisionValue, - *memOrderValue, *memScopeValue)); -} -MlirType -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -uint32_t -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getReduction()); -} -MlirAttribute -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision( - MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getLdReduceAccPrecision())); -} -uint32_t -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getMemOrder()); -} -uint32_t -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getMemScope()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t reduction, - uint32_t memOrder, uint32_t memScope) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto reductionValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(reduction); - if (!reductionValue) - return {nullptr}; - auto memOrderValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); - if (!memOrderValue) - return {nullptr}; - auto memScopeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); - if (!memScopeValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSIMTMultimemRedType::get( - ctx, valTypeValue, copyBits, *reductionValue, *memOrderValue, - *memScopeValue)); -} -MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getReduction()); -} -uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getMemOrder()); -} -uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getMemScope()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(MlirContext context, - MlirType valType, - int copyBits, - uint32_t memOrder, - uint32_t memScope) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto memOrderValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemOrderKind(memOrder); - if (!memOrderValue) - return {nullptr}; - auto memScopeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMemScopeKind(memScope); - if (!memScopeValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSIMTMultimemStType::get( - ctx, valTypeValue, copyBits, *memOrderValue, *memScopeValue)); -} -MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getMemOrder()); -} -uint32_t mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getMemScope()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(MlirContext context, - MlirType valType, - int copyBits, - uint32_t srcSpace, - uint32_t dstSpace) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto srcSpaceValue = - ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(srcSpace); - if (!srcSpaceValue) - return {nullptr}; - auto dstSpaceValue = - ::mlir::cutlass_compiler::cute::symbolizeAddressSpace(dstSpace); - if (!dstSpaceValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSIMTSyncCopyType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, copyBits, *srcSpaceValue, *dstSpaceValue)); -} -MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -uint32_t mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getSrcSpace()); -} -uint32_t mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getDstSpace()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(MlirContext context, - MlirType valType, int numDp, - int numBit, int numCta, - uint32_t broadcast) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto broadcastValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast( - broadcast); - if (!broadcastValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSM100CopyS2TType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, numDp, numBit, numCta, *broadcastValue)); -} -MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumDp(); -} -int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumBit(); -} -int mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumCta(); -} -uint32_t mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getBroadcast()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(MlirContext context, - MlirType valType, int numDp, - int numBit, int numCta, - uint32_t smemMajor, - uint32_t broadcast) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto smemMajorValue = - ::mlir::cutlass_compiler::cute::symbolizeMajorMode(smemMajor); - if (!smemMajorValue) - return {nullptr}; - auto broadcastValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeCopyS2TBroadcast( - broadcast); - if (!broadcastValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSM100S2TCopyV2Type::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, numDp, numBit, numCta, *smemMajorValue, *broadcastValue)); -} -MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumDp(); -} -int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumBit(); -} -int mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumCta(); -} -uint32_t mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getSmemMajor()); -} -uint32_t mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getBroadcast()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(MlirContext context, - MlirType valType, int numDp, - int numBit, uint32_t numRep, - bool pack16b) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSM100TmemLoadType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, numDp, numBit, numRep, - pack16b ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); -} -MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumDp(); -} -int mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumBit(); -} -uint32_t mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumRep(); -} -bool mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getPack16b()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(MlirContext context, - MlirType valType, int numDp, - int numBit, uint32_t numRep, - bool expand16b) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSM100TmemStoreType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, numDp, numBit, numRep, - expand16b ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); -} -MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumDp(); -} -int mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumBit(); -} -uint32_t mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumRep(); -} -bool mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getExpand16b()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet( - MlirContext context, MlirType valType, int numDp, int numBit, - uint32_t numRep, uint32_t redOp, bool red, bool nan) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto redOpValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(redOp); - if (!redOpValue) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSM107TmemLoadSPCompressType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, numDp, numBit, numRep, *redOpValue, - red ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), - nan ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); -} -MlirType -mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(MlirType type) { - return llvm::cast( - unwrap(type)) - .getNumDp(); -} -int mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(MlirType type) { - return llvm::cast( - unwrap(type)) - .getNumBit(); -} -uint32_t -mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(MlirType type) { - return llvm::cast( - unwrap(type)) - .getNumRep(); -} -uint32_t -mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getRedOp()); -} -bool mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getRed()); -} -bool mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getNan()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( - MlirContext context, MlirType valType, int numDp, int numBit, - uint32_t numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto redOpValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmemLoadRedOp(redOp); - if (!redOpValue) - return {nullptr}; - ::mlir::IntegerAttr halfSplitOffValue; - if (!unwrapAs(ctx, halfSplitOff, true, halfSplitOffValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomSM10xTmemLoadRedType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, numDp, numBit, numRep, *redOpValue, - nan ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), halfSplitOffValue)); -} -MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumDp(); -} -int mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumBit(); -} -uint32_t mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumRep(); -} -uint32_t mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getRedOp()); -} -bool mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getNan()); -} -MlirAttribute -mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getHalfSplitOff())); -} - -bool mlirTypeIsACuteNVGPUCopyAtomStsm(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomStsmTypeGet(MlirContext context, MlirType valType, - MlirAttribute mode, int numMatrices, - bool transpose) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::ShapeAttr modeValue; - if (!unwrapAs(ctx, mode, false, modeValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomStsmType::get( - ctx, valTypeValue, modeValue, numMatrices, - transpose ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); -} -MlirType mlirCuteNVGPUCopyAtomStsmTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getValType())); -} -MlirAttribute mlirCuteNVGPUCopyAtomStsmTypeGetMode(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getMode())); -} -int mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumMatrices(); -} -bool mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getTranspose()); -} - -bool mlirTypeIsACuteNVGPUCopyAtomTmaLoad(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGet( - MlirContext context, MlirType valType, int sparsity, int copyBits, - uint32_t mode, int numCta, MlirType gStride, bool mcast, MlirType tmaGbasis, - bool override, bool noFullyOobTile) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto modeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaLoadMode(mode); - if (!modeValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute::StrideType gStrideValue; - if (!unwrapAs(ctx, gStride, false, gStrideValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomTmaLoadType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, sparsity, copyBits, *modeValue, numCta, gStrideValue, mcast, - tmaGbasisValue, override, noFullyOobTile)); -} -MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getValType())); -} -int mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(MlirType type) { - return llvm::cast(unwrap(type)) - .getSparsity(); -} -int mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -uint32_t mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getMode()); -} -int mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(MlirType type) { - return llvm::cast(unwrap(type)).getNumCta(); -} -MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getGStride())); -} -bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(MlirType type) { - return llvm::cast(unwrap(type)).getMcast(); -} -MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getTmaGbasis())); -} -bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(MlirType type) { - return llvm::cast(unwrap(type)) - .getOverride(); -} -bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(MlirType type) { - return llvm::cast(unwrap(type)) - .getNoFullyOobTile(); -} - -bool mlirTypeIsACuteNVGPUCopyAtomTmaReduce(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGet(MlirContext context, - MlirType valType, int copyBits, - uint32_t mode, uint32_t kind, - MlirType gStride, - MlirType tmaGbasis) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto modeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(mode); - if (!modeValue) - return {nullptr}; - auto kindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeReductionKind(kind); - if (!kindValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute::StrideType gStrideValue; - if (!unwrapAs(ctx, gStride, false, gStrideValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomTmaReduceType::get( - ctx, valTypeValue, copyBits, *modeValue, *kindValue, gStrideValue, - tmaGbasisValue)); -} -MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getValType())); -} -int mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -uint32_t mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getMode()); -} -uint32_t mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getKind()); -} -MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getGStride())); -} -MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getTmaGbasis())); -} - -bool mlirTypeIsACuteNVGPUCopyAtomTmaStore(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGet(MlirContext context, - MlirType valType, int sparsity, - int copyBits, uint32_t mode, - MlirType gStride, - MlirType tmaGbasis, bool override, - bool noFullyOobTile) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type valTypeValue; - if (!unwrapAs(ctx, valType, false, valTypeValue)) - return {nullptr}; - auto modeValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeTmaStoreMode(mode); - if (!modeValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute::StrideType gStrideValue; - if (!unwrapAs(ctx, gStride, false, gStrideValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutType tmaGbasisValue; - if (!unwrapAs(ctx, tmaGbasis, true, tmaGbasisValue)) - return {nullptr}; - return wrap(cute_nvgpu::CopyAtomTmaStoreType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - valTypeValue, sparsity, copyBits, *modeValue, gStrideValue, - tmaGbasisValue, override, noFullyOobTile)); -} -MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getValType())); -} -int mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(MlirType type) { - return llvm::cast(unwrap(type)) - .getSparsity(); -} -int mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(MlirType type) { - return llvm::cast(unwrap(type)) - .getCopyBits(); -} -uint32_t mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getMode()); -} -MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getGStride())); -} -MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getTmaGbasis())); -} -bool mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(MlirType type) { - return llvm::cast(unwrap(type)) - .getOverride(); -} -bool mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(MlirType type) { - return llvm::cast(unwrap(type)) - .getNoFullyOobTile(); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse(MlirType type) { - return llvm::isa_and_nonnull< - cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType>(unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, - int vecSize, uint32_t archPromote, MlirAttribute intOverflow) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); - if (!aMajorValue) - return {nullptr}; - auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); - if (!bMajorValue) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - mlir::Type sfTypeValue; - if (!unwrapAs(ctx, sfType, false, sfTypeValue)) - return {nullptr}; - auto sparseMetadataFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( - sparseMetadataFormat); - if (!sparseMetadataFormatValue) - return {nullptr}; - auto aFragKindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); - if (!aFragKindValue) - return {nullptr}; - auto archPromoteValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeArch(archPromote); - if (!archPromoteValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; - if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM100UMMABlockScaledSparseType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, - cTypeValue, sfTypeValue, *sparseMetadataFormatValue, *aFragKindValue, - vecSize, *archPromoteValue, intOverflowValue)); -} -MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getShapeMnk())); -} -int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(MlirType type) { - return llvm::cast( - unwrap(type)) - .getNumCta(); -} -uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(MlirType type) { - return static_cast( - llvm::cast( - unwrap(type)) - .getAMajor()); -} -uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(MlirType type) { - return static_cast( - llvm::cast( - unwrap(type)) - .getBMajor()); -} -MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getAType())); -} -MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getBType())); -} -MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getCType())); -} -MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getSfType())); -} -uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat( - MlirType type) { - return static_cast( - llvm::cast( - unwrap(type)) - .getSparseMetadataFormat()); -} -uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(MlirType type) { - return static_cast( - llvm::cast( - unwrap(type)) - .getAFragKind()); -} -int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize( - MlirType type) { - return llvm::cast( - unwrap(type)) - .getVecSize(); -} -uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote( - MlirType type) { - return static_cast( - llvm::cast( - unwrap(type)) - .getArchPromote()); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow( - MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getIntOverflow())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t archPromote, - MlirAttribute intOverflow) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); - if (!aMajorValue) - return {nullptr}; - auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); - if (!bMajorValue) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - mlir::Type sfTypeValue; - if (!unwrapAs(ctx, sfType, false, sfTypeValue)) - return {nullptr}; - auto aFragKindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); - if (!aFragKindValue) - return {nullptr}; - auto archPromoteValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeArch(archPromote); - if (!archPromoteValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; - if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM100UMMABlockScaledType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, - cTypeValue, sfTypeValue, *aFragKindValue, vecSize, *archPromoteValue, - intOverflowValue)); -} -MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getShapeMnk())); -} -int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumCta(); -} -uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getAMajor()); -} -uint32_t mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getBMajor()); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getAType())); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getBType())); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getCType())); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getSfType())); -} -uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getAFragKind()); -} -int mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(MlirType type) { - return llvm::cast(unwrap(type)) - .getVecSize(); -} -uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getArchPromote()); -} -MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getIntOverflow())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, - int cScaleExp, MlirAttribute intOverflow) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); - if (!aMajorValue) - return {nullptr}; - auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); - if (!bMajorValue) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - mlir::Type eTypeValue; - if (!unwrapAs(ctx, eType, false, eTypeValue)) - return {nullptr}; - auto sparseMetadataFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( - sparseMetadataFormat); - if (!sparseMetadataFormatValue) - return {nullptr}; - auto aFragKindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); - if (!aFragKindValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; - if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM100UMMASparseType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, - cTypeValue, eTypeValue, *sparseMetadataFormatValue, *aFragKindValue, - cScaleExp, intOverflowValue)); -} -MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getShapeMnk())); -} -int mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumCta(); -} -uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getAMajor()); -} -uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getBMajor()); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getAType())); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getBType())); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getCType())); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getEType())); -} -uint32_t -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getSparseMetadataFormat()); -} -uint32_t mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getAFragKind()); -} -int mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(MlirType type) { - return llvm::cast(unwrap(type)) - .getCScaleExp(); -} -MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getIntOverflow())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMA(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - uint32_t aFragKind, int cScaleExp, MlirAttribute intOverflow) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); - if (!aMajorValue) - return {nullptr}; - auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); - if (!bMajorValue) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - auto aFragKindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); - if (!aFragKindValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; - if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM100UMMAType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, - cTypeValue, *aFragKindValue, cScaleExp, intOverflowValue)); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getShapeMnk())); -} -int mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(MlirType type) { - return llvm::cast(unwrap(type)).getNumCta(); -} -uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getAMajor()); -} -uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getBMajor()); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAType())); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getBType())); -} -MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getCType())); -} -uint32_t mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getAFragKind()); -} -int mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(MlirType type) { - return llvm::cast(unwrap(type)) - .getCScaleExp(); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getIntOverflow())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse(MlirType type) { - return llvm::isa_and_nonnull< - cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType>(unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, - int vecSize, uint32_t aCollectorOp, uint32_t bCollectorOp, - MlirAttribute intOverflow) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); - if (!aMajorValue) - return {nullptr}; - auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); - if (!bMajorValue) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - mlir::Type sfTypeValue; - if (!unwrapAs(ctx, sfType, false, sfTypeValue)) - return {nullptr}; - auto sparseMetadataFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( - sparseMetadataFormat); - if (!sparseMetadataFormatValue) - return {nullptr}; - auto aFragKindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); - if (!aFragKindValue) - return {nullptr}; - auto aCollectorOpValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( - aCollectorOp); - if (!aCollectorOpValue) - return {nullptr}; - auto bCollectorOpValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( - bCollectorOp); - if (!bCollectorOpValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; - if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM107UMMABlockScaledSparseType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, - cTypeValue, sfTypeValue, *sparseMetadataFormatValue, *aFragKindValue, - vecSize, *aCollectorOpValue, *bCollectorOpValue, intOverflowValue)); -} -MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getShapeMnk())); -} -int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(MlirType type) { - return llvm::cast( - unwrap(type)) - .getNumCta(); -} -uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(MlirType type) { - return static_cast( - llvm::cast( - unwrap(type)) - .getAMajor()); -} -uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(MlirType type) { - return static_cast( - llvm::cast( - unwrap(type)) - .getBMajor()); -} -MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getAType())); -} -MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getBType())); -} -MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getCType())); -} -MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getSfType())); -} -uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat( - MlirType type) { - return static_cast( - llvm::cast( - unwrap(type)) - .getSparseMetadataFormat()); -} -uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(MlirType type) { - return static_cast( - llvm::cast( - unwrap(type)) - .getAFragKind()); -} -int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize( - MlirType type) { - return llvm::cast( - unwrap(type)) - .getVecSize(); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp( - MlirType type) { - return static_cast( - llvm::cast( - unwrap(type)) - .getACollectorOp()); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp( - MlirType type) { - return static_cast( - llvm::cast( - unwrap(type)) - .getBCollectorOp()); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow( - MlirType type) { - return wrap(static_cast( - llvm::cast( - unwrap(type)) - .getIntOverflow())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t aCollectorOp, - uint32_t bCollectorOp, MlirAttribute intOverflow) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); - if (!aMajorValue) - return {nullptr}; - auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); - if (!bMajorValue) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - mlir::Type sfTypeValue; - if (!unwrapAs(ctx, sfType, false, sfTypeValue)) - return {nullptr}; - auto aFragKindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); - if (!aFragKindValue) - return {nullptr}; - auto aCollectorOpValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( - aCollectorOp); - if (!aCollectorOpValue) - return {nullptr}; - auto bCollectorOpValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( - bCollectorOp); - if (!bCollectorOpValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; - if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM107UMMABlockScaledType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, - cTypeValue, sfTypeValue, *aFragKindValue, vecSize, *aCollectorOpValue, - *bCollectorOpValue, intOverflowValue)); -} -MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getShapeMnk())); -} -int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumCta(); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getAMajor()); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getBMajor()); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getAType())); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getBType())); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getCType())); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getSfType())); -} -uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getAFragKind()); -} -int mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(MlirType type) { - return llvm::cast(unwrap(type)) - .getVecSize(); -} -uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getACollectorOp()); -} -uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getBCollectorOp()); -} -MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getIntOverflow())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, - int cScaleExp, uint32_t aCollectorOp, uint32_t bCollectorOp, - MlirAttribute intOverflow) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); - if (!aMajorValue) - return {nullptr}; - auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); - if (!bMajorValue) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - mlir::Type eTypeValue; - if (!unwrapAs(ctx, eType, false, eTypeValue)) - return {nullptr}; - auto sparseMetadataFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( - sparseMetadataFormat); - if (!sparseMetadataFormatValue) - return {nullptr}; - auto aFragKindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); - if (!aFragKindValue) - return {nullptr}; - auto aCollectorOpValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( - aCollectorOp); - if (!aCollectorOpValue) - return {nullptr}; - auto bCollectorOpValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( - bCollectorOp); - if (!bCollectorOpValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; - if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM107UMMASparseType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, - cTypeValue, eTypeValue, *sparseMetadataFormatValue, *aFragKindValue, - cScaleExp, *aCollectorOpValue, *bCollectorOpValue, intOverflowValue)); -} -MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getShapeMnk())); -} -int mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(MlirType type) { - return llvm::cast(unwrap(type)) - .getNumCta(); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getAMajor()); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getBMajor()); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getAType())); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getBType())); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getCType())); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getEType())); -} -uint32_t -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getSparseMetadataFormat()); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getAFragKind()); -} -int mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(MlirType type) { - return llvm::cast(unwrap(type)) - .getCScaleExp(); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getACollectorOp()); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getBCollectorOp()); -} -MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getIntOverflow())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMA(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - uint32_t aFragKind, int cScaleExp, uint32_t aCollectorOp, - uint32_t bCollectorOp, MlirAttribute intOverflow) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); - if (!aMajorValue) - return {nullptr}; - auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); - if (!bMajorValue) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - auto aFragKindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); - if (!aFragKindValue) - return {nullptr}; - auto aCollectorOpValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( - aCollectorOp); - if (!aCollectorOpValue) - return {nullptr}; - auto bCollectorOpValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaCollectorOp( - bCollectorOp); - if (!bCollectorOpValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; - if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM107UMMAType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, numCta, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, - cTypeValue, *aFragKindValue, cScaleExp, *aCollectorOpValue, - *bCollectorOpValue, intOverflowValue)); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getShapeMnk())); -} -int mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(MlirType type) { - return llvm::cast(unwrap(type)).getNumCta(); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getAMajor()); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getBMajor()); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAType())); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getBType())); -} -MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getCType())); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getAFragKind()); -} -int mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(MlirType type) { - return llvm::cast(unwrap(type)) - .getCScaleExp(); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getACollectorOp()); -} -uint32_t mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getBCollectorOp()); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getIntOverflow())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet( - MlirContext context, MlirAttribute shapeMnk, int vecSize, MlirType aType, - MlirType bType, MlirType cType, MlirType sfType, bool useSfLayoutTV) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - mlir::Type sfTypeValue; - if (!unwrapAs(ctx, sfType, false, sfTypeValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM120BlockScaledType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, vecSize, aTypeValue, bTypeValue, cTypeValue, sfTypeValue, - useSfLayoutTV)); -} -MlirAttribute -mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getShapeMnk())); -} -int mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(MlirType type) { - return llvm::cast(unwrap(type)) - .getVecSize(); -} -MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getAType())); -} -MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getBType())); -} -MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getCType())); -} -MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getSfType())); -} -bool mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(MlirType type) { - return llvm::cast(unwrap(type)) - .getUseSfLayout_TV(); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet( - MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, - MlirType cType, uint32_t sparseMetadataFormat, MlirAttribute intOverflow) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - auto sparseMetadataFormatValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeSparseMetadataFormat( - sparseMetadataFormat); - if (!sparseMetadataFormatValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; - if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM80SparseType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, aTypeValue, bTypeValue, cTypeValue, - *sparseMetadataFormatValue, intOverflowValue)); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getShapeMnk())); -} -MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAType())); -} -MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getBType())); -} -MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getCType())); -} -uint32_t -mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)) - .getSparseMetadataFormat()); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getIntOverflow())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM80TypeGet(MlirContext context, - MlirAttribute shapeMnk, MlirType aType, - MlirType bType, MlirType cType, - MlirAttribute intOverflow, - MlirAttribute binaryOp) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; - if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::BinaryOpAttr binaryOpValue; - if (!unwrapAs(ctx, binaryOp, true, binaryOpValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM80Type::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, aTypeValue, bTypeValue, cTypeValue, intOverflowValue, - binaryOpValue)); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getShapeMnk())); -} -MlirType mlirCuteNVGPUMmaAtomSM80TypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAType())); -} -MlirType mlirCuteNVGPUMmaAtomSM80TypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getBType())); -} -MlirType mlirCuteNVGPUMmaAtomSM80TypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getCType())); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getIntOverflow())); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getBinaryOp())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, - MlirAttribute shapeMnk, MlirType aType, - MlirType bType, MlirType cType) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM89Type::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, aTypeValue, bTypeValue, cTypeValue)); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getShapeMnk())); -} -MlirType mlirCuteNVGPUMmaAtomSM89TypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAType())); -} -MlirType mlirCuteNVGPUMmaAtomSM89TypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getBType())); -} -MlirType mlirCuteNVGPUMmaAtomSM89TypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getCType())); -} - -bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUMmaAtomSM90TypeGet( - MlirContext context, MlirAttribute shapeMnk, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - uint32_t aFragKind, MlirAttribute intOverflow, bool aNeg, bool bNeg) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - auto aMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(aMajor); - if (!aMajorValue) - return {nullptr}; - auto bMajorValue = ::mlir::cutlass_compiler::cute::symbolizeMajorMode(bMajor); - if (!bMajorValue) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - auto aFragKindValue = - ::mlir::cutlass_compiler::cute_nvgpu::symbolizeMmaFragKind(aFragKind); - if (!aFragKindValue) - return {nullptr}; - ::mlir::cutlass_compiler::cute_nvgpu::MMAIntOverflowAttr intOverflowValue; - if (!unwrapAs(ctx, intOverflow, true, intOverflowValue)) - return {nullptr}; - return wrap(cute_nvgpu::MmaAtomSM90Type::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, *aMajorValue, *bMajorValue, aTypeValue, bTypeValue, - cTypeValue, *aFragKindValue, intOverflowValue, - aNeg ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr(), - bNeg ? mlir::UnitAttr::get(ctx) : mlir::UnitAttr())); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getShapeMnk())); -} -uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getAMajor()); -} -uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getBMajor()); -} -MlirType mlirCuteNVGPUMmaAtomSM90TypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAType())); -} -MlirType mlirCuteNVGPUMmaAtomSM90TypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getBType())); -} -MlirType mlirCuteNVGPUMmaAtomSM90TypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getCType())); -} -uint32_t mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getAFragKind()); -} -MlirAttribute mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getIntOverflow())); -} -bool mlirCuteNVGPUMmaAtomSM90TypeGetANeg(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getANeg()); -} -bool mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(MlirType type) { - return static_cast( - llvm::cast(unwrap(type)).getBNeg()); -} - -bool mlirTypeIsACuteNVGPUSmemDescCircularSM103(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType -mlirCuteNVGPUSmemDescCircularSM103TypeGet(MlirContext context, - MlirAttribute blockLayoutBytes) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::LayoutAttr blockLayoutBytesValue; - if (!unwrapAs(ctx, blockLayoutBytes, false, blockLayoutBytesValue)) - return {nullptr}; - return wrap(cute_nvgpu::SmemDescCircularSM103Type::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - blockLayoutBytesValue)); -} -MlirAttribute -mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getBlockLayoutBytes())); -} - -bool mlirTypeIsACuteNVGPUSmemDescCircularSM107(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType -mlirCuteNVGPUSmemDescCircularSM107TypeGet(MlirContext context, - MlirAttribute blockLayoutBytes) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::LayoutAttr blockLayoutBytesValue; - if (!unwrapAs(ctx, blockLayoutBytes, false, blockLayoutBytesValue)) - return {nullptr}; - return wrap(cute_nvgpu::SmemDescCircularSM107Type::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - blockLayoutBytesValue)); -} -MlirAttribute -mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getBlockLayoutBytes())); -} - -bool mlirTypeIsACuteNVGPUSmemDescSM107(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUSmemDescSM107TypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute_nvgpu::SmemDescSM107Type::get(ctx)); -} - -bool mlirTypeIsACuteNVGPUSmemDesc(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUSmemDescTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute_nvgpu::SmemDescType::get(ctx)); -} - -bool mlirTypeIsACuteNVGPUSmemDescView(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUSmemDescViewTypeGet(MlirContext context, MlirType desc, - MlirAttribute layout) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - mlir::Type descValue; - if (!unwrapAs(ctx, desc, false, descValue)) - return {nullptr}; - ::mlir::cutlass_compiler::cute::LayoutAttr layoutValue; - if (!unwrapAs(ctx, layout, false, layoutValue)) - return {nullptr}; - return wrap(cute_nvgpu::SmemDescViewType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - descValue, layoutValue)); -} -MlirType mlirCuteNVGPUSmemDescViewTypeGetDesc(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getDesc())); -} -MlirAttribute mlirCuteNVGPUSmemDescViewTypeGetLayout(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getLayout())); -} - -bool mlirTypeIsACuteNVGPUTmaDescriptorIm2Col(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute_nvgpu::TmaDescriptorIm2ColType::get(ctx)); -} - -bool mlirTypeIsACuteNVGPUTmaDescriptorTiled(MlirType type) { - return llvm::isa_and_nonnull( - unwrap(type)); -} -MlirType mlirCuteNVGPUTmaDescriptorTiledTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute_nvgpu::TmaDescriptorTiledType::get(ctx)); -} - -bool mlirTypeIsACuteNVGPUUniversalFmaAtom(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUUniversalFmaAtomTypeGet(MlirContext context, - MlirAttribute shapeMnk, - MlirType aType, MlirType bType, - MlirType cType) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::cutlass_compiler::cute::ShapeAttr shapeMnkValue; - if (!unwrapAs(ctx, shapeMnk, false, shapeMnkValue)) - return {nullptr}; - mlir::Type aTypeValue; - if (!unwrapAs(ctx, aType, false, aTypeValue)) - return {nullptr}; - mlir::Type bTypeValue; - if (!unwrapAs(ctx, bType, false, bTypeValue)) - return {nullptr}; - mlir::Type cTypeValue; - if (!unwrapAs(ctx, cType, false, cTypeValue)) - return {nullptr}; - return wrap(cute_nvgpu::UniversalFmaAtomType::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - shapeMnkValue, aTypeValue, bTypeValue, cTypeValue)); -} -MlirAttribute mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)) - .getShapeMnk())); -} -MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetAType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getAType())); -} -MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetBType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getBType())); -} -MlirType mlirCuteNVGPUUniversalFmaAtomTypeGetCType(MlirType type) { - return wrap(static_cast( - llvm::cast(unwrap(type)).getCType())); -} - -bool mlirTypeIsACuteNVGPUWorkIdResponse(MlirType type) { - return llvm::isa_and_nonnull(unwrap(type)); -} -MlirType mlirCuteNVGPUWorkIdResponseTypeGet(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(cute_nvgpu::WorkIdResponseType::get(ctx)); -} diff --git a/mlir/dialects/cute_nvgpu/capi.h b/mlir/dialects/cute_nvgpu/capi.h deleted file mode 100644 index f1e4e999..00000000 --- a/mlir/dialects/cute_nvgpu/capi.h +++ /dev/null @@ -1,1481 +0,0 @@ -//===- capi.h - C API for the cute_nvgpu dialect ------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_CUTE_NVGPU_CAPI_H -#define MLIR_DIALECTS_CUTE_NVGPU_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Registration - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVGPU, cute_nvgpu); - -// Attributes - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles, and std::optional enums a -// negative value. Enums are their integer values (the dialect's *Enums.td). -// Getters require a value accepted by the matching IsA. - -// `#cute_nvgpu.atom_copy_field_bulkg2s`: Fields stored in Bulk Load Copy Atom -// type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_bulks2g`: Fields stored in Bulk Store Copy Atom -// type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_bulks2s`: Fields stored in Bulk CTA to Cluster -// Copy Atom type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_dsmem_store`: Fields stored in -// CopyAtomDsmemStoreType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_g2r`: Fields stored in CopyAtomG2RType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_2d_gather4_tma_load`: Fields stored in -// CopyAtomNonExec2DGather4TmaLoadType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad( - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue( - MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_2d_scatter4_tma_store`: Fields stored -// in CopyAtomNonExec2DScatter4TmaStoreType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore( - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue( - MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_load`: Fields stored in -// CopyAtomNonExecIm2ColTmaLoadType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_store`: Fields stored in -// CopyAtomNonExecIm2ColTmaStoreType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_tma_load`: Fields stored in -// CopyAtomNonExecTiledTmaLoadType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_tma_reduce`: Fields stored in -// CopyAtomNonExecTiledTmaReduceType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_non_exec_tma_store`: Fields stored in -// CopyAtomNonExecTiledTmaStoreType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_r2g`: Fields stored in CopyAtomR2GType. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_tmaload`: Fields stored in Tma Load Copy Atom -// type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_tmareduce`: Fields stored in Tma Reduce Copy -// Atom type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_copy_field_tmastore`: Fields stored in Tma Store Copy Atom -// type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm100`: Fields stored in MMA Atom types for -// SM100. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM100(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM100AttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm100_block_scaled`: Fields stored in MMA Atom -// types for SM100 (block scaled). -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm100_block_scaled_sparse`: Fields stored in MMA -// Atom types for SM100 (block scaled). -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm100_sparse`: Fields stored in sparse MMA Atom -// types for SM100. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm120_block_scaled`: Fields stored in MMA Atom -// types for SM120 (block scaled). -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm80_sparse`: Fields stored in MMA Atom types for -// SM80. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.atom_mma_field_sm90`: Fields stored in MMA Atom types for SM90. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUAtomMmaFieldSM90(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUAtomMmaFieldSM90AttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.bin_op`: Binary operation for single-bit MMA operations. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUBinaryOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUBinaryOpAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUBinaryOpAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.copy_s2t_broadcast_mode`: Broadcast modes for the different -// utccp instructions. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUCopyS2TBroadcast(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyS2TBroadcastAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.gather_scatter_tma_load`: The various kinds of TMA loads in -// gather/scatter mode. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUGatherScatterTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUGatherScatterTmaLoadAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.im2col_tma_load`: The various kinds of TMA loads in im2col mode. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUIm2ColTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUIm2ColTmaLoadAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.ld_reduce_acc_precision_kind`: multimem ld_reduce accumulation -// precision kind. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPULdReduceAccPrecisionKindAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.ldsm_sz_pattern`: LDSM's sz pattern, describing the bit size. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPULdsmSzPattern(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPULdsmSzPatternAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPULdsmSzPatternAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.load_cache_mode`: Cache modes for the load instructions. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPULoadCacheMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPULoadCacheModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPULoadCacheModeAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.mma_int_overflow`: MMA overflow options. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUMMAIntOverflow(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMMAIntOverflowAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMMAIntOverflowAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.major`: Major mode for MMA operations. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPUMajorMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMajorModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMajorModeAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.mma_collector_op`: Enums for the mma collector op. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUMmaCollectorOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaCollectorOpAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaCollectorOpAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.mma_frag_kind`: Enums for the mma frag type. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUMmaFragKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaFragKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaFragKindAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.not_implemented_frg`: Fragment of an atom that has none. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUNotImplementedFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUNotImplementedFrgAttrGet(MlirContext context); - -// `#cute_nvgpu.tma_reduce_kind`: Op for the TMASTORE instruction. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUReductionKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUReductionKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUReductionKindAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.rmem_frg`: Register-memory fragment of an MMA operand. -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVGPURmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPURmemFrgAttrGet(MlirContext context, MlirType valueType, - uint32_t operand, bool deriveElemType); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPURmemFrgAttrGetValueType(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPURmemFrgAttrGetOperand(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPURmemFrgAttrGetDeriveElemType(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm100.circular_smem_frg`: SM100 circular shared-memory -// fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM100CircularSmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100CircularSmemFrgAttrGet( - MlirContext context, uint32_t majorMode); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm100.smem_frg`: SM100 shared-memory fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM100SmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUSM100SmemFrgAttrGet(MlirContext context, uint32_t majorMode); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm100.tmem_e_frg`: SM100 tensor-memory sparse-metadata -// fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM100TmemEFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemEFrgAttrGet( - MlirContext context, MlirType aType, MlirType eType); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSM100TmemEFrgAttrGetAType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSM100TmemEFrgAttrGetEType(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm100.tmem_frg`: SM100 tensor-memory fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM100TmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemFrgAttrGet( - MlirContext context, MlirType dataType, MlirType storageType, int ctaGroup, - uint32_t tmemAllocMode); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSM100TmemFrgAttrGetDataType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(MlirAttribute attr); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm100.tmem_sf_frg`: SM100 tensor-memory scale-factor -// fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM100TmemSfFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVGPUSM100TmemSfFrgAttrGet( - MlirContext context, MlirType sfType, int sfVecSize, int ctaGroup, - bool isSfa, uint32_t tmemAllocMode); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(MlirAttribute attr); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(MlirAttribute attr); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm107.smem_frg`: SM107 shared-memory fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM107SmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUSM107SmemFrgAttrGet(MlirContext context, uint32_t majorMode); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(MlirAttribute attr); - -// `#cute_nvgpu.arch.sm90.smem_frg`: SM90 shared-memory fragment. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUSM90SmemFrg(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUSM90SmemFrgAttrGet(MlirContext context, uint32_t majorMode); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(MlirAttribute attr); - -// `#cute_nvgpu.tiled_tma_load`: The various kinds of TMA loads in tiled mode. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTiledTmaLoad(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTiledTmaLoadAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTiledTmaLoadAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.tma_data_format`: Bits 74-71 of the TMA descriptor. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTmaDataFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTmaDataFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTmaDataFormatAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.tma_load_mode`: Modes for the TMA load instruction. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTmaLoadMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTmaLoadModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTmaLoadModeAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.tma_store_mode`: Modes for the TMA store instruction. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTmaStoreMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTmaStoreModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTmaStoreModeAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.tmem_alloc_mode`: Modes for the Tmem allocation. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTmemAllocMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTmemAllocModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTmemAllocModeAttrGetValue(MlirAttribute attr); - -// `#cute_nvgpu.tmem_load_red_op`: Reduce operation for the Tmem load -// instruction. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVGPUTmemLoadRedOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUTmemLoadRedOpAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUTmemLoadRedOpAttrGetValue(MlirAttribute attr); - -// Types - -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUType(MlirType type); - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles, and std::optional enums a -// negative value. Enums are their integer values (the dialect's *Enums.td). -// Getters require a value accepted by the matching IsA. - -// `!cute_nvgpu.atom.bulk_copy_g2s`: Bulk (TMA non-tensor) copy atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet( - MlirContext context, MlirType valType, int copyBits, bool mcast); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(MlirType type); - -// `!cute_nvgpu.atom.bulk_copy_s2g`: Bulk (TMA non-tensor) copy atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet( - MlirContext context, MlirType valType, int copyBits, bool mask); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(MlirType type); - -// `!cute_nvgpu.atom.bulk_copy_s2s`: Bulk (TMA non-tensor) copy atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet( - MlirContext context, MlirType valType, int copyBits); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(MlirType type); - -// `!cute_nvgpu.atom.dsmem_store`: Distributed shared-memory store atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomDsmemStore(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomDsmemStoreTypeGet( - MlirContext context, MlirType valType, int copyBits); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(MlirType type); - -// `!cute_nvgpu.atom.g2r`: Global-to-register load atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomG2R(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomG2RTypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, - uint32_t memScope, uint32_t l2PrefetchSize, uint32_t l1CacheEvictPriority, - uint32_t loadCacheMode, uint32_t sharedSpace, bool invariant); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomG2RTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(MlirType type); -MLIR_CAPI_EXPORTED bool mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(MlirType type); - -// `!cute_nvgpu.atom.im2col_tma_load`: Executable im2col TMA load atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet( - MlirContext context, MlirType valType, int copyBits, int numCta, - MlirType gStride, bool mcast, MlirType tmaGbasis); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(MlirType type); - -// `!cute_nvgpu.atom.im2col_tma_store`: Executable im2col TMA store atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet( - MlirContext context, MlirType valType, int copyBits, MlirType gStride, - MlirType tmaGbasis); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(MlirType type); - -// `!cute_nvgpu.atom.ldsm`: ldmatrix atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomLdsm(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomLdsmTypeGet( - MlirContext context, MlirType valType, MlirAttribute mode, - uint32_t szPattern, int numMatrices, bool transpose); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomLdsmTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomLdsmTypeGetMode(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(MlirType type); - -// `!cute_nvgpu.atom.non_exec_2d_gather4_tma_load`: Non-executable 2-D gather4 -// TMA load atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet( - MlirContext context, uint32_t kind, MlirType valType, int copyBits, - MlirType tmaGbasis, int64_t tmaFormat); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(MlirType type); -MLIR_CAPI_EXPORTED int64_t -mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(MlirType type); - -// `!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store`: Non-executable 2-D -// scatter4 TMA store atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(MlirContext context, - MlirType valType, - int copyBits, - MlirType tmaGbasis, - int64_t tmaFormat); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(MlirType type); -MLIR_CAPI_EXPORTED int64_t -mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(MlirType type); - -// `!cute_nvgpu.atom.non_exec_im2col_tma_load`: Non-executable im2col TMA load -// atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet( - MlirContext context, uint32_t kind, MlirType valType, int copyBits, - MlirType tmaGbasis, int64_t tmaFormat); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(MlirType type); -MLIR_CAPI_EXPORTED int64_t -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(MlirType type); - -// `!cute_nvgpu.atom.non_exec_im2col_tma_store`: Non-executable im2col TMA store -// atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet( - MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, - int64_t tmaFormat); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(MlirType type); -MLIR_CAPI_EXPORTED int64_t -mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(MlirType type); - -// `!cute_nvgpu.atom.non_exec_tiled_tma_load`: Non-executable tiled TMA load -// atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet( - MlirContext context, uint32_t kind, MlirType valType, int copyBits, - MlirType tmaGbasis, int64_t tmaFormat); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(MlirType type); -MLIR_CAPI_EXPORTED int64_t -mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(MlirType type); - -// `!cute_nvgpu.atom.non_exec_tiled_tma_reduce`: Non-executable tiled TMA reduce -// atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet( - MlirContext context, uint32_t kind, MlirType valType, int copyBits, - MlirType tmaGbasis, int64_t tmaFormat); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(MlirType type); -MLIR_CAPI_EXPORTED int64_t -mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(MlirType type); - -// `!cute_nvgpu.atom.non_exec_tiled_tma_store`: Non-executable tiled TMA store -// atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet( - MlirContext context, MlirType valType, int copyBits, MlirType tmaGbasis, - int64_t tmaFormat); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(MlirType type); -MLIR_CAPI_EXPORTED int64_t -mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(MlirType type); - -// `!cute_nvgpu.atom.r2g`: Register-to-global store atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomR2G(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomR2GTypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, - uint32_t memScope, uint32_t l1CacheEvictPriority, uint32_t storeCacheMode, - uint32_t sharedSpace); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomR2GTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(MlirType type); - -// `!cute_nvgpu.atom.r2s`: Register-to-shared store atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomR2S(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomR2STypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, - uint32_t memScope, uint32_t sharedSpace); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomR2STypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomR2STypeGetMemScope(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(MlirType type); - -// `!cute_nvgpu.atom.s2r`: Shared-to-register load atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomS2R(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomS2RTypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, - uint32_t memScope, uint32_t sharedSpace); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomS2RTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(MlirType type); - -// `!cute_nvgpu.atom.simt_async_copy`: cp.async copy atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet( - MlirContext context, MlirType valType, uint32_t cache, int copyBits); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(MlirType type); - -// `!cute_nvgpu.atom.simt_multimem_ld_reduce`: multimem.ld_reduce atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t reduction, - MlirAttribute ldReduceAccPrecision, uint32_t memOrder, uint32_t memScope); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision( - MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(MlirType type); - -// `!cute_nvgpu.atom.simt_multimem_red`: multimem.red atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t reduction, - uint32_t memOrder, uint32_t memScope); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(MlirType type); - -// `!cute_nvgpu.atom.simt_multimem_st`: multimem.st atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t memOrder, - uint32_t memScope); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(MlirType type); - -// `!cute_nvgpu.atom.universal_copy`: Universal (SIMT) copy atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t srcSpace, - uint32_t dstSpace); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(MlirType type); - -// `!cute_nvgpu.atom.s2t_copy`: tcgen05.cp (shared to tensor memory) atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet( - MlirContext context, MlirType valType, int numDp, int numBit, int numCta, - uint32_t broadcast); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(MlirType type); - -// `!cute_nvgpu.atom.sm100_s2t_copy_v2`: tcgen05.cp atom with the shared-memory -// major mode. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet( - MlirContext context, MlirType valType, int numDp, int numBit, int numCta, - uint32_t smemMajor, uint32_t broadcast); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(MlirType type); - -// `!cute_nvgpu.atom.tmem_load`: tcgen05.ld atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet( - MlirContext context, MlirType valType, int numDp, int numBit, - uint32_t numRep, bool pack16b); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(MlirType type); - -// `!cute_nvgpu.atom.tmem_store`: tcgen05.st atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet( - MlirContext context, MlirType valType, int numDp, int numBit, - uint32_t numRep, bool expand16b); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(MlirType type); - -// `!cute_nvgpu.atom.tmem_load_spcompress`: SM107 tensor-memory load with sparse -// compression. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet( - MlirContext context, MlirType valType, int numDp, int numBit, - uint32_t numRep, uint32_t redOp, bool red, bool nan); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(MlirType type); - -// `!cute_nvgpu.atom.tmem_load_red`: tcgen05.ld.red atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet( - MlirContext context, MlirType valType, int numDp, int numBit, - uint32_t numRep, uint32_t redOp, bool nan, MlirAttribute halfSplitOff); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(MlirType type); - -// `!cute_nvgpu.atom.stsm`: stmatrix atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomStsm(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomStsmTypeGet( - MlirContext context, MlirType valType, MlirAttribute mode, int numMatrices, - bool transpose); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomStsmTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUCopyAtomStsmTypeGetMode(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(MlirType type); - -// `!cute_nvgpu.atom.tma_load`: Executable TMA load atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaLoad(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaLoadTypeGet( - MlirContext context, MlirType valType, int sparsity, int copyBits, - uint32_t mode, int numCta, MlirType gStride, bool mcast, MlirType tmaGbasis, - bool override, bool noFullyOobTile); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(MlirType type); -MLIR_CAPI_EXPORTED int mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(MlirType type); -MLIR_CAPI_EXPORTED bool mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(MlirType type); - -// `!cute_nvgpu.atom.tma_reduce`: Executable TMA reduce atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaReduce(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaReduceTypeGet( - MlirContext context, MlirType valType, int copyBits, uint32_t mode, - uint32_t kind, MlirType gStride, MlirType tmaGbasis); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(MlirType type); - -// `!cute_nvgpu.atom.tma_store`: Executable TMA store atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUCopyAtomTmaStore(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUCopyAtomTmaStoreTypeGet( - MlirContext context, MlirType valType, int sparsity, int copyBits, - uint32_t mode, MlirType gStride, MlirType tmaGbasis, bool override, - bool noFullyOobTile); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(MlirType type); - -// `!cute_nvgpu.sm100.mma_bs_sp`: SM100 block-scaled sparse UMMA MMA atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, - int vecSize, uint32_t archPromote, MlirAttribute intOverflow); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat( - MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(MlirType type); - -// `!cute_nvgpu.sm100.mma_bs`: SM100 block-scaled UMMA MMA atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t archPromote, - MlirAttribute intOverflow); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(MlirType type); - -// `!cute_nvgpu.sm100.mma_sp`: SM100 sparse UMMA MMA atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, - int cScaleExp, MlirAttribute intOverflow); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(MlirType type); - -// `!cute_nvgpu.sm100.mma`: SM100 UMMA MMA atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM100UMMA(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM100UMMATypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - uint32_t aFragKind, int cScaleExp, MlirAttribute intOverflow); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(MlirType type); - -// `!cute_nvgpu.sm107.mma_bs_sp`: SM107 block-scaled sparse UMMA MMA atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType sfType, uint32_t sparseMetadataFormat, uint32_t aFragKind, - int vecSize, uint32_t aCollectorOp, uint32_t bCollectorOp, - MlirAttribute intOverflow); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat( - MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp( - MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp( - MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(MlirType type); - -// `!cute_nvgpu.sm107.mma_bs`: SM107 block-scaled UMMA MMA atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType sfType, uint32_t aFragKind, int vecSize, uint32_t aCollectorOp, - uint32_t bCollectorOp, MlirAttribute intOverflow); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(MlirType type); - -// `!cute_nvgpu.sm107.mma_sp`: SM107 sparse UMMA MMA atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - MlirType eType, uint32_t sparseMetadataFormat, uint32_t aFragKind, - int cScaleExp, uint32_t aCollectorOp, uint32_t bCollectorOp, - MlirAttribute intOverflow); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(MlirType type); - -// `!cute_nvgpu.sm107.mma`: SM107 UMMA MMA atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM107UMMA(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM107UMMATypeGet( - MlirContext context, MlirAttribute shapeMnk, int numCta, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - uint32_t aFragKind, int cScaleExp, uint32_t aCollectorOp, - uint32_t bCollectorOp, MlirAttribute intOverflow); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(MlirType type); - -// `!cute_nvgpu.SM120.mma_bs`: SM120 block-scaled mma.sync MMA atom. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet( - MlirContext context, MlirAttribute shapeMnk, int vecSize, MlirType aType, - MlirType bType, MlirType cType, MlirType sfType, bool useSfLayoutTV); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED int -mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(MlirType type); -MLIR_CAPI_EXPORTED bool -mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(MlirType type); - -// `!cute_nvgpu.sm80.sparse_mma`: SM80 sparse mma.sync MMA atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80Sparse(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80SparseTypeGet( - MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, - MlirType cType, uint32_t sparseMetadataFormat, MlirAttribute intOverflow); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(MlirType type); - -// `!cute_nvgpu.sm80.mma`: SM80 mma.sync MMA atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM80(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGet( - MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, - MlirType cType, MlirAttribute intOverflow, MlirAttribute binaryOp); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM80TypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(MlirType type); - -// `!cute_nvgpu.sm89.mma`: SM89 fp8 mma.sync MMA atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM89(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUMmaAtomSM89TypeGet(MlirContext context, MlirAttribute shapeMnk, - MlirType aType, MlirType bType, MlirType cType); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM89TypeGetCType(MlirType type); - -// `!cute_nvgpu.sm90.mma`: SM90 wgmma MMA atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUMmaAtomSM90(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGet( - MlirContext context, MlirAttribute shapeMnk, uint32_t aMajor, - uint32_t bMajor, MlirType aType, MlirType bType, MlirType cType, - uint32_t aFragKind, MlirAttribute intOverflow, bool aNeg, bool bNeg); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUMmaAtomSM90TypeGetCType(MlirType type); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(MlirType type); -MLIR_CAPI_EXPORTED bool mlirCuteNVGPUMmaAtomSM90TypeGetANeg(MlirType type); -MLIR_CAPI_EXPORTED bool mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(MlirType type); - -// `!cute_nvgpu.sm103.smem_desc_circular`: Circular shared-memory descriptor -// over a block layout in bytes. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUSmemDescCircularSM103(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescCircularSM103TypeGet( - MlirContext context, MlirAttribute blockLayoutBytes); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(MlirType type); - -// `!cute_nvgpu.sm107.smem_desc_circular`: Circular shared-memory descriptor -// over a block layout in bytes. -MLIR_CAPI_EXPORTED bool -mlirTypeIsACuteNVGPUSmemDescCircularSM107(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescCircularSM107TypeGet( - MlirContext context, MlirAttribute blockLayoutBytes); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(MlirType type); - -// `!cute_nvgpu.sm107.smem_desc`: SM107 shared-memory descriptor. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDescSM107(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUSmemDescSM107TypeGet(MlirContext context); - -// `!cute_nvgpu.smem_desc`: UMMA/GMMA shared-memory descriptor. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDesc(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescTypeGet(MlirContext context); - -// `!cute_nvgpu.smem_desc_view`: Shared-memory descriptor iterator with a -// layout. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUSmemDescView(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescViewTypeGet( - MlirContext context, MlirType desc, MlirAttribute layout); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUSmemDescViewTypeGetDesc(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUSmemDescViewTypeGetLayout(MlirType type); - -// `!cute_nvgpu.tma_descriptor_im2col`: Im2col TMA descriptor. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTmaDescriptorIm2Col(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(MlirContext context); - -// `!cute_nvgpu.tma_descriptor_tiled`: Tiled TMA descriptor. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUTmaDescriptorTiled(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUTmaDescriptorTiledTypeGet(MlirContext context); - -// `!cute_nvgpu.atom.universal_fma`: Universal FMA MMA atom. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUUniversalFmaAtom(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirCuteNVGPUUniversalFmaAtomTypeGet( - MlirContext context, MlirAttribute shapeMnk, MlirType aType, MlirType bType, - MlirType cType); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUUniversalFmaAtomTypeGetAType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUUniversalFmaAtomTypeGetBType(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUUniversalFmaAtomTypeGetCType(MlirType type); - -// `!cute_nvgpu.workid_response`: Cluster launch control work-id response. -MLIR_CAPI_EXPORTED bool mlirTypeIsACuteNVGPUWorkIdResponse(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirCuteNVGPUWorkIdResponseTypeGet(MlirContext context); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_CUTE_NVGPU_CAPI_H diff --git a/mlir/dialects/cute_nvgpu/cute_nvgpu.zig b/mlir/dialects/cute_nvgpu/cute_nvgpu.zig deleted file mode 100644 index ac92ab5a..00000000 --- a/mlir/dialects/cute_nvgpu/cute_nvgpu.zig +++ /dev/null @@ -1,5225 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -const cute_ir = @import("mlir/dialects/cute_ir"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("cute_nvgpu"); -} - -/// `cute_nvgpu.grid_constant`: marks a kernel argument passed as a -/// `__grid_constant__` parameter. -pub fn gridConstant(ctx: *mlir.Context) mlir.NamedAttribute { - return .named(ctx, "cute_nvgpu.grid_constant", .unit(ctx)); -} - -pub const NVGPUType = opaque { - const M = mlir.Methods(NVGPUType, c.MlirType); - - pub const isAFn = c.mlirTypeIsACuteNVGPUType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; -}; - -// Enums, generated from the dialect's .td files; values are the compiler's. - -/// NVGpu arch number like sm_103. -pub const Arch = enum(u32) { - sm_52 = 520, - sm_70 = 700, - sm_75 = 750, - sm_80 = 800, - sm_86 = 860, - sm_90 = 900, - sm_100 = 1000, - sm_103 = 1030, - sm_107 = 1070, -}; - -/// Fields stored in Bulk Load Copy Atom type. -pub const AtomCopyFieldBulkCopyG2S = enum(u32) { - tma_bar = 1, - mcast_mask = 2, - cache_policy = 7, -}; - -/// Fields stored in Bulk Store Copy Atom type. -pub const AtomCopyFieldBulkCopyS2G = enum(u32) { - byte_mask = 4, - cache_policy = 7, -}; - -/// Fields stored in Bulk CTA to Cluster Copy Atom type. -pub const AtomCopyFieldBulkCopyS2S = enum(u32) { - tma_bar = 1, - cta_rank = 5, -}; - -/// Fields stored in CopyAtomDsmemStoreType. -pub const AtomCopyFieldDsmemStore = enum(u32) { - mbar_ptr = 8, -}; - -/// Fields stored in CopyAtomG2RType. -pub const AtomCopyFieldLoadGlobal = enum(u32) { - cache_policy = 7, -}; - -/// Fields stored in CopyAtomNonExec2DGather4TmaLoadType. -pub const AtomCopyFieldNonExec2DGather4TmaLoad = enum(u32) { - tma_desc = 3, -}; - -/// Fields stored in CopyAtomNonExec2DScatter4TmaStoreType. -pub const AtomCopyFieldNonExec2DScatter4TmaStore = enum(u32) { - tma_desc = 3, -}; - -/// Fields stored in CopyAtomNonExecIm2ColTmaLoadType. -pub const AtomCopyFieldNonExecIm2ColTmaLoad = enum(u32) { - im2col_tma_desc = 6, -}; - -/// Fields stored in CopyAtomNonExecIm2ColTmaStoreType. -pub const AtomCopyFieldNonExecIm2ColTmaStore = enum(u32) { - im2col_tma_desc = 6, -}; - -/// Fields stored in CopyAtomNonExecTiledTmaLoadType. -pub const AtomCopyFieldNonExecTiledTmaLoad = enum(u32) { - tma_desc = 3, -}; - -/// Fields stored in CopyAtomNonExecTiledTmaReduceType. -pub const AtomCopyFieldNonExecTiledTmaReduce = enum(u32) { - tma_desc = 3, -}; - -/// Fields stored in CopyAtomNonExecTiledTmaStoreType. -pub const AtomCopyFieldNonExecTiledTmaStore = enum(u32) { - tma_desc = 3, -}; - -/// Fields stored in CopyAtomR2GType. -pub const AtomCopyFieldStoreGlobal = enum(u32) { - cache_policy = 7, -}; - -/// Fields stored in Tma Load Copy Atom type. -pub const AtomCopyFieldTmaLoad = enum(u32) { - tma_descriptor_ptr = 0, - tma_bar = 1, - mcast_mask = 2, - cache_policy = 7, - g_stride = 9, -}; - -/// Fields stored in Tma Reduce Copy Atom type. -pub const AtomCopyFieldTmaReduce = enum(u32) { - tma_descriptor_ptr = 0, - cache_policy = 7, - g_stride = 9, -}; - -/// Fields stored in Tma Store Copy Atom type. -pub const AtomCopyFieldTmaStore = enum(u32) { - tma_descriptor_ptr = 0, - cache_policy = 7, - g_stride = 9, -}; - -/// Fields stored in MMA Atom types for SM100. -pub const AtomMmaFieldSM100 = enum(u32) { - accum_c = 0, - neg_a = 1, - neg_b = 2, - disable_output_lane = 6, -}; - -/// Fields stored in MMA Atom types for SM100 (block scaled). -pub const AtomMmaFieldSM100BlockScaled = enum(u32) { - accum_c = 0, - neg_a = 1, - neg_b = 2, - sf_a = 3, - sf_b = 4, -}; - -/// Fields stored in MMA Atom types for SM100 (block scaled). -pub const AtomMmaFieldSM100BlockScaledSparse = enum(u32) { - accum_c = 0, - neg_a = 1, - neg_b = 2, - sf_a = 3, - sf_b = 4, - e = 5, -}; - -/// Fields stored in sparse MMA Atom types for SM100. -pub const AtomMmaFieldSM100Sparse = enum(u32) { - accum_c = 0, - neg_a = 1, - neg_b = 2, - e = 5, -}; - -/// Fields stored in MMA Atom types for SM120 (block scaled). -pub const AtomMmaFieldSM120BlockScaled = enum(u32) { - sf_a = 0, - sf_b = 1, -}; - -/// Fields stored in MMA Atom types for SM80. -pub const AtomMmaFieldSM80Sparse = enum(u32) { - e = 0, -}; - -/// Fields stored in MMA Atom types for SM90. -pub const AtomMmaFieldSM90 = enum(u32) { - accum_c = 0, -}; - -/// Binary operation for single-bit MMA operations. -pub const BinaryOp = enum(u32) { - none = 0, - xor_popc = 1, - and_popc = 2, -}; - -/// Cute Cache Eviction Priority kind. -pub const CacheEvictionPriority = enum(u32) { - EVICT_NORMAL = 0, - EVICT_FIRST = 1, - EVICT_LAST = 2, - EVICT_UNCHANGED = 3, - NO_ALLOCATE = 4, -}; - -/// Broadcast modes for the different utccp instructions. -pub const CopyS2TBroadcast = enum(u32) { - none = 0, - lw_0213 = 1, - lw_0123 = 2, - x4 = 3, -}; - -/// The various kinds of TMA loads in gather/scatter mode. -pub const GatherScatterTmaLoad = enum(u32) { - sm_100 = 0, - sm_100_multicast = 1, - sm_100_2sm = 2, - sm_100_2sm_multicast = 3, -}; - -/// The various kinds of TMA loads in im2col mode. -pub const Im2ColTmaLoad = enum(u32) { - sm_90 = 0, - sm_90_multicast = 1, - sm_100_2sm = 2, - sm_100_2sm_multicast = 3, -}; - -/// CuTe L2 prefetch size options. -pub const L2PrefetchSize = enum(u32) { - NONE = 0, - RESERVED = 1, - SIZE_64B = 2, - SIZE_128B = 3, - SIZE_256B = 4, -}; - -/// multimem ld_reduce accumulation precision kind. -pub const LdReduceAccPrecisionKind = enum(u32) { - F16 = 0, - F32 = 1, -}; - -/// LDSM's sz pattern, describing the bit size. -pub const LdsmSzPattern = enum(u32) { - u16 = 0, - u4to8 = 1, - s4to8 = 2, - u2to4 = 3, - s2to4 = 4, - u4x16p64to8 = 5, - u6x16p32to8 = 6, - u8 = 7, -}; - -/// Cache modes for the load instructions. -pub const LoadCacheMode = enum(u32) { - always = 0, - global = 1, - streaming = 2, - last_use = 3, - none = 4, -}; - -/// MMA overflow options. -pub const MMAIntOverflow = enum(u32) { - satfinite = 1, - wrapped = 0, -}; - -/// Memory Ordering kind. -pub const MemOrderKind = enum(u32) { - WEAK = 0, - RELAXED = 1, - ACQUIRE = 2, - RELEASE = 3, - ACQ_REL = 4, - SC = 5, - MMIO = 6, - CONSTANT = 7, - VOLATILE = 8, -}; - -/// Cute Memory Scope kind. -pub const MemScopeKind = enum(u32) { - CTA = 0, - CLUSTER = 1, - GPU = 2, - SYS = 3, -}; - -/// Enums for the mma collector op. -pub const MmaCollectorOp = enum(u32) { - discard = 0, - lastuse = 1, - fill = 2, - use = 3, -}; - -/// Enums for the mma frag type. -pub const MmaFragKind = enum(u32) { - smem_desc = 0, - sparse_smem_desc = 1, - tmem = 2, - tmem_ws = 3, - tmem_e = 4, - tmem_sf = 5, - rmem = 6, -}; - -/// Op for the TMASTORE instruction. -pub const ReductionKind = enum(u32) { - ADD = 0, - MIN = 1, - MAX = 2, - INC = 3, - DEC = 4, - AND = 5, - OR = 6, - XOR = 7, -}; - -/// CuTe Shared Space kind. -pub const SharedSpace = enum(u32) { - CTA = 0, - CLUSTER = 1, -}; - -/// Metadata format used in sparse MMA. -pub const SparseMetadataFormat = enum(u32) { - tid = 0, -}; - -/// Cache modes for the store instructions. -pub const StoreCacheMode = enum(u32) { - write_back = 0, - global = 1, - streaming = 2, - write_through = 3, - none = 4, -}; - -/// The various kinds of TMA loads in tiled mode. -pub const TiledTmaLoad = enum(u32) { - sm_90 = 0, - sm_90_multicast = 1, - sm_100_2sm = 2, - sm_100_2sm_multicast = 3, -}; - -/// Bits 74-71 of the TMA descriptor. -pub const TmaDataFormat = enum(u32) { - U8 = 0, - U16 = 1, - U32 = 2, - S32 = 3, - U64 = 4, - S64 = 5, - F16_RN = 6, - F32_RN = 7, - F32_FTZ_RN = 8, - F64_RN = 9, - BF16_RN = 10, - U4 = 11, - U4_UNPACK_U8 = 12, - U6_UNPACK_U8 = 13, - TF32_RN = 14, - TF32_FTZ_RN = 15, -}; - -/// Modes for the TMA load instruction. -pub const TmaLoadMode = enum(u32) { - tiled = 0, - im2col = 1, - w128 = 2, - w = 3, - gather4 = 4, -}; - -/// Modes for the TMA store instruction. -pub const TmaStoreMode = enum(u32) { - tiled = 0, - im2col = 1, - scatter4 = 2, -}; - -/// Modes for the Tmem allocation. -pub const TmemAllocMode = enum(u32) { - Interleaved = 0, - NonInterleaved = 1, - Duplicated = 2, - SfDuplicated_4x1 = 3, - SfDuplicated_2x2 = 4, -}; - -/// Reduce operation for the Tmem load instruction. -pub const TmemLoadRedOp = enum(u32) { - max = 0, - maxabs = 1, - min = 2, - minabs = 3, -}; - -// Types and attributes, generated from the dialect's .td files. - -/// `!cute_nvgpu.atom.bulk_copy_g2s`: Bulk (TMA non-tensor) copy atom. -pub const CopyAtomBulkCopyG2SType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyG2S; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.bulk_copy_g2s"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int = 0, - mcast: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.mcast); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetCopyBits(self.ptr()); - } - pub fn mcast(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomBulkCopyG2STypeGetMcast(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.bulk_copy_s2g`: Bulk (TMA non-tensor) copy atom. -pub const CopyAtomBulkCopyS2GType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2G; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.bulk_copy_s2g"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int = 0, - mask: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.mask); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetCopyBits(self.ptr()); - } - pub fn mask(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomBulkCopyS2GTypeGetMask(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.bulk_copy_s2s`: Bulk (TMA non-tensor) copy atom. -pub const CopyAtomBulkCopyS2SType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomBulkCopyS2S; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.bulk_copy_s2s"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int = 0, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomBulkCopyS2STypeGetCopyBits(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.dsmem_store`: Distributed shared-memory store atom. -pub const CopyAtomDsmemStoreType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomDsmemStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.dsmem_store"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomDsmemStoreTypeGetCopyBits(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.g2r`: Global-to-register load atom. -pub const CopyAtomG2RType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomG2R; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.g2r"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - memOrder: MemOrderKind = .WEAK, - memScope: MemScopeKind = .CTA, - l2PrefetchSize: L2PrefetchSize = .NONE, - l1CacheEvictPriority: CacheEvictionPriority = .EVICT_NORMAL, - loadCacheMode: LoadCacheMode = .always, - sharedSpace: SharedSpace = .CTA, - invariant: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomG2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l2PrefetchSize), @backingInt(args.l1CacheEvictPriority), @backingInt(args.loadCacheMode), @backingInt(args.sharedSpace), args.invariant); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomG2RTypeGetCopyBits(self.ptr()); - } - pub fn memOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemOrder(self.ptr()))); - } - pub fn memScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetMemScope(self.ptr()))); - } - pub fn l2PrefetchSize(self: *const Self) L2PrefetchSize { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL2PrefetchSize(self.ptr()))); - } - pub fn l1CacheEvictPriority(self: *const Self) CacheEvictionPriority { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetL1CacheEvictPriority(self.ptr()))); - } - pub fn loadCacheMode(self: *const Self) LoadCacheMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetLoadCacheMode(self.ptr()))); - } - pub fn sharedSpace(self: *const Self) SharedSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomG2RTypeGetSharedSpace(self.ptr()))); - } - pub fn invariant(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomG2RTypeGetInvariant(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.im2col_tma_load`: Executable im2col TMA load atom. -pub const CopyAtomIm2ColTmaLoadType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.im2col_tma_load"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - numCta: c_int, - gStride: *const mlir.Type, - mcast: bool = false, - tmaGbasis: ?*const mlir.Type = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetCopyBits(self.ptr()); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetNumCta(self.ptr()); - } - pub fn gStride(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetGStride(self.ptr()).ptr.?); - } - pub fn mcast(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetMcast(self.ptr()); - } - pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaLoadTypeGetTmaGbasis(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.atom.im2col_tma_store`: Executable im2col TMA store atom. -pub const CopyAtomIm2ColTmaStoreType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomIm2ColTmaStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.im2col_tma_store"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - gStride: *const mlir.Type, - tmaGbasis: ?*const mlir.Type = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetCopyBits(self.ptr()); - } - pub fn gStride(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetGStride(self.ptr()).ptr.?); - } - pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomIm2ColTmaStoreTypeGetTmaGbasis(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.atom.ldsm`: ldmatrix atom. -pub const CopyAtomLdsmType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomLdsm; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.ldsm"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - mode: *const mlir.Attribute, - szPattern: LdsmSzPattern, - numMatrices: c_int, - transpose: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomLdsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), @backingInt(args.szPattern), args.numMatrices, args.transpose); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetValType(self.ptr()).ptr.?); - } - pub fn mode(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetMode(self.ptr()).ptr.?); - } - pub fn szPattern(self: *const Self) LdsmSzPattern { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomLdsmTypeGetSzPattern(self.ptr()))); - } - pub fn numMatrices(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomLdsmTypeGetNumMatrices(self.ptr()); - } - pub fn transpose(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomLdsmTypeGetTranspose(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.non_exec_2d_gather4_tma_load`: Non-executable 2-D gather4 TMA load atom. -pub const CopyAtomNonExec2DGather4TmaLoadType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExec2DGather4TmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.non_exec_2d_gather4_tma_load"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - kind: GatherScatterTmaLoad, - valType: *const mlir.Type, - copyBits: c_int, - tmaGbasis: *const mlir.Type, - tmaFormat: ?TmaDataFormat = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn kind(self: *const Self) GatherScatterTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetKind(self.ptr()))); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetCopyBits(self.ptr()); - } - pub fn tmaGbasis(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); - } - pub fn tmaFormat(self: *const Self) ?TmaDataFormat { - const value = c.mlirCuteNVGPUCopyAtomNonExec2DGather4TmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); - } -}; - -/// `!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store`: Non-executable 2-D scatter4 TMA store atom. -pub const CopyAtomNonExec2DScatter4TmaStoreType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExec2DScatter4TmaStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.non_exec_2d_scatter4_tma_store"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - tmaGbasis: *const mlir.Type, - tmaFormat: ?TmaDataFormat = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetCopyBits(self.ptr()); - } - pub fn tmaGbasis(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); - } - pub fn tmaFormat(self: *const Self) ?TmaDataFormat { - const value = c.mlirCuteNVGPUCopyAtomNonExec2DScatter4TmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); - } -}; - -/// `!cute_nvgpu.atom.non_exec_im2col_tma_load`: Non-executable im2col TMA load atom. -pub const CopyAtomNonExecIm2ColTmaLoadType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.non_exec_im2col_tma_load"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - kind: Im2ColTmaLoad, - valType: *const mlir.Type, - copyBits: c_int, - tmaGbasis: *const mlir.Type, - tmaFormat: ?TmaDataFormat = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn kind(self: *const Self) Im2ColTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetKind(self.ptr()))); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetCopyBits(self.ptr()); - } - pub fn tmaGbasis(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); - } - pub fn tmaFormat(self: *const Self) ?TmaDataFormat { - const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); - } -}; - -/// `!cute_nvgpu.atom.non_exec_im2col_tma_store`: Non-executable im2col TMA store atom. -pub const CopyAtomNonExecIm2ColTmaStoreType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecIm2ColTmaStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.non_exec_im2col_tma_store"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - tmaGbasis: *const mlir.Type, - tmaFormat: ?TmaDataFormat = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetCopyBits(self.ptr()); - } - pub fn tmaGbasis(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); - } - pub fn tmaFormat(self: *const Self) ?TmaDataFormat { - const value = c.mlirCuteNVGPUCopyAtomNonExecIm2ColTmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); - } -}; - -/// `!cute_nvgpu.atom.non_exec_tiled_tma_load`: Non-executable tiled TMA load atom. -pub const CopyAtomNonExecTiledTmaLoadType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.non_exec_tiled_tma_load"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - kind: TiledTmaLoad, - valType: *const mlir.Type, - copyBits: c_int, - tmaGbasis: *const mlir.Type, - tmaFormat: ?TmaDataFormat = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn kind(self: *const Self) TiledTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetKind(self.ptr()))); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetCopyBits(self.ptr()); - } - pub fn tmaGbasis(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaGbasis(self.ptr()).ptr.?); - } - pub fn tmaFormat(self: *const Self) ?TmaDataFormat { - const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaLoadTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); - } -}; - -/// `!cute_nvgpu.atom.non_exec_tiled_tma_reduce`: Non-executable tiled TMA reduce atom. -pub const CopyAtomNonExecTiledTmaReduceType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaReduce; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.non_exec_tiled_tma_reduce"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - kind: ReductionKind, - valType: *const mlir.Type, - copyBits: c_int, - tmaGbasis: *const mlir.Type, - tmaFormat: ?TmaDataFormat = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGet(ctx.ptr(), @backingInt(args.kind), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn kind(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetKind(self.ptr()))); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetCopyBits(self.ptr()); - } - pub fn tmaGbasis(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaGbasis(self.ptr()).ptr.?); - } - pub fn tmaFormat(self: *const Self) ?TmaDataFormat { - const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaReduceTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); - } -}; - -/// `!cute_nvgpu.atom.non_exec_tiled_tma_store`: Non-executable tiled TMA store atom. -pub const CopyAtomNonExecTiledTmaStoreType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomNonExecTiledTmaStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.non_exec_tiled_tma_store"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - tmaGbasis: *const mlir.Type, - tmaFormat: ?TmaDataFormat = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, args.tmaGbasis.ptr(), if (args.tmaFormat) |v| @backingInt(v) else -1); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetCopyBits(self.ptr()); - } - pub fn tmaGbasis(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaGbasis(self.ptr()).ptr.?); - } - pub fn tmaFormat(self: *const Self) ?TmaDataFormat { - const value = c.mlirCuteNVGPUCopyAtomNonExecTiledTmaStoreTypeGetTmaFormat(self.ptr()); - return if (value < 0) null else @fromBackingInt(@intCast(value)); - } -}; - -/// `!cute_nvgpu.atom.r2g`: Register-to-global store atom. -pub const CopyAtomR2GType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2G; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.r2g"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - memOrder: MemOrderKind = .WEAK, - memScope: MemScopeKind = .CTA, - l1CacheEvictPriority: CacheEvictionPriority = .EVICT_NORMAL, - storeCacheMode: StoreCacheMode = .write_back, - sharedSpace: SharedSpace = .CTA, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomR2GTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.l1CacheEvictPriority), @backingInt(args.storeCacheMode), @backingInt(args.sharedSpace)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomR2GTypeGetCopyBits(self.ptr()); - } - pub fn memOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemOrder(self.ptr()))); - } - pub fn memScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetMemScope(self.ptr()))); - } - pub fn l1CacheEvictPriority(self: *const Self) CacheEvictionPriority { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetL1CacheEvictPriority(self.ptr()))); - } - pub fn storeCacheMode(self: *const Self) StoreCacheMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetStoreCacheMode(self.ptr()))); - } - pub fn sharedSpace(self: *const Self) SharedSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2GTypeGetSharedSpace(self.ptr()))); - } -}; - -/// `!cute_nvgpu.atom.r2s`: Register-to-shared store atom. -pub const CopyAtomR2SType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomR2S; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.r2s"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - memOrder: MemOrderKind = .WEAK, - memScope: MemScopeKind = .CTA, - sharedSpace: SharedSpace = .CTA, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomR2STypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomR2STypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomR2STypeGetCopyBits(self.ptr()); - } - pub fn memOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemOrder(self.ptr()))); - } - pub fn memScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetMemScope(self.ptr()))); - } - pub fn sharedSpace(self: *const Self) SharedSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomR2STypeGetSharedSpace(self.ptr()))); - } -}; - -/// `!cute_nvgpu.atom.s2r`: Shared-to-register load atom. -pub const CopyAtomS2RType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomS2R; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.s2r"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - memOrder: MemOrderKind = .WEAK, - memScope: MemScopeKind = .CTA, - sharedSpace: SharedSpace = .CTA, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomS2RTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope), @backingInt(args.sharedSpace)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomS2RTypeGetCopyBits(self.ptr()); - } - pub fn memOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemOrder(self.ptr()))); - } - pub fn memScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetMemScope(self.ptr()))); - } - pub fn sharedSpace(self: *const Self) SharedSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomS2RTypeGetSharedSpace(self.ptr()))); - } -}; - -/// `!cute_nvgpu.atom.simt_async_copy`: cp.async copy atom. -pub const CopyAtomSIMTAsyncCopyType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTAsyncCopy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.simt_async_copy"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - cache: LoadCacheMode, - copyBits: c_int, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), @backingInt(args.cache), args.copyBits); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetValType(self.ptr()).ptr.?); - } - pub fn cache(self: *const Self) LoadCacheMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCache(self.ptr()))); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSIMTAsyncCopyTypeGetCopyBits(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.simt_multimem_ld_reduce`: multimem.ld_reduce atom. -pub const CopyAtomSIMTMultimemLdReduceType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemLdReduce; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.simt_multimem_ld_reduce"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int = 0, - reduction: ReductionKind = .ADD, - ldReduceAccPrecision: ?*const mlir.Attribute = null, - memOrder: MemOrderKind = .RELAXED, - memScope: MemScopeKind = .SYS, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), if (args.ldReduceAccPrecision) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, @backingInt(args.memOrder), @backingInt(args.memScope)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetCopyBits(self.ptr()); - } - pub fn reduction(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetReduction(self.ptr()))); - } - pub fn ldReduceAccPrecision(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetLdReduceAccPrecision(self.ptr()).ptr); - } - pub fn memOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemOrder(self.ptr()))); - } - pub fn memScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemLdReduceTypeGetMemScope(self.ptr()))); - } -}; - -/// `!cute_nvgpu.atom.simt_multimem_red`: multimem.red atom. -pub const CopyAtomSIMTMultimemRedType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemRed; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.simt_multimem_red"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - reduction: ReductionKind = .ADD, - memOrder: MemOrderKind = .RELAXED, - memScope: MemScopeKind = .SYS, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.reduction), @backingInt(args.memOrder), @backingInt(args.memScope)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetCopyBits(self.ptr()); - } - pub fn reduction(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetReduction(self.ptr()))); - } - pub fn memOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemOrder(self.ptr()))); - } - pub fn memScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemRedTypeGetMemScope(self.ptr()))); - } -}; - -/// `!cute_nvgpu.atom.simt_multimem_st`: multimem.st atom. -pub const CopyAtomSIMTMultimemStType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTMultimemSt; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.simt_multimem_st"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int = 0, - memOrder: MemOrderKind = .RELAXED, - memScope: MemScopeKind = .SYS, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.memOrder), @backingInt(args.memScope)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetCopyBits(self.ptr()); - } - pub fn memOrder(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemOrder(self.ptr()))); - } - pub fn memScope(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTMultimemStTypeGetMemScope(self.ptr()))); - } -}; - -/// `!cute_nvgpu.atom.universal_copy`: Universal (SIMT) copy atom. -pub const CopyAtomSIMTSyncCopyType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSIMTSyncCopy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.universal_copy"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int = 0, - srcSpace: cute_ir.AddressSpace = .generic, - dstSpace: cute_ir.AddressSpace = .generic, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.srcSpace), @backingInt(args.dstSpace)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetCopyBits(self.ptr()); - } - pub fn srcSpace(self: *const Self) cute_ir.AddressSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetSrcSpace(self.ptr()))); - } - pub fn dstSpace(self: *const Self) cute_ir.AddressSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSIMTSyncCopyTypeGetDstSpace(self.ptr()))); - } -}; - -/// `!cute_nvgpu.atom.s2t_copy`: tcgen05.cp (shared to tensor memory) atom. -pub const CopyAtomSM100CopyS2TType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100CopyS2T; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.s2t_copy"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - numDp: c_int, - numBit: c_int, - numCta: c_int, - broadcast: CopyS2TBroadcast = .none, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.broadcast)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetValType(self.ptr()).ptr.?); - } - pub fn numDp(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumDp(self.ptr()); - } - pub fn numBit(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumBit(self.ptr()); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetNumCta(self.ptr()); - } - pub fn broadcast(self: *const Self) CopyS2TBroadcast { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100CopyS2TTypeGetBroadcast(self.ptr()))); - } -}; - -/// `!cute_nvgpu.atom.sm100_s2t_copy_v2`: tcgen05.cp atom with the shared-memory major mode. -pub const CopyAtomSM100S2TCopyV2Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100S2TCopyV2; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.sm100_s2t_copy_v2"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - numDp: c_int, - numBit: c_int, - numCta: c_int, - smemMajor: cute_ir.MajorMode, - broadcast: CopyS2TBroadcast = .none, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numCta, @backingInt(args.smemMajor), @backingInt(args.broadcast)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetValType(self.ptr()).ptr.?); - } - pub fn numDp(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumDp(self.ptr()); - } - pub fn numBit(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumBit(self.ptr()); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetNumCta(self.ptr()); - } - pub fn smemMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetSmemMajor(self.ptr()))); - } - pub fn broadcast(self: *const Self) CopyS2TBroadcast { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM100S2TCopyV2TypeGetBroadcast(self.ptr()))); - } -}; - -/// `!cute_nvgpu.atom.tmem_load`: tcgen05.ld atom. -pub const CopyAtomSM100TmemLoadType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.tmem_load"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - numDp: c_int, - numBit: c_int, - numRep: u32, - pack16b: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, args.pack16b); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetValType(self.ptr()).ptr.?); - } - pub fn numDp(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumDp(self.ptr()); - } - pub fn numBit(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumBit(self.ptr()); - } - pub fn numRep(self: *const Self) u32 { - return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetNumRep(self.ptr()); - } - pub fn pack16b(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomSM100TmemLoadTypeGetPack16b(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.tmem_store`: tcgen05.st atom. -pub const CopyAtomSM100TmemStoreType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM100TmemStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.tmem_store"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - numDp: c_int, - numBit: c_int, - numRep: u32, - expand16b: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, args.expand16b); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetValType(self.ptr()).ptr.?); - } - pub fn numDp(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumDp(self.ptr()); - } - pub fn numBit(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumBit(self.ptr()); - } - pub fn numRep(self: *const Self) u32 { - return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetNumRep(self.ptr()); - } - pub fn expand16b(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomSM100TmemStoreTypeGetExpand16b(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.tmem_load_spcompress`: SM107 tensor-memory load with sparse compression. -pub const CopyAtomSM107TmemLoadSPCompressType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM107TmemLoadSPCompress; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.tmem_load_spcompress"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - numDp: c_int, - numBit: c_int, - numRep: u32, - redOp: TmemLoadRedOp, - red: bool = false, - nan: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.red, args.nan); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetValType(self.ptr()).ptr.?); - } - pub fn numDp(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumDp(self.ptr()); - } - pub fn numBit(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumBit(self.ptr()); - } - pub fn numRep(self: *const Self) u32 { - return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNumRep(self.ptr()); - } - pub fn redOp(self: *const Self) TmemLoadRedOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRedOp(self.ptr()))); - } - pub fn red(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetRed(self.ptr()); - } - pub fn nan(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomSM107TmemLoadSPCompressTypeGetNan(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.tmem_load_red`: tcgen05.ld.red atom. -pub const CopyAtomSM10xTmemLoadRedType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomSM10xTmemLoadRed; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.tmem_load_red"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - numDp: c_int, - numBit: c_int, - numRep: u32, - redOp: TmemLoadRedOp, - nan: bool = false, - halfSplitOff: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGet(ctx.ptr(), args.valType.ptr(), args.numDp, args.numBit, args.numRep, @backingInt(args.redOp), args.nan, if (args.halfSplitOff) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetValType(self.ptr()).ptr.?); - } - pub fn numDp(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumDp(self.ptr()); - } - pub fn numBit(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumBit(self.ptr()); - } - pub fn numRep(self: *const Self) u32 { - return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNumRep(self.ptr()); - } - pub fn redOp(self: *const Self) TmemLoadRedOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetRedOp(self.ptr()))); - } - pub fn nan(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetNan(self.ptr()); - } - pub fn halfSplitOff(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomSM10xTmemLoadRedTypeGetHalfSplitOff(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.atom.stsm`: stmatrix atom. -pub const CopyAtomStsmType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomStsm; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.stsm"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - mode: *const mlir.Attribute, - numMatrices: c_int, - transpose: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomStsmTypeGet(ctx.ptr(), args.valType.ptr(), args.mode.ptr(), args.numMatrices, args.transpose); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetValType(self.ptr()).ptr.?); - } - pub fn mode(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUCopyAtomStsmTypeGetMode(self.ptr()).ptr.?); - } - pub fn numMatrices(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomStsmTypeGetNumMatrices(self.ptr()); - } - pub fn transpose(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomStsmTypeGetTranspose(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.tma_load`: Executable TMA load atom. -pub const CopyAtomTmaLoadType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.tma_load"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - sparsity: c_int = 1, - copyBits: c_int, - mode: TmaLoadMode, - numCta: c_int, - gStride: *const mlir.Type, - mcast: bool = false, - tmaGbasis: ?*const mlir.Type = null, - override: bool = false, - noFullyOobTile: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaLoadTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.numCta, args.gStride.ptr(), args.mcast, if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetValType(self.ptr()).ptr.?); - } - pub fn sparsity(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetSparsity(self.ptr()); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetCopyBits(self.ptr()); - } - pub fn mode(self: *const Self) TmaLoadMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMode(self.ptr()))); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNumCta(self.ptr()); - } - pub fn gStride(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetGStride(self.ptr()).ptr.?); - } - pub fn mcast(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetMcast(self.ptr()); - } - pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetTmaGbasis(self.ptr()).ptr); - } - pub fn override(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetOverride(self.ptr()); - } - pub fn noFullyOobTile(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomTmaLoadTypeGetNoFullyOobTile(self.ptr()); - } -}; - -/// `!cute_nvgpu.atom.tma_reduce`: Executable TMA reduce atom. -pub const CopyAtomTmaReduceType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaReduce; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.tma_reduce"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - copyBits: c_int, - mode: TmaStoreMode, - kind: ReductionKind, - gStride: *const mlir.Type, - tmaGbasis: ?*const mlir.Type = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaReduceTypeGet(ctx.ptr(), args.valType.ptr(), args.copyBits, @backingInt(args.mode), @backingInt(args.kind), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetValType(self.ptr()).ptr.?); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetCopyBits(self.ptr()); - } - pub fn mode(self: *const Self) TmaStoreMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetMode(self.ptr()))); - } - pub fn kind(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetKind(self.ptr()))); - } - pub fn gStride(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetGStride(self.ptr()).ptr.?); - } - pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaReduceTypeGetTmaGbasis(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.atom.tma_store`: Executable TMA store atom. -pub const CopyAtomTmaStoreType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUCopyAtomTmaStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.tma_store"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - valType: *const mlir.Type, - sparsity: c_int = 1, - copyBits: c_int, - mode: TmaStoreMode, - gStride: *const mlir.Type, - tmaGbasis: ?*const mlir.Type = null, - override: bool = false, - noFullyOobTile: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyAtomTmaStoreTypeGet(ctx.ptr(), args.valType.ptr(), args.sparsity, args.copyBits, @backingInt(args.mode), args.gStride.ptr(), if (args.tmaGbasis) |v| v.ptr() else c.MlirType{ .ptr = null }, args.override, args.noFullyOobTile); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetValType(self.ptr()).ptr.?); - } - pub fn sparsity(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetSparsity(self.ptr()); - } - pub fn copyBits(self: *const Self) c_int { - return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetCopyBits(self.ptr()); - } - pub fn mode(self: *const Self) TmaStoreMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetMode(self.ptr()))); - } - pub fn gStride(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetGStride(self.ptr()).ptr.?); - } - pub fn tmaGbasis(self: *const Self) ?*const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetTmaGbasis(self.ptr()).ptr); - } - pub fn override(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetOverride(self.ptr()); - } - pub fn noFullyOobTile(self: *const Self) bool { - return c.mlirCuteNVGPUCopyAtomTmaStoreTypeGetNoFullyOobTile(self.ptr()); - } -}; - -/// `!cute_nvgpu.sm100.mma_bs_sp`: SM100 block-scaled sparse UMMA MMA atom. -pub const MmaAtomSM100UMMABlockScaledSparseType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaledSparse; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm100.mma_bs_sp"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - numCta: c_int, - aMajor: cute_ir.MajorMode, - bMajor: cute_ir.MajorMode, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - sfType: *const mlir.Type, - sparseMetadataFormat: SparseMetadataFormat, - aFragKind: MmaFragKind, - vecSize: c_int, - archPromote: Arch = .sm_100, - intOverflow: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetNumCta(self.ptr()); - } - pub fn aMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); - } - pub fn bMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetCType(self.ptr()).ptr.?); - } - pub fn sfType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); - } - pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); - } - pub fn aFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); - } - pub fn vecSize(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetVecSize(self.ptr()); - } - pub fn archPromote(self: *const Self) Arch { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetArchPromote(self.ptr()))); - } - pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.sm100.mma_bs`: SM100 block-scaled UMMA MMA atom. -pub const MmaAtomSM100UMMABlockScaledType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMABlockScaled; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm100.mma_bs"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - numCta: c_int, - aMajor: cute_ir.MajorMode, - bMajor: cute_ir.MajorMode, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - sfType: *const mlir.Type, - aFragKind: MmaFragKind, - vecSize: c_int, - archPromote: Arch = .sm_100, - intOverflow: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.archPromote), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetNumCta(self.ptr()); - } - pub fn aMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAMajor(self.ptr()))); - } - pub fn bMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBMajor(self.ptr()))); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetCType(self.ptr()).ptr.?); - } - pub fn sfType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); - } - pub fn aFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetAFragKind(self.ptr()))); - } - pub fn vecSize(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetVecSize(self.ptr()); - } - pub fn archPromote(self: *const Self) Arch { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetArchPromote(self.ptr()))); - } - pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.sm100.mma_sp`: SM100 sparse UMMA MMA atom. -pub const MmaAtomSM100UMMASparseType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMASparse; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm100.mma_sp"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - numCta: c_int, - aMajor: cute_ir.MajorMode, - bMajor: cute_ir.MajorMode, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - eType: *const mlir.Type, - sparseMetadataFormat: SparseMetadataFormat, - aFragKind: MmaFragKind, - cScaleExp: c_int, - intOverflow: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetNumCta(self.ptr()); - } - pub fn aMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAMajor(self.ptr()))); - } - pub fn bMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBMajor(self.ptr()))); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCType(self.ptr()).ptr.?); - } - pub fn eType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetEType(self.ptr()).ptr.?); - } - pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); - } - pub fn aFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetAFragKind(self.ptr()))); - } - pub fn cScaleExp(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetCScaleExp(self.ptr()); - } - pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMASparseTypeGetIntOverflow(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.sm100.mma`: SM100 UMMA MMA atom. -pub const MmaAtomSM100UMMAType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM100UMMA; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm100.mma"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - numCta: c_int, - aMajor: cute_ir.MajorMode, - bMajor: cute_ir.MajorMode, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - aFragKind: MmaFragKind, - cScaleExp: c_int, - intOverflow: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM100UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetNumCta(self.ptr()); - } - pub fn aMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAMajor(self.ptr()))); - } - pub fn bMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBMajor(self.ptr()))); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCType(self.ptr()).ptr.?); - } - pub fn aFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetAFragKind(self.ptr()))); - } - pub fn cScaleExp(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetCScaleExp(self.ptr()); - } - pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM100UMMATypeGetIntOverflow(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.sm107.mma_bs_sp`: SM107 block-scaled sparse UMMA MMA atom. -pub const MmaAtomSM107UMMABlockScaledSparseType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaledSparse; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm107.mma_bs_sp"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - numCta: c_int, - aMajor: cute_ir.MajorMode, - bMajor: cute_ir.MajorMode, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - sfType: *const mlir.Type, - sparseMetadataFormat: SparseMetadataFormat, - aFragKind: MmaFragKind, - vecSize: c_int, - aCollectorOp: MmaCollectorOp, - bCollectorOp: MmaCollectorOp, - intOverflow: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetNumCta(self.ptr()); - } - pub fn aMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAMajor(self.ptr()))); - } - pub fn bMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBMajor(self.ptr()))); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetCType(self.ptr()).ptr.?); - } - pub fn sfType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSfType(self.ptr()).ptr.?); - } - pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetSparseMetadataFormat(self.ptr()))); - } - pub fn aFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetAFragKind(self.ptr()))); - } - pub fn vecSize(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetVecSize(self.ptr()); - } - pub fn aCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetACollectorOp(self.ptr()))); - } - pub fn bCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetBCollectorOp(self.ptr()))); - } - pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledSparseTypeGetIntOverflow(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.sm107.mma_bs`: SM107 block-scaled UMMA MMA atom. -pub const MmaAtomSM107UMMABlockScaledType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMABlockScaled; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm107.mma_bs"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - numCta: c_int, - aMajor: cute_ir.MajorMode, - bMajor: cute_ir.MajorMode, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - sfType: *const mlir.Type, - aFragKind: MmaFragKind, - vecSize: c_int, - aCollectorOp: MmaCollectorOp, - bCollectorOp: MmaCollectorOp, - intOverflow: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), @backingInt(args.aFragKind), args.vecSize, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetNumCta(self.ptr()); - } - pub fn aMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAMajor(self.ptr()))); - } - pub fn bMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBMajor(self.ptr()))); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetCType(self.ptr()).ptr.?); - } - pub fn sfType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetSfType(self.ptr()).ptr.?); - } - pub fn aFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetAFragKind(self.ptr()))); - } - pub fn vecSize(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetVecSize(self.ptr()); - } - pub fn aCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetACollectorOp(self.ptr()))); - } - pub fn bCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetBCollectorOp(self.ptr()))); - } - pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMABlockScaledTypeGetIntOverflow(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.sm107.mma_sp`: SM107 sparse UMMA MMA atom. -pub const MmaAtomSM107UMMASparseType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMASparse; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm107.mma_sp"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - numCta: c_int, - aMajor: cute_ir.MajorMode, - bMajor: cute_ir.MajorMode, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - eType: *const mlir.Type, - sparseMetadataFormat: SparseMetadataFormat, - aFragKind: MmaFragKind, - cScaleExp: c_int, - aCollectorOp: MmaCollectorOp, - bCollectorOp: MmaCollectorOp, - intOverflow: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.eType.ptr(), @backingInt(args.sparseMetadataFormat), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetNumCta(self.ptr()); - } - pub fn aMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAMajor(self.ptr()))); - } - pub fn bMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBMajor(self.ptr()))); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCType(self.ptr()).ptr.?); - } - pub fn eType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetEType(self.ptr()).ptr.?); - } - pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetSparseMetadataFormat(self.ptr()))); - } - pub fn aFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetAFragKind(self.ptr()))); - } - pub fn cScaleExp(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetCScaleExp(self.ptr()); - } - pub fn aCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetACollectorOp(self.ptr()))); - } - pub fn bCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetBCollectorOp(self.ptr()))); - } - pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMASparseTypeGetIntOverflow(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.sm107.mma`: SM107 UMMA MMA atom. -pub const MmaAtomSM107UMMAType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM107UMMA; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm107.mma"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - numCta: c_int, - aMajor: cute_ir.MajorMode, - bMajor: cute_ir.MajorMode, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - aFragKind: MmaFragKind, - cScaleExp: c_int, - aCollectorOp: MmaCollectorOp, - bCollectorOp: MmaCollectorOp, - intOverflow: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM107UMMATypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.numCta, @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), args.cScaleExp, @backingInt(args.aCollectorOp), @backingInt(args.bCollectorOp), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn numCta(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetNumCta(self.ptr()); - } - pub fn aMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAMajor(self.ptr()))); - } - pub fn bMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBMajor(self.ptr()))); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCType(self.ptr()).ptr.?); - } - pub fn aFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetAFragKind(self.ptr()))); - } - pub fn cScaleExp(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetCScaleExp(self.ptr()); - } - pub fn aCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetACollectorOp(self.ptr()))); - } - pub fn bCollectorOp(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetBCollectorOp(self.ptr()))); - } - pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM107UMMATypeGetIntOverflow(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.SM120.mma_bs`: SM120 block-scaled mma.sync MMA atom. -pub const MmaAtomSM120BlockScaledType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM120BlockScaled; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "SM120.mma_bs"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - vecSize: c_int, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - sfType: *const mlir.Type, - useSfLayoutTV: bool, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.vecSize, args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), args.sfType.ptr(), args.useSfLayoutTV); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn vecSize(self: *const Self) c_int { - return c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetVecSize(self.ptr()); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetCType(self.ptr()).ptr.?); - } - pub fn sfType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetSfType(self.ptr()).ptr.?); - } - pub fn useSfLayoutTV(self: *const Self) bool { - return c.mlirCuteNVGPUMmaAtomSM120BlockScaledTypeGetUseSfLayoutTV(self.ptr()); - } -}; - -/// `!cute_nvgpu.sm80.sparse_mma`: SM80 sparse mma.sync MMA atom. -pub const MmaAtomSM80SparseType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80Sparse; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm80.sparse_mma"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - sparseMetadataFormat: SparseMetadataFormat, - intOverflow: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM80SparseTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.sparseMetadataFormat), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetCType(self.ptr()).ptr.?); - } - pub fn sparseMetadataFormat(self: *const Self) SparseMetadataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetSparseMetadataFormat(self.ptr()))); - } - pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80SparseTypeGetIntOverflow(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.sm80.mma`: SM80 mma.sync MMA atom. -pub const MmaAtomSM80Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM80; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm80.mma"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - intOverflow: ?*const mlir.Attribute = null, - binaryOp: ?*const mlir.Attribute = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM80TypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, if (args.binaryOp) |v| v.ptr() else c.MlirAttribute{ .ptr = null }); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetCType(self.ptr()).ptr.?); - } - pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetIntOverflow(self.ptr()).ptr); - } - pub fn binaryOp(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM80TypeGetBinaryOp(self.ptr()).ptr); - } -}; - -/// `!cute_nvgpu.sm89.mma`: SM89 fp8 mma.sync MMA atom. -pub const MmaAtomSM89Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM89; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm89.mma"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM89TypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM89TypeGetCType(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.sm90.mma`: SM90 wgmma MMA atom. -pub const MmaAtomSM90Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUMmaAtomSM90; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm90.mma"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - aMajor: cute_ir.MajorMode, - bMajor: cute_ir.MajorMode, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - aFragKind: MmaFragKind, - intOverflow: ?*const mlir.Attribute = null, - aNeg: bool = false, - bNeg: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaAtomSM90TypeGet(ctx.ptr(), args.shapeMnk.ptr(), @backingInt(args.aMajor), @backingInt(args.bMajor), args.aType.ptr(), args.bType.ptr(), args.cType.ptr(), @backingInt(args.aFragKind), if (args.intOverflow) |v| v.ptr() else c.MlirAttribute{ .ptr = null }, args.aNeg, args.bNeg); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn aMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAMajor(self.ptr()))); - } - pub fn bMajor(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBMajor(self.ptr()))); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetCType(self.ptr()).ptr.?); - } - pub fn aFragKind(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetAFragKind(self.ptr()))); - } - pub fn intOverflow(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUMmaAtomSM90TypeGetIntOverflow(self.ptr()).ptr); - } - pub fn aNeg(self: *const Self) bool { - return c.mlirCuteNVGPUMmaAtomSM90TypeGetANeg(self.ptr()); - } - pub fn bNeg(self: *const Self) bool { - return c.mlirCuteNVGPUMmaAtomSM90TypeGetBNeg(self.ptr()); - } -}; - -/// `!cute_nvgpu.sm103.smem_desc_circular`: Circular shared-memory descriptor over a block layout in bytes. -pub const SmemDescCircularSM103Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescCircularSM103; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm103.smem_desc_circular"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - blockLayoutBytes: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSmemDescCircularSM103TypeGet(ctx.ptr(), args.blockLayoutBytes.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn blockLayoutBytes(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM103TypeGetBlockLayoutBytes(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.sm107.smem_desc_circular`: Circular shared-memory descriptor over a block layout in bytes. -pub const SmemDescCircularSM107Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescCircularSM107; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm107.smem_desc_circular"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - blockLayoutBytes: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSmemDescCircularSM107TypeGet(ctx.ptr(), args.blockLayoutBytes.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn blockLayoutBytes(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUSmemDescCircularSM107TypeGetBlockLayoutBytes(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.sm107.smem_desc`: SM107 shared-memory descriptor. -pub const SmemDescSM107Type = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescSM107; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "sm107.smem_desc"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCuteNVGPUSmemDescSM107TypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cute_nvgpu.smem_desc`: UMMA/GMMA shared-memory descriptor. -pub const SmemDescType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDesc; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "smem_desc"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCuteNVGPUSmemDescTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cute_nvgpu.smem_desc_view`: Shared-memory descriptor iterator with a layout. -pub const SmemDescViewType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUSmemDescView; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "smem_desc_view"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - desc: *const mlir.Type, - layout: *const mlir.Attribute, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSmemDescViewTypeGet(ctx.ptr(), args.desc.ptr(), args.layout.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn desc(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetDesc(self.ptr()).ptr.?); - } - pub fn layout(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUSmemDescViewTypeGetLayout(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.tma_descriptor_im2col`: Im2col TMA descriptor. -pub const TmaDescriptorIm2ColType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorIm2Col; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "tma_descriptor_im2col"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCuteNVGPUTmaDescriptorIm2ColTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cute_nvgpu.tma_descriptor_tiled`: Tiled TMA descriptor. -pub const TmaDescriptorTiledType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUTmaDescriptorTiled; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "tma_descriptor_tiled"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCuteNVGPUTmaDescriptorTiledTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!cute_nvgpu.atom.universal_fma`: Universal FMA MMA atom. -pub const UniversalFmaAtomType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUUniversalFmaAtom; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "atom.universal_fma"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct { - shapeMnk: *const mlir.Attribute, - aType: *const mlir.Type, - bType: *const mlir.Type, - cType: *const mlir.Type, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUUniversalFmaAtomTypeGet(ctx.ptr(), args.shapeMnk.ptr(), args.aType.ptr(), args.bType.ptr(), args.cType.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn shapeMnk(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetShapeMnk(self.ptr()).ptr.?); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetAType(self.ptr()).ptr.?); - } - pub fn bType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetBType(self.ptr()).ptr.?); - } - pub fn cType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUUniversalFmaAtomTypeGetCType(self.ptr()).ptr.?); - } -}; - -/// `!cute_nvgpu.workid_response`: Cluster launch control work-id response. -pub const WorkIdResponseType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsACuteNVGPUWorkIdResponse; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "workid_response"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCuteNVGPUWorkIdResponseTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `#cute_nvgpu.atom_copy_field_bulkg2s`: Fields stored in Bulk Load Copy Atom type. -pub const AtomCopyFieldBulkCopyG2SAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyG2S; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_bulkg2s"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldBulkCopyG2S, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldBulkCopyG2S { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyG2SAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_bulks2g`: Fields stored in Bulk Store Copy Atom type. -pub const AtomCopyFieldBulkCopyS2GAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2G; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_bulks2g"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldBulkCopyS2G, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldBulkCopyS2G { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2GAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_bulks2s`: Fields stored in Bulk CTA to Cluster Copy Atom type. -pub const AtomCopyFieldBulkCopyS2SAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldBulkCopyS2S; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_bulks2s"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldBulkCopyS2S, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldBulkCopyS2S { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldBulkCopyS2SAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_dsmem_store`: Fields stored in CopyAtomDsmemStoreType. -pub const AtomCopyFieldDsmemStoreAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldDsmemStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_dsmem_store"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldDsmemStore, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldDsmemStore { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldDsmemStoreAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_g2r`: Fields stored in CopyAtomG2RType. -pub const AtomCopyFieldLoadGlobalAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldLoadGlobal; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_g2r"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldLoadGlobal, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldLoadGlobal { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldLoadGlobalAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_non_exec_2d_gather4_tma_load`: Fields stored in CopyAtomNonExec2DGather4TmaLoadType. -pub const AtomCopyFieldNonExec2DGather4TmaLoadAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_non_exec_2d_gather4_tma_load"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldNonExec2DGather4TmaLoad, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldNonExec2DGather4TmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DGather4TmaLoadAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_non_exec_2d_scatter4_tma_store`: Fields stored in CopyAtomNonExec2DScatter4TmaStoreType. -pub const AtomCopyFieldNonExec2DScatter4TmaStoreAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_non_exec_2d_scatter4_tma_store"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldNonExec2DScatter4TmaStore, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldNonExec2DScatter4TmaStore { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExec2DScatter4TmaStoreAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_load`: Fields stored in CopyAtomNonExecIm2ColTmaLoadType. -pub const AtomCopyFieldNonExecIm2ColTmaLoadAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_non_exec_im2col_tma_load"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldNonExecIm2ColTmaLoad, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldNonExecIm2ColTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaLoadAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_non_exec_im2col_tma_store`: Fields stored in CopyAtomNonExecIm2ColTmaStoreType. -pub const AtomCopyFieldNonExecIm2ColTmaStoreAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecIm2ColTmaStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_non_exec_im2col_tma_store"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldNonExecIm2ColTmaStore, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldNonExecIm2ColTmaStore { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecIm2ColTmaStoreAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_non_exec_tma_load`: Fields stored in CopyAtomNonExecTiledTmaLoadType. -pub const AtomCopyFieldNonExecTiledTmaLoadAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_non_exec_tma_load"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldNonExecTiledTmaLoad, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaLoadAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_non_exec_tma_reduce`: Fields stored in CopyAtomNonExecTiledTmaReduceType. -pub const AtomCopyFieldNonExecTiledTmaReduceAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaReduce; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_non_exec_tma_reduce"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldNonExecTiledTmaReduce, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaReduce { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaReduceAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_non_exec_tma_store`: Fields stored in CopyAtomNonExecTiledTmaStoreType. -pub const AtomCopyFieldNonExecTiledTmaStoreAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldNonExecTiledTmaStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_non_exec_tma_store"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldNonExecTiledTmaStore, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldNonExecTiledTmaStore { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldNonExecTiledTmaStoreAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_r2g`: Fields stored in CopyAtomR2GType. -pub const AtomCopyFieldStoreGlobalAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldStoreGlobal; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_r2g"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldStoreGlobal, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldStoreGlobal { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldStoreGlobalAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_tmaload`: Fields stored in Tma Load Copy Atom type. -pub const AtomCopyFieldTmaLoadAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_tmaload"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldTmaLoad, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaLoadAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_tmareduce`: Fields stored in Tma Reduce Copy Atom type. -pub const AtomCopyFieldTmaReduceAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaReduce; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_tmareduce"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldTmaReduce, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldTmaReduce { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaReduceAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_copy_field_tmastore`: Fields stored in Tma Store Copy Atom type. -pub const AtomCopyFieldTmaStoreAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomCopyFieldTmaStore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_copy_field_tmastore"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomCopyFieldTmaStore, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomCopyFieldTmaStore { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomCopyFieldTmaStoreAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_mma_field_sm100`: Fields stored in MMA Atom types for SM100. -pub const AtomMmaFieldSM100Attr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_mma_field_sm100"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomMmaFieldSM100, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100AttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomMmaFieldSM100 { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100AttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_mma_field_sm100_block_scaled`: Fields stored in MMA Atom types for SM100 (block scaled). -pub const AtomMmaFieldSM100BlockScaledAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaled; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_mma_field_sm100_block_scaled"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomMmaFieldSM100BlockScaled, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomMmaFieldSM100BlockScaled { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_mma_field_sm100_block_scaled_sparse`: Fields stored in MMA Atom types for SM100 (block scaled). -pub const AtomMmaFieldSM100BlockScaledSparseAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100BlockScaledSparse; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_mma_field_sm100_block_scaled_sparse"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomMmaFieldSM100BlockScaledSparse, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomMmaFieldSM100BlockScaledSparse { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100BlockScaledSparseAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_mma_field_sm100_sparse`: Fields stored in sparse MMA Atom types for SM100. -pub const AtomMmaFieldSM100SparseAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM100Sparse; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_mma_field_sm100_sparse"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomMmaFieldSM100Sparse, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomMmaFieldSM100Sparse { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM100SparseAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_mma_field_sm120_block_scaled`: Fields stored in MMA Atom types for SM120 (block scaled). -pub const AtomMmaFieldSM120BlockScaledAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM120BlockScaled; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_mma_field_sm120_block_scaled"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomMmaFieldSM120BlockScaled, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomMmaFieldSM120BlockScaled { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM120BlockScaledAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_mma_field_sm80_sparse`: Fields stored in MMA Atom types for SM80. -pub const AtomMmaFieldSM80SparseAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM80Sparse; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_mma_field_sm80_sparse"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomMmaFieldSM80Sparse, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomMmaFieldSM80Sparse { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM80SparseAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.atom_mma_field_sm90`: Fields stored in MMA Atom types for SM90. -pub const AtomMmaFieldSM90Attr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUAtomMmaFieldSM90; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atom_mma_field_sm90"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomMmaFieldSM90, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUAtomMmaFieldSM90AttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomMmaFieldSM90 { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUAtomMmaFieldSM90AttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.bin_op`: Binary operation for single-bit MMA operations. -pub const BinaryOpAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUBinaryOp; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "bin_op"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: BinaryOp, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUBinaryOpAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) BinaryOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUBinaryOpAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.copy_s2t_broadcast_mode`: Broadcast modes for the different utccp instructions. -pub const CopyS2TBroadcastAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUCopyS2TBroadcast; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "copy_s2t_broadcast_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: CopyS2TBroadcast, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUCopyS2TBroadcastAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) CopyS2TBroadcast { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUCopyS2TBroadcastAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.gather_scatter_tma_load`: The various kinds of TMA loads in gather/scatter mode. -pub const GatherScatterTmaLoadAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUGatherScatterTmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "gather_scatter_tma_load"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: GatherScatterTmaLoad, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUGatherScatterTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) GatherScatterTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUGatherScatterTmaLoadAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.im2col_tma_load`: The various kinds of TMA loads in im2col mode. -pub const Im2ColTmaLoadAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUIm2ColTmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "im2col_tma_load"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Im2ColTmaLoad, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUIm2ColTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Im2ColTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUIm2ColTmaLoadAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.ld_reduce_acc_precision_kind`: multimem ld_reduce accumulation precision kind. -pub const LdReduceAccPrecisionKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPULdReduceAccPrecisionKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "ld_reduce_acc_precision_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: LdReduceAccPrecisionKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) LdReduceAccPrecisionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPULdReduceAccPrecisionKindAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.ldsm_sz_pattern`: LDSM's sz pattern, describing the bit size. -pub const LdsmSzPatternAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPULdsmSzPattern; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "ldsm_sz_pattern"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: LdsmSzPattern, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULdsmSzPatternAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) LdsmSzPattern { - return @fromBackingInt(@intCast(c.mlirCuteNVGPULdsmSzPatternAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.load_cache_mode`: Cache modes for the load instructions. -pub const LoadCacheModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPULoadCacheMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "load_cache_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: LoadCacheMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPULoadCacheModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) LoadCacheMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPULoadCacheModeAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.mma_int_overflow`: MMA overflow options. -pub const MMAIntOverflowAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUMMAIntOverflow; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mma_int_overflow"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MMAIntOverflow, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MMAIntOverflow { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMMAIntOverflowAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.major`: Major mode for MMA operations. -pub const MajorModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUMajorMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "major"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: cute_ir.MajorMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMajorModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMajorModeAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.mma_collector_op`: Enums for the mma collector op. -pub const MmaCollectorOpAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUMmaCollectorOp; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mma_collector_op"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MmaCollectorOp, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaCollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MmaCollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaCollectorOpAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.mma_frag_kind`: Enums for the mma frag type. -pub const MmaFragKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUMmaFragKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mma_frag_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MmaFragKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUMmaFragKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MmaFragKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUMmaFragKindAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.not_implemented_frg`: Fragment of an atom that has none. -pub const NotImplementedFrgAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUNotImplementedFrg; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "not_implemented_frg"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirCuteNVGPUNotImplementedFrgAttrGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `#cute_nvgpu.tma_reduce_kind`: Op for the TMASTORE instruction. -pub const ReductionKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUReductionKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tma_reduce_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ReductionKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUReductionKindAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.rmem_frg`: Register-memory fragment of an MMA operand. -pub const RmemFrgAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPURmemFrg; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "rmem_frg"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - valueType: *const mlir.Type, - operand: cute_ir.MmaOperand, - deriveElemType: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPURmemFrgAttrGet(ctx.ptr(), args.valueType.ptr(), @backingInt(args.operand), args.deriveElemType); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn valueType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPURmemFrgAttrGetValueType(self.ptr()).ptr.?); - } - pub fn operand(self: *const Self) cute_ir.MmaOperand { - return @fromBackingInt(@intCast(c.mlirCuteNVGPURmemFrgAttrGetOperand(self.ptr()))); - } - pub fn deriveElemType(self: *const Self) bool { - return c.mlirCuteNVGPURmemFrgAttrGetDeriveElemType(self.ptr()); - } -}; - -/// `#cute_nvgpu.arch.sm100.circular_smem_frg`: SM100 circular shared-memory fragment. -pub const SM100CircularSmemFrgAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100CircularSmemFrg; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "arch.sm100.circular_smem_frg"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - majorMode: cute_ir.MajorMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100CircularSmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn majorMode(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100CircularSmemFrgAttrGetMajorMode(self.ptr()))); - } -}; - -/// `#cute_nvgpu.arch.sm100.smem_frg`: SM100 shared-memory fragment. -pub const SM100SmemFrgAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100SmemFrg; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "arch.sm100.smem_frg"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - majorMode: cute_ir.MajorMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn majorMode(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100SmemFrgAttrGetMajorMode(self.ptr()))); - } -}; - -/// `#cute_nvgpu.arch.sm100.tmem_e_frg`: SM100 tensor-memory sparse-metadata fragment. -pub const SM100TmemEFrgAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemEFrg; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "arch.sm100.tmem_e_frg"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - aType: *const mlir.Type, - eType: *const mlir.Type, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100TmemEFrgAttrGet(ctx.ptr(), args.aType.ptr(), args.eType.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn aType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUSM100TmemEFrgAttrGetAType(self.ptr()).ptr.?); - } - pub fn eType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUSM100TmemEFrgAttrGetEType(self.ptr()).ptr.?); - } -}; - -/// `#cute_nvgpu.arch.sm100.tmem_frg`: SM100 tensor-memory fragment. -pub const SM100TmemFrgAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemFrg; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "arch.sm100.tmem_frg"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - dataType: *const mlir.Type, - storageType: *const mlir.Type, - ctaGroup: c_int, - tmemAllocMode: TmemAllocMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100TmemFrgAttrGet(ctx.ptr(), args.dataType.ptr(), args.storageType.ptr(), args.ctaGroup, @backingInt(args.tmemAllocMode)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn dataType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetDataType(self.ptr()).ptr.?); - } - pub fn storageType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetStorageType(self.ptr()).ptr.?); - } - pub fn ctaGroup(self: *const Self) c_int { - return c.mlirCuteNVGPUSM100TmemFrgAttrGetCtaGroup(self.ptr()); - } - pub fn tmemAllocMode(self: *const Self) TmemAllocMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemFrgAttrGetTmemAllocMode(self.ptr()))); - } -}; - -/// `#cute_nvgpu.arch.sm100.tmem_sf_frg`: SM100 tensor-memory scale-factor fragment. -pub const SM100TmemSfFrgAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUSM100TmemSfFrg; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "arch.sm100.tmem_sf_frg"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - sfType: *const mlir.Type, - sfVecSize: c_int, - ctaGroup: c_int, - isSfa: bool, - tmemAllocMode: TmemAllocMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM100TmemSfFrgAttrGet(ctx.ptr(), args.sfType.ptr(), args.sfVecSize, args.ctaGroup, args.isSfa, @backingInt(args.tmemAllocMode)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn sfType(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetSfType(self.ptr()).ptr.?); - } - pub fn sfVecSize(self: *const Self) c_int { - return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetSfVecSize(self.ptr()); - } - pub fn ctaGroup(self: *const Self) c_int { - return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetCtaGroup(self.ptr()); - } - pub fn isSfa(self: *const Self) bool { - return c.mlirCuteNVGPUSM100TmemSfFrgAttrGetIsSfa(self.ptr()); - } - pub fn tmemAllocMode(self: *const Self) TmemAllocMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM100TmemSfFrgAttrGetTmemAllocMode(self.ptr()))); - } -}; - -/// `#cute_nvgpu.arch.sm107.smem_frg`: SM107 shared-memory fragment. -pub const SM107SmemFrgAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUSM107SmemFrg; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "arch.sm107.smem_frg"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - majorMode: cute_ir.MajorMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM107SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn majorMode(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM107SmemFrgAttrGetMajorMode(self.ptr()))); - } -}; - -/// `#cute_nvgpu.arch.sm90.smem_frg`: SM90 shared-memory fragment. -pub const SM90SmemFrgAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUSM90SmemFrg; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "arch.sm90.smem_frg"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - majorMode: cute_ir.MajorMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUSM90SmemFrgAttrGet(ctx.ptr(), @backingInt(args.majorMode)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn majorMode(self: *const Self) cute_ir.MajorMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUSM90SmemFrgAttrGetMajorMode(self.ptr()))); - } -}; - -/// `#cute_nvgpu.tiled_tma_load`: The various kinds of TMA loads in tiled mode. -pub const TiledTmaLoadAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUTiledTmaLoad; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tiled_tma_load"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TiledTmaLoad, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTiledTmaLoadAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TiledTmaLoad { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTiledTmaLoadAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.tma_data_format`: Bits 74-71 of the TMA descriptor. -pub const TmaDataFormatAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaDataFormat; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tma_data_format"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TmaDataFormat, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaDataFormatAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TmaDataFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaDataFormatAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.tma_load_mode`: Modes for the TMA load instruction. -pub const TmaLoadModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaLoadMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tma_load_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TmaLoadMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaLoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TmaLoadMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaLoadModeAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.tma_store_mode`: Modes for the TMA store instruction. -pub const TmaStoreModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUTmaStoreMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tma_store_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TmaStoreMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmaStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TmaStoreMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmaStoreModeAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.tmem_alloc_mode`: Modes for the Tmem allocation. -pub const TmemAllocModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUTmemAllocMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tmem_alloc_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TmemAllocMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmemAllocModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TmemAllocMode { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemAllocModeAttrGetValue(self.ptr()))); - } -}; - -/// `#cute_nvgpu.tmem_load_red_op`: Reduce operation for the Tmem load instruction. -pub const TmemLoadRedOpAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVGPUTmemLoadRedOp; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tmem_load_red_op"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TmemLoadRedOp, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVGPUTmemLoadRedOpAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TmemLoadRedOp { - return @fromBackingInt(@intCast(c.mlirCuteNVGPUTmemLoadRedOpAttrGetValue(self.ptr()))); - } -}; - -// Operation builders, generated from the operations' .td. -/// `cute_nvgpu.arch.alloc_rmem`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_alloc_rmem(ctx: *mlir.Context, input: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.alloc_rmem", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{ptr_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.alloc_smem`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_alloc_smem(ctx: *mlir.Context, ptr_type: *const mlir.Type, input: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.alloc_smem", .{ - .results = .{ .flat = &.{ptr_type} }, - .attributes = &.{ - .named(ctx, "input", input), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM100.copy_s2t`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM100_copy_s2t(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, dp: *const mlir.Attribute, bits: *const mlir.Attribute, cta: *const mlir.Attribute, broadcast: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.copy_s2t", .{ - .operands = .{ .flat = &.{ src, dst } }, - .attributes = &.{ - .named(ctx, "dp", dp), - .named(ctx, "bits", bits), - .named(ctx, "cta", cta), - .named(ctx, "broadcast", broadcast), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM100.tma_load`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM100_tma_load( - ctx: *mlir.Context, - src_desc: *const mlir.Value, - dsmem_data_addr: *const mlir.Value, - dsmem_bar_addr: *const mlir.Value, - coord: []const *const mlir.Value, - offsets: []const *const mlir.Value, - mode: *const mlir.Attribute, - num_cta: *const mlir.Attribute, - opts: struct { - multicast_mask: ?*const mlir.Value = null, - cache_policy: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_load", .{ - .operands = .{ .variadic = &.{ - &.{src_desc}, - &.{dsmem_data_addr}, - &.{dsmem_bar_addr}, - coord, - if (opts.multicast_mask) |value| &.{value} else &.{}, - offsets, - if (opts.cache_policy) |value| &.{value} else &.{}, - } }, - .attributes = &.{ - .named(ctx, "mode", mode), - .named(ctx, "num_cta", num_cta), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM100.tma_reduce`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM100_tma_reduce( - ctx: *mlir.Context, - dst_desc: *const mlir.Value, - src_smem_data_addr: *const mlir.Value, - coord: []const *const mlir.Value, - mode: *const mlir.Attribute, - kind: *const mlir.Attribute, - opts: struct { - cache_policy: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_reduce", .{ - .operands = .{ .variadic = &.{ - &.{dst_desc}, - &.{src_smem_data_addr}, - coord, - if (opts.cache_policy) |value| &.{value} else &.{}, - } }, - .attributes = &.{ - .named(ctx, "mode", mode), - .named(ctx, "kind", kind), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM100.tma_store`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM100_tma_store( - ctx: *mlir.Context, - dst_desc: *const mlir.Value, - src_smem_data_addr: *const mlir.Value, - coord: []const *const mlir.Value, - mode: *const mlir.Attribute, - opts: struct { - cache_policy: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tma_store", .{ - .operands = .{ .variadic = &.{ - &.{dst_desc}, - &.{src_smem_data_addr}, - coord, - if (opts.cache_policy) |value| &.{value} else &.{}, - } }, - .attributes = &.{ - .named(ctx, "mode", mode), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM100.tmem_load`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM100_tmem_load( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - num_dp: *const mlir.Attribute, - num_b: *const mlir.Attribute, - num_rep: *const mlir.Attribute, - opts: struct { - pack_16: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "num_dp", num_dp)); - attributes.appendAssumeCapacity(.named(ctx, "num_b", num_b)); - attributes.appendAssumeCapacity(.named(ctx, "num_rep", num_rep)); - if (opts.pack_16) |value| attributes.appendAssumeCapacity(.named(ctx, "pack_16", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tmem_load", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM100.tmem_store`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM100_tmem_store( - ctx: *mlir.Context, - dst: *const mlir.Value, - src: *const mlir.Value, - num_dp: *const mlir.Attribute, - num_b: *const mlir.Attribute, - num_rep: *const mlir.Attribute, - opts: struct { - expand_16: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "num_dp", num_dp)); - attributes.appendAssumeCapacity(.named(ctx, "num_b", num_b)); - attributes.appendAssumeCapacity(.named(ctx, "num_rep", num_rep)); - if (opts.expand_16) |value| attributes.appendAssumeCapacity(.named(ctx, "expand_16", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM100.tmem_store", .{ - .operands = .{ .flat = &.{ dst, src } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM107.tma_load_override`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM107_tma_load_override( - ctx: *mlir.Context, - src_desc: *const mlir.Value, - dsmem_data_addr: *const mlir.Value, - dsmem_bar_addr: *const mlir.Value, - override_addr: *const mlir.Value, - tensor_size: []const *const mlir.Value, - lower_stride: []const *const mlir.Value, - coord: []const *const mlir.Value, - mode: *const mlir.Attribute, - num_cta: *const mlir.Attribute, - opts: struct { - upper_stride: ?*const mlir.Value = null, - multicast_mask: ?*const mlir.Value = null, - cache_policy: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM107.tma_load_override", .{ - .operands = .{ .variadic = &.{ - &.{src_desc}, - &.{dsmem_data_addr}, - &.{dsmem_bar_addr}, - &.{override_addr}, - tensor_size, - lower_stride, - if (opts.upper_stride) |value| &.{value} else &.{}, - coord, - if (opts.multicast_mask) |value| &.{value} else &.{}, - if (opts.cache_policy) |value| &.{value} else &.{}, - } }, - .attributes = &.{ - .named(ctx, "mode", mode), - .named(ctx, "num_cta", num_cta), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM107.tma_store_override`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM107_tma_store_override( - ctx: *mlir.Context, - dst_desc: *const mlir.Value, - src_smem_data_addr: *const mlir.Value, - override_addr: *const mlir.Value, - tensor_size: []const *const mlir.Value, - lower_stride: []const *const mlir.Value, - coord: []const *const mlir.Value, - mode: *const mlir.Attribute, - opts: struct { - upper_stride: ?*const mlir.Value = null, - cache_policy: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM107.tma_store_override", .{ - .operands = .{ .variadic = &.{ - &.{dst_desc}, - &.{src_smem_data_addr}, - &.{override_addr}, - tensor_size, - lower_stride, - if (opts.upper_stride) |value| &.{value} else &.{}, - coord, - if (opts.cache_policy) |value| &.{value} else &.{}, - } }, - .attributes = &.{ - .named(ctx, "mode", mode), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM90.bulk_copy_g2s`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM90_bulk_copy_g2s( - ctx: *mlir.Context, - gmem_data_addr: *const mlir.Value, - dsmem_data_addr: *const mlir.Value, - dsmem_bar_addr: *const mlir.Value, - size: *const mlir.Attribute, - opts: struct { - multicast_mask: ?*const mlir.Value = null, - cache_policy: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", .{ - .operands = .{ .variadic = &.{ - &.{gmem_data_addr}, - &.{dsmem_data_addr}, - &.{dsmem_bar_addr}, - if (opts.multicast_mask) |value| &.{value} else &.{}, - if (opts.cache_policy) |value| &.{value} else &.{}, - } }, - .attributes = &.{ - .named(ctx, "size", size), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2g`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM90_bulk_copy_s2g( - ctx: *mlir.Context, - smem_data_addr: *const mlir.Value, - gmem_data_addr: *const mlir.Value, - size: *const mlir.Attribute, - opts: struct { - byte_mask: ?*const mlir.Value = null, - cache_policy: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", .{ - .operands = .{ .variadic = &.{ - &.{smem_data_addr}, - &.{gmem_data_addr}, - if (opts.byte_mask) |value| &.{value} else &.{}, - if (opts.cache_policy) |value| &.{value} else &.{}, - } }, - .attributes = &.{ - .named(ctx, "size", size), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.copy.SM90.bulk_copy_s2s`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_copy_SM90_bulk_copy_s2s(ctx: *mlir.Context, smem_data_addr: *const mlir.Value, dsmem_data_addr: *const mlir.Value, dsmem_bar_addr: *const mlir.Value, cta_rank: *const mlir.Value, size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", .{ - .operands = .{ .flat = &.{ smem_data_addr, dsmem_data_addr, dsmem_bar_addr, cta_rank } }, - .attributes = &.{ - .named(ctx, "size", size), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.get_dyn_smem`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_get_dyn_smem(ctx: *mlir.Context, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.get_dyn_smem", .{ - .results = .{ .flat = &.{ptr_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.get_dyn_smem_size`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_get_dyn_smem_size(ctx: *mlir.Context, size_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.get_dyn_smem_size", .{ - .results = .{ .flat = &.{size_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.make_warp_uniform`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_make_warp_uniform(ctx: *mlir.Context, val: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.make_warp_uniform", .{ - .operands = .{ .flat = &.{val} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.mma.SM107.umma`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_mma_SM107_umma(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, disable_output_lane: *const mlir.Value, num_cta: *const mlir.Attribute, scale: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma", .{ - .operands = .{ .flat = &.{ a, b, cd, idesc, accum, disable_output_lane } }, - .attributes = &.{ - .named(ctx, "num_cta", num_cta), - .named(ctx, "scale", scale), - .named(ctx, "a_type", a_type), - .named(ctx, "b_type", b_type), - .named(ctx, "shape_k", shape_k), - .named(ctx, "a_collector_op", a_collector_op), - .named(ctx, "b_collector_op", b_collector_op), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.mma.SM107.umma_block_scaled`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_mma_SM107_umma_block_scaled(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, sf_a: *const mlir.Value, sf_b: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, vec_size: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_block_scaled", .{ - .operands = .{ .flat = &.{ a, b, cd, sf_a, sf_b, idesc, accum } }, - .attributes = &.{ - .named(ctx, "num_cta", num_cta), - .named(ctx, "vec_size", vec_size), - .named(ctx, "a_type", a_type), - .named(ctx, "b_type", b_type), - .named(ctx, "shape_k", shape_k), - .named(ctx, "a_collector_op", a_collector_op), - .named(ctx, "b_collector_op", b_collector_op), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_mma_SM107_umma_block_scaled_sparse(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, sf_a: *const mlir.Value, sf_b: *const mlir.Value, e_op: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, vec_size: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse", .{ - .operands = .{ .flat = &.{ a, b, cd, sf_a, sf_b, e_op, idesc, accum } }, - .attributes = &.{ - .named(ctx, "num_cta", num_cta), - .named(ctx, "vec_size", vec_size), - .named(ctx, "a_type", a_type), - .named(ctx, "b_type", b_type), - .named(ctx, "shape_k", shape_k), - .named(ctx, "a_collector_op", a_collector_op), - .named(ctx, "b_collector_op", b_collector_op), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.mma.SM107.umma_sparse`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_mma_SM107_umma_sparse(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, cd: *const mlir.Value, e: *const mlir.Value, idesc: *const mlir.Value, accum: *const mlir.Value, num_cta: *const mlir.Attribute, scale: *const mlir.Attribute, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, shape_k: *const mlir.Attribute, a_collector_op: *const mlir.Attribute, b_collector_op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM107.umma_sparse", .{ - .operands = .{ .flat = &.{ a, b, cd, e, idesc, accum } }, - .attributes = &.{ - .named(ctx, "num_cta", num_cta), - .named(ctx, "scale", scale), - .named(ctx, "a_type", a_type), - .named(ctx, "b_type", b_type), - .named(ctx, "shape_k", shape_k), - .named(ctx, "a_collector_op", a_collector_op), - .named(ctx, "b_collector_op", b_collector_op), - }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.mma.SM120.block_scaled`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_mma_SM120_block_scaled( - ctx: *mlir.Context, - opA: []const *const mlir.Value, - opB: []const *const mlir.Value, - opC: []const *const mlir.Value, - sfA: *const mlir.Value, - sfB: *const mlir.Value, - res_types: []const *const mlir.Type, - shape_MNK: *const mlir.Attribute, - vec_size: *const mlir.Attribute, - a_type: *const mlir.Attribute, - b_type: *const mlir.Attribute, - sf_type: *const mlir.Attribute, - opts: struct { - byte_id_a: ?*const mlir.Value = null, - byte_id_b: ?*const mlir.Value = null, - thread_id_a: ?*const mlir.Attribute = null, - thread_id_b: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "shape_MNK", shape_MNK)); - attributes.appendAssumeCapacity(.named(ctx, "vec_size", vec_size)); - if (opts.thread_id_a) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_a", value)); - if (opts.thread_id_b) |value| attributes.appendAssumeCapacity(.named(ctx, "thread_id_b", value)); - attributes.appendAssumeCapacity(.named(ctx, "a_type", a_type)); - attributes.appendAssumeCapacity(.named(ctx, "b_type", b_type)); - attributes.appendAssumeCapacity(.named(ctx, "sf_type", sf_type)); - return mlir.Operation.make(ctx, "cute_nvgpu.arch.mma.SM120.block_scaled", .{ - .operands = .{ .variadic = &.{ - opA, - opB, - opC, - &.{sfA}, - &.{sfB}, - if (opts.byte_id_a) |value| &.{value} else &.{}, - if (opts.byte_id_b) |value| &.{value} else &.{}, - } }, - .results = .{ .variadic = &.{ - res_types, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.arch.prefetch_tma_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_prefetch_tma_desc(ctx: *mlir.Context, tma_descriptor_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.prefetch_tma_desc", .{ - .operands = .{ .flat = &.{tma_descriptor_ptr} }, - .location = location, - }); -} - -/// `cute_nvgpu.arch.sm100.alloc_tmem`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_sm100_alloc_tmem( - ctx: *mlir.Context, - n_cols: *const mlir.Value, - dst_ptr: *const mlir.Value, - opts: struct { - is_two_cta: ?*const mlir.Attribute = null, - exclusive: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); - if (opts.exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.alloc_tmem", .{ - .operands = .{ .flat = &.{ n_cols, dst_ptr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.arch.sm100.dealloc_tmem`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_sm100_dealloc_tmem( - ctx: *mlir.Context, - tmem_ptr: *const mlir.Value, - n_cols: *const mlir.Value, - opts: struct { - is_two_cta: ?*const mlir.Attribute = null, - exclusive: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); - if (opts.exclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "exclusive", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.dealloc_tmem", .{ - .operands = .{ .flat = &.{ tmem_ptr, n_cols } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_sm100_relinquish_tmem_alloc_permit( - ctx: *mlir.Context, - opts: struct { - is_two_cta: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.is_two_cta) |value| attributes.appendAssumeCapacity(.named(ctx, "is_two_cta", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.arch.sm100.retrieve_tmem_ptr`. Result types are explicit; attributes use mlir.Attribute. -pub fn arch_sm100_retrieve_tmem_ptr(ctx: *mlir.Context, smem_ptr: *const mlir.Value, tmem_ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.arch.sm100.retrieve_tmem_ptr", .{ - .operands = .{ .flat = &.{smem_ptr} }, - .results = .{ .flat = &.{tmem_ptr_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.atom.get_coord_tensor`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_get_coord_tensor( - ctx: *mlir.Context, - atom: *const mlir.Value, - shape: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - offset_b: ?*const mlir.Value = null, - layout_b: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_coord_tensor", .{ - .operands = .{ .variadic = &.{ - &.{atom}, - &.{shape}, - if (opts.offset_b) |value| &.{value} else &.{}, - if (opts.layout_b) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.atom.get_copy_s2t_smem_desc_view`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_get_copy_s2t_smem_desc_view(ctx: *mlir.Context, atom: *const mlir.Value, view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_copy_s2t_smem_desc_view", .{ - .operands = .{ .flat = &.{ atom, view } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.atom.get_value`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_get_value(ctx: *mlir.Context, atom: *const mlir.Value, result_type: *const mlir.Type, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.atom.get_value", .{ - .operands = .{ .flat = &.{atom} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "field", field), - }, - .location = location, - }); -} - -/// `cute_nvgpu.atom.make_exec_tma`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_exec_tma( - ctx: *mlir.Context, - input: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - override: ?*const mlir.Attribute = null, - no_fully_oob_tile: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.override) |value| attributes.appendAssumeCapacity(.named(ctx, "override", value)); - if (opts.no_fully_oob_tile) |value| attributes.appendAssumeCapacity(.named(ctx, "no_fully_oob_tile", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_exec_tma", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_2d_gather4_tma_load( - ctx: *mlir.Context, - gmem_tensor: *const mlir.Value, - gmem_coord_layout: *const mlir.Value, - smem_layout: *const mlir.Value, - cta_v_map: *const mlir.Value, - non_exec_atom_type: *const mlir.Type, - tma_tensor_type: *const mlir.Type, - kind: *const mlir.Attribute, - opts: struct { - num_multicast: ?*const mlir.Attribute = null, - tma_format: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", .{ - .operands = .{ .flat = &.{ gmem_tensor, gmem_coord_layout, smem_layout, cta_v_map } }, - .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_2d_scatter4_tma_store( - ctx: *mlir.Context, - gmem_tensor: *const mlir.Value, - gmem_index_layout: *const mlir.Value, - smem_layout: *const mlir.Value, - cta_v_map: *const mlir.Value, - non_exec_atom_type: *const mlir.Type, - tma_tensor_type: *const mlir.Type, - opts: struct { - tma_format: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store", .{ - .operands = .{ .flat = &.{ gmem_tensor, gmem_index_layout, smem_layout, cta_v_map } }, - .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.atom.make_non_exec_im2col_tma_load`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_im2col_tma_load( - ctx: *mlir.Context, - gmem_tensor: *const mlir.Value, - smem_layout: *const mlir.Value, - cta_v_map: *const mlir.Value, - lower_corner_whd: *const mlir.Value, - upper_corner_whd: *const mlir.Value, - lower_padding_whd: *const mlir.Value, - upper_padding_whd: *const mlir.Value, - stride_whd: *const mlir.Value, - lower_srt: *const mlir.Value, - stride_srt: *const mlir.Value, - non_exec_atom_type: *const mlir.Type, - tma_tensor_type: *const mlir.Type, - kind: *const mlir.Attribute, - opts: struct { - num_multicast: ?*const mlir.Attribute = null, - tma_format: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_im2col_tma_load", .{ - .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt } }, - .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.atom.make_non_exec_im2col_tma_store`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_im2col_tma_store( - ctx: *mlir.Context, - gmem_tensor: *const mlir.Value, - smem_layout: *const mlir.Value, - cta_v_map: *const mlir.Value, - non_exec_atom_type: *const mlir.Type, - tma_tensor_type: *const mlir.Type, - opts: struct { - tma_format: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_im2col_tma_store", .{ - .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, - .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.atom.make_non_exec_tiled_tma_load`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_tiled_tma_load( - ctx: *mlir.Context, - gmem_tensor: *const mlir.Value, - smem_layout: *const mlir.Value, - cta_v_map: *const mlir.Value, - non_exec_atom_type: *const mlir.Type, - tma_tensor_type: *const mlir.Type, - kind: *const mlir.Attribute, - opts: struct { - num_multicast: ?*const mlir.Attribute = null, - tma_format: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_load", .{ - .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, - .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.atom.make_non_exec_tiled_tma_reduce`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_tiled_tma_reduce( - ctx: *mlir.Context, - gmem_tensor: *const mlir.Value, - smem_layout: *const mlir.Value, - cta_v_map: *const mlir.Value, - non_exec_atom_type: *const mlir.Type, - tma_tensor_type: *const mlir.Type, - kind: *const mlir.Attribute, - opts: struct { - tma_format: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", .{ - .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, - .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.atom.make_non_exec_tiled_tma_store`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_non_exec_tiled_tma_store( - ctx: *mlir.Context, - gmem_tensor: *const mlir.Value, - smem_layout: *const mlir.Value, - cta_v_map: *const mlir.Value, - non_exec_atom_type: *const mlir.Type, - tma_tensor_type: *const mlir.Type, - opts: struct { - tma_format: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_non_exec_tiled_tma_store", .{ - .operands = .{ .flat = &.{ gmem_tensor, smem_layout, cta_v_map } }, - .results = .{ .flat = &.{ non_exec_atom_type, tma_tensor_type } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.atom.make_s2t_copy`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_s2t_copy(ctx: *mlir.Context, copy_s2t_atom: *const mlir.Value, tmem_memref: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_s2t_copy", .{ - .operands = .{ .flat = &.{ copy_s2t_atom, tmem_memref } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.atom.make_tma_residue_tensor`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_tma_residue_tensor(ctx: *mlir.Context, tma_atom: *const mlir.Value, gmem_tensor: *const mlir.Value, residue_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_tma_residue_tensor", .{ - .operands = .{ .flat = &.{ tma_atom, gmem_tensor } }, - .results = .{ .flat = &.{residue_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.atom.make_tmem_copy`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_make_tmem_copy(ctx: *mlir.Context, atom: *const mlir.Value, tmem_memref: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.atom.make_tmem_copy", .{ - .operands = .{ .flat = &.{ atom, tmem_memref } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.atom.set_value`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_set_value(ctx: *mlir.Context, atom: *const mlir.Value, value_: *const mlir.Value, result_type: *const mlir.Type, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.atom.set_value", .{ - .operands = .{ .flat = &.{ atom, value_ } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "field", field), - }, - .location = location, - }); -} - -/// `cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_sm107_get_copy_s2t_smem_desc_view(ctx: *mlir.Context, atom: *const mlir.Value, view: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view", .{ - .operands = .{ .flat = &.{ atom, view } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.atom.tma_partition`. Result types are explicit; attributes use mlir.Attribute. -pub fn atom_tma_partition(ctx: *mlir.Context, tma_atom: *const mlir.Value, cta_coord: *const mlir.Value, cta_layout: *const mlir.Value, smem_tensor: *const mlir.Value, target_tensors: []const *const mlir.Value, res_smem_tensor_type: *const mlir.Type, res_target_tensors_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.atom.tma_partition", .{ - .operands = .{ .variadic = &.{ - &.{tma_atom}, - &.{cta_coord}, - &.{cta_layout}, - &.{smem_tensor}, - target_tensors, - } }, - .results = .{ .variadic = &.{ - &.{res_smem_tensor_type}, - res_target_tensors_types, - } }, - .location = location, - }); -} - -/// `cute_nvgpu.cast_tma_desc_to_integer`. Result types are explicit; attributes use mlir.Attribute. -pub fn cast_tma_desc_to_integer(ctx: *mlir.Context, src: *const mlir.Value, dst_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.cast_tma_desc_to_integer", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.copy_tma_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn copy_tma_desc(ctx: *mlir.Context, atom: *const mlir.Value, tma_desc_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.copy_tma_desc", .{ - .operands = .{ .flat = &.{ atom, tma_desc_ptr } }, - .location = location, - }); -} - -/// `cute_nvgpu.get_grid_constant_pointer`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_grid_constant_pointer(ctx: *mlir.Context, input: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.get_grid_constant_pointer", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.get_tma_desc_addr`. Result types are explicit; attributes use mlir.Attribute. -pub fn get_tma_desc_addr(ctx: *mlir.Context, src: *const mlir.Value, ptr_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.get_tma_desc_addr", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{ptr_type} }, - .location = location, - }); -} - -/// `cute_nvgpu.make_gmma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_gmma_smem_desc(ctx: *mlir.Context, src: *const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, major: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.make_gmma_smem_desc", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "layout", layout), - .named(ctx, "major", major), - }, - .location = location, - }); -} - -/// `cute_nvgpu.make_tma_desc_im2col`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tma_desc_im2col( - ctx: *mlir.Context, - gmem_view: *const mlir.Value, - smem_layout: *const mlir.Value, - cta_value_tile: *const mlir.Value, - lower_corner_whd: *const mlir.Value, - upper_corner_whd: *const mlir.Value, - lower_padding_whd: *const mlir.Value, - upper_padding_whd: *const mlir.Value, - stride_whd: *const mlir.Value, - lower_srt: *const mlir.Value, - stride_srt: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - num_multicast: ?*const mlir.Attribute = null, - tma_format: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col", .{ - .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt } }, - .results = .{ .flat = &.{result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.make_tma_desc_im2col_at`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tma_desc_im2col_at( - ctx: *mlir.Context, - gmem_view: *const mlir.Value, - smem_layout: *const mlir.Value, - cta_value_tile: *const mlir.Value, - lower_corner_whd: *const mlir.Value, - upper_corner_whd: *const mlir.Value, - lower_padding_whd: *const mlir.Value, - upper_padding_whd: *const mlir.Value, - stride_whd: *const mlir.Value, - lower_srt: *const mlir.Value, - stride_srt: *const mlir.Value, - tma_desc_addr: *const mlir.Value, - opts: struct { - num_multicast: ?*const mlir.Attribute = null, - tma_format: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_im2col_at", .{ - .operands = .{ .flat = &.{ gmem_view, smem_layout, cta_value_tile, lower_corner_whd, upper_corner_whd, lower_padding_whd, upper_padding_whd, stride_whd, lower_srt, stride_srt, tma_desc_addr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.make_tma_desc_tiled`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tma_desc_tiled( - ctx: *mlir.Context, - gmem_view: *const mlir.Value, - smem_layout: *const mlir.Value, - cta_value_tile: *const mlir.Value, - result_type: *const mlir.Type, - mode: *const mlir.Attribute, - opts: struct { - traversal_stride: ?*const mlir.Value = null, - num_multicast: ?*const mlir.Attribute = null, - tma_format: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); - if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled", .{ - .operands = .{ .variadic = &.{ - &.{gmem_view}, - &.{smem_layout}, - &.{cta_value_tile}, - if (opts.traversal_stride) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.make_tma_desc_tiled_at`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_tma_desc_tiled_at( - ctx: *mlir.Context, - gmem_view: *const mlir.Value, - smem_layout: *const mlir.Value, - cta_value_tile: *const mlir.Value, - tma_desc_addr: *const mlir.Value, - mode: *const mlir.Attribute, - opts: struct { - traversal_stride: ?*const mlir.Value = null, - num_multicast: ?*const mlir.Attribute = null, - tma_format: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); - if (opts.num_multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "num_multicast", value)); - if (opts.tma_format) |value| attributes.appendAssumeCapacity(.named(ctx, "tma_format", value)); - return mlir.Operation.make(ctx, "cute_nvgpu.make_tma_desc_tiled_at", .{ - .operands = .{ .variadic = &.{ - &.{gmem_view}, - &.{smem_layout}, - &.{cta_value_tile}, - &.{tma_desc_addr}, - if (opts.traversal_stride) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `cute_nvgpu.make_umma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn make_umma_smem_desc( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - layout: *const mlir.Attribute, - major: *const mlir.Attribute, - opts: struct { - nextSrc: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.make_umma_smem_desc", .{ - .operands = .{ .variadic = &.{ - &.{src}, - if (opts.nextSrc) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "layout", layout), - .named(ctx, "major", major), - }, - .location = location, - }); -} - -/// `cute_nvgpu.prefetch_tma_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn prefetch_tma_desc(ctx: *mlir.Context, atom: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.prefetch_tma_desc", .{ - .operands = .{ .flat = &.{atom} }, - .location = location, - }); -} - -/// `cute_nvgpu.sm103.make_umma_smem_desc_circular`. Result types are explicit; attributes use mlir.Attribute. -pub fn sm103_make_umma_smem_desc_circular( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - layout: *const mlir.Attribute, - major: *const mlir.Attribute, - opts: struct { - nextSrc: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.sm103.make_umma_smem_desc_circular", .{ - .operands = .{ .variadic = &.{ - &.{src}, - if (opts.nextSrc) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "layout", layout), - .named(ctx, "major", major), - }, - .location = location, - }); -} - -/// `cute_nvgpu.sm107.make_umma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn sm107_make_umma_smem_desc( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - layout: *const mlir.Attribute, - major: *const mlir.Attribute, - opts: struct { - nextSrc: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.sm107.make_umma_smem_desc", .{ - .operands = .{ .variadic = &.{ - &.{src}, - if (opts.nextSrc) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "layout", layout), - .named(ctx, "major", major), - }, - .location = location, - }); -} - -/// `cute_nvgpu.sm107.make_umma_smem_desc_circular`. Result types are explicit; attributes use mlir.Attribute. -pub fn sm107_make_umma_smem_desc_circular( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - layout: *const mlir.Attribute, - major: *const mlir.Attribute, - opts: struct { - nextSrc: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.sm107.make_umma_smem_desc_circular", .{ - .operands = .{ .variadic = &.{ - &.{src}, - if (opts.nextSrc) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "layout", layout), - .named(ctx, "major", major), - }, - .location = location, - }); -} - -/// `cute_nvgpu.update_tma_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn update_tma_desc(ctx: *mlir.Context, tma_atom: *const mlir.Value, gmem_tensor: *const mlir.Value, tma_desc_ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "cute_nvgpu.update_tma_desc", .{ - .operands = .{ .flat = &.{ tma_atom, gmem_tensor, tma_desc_ptr } }, - .location = location, - }); -} diff --git a/mlir/dialects/cute_nvgpu/test.zig b/mlir/dialects/cute_nvgpu/test.zig deleted file mode 100644 index 3c7de6ce..00000000 --- a/mlir/dialects/cute_nvgpu/test.zig +++ /dev/null @@ -1,219 +0,0 @@ -const std = @import("std"); - -const cute_ir = @import("mlir/dialects/cute_ir"); -const cute_nvgpu = @import("mlir/dialects/cute_nvgpu"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "cute_nvgpu.arch.alloc_rmem", - "cute_nvgpu.arch.alloc_smem", - "cute_nvgpu.arch.copy.SM100.copy_s2t", - "cute_nvgpu.arch.copy.SM100.tma_load", - "cute_nvgpu.arch.copy.SM100.tma_reduce", - "cute_nvgpu.arch.copy.SM100.tma_store", - "cute_nvgpu.arch.copy.SM100.tmem_load", - "cute_nvgpu.arch.copy.SM100.tmem_store", - "cute_nvgpu.arch.copy.SM107.tma_load_override", - "cute_nvgpu.arch.copy.SM107.tma_store_override", - "cute_nvgpu.arch.copy.SM90.bulk_copy_g2s", - "cute_nvgpu.arch.copy.SM90.bulk_copy_s2g", - "cute_nvgpu.arch.copy.SM90.bulk_copy_s2s", - "cute_nvgpu.arch.get_dyn_smem", - "cute_nvgpu.arch.get_dyn_smem_size", - "cute_nvgpu.arch.make_warp_uniform", - "cute_nvgpu.arch.mma.SM107.umma", - "cute_nvgpu.arch.mma.SM107.umma_block_scaled", - "cute_nvgpu.arch.mma.SM107.umma_block_scaled_sparse", - "cute_nvgpu.arch.mma.SM107.umma_sparse", - "cute_nvgpu.arch.mma.SM120.block_scaled", - "cute_nvgpu.arch.prefetch_tma_desc", - "cute_nvgpu.arch.sm100.alloc_tmem", - "cute_nvgpu.arch.sm100.dealloc_tmem", - "cute_nvgpu.arch.sm100.relinquish_tmem_alloc_permit", - "cute_nvgpu.arch.sm100.retrieve_tmem_ptr", - "cute_nvgpu.atom.get_coord_tensor", - "cute_nvgpu.atom.get_copy_s2t_smem_desc_view", - "cute_nvgpu.atom.get_value", - "cute_nvgpu.atom.make_exec_tma", - "cute_nvgpu.atom.make_non_exec_2d_gather4_tma_load", - "cute_nvgpu.atom.make_non_exec_2d_scatter4_tma_store", - "cute_nvgpu.atom.make_non_exec_im2col_tma_load", - "cute_nvgpu.atom.make_non_exec_im2col_tma_store", - "cute_nvgpu.atom.make_non_exec_tiled_tma_load", - "cute_nvgpu.atom.make_non_exec_tiled_tma_reduce", - "cute_nvgpu.atom.make_non_exec_tiled_tma_store", - "cute_nvgpu.atom.make_s2t_copy", - "cute_nvgpu.atom.make_tma_residue_tensor", - "cute_nvgpu.atom.make_tmem_copy", - "cute_nvgpu.atom.set_value", - "cute_nvgpu.atom.sm107.get_copy_s2t_smem_desc_view", - "cute_nvgpu.atom.tma_partition", - "cute_nvgpu.cast_tma_desc_to_integer", - "cute_nvgpu.copy_tma_desc", - "cute_nvgpu.get_grid_constant_pointer", - "cute_nvgpu.get_tma_desc_addr", - "cute_nvgpu.make_gmma_smem_desc", - "cute_nvgpu.make_tma_desc_im2col", - "cute_nvgpu.make_tma_desc_im2col_at", - "cute_nvgpu.make_tma_desc_tiled", - "cute_nvgpu.make_tma_desc_tiled_at", - "cute_nvgpu.make_umma_smem_desc", - "cute_nvgpu.prefetch_tma_desc", - "cute_nvgpu.sm103.make_umma_smem_desc_circular", - "cute_nvgpu.sm107.make_umma_smem_desc", - "cute_nvgpu.sm107.make_umma_smem_desc_circular", - "cute_nvgpu.update_tma_desc", -}; - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - cute_ir.registerDialect(registry); - cute_nvgpu.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(cute_nvgpu); - inline for (comptime std.meta.declarations(cute_nvgpu)) |decl| { - const T = @field(cute_nvgpu, decl); - if (@TypeOf(T) == type) std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const ctx = try testContext(); - defer ctx.deinit(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} - -test "grid_constant is a unit attribute of the cute_nvgpu dialect" { - const ctx = try testContext(); - defer ctx.deinit(); - var buf: [64]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{cute_nvgpu.gridConstant(ctx)})}); - try std.testing.expectEqualStrings("{cute_nvgpu.grid_constant}", w.buffered()); -} - -test "generated types and attributes rebuild from their getters" { - @setEvalBranchQuota(100_000); - const ctx = try testContext(); - defer ctx.deinit(); - inline for (.{ - .{ cute_nvgpu.CopyAtomBulkCopyG2SType, "!cute_nvgpu.atom.bulk_copy_g2s" }, - .{ cute_nvgpu.CopyAtomBulkCopyS2GType, "!cute_nvgpu.atom.bulk_copy_s2g" }, - .{ cute_nvgpu.CopyAtomBulkCopyS2SType, "!cute_nvgpu.atom.bulk_copy_s2s" }, - .{ cute_nvgpu.CopyAtomDsmemStoreType, "!cute_nvgpu.atom.dsmem_store" }, - .{ cute_nvgpu.CopyAtomG2RType, "!cute_nvgpu.atom.g2r" }, - .{ cute_nvgpu.CopyAtomIm2ColTmaLoadType, "!cute_nvgpu.atom.im2col_tma_load mcast = true tma_gbasis = <\"(64,128):(1@1,1@0)\">>" }, - .{ cute_nvgpu.CopyAtomIm2ColTmaStoreType, "!cute_nvgpu.atom.im2col_tma_store tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, - .{ cute_nvgpu.CopyAtomLdsmType, "!cute_nvgpu.atom.ldsm, sz_pattern = u16, num_matrices = 4, t>" }, - .{ cute_nvgpu.CopyAtomNonExec2DGather4TmaLoadType, "!cute_nvgpu.atom.non_exec_2d_gather4_tma_load, tma_format = F16_RN>" }, - .{ cute_nvgpu.CopyAtomNonExec2DScatter4TmaStoreType, "!cute_nvgpu.atom.non_exec_2d_scatter4_tma_store, tma_format = F16_RN>" }, - .{ cute_nvgpu.CopyAtomNonExecIm2ColTmaLoadType, "!cute_nvgpu.atom.non_exec_im2col_tma_load, tma_format = F16_RN>" }, - .{ cute_nvgpu.CopyAtomNonExecIm2ColTmaStoreType, "!cute_nvgpu.atom.non_exec_im2col_tma_store, tma_format = F16_RN>" }, - .{ cute_nvgpu.CopyAtomNonExecTiledTmaLoadType, "!cute_nvgpu.atom.non_exec_tiled_tma_load, tma_format = BF16_RN>" }, - .{ cute_nvgpu.CopyAtomNonExecTiledTmaReduceType, "!cute_nvgpu.atom.non_exec_tiled_tma_reduce, tma_format = F32_RN>" }, - .{ cute_nvgpu.CopyAtomNonExecTiledTmaStoreType, "!cute_nvgpu.atom.non_exec_tiled_tma_store, tma_format = BF16_RN>" }, - .{ cute_nvgpu.CopyAtomR2GType, "!cute_nvgpu.atom.r2g" }, - .{ cute_nvgpu.CopyAtomR2SType, "!cute_nvgpu.atom.r2s" }, - .{ cute_nvgpu.CopyAtomS2RType, "!cute_nvgpu.atom.s2r" }, - .{ cute_nvgpu.CopyAtomSIMTAsyncCopyType, "!cute_nvgpu.atom.simt_async_copy" }, - .{ cute_nvgpu.CopyAtomSIMTMultimemLdReduceType, "!cute_nvgpu.atom.simt_multimem_ld_reduce, acquire, gpu>" }, - .{ cute_nvgpu.CopyAtomSIMTMultimemRedType, "!cute_nvgpu.atom.simt_multimem_red" }, - .{ cute_nvgpu.CopyAtomSIMTMultimemStType, "!cute_nvgpu.atom.simt_multimem_st" }, - .{ cute_nvgpu.CopyAtomSIMTSyncCopyType, "!cute_nvgpu.atom.universal_copy" }, - .{ cute_nvgpu.CopyAtomSM100CopyS2TType, "!cute_nvgpu.atom.s2t_copy" }, - .{ cute_nvgpu.CopyAtomSM100S2TCopyV2Type, "!cute_nvgpu.atom.sm100_s2t_copy_v2" }, - .{ cute_nvgpu.CopyAtomSM100TmemLoadType, "!cute_nvgpu.atom.tmem_load" }, - .{ cute_nvgpu.CopyAtomSM100TmemStoreType, "!cute_nvgpu.atom.tmem_store" }, - .{ cute_nvgpu.CopyAtomSM107TmemLoadSPCompressType, "!cute_nvgpu.atom.tmem_load_spcompress" }, - .{ cute_nvgpu.CopyAtomSM10xTmemLoadRedType, "!cute_nvgpu.atom.tmem_load_red" }, - .{ cute_nvgpu.CopyAtomStsmType, "!cute_nvgpu.atom.stsm, num_matrices = 4, n>" }, - .{ cute_nvgpu.CopyAtomTmaLoadType, "!cute_nvgpu.atom.tma_load mcast = true tma_gbasis = <\"(64,1):(1@0,1@1)\">>" }, - .{ cute_nvgpu.CopyAtomTmaReduceType, "!cute_nvgpu.atom.tma_reduce tma_gbasis = <\"(32,128):(1@1,1@0)\">>" }, - .{ cute_nvgpu.CopyAtomTmaStoreType, "!cute_nvgpu.atom.tma_store tma_gbasis = <\"(64,16):(1@0,1@1)\">>" }, - .{ cute_nvgpu.MmaAtomSM100UMMABlockScaledSparseType, "!cute_nvgpu.sm100.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32>" }, - .{ cute_nvgpu.MmaAtomSM100UMMABlockScaledType, "!cute_nvgpu.sm100.mma_bs<128x128x96, num_cta = 1, ab_major = (k, k), elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 16, arch_promote = sm_103>" }, - .{ cute_nvgpu.MmaAtomSM100UMMASparseType, "!cute_nvgpu.sm100.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0>" }, - .{ cute_nvgpu.MmaAtomSM100UMMAType, "!cute_nvgpu.sm100.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0>" }, - .{ cute_nvgpu.MmaAtomSM107UMMABlockScaledSparseType, "!cute_nvgpu.sm107.mma_bs_sp<128x128x64, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, sparse_metadata_format = tid, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, discard)>" }, - .{ cute_nvgpu.MmaAtomSM107UMMABlockScaledType, "!cute_nvgpu.sm107.mma_bs<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f8E4M3FN, f8E4M3FN, f32), sf_type = f8E8M0FNU, frag_kind = ss, vec_size = 32, ab_collector_op = (discard, use), int_overflow = >" }, - .{ cute_nvgpu.MmaAtomSM107UMMASparseType, "!cute_nvgpu.sm107.mma_sp<128x128x32, num_cta = 1, ab_major = (k, k), elem_type = (f16, f16, f32), e_type = i8, sparse_metadata_format = tid, frag_kind = ss, c_scale_exp = 0, ab_collector_op = (discard, discard)>" }, - .{ cute_nvgpu.MmaAtomSM107UMMAType, "!cute_nvgpu.sm107.mma<128x128x16, num_cta = 1, ab_major = (k, mn), elem_type = (f16, f16, f32), frag_kind = ss, c_scale_exp = 0, ab_collector_op = (lastuse, fill)>" }, - .{ cute_nvgpu.MmaAtomSM120BlockScaledType, "!cute_nvgpu.SM120.mma_bs<16x8x64, vec_size = 32, elem_type = (f4E2M1FN, f4E2M1FN, f32), sf_type = f8E8M0FNU, use_sf_layout_TV = true>" }, - .{ cute_nvgpu.MmaAtomSM80SparseType, "!cute_nvgpu.sm80.sparse_mma<16x8x64, elem_type = (i8, i8, i32), md_format = tid, int_overflow = >" }, - .{ cute_nvgpu.MmaAtomSM80Type, "!cute_nvgpu.sm80.mma<16x8x32, elem_type = (i8, i8, i32), int_overflow = , binary_op = >" }, - .{ cute_nvgpu.MmaAtomSM89Type, "!cute_nvgpu.sm89.mma<16x8x32, elem_type = (f8E4M3FN, f8E4M3FN, f32) >" }, - .{ cute_nvgpu.MmaAtomSM90Type, "!cute_nvgpu.sm90.mma<64x128x32, ab_major = (k, k), elem_type = (i8, i8, i32), frag_kind = ss, int_overflow = , a_neg, b_neg>" }, - .{ cute_nvgpu.SmemDescCircularSM103Type, "!cute_nvgpu.sm103.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, - .{ cute_nvgpu.SmemDescCircularSM107Type, "!cute_nvgpu.sm107.smem_desc_circular<<\"(8,128,2):(128,1,1024)\">>" }, - .{ cute_nvgpu.SmemDescSM107Type, "!cute_nvgpu.sm107.smem_desc" }, - .{ cute_nvgpu.SmemDescType, "!cute_nvgpu.smem_desc" }, - .{ cute_nvgpu.SmemDescViewType, "!cute_nvgpu.smem_desc_view" }, - .{ cute_nvgpu.TmaDescriptorIm2ColType, "!cute_nvgpu.tma_descriptor_im2col" }, - .{ cute_nvgpu.TmaDescriptorTiledType, "!cute_nvgpu.tma_descriptor_tiled" }, - .{ cute_nvgpu.UniversalFmaAtomType, "!cute_nvgpu.atom.universal_fma<1x1x1, (f32, f32) -> f32 >" }, - .{ cute_nvgpu.WorkIdResponseType, "!cute_nvgpu.workid_response" }, - .{ cute_nvgpu.NotImplementedFrgAttr, "#cute_nvgpu.not_implemented_frg<>" }, - .{ cute_nvgpu.RmemFrgAttr, "#cute_nvgpu.rmem_frg" }, - .{ cute_nvgpu.SM100CircularSmemFrgAttr, "#cute_nvgpu.arch.sm100.circular_smem_frg" }, - .{ cute_nvgpu.SM100SmemFrgAttr, "#cute_nvgpu.arch.sm100.smem_frg" }, - .{ cute_nvgpu.SM100TmemEFrgAttr, "#cute_nvgpu.arch.sm100.tmem_e_frg" }, - .{ cute_nvgpu.SM100TmemFrgAttr, "#cute_nvgpu.arch.sm100.tmem_frg" }, - .{ cute_nvgpu.SM100TmemSfFrgAttr, "#cute_nvgpu.arch.sm100.tmem_sf_frg" }, - .{ cute_nvgpu.SM107SmemFrgAttr, "#cute_nvgpu.arch.sm107.smem_frg" }, - .{ cute_nvgpu.SM90SmemFrgAttr, "#cute_nvgpu.arch.sm90.smem_frg" }, - }) |example| { - const T = example[0]; - const is_type = @hasDecl(T, "type_"); - const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); - const value = parsed.isA(T) orelse return error.TestUnexpectedResult; - try std.testing.expect((try rebuild(T, ctx, value)).eql(value)); - var text: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer text.deinit(); - try text.writer.print("{f}", .{value}); - try std.testing.expect(parsed.eql(if (is_type) try mlir.Type.parse(ctx, text.written()) else try mlir.Attribute.parse(ctx, text.written()))); - } - // Every case of every enum attribute. - inline for (.{cute_nvgpu}) |namespace| { - inline for (comptime std.meta.declarations(namespace)) |decl| { - const T = @field(namespace, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque" and @hasDecl(T, "InitArgs")) { - const fields = @typeInfo(T.InitArgs).@"struct".field_names; - if (fields.len == 1 and comptime std.mem.eql(u8, fields[0], "value")) { - inline for (@typeInfo(@FieldType(T.InitArgs, fields[0])).@"enum".field_names) |case| { - const case_value = @field(@FieldType(T.InitArgs, fields[0]), case); - const attr = try T.get(ctx, .{ .value = case_value }); - try std.testing.expectEqual(@backingInt(case_value), @backingInt(attr.value())); - try std.testing.expect((try rebuild(T, ctx, attr)).eql(attr)); - } - } - } - } - } - try std.testing.expect((try cute_nvgpu.SmemDescType.get(ctx, .{})).type_().isA(cute_nvgpu.NVGPUType) != null); -} - -fn rebuild(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { - var args: T.InitArgs = undefined; - var elements: [8]*const mlir.Type = undefined; - inline for (@typeInfo(T.InitArgs).@"struct".field_names) |field| { - const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; - const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; - const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; - if (@FieldType(T.InitArgs, field) == []const *const mlir.Type) { - const count = @field(T, "num" ++ name)(value); - for (elements[0..count], 0..) |*element, i| element.* = @field(T, noun[0 .. noun.len - 1])(value, i); - @field(args, field) = elements[0..count]; - } else { - @field(args, field) = @field(T, accessor)(value); - } - } - return T.get(ctx, args); -} diff --git a/mlir/dialects/dialects.zig b/mlir/dialects/dialects.zig new file mode 100644 index 00000000..3deac104 --- /dev/null +++ b/mlir/dialects/dialects.zig @@ -0,0 +1,8 @@ +const std = @import("std"); + +pub const stablehlo = @import("mlir/dialects/stablehlo"); +pub const func = @import("func.zig"); + +test { + std.testing.refAllDecls(@This()); +} diff --git a/mlir/dialects/fly/BUILD.bazel b/mlir/dialects/fly/BUILD.bazel deleted file mode 100644 index 87fe2405..00000000 --- a/mlir/dialects/fly/BUILD.bazel +++ /dev/null @@ -1,56 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -cc_library( - name = "capi", - srcs = [ - "capi.cc", - ], - hdrs = [ - "capi.h", - ], - tags = ["manual"], - deps = [ - "@flydsl//:FlyDialect", - "@flydsl//:TiledOpTraits", - "@llvm-project//mlir:CAPIIR", - ], -) - -zig_library( - name = "fly", - srcs = [ - "attributes.zig", - "ops.zig", - "types.zig", - ], - import_name = "mlir/dialects/fly", - main = "fly.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":fly", - "//mlir", - "//mlir/dialects/arith", - "//mlir/dialects/fly_rocdl", - "//mlir/dialects/func", - "//mlir/dialects/gpu", - "//mlir/dialects/llvm", - "//mlir/dialects/math", - "//mlir/dialects/memref", - "//mlir/dialects/rocdl", - "//mlir/dialects/scf", - "//mlir/dialects/ub", - "//mlir/dialects/vector", - ], -) diff --git a/mlir/dialects/fly/REGENERATING.md b/mlir/dialects/fly/REGENERATING.md deleted file mode 100644 index 9673870f..00000000 --- a/mlir/dialects/fly/REGENERATING.md +++ /dev/null @@ -1,79 +0,0 @@ -# Regenerating the operation builders - -These are instructions for an agent. They describe what `ops.zig` and -`../fly_rocdl/ops.zig` must contain after a FlyDSL bump, where to derive them from, -and how to check the result. Scripts and intermediate files are not -committed; only the generated builders are. - -Paths below are relative to this directory. - -## Goal - -`ops.zig` binds every operation of the `fly` dialect and `../fly_rocdl/ops.zig` every -operation of the `fly_rocdl` dialect, one builder per operation, so that no -code builds a FlyDSL operation from its name. The attributes and types -(`attributes.zig`, `types.zig`, `../fly_rocdl/fly_rocdl.zig`, `capi.*`, `../fly_rocdl/capi.*`) are maintained by -hand and are not covered here. - -## Sources - -The FlyDSL commit is the one `third_party/flydsl/repo.bzl` pins, with its -patches applied: read the `.td` files from the fetched repository -(`$(bazel info output_base)/external/+non_module_deps+flydsl`), never from an -unpatched checkout. - -- `include/flydsl/Dialect/Fly/IR/FlyOps.td` for `ops.zig`. -- `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` for `../fly_rocdl/ops.zig`. - -Dump their records with the LLVM the workspace builds: - -```sh -bazel build @llvm-project//llvm:llvm-tblgen -llvm-tblgen --dump-json \ - -I /include -I /external/+llvm+llvm-project/mlir/include \ - /include/flydsl/Dialect/Fly/IR/FlyOps.td -o FlyOps.json -``` - -Every record in `!instanceof.Op` whose `opDialect` names the dialect is an -operation. Do not regex the `.td` files. - -## What each builder must be - -Builder names, parameter order, `opts`, operand and result types, doc comments -and `operation_names` follow `mlir/dialects/README.md`. Specific to FlyDSL: - -- Builders are sorted by mnemonic. The doc comment gives the `summary` and - whether result types are inferred, explicit or absent. -- Unlike the README rule, parameter names are the ODS names in snake case. The - reserved locals `ctx`, `opts`, `location`, `make`, `operands`, `attributes` - and `present` also take a trailing underscore. -- Operations with `AttrSizedOperandSegments` pass every operand group through - `Operation.make`'s `.variadic` operands, which writes `operandSegmentSizes`; - others pass a flat list (an optional operand may only be last). -- Inferred operations set `.result_type_inference = true`. Inference comes - from the flattened trait list (trait lists such as `Pure` expand to their - members); an `OpInterface` record whose `cppInterfaceName` is - `InferTypeOpInterface` marks it. -- Attributes are `*const mlir.Attribute`, set under their ODS name. -- Builders call `make` from `fly.zig`, which verifies the operation and names - the operand types when creation fails. -- Each file starts with a header naming the FlyDSL commit and the `.td` it - was generated from. - -Regions, successors and `VariadicOfVariadic` operands have no use in either -dialect today; when one appears, extend the generator and this file. - -## Checks - -- Format and run the conformance checks of `mlir/dialects/README.md`. -- `bazel test //mlir/dialects/fly:test //dsl/fly:test`, which includes the - registration tests of the `operation_names` lists in both packages' - `test.zig`. -- Callers that no longer compile are the point of generating: an operation - whose signature changed in FlyDSL now fails at the call site. Fix them in - the same change (`dsl/fly/builder.zig`, `zml/kernel/fly/fly.zig`, `zml/kernels/*/fly/`, - and downstream llmd's `llmd/kernels/*/fly/`). -- Emit the IR of the fly kernels before and after (for instance - llmd's `sparse_mla.Main2D.emit` and the `zml/kernels/moe/fly/` kernels with - representative configs) and - diff it: a regeneration without an upstream change must be byte-identical. diff --git a/mlir/dialects/fly/attributes.zig b/mlir/dialects/fly/attributes.zig deleted file mode 100644 index 3fc86ad3..00000000 --- a/mlir/dialects/fly/attributes.zig +++ /dev/null @@ -1,223 +0,0 @@ -const c = @import("c"); -const mlir = @import("mlir"); - -/// `#fly.int`, the unified static/dynamic integer leaf. -pub const IntAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAFlyInt; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn getStatic(ctx: *mlir.Context, value: i32) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyIntAttrStaticGet(ctx.ptr(), value).ptr orelse return error.InvalidMlir); - } - - pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i32) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyIntAttrDynamicGet(ctx.ptr(), width, divisibility).ptr orelse return error.InvalidMlir); - } - - pub fn getNone(ctx: *mlir.Context) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyIntAttrNoneGet(ctx.ptr()).ptr orelse return error.InvalidMlir); - } -}; - -/// `#fly.int_tuple`: a leaf, a basis element, or a nest of those. -pub const IntTupleAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAFlyIntTuple; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub const InitArgs = struct { - value: *const mlir.Attribute, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyIntTupleAttrGet(ctx.ptr(), args.value.ptr()).ptr orelse return error.InvalidMlir); - } - - pub fn getStatic(ctx: *mlir.Context, value: i32) mlir.Error!*const Self { - return get(ctx, .{ .value = (try IntAttr.getStatic(ctx, value)).attribute() }); - } - - pub fn getDynamic(ctx: *mlir.Context, width: i32, divisibility: i32) mlir.Error!*const Self { - return get(ctx, .{ .value = (try IntAttr.getDynamic(ctx, width, divisibility)).attribute() }); - } - - pub fn getNone(ctx: *mlir.Context) mlir.Error!*const Self { - return get(ctx, .{ .value = (try IntAttr.getNone(ctx)).attribute() }); - } - - pub fn getBasis(ctx: *mlir.Context, value: *const IntAttr, modes: []const i32) mlir.Error!*const Self { - const result = c.mlirFlyIntTupleAttrBasisGet(ctx.ptr(), value.ptr(), @intCast(modes.len), modes.ptr); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - - pub fn getTuple(ctx: *mlir.Context, elements: []const *const Self) mlir.Error!*const Self { - const result = c.mlirFlyIntTupleAttrTupleGet(ctx.ptr(), @intCast(elements.len), @ptrCast(elements.ptr)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `#fly.layout`, a `shape:stride` pair. -pub const LayoutAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAFlyLayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub const InitArgs = struct { - shape: *const IntTupleAttr, - stride: *const IntTupleAttr, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyLayoutAttrGet(ctx.ptr(), args.shape.ptr(), args.stride.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `#fly.tile`: a mode, or a `[a|b]` nest of modes. -pub const TileAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAFlyTile; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub const InitArgs = struct { - value: *const mlir.Attribute, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyTileAttrGet(ctx.ptr(), args.value.ptr()).ptr orelse return error.InvalidMlir); - } - - pub fn getModes(ctx: *mlir.Context, modes: []const *const mlir.Attribute) mlir.Error!*const Self { - const result = c.mlirFlyTileAttrModesGet(ctx.ptr(), @intCast(modes.len), @ptrCast(modes.ptr)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `S`. -pub const SwizzleAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAFlySwizzle; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub const InitArgs = struct { - mask: i32, - base: i32, - shift: i32, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlySwizzleAttrGet(ctx.ptr(), args.mask, args.base, args.shift); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `align`, in bytes. -pub const AlignAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAFlyAlign; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub const InitArgs = struct { - alignment: i32, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyAlignAttrGet(ctx.ptr(), args.alignment).ptr orelse return error.InvalidMlir); - } -}; - -pub const AddressSpace = enum(i32) { generic = 0, global = 1, shared = 2, register = 3 }; - -/// `global`, `shared`, ... A pointer or memref may also carry a foreign -/// address-space attribute, such as `#fly_rocdl.buffer_desc`. -pub const AddressSpaceAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAFlyAddressSpace; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub const InitArgs = struct { - addressSpace: AddressSpace, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyAddressSpaceAttrGet(ctx.ptr(), @backingInt(args.addressSpace)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -pub const MmaOperand = enum(i32) { a = 0, b = 1, c = 2, d = 3 }; - -pub fn mmaOperandAttr(ctx: *mlir.Context, operand: MmaOperand) mlir.Error!*const mlir.Attribute { - const result = c.mlirFlyMmaOperandAttrGet(ctx.ptr(), @backingInt(operand)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); -} - -pub const GemmTraversalOrder = enum(i32) { - kmn = 0, - knm = 1, - mkn = 2, - mnk = 3, - nkm = 4, - nmk = 5, - kmn_serpentine = 6, - knm_serpentine = 7, - mkn_serpentine = 8, - mnk_serpentine = 9, - nkm_serpentine = 10, - nmk_serpentine = 11, -}; - -pub fn gemmTraversalOrderAttr(ctx: *mlir.Context, order: GemmTraversalOrder) mlir.Error!*const mlir.Attribute { - const result = c.mlirFlyGemmTraversalOrderAttrGet(ctx.ptr(), @backingInt(order)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); -} diff --git a/mlir/dialects/fly/capi.cc b/mlir/dialects/fly/capi.cc deleted file mode 100644 index 1f4913bd..00000000 --- a/mlir/dialects/fly/capi.cc +++ /dev/null @@ -1,529 +0,0 @@ -//===- capi.cc - C API for the fly dialect --------------------------------===// - -#include "capi.h" - -#include "TiledOpTraits.h" -#include "flydsl/Dialect/Fly/IR/FlyDialect.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Fly, fly, mlir::fly::FlyDialect) - -using mlir::cast; -using mlir::isa; -using namespace mlir::fly; - -static mlir::MLIRContext *loadFly(MlirContext context) { - mlir::MLIRContext *ctx = unwrap(context); - if (ctx) - ctx->getOrLoadDialect(); - return ctx; -} - -/// `!attr` and `attr == other` are value arithmetic on a fly IntAttr, not a -/// null test: every handle check here goes through these two helpers. -template static bool isNull(T value) { - return !static_cast(value); -} - -template -static T checked(Handle handle, mlir::MLIRContext *ctx) { - auto value = mlir::dyn_cast_if_present(unwrap(handle)); - if (isNull(value) || value.getContext() != ctx) - return T{}; - return value; -} - -#define FLY_ATTR_ISA(Name, Attr) \ - bool mlirAttributeIsAFly##Name(MlirAttribute attr) { \ - return mlir::isa_and_nonnull(unwrap(attr)); \ - } - -FLY_ATTR_ISA(Int, IntAttr) -FLY_ATTR_ISA(IntTuple, IntTupleAttr) -FLY_ATTR_ISA(Layout, LayoutAttr) -FLY_ATTR_ISA(Tile, TileAttr) -FLY_ATTR_ISA(Swizzle, SwizzleAttr) -FLY_ATTR_ISA(Align, AlignAttr) -FLY_ATTR_ISA(AddressSpace, AddressSpaceAttr) - -#undef FLY_ATTR_ISA - -#define FLY_TYPE_ISA(Name, Type) \ - bool mlirTypeIsAFly##Name(MlirType type) { \ - return mlir::isa_and_nonnull(unwrap(type)); \ - } - -FLY_TYPE_ISA(IntTuple, IntTupleType) -FLY_TYPE_ISA(Layout, LayoutType) -FLY_TYPE_ISA(ComposedLayout, ComposedLayoutType) -FLY_TYPE_ISA(Tile, TileType) -FLY_TYPE_ISA(Swizzle, SwizzleType) -FLY_TYPE_ISA(Pointer, PointerType) -FLY_TYPE_ISA(MemRef, mlir::fly::MemRefType) -FLY_TYPE_ISA(CoordTensor, CoordTensorType) -FLY_TYPE_ISA(CopyAtom, CopyAtomType) -FLY_TYPE_ISA(MmaAtom, MmaAtomType) -FLY_TYPE_ISA(TiledCopy, TiledCopyType) -FLY_TYPE_ISA(TiledMma, TiledMmaType) -FLY_TYPE_ISA(CopyOpUniversalCopy, CopyOpUniversalCopyType) - -#undef FLY_TYPE_ISA - -MlirAttribute mlirFlyIntAttrStaticGet(MlirContext context, int32_t value) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - return wrap(static_cast(IntAttr::getStatic(ctx, value))); -} - -MlirAttribute mlirFlyIntAttrDynamicGet(MlirContext context, int32_t width, - int32_t divisibility) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - return wrap(static_cast( - IntAttr::getDynamic(ctx, width, divisibility))); -} - -MlirAttribute mlirFlyIntAttrNoneGet(MlirContext context) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - return wrap(static_cast(IntAttr::getNone(ctx))); -} - -MlirAttribute mlirFlyIntTupleAttrGet(MlirContext context, MlirAttribute value) { - mlir::MLIRContext *ctx = loadFly(context); - mlir::Attribute inner = unwrap(value); - if (!ctx || isNull(inner) || inner.getContext() != ctx) - return {nullptr}; - return wrap(static_cast(IntTupleAttr::get(ctx, inner))); -} - -MlirAttribute mlirFlyIntTupleAttrBasisGet(MlirContext context, - MlirAttribute value, intptr_t nModes, - const int32_t *modes) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - IntAttr intValue = checked(value, ctx); - if (isNull(intValue) || (nModes > 0 && !modes)) - return {nullptr}; - return wrap(static_cast( - IntTupleAttr::get(intValue, llvm::ArrayRef(modes, nModes)))); -} - -MlirAttribute mlirFlyIntTupleAttrTupleGet(MlirContext context, - intptr_t nElements, - MlirAttribute const *elements) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx || (nElements > 0 && !elements)) - return {nullptr}; - llvm::SmallVector values; - values.reserve(nElements); - for (intptr_t i = 0; i < nElements; ++i) { - IntTupleAttr element = checked(elements[i], ctx); - if (isNull(element)) - return {nullptr}; - values.push_back(element); - } - return wrap(static_cast( - IntTupleAttr::get(ctx, mlir::ArrayAttr::get(ctx, values)))); -} - -MlirAttribute mlirFlyLayoutAttrGet(MlirContext context, MlirAttribute shape, - MlirAttribute stride) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - IntTupleAttr shapeAttr = checked(shape, ctx); - IntTupleAttr strideAttr = checked(stride, ctx); - if (isNull(shapeAttr) || isNull(strideAttr)) - return {nullptr}; - return wrap(static_cast( - LayoutAttr::get(ctx, shapeAttr, strideAttr))); -} - -MlirAttribute mlirFlyTileAttrGet(MlirContext context, MlirAttribute value) { - mlir::MLIRContext *ctx = loadFly(context); - mlir::Attribute inner = unwrap(value); - if (!ctx || isNull(inner) || inner.getContext() != ctx) - return {nullptr}; - return wrap(static_cast(TileAttr::get(ctx, inner))); -} - -MlirAttribute mlirFlyTileAttrModesGet(MlirContext context, intptr_t nModes, - MlirAttribute const *modes) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx || (nModes > 0 && !modes)) - return {nullptr}; - llvm::SmallVector values; - values.reserve(nModes); - for (intptr_t i = 0; i < nModes; ++i) { - mlir::Attribute mode = unwrap(modes[i]); - if (isNull(mode) || mode.getContext() != ctx) - return {nullptr}; - values.push_back(mode); - } - return wrap(static_cast( - TileAttr::get(ctx, mlir::ArrayAttr::get(ctx, values)))); -} - -MlirAttribute mlirFlySwizzleAttrGet(MlirContext context, int32_t mask, - int32_t base, int32_t shift) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - return wrap( - static_cast(SwizzleAttr::get(ctx, mask, base, shift))); -} - -MlirAttribute mlirFlyAlignAttrGet(MlirContext context, int32_t alignment) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - return wrap(static_cast(AlignAttr::get(ctx, alignment))); -} - -MlirAttribute mlirFlyAddressSpaceAttrGet(MlirContext context, - int32_t addressSpace) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - return wrap(static_cast( - AddressSpaceAttr::get(ctx, static_cast(addressSpace)))); -} - -MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext context, int32_t operand) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - return wrap(static_cast( - MmaOperandAttr::get(ctx, static_cast(operand)))); -} - -MlirAttribute mlirFlyGemmTraversalOrderAttrGet(MlirContext context, - int32_t order) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - return wrap(static_cast(GemmTraversalOrderAttr::get( - ctx, static_cast(order)))); -} - -MlirType mlirFlyIntTupleTypeGet(MlirContext context, MlirAttribute attr) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - IntTupleAttr value = checked(attr, ctx); - if (isNull(value)) - return {nullptr}; - return wrap(static_cast(IntTupleType::get(value))); -} - -MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type) { - return wrap( - static_cast(cast(unwrap(type)).getAttr())); -} - -intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type) { - return cast(unwrap(type)).rank(); -} - -MlirType mlirFlyIntTupleTypeGetElement(MlirType type, intptr_t pos) { - return wrap( - static_cast(cast(unwrap(type)).at(pos))); -} - -bool mlirFlyIntTupleTypeIsLeaf(MlirType type) { - return cast(unwrap(type)).isLeaf(); -} - -bool mlirFlyIntTupleTypeIsStatic(MlirType type) { - return cast(unwrap(type)).isStatic(); -} - -bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type) { - IntTupleAttr attr = cast(unwrap(type)).getAttr(); - return attr.isLeafInt() && attr.getLeafAsInt().isStatic(); -} - -bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type) { - return cast(unwrap(type)).getAttr().isLeafNone(); -} - -bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type) { - return cast(unwrap(type)).getAttr().isLeafBasis(); -} - -int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type) { - return cast(unwrap(type)).getAttr().getLeafAsInt().getValue(); -} - -MlirType mlirFlyLayoutTypeGet(MlirContext context, MlirAttribute attr) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - LayoutAttr value = checked(attr, ctx); - if (isNull(value)) - return {nullptr}; - return wrap(static_cast(LayoutType::get(value))); -} - -MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type) { - return wrap( - static_cast(cast(unwrap(type)).getAttr())); -} - -MlirType mlirFlyLayoutTypeGetShape(MlirType type) { - return wrap(static_cast( - IntTupleType::get(cast(unwrap(type)).getAttr().getShape()))); -} - -MlirType mlirFlyLayoutTypeGetStride(MlirType type) { - return wrap(static_cast( - IntTupleType::get(cast(unwrap(type)).getAttr().getStride()))); -} - -MlirAttribute mlirFlyComposedLayoutTypeGetAttr(MlirType type) { - return wrap(static_cast( - cast(unwrap(type)).getAttr())); -} - -MlirType mlirFlyTileTypeGet(MlirContext context, MlirAttribute attr) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - TileAttr value = checked(attr, ctx); - if (isNull(value)) - return {nullptr}; - return wrap(static_cast(TileType::get(value))); -} - -MlirAttribute mlirFlyTileTypeGetAttr(MlirType type) { - return wrap( - static_cast(cast(unwrap(type)).getAttr())); -} - -MlirType mlirFlySwizzleTypeGet(MlirContext context, MlirAttribute attr) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - SwizzleAttr value = checked(attr, ctx); - if (isNull(value)) - return {nullptr}; - return wrap(static_cast(SwizzleType::get(value))); -} - -MlirType mlirFlyPointerTypeGet(MlirContext context, MlirType elemTy, - MlirAttribute addressSpace, - MlirAttribute alignment, MlirAttribute swizzle) { - mlir::MLIRContext *ctx = loadFly(context); - mlir::Type element = unwrap(elemTy); - mlir::Attribute space = unwrap(addressSpace); - if (!ctx || isNull(element) || element.getContext() != ctx) - return {nullptr}; - if (isNull(space) || space.getContext() != ctx) - return {nullptr}; - AlignAttr align = alignment.ptr ? checked(alignment, ctx) - : AlignAttr::getTrivialAlignment(element); - SwizzleAttr sw = swizzle.ptr ? checked(swizzle, ctx) - : SwizzleAttr::getTrivialSwizzle(ctx); - if (isNull(align) || isNull(sw)) - return {nullptr}; - return wrap(static_cast( - PointerType::get(ctx, element, space, align, sw))); -} - -MlirType mlirFlyPointerTypeGetElemTy(MlirType type) { - return wrap(cast(unwrap(type)).getElemTy()); -} - -MlirAttribute mlirFlyPointerTypeGetAddressSpace(MlirType type) { - return wrap(cast(unwrap(type)).getAddressSpace()); -} - -MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type) { - return wrap(static_cast( - cast(unwrap(type)).getAlignment())); -} - -MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type) { - return wrap(static_cast( - cast(unwrap(type)).getSwizzle())); -} - -MlirType mlirFlyMemRefTypeGet(MlirContext context, MlirType elemTy, - MlirAttribute addressSpace, MlirAttribute layout, - MlirAttribute alignment, MlirAttribute swizzle) { - mlir::MLIRContext *ctx = loadFly(context); - mlir::Type element = unwrap(elemTy); - mlir::Attribute space = unwrap(addressSpace); - mlir::Attribute lay = unwrap(layout); - if (!ctx || isNull(element) || element.getContext() != ctx) - return {nullptr}; - if (isNull(space) || space.getContext() != ctx) - return {nullptr}; - if (isNull(lay) || lay.getContext() != ctx) - return {nullptr}; - AlignAttr align = alignment.ptr ? checked(alignment, ctx) - : AlignAttr::getTrivialAlignment(element); - SwizzleAttr sw = swizzle.ptr ? checked(swizzle, ctx) - : SwizzleAttr::getTrivialSwizzle(ctx); - if (isNull(align) || isNull(sw)) - return {nullptr}; - return wrap(static_cast( - mlir::fly::MemRefType::get(ctx, element, space, lay, align, sw))); -} - -MlirType mlirFlyMemRefTypeGetElemTy(MlirType type) { - return wrap(cast(unwrap(type)).getElemTy()); -} - -MlirAttribute mlirFlyMemRefTypeGetAddressSpace(MlirType type) { - return wrap(cast(unwrap(type)).getAddressSpace()); -} - -MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type) { - return wrap(cast(unwrap(type)).getLayout()); -} - -MlirType mlirFlyCoordTensorTypeGet(MlirContext context, MlirAttribute base, - MlirAttribute layout) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - IntTupleAttr baseAttr = checked(base, ctx); - mlir::Attribute lay = unwrap(layout); - if (isNull(baseAttr) || isNull(lay) || lay.getContext() != ctx) - return {nullptr}; - return wrap( - static_cast(CoordTensorType::get(ctx, baseAttr, lay))); -} - -static mlir::Attribute outerLayoutAttr(mlir::Attribute layout) { - while (auto composed = mlir::dyn_cast(layout)) - layout = composed.getOuter(); - return layout; -} - -MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type) { - mlir::Type ty = unwrap(type); - mlir::Attribute layout; - if (auto l = mlir::dyn_cast(ty)) { - layout = l.getAttr(); - } else if (auto c = mlir::dyn_cast(ty)) { - layout = c.getAttr(); - } else if (auto m = mlir::dyn_cast(ty)) { - layout = m.getLayout(); - } else { - layout = cast(ty).getLayout(); - } - return wrap(static_cast( - IntTupleType::get(cast(outerLayoutAttr(layout)).getShape()))); -} - -MlirType mlirFlyCopyAtomTypeGet(MlirContext context, MlirType copyOp, - int32_t valBits) { - mlir::MLIRContext *ctx = loadFly(context); - mlir::Type op = unwrap(copyOp); - if (!ctx || isNull(op) || op.getContext() != ctx) - return {nullptr}; - return wrap(static_cast(CopyAtomType::get(ctx, op, valBits))); -} - -MlirType mlirFlyMmaAtomTypeGet(MlirContext context, MlirType mmaOp) { - mlir::MLIRContext *ctx = loadFly(context); - mlir::Type op = unwrap(mmaOp); - if (!ctx || isNull(op) || op.getContext() != ctx) - return {nullptr}; - return wrap(static_cast(MmaAtomType::get(ctx, op))); -} - -#define FLY_ATOM_LAYOUT(Name, AtomType, getter) \ - MlirType Name(MlirType type) { \ - return wrap(static_cast(LayoutType::get( \ - cast(cast(unwrap(type)).getter())))); \ - } - -FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrLayout, CopyAtomType, getThrLayout) -FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutSrc, CopyAtomType, - getThrValLayoutSrc) -FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutDst, CopyAtomType, - getThrValLayoutDst) -FLY_ATOM_LAYOUT(mlirFlyCopyAtomTypeGetThrValLayoutRef, CopyAtomType, - getThrValLayoutRef) -FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrLayout, MmaAtomType, getThrLayout) -FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutA, MmaAtomType, - getThrValLayoutA) -FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutB, MmaAtomType, - getThrValLayoutB) -FLY_ATOM_LAYOUT(mlirFlyMmaAtomTypeGetThrValLayoutC, MmaAtomType, - getThrValLayoutC) - -#undef FLY_ATOM_LAYOUT - -MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type) { - return wrap(static_cast(IntTupleType::get( - cast(cast(unwrap(type)).getShapeMNK())))); -} - -MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type) { - auto ty = cast(unwrap(type)); - return wrap( - static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutSrc( - cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), - ty.getTileMN().getAttr())))); -} - -MlirType mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type) { - auto ty = cast(unwrap(type)); - return wrap( - static_cast(LayoutType::get(tiledCopyGetTiledThrValLayoutDst( - cast(ty.getCopyAtom()), ty.getLayoutThrVal().getAttr(), - ty.getTileMN().getAttr())))); -} - -MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type) { - auto ty = cast(unwrap(type)); - return wrap(static_cast(IntTupleType::get(tiledMmaGetTileSizeMNK( - cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), - ty.getPermutation().getAttr())))); -} - -MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type) { - auto ty = cast(unwrap(type)); - return wrap(static_cast(LayoutType::get(tiledMmaGetThrLayoutVMNK( - cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr())))); -} - -static MlirType tiledMmaTiledThrVal(MlirType tiledMma, MmaOperand operand) { - auto ty = cast(unwrap(tiledMma)); - return wrap( - static_cast(LayoutType::get(tiledMmaGetTiledThrValLayout( - cast(ty.getMmaAtom()), ty.getAtomLayout().getAttr(), - ty.getPermutation().getAttr(), operand)))); -} - -MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type) { - return tiledMmaTiledThrVal(type, MmaOperand::A); -} - -MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type) { - return tiledMmaTiledThrVal(type, MmaOperand::B); -} - -MlirType mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type) { - return tiledMmaTiledThrVal(type, MmaOperand::C); -} - -MlirType mlirFlyCopyOpUniversalCopyTypeGet(MlirContext context, - int32_t bitSize) { - mlir::MLIRContext *ctx = loadFly(context); - if (!ctx) - return {nullptr}; - return wrap( - static_cast(CopyOpUniversalCopyType::get(ctx, bitSize))); -} diff --git a/mlir/dialects/fly/capi.h b/mlir/dialects/fly/capi.h deleted file mode 100644 index 50fb0f49..00000000 --- a/mlir/dialects/fly/capi.h +++ /dev/null @@ -1,168 +0,0 @@ -//===- capi.h - C API for the fly dialect -------------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_FLY_CAPI_H -#define MLIR_DIALECTS_FLY_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Fly, fly); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyInt(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrStaticGet(MlirContext context, - int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrDynamicGet(MlirContext context, - int32_t width, - int32_t divisibility); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntAttrNoneGet(MlirContext context); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyIntTuple(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrGet(MlirContext context, - MlirAttribute value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirFlyIntTupleAttrBasisGet(MlirContext context, MlirAttribute value, - intptr_t nModes, const int32_t *modes); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleAttrTupleGet( - MlirContext context, intptr_t nElements, MlirAttribute const *elements); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutAttrGet(MlirContext context, - MlirAttribute shape, - MlirAttribute stride); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyTile(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrGet(MlirContext context, - MlirAttribute value); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileAttrModesGet( - MlirContext context, intptr_t nModes, MlirAttribute const *modes); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlySwizzle(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlySwizzleAttrGet(MlirContext context, - int32_t mask, - int32_t base, - int32_t shift); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAlign(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyAlignAttrGet(MlirContext context, - int32_t alignment); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAFlyAddressSpace(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirFlyAddressSpaceAttrGet(MlirContext context, int32_t addressSpace); - -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMmaOperandAttrGet(MlirContext context, - int32_t operand); -MLIR_CAPI_EXPORTED MlirAttribute -mlirFlyGemmTraversalOrderAttrGet(MlirContext context, int32_t order); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyIntTuple(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyIntTupleTypeGetAttr(MlirType type); -MLIR_CAPI_EXPORTED intptr_t mlirFlyIntTupleTypeGetNumElements(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyIntTupleTypeGetElement(MlirType type, - intptr_t pos); -MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsLeaf(MlirType type); -MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStatic(MlirType type); -MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsStaticLeaf(MlirType type); -MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsNoneLeaf(MlirType type); -MLIR_CAPI_EXPORTED bool mlirFlyIntTupleTypeIsBasisLeaf(MlirType type); -MLIR_CAPI_EXPORTED int64_t mlirFlyIntTupleTypeGetStaticValue(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyLayoutTypeGetAttr(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetShape(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutTypeGetStride(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyComposedLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirFlyComposedLayoutTypeGetAttr(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTile(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyTileTypeGet(MlirContext context, - MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyTileTypeGetAttr(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlySwizzle(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlySwizzleTypeGet(MlirContext context, - MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyPointer(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGet(MlirContext context, - MlirType elemTy, - MlirAttribute addressSpace, - MlirAttribute alignment, - MlirAttribute swizzle); -MLIR_CAPI_EXPORTED MlirType mlirFlyPointerTypeGetElemTy(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirFlyPointerTypeGetAddressSpace(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetAlignment(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyPointerTypeGetSwizzle(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMemRef(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGet( - MlirContext context, MlirType elemTy, MlirAttribute addressSpace, - MlirAttribute layout, MlirAttribute alignment, MlirAttribute swizzle); -MLIR_CAPI_EXPORTED MlirType mlirFlyMemRefTypeGetElemTy(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute -mlirFlyMemRefTypeGetAddressSpace(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirFlyMemRefTypeGetLayout(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCoordTensor(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyCoordTensorTypeGet(MlirContext context, - MlirAttribute base, - MlirAttribute layout); - -MLIR_CAPI_EXPORTED MlirType mlirFlyLayoutLikeTypeGetShape(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyAtom(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGet(MlirContext context, - MlirType copyOp, - int32_t valBits); -MLIR_CAPI_EXPORTED MlirType mlirFlyCopyAtomTypeGetThrLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirFlyCopyAtomTypeGetThrValLayoutSrc(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirFlyCopyAtomTypeGetThrValLayoutDst(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirFlyCopyAtomTypeGetThrValLayoutRef(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyMmaAtom(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGet(MlirContext context, - MlirType mmaOp); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrLayout(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetShapeMNK(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutA(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutB(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyMmaAtomTypeGetThrValLayoutC(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledCopy(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyTiledMma(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetTileSizeMNK(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyTiledMmaTypeGetThrLayoutVMNK(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirFlyTiledMmaTypeGetTiledThrValLayoutA(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirFlyTiledMmaTypeGetTiledThrValLayoutB(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirFlyTiledMmaTypeGetTiledThrValLayoutC(MlirType type); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyCopyOpUniversalCopy(MlirType type); -MLIR_CAPI_EXPORTED MlirType -mlirFlyCopyOpUniversalCopyTypeGet(MlirContext context, int32_t bitSize); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_FLY_CAPI_H diff --git a/mlir/dialects/fly/fly.zig b/mlir/dialects/fly/fly.zig deleted file mode 100644 index 15530388..00000000 --- a/mlir/dialects/fly/fly.zig +++ /dev/null @@ -1,120 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); - -pub const attributes = @import("attributes.zig"); -pub const ops = @import("ops.zig"); -pub const types = @import("types.zig"); - -pub const AddressSpace = attributes.AddressSpace; -pub const GemmTraversalOrder = attributes.GemmTraversalOrder; -pub const MmaOperand = attributes.MmaOperand; - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("fly"); -} - -/// `Operation.make` that names the op and its operand types on failure: a -/// rejected `inferReturnTypes` otherwise panics with nothing to go on. -pub fn make(ctx: *mlir.Context, name: []const u8, args: mlir.Operation.MakeArgs) *mlir.Operation { - return mlir.Operation.try_make(ctx, name, args) catch |err| { - var buf: [4096]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - if (args.operands) |operands| switch (operands) { - .flat => |vs| for (vs, 0..) |v, i| { - w.print("\n operand {d}: {f}", .{ i, v.type_() }) catch break; - }, - .variadic => |segments| for (segments) |seg| for (seg) |v| { - w.print("\n operand: {f}", .{v.type_()}) catch break; - }, - }; - if (args.results) |results| switch (results) { - .flat => |ts| for (ts, 0..) |t, i| { - w.print("\n result {d}: {f}", .{ i, t }) catch break; - }, - .variadic => {}, - }; - std.debug.panic("{s}: cannot create operation ({}){s}", .{ name, err, w.buffered() }); - }; -} - -pub const TypeKind = enum { - memref, - coord_tensor, - ptr, - int_tuple, - layout, - composed_layout, - tile, - swizzle, - copy_atom, - mma_atom, - tiled_copy, - tiled_mma, - other, -}; - -pub fn KindType(comptime kind: TypeKind) type { - return switch (kind) { - .memref => types.MemRefType, - .coord_tensor => types.CoordTensorType, - .ptr => types.PointerType, - .int_tuple => types.IntTupleType, - .layout => types.LayoutType, - .composed_layout => types.ComposedLayoutType, - .tile => types.TileType, - .swizzle => types.SwizzleType, - .copy_atom => types.CopyAtomType, - .mma_atom => types.MmaAtomType, - .tiled_copy => types.TiledCopyType, - .tiled_mma => types.TiledMmaType, - .other => @compileError("fly.KindType: `.other` is not a fly type"), - }; -} - -pub fn typeKind(ty: *const mlir.Type) TypeKind { - inline for (comptime std.meta.tags(TypeKind)) |kind| { - if (kind != .other and ty.isA(KindType(kind)) != null) return kind; - } - return .other; -} - -/// Panics if `ty` is not a `!fly.`. -pub fn expect(ty: *const mlir.Type, comptime kind: TypeKind) *const KindType(kind) { - return ty.isA(KindType(kind)) orelse - std.debug.panic("fly: `{f}` is not a !fly.{s}", .{ ty, @tagName(kind) }); -} - -/// The shape of the outermost plain layout of any layout-carrying type. -pub fn layoutLikeShape(ty: *const mlir.Type) *const types.IntTupleType { - if (ty.isA(types.LayoutType) == null and ty.isA(types.ComposedLayoutType) == null and - ty.isA(types.MemRefType) == null and ty.isA(types.CoordTensorType) == null) - { - std.debug.panic("fly: `{f}` carries no layout", .{ty}); - } - return @ptrCast(c.mlirFlyLayoutLikeTypeGetShape(ty.ptr()).ptr.?); -} - -pub fn elemType(ty: *const mlir.Type) *const mlir.Type { - if (ty.isA(types.MemRefType)) |m| return m.elemTy(); - if (ty.isA(types.PointerType)) |p| return p.elemTy(); - std.debug.panic("fly: `{f}` has no element type", .{ty}); -} - -pub fn addressSpace(ty: *const mlir.Type) *const mlir.Attribute { - if (ty.isA(types.MemRefType)) |m| return m.addressSpace(); - if (ty.isA(types.PointerType)) |p| return p.addressSpace(); - std.debug.panic("fly: `{f}` has no address space", .{ty}); -} - -/// Keeps the address space, alignment and swizzle. -pub fn ptrWithElem(ctx: *mlir.Context, ptr: *const mlir.Type, elem: *const mlir.Type) mlir.Error!*const types.PointerType { - const p = expect(ptr, .ptr); - return types.PointerType.get(ctx, .{ - .elemTy = elem, - .addressSpace = p.addressSpace(), - .alignment = p.alignment(), - .swizzle = p.swizzle(), - }); -} diff --git a/mlir/dialects/fly/ops.zig b/mlir/dialects/fly/ops.zig deleted file mode 100644 index b9753136..00000000 --- a/mlir/dialects/fly/ops.zig +++ /dev/null @@ -1,1152 +0,0 @@ -//! `fly` operation builders, generated from FlyDSL's `include/flydsl/Dialect/Fly/IR/FlyOps.td` at -//! 1941889400621f1fc3d8f03bada1a4e7380cdf7e; do not edit, see REGENERATING.md. -//! -//! Builders take the operands, then the result types (only for operations -//! that do not infer them), then the required attributes, then an `opts` -//! struct of optional operands and attributes (pass `.{}` to omit them), then -//! the location. Parameters are the ODS names in snake case, with a trailing -//! underscore where needed. Pass `&.{}` for empty variadic operands; -//! variable-length operand groups of -//! `AttrSizedOperandSegments` operations get their segment sizes from -//! `Operation.make`. Attributes use `mlir.Attribute`. - -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -const make = @import("fly.zig").make; - -/// `fly.add_offset`. Result types are inferred. -pub fn add_offset(ctx: *mlir.Context, ptr: *const mlir.Value, offset: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.add_offset", .{ - .operands = .{ .flat = &.{ ptr, offset } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.append`. Result types are inferred. -pub fn append( - ctx: *mlir.Context, - tuple: *const mlir.Value, - elem: *const mlir.Value, - opts: struct { - n: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); - return make(ctx, "fly.append", .{ - .operands = .{ .flat = &.{ tuple, elem } }, - .attributes = attributes.constSlice(), - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.apply_swizzle`. Result types are inferred. -pub fn apply_swizzle(ctx: *mlir.Context, ptr: *const mlir.Value, swizzle: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.apply_swizzle", .{ - .operands = .{ .flat = &.{ ptr, swizzle } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.assume`. Result types are explicit. -pub fn assume(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.assume", .{ - .operands = .{ .flat = &.{ dst, src } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.atom.set_value`. Result types are inferred. -pub fn atom_set_value(ctx: *mlir.Context, atom: *const mlir.Value, value: *const mlir.Value, field: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.atom.set_value", .{ - .operands = .{ .flat = &.{ atom, value } }, - .attributes = &.{ - .named(ctx, "field", field), - }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.blocked_product`. Result types are inferred. -pub fn blocked_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.blocked_product", .{ - .operands = .{ .flat = &.{ layout, tile } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.ceil_div`. Result types are inferred. -pub fn ceil_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.ceil_div", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.coalesce`. Result types are inferred. -pub fn coalesce( - ctx: *mlir.Context, - layout: *const mlir.Value, - opts: struct { - pattern: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return make(ctx, "fly.coalesce", .{ - .operands = .{ .flat = if (opts.pattern) |present| &.{ layout, present } else &.{layout} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.complement`. Result types are inferred. -pub fn complement( - ctx: *mlir.Context, - layout: *const mlir.Value, - opts: struct { - codomain_size: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return make(ctx, "fly.complement", .{ - .operands = .{ .flat = if (opts.codomain_size) |present| &.{ layout, present } else &.{layout} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.composed_get_inner`. Result types are inferred. -pub fn composed_get_inner(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.composed_get_inner", .{ - .operands = .{ .flat = &.{input} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.composed_get_offset`. Result types are inferred. -pub fn composed_get_offset(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.composed_get_offset", .{ - .operands = .{ .flat = &.{input} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.composed_get_outer`. Result types are inferred. -pub fn composed_get_outer(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.composed_get_outer", .{ - .operands = .{ .flat = &.{input} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.composition`. Result types are inferred. -pub fn composition(ctx: *mlir.Context, outer: *const mlir.Value, inner: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.composition", .{ - .operands = .{ .flat = &.{ outer, inner } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.coprofile`. Result types are inferred. -pub fn coprofile(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.coprofile", .{ - .operands = .{ .flat = &.{layout} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.copy`. No results. -pub fn copy( - ctx: *mlir.Context, - copy_atom: *const mlir.Value, - src: *const mlir.Value, - dst: *const mlir.Value, - opts: struct { - pred: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return make(ctx, "fly.copy", .{ - .operands = .{ .flat = if (opts.pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, - .location = location, - }); -} - -/// `fly.copy_atom_call`. No results. -pub fn copy_atom_call( - ctx: *mlir.Context, - copy_atom: *const mlir.Value, - src: *const mlir.Value, - dst: *const mlir.Value, - opts: struct { - pred: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return make(ctx, "fly.copy_atom_call", .{ - .operands = .{ .flat = if (opts.pred) |present| &.{ copy_atom, src, dst, present } else &.{ copy_atom, src, dst } }, - .location = location, - }); -} - -/// `fly.copy_atom_call_ssa`. Result types are explicit. -pub fn copy_atom_call_ssa( - ctx: *mlir.Context, - copy_atom: *const mlir.Value, - src: *const mlir.Value, - results_types: []const *const mlir.Type, - opts: struct { - dst: ?*const mlir.Value = null, - pred: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return make(ctx, "fly.copy_atom_call_ssa", .{ - .operands = .{ .variadic = &.{ - &.{copy_atom}, - &.{src}, - if (opts.dst) |present| &.{present} else &.{}, - if (opts.pred) |present| &.{present} else &.{}, - } }, - .results = .{ .flat = results_types }, - .location = location, - }); -} - -/// `fly.coshape`. Result types are inferred. -pub fn coshape(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.coshape", .{ - .operands = .{ .flat = &.{layout} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.cosize`. Result types are inferred. -pub fn cosize(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.cosize", .{ - .operands = .{ .flat = &.{layout} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.crd2idx`. Result types are inferred. -pub fn crd2idx(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.crd2idx", .{ - .operands = .{ .flat = &.{ coord, layout } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.decomposition`. Result types are inferred. -pub fn decomposition(ctx: *mlir.Context, tensor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.decomposition", .{ - .operands = .{ .flat = &.{tensor} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.dice`. Result types are inferred. -pub fn dice(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.dice", .{ - .operands = .{ .flat = &.{ src, coord } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.elem_less`. Result types are explicit. -pub fn elem_less(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.elem_less", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.equal`. Result types are explicit. -pub fn equal(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.equal", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.extract_aligned_pointer_as_index`. Extract the raw pointer from a fly.memref. Result types are explicit. -pub fn extract_aligned_pointer_as_index(ctx: *mlir.Context, source: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.extract_aligned_pointer_as_index", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.flat_divide`. Result types are inferred. -pub fn flat_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.flat_divide", .{ - .operands = .{ .flat = &.{ layout, divisor } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.flat_product`. Result types are inferred. -pub fn flat_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.flat_product", .{ - .operands = .{ .flat = &.{ layout, tile } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.gemm`. Fully expand tiled GEMM with variadic A and B operand groups. No results. -pub fn gemm( - ctx: *mlir.Context, - mma_atom: *const mlir.Value, - d: *const mlir.Value, - a: []const *const mlir.Value, - b: []const *const mlir.Value, - c: *const mlir.Value, - opts: struct { - traversal_layout: ?*const mlir.Value = null, - traversal_order: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.traversal_order) |present| attributes.appendAssumeCapacity(.named(ctx, "traversalOrder", present)); - return make(ctx, "fly.gemm", .{ - .operands = .{ .variadic = &.{ - &.{mma_atom}, - &.{d}, - a, - b, - &.{c}, - if (opts.traversal_layout) |present| &.{present} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `fly.get`. Result types are inferred. -pub fn get(ctx: *mlir.Context, input: *const mlir.Value, mode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.get", .{ - .operands = .{ .flat = &.{input} }, - .attributes = &.{ - .named(ctx, "mode", mode), - }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.get_1d_coord`. Result types are inferred. -pub fn get_1d_coord(ctx: *mlir.Context, index: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.get_1d_coord", .{ - .operands = .{ .flat = &.{ index, layout } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.get_copy_atom`. Extract the copy atom and its runtime state from a tiled copy. Result types are inferred. -pub fn get_copy_atom(ctx: *mlir.Context, tiled_copy: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.get_copy_atom", .{ - .operands = .{ .flat = &.{tiled_copy} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.get_dyn_shared`. Result types are inferred. -pub fn get_dyn_shared(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.get_dyn_shared", .{ - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.get_flat_coord`. Result types are inferred. -pub fn get_flat_coord(ctx: *mlir.Context, index: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.get_flat_coord", .{ - .operands = .{ .flat = &.{ index, layout } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.get_iter`. Result types are inferred. -pub fn get_iter(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.get_iter", .{ - .operands = .{ .flat = &.{memref} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.get_layout`. Result types are inferred. -pub fn get_layout(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.get_layout", .{ - .operands = .{ .flat = &.{memref} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.get_leaves`. Result types are inferred. -pub fn get_leaves( - ctx: *mlir.Context, - input: *const mlir.Value, - opts: struct { - dynamic_only: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.dynamic_only) |present| attributes.appendAssumeCapacity(.named(ctx, "dynamicOnly", present)); - return make(ctx, "fly.get_leaves", .{ - .operands = .{ .flat = &.{input} }, - .attributes = attributes.constSlice(), - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.get_mma_atom`. Extract the MMA atom and its runtime state from a tiled MMA. Result types are inferred. -pub fn get_mma_atom(ctx: *mlir.Context, tiled_mma: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.get_mma_atom", .{ - .operands = .{ .flat = &.{tiled_mma} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.get_scalar`. Result types are inferred. -pub fn get_scalar(ctx: *mlir.Context, int_tuple: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.get_scalar", .{ - .operands = .{ .flat = &.{int_tuple} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.get_shape`. Result types are inferred. -pub fn get_shape(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.get_shape", .{ - .operands = .{ .flat = &.{layout} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.get_stride`. Result types are inferred. -pub fn get_stride(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.get_stride", .{ - .operands = .{ .flat = &.{layout} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.group`. Result types are inferred. -pub fn group(ctx: *mlir.Context, tuple: *const mlir.Value, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.group", .{ - .operands = .{ .flat = &.{tuple} }, - .attributes = &.{ - .named(ctx, "begin", begin), - .named(ctx, "end", end), - }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.idx2crd`. Result types are inferred. -pub fn idx2crd(ctx: *mlir.Context, coord: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.idx2crd", .{ - .operands = .{ .flat = &.{ coord, layout } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.int_tuple_add`. Result types are inferred. -pub fn int_tuple_add(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.int_tuple_add", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.int_tuple_div`. Result types are inferred. -pub fn int_tuple_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.int_tuple_div", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.int_tuple_mod`. Result types are inferred. -pub fn int_tuple_mod(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.int_tuple_mod", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.int_tuple_mul`. Result types are inferred. -pub fn int_tuple_mul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.int_tuple_mul", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.int_tuple_product`. Result types are inferred. -pub fn int_tuple_product(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.int_tuple_product", .{ - .operands = .{ .flat = &.{input} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.int_tuple_product_each`. Result types are inferred. -pub fn int_tuple_product_each(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.int_tuple_product_each", .{ - .operands = .{ .flat = &.{input} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.int_tuple_product_like`. Result types are inferred. -pub fn int_tuple_product_like(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.int_tuple_product_like", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.int_tuple_sub`. Result types are inferred. -pub fn int_tuple_sub(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.int_tuple_sub", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.inttoptr`. Result types are explicit. -pub fn inttoptr(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.inttoptr", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.left_inverse`. Result types are inferred. -pub fn left_inverse(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.left_inverse", .{ - .operands = .{ .flat = &.{layout} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.logical_divide`. Result types are inferred. -pub fn logical_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.logical_divide", .{ - .operands = .{ .flat = &.{ layout, divisor } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.logical_product`. Result types are inferred. -pub fn logical_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.logical_product", .{ - .operands = .{ .flat = &.{ layout, tile } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.make_composed_layout`. Result types are inferred. -pub fn make_composed_layout(ctx: *mlir.Context, inner: *const mlir.Value, offset: *const mlir.Value, outer: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_composed_layout", .{ - .operands = .{ .flat = &.{ inner, offset, outer } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.make_coord`. Result types are explicit. -pub fn make_coord(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_coord", .{ - .operands = .{ .flat = dync_elems }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.make_copy_atom`. Result types are explicit. -pub fn make_copy_atom(ctx: *mlir.Context, args: []const *const mlir.Value, result_type: *const mlir.Type, val_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_copy_atom", .{ - .operands = .{ .flat = args }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "valBits", val_bits), - }, - .location = location, - }); -} - -/// `fly.make_fragment_layout_like`. Result types are inferred. -pub fn make_fragment_layout_like(ctx: *mlir.Context, src: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_fragment_layout_like", .{ - .operands = .{ .flat = &.{src} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.make_fragment_like`. Result types are inferred. -pub fn make_fragment_like( - ctx: *mlir.Context, - src: *const mlir.Value, - opts: struct { - dtype: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.dtype) |present| attributes.appendAssumeCapacity(.named(ctx, "dtype", present)); - return make(ctx, "fly.make_fragment_like", .{ - .operands = .{ .flat = &.{src} }, - .attributes = attributes.constSlice(), - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.make_identity_layout`. Result types are inferred. -pub fn make_identity_layout(ctx: *mlir.Context, shape: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_identity_layout", .{ - .operands = .{ .flat = &.{shape} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.make_int_tuple`. Result types are explicit. -pub fn make_int_tuple(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_int_tuple", .{ - .operands = .{ .flat = dync_elems }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.make_layout`. Result types are inferred. -pub fn make_layout(ctx: *mlir.Context, shape: *const mlir.Value, stride: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_layout", .{ - .operands = .{ .flat = &.{ shape, stride } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.make_layout_like`. Result types are inferred. -pub fn make_layout_like(ctx: *mlir.Context, ref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_layout_like", .{ - .operands = .{ .flat = &.{ref} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.make_mma_atom`. Result types are explicit. -pub fn make_mma_atom(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_mma_atom", .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.make_ordered_layout`. Result types are inferred. -pub fn make_ordered_layout(ctx: *mlir.Context, shape: *const mlir.Value, order: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_ordered_layout", .{ - .operands = .{ .flat = &.{ shape, order } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.make_ptr`. Result types are explicit. -pub fn make_ptr( - ctx: *mlir.Context, - args: []const *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - dict_attrs: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.dict_attrs) |present| attributes.appendAssumeCapacity(.named(ctx, "dictAttrs", present)); - return make(ctx, "fly.make_ptr", .{ - .operands = .{ .flat = args }, - .results = .{ .flat = &.{result_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `fly.make_shape`. Result types are explicit. -pub fn make_shape(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_shape", .{ - .operands = .{ .flat = dync_elems }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.make_stride`. Result types are explicit. -pub fn make_stride(ctx: *mlir.Context, dync_elems: []const *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_stride", .{ - .operands = .{ .flat = dync_elems }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.make_tiled_copy`. Result types are inferred. -pub fn make_tiled_copy(ctx: *mlir.Context, copy_atom: *const mlir.Value, layout_thr_val: *const mlir.Value, tile_mn: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_tiled_copy", .{ - .operands = .{ .flat = &.{ copy_atom, layout_thr_val, tile_mn } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.make_tiled_mma`. Result types are inferred. -pub fn make_tiled_mma( - ctx: *mlir.Context, - mma_atom: *const mlir.Value, - atom_layout: *const mlir.Value, - opts: struct { - permutation: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return make(ctx, "fly.make_tiled_mma", .{ - .operands = .{ .flat = if (opts.permutation) |present| &.{ mma_atom, atom_layout, present } else &.{ mma_atom, atom_layout } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.make_view`. Result types are inferred. -pub fn make_view(ctx: *mlir.Context, iter: *const mlir.Value, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.make_view", .{ - .operands = .{ .flat = &.{ iter, layout } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.memref.alloca`. Result types are explicit. -pub fn memref_alloca(ctx: *mlir.Context, layout: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.memref.alloca", .{ - .operands = .{ .flat = &.{layout} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.memref.load`. Result types are inferred. -pub fn memref_load(ctx: *mlir.Context, memref: *const mlir.Value, indices: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.memref.load", .{ - .operands = .{ .flat = &.{ memref, indices } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.memref.load_vec`. Result types are inferred. -pub fn memref_load_vec(ctx: *mlir.Context, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.memref.load_vec", .{ - .operands = .{ .flat = &.{memref} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.memref.store`. No results. -pub fn memref_store(ctx: *mlir.Context, value: *const mlir.Value, memref: *const mlir.Value, indices: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.memref.store", .{ - .operands = .{ .flat = &.{ value, memref, indices } }, - .location = location, - }); -} - -/// `fly.memref.store_vec`. No results. -pub fn memref_store_vec(ctx: *mlir.Context, vector: *const mlir.Value, memref: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.memref.store_vec", .{ - .operands = .{ .flat = &.{ vector, memref } }, - .location = location, - }); -} - -/// `fly.mma.make_fragment`. Result types are inferred. -pub fn mma_make_fragment( - ctx: *mlir.Context, - tiled_mma: *const mlir.Value, - input: *const mlir.Value, - operand_id: *const mlir.Attribute, - opts: struct { - stages: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "operand_id", operand_id)); - if (opts.stages) |present| attributes.appendAssumeCapacity(.named(ctx, "stages", present)); - return make(ctx, "fly.mma.make_fragment", .{ - .operands = .{ .flat = &.{ tiled_mma, input } }, - .attributes = attributes.constSlice(), - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.mma_atom_call`. No results. -pub fn mma_atom_call(ctx: *mlir.Context, mma_atom: *const mlir.Value, d: *const mlir.Value, a: []const *const mlir.Value, b: []const *const mlir.Value, c: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.mma_atom_call", .{ - .operands = .{ .variadic = &.{ - &.{mma_atom}, - &.{d}, - a, - b, - &.{c}, - } }, - .location = location, - }); -} - -/// `fly.mma_atom_call_ssa`. Result types are explicit. -pub fn mma_atom_call_ssa( - ctx: *mlir.Context, - mma_atom: *const mlir.Value, - a: []const *const mlir.Value, - b: []const *const mlir.Value, - c: *const mlir.Value, - results_types: []const *const mlir.Type, - opts: struct { - d: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return make(ctx, "fly.mma_atom_call_ssa", .{ - .operands = .{ .variadic = &.{ - &.{mma_atom}, - if (opts.d) |present| &.{present} else &.{}, - a, - b, - &.{c}, - } }, - .results = .{ .flat = results_types }, - .location = location, - }); -} - -/// `fly.prepend`. Result types are inferred. -pub fn prepend( - ctx: *mlir.Context, - tuple: *const mlir.Value, - elem: *const mlir.Value, - opts: struct { - n: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.n) |present| attributes.appendAssumeCapacity(.named(ctx, "n", present)); - return make(ctx, "fly.prepend", .{ - .operands = .{ .flat = &.{ tuple, elem } }, - .attributes = attributes.constSlice(), - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.print`. No results. -pub fn print(ctx: *mlir.Context, values: []const *const mlir.Value, format: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.print", .{ - .operands = .{ .flat = values }, - .attributes = &.{ - .named(ctx, "format", format), - }, - .location = location, - }); -} - -/// `fly.ptr.load`. Result types are explicit. -pub fn ptr_load(ctx: *mlir.Context, ptr: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.ptr.load", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.ptr.store`. No results. -pub fn ptr_store(ctx: *mlir.Context, value: *const mlir.Value, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.ptr.store", .{ - .operands = .{ .flat = &.{ value, ptr } }, - .location = location, - }); -} - -/// `fly.ptrtoint`. Result types are inferred. -pub fn ptrtoint(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.ptrtoint", .{ - .operands = .{ .flat = &.{ptr} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.raked_product`. Result types are inferred. -pub fn raked_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.raked_product", .{ - .operands = .{ .flat = &.{ layout, tile } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.recast_iter`. Result types are explicit. -pub fn recast_iter(ctx: *mlir.Context, src: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.recast_iter", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.recast_layout`. Result types are inferred. -pub fn recast_layout(ctx: *mlir.Context, src: *const mlir.Value, new_type_bits: *const mlir.Attribute, old_type_bits: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.recast_layout", .{ - .operands = .{ .flat = &.{src} }, - .attributes = &.{ - .named(ctx, "new_type_bits", new_type_bits), - .named(ctx, "old_type_bits", old_type_bits), - }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.right_inverse`. Result types are inferred. -pub fn right_inverse(ctx: *mlir.Context, layout: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.right_inverse", .{ - .operands = .{ .flat = &.{layout} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.select`. Result types are inferred. -pub fn select(ctx: *mlir.Context, tuple: *const mlir.Value, indices: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.select", .{ - .operands = .{ .flat = &.{tuple} }, - .attributes = &.{ - .named(ctx, "indices", indices), - }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.shape_div`. Result types are inferred. -pub fn shape_div(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.shape_div", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.size`. Result types are inferred. -pub fn size(ctx: *mlir.Context, int_tuple: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.size", .{ - .operands = .{ .flat = &.{int_tuple} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.slice`. Result types are inferred. -pub fn slice(ctx: *mlir.Context, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.slice", .{ - .operands = .{ .flat = &.{ src, coord } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.static`. Result types are explicit. -pub fn static(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.static", .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `fly.take`. Result types are inferred. -pub fn take(ctx: *mlir.Context, tuple: *const mlir.Value, begin: *const mlir.Attribute, end: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.take", .{ - .operands = .{ .flat = &.{tuple} }, - .attributes = &.{ - .named(ctx, "begin", begin), - .named(ctx, "end", end), - }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.tile_to_shape`. Result types are inferred. -pub fn tile_to_shape(ctx: *mlir.Context, block: *const mlir.Value, trg_shape: *const mlir.Value, ord_shape: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.tile_to_shape", .{ - .operands = .{ .flat = &.{ block, trg_shape, ord_shape } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.tiled_copy.partition_dst`. Result types are inferred. -pub fn tiled_copy_partition_dst(ctx: *mlir.Context, tiled_copy: *const mlir.Value, dst: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.tiled_copy.partition_dst", .{ - .operands = .{ .flat = &.{ tiled_copy, dst, coord } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.tiled_copy.partition_src`. Result types are inferred. -pub fn tiled_copy_partition_src(ctx: *mlir.Context, tiled_copy: *const mlir.Value, src: *const mlir.Value, coord: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.tiled_copy.partition_src", .{ - .operands = .{ .flat = &.{ tiled_copy, src, coord } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.tiled_copy.retile`. Result types are inferred. -pub fn tiled_copy_retile(ctx: *mlir.Context, tiled_copy: *const mlir.Value, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.tiled_copy.retile", .{ - .operands = .{ .flat = &.{ tiled_copy, input } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.tiled_divide`. Result types are inferred. -pub fn tiled_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.tiled_divide", .{ - .operands = .{ .flat = &.{ layout, divisor } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.tiled_mma.partition`. Result types are inferred. -pub fn tiled_mma_partition(ctx: *mlir.Context, tiled_mma: *const mlir.Value, input: *const mlir.Value, coord: *const mlir.Value, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.tiled_mma.partition", .{ - .operands = .{ .flat = &.{ tiled_mma, input, coord } }, - .attributes = &.{ - .named(ctx, "operand_id", operand_id), - }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.tiled_mma.partition_shape`. Result types are inferred. -pub fn tiled_mma_partition_shape(ctx: *mlir.Context, tiled_mma: *const mlir.Value, shape: *const mlir.Value, operand_id: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.tiled_mma.partition_shape", .{ - .operands = .{ .flat = &.{ tiled_mma, shape } }, - .attributes = &.{ - .named(ctx, "operand_id", operand_id), - }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.tiled_product`. Result types are inferred. -pub fn tiled_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.tiled_product", .{ - .operands = .{ .flat = &.{ layout, tile } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.to_llvm_ptr`. Result types are inferred. -pub fn to_llvm_ptr(ctx: *mlir.Context, ptr: *const mlir.Value, llvm_address_space: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.to_llvm_ptr", .{ - .operands = .{ .flat = &.{ptr} }, - .attributes = &.{ - .named(ctx, "llvm_address_space", llvm_address_space), - }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.zipped_divide`. Result types are inferred. -pub fn zipped_divide(ctx: *mlir.Context, layout: *const mlir.Value, divisor: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.zipped_divide", .{ - .operands = .{ .flat = &.{ layout, divisor } }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly.zipped_product`. Result types are inferred. -pub fn zipped_product(ctx: *mlir.Context, layout: *const mlir.Value, tile: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly.zipped_product", .{ - .operands = .{ .flat = &.{ layout, tile } }, - .result_type_inference = true, - .location = location, - }); -} diff --git a/mlir/dialects/fly/test.zig b/mlir/dialects/fly/test.zig deleted file mode 100644 index a30172ff..00000000 --- a/mlir/dialects/fly/test.zig +++ /dev/null @@ -1,378 +0,0 @@ -const std = @import("std"); - -const arith = @import("mlir/dialects/arith"); -const fly = @import("mlir/dialects/fly"); -const fly_rocdl = @import("mlir/dialects/fly_rocdl"); -const func = @import("mlir/dialects/func"); -const gpu = @import("mlir/dialects/gpu"); -const llvm = @import("mlir/dialects/llvm"); -const math = @import("mlir/dialects/math"); -const memref = @import("mlir/dialects/memref"); -const mlir = @import("mlir"); -const rocdl = @import("mlir/dialects/rocdl"); -const scf = @import("mlir/dialects/scf"); -const ub = @import("mlir/dialects/ub"); -const vector = @import("mlir/dialects/vector"); - -const attributes = fly.attributes; -const ops = fly.ops; -const types = fly.types; -const TypeKind = fly.TypeKind; -const typeKind = fly.typeKind; -const expect = fly.expect; -const layoutLikeShape = fly.layoutLikeShape; -const elemType = fly.elemType; -const ptrWithElem = fly.ptrWithElem; - -const operation_names: []const []const u8 = &.{ - "fly.add_offset", - "fly.append", - "fly.apply_swizzle", - "fly.assume", - "fly.atom.set_value", - "fly.blocked_product", - "fly.ceil_div", - "fly.coalesce", - "fly.complement", - "fly.composed_get_inner", - "fly.composed_get_offset", - "fly.composed_get_outer", - "fly.composition", - "fly.coprofile", - "fly.copy", - "fly.copy_atom_call", - "fly.copy_atom_call_ssa", - "fly.coshape", - "fly.cosize", - "fly.crd2idx", - "fly.decomposition", - "fly.dice", - "fly.elem_less", - "fly.equal", - "fly.extract_aligned_pointer_as_index", - "fly.flat_divide", - "fly.flat_product", - "fly.gemm", - "fly.get", - "fly.get_1d_coord", - "fly.get_copy_atom", - "fly.get_dyn_shared", - "fly.get_flat_coord", - "fly.get_iter", - "fly.get_layout", - "fly.get_leaves", - "fly.get_mma_atom", - "fly.get_scalar", - "fly.get_shape", - "fly.get_stride", - "fly.group", - "fly.idx2crd", - "fly.int_tuple_add", - "fly.int_tuple_div", - "fly.int_tuple_mod", - "fly.int_tuple_mul", - "fly.int_tuple_product", - "fly.int_tuple_product_each", - "fly.int_tuple_product_like", - "fly.int_tuple_sub", - "fly.inttoptr", - "fly.left_inverse", - "fly.logical_divide", - "fly.logical_product", - "fly.make_composed_layout", - "fly.make_coord", - "fly.make_copy_atom", - "fly.make_fragment_layout_like", - "fly.make_fragment_like", - "fly.make_identity_layout", - "fly.make_int_tuple", - "fly.make_layout", - "fly.make_layout_like", - "fly.make_mma_atom", - "fly.make_ordered_layout", - "fly.make_ptr", - "fly.make_shape", - "fly.make_stride", - "fly.make_tiled_copy", - "fly.make_tiled_mma", - "fly.make_view", - "fly.memref.alloca", - "fly.memref.load", - "fly.memref.load_vec", - "fly.memref.store", - "fly.memref.store_vec", - "fly.mma.make_fragment", - "fly.mma_atom_call", - "fly.mma_atom_call_ssa", - "fly.prepend", - "fly.print", - "fly.ptr.load", - "fly.ptr.store", - "fly.ptrtoint", - "fly.raked_product", - "fly.recast_iter", - "fly.recast_layout", - "fly.right_inverse", - "fly.select", - "fly.shape_div", - "fly.size", - "fly.slice", - "fly.static", - "fly.take", - "fly.tile_to_shape", - "fly.tiled_copy.partition_dst", - "fly.tiled_copy.partition_src", - "fly.tiled_copy.retile", - "fly.tiled_divide", - "fly.tiled_mma.partition", - "fly.tiled_mma.partition_shape", - "fly.tiled_product", - "fly.to_llvm_ptr", - "fly.zipped_divide", - "fly.zipped_product", -}; - -test { - std.testing.refAllDecls(fly); - inline for (.{ types, attributes, ops, fly_rocdl, fly_rocdl.ops }) |module| { - std.testing.refAllDecls(module); - inline for (comptime std.meta.declarations(module)) |decl| { - const value = @field(module, decl); - if (@TypeOf(value) == type and @typeInfo(value) == .@"opaque") std.testing.refAllDecls(value); - } - } -} - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - func.registerDialect(registry); - arith.registerDialect(registry); - scf.registerDialect(registry); - math.registerDialect(registry); - vector.registerDialect(registry); - memref.registerDialect(registry); - llvm.registerDialect(registry); - gpu.registerDialect(registry); - rocdl.registerDialect(registry); - ub.registerDialect(registry); - fly.registerDialect(registry); - fly_rocdl.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -fn expectPrints(expected: []const u8, value: anytype) !void { - var buf: [512]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(expected, w.buffered()); -} - -test "fly dialects register" { - const ctx = try testContext(); - defer ctx.deinit(); - - try std.testing.expect(ctx.isRegisteredOperation("fly.make_int_tuple")); - try std.testing.expect(ctx.isRegisteredOperation("fly.make_layout")); - try std.testing.expect(ctx.isRegisteredOperation("fly.ptr.load")); - try std.testing.expect(ctx.isRegisteredOperation("fly.copy_atom_call")); - try std.testing.expect(ctx.isRegisteredOperation("gpu.func")); - try std.testing.expect(ctx.isRegisteredOperation("rocdl.sched.barrier")); - try std.testing.expect(ctx.isRegisteredOperation("ub.poison")); -} - -test "fly types parse and print round-trip" { - const ctx = try testContext(); - defer ctx.deinit(); - - const cases = [_][]const u8{ - "!fly.int_tuple<((2, 4), 8)>", - "!fly.int_tuple<((1, 4), (32, 64))>", - "!fly.layout<((2, 3), 3) : ((57, 69), 19)>", - "!fly.memref", - "!fly.memref", - "!fly.ptr", - "!fly.ptr", - "!fly.ptr", - "!fly.memref", - "!fly_rocdl.cdna3.buffer_copy<128>", - "!fly_rocdl.cdna3.buffer_copy_lds<32>", - "!fly_rocdl.cdna3.mfma<16x16x16, (f16, f16) -> f32>", - "!fly.copy_atom, 32>", - "!fly.tile<[8|64]>", - "!fly.coord_tensor<(0,0), (100,1000):(1E0,1E1)>", - }; - for (cases) |src| { - const ty = mlir.Type.parse(ctx, src) catch |err| { - std.debug.print("failed to parse `{s}`: {}\n", .{ src, err }); - return err; - }; - var buf: [512]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{ty}); - const ty2 = try mlir.Type.parse(ctx, w.buffered()); - try std.testing.expect(ty.eql(ty2)); - } -} - -test "types and attributes are built through the C API" { - const ctx = try testContext(); - defer ctx.deinit(); - - const two = try attributes.IntTupleAttr.getStatic(ctx, 2); - const four = try attributes.IntTupleAttr.getStatic(ctx, 4); - const one = try attributes.IntTupleAttr.getStatic(ctx, 1); - const shape = try attributes.IntTupleAttr.getTuple(ctx, &.{ two, four }); - const stride = try attributes.IntTupleAttr.getTuple(ctx, &.{ four, one }); - const layout = try types.LayoutType.get(ctx, .{ .attr = try .get(ctx, .{ .shape = shape, .stride = stride }) }); - try expectPrints("!fly.layout<(2,4):(4,1)>", layout); - try expectPrints("!fly.int_tuple<(2,4)>", layout.shape()); - try std.testing.expectEqual(@as(usize, 2), layout.shape().numElements()); - try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, layout.shape().element(1).leaf()); - - const global = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .global }); - const memref_type = try types.MemRefType.get(ctx, .{ - .elemTy = .float(ctx, .f16), - .addressSpace = global.attribute(), - .layout = layout.attr().attribute(), - }); - try expectPrints("!fly.memref", memref_type); - try expectPrints("!fly.int_tuple<(2,4)>", layoutLikeShape(memref_type.type_())); - - const shared = try attributes.AddressSpaceAttr.get(ctx, .{ .addressSpace = .shared }); - const ptr = try types.PointerType.get(ctx, .{ - .elemTy = .int(ctx, .i8), - .addressSpace = shared.attribute(), - .alignment = try .get(ctx, .{ .alignment = 16 }), - }); - try expectPrints("!fly.ptr>", ptr); - try expectPrints("!fly.ptr>", try ptrWithElem(ctx, ptr.type_(), .float(ctx, .f32))); - - const tile = try types.TileType.get(ctx, .{ .attr = try .getModes(ctx, &.{ - (try attributes.IntAttr.getStatic(ctx, 8)).attribute(), - (try attributes.IntAttr.getStatic(ctx, 64)).attribute(), - }) }); - try expectPrints("!fly.tile<[8|64]>", tile); - - const copy_atom = try types.CopyAtomType.get(ctx, .{ - .copyOp = (try types.CopyOpUniversalCopyType.get(ctx, .{ .bitSize = 128 })).type_(), - .valBits = 32, - }); - try expectPrints("!fly.copy_atom, 32>", copy_atom); - - const buffer_desc = try fly_rocdl.BufferDescAddressAttr.get(ctx); - try expectPrints("!fly.ptr", try types.PointerType.get(ctx, .{ - .elemTy = .float(ctx, .f16), - .addressSpace = buffer_desc.attribute(), - })); -} - -test "fly enum attributes" { - const ctx = try testContext(); - defer ctx.deinit(); - _ = try attributes.mmaOperandAttr(ctx, .a); - _ = try attributes.gemmTraversalOrderAttr(ctx, .kmn); -} - -test "inferred fly ops compute layout algebra" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc: *const mlir.Location = .unknown(ctx); - const module: *mlir.Module = .init(loc); - defer module.deinit(); - const block = module.body(); - - // raked_product((8,16):(16,1), (1,4):(1,1)) — the vectorAdd TV layout. - const thr = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(8,16):(16,1)>"), loc).appendTo(block); - const val = ops.static(ctx, try mlir.Type.parse(ctx, "!fly.layout<(1,4):(1,1)>"), loc).appendTo(block); - const mn = ops.raked_product(ctx, thr.result(0), val.result(0), loc).appendTo(block); - const shape = ops.get_shape(ctx, mn.result(0), loc).appendTo(block); - const tiler = ops.int_tuple_product_each(ctx, shape.result(0), loc).appendTo(block); - - try expectPrints("!fly.int_tuple<(8,64)>", tiler.result(0).type_()); - try std.testing.expect(module.operation().verify()); -} - -test "atom traits" { - const ctx = try testContext(); - defer ctx.deinit(); - - // MFMA 16x16x4 f32 tiled over a (2,2,1) atom layout. - const tm = expect(try mlir.Type.parse(ctx, "!fly.tiled_mma f32>>, <(2,2,1):(1,2,0)>>"), .tiled_mma); - try expectPrints("!fly.int_tuple<(32,32,4)>", tm.tileSizeMNK()); - _ = tm.tiledThrValLayoutA(); - _ = tm.thrLayoutVMNK(); - - const atom = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try fly_rocdl.MmaOpCDNA3MFMAType.get(ctx, .{ - .m = 16, - .n = 16, - .k = 4, - .elemTyA = .float(ctx, .f32), - .elemTyB = .float(ctx, .f32), - .elemTyAcc = .float(ctx, .f32), - })).type_() }); - try expectPrints("!fly.mma_atom f32>>", atom); - try expectPrints("!fly.int_tuple<(16,16,4)>", atom.shapeMNK()); - try expectPrints("!fly.layout<64:1>", atom.thrLayout()); - - const wmma = try types.MmaAtomType.get(ctx, .{ .mmaOp = (try fly_rocdl.MmaOpGFX11WMMAType.get(ctx, .{ - .m = 16, - .n = 16, - .k = 16, - .elemTyA = .float(ctx, .f16), - .elemTyB = .float(ctx, .f16), - .elemTyAcc = .float(ctx, .f32), - })).type_() }); - try expectPrints("!fly.layout<32:1>", wmma.thrLayout()); - - const ca = expect(try mlir.Type.parse(ctx, "!fly.copy_atom, 32>"), .copy_atom); - _ = ca.thrValLayoutSrc(); -} - -test "type kinds" { - const ctx = try testContext(); - defer ctx.deinit(); - const cases = .{ - .{ "!fly.memref", TypeKind.memref }, - .{ "!fly.ptr", TypeKind.ptr }, - .{ "!fly.int_tuple<(2,4)>", TypeKind.int_tuple }, - .{ "!fly.layout<(8,4):(4,1)>", TypeKind.layout }, - .{ "!fly.tile<[8|4]>", TypeKind.tile }, - .{ "!fly.copy_atom, 32>", TypeKind.copy_atom }, - .{ "!fly.mma_atom f32>>", TypeKind.mma_atom }, - }; - inline for (cases) |case| { - try std.testing.expectEqual(case[1], typeKind(try mlir.Type.parse(ctx, case[0]))); - } - try std.testing.expectEqual(TypeKind.other, typeKind(.float(ctx, .f32))); -} - -test "structural readers" { - const ctx = try testContext(); - defer ctx.deinit(); - const it = expect(try mlir.Type.parse(ctx, "!fly.int_tuple<((2,4),?)>"), .int_tuple); - try std.testing.expectEqual(@as(usize, 2), it.numElements()); - try std.testing.expect(!it.isLeaf()); - try std.testing.expect(!it.isStatic()); - const inner = it.element(0); - try std.testing.expectEqual(@as(usize, 2), inner.numElements()); - try std.testing.expectEqual(types.IntTupleType.Leaf{ .static = 4 }, inner.element(1).leaf()); - try std.testing.expectEqual(types.IntTupleType.Leaf.dynamic, it.element(1).leaf()); - - const m = try mlir.Type.parse(ctx, "!fly.memref o 0 o (64,32):(32,1), align<16>>"); - try expectPrints("!fly.int_tuple<(64,32)>", layoutLikeShape(m)); - try expectPrints("bf16", elemType(m)); -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - fly.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/fly/types.zig b/mlir/dialects/fly/types.zig deleted file mode 100644 index 27966997..00000000 --- a/mlir/dialects/fly/types.zig +++ /dev/null @@ -1,429 +0,0 @@ -const c = @import("c"); -const mlir = @import("mlir"); - -const attributes = @import("attributes.zig"); - -fn optAttr(value: anytype) c.MlirAttribute { - return if (value) |v| v.ptr() else .{ .ptr = null }; -} - -/// `!fly.int_tuple`. -pub const IntTupleType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyIntTuple; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - attr: *const attributes.IntTupleAttr, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyIntTupleTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); - } - - pub fn attr(self: *const Self) *const attributes.IntTupleAttr { - return @ptrCast(c.mlirFlyIntTupleTypeGetAttr(self.ptr()).ptr.?); - } - - /// 1 for a leaf. - pub fn numElements(self: *const Self) usize { - return @intCast(c.mlirFlyIntTupleTypeGetNumElements(self.ptr())); - } - - pub fn element(self: *const Self, pos: usize) *const Self { - return @ptrCast(c.mlirFlyIntTupleTypeGetElement(self.ptr(), @intCast(pos)).ptr.?); - } - - pub fn isLeaf(self: *const Self) bool { - return c.mlirFlyIntTupleTypeIsLeaf(self.ptr()); - } - - pub fn isStatic(self: *const Self) bool { - return c.mlirFlyIntTupleTypeIsStatic(self.ptr()); - } - - pub const Leaf = union(enum) { static: i64, dynamic, none, basis }; - - /// Asserts the tuple is a leaf. - pub fn leaf(self: *const Self) Leaf { - if (c.mlirFlyIntTupleTypeIsStaticLeaf(self.ptr())) { - return .{ .static = c.mlirFlyIntTupleTypeGetStaticValue(self.ptr()) }; - } - if (c.mlirFlyIntTupleTypeIsNoneLeaf(self.ptr())) return .none; - if (c.mlirFlyIntTupleTypeIsBasisLeaf(self.ptr())) return .basis; - return .dynamic; - } -}; - -/// `!fly.layout`. -pub const LayoutType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyLayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - attr: *const attributes.LayoutAttr, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyLayoutTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); - } - - pub fn attr(self: *const Self) *const attributes.LayoutAttr { - return @ptrCast(c.mlirFlyLayoutTypeGetAttr(self.ptr()).ptr.?); - } - - pub fn shape(self: *const Self) *const IntTupleType { - return @ptrCast(c.mlirFlyLayoutTypeGetShape(self.ptr()).ptr.?); - } - - pub fn stride(self: *const Self) *const IntTupleType { - return @ptrCast(c.mlirFlyLayoutTypeGetStride(self.ptr()).ptr.?); - } -}; - -/// `!fly.composed_layout`. -pub const ComposedLayoutType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyComposedLayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub fn attr(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirFlyComposedLayoutTypeGetAttr(self.ptr()).ptr.?); - } -}; - -/// `!fly.tile`. -pub const TileType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyTile; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - attr: *const attributes.TileAttr, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyTileTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); - } - - pub fn attr(self: *const Self) *const attributes.TileAttr { - return @ptrCast(c.mlirFlyTileTypeGetAttr(self.ptr()).ptr.?); - } -}; - -/// `!fly.swizzle`. -pub const SwizzleType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlySwizzle; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - attr: *const attributes.SwizzleAttr, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirFlySwizzleTypeGet(ctx.ptr(), args.attr.ptr()).ptr orelse return error.InvalidMlir); - } -}; - -/// `!fly.ptr`. -pub const PointerType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyPointer; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - elemTy: *const mlir.Type, - addressSpace: *const mlir.Attribute, - alignment: ?*const attributes.AlignAttr = null, - swizzle: ?*const attributes.SwizzleAttr = null, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyPointerTypeGet(ctx.ptr(), args.elemTy.ptr(), args.addressSpace.ptr(), optAttr(args.alignment), optAttr(args.swizzle)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - - pub fn elemTy(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirFlyPointerTypeGetElemTy(self.ptr()).ptr.?); - } - - pub fn addressSpace(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirFlyPointerTypeGetAddressSpace(self.ptr()).ptr.?); - } - - pub fn alignment(self: *const Self) *const attributes.AlignAttr { - return @ptrCast(c.mlirFlyPointerTypeGetAlignment(self.ptr()).ptr.?); - } - - pub fn swizzle(self: *const Self) *const attributes.SwizzleAttr { - return @ptrCast(c.mlirFlyPointerTypeGetSwizzle(self.ptr()).ptr.?); - } -}; - -/// `!fly.memref`. -pub const MemRefType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyMemRef; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - elemTy: *const mlir.Type, - addressSpace: *const mlir.Attribute, - layout: *const mlir.Attribute, - alignment: ?*const attributes.AlignAttr = null, - swizzle: ?*const attributes.SwizzleAttr = null, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyMemRefTypeGet(ctx.ptr(), args.elemTy.ptr(), args.addressSpace.ptr(), args.layout.ptr(), optAttr(args.alignment), optAttr(args.swizzle)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - - pub fn elemTy(self: *const Self) *const mlir.Type { - return @ptrCast(c.mlirFlyMemRefTypeGetElemTy(self.ptr()).ptr.?); - } - - pub fn addressSpace(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirFlyMemRefTypeGetAddressSpace(self.ptr()).ptr.?); - } - - pub fn layout(self: *const Self) *const mlir.Attribute { - return @ptrCast(c.mlirFlyMemRefTypeGetLayout(self.ptr()).ptr.?); - } -}; - -/// `!fly.coord_tensor`. -pub const CoordTensorType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyCoordTensor; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - base: *const attributes.IntTupleAttr, - layout: *const mlir.Attribute, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyCoordTensorTypeGet(ctx.ptr(), args.base.ptr(), args.layout.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!fly.copy_atom`. -pub const CopyAtomType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyCopyAtom; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - copyOp: *const mlir.Type, - valBits: i32, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyCopyAtomTypeGet(ctx.ptr(), args.copyOp.ptr(), args.valBits); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - - pub fn thrLayout(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyCopyAtomTypeGetThrLayout(self.ptr()).ptr.?); - } - - pub fn thrValLayoutSrc(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutSrc(self.ptr()).ptr.?); - } - - pub fn thrValLayoutDst(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutDst(self.ptr()).ptr.?); - } - - pub fn thrValLayoutRef(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyCopyAtomTypeGetThrValLayoutRef(self.ptr()).ptr.?); - } -}; - -/// `!fly.mma_atom`. -pub const MmaAtomType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyMmaAtom; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - mmaOp: *const mlir.Type, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyMmaAtomTypeGet(ctx.ptr(), args.mmaOp.ptr()).ptr orelse return error.InvalidMlir); - } - - pub fn thrLayout(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyMmaAtomTypeGetThrLayout(self.ptr()).ptr.?); - } - - pub fn shapeMNK(self: *const Self) *const IntTupleType { - return @ptrCast(c.mlirFlyMmaAtomTypeGetShapeMNK(self.ptr()).ptr.?); - } - - pub fn thrValLayoutA(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutA(self.ptr()).ptr.?); - } - - pub fn thrValLayoutB(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutB(self.ptr()).ptr.?); - } - - pub fn thrValLayoutC(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyMmaAtomTypeGetThrValLayoutC(self.ptr()).ptr.?); - } -}; - -/// `!fly.tiled_copy`. -pub const TiledCopyType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyTiledCopy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub fn tiledThrValLayoutSrc(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyTiledCopyTypeGetTiledThrValLayoutSrc(self.ptr()).ptr.?); - } - - pub fn tiledThrValLayoutDst(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyTiledCopyTypeGetTiledThrValLayoutDst(self.ptr()).ptr.?); - } -}; - -/// `!fly.tiled_mma`. -pub const TiledMmaType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyTiledMma; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub fn tileSizeMNK(self: *const Self) *const IntTupleType { - return @ptrCast(c.mlirFlyTiledMmaTypeGetTileSizeMNK(self.ptr()).ptr.?); - } - - pub fn thrLayoutVMNK(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyTiledMmaTypeGetThrLayoutVMNK(self.ptr()).ptr.?); - } - - pub fn tiledThrValLayoutA(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutA(self.ptr()).ptr.?); - } - - pub fn tiledThrValLayoutB(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutB(self.ptr()).ptr.?); - } - - pub fn tiledThrValLayoutC(self: *const Self) *const LayoutType { - return @ptrCast(c.mlirFlyTiledMmaTypeGetTiledThrValLayoutC(self.ptr()).ptr.?); - } -}; - -/// `!fly.universal_copy`. -pub const CopyOpUniversalCopyType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyCopyOpUniversalCopy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - bitSize: i32, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyCopyOpUniversalCopyTypeGet(ctx.ptr(), args.bitSize); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; diff --git a/mlir/dialects/fly_rocdl/BUILD.bazel b/mlir/dialects/fly_rocdl/BUILD.bazel deleted file mode 100644 index 334e520b..00000000 --- a/mlir/dialects/fly_rocdl/BUILD.bazel +++ /dev/null @@ -1,43 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -cc_library( - name = "capi", - srcs = [ - "capi.cc", - ], - hdrs = [ - "capi.h", - ], - tags = ["manual"], - deps = [ - "@flydsl//:FlyROCDLDialect", - "@llvm-project//mlir:CAPIIR", - ], -) - -zig_library( - name = "fly_rocdl", - srcs = [ - "ops.zig", - ], - import_name = "mlir/dialects/fly_rocdl", - main = "fly_rocdl.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//mlir/dialects/fly", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":fly_rocdl", - "//mlir", - ], -) diff --git a/mlir/dialects/fly_rocdl/capi.cc b/mlir/dialects/fly_rocdl/capi.cc deleted file mode 100644 index b055e993..00000000 --- a/mlir/dialects/fly_rocdl/capi.cc +++ /dev/null @@ -1,107 +0,0 @@ -//===- capi.cc - C API for the fly_rocdl dialect --------------------------===// - -#include "capi.h" - -#include "flydsl/Dialect/FlyROCDL/IR/Dialect.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl, - mlir::fly_rocdl::FlyROCDLDialect) - -static mlir::MLIRContext *loadFlyROCDL(MlirContext context) { - mlir::MLIRContext *ctx = unwrap(context); - if (ctx) - ctx->getOrLoadDialect(); - return ctx; -} - -/// Null test for unwrapped handles, kept in one place as in fly's C API, where -/// `!attr` is value arithmetic on a fly IntAttr rather than a null test. -template static bool isNull(T value) { - return !static_cast(value); -} - -template -static MlirType mmaOpGet(MlirContext context, int32_t m, int32_t n, int32_t k, - MlirType elemTyA, MlirType elemTyB, MlirType elemTyAcc, - Flags... flags) { - mlir::MLIRContext *ctx = loadFlyROCDL(context); - mlir::Type a = unwrap(elemTyA), b = unwrap(elemTyB), acc = unwrap(elemTyAcc); - if (!ctx || isNull(a) || isNull(b) || isNull(acc)) - return {nullptr}; - if (a.getContext() != ctx || b.getContext() != ctx || acc.getContext() != ctx) - return {nullptr}; - return wrap( - static_cast(Type::get(ctx, m, n, k, a, b, acc, flags...))); -} - -#define FLY_ROCDL_TYPE_ISA(Name, Type) \ - bool mlirTypeIsAFlyROCDL##Name(MlirType type) { \ - return mlir::isa_and_nonnull(unwrap(type)); \ - } - -FLY_ROCDL_TYPE_ISA(CopyOpCDNA3BufferCopy, - mlir::fly_rocdl::CopyOpCDNA3BufferCopyType) -FLY_ROCDL_TYPE_ISA(CopyOpCDNA3BufferCopyLDS, - mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType) -FLY_ROCDL_TYPE_ISA(MmaOpCDNA3MFMA, mlir::fly_rocdl::MmaOpCDNA3_MFMAType) -FLY_ROCDL_TYPE_ISA(MmaOpGFX11WMMA, mlir::fly_rocdl::MmaOpGFX11_WMMAType) -FLY_ROCDL_TYPE_ISA(MmaOpGFX120XWMMA, mlir::fly_rocdl::MmaOpGFX120X_WMMAType) - -#undef FLY_ROCDL_TYPE_ISA - -MlirAttribute mlirFlyROCDLBufferDescAddressAttrGet(MlirContext context) { - mlir::MLIRContext *ctx = loadFlyROCDL(context); - if (!ctx) - return {nullptr}; - return wrap(static_cast( - mlir::fly_rocdl::BufferDescAddressAttr::get(ctx))); -} - -MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(MlirContext context, - int32_t bitSize, - int32_t cacheModifier) { - mlir::MLIRContext *ctx = loadFlyROCDL(context); - if (!ctx) - return {nullptr}; - return wrap( - static_cast(mlir::fly_rocdl::CopyOpCDNA3BufferCopyType::get( - ctx, bitSize, cacheModifier))); -} - -MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(MlirContext context, - int32_t bitSize) { - mlir::MLIRContext *ctx = loadFlyROCDL(context); - if (!ctx) - return {nullptr}; - return wrap(static_cast( - mlir::fly_rocdl::CopyOpCDNA3BufferCopyLDSType::get(ctx, bitSize))); -} - -MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet(MlirContext context, int32_t m, - int32_t n, int32_t k, - MlirType elemTyA, MlirType elemTyB, - MlirType elemTyAcc) { - return mmaOpGet( - context, m, n, k, elemTyA, elemTyB, elemTyAcc); -} - -MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet(MlirContext context, int32_t m, - int32_t n, int32_t k, - MlirType elemTyA, MlirType elemTyB, - MlirType elemTyAcc, bool signA, - bool signB, bool clamp) { - return mmaOpGet( - context, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); -} - -MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet(MlirContext context, int32_t m, - int32_t n, int32_t k, - MlirType elemTyA, MlirType elemTyB, - MlirType elemTyAcc, bool signA, - bool signB, bool clamp) { - return mmaOpGet( - context, m, n, k, elemTyA, elemTyB, elemTyAcc, signA, signB, clamp); -} diff --git a/mlir/dialects/fly_rocdl/capi.h b/mlir/dialects/fly_rocdl/capi.h deleted file mode 100644 index 0ee248bb..00000000 --- a/mlir/dialects/fly_rocdl/capi.h +++ /dev/null @@ -1,45 +0,0 @@ -//===- capi.h - C API for the fly_rocdl dialect -------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_FLY_ROCDL_CAPI_H -#define MLIR_DIALECTS_FLY_ROCDL_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(FlyROCDL, fly_rocdl); - -MLIR_CAPI_EXPORTED MlirAttribute -mlirFlyROCDLBufferDescAddressAttrGet(MlirContext context); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopy(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet( - MlirContext context, int32_t bitSize, int32_t cacheModifier); - -MLIR_CAPI_EXPORTED bool -mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopyLDS(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet( - MlirContext context, int32_t bitSize); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpCDNA3MFMA(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpCDNA3MFMATypeGet( - MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, - MlirType elemTyB, MlirType elemTyAcc); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX11WMMA(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX11WMMATypeGet( - MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, - MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); - -MLIR_CAPI_EXPORTED bool mlirTypeIsAFlyROCDLMmaOpGFX120XWMMA(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirFlyROCDLMmaOpGFX120XWMMATypeGet( - MlirContext context, int32_t m, int32_t n, int32_t k, MlirType elemTyA, - MlirType elemTyB, MlirType elemTyAcc, bool signA, bool signB, bool clamp); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_FLY_ROCDL_CAPI_H diff --git a/mlir/dialects/fly_rocdl/fly_rocdl.zig b/mlir/dialects/fly_rocdl/fly_rocdl.zig deleted file mode 100644 index 7a91a925..00000000 --- a/mlir/dialects/fly_rocdl/fly_rocdl.zig +++ /dev/null @@ -1,156 +0,0 @@ -const c = @import("c"); -const mlir = @import("mlir"); - -pub const ops = @import("ops.zig"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("fly_rocdl"); -} - -/// `#fly_rocdl.buffer_desc`, an address space for buffer-descriptor pointers. -pub const BufferDescAddressAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn get(ctx: *mlir.Context) mlir.Error!*const Self { - return @ptrCast(c.mlirFlyROCDLBufferDescAddressAttrGet(ctx.ptr()).ptr orelse return error.InvalidMlir); - } -}; - -/// `!fly_rocdl.cdna3.buffer_copy`. -pub const CopyOpCDNA3BufferCopyType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - bitSize: i32, - /// 0 = cached, 2 = non-temporal. - cacheModifier: i32 = 0, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyROCDLCopyOpCDNA3BufferCopyTypeGet(ctx.ptr(), args.bitSize, args.cacheModifier); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!fly_rocdl.cdna3.buffer_copy_lds`. -pub const CopyOpCDNA3BufferCopyLDSType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyROCDLCopyOpCDNA3BufferCopyLDS; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - bitSize: i32, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyROCDLCopyOpCDNA3BufferCopyLDSTypeGet(ctx.ptr(), args.bitSize); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!fly_rocdl.cdna3.mfma`. -pub const MmaOpCDNA3MFMAType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpCDNA3MFMA; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = struct { - m: i32, - n: i32, - k: i32, - elemTyA: *const mlir.Type, - elemTyB: *const mlir.Type, - elemTyAcc: *const mlir.Type, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyROCDLMmaOpCDNA3MFMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!fly_rocdl.gfx11.wmma`. -pub const MmaOpGFX11WMMAType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpGFX11WMMA; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = WmmaInitArgs; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyROCDLMmaOpGFX11WMMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr(), args.signA, args.signB, args.clamp); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!fly_rocdl.gfx120x.wmma`. -pub const MmaOpGFX120XWMMAType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsAFlyROCDLMmaOpGFX120XWMMA; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - - pub const InitArgs = WmmaInitArgs; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirFlyROCDLMmaOpGFX120XWMMATypeGet(ctx.ptr(), args.m, args.n, args.k, args.elemTyA.ptr(), args.elemTyB.ptr(), args.elemTyAcc.ptr(), args.signA, args.signB, args.clamp); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `signA`/`signB`/`clamp` drive the integer WMMA intrinsics and must stay -/// false on the float paths. -pub const WmmaInitArgs = struct { - m: i32, - n: i32, - k: i32, - elemTyA: *const mlir.Type, - elemTyB: *const mlir.Type, - elemTyAcc: *const mlir.Type, - signA: bool = false, - signB: bool = false, - clamp: bool = false, -}; diff --git a/mlir/dialects/fly_rocdl/ops.zig b/mlir/dialects/fly_rocdl/ops.zig deleted file mode 100644 index 6cff2bf6..00000000 --- a/mlir/dialects/fly_rocdl/ops.zig +++ /dev/null @@ -1,83 +0,0 @@ -//! `fly_rocdl` operation builders, generated from FlyDSL's `include/flydsl/Dialect/FlyROCDL/IR/Ops.td` at -//! 1941889400621f1fc3d8f03bada1a4e7380cdf7e; do not edit, see ../fly/REGENERATING.md. -//! -//! Builders take the operands, then the result types (only for operations -//! that do not infer them), then the required attributes, then an `opts` -//! struct of optional operands and attributes (pass `.{}` to omit them), then -//! the location. Parameters are the ODS names in snake case, with a trailing -//! underscore where needed. Pass `&.{}` for empty variadic operands; -//! variable-length operand groups of -//! `AttrSizedOperandSegments` operations get their segment sizes from -//! `Operation.make`. Attributes use `mlir.Attribute`. - -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -const make = @import("mlir/dialects/fly").make; - -/// `fly_rocdl.get_buffer_rsrc`. Extract the raw ROCDL buffer resource from a buffer-descriptor pointer. Result types are inferred. -pub fn get_buffer_rsrc(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return make(ctx, "fly_rocdl.get_buffer_rsrc", .{ - .operands = .{ .flat = &.{ptr} }, - .result_type_inference = true, - .location = location, - }); -} - -/// `fly_rocdl.make_tiled_tdm_load_atom`. Build a CDNA5 TDM load atom and the coordinate tensor that addresses it. Result types are inferred. -pub fn make_tiled_tdm_load_atom( - ctx: *mlir.Context, - tensor: *const mlir.Value, - smem_layout: *const mlir.Value, - tiler: *const mlir.Value, - opts: struct { - num_warps: ?*const mlir.Attribute = null, - init_boundary_check: ?*const mlir.Attribute = null, - cache_modifier: ?*const mlir.Attribute = null, - atomic_barrier: ?*const mlir.Attribute = null, - internal_type: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); - if (opts.init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); - if (opts.cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); - if (opts.atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); - if (opts.internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); - return make(ctx, "fly_rocdl.make_tiled_tdm_load_atom", .{ - .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, - .attributes = attributes.constSlice(), - .result_type_inference = true, - .location = location, - }); -} - -/// `fly_rocdl.make_tiled_tdm_store_atom`. Build a CDNA5 TDM store atom and the coordinate tensor that addresses it. Result types are inferred. -pub fn make_tiled_tdm_store_atom( - ctx: *mlir.Context, - tensor: *const mlir.Value, - smem_layout: *const mlir.Value, - tiler: *const mlir.Value, - opts: struct { - num_warps: ?*const mlir.Attribute = null, - init_boundary_check: ?*const mlir.Attribute = null, - cache_modifier: ?*const mlir.Attribute = null, - atomic_barrier: ?*const mlir.Attribute = null, - internal_type: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.num_warps) |present| attributes.appendAssumeCapacity(.named(ctx, "numWarps", present)); - if (opts.init_boundary_check) |present| attributes.appendAssumeCapacity(.named(ctx, "initBoundaryCheck", present)); - if (opts.cache_modifier) |present| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", present)); - if (opts.atomic_barrier) |present| attributes.appendAssumeCapacity(.named(ctx, "atomicBarrier", present)); - if (opts.internal_type) |present| attributes.appendAssumeCapacity(.named(ctx, "internalType", present)); - return make(ctx, "fly_rocdl.make_tiled_tdm_store_atom", .{ - .operands = .{ .flat = &.{ tensor, smem_layout, tiler } }, - .attributes = attributes.constSlice(), - .result_type_inference = true, - .location = location, - }); -} diff --git a/mlir/dialects/fly_rocdl/test.zig b/mlir/dialects/fly_rocdl/test.zig deleted file mode 100644 index bb85d6e5..00000000 --- a/mlir/dialects/fly_rocdl/test.zig +++ /dev/null @@ -1,30 +0,0 @@ -const std = @import("std"); - -const fly_rocdl = @import("mlir/dialects/fly_rocdl"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "fly_rocdl.get_buffer_rsrc", - "fly_rocdl.make_tiled_tdm_load_atom", - "fly_rocdl.make_tiled_tdm_store_atom", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(fly_rocdl); - inline for (comptime std.meta.declarations(fly_rocdl)) |decl| { - const T = @field(fly_rocdl, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - fly_rocdl.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/func/func.zig b/mlir/dialects/func.zig similarity index 73% rename from mlir/dialects/func/func.zig rename to mlir/dialects/func.zig index 45c74674..b9f8b7f5 100644 --- a/mlir/dialects/func/func.zig +++ b/mlir/dialects/func.zig @@ -1,12 +1,8 @@ -const c = @import("c"); +const std = @import("std"); const mlir = @import("mlir"); const stdx = @import("stdx"); -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("func"); -} - const FuncOpArgs = struct { pub const Visibility = enum { public, @@ -21,22 +17,18 @@ const FuncOpArgs = struct { block: *mlir.Block, location: *const mlir.Location, no_inline: bool = false, - visibility: ?Visibility = null, - verify: bool = true, - extra_attributes: []const mlir.NamedAttribute = &.{}, + visibility: Visibility = .public, }; pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { - var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .empty; - var results_buffer: stdx.BoundedArray(*const mlir.Type, 64) = .empty; + var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .{}; + var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .{}; - var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_name", .string(ctx, args.name))); - if (args.visibility) |v| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "sym_visibility", .string(ctx, @tagName(v)))); - } + var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .{}; attr_tuples_buffer.appendSliceAssumeCapacity(&.{ - .named(ctx, "function_type", .typeAttr(.function( + .named(ctx, "sym_name", mlir.stringAttribute(ctx, args.name)), + .named(ctx, "sym_visibility", mlir.stringAttribute(ctx, @tagName(args.visibility))), + .named(ctx, "function_type", mlir.typeAttribute(mlir.functionType( ctx, args.args orelse args: { for (0..args.block.numArguments()) |i| { @@ -57,23 +49,19 @@ pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { }); if (args.args_attributes) |args_attributes| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, args_attributes))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", mlir.arrayAttribute(ctx, args_attributes))); } if (args.results_attributes) |results_attributes| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, results_attributes))); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", mlir.arrayAttribute(ctx, results_attributes))); } if (args.no_inline) { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", .unit(ctx))); - } - for (args.extra_attributes) |a| { - attr_tuples_buffer.appendAssumeCapacity(a); + attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "no_inline", mlir.unitAttribute(ctx))); } return mlir.Operation.make(ctx, "func.func", .{ .blocks = &.{args.block}, .attributes = attr_tuples_buffer.constSlice(), .location = args.location, - .verify = args.verify, }); } @@ -82,7 +70,7 @@ pub fn call(ctx: *mlir.Context, name: []const u8, values: []const *const mlir.Va .operands = .{ .variadic = &.{values} }, .results = .{ .flat = results }, .attributes = &.{ - .named(ctx, "callee", .flatSymbolRef(ctx, name)), + .named(ctx, "callee", mlir.flatSymbolRefAttribute(ctx, name)), }, .location = location, }); @@ -103,7 +91,3 @@ pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: .location = location, }); } - -pub fn registerExtensions(registry: *mlir.DialectRegistry) void { - c.mlirRegisterFuncExtensions(registry.ptr()); -} diff --git a/mlir/dialects/func/BUILD.bazel b/mlir/dialects/func/BUILD.bazel deleted file mode 100644 index 4524f2dd..00000000 --- a/mlir/dialects/func/BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:FuncExtensions", - ], -) - -zig_library( - name = "func", - import_name = "mlir/dialects/func", - main = "func.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":func", - "//mlir", - ], -) diff --git a/mlir/dialects/func/capi.cc b/mlir/dialects/func/capi.cc deleted file mode 100644 index 11a5e006..00000000 --- a/mlir/dialects/func/capi.cc +++ /dev/null @@ -1,10 +0,0 @@ -//===- capi.cc - C API for the func dialect -------------------------------===// - -#include "capi.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/Dialect/Func/Extensions/AllExtensions.h" - -void mlirRegisterFuncExtensions(MlirDialectRegistry registry) { - mlir::func::registerAllExtensions(*unwrap(registry)); -} diff --git a/mlir/dialects/func/capi.h b/mlir/dialects/func/capi.h deleted file mode 100644 index cd522b2d..00000000 --- a/mlir/dialects/func/capi.h +++ /dev/null @@ -1,20 +0,0 @@ -//===- capi.h - C API for the func dialect ------------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_FUNC_CAPI_H -#define MLIR_DIALECTS_FUNC_CAPI_H - -#include "mlir-c/Dialect/Func.h" -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_CAPI_EXPORTED void -mlirRegisterFuncExtensions(MlirDialectRegistry registry); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_FUNC_CAPI_H diff --git a/mlir/dialects/func/test.zig b/mlir/dialects/func/test.zig deleted file mode 100644 index 7a3b17f9..00000000 --- a/mlir/dialects/func/test.zig +++ /dev/null @@ -1,41 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "func.call", - "func.func", - "func.return", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(func); - inline for (comptime std.meta.declarations(func)) |decl| { - const T = @field(func, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "func dialect and extensions register" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - func.registerDialect(registry); - func.registerExtensions(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - try std.testing.expect(ctx.isRegisteredOperation("func.func")); -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - func.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/gpu/BUILD.bazel b/mlir/dialects/gpu/BUILD.bazel deleted file mode 100644 index 7264487a..00000000 --- a/mlir/dialects/gpu/BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:GPUDialect", - ], -) - -zig_library( - name = "gpu", - import_name = "mlir/dialects/gpu", - main = "gpu.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":gpu", - "//mlir", - ], -) diff --git a/mlir/dialects/gpu/capi.cc b/mlir/dialects/gpu/capi.cc deleted file mode 100644 index 2e215d58..00000000 --- a/mlir/dialects/gpu/capi.cc +++ /dev/null @@ -1,52 +0,0 @@ -//===- capi.cc - C API for the gpu dialect --------------------------------===// - -#include "capi.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" -#include "mlir/Dialect/GPU/IR/GPUDialect.h" - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(GPU, gpu, mlir::gpu::GPUDialect) - -static mlir::MLIRContext *loadGPU(MlirContext context) { - mlir::MLIRContext *ctx = unwrap(context); - if (ctx) - ctx->getOrLoadDialect(); - return ctx; -} - -bool mlirAttributeIsAGPUDimension(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirGPUDimensionAttrGet(MlirContext context, uint32_t value) { - mlir::MLIRContext *ctx = loadGPU(context); - std::optional dimension = - mlir::gpu::symbolizeDimension(value); - if (!ctx || !dimension) - return {nullptr}; - return wrap(mlir::gpu::DimensionAttr::get(ctx, *dimension)); -} - -uint32_t mlirGPUDimensionAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAGPUShuffleMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext context, uint32_t value) { - mlir::MLIRContext *ctx = loadGPU(context); - std::optional mode = - mlir::gpu::symbolizeShuffleMode(value); - if (!ctx || !mode) - return {nullptr}; - return wrap(mlir::gpu::ShuffleModeAttr::get(ctx, *mode)); -} - -uint32_t mlirGPUShuffleModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} diff --git a/mlir/dialects/gpu/capi.h b/mlir/dialects/gpu/capi.h deleted file mode 100644 index 670e23a9..00000000 --- a/mlir/dialects/gpu/capi.h +++ /dev/null @@ -1,37 +0,0 @@ -//===- capi.h - C API for the gpu dialect -------------------------*- C -*-===// -// -// C entry points for the `gpu` dialect that upstream's C API leaves out: its -// registration (outside `CAPIGPU`, which also brings the GPU passes) and the -// enum attributes kernels build. - -#ifndef MLIR_DIALECTS_GPU_CAPI_H -#define MLIR_DIALECTS_GPU_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(GPU, gpu); - -// Enum attributes take and return the C++ enum's integer value; `Get` returns -// a null attribute for a value outside the enum. - -// #gpu -MLIR_CAPI_EXPORTED bool mlirAttributeIsAGPUDimension(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirGPUDimensionAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirGPUDimensionAttrGetValue(MlirAttribute attr); - -// #gpu -MLIR_CAPI_EXPORTED bool mlirAttributeIsAGPUShuffleMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirGPUShuffleModeAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirGPUShuffleModeAttrGetValue(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_GPU_CAPI_H diff --git a/mlir/dialects/gpu/gpu.zig b/mlir/dialects/gpu/gpu.zig deleted file mode 100644 index 4b9df47f..00000000 --- a/mlir/dialects/gpu/gpu.zig +++ /dev/null @@ -1,169 +0,0 @@ -const c = @import("c"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("gpu"); -} - -/// A dimension, either `x`, `y` or `z`. -pub const Dim = enum(u32) { - x = 0, - y = 1, - z = 2, -}; - -/// Indexing modes supported by `gpu.shuffle`. -pub const ShuffleMode = enum(u32) { - xor = 0, - down = 1, - up = 2, - idx = 3, -}; - -/// `#gpu`. -pub const DimensionAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAGPUDimension; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "dim"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Dim, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirGPUDimensionAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Dim { - return @fromBackingInt(@intCast(c.mlirGPUDimensionAttrGetValue(self.ptr()))); - } -}; - -/// `#gpu`. -pub const ShuffleModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAGPUShuffleMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "shuffle_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ShuffleMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirGPUShuffleModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ShuffleMode { - return @fromBackingInt(@intCast(c.mlirGPUShuffleModeAttrGetValue(self.ptr()))); - } -}; - -/// `gpu.container_module`: marks a module whose `gpu.module`s hold the kernels -/// it launches. -pub fn containerModule(ctx: *mlir.Context) mlir.NamedAttribute { - return .named(ctx, "gpu.container_module", .unit(ctx)); -} - -/// `gpu.kernel`: marks a function as a kernel entry point. -pub fn kernel(ctx: *mlir.Context) mlir.NamedAttribute { - return .named(ctx, "gpu.kernel", .unit(ctx)); -} - -pub fn module(ctx: *mlir.Context, name: []const u8, body: *mlir.Block, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "gpu.module", .{ - .blocks = &.{body}, - .attributes = &.{ - .named(ctx, "sym_name", .string(ctx, name)), - }, - .location = location, - .verify = false, - }); -} - -pub const FuncArgs = struct { - name: []const u8, - args: []const *const mlir.Type, - block: *mlir.Block, - location: *const mlir.Location, - kernel: bool = true, - extra_attributes: []const mlir.NamedAttribute = &.{}, -}; - -/// `gpu.func @name(args) kernel { ... gpu.return }`. -pub fn func(ctx: *mlir.Context, args: FuncArgs) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; - attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "sym_name", .string(ctx, args.name)), - .named(ctx, "function_type", .typeAttr(.function(ctx, args.args, &.{}))), - }); - if (args.kernel) attrs.appendAssumeCapacity(kernel(ctx)); - attrs.appendSliceAssumeCapacity(args.extra_attributes); - return mlir.Operation.make(ctx, "gpu.func", .{ - .blocks = &.{args.block}, - .attributes = attrs.constSlice(), - .location = args.location, - .verify = false, - }); -} - -pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "gpu.return", .{ - .operands = .{ .flat = values }, - .location = location, - .verify = false, - }); -} - -fn idOp(ctx: *mlir.Context, comptime name: []const u8, dim: Dim, location: *const mlir.Location) *mlir.Operation { - const dimension = DimensionAttr.get(ctx, .{ .value = dim }) catch unreachable; - return mlir.Operation.make(ctx, name, .{ - .results = .{ .flat = &.{.index(ctx)} }, - .attributes = &.{.named(ctx, "dimension", dimension.attribute())}, - .location = location, - }); -} - -pub fn thread_id(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { - return idOp(ctx, "gpu.thread_id", dim, location); -} - -pub fn block_id(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { - return idOp(ctx, "gpu.block_id", dim, location); -} - -pub fn block_dim(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { - return idOp(ctx, "gpu.block_dim", dim, location); -} - -pub fn grid_dim(ctx: *mlir.Context, dim: Dim, location: *const mlir.Location) *mlir.Operation { - return idOp(ctx, "gpu.grid_dim", dim, location); -} - -/// `gpu.shuffle`: `value` from lane `offset` (per `mode`) within `width` lanes, -/// and whether that lane was valid (i1). -pub fn shuffle(ctx: *mlir.Context, value: *const mlir.Value, offset: *const mlir.Value, width: *const mlir.Value, mode: ShuffleMode, location: *const mlir.Location) *mlir.Operation { - const mode_attr = ShuffleModeAttr.get(ctx, .{ .value = mode }) catch unreachable; - return mlir.Operation.make(ctx, "gpu.shuffle", .{ - .operands = .{ .flat = &.{ value, offset, width } }, - .results = .{ .flat = &.{ value.type_(), .int(ctx, .i1) } }, - .attributes = &.{.named(ctx, "mode", mode_attr.attribute())}, - .location = location, - }); -} - -pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "gpu.barrier", .{ .location = location }); -} diff --git a/mlir/dialects/gpu/test.zig b/mlir/dialects/gpu/test.zig deleted file mode 100644 index d0f4a80d..00000000 --- a/mlir/dialects/gpu/test.zig +++ /dev/null @@ -1,71 +0,0 @@ -const std = @import("std"); - -const gpu = @import("mlir/dialects/gpu"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "gpu.barrier", - "gpu.block_dim", - "gpu.block_id", - "gpu.func", - "gpu.grid_dim", - "gpu.module", - "gpu.return", - "gpu.shuffle", - "gpu.thread_id", -}; - -fn expectPrints(expected: []const u8, value: anytype) !void { - var buf: [128]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(expected, w.buffered()); -} - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(gpu); - inline for (comptime std.meta.declarations(gpu)) |decl| { - const T = @field(gpu, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "gpu enum attributes are built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - inline for (@typeInfo(gpu.Dim).@"enum".field_names) |name| { - const value: gpu.Dim = @field(gpu.Dim, name); - const attr = try gpu.DimensionAttr.get(ctx, .{ .value = value }); - try std.testing.expectEqual(value, attr.value()); - try std.testing.expect(attr.attribute().isA(gpu.DimensionAttr) != null); - try expectPrints("#gpu.dim<" ++ name ++ ">", attr); - } - inline for (@typeInfo(gpu.ShuffleMode).@"enum".field_names) |name| { - const value: gpu.ShuffleMode = @field(gpu.ShuffleMode, name); - const attr = try gpu.ShuffleModeAttr.get(ctx, .{ .value = value }); - try std.testing.expectEqual(value, attr.value()); - try expectPrints("#gpu.shuffle_mode<" ++ name ++ ">", attr); - } -} - -test "container_module and kernel are unit attributes of the gpu dialect" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - var buf: [128]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ gpu.containerModule(ctx), gpu.kernel(ctx) })}); - try std.testing.expectEqualStrings("{gpu.container_module, gpu.kernel}", w.buffered()); -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - gpu.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/llvm/BUILD.bazel b/mlir/dialects/llvm/BUILD.bazel deleted file mode 100644 index a03d215e..00000000 --- a/mlir/dialects/llvm/BUILD.bazel +++ /dev/null @@ -1,38 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:CAPILLVM", - "@llvm-project//mlir:LLVMDialect", - ], -) - -zig_library( - name = "llvm", - srcs = ["ops.zig"], - import_name = "mlir/dialects/llvm", - main = "llvm.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//stdx", - "@llvm-project//mlir:CAPILLVM", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":llvm", - "//mlir", - ], -) diff --git a/mlir/dialects/llvm/capi.cc b/mlir/dialects/llvm/capi.cc deleted file mode 100644 index 85b480e3..00000000 --- a/mlir/dialects/llvm/capi.cc +++ /dev/null @@ -1,55 +0,0 @@ -//===- capi.cc - C API for the llvm dialect -------------------------------===// - -#include "capi.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" - -using namespace mlir; - -static MLIRContext *loadLLVM(MlirContext context) { - MLIRContext *ctx = unwrap(context); - if (ctx) - ctx->getOrLoadDialect(); - return ctx; -} - -template static T optional(MlirAttribute attr) { - return llvm::dyn_cast_or_null(unwrap(attr)); -} - -bool mlirAttributeIsALLVMLoopUnroll(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirLLVMLoopUnrollAttrGet(MlirContext context, - MlirAttribute disable, - MlirAttribute count, - MlirAttribute runtimeDisable, - MlirAttribute full) { - MLIRContext *ctx = loadLLVM(context); - if (!ctx) - return {nullptr}; - return wrap(LLVM::LoopUnrollAttr::get(ctx, optional(disable), - optional(count), - optional(runtimeDisable), - optional(full), {}, {}, {})); -} - -bool mlirAttributeIsALLVMLoopAnnotation(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirLLVMLoopAnnotationAttrGet(MlirContext context, - MlirAttribute disableNonforced, - MlirAttribute unroll, - MlirAttribute mustProgress) { - MLIRContext *ctx = loadLLVM(context); - if (!ctx) - return {nullptr}; - return wrap(LLVM::LoopAnnotationAttr::get( - ctx, optional(disableNonforced), {}, {}, - optional(unroll), {}, {}, {}, {}, {}, {}, - optional(mustProgress), {}, {}, {}, {})); -} diff --git a/mlir/dialects/llvm/capi.h b/mlir/dialects/llvm/capi.h deleted file mode 100644 index 04528ddc..00000000 --- a/mlir/dialects/llvm/capi.h +++ /dev/null @@ -1,36 +0,0 @@ -//===- capi.h - C API for the llvm dialect ------------------------*- C -*-===// -// -// C entry points for the `llvm` dialect's loop annotations, which upstream's -// C API leaves out. - -#ifndef MLIR_DIALECTS_LLVM_CAPI_H -#define MLIR_DIALECTS_LLVM_CAPI_H - -#include "mlir-c/Dialect/LLVM.h" -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Null parameters are unset. - -MLIR_CAPI_EXPORTED bool mlirAttributeIsALLVMLoopUnroll(MlirAttribute attr); -/// `disable`, `runtimeDisable` and `full` are bool attributes, `count` an -/// integer attribute. -MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMLoopUnrollAttrGet( - MlirContext context, MlirAttribute disable, MlirAttribute count, - MlirAttribute runtimeDisable, MlirAttribute full); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsALLVMLoopAnnotation(MlirAttribute attr); -/// `disableNonforced` and `mustProgress` are bool attributes, `unroll` a -/// loop unroll attribute; the other parameters are unset. -MLIR_CAPI_EXPORTED MlirAttribute mlirLLVMLoopAnnotationAttrGet( - MlirContext context, MlirAttribute disableNonforced, MlirAttribute unroll, - MlirAttribute mustProgress); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_LLVM_CAPI_H diff --git a/mlir/dialects/llvm/llvm.zig b/mlir/dialects/llvm/llvm.zig deleted file mode 100644 index d1197317..00000000 --- a/mlir/dialects/llvm/llvm.zig +++ /dev/null @@ -1,52 +0,0 @@ -// Keep function definitions in ops.zig: LLVM reserves symbols starting with "llvm.". -const ops = @import("ops.zig"); - -pub const NVPTXAddressSpace = ops.NVPTXAddressSpace; -pub const pointerType = ops.pointerType; -pub const structType = ops.structType; -pub const identifiedStructType = ops.identifiedStructType; -pub const arrayType = ops.arrayType; -pub const LoopUnrollAttr = ops.LoopUnrollAttr; -pub const LoopAnnotationAttr = ops.LoopAnnotationAttr; -pub const Linkage = ops.Linkage; -pub const linkageAttr = ops.linkageAttr; -pub const emitCInterface = ops.emitCInterface; -pub const linkage = ops.linkage; -pub const inlineHint = ops.inlineHint; -pub const noundef = ops.noundef; -pub const zeroext = ops.zeroext; -pub const signext = ops.signext; -pub const alignment = ops.alignment; -pub const dereferenceable = ops.dereferenceable; -pub const elementtype = ops.elementtype; -pub const AtomicBinOp = ops.AtomicBinOp; -pub const AtomicOrdering = ops.AtomicOrdering; -pub const inttoptr = ops.inttoptr; -pub const ptrtoint = ops.ptrtoint; -pub const bitcast = ops.bitcast; -pub const fmul = ops.fmul; -pub const extractvalue = ops.extractvalue; -pub const LoadOpts = ops.LoadOpts; -pub const load = ops.load; -pub const StoreOpts = ops.StoreOpts; -pub const store = ops.store; -pub const GEPIndex = ops.GEPIndex; -pub const getelementptr = ops.getelementptr; -pub const alloca = ops.alloca; -pub const atomicrmw = ops.atomicrmw; -pub const intr_assume = ops.intr_assume; -pub const intr_lifetime_start = ops.intr_lifetime_start; -pub const intr_lifetime_end = ops.intr_lifetime_end; -pub const extractelement = ops.extractelement; -pub const insertelement = ops.insertelement; -pub const shufflevector = ops.shufflevector; -pub const mlir_undef = ops.mlir_undef; -pub const mlir_poison = ops.mlir_poison; -pub const freeze = ops.freeze; -pub const BrOpts = ops.BrOpts; -pub const br = ops.br; -pub const unreachable_ = ops.unreachable_; -pub const call_intrinsic = ops.call_intrinsic; -pub const InlineAsmOpts = ops.InlineAsmOpts; -pub const inline_asm = ops.inline_asm; -pub const registerDialect = ops.registerDialect; diff --git a/mlir/dialects/llvm/ops.zig b/mlir/dialects/llvm/ops.zig deleted file mode 100644 index d1fb5771..00000000 --- a/mlir/dialects/llvm/ops.zig +++ /dev/null @@ -1,567 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("llvm"); -} - -// ============================================================================= -// Types -// ============================================================================= - -/// Address spaces of `!llvm.ptr` on NVPTX. -pub const NVPTXAddressSpace = enum(u32) { - generic = 0, - global = 1, - shared = 3, - constant = 4, - local = 5, - /// Tensor memory (SM100 TMEM). - tensor = 6, - /// Shared memory of any CTA of the cluster (`shared::cluster`). - shared_cluster = 7, -}; - -/// `!llvm.ptr`, an opaque pointer. Loads the LLVM dialect into `ctx`. -pub fn pointerType(ctx: *mlir.Context, space: anytype) *const mlir.Type { - _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); - return @ptrCast(c.mlirLLVMPointerTypeGet(ctx.ptr(), @intCast(@backingInt(space))).ptr); -} - -/// `!llvm.struct<(fields...)>`, a literal struct. -pub fn structType(ctx: *mlir.Context, fields: []const *const mlir.Type) *const mlir.Type { - _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); - return @ptrCast(c.mlirLLVMStructTypeLiteralGet(ctx.ptr(), @intCast(fields.len), @ptrCast(fields), false).ptr); -} - -/// `!llvm.struct<"name", (fields...)>`, an identified struct; null if `name` -/// already names a struct of other fields. -pub fn identifiedStructType(ctx: *mlir.Context, name: []const u8, fields: []const *const mlir.Type) ?*const mlir.Type { - _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); - const t = c.mlirLLVMStructTypeIdentifiedGet(ctx.ptr(), mlir.stringRef(name)); - if (c.mlirLLVMStructTypeSetBody(t, @intCast(fields.len), @ptrCast(fields), false).value == 0) return null; - return @ptrCast(t.ptr); -} - -/// `!llvm.array`. -pub fn arrayType(element_type: *const mlir.Type, count: u32) *const mlir.Type { - return @ptrCast(c.mlirLLVMArrayTypeGet(element_type.ptr(), count).ptr); -} - -// ============================================================================= -// Attributes -// ============================================================================= - -fn optionalBool(ctx: *mlir.Context, value: ?bool) c.MlirAttribute { - return if (value) |v| mlir.Attribute.boolean(ctx, v).ptr() else .{ .ptr = null }; -} - -/// `#llvm.loop_unroll`: the `llvm.loop.unroll.*` loop metadata. -pub const LoopUnrollAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsALLVMLoopUnroll; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - disable: ?bool = null, - count: ?u32 = null, - runtimeDisable: ?bool = null, - full: ?bool = null, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const Self { - const count: c.MlirAttribute = if (args.count) |n| mlir.Attribute.int(ctx, .i32, n).ptr() else .{ .ptr = null }; - return @ptrCast(c.mlirLLVMLoopUnrollAttrGet(ctx.ptr(), optionalBool(ctx, args.disable), count, optionalBool(ctx, args.runtimeDisable), optionalBool(ctx, args.full)).ptr); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// A global's linkage, as `#llvm.linkage` spells it. -pub const Linkage = enum(c_uint) { - external = c.MlirLLVMLinkageExternal, - available_externally = c.MlirLLVMLinkageAvailableExternally, - linkonce = c.MlirLLVMLinkageLinkonce, - linkonce_odr = c.MlirLLVMLinkageLinkonceODR, - weak = c.MlirLLVMLinkageWeak, - weak_odr = c.MlirLLVMLinkageWeakODR, - appending = c.MlirLLVMLinkageAppending, - internal = c.MlirLLVMLinkageInternal, - private = c.MlirLLVMLinkagePrivate, - extern_weak = c.MlirLLVMLinkageExternWeak, - common = c.MlirLLVMLinkageCommon, -}; - -/// `#llvm.linkage`; `llvm.linkage` on a `func.func` sets its -/// linkage once lowered to LLVM. Loads the LLVM dialect into `ctx`, which -/// must have it registered. -pub fn linkageAttr(ctx: *mlir.Context, value: Linkage) *const mlir.Attribute { - _ = c.mlirContextGetOrLoadDialect(ctx.ptr(), mlir.stringRef("llvm")); - return @ptrCast(c.mlirLLVMLinkageAttrGet(ctx.ptr(), @backingInt(value)).ptr); -} - -/// `llvm.emit_c_interface`: has the lowering of a `func.func` emit a -/// `_mlir_ciface_` wrapper that takes its memrefs by pointer. -pub fn emitCInterface(ctx: *mlir.Context) mlir.NamedAttribute { - return .named(ctx, "llvm.emit_c_interface", .unit(ctx)); -} - -/// `llvm.linkage = #llvm.linkage` on a `func.func`. -pub fn linkage(ctx: *mlir.Context, value: Linkage) mlir.NamedAttribute { - return .named(ctx, "llvm.linkage", linkageAttr(ctx, value)); -} - -/// `llvm.inline_hint` on a `func.func`: `inlinehint` once lowered. -pub fn inlineHint(ctx: *mlir.Context) mlir.NamedAttribute { - return .named(ctx, "llvm.inline_hint", .unit(ctx)); -} - -/// `llvm.noundef` on an argument: never undef or poison. -pub fn noundef(ctx: *mlir.Context) mlir.NamedAttribute { - return .named(ctx, "llvm.noundef", .unit(ctx)); -} - -/// `llvm.zeroext` on an argument: a narrow integer passed zero-extended. -pub fn zeroext(ctx: *mlir.Context) mlir.NamedAttribute { - return .named(ctx, "llvm.zeroext", .unit(ctx)); -} - -/// `llvm.signext` on an argument: a narrow integer passed sign-extended. -pub fn signext(ctx: *mlir.Context) mlir.NamedAttribute { - return .named(ctx, "llvm.signext", .unit(ctx)); -} - -/// `llvm.align = bytes` on a pointer argument. -pub fn alignment(ctx: *mlir.Context, bytes: u64) mlir.NamedAttribute { - return .named(ctx, "llvm.align", .int(ctx, .i64, @as(i64, @intCast(bytes)))); -} - -/// `llvm.dereferenceable = bytes` on a pointer argument: that many bytes are -/// readable. -pub fn dereferenceable(ctx: *mlir.Context, bytes: u64) mlir.NamedAttribute { - return .named(ctx, "llvm.dereferenceable", .int(ctx, .i64, @as(i64, @intCast(bytes)))); -} - -/// `llvm.elementtype = type` on a pointer argument: its pointee, which -/// typed-pointer IR prints. -pub fn elementtype(ctx: *mlir.Context, pointee: *const mlir.Type) mlir.NamedAttribute { - return .named(ctx, "llvm.elementtype", .typeAttr(pointee)); -} - -/// The operation of `llvm.atomicrmw`, as the LLVM dialect numbers it. -pub const AtomicBinOp = enum(i64) { - xchg = 0, - add = 1, - sub = 2, - _and = 3, - nand = 4, - _or = 5, - _xor = 6, - max = 7, - min = 8, - umax = 9, - umin = 10, - fadd = 11, - fsub = 12, - fmax = 13, - fmin = 14, - uinc_wrap = 15, - udec_wrap = 16, - usub_cond = 17, - usub_sat = 18, - fmaximum = 19, - fminimum = 20, - fmaximumnum = 21, - fminimumnum = 22, -}; - -/// The memory ordering of an atomic operation, as the LLVM dialect numbers it. -pub const AtomicOrdering = enum(i64) { - not_atomic = 0, - unordered = 1, - monotonic = 2, - acquire = 4, - release = 5, - acq_rel = 6, - seq_cst = 7, -}; - -/// `#llvm.loop_annotation`: the `llvm.loop` metadata of a loop. -pub const LoopAnnotationAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsALLVMLoopAnnotation; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - disableNonforced: ?bool = null, - unroll: ?*const LoopUnrollAttr = null, - mustProgress: ?bool = null, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const Self { - const unroll: c.MlirAttribute = if (args.unroll) |u| u.ptr() else .{ .ptr = null }; - return @ptrCast(c.mlirLLVMLoopAnnotationAttrGet(ctx.ptr(), optionalBool(ctx, args.disableNonforced), unroll, optionalBool(ctx, args.mustProgress)).ptr); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -// ============================================================================= -// Casts and arithmetic -// ============================================================================= - -/// llvm.inttoptr — an integer address as a pointer of type `result_type`. -pub fn inttoptr(ctx: *mlir.Context, address: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.inttoptr", .{ - .operands = .{ .flat = &.{address} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// llvm.ptrtoint — the integer address of an LLVM pointer. -pub fn ptrtoint(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.ptrtoint", .{ - .operands = .{ .flat = &.{pointer} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// llvm.bitcast — reinterpret `value` as `result_type` (same bit width). -pub fn bitcast(ctx: *mlir.Context, value: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.bitcast", .{ - .operands = .{ .flat = &.{value} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// llvm.fmul — floating-point (or floating-point vector) product of type `result_type`. -pub fn fmul(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.fmul", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -// ============================================================================= -// Aggregates -// ============================================================================= - -/// llvm.extractvalue — the element of type `result_type` at `position` in the -/// struct or array `container`. -pub fn extractvalue(ctx: *mlir.Context, container: *const mlir.Value, result_type: *const mlir.Type, position: []const i64, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.extractvalue", .{ - .operands = .{ .flat = &.{container} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{.named(ctx, "position", .denseArray(ctx, .i64, position))}, - .location = location, - }); -} - -// ============================================================================= -// Memory -// ============================================================================= - -pub const LoadOpts = struct { - /// Alignment in bytes (none: the natural alignment of the type). - alignment: ?u64 = null, - /// The memory is not written while the kernel runs (`!invariant.load`; on NVPTX - /// global memory this selects the non-coherent `ld.global.nc`). - invariant: bool = false, -}; - -/// llvm.load — a value of type `result_type` through `pointer`. -pub fn load(ctx: *mlir.Context, pointer: *const mlir.Value, result_type: *const mlir.Type, opts: LoadOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.alignment) |bytes| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); - } - if (opts.invariant) { - attrs.appendAssumeCapacity(.named(ctx, "invariant", .unit(ctx))); - } - return mlir.Operation.make(ctx, "llvm.load", .{ - .operands = .{ .flat = &.{pointer} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -pub const StoreOpts = struct { - /// Alignment in bytes (none: the natural alignment of the type). - alignment: ?u64 = null, -}; - -/// llvm.store — `value` through `pointer`. -pub fn store(ctx: *mlir.Context, value: *const mlir.Value, pointer: *const mlir.Value, opts: StoreOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.alignment) |bytes| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); - } - return mlir.Operation.make(ctx, "llvm.store", .{ - .operands = .{ .flat = &.{ value, pointer } }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// An index of `getelementptr`: a constant, or an SSA integer. -pub const GEPIndex = union(enum) { - constant: i32, - value: *const mlir.Value, -}; - -/// llvm.getelementptr — `base` offset by `indices` into `elem_type`, with -/// `base`'s pointer type. -pub fn getelementptr( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const GEPIndex, - elem_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - const dynamic_index = std.math.minInt(i32); - var operands: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - var raw: stdx.BoundedArray(i32, 16) = .empty; - operands.appendAssumeCapacity(base); - for (indices) |index| switch (index) { - .constant => |c_| raw.appendAssumeCapacity(c_), - .value => |v| { - raw.appendAssumeCapacity(dynamic_index); - operands.appendAssumeCapacity(v); - }, - }; - return mlir.Operation.make(ctx, "llvm.getelementptr", .{ - .operands = .{ .flat = operands.constSlice() }, - .results = .{ .flat = &.{base.type_()} }, - .attributes = &.{ - .named(ctx, "rawConstantIndices", .denseArray(ctx, .i32, raw.constSlice())), - .named(ctx, "elem_type", .typeAttr(elem_type)), - }, - .location = location, - }); -} - -/// llvm.alloca — stack storage for `array_size` elements of `elem_type`, as a -/// pointer of type `result_type`. -pub fn alloca( - ctx: *mlir.Context, - array_size: *const mlir.Value, - result_type: *const mlir.Type, - elem_type: *const mlir.Type, - opts: struct { - alignment: ?u64 = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "elem_type", .typeAttr(elem_type))); - if (opts.alignment) |bytes| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); - return mlir.Operation.make(ctx, "llvm.alloca", .{ - .operands = .{ .flat = &.{array_size} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// llvm.atomicrmw — combine `val` into the value at `ptr` with `bin_op`, -/// atomically; the result is the previous value, of `val`'s type. -pub fn atomicrmw( - ctx: *mlir.Context, - ptr: *const mlir.Value, - val: *const mlir.Value, - bin_op: AtomicBinOp, - ordering: AtomicOrdering, - opts: struct { - /// Synchronization scope, e.g. `"agent"` (none: the whole system). - syncscope: ?[]const u8 = null, - /// Alignment in bytes (none: the natural alignment of the type). - alignment: ?u64 = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "bin_op", .int(ctx, .i64, @backingInt(bin_op))), - .named(ctx, "ordering", .int(ctx, .i64, @backingInt(ordering))), - }); - if (opts.syncscope) |syncscope| attrs.appendAssumeCapacity(.named(ctx, "syncscope", .string(ctx, syncscope))); - if (opts.alignment) |bytes| attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, bytes))); - return mlir.Operation.make(ctx, "llvm.atomicrmw", .{ - .operands = .{ .flat = &.{ ptr, val } }, - .results = .{ .flat = &.{val.type_()} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -// ============================================================================= -// Intrinsics and inline assembly -// ============================================================================= - -/// llvm.intr.assume — let the optimizer assume that `cond` (i1) holds. -pub fn intr_assume(ctx: *mlir.Context, cond: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.intr.assume", .{ - .operands = .{ .flat = &.{cond} }, - .attributes = &.{ - .named(ctx, "op_bundle_sizes", .denseArray(ctx, .i32, &.{})), - .named(ctx, "op_bundle_tags", .array(ctx, &.{})), - }, - .location = location, - }); -} - -pub fn intr_lifetime_start(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.intr.lifetime.start", .{ - .operands = .{ .flat = &.{ptr} }, - .location = location, - }); -} - -pub fn intr_lifetime_end(ctx: *mlir.Context, ptr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.intr.lifetime.end", .{ - .operands = .{ .flat = &.{ptr} }, - .location = location, - }); -} - -pub fn extractelement(ctx: *mlir.Context, vector: *const mlir.Value, position: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.extractelement", .{ - .operands = .{ .flat = &.{ vector, position } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn insertelement(ctx: *mlir.Context, vector: *const mlir.Value, value: *const mlir.Value, position: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.insertelement", .{ - .operands = .{ .flat = &.{ vector, value, position } }, - .results = .{ .flat = &.{vector.type_()} }, - .location = location, - }); -} - -pub fn shufflevector(ctx: *mlir.Context, v1: *const mlir.Value, v2: *const mlir.Value, result_type: *const mlir.Type, mask: []const i32, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.shufflevector", .{ - .operands = .{ .flat = &.{ v1, v2 } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{.named(ctx, "mask", .denseArray(ctx, .i32, mask))}, - .location = location, - }); -} - -pub fn mlir_undef(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.mlir.undef", .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn mlir_poison(ctx: *mlir.Context, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.mlir.poison", .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn freeze(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.freeze", .{ - .operands = .{ .flat = &.{value} }, - .results = .{ .flat = &.{value.type_()} }, - .location = location, - }); -} - -pub const BrOpts = struct { - loop_annotation: ?*const LoopAnnotationAttr = null, -}; - -/// `llvm.br`. Not verified: a terminator only verifies inside its block. -pub fn br(ctx: *mlir.Context, dest: *mlir.Block, dest_operands: []const *const mlir.Value, opts: BrOpts, location: *const mlir.Location) *mlir.Operation { - var state: mlir.OperationState = .init("llvm.br", location); - state.addOperands(dest_operands); - var successors: [1]*const mlir.Block = .{dest}; - state.addSuccessors(&successors); - if (opts.loop_annotation) |annotation| state.addAttributes(&.{.named(ctx, "loop_annotation", annotation.attribute())}); - return mlir.Operation.init(&state) catch @panic("llvm.br: failed to create the operation"); -} - -/// `llvm.unreachable`. Not verified: a terminator only verifies inside its -/// block. -pub fn unreachable_(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.unreachable", .{ .location = location, .verify = false }); -} - -/// llvm.call_intrinsic — call the LLVM intrinsic `name` (e.g. `llvm.nvvm.*`), with an -/// optional result of type `result_type`. -pub fn call_intrinsic( - ctx: *mlir.Context, - name: []const u8, - operands: []const *const mlir.Value, - opts: struct { - result_type: ?*const mlir.Type = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "llvm.call_intrinsic", .{ - .operands = .{ .flat = operands }, - .results = .{ .flat = if (opts.result_type) |r| &.{r} else &.{} }, - .attributes = &.{ - .named(ctx, "intrin", .string(ctx, name)), - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ @intCast(operands.len), 0 })), - .named(ctx, "op_bundle_sizes", .denseArray(ctx, .i32, &.{})), - }, - .location = location, - }); -} - -pub const InlineAsmOpts = struct { - result_type: ?*const mlir.Type = null, - - /// LLVM constraint string, e.g. `"=r,r"` (outputs first). - constraints: []const u8 = "", - /// The assembly has effects beyond its outputs (it is never removed or merged). - has_side_effects: bool = false, -}; - -/// llvm.inline_asm — the PTX (or other target) assembly `asm_string`, with `$i` -/// placeholders bound to the result then `operands`. -pub fn inline_asm( - ctx: *mlir.Context, - asm_string: []const u8, - operands: []const *const mlir.Value, - opts: InlineAsmOpts, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "asm_string", .string(ctx, asm_string))); - attrs.appendAssumeCapacity(.named(ctx, "constraints", .string(ctx, opts.constraints))); - if (opts.has_side_effects) { - attrs.appendAssumeCapacity(.named(ctx, "has_side_effects", .unit(ctx))); - } - return mlir.Operation.make(ctx, "llvm.inline_asm", .{ - .operands = .{ .flat = operands }, - .results = .{ .flat = if (opts.result_type) |r| &.{r} else &.{} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} diff --git a/mlir/dialects/llvm/test.zig b/mlir/dialects/llvm/test.zig deleted file mode 100644 index 6ba84b4d..00000000 --- a/mlir/dialects/llvm/test.zig +++ /dev/null @@ -1,210 +0,0 @@ -const std = @import("std"); - -const llvm = @import("mlir/dialects/llvm"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "llvm.alloca", - "llvm.atomicrmw", - "llvm.bitcast", - "llvm.br", - "llvm.call_intrinsic", - "llvm.extractelement", - "llvm.extractvalue", - "llvm.fmul", - "llvm.freeze", - "llvm.getelementptr", - "llvm.inline_asm", - "llvm.insertelement", - "llvm.intr.assume", - "llvm.intr.lifetime.end", - "llvm.intr.lifetime.start", - "llvm.inttoptr", - "llvm.ptrtoint", - "llvm.load", - "llvm.mlir.poison", - "llvm.mlir.undef", - "llvm.shufflevector", - "llvm.store", - "llvm.unreachable", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(llvm); - inline for (comptime std.meta.declarations(llvm)) |decl| { - const T = @field(llvm, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - llvm.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} - -test "literal struct types build from their fields" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - llvm.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - const struct_type = llvm.structType(ctx, &.{ .int(ctx, .i32), .float(ctx, .f32) }); - try std.testing.expect(struct_type.eql(try mlir.Type.parse(ctx, "!llvm.struct<(i32, f32)>"))); -} - -test "identified struct types build from their name and fields" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - llvm.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - const fields: []const *const mlir.Type = &.{ .int(ctx, .i32), .float(ctx, .f32) }; - const struct_type = llvm.identifiedStructType(ctx, "struct.args", fields).?; - try std.testing.expect(struct_type.eql(try mlir.Type.parse(ctx, "!llvm.struct<\"struct.args\", (i32, f32)>"))); - try std.testing.expect(llvm.identifiedStructType(ctx, "struct.args", fields).?.eql(struct_type)); - try std.testing.expect(llvm.identifiedStructType(ctx, "struct.args", fields[0..1]) == null); -} - -test "getelementptr and alloca build verified operations" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - llvm.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - const loc: *const mlir.Location = .unknown(ctx); - const ptr = llvm.pointerType(ctx, llvm.NVPTXAddressSpace.global); - const block = mlir.Block.init(&.{ ptr, .int(ctx, .i64) }, &.{ loc, loc }); - defer block.deinit(); - const array = llvm.arrayType(.float(ctx, .f32), 8); - const gep = llvm.getelementptr(ctx, block.argument(0), &.{ .{ .constant = 0 }, .{ .value = block.argument(1) } }, array, loc).appendTo(block); - try std.testing.expect(gep.verify()); - try std.testing.expect(gep.result(0).type_().eql(ptr)); - const alloca = llvm.alloca(ctx, block.argument(1), llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic), array, .{ .alignment = 4 }, loc).appendTo(block); - try std.testing.expect(alloca.verify()); -} - -test "extractvalue, atomicrmw and intr.assume build verified operations" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - llvm.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - const loc: *const mlir.Location = .unknown(ctx); - const pair = llvm.structType(ctx, &.{ .int(ctx, .i32), .float(ctx, .f32) }); - const block = mlir.Block.init(&.{ pair, llvm.pointerType(ctx, llvm.NVPTXAddressSpace.global), .int(ctx, .i1) }, &.{ loc, loc, loc }); - defer block.deinit(); - const first = llvm.extractvalue(ctx, block.argument(0), .int(ctx, .i32), &.{0}, loc).appendTo(block); - try std.testing.expect(first.verify()); - const previous = llvm.atomicrmw(ctx, block.argument(1), first.result(0), .add, .monotonic, .{ .syncscope = "agent", .alignment = 4 }, loc).appendTo(block); - try std.testing.expect(previous.verify()); - try std.testing.expect(previous.result(0).type_().eql(.int(ctx, .i32))); - var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer writer.deinit(); - try writer.writer.print("{f}", .{previous}); - try std.testing.expect(std.mem.indexOf(u8, writer.written(), "llvm.atomicrmw add ") != null); - try std.testing.expect(std.mem.indexOf(u8, writer.written(), " monotonic ") != null); - try std.testing.expect(llvm.intr_assume(ctx, block.argument(2), loc).appendTo(block).verify()); -} - -test "vector, freeze and lifetime operations build verified operations" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - llvm.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - const loc: *const mlir.Location = .unknown(ctx); - const i64x2: *const mlir.Type = .vector(&.{2}, .int(ctx, .i64)); - const block = mlir.Block.init(&.{ i64x2, .int(ctx, .i64), .int(ctx, .i32), llvm.pointerType(ctx, llvm.NVPTXAddressSpace.generic) }, &.{ loc, loc, loc, loc }); - defer block.deinit(); - const v = block.argument(0); - const inserted = llvm.insertelement(ctx, v, block.argument(1), block.argument(2), loc).appendTo(block); - try std.testing.expect(inserted.verify()); - try std.testing.expect(inserted.result(0).type_().eql(i64x2)); - try std.testing.expect(llvm.extractelement(ctx, v, block.argument(2), .int(ctx, .i64), loc).appendTo(block).verify()); - const poison = llvm.mlir_poison(ctx, i64x2, loc).appendTo(block); - try std.testing.expect(poison.verify()); - try std.testing.expect(llvm.mlir_undef(ctx, i64x2, loc).appendTo(block).verify()); - try std.testing.expect(llvm.shufflevector(ctx, v, poison.result(0), i64x2, &.{ 1, 0 }, loc).appendTo(block).verify()); - try std.testing.expect(llvm.freeze(ctx, v, loc).appendTo(block).verify()); - try std.testing.expect(llvm.intr_lifetime_start(ctx, block.argument(3), loc).appendTo(block).verify()); - try std.testing.expect(llvm.intr_lifetime_end(ctx, block.argument(3), loc).appendTo(block).verify()); -} - -test "br carries its loop annotation" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - llvm.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - const loc: *const mlir.Location = .unknown(ctx); - const dest = mlir.Block.init(&.{}, &.{}); - defer dest.deinit(); - const annotation = llvm.LoopAnnotationAttr.get(ctx, .{ .mustProgress = true }); - const op = llvm.br(ctx, dest, &.{}, .{ .loop_annotation = annotation }, loc); - defer op.deinit(); - try std.testing.expect(op.attributeByName("loop_annotation").?.eql(annotation.attribute())); - const end = llvm.unreachable_(ctx, loc); - defer end.deinit(); -} - -test "argument and function attributes are named by the LLVM dialect" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - llvm.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - var buf: [512]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ - llvm.noundef(ctx), llvm.zeroext(ctx), llvm.signext(ctx), - llvm.alignment(ctx, 16), llvm.dereferenceable(ctx, 64), llvm.elementtype(ctx, .float(ctx, .f32)), - llvm.linkage(ctx, .internal), llvm.inlineHint(ctx), - })}); - try std.testing.expectEqualStrings("{llvm.align = 16 : i64, llvm.dereferenceable = 64 : i64, llvm.elementtype = f32, llvm.inline_hint, llvm.linkage = #llvm.linkage, llvm.noundef, llvm.signext, llvm.zeroext}", w.buffered()); -} - -test "emit_c_interface is a unit attribute of the LLVM dialect" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer writer.deinit(); - try writer.writer.print("{f}", .{mlir.Attribute.dict(ctx, &.{llvm.emitCInterface(ctx)})}); - try std.testing.expectEqualStrings("{llvm.emit_c_interface}", writer.written()); -} - -test "loop annotations are built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - const unroll = llvm.LoopUnrollAttr.get(ctx, .{ .full = true }); - try std.testing.expect(unroll.attribute().isA(llvm.LoopUnrollAttr) != null); - const annotation = llvm.LoopAnnotationAttr.get(ctx, .{ .unroll = unroll, .mustProgress = true }); - var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer writer.deinit(); - try writer.writer.print("{f}", .{annotation}); - try std.testing.expectEqualStrings("#llvm.loop_annotation, mustProgress = true>", writer.written()); - try std.testing.expect((try mlir.Attribute.parse(ctx, writer.written())).eql(annotation.attribute())); -} - -test "linkage attributes are built through the C API" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - llvm.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - var writer: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer writer.deinit(); - try writer.writer.print("{f}", .{llvm.linkageAttr(ctx, .weak_odr)}); - try std.testing.expectEqualStrings("#llvm.linkage", writer.written()); -} diff --git a/mlir/dialects/math/BUILD.bazel b/mlir/dialects/math/BUILD.bazel deleted file mode 100644 index 3845c969..00000000 --- a/mlir/dialects/math/BUILD.bazel +++ /dev/null @@ -1,22 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "math", - import_name = "mlir/dialects/math", - main = "math.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//mlir", - "@llvm-project//mlir:CAPIMath", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":math", - "//mlir", - ], -) diff --git a/mlir/dialects/math/math.zig b/mlir/dialects/math/math.zig deleted file mode 100644 index 4ca6ee6d..00000000 --- a/mlir/dialects/math/math.zig +++ /dev/null @@ -1,210 +0,0 @@ -const mlir = @import("mlir"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("math"); -} - -// ============================================================================= -// Float unary — SameOperandsAndResultType. Result type inferred from operand. -// ============================================================================= - -fn floatUnaryOp(comptime op_name: []const u8) type { - return struct { - pub fn call( - ctx: *mlir.Context, - src: *const mlir.Value, - location: *const mlir.Location, - ) *mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = .{ .flat = &.{src} }, - .result_type_inference = true, - .location = location, - }); - } - }; -} - -// Exponentials / logs -pub const exp = floatUnaryOp("math.exp").call; -pub const exp2 = floatUnaryOp("math.exp2").call; -pub const expm1 = floatUnaryOp("math.expm1").call; -pub const log = floatUnaryOp("math.log").call; -pub const log2 = floatUnaryOp("math.log2").call; -pub const log10 = floatUnaryOp("math.log10").call; -pub const log1p = floatUnaryOp("math.log1p").call; - -// Trig / hyperbolic -pub const sin = floatUnaryOp("math.sin").call; -pub const cos = floatUnaryOp("math.cos").call; -pub const tan = floatUnaryOp("math.tan").call; -pub const asin = floatUnaryOp("math.asin").call; -pub const acos = floatUnaryOp("math.acos").call; -pub const atan = floatUnaryOp("math.atan").call; -pub const sinh = floatUnaryOp("math.sinh").call; -pub const cosh = floatUnaryOp("math.cosh").call; -pub const tanh = floatUnaryOp("math.tanh").call; -pub const asinh = floatUnaryOp("math.asinh").call; -pub const acosh = floatUnaryOp("math.acosh").call; -pub const atanh = floatUnaryOp("math.atanh").call; - -// Roots / errors / abs -pub const sqrt = floatUnaryOp("math.sqrt").call; -pub const rsqrt = floatUnaryOp("math.rsqrt").call; -pub const cbrt = floatUnaryOp("math.cbrt").call; -pub const erf = floatUnaryOp("math.erf").call; -pub const erfc = floatUnaryOp("math.erfc").call; -pub const absf = floatUnaryOp("math.absf").call; - -// Rounding -pub const floor = floatUnaryOp("math.floor").call; -pub const ceil = floatUnaryOp("math.ceil").call; -pub const round = floatUnaryOp("math.round").call; -pub const roundeven = floatUnaryOp("math.roundeven").call; -pub const trunc = floatUnaryOp("math.trunc").call; - -// ============================================================================= -// Float binary — SameOperandsAndResultType. -// ============================================================================= - -fn floatBinaryOp(comptime op_name: []const u8) type { - return struct { - pub fn call( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - location: *const mlir.Location, - ) *mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .result_type_inference = true, - .location = location, - }); - } - }; -} - -pub const powf = floatBinaryOp("math.powf").call; -pub const atan2 = floatBinaryOp("math.atan2").call; -pub const copysign = floatBinaryOp("math.copysign").call; - -// ============================================================================= -// Float ternary — SameOperandsAndResultType. -// ============================================================================= - -pub fn fma( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "math.fma", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .result_type_inference = true, - .location = location, - }); -} - -/// math.clampf value, min, max — clamp to [min, max], same type throughout. -/// Distinct from `tt.clampf` (which has an explicit `propagateNan` attr); the -/// math version uses fastmath for NaN behaviour. -pub fn clampf( - ctx: *mlir.Context, - value: *const mlir.Value, - min: *const mlir.Value, - max: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "math.clampf", .{ - .operands = .{ .flat = &.{ value, min, max } }, - .result_type_inference = true, - .location = location, - }); -} - -// ============================================================================= -// Sincos — operand → (sin, cos), both same type as operand. -// ============================================================================= - -pub fn sincos( - ctx: *mlir.Context, - operand: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "math.sincos", .{ - .operands = .{ .flat = &.{operand} }, - .results = .{ .flat = &.{ operand.type_(), operand.type_() } }, - .location = location, - }); -} - -// ============================================================================= -// FPowI — float base, signless integer exponent → float (same type as base). -// ============================================================================= - -pub fn fpowi( - ctx: *mlir.Context, - base: *const mlir.Value, - power: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "math.fpowi", .{ - .operands = .{ .flat = &.{ base, power } }, - .results = .{ .flat = &.{base.type_()} }, - .location = location, - }); -} - -// ============================================================================= -// Integer unary -// ============================================================================= - -fn intUnaryOp(comptime op_name: []const u8) type { - return struct { - pub fn call( - ctx: *mlir.Context, - src: *const mlir.Value, - location: *const mlir.Location, - ) *mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = .{ .flat = &.{src} }, - .result_type_inference = true, - .location = location, - }); - } - }; -} - -pub const absi = intUnaryOp("math.absi").call; -pub const ctlz = intUnaryOp("math.ctlz").call; -pub const cttz = intUnaryOp("math.cttz").call; -pub const ctpop = intUnaryOp("math.ctpop").call; - -/// math.ipowi(base, exp) — integer power. Result type matches base. -pub const ipowi = floatBinaryOp("math.ipowi").call; - -// ============================================================================= -// Float classification — operand is float-like, result is i1-like. -// ============================================================================= - -fn classifyOp(comptime op_name: []const u8) type { - return struct { - pub fn call( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, - ) *mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); - } - }; -} - -pub const isfinite = classifyOp("math.isfinite").call; -pub const isinf = classifyOp("math.isinf").call; -pub const isnan = classifyOp("math.isnan").call; -pub const isnormal = classifyOp("math.isnormal").call; diff --git a/mlir/dialects/math/test.zig b/mlir/dialects/math/test.zig deleted file mode 100644 index fe2b7bc4..00000000 --- a/mlir/dialects/math/test.zig +++ /dev/null @@ -1,73 +0,0 @@ -const std = @import("std"); - -const math = @import("mlir/dialects/math"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "math.absf", - "math.absi", - "math.acos", - "math.acosh", - "math.asin", - "math.asinh", - "math.atan", - "math.atan2", - "math.atanh", - "math.cbrt", - "math.ceil", - "math.clampf", - "math.copysign", - "math.cos", - "math.cosh", - "math.ctlz", - "math.ctpop", - "math.cttz", - "math.erf", - "math.erfc", - "math.exp", - "math.exp2", - "math.expm1", - "math.floor", - "math.fma", - "math.fpowi", - "math.ipowi", - "math.isfinite", - "math.isinf", - "math.isnan", - "math.isnormal", - "math.log", - "math.log10", - "math.log1p", - "math.log2", - "math.powf", - "math.round", - "math.roundeven", - "math.rsqrt", - "math.sin", - "math.sincos", - "math.sinh", - "math.sqrt", - "math.tan", - "math.tanh", - "math.trunc", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(math); - inline for (comptime std.meta.declarations(math)) |decl| { - const T = @field(math, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - math.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/memref/BUILD.bazel b/mlir/dialects/memref/BUILD.bazel deleted file mode 100644 index be1a16bb..00000000 --- a/mlir/dialects/memref/BUILD.bazel +++ /dev/null @@ -1,23 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "memref", - import_name = "mlir/dialects/memref", - main = "memref.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//mlir", - "//stdx", - "@llvm-project//mlir:CAPIMemRef", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":memref", - "//mlir", - ], -) diff --git a/mlir/dialects/memref/memref.zig b/mlir/dialects/memref/memref.zig deleted file mode 100644 index 6c45eb33..00000000 --- a/mlir/dialects/memref/memref.zig +++ /dev/null @@ -1,670 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("memref"); -} - -/// Sentinel matching `mlir::ShapedType::kDynamic` (== INT64_MIN). -/// Use this in static-size/offset/stride arrays to mark a slot as dynamic. -pub const kDynamic: i64 = std.math.minInt(i64); - -// ============================================================================= -// Allocation — alloc / alloca / dealloc / realloc -// ============================================================================= - -pub const AllocOpts = struct { - alignment: ?i64 = null, -}; - -pub const ReallocOpts = struct { - alignment: ?i64 = null, - dynamic_size: ?*const mlir.Value = null, -}; - -/// memref.alloc — heap allocation. `dynamic_sizes` provides one Index value per -/// dynamic dim of `result_type` (left-to-right). `symbol_operands` binds to -/// the symbols of the layout map (rare; pass &.{} when none). -pub fn alloc( - ctx: *mlir.Context, - result_type: *const mlir.Type, - dynamic_sizes: []const *const mlir.Value, - symbol_operands: []const *const mlir.Value, - opts: AllocOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; - operands_buf.appendSliceAssumeCapacity(dynamic_sizes); - operands_buf.appendSliceAssumeCapacity(symbol_operands); - - const seg_sizes = [2]i32{ @intCast(dynamic_sizes.len), @intCast(symbol_operands.len) }; - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); - if (opts.alignment) |a| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); - } - - return mlir.Operation.make(ctx, "memref.alloc", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// memref.alloca — stack allocation; same operand/attribute layout as alloc. -/// Unverified: it only verifies inside an `AutomaticAllocationScope`. -pub fn alloca( - ctx: *mlir.Context, - result_type: *const mlir.Type, - dynamic_sizes: []const *const mlir.Value, - symbol_operands: []const *const mlir.Value, - opts: AllocOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; - operands_buf.appendSliceAssumeCapacity(dynamic_sizes); - operands_buf.appendSliceAssumeCapacity(symbol_operands); - - const seg_sizes = [2]i32{ @intCast(dynamic_sizes.len), @intCast(symbol_operands.len) }; - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes))); - if (opts.alignment) |a| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); - } - - return mlir.Operation.make(ctx, "memref.alloca", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .verify = false, - .location = location, - }); -} - -pub fn dealloc( - ctx: *mlir.Context, - memref: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.dealloc", .{ - .operands = .{ .flat = &.{memref} }, - .location = location, - }); -} - -/// memref.realloc — grow/shrink an existing allocation. `dynamic_size` is -/// optional; omit `opts.dynamic_size` to keep the size encoded in `result_type`. -pub fn realloc( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - opts: ReallocOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; - operands_buf.appendAssumeCapacity(src); - if (opts.dynamic_size) |s| operands_buf.appendAssumeCapacity(s); - - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.alignment) |a| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); - } - - return mlir.Operation.make(ctx, "memref.realloc", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -// ============================================================================= -// Load / Store — plain (non-affine) memref load/store -// ============================================================================= - -pub const LoadOpts = struct { - /// Hint that the lowering may use a non-temporal load. - nontemporal: bool = false, -}; - -/// memref.load — load a single element from a memref. -pub fn load( - ctx: *mlir.Context, - memref: *const mlir.Value, - indices: []const *const mlir.Value, - opts: LoadOpts, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - buf.appendAssumeCapacity(memref); - buf.appendSliceAssumeCapacity(indices); - - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.nontemporal) { - attrs.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); - } - - return mlir.Operation.make(ctx, "memref.load", .{ - .operands = .{ .flat = buf.constSlice() }, - .result_type_inference = true, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -pub const StoreOpts = struct { - nontemporal: bool = false, -}; - -/// memref.store — store a single element into a memref. -pub fn store( - ctx: *mlir.Context, - value: *const mlir.Value, - memref: *const mlir.Value, - indices: []const *const mlir.Value, - opts: StoreOpts, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - buf.appendSliceAssumeCapacity(&.{ value, memref }); - buf.appendSliceAssumeCapacity(indices); - - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.nontemporal) { - attrs.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); - } - - return mlir.Operation.make(ctx, "memref.store", .{ - .operands = .{ .flat = buf.constSlice() }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -// ============================================================================= -// Atomic RMW -// ============================================================================= - -/// Matches `mlir::arith::AtomicRMWKind` (I64EnumAttr) in upstream `Arith` td. -/// memref.atomic_rmw and affine.parallel both encode reductions through this. -pub const AtomicRMWKind = enum(i64) { - addf = 0, - addi = 1, - andi = 2, - assign = 3, - maximumf = 4, - maxnumf = 5, - maxs = 6, - maxu = 7, - minimumf = 8, - minnumf = 9, - mins = 10, - minu = 11, - mulf = 12, - muli = 13, - ori = 14, - xori = 15, -}; - -/// memref.atomic_rmw — atomic read-modify-write. -pub fn atomic_rmw( - ctx: *mlir.Context, - kind: AtomicRMWKind, - value: *const mlir.Value, - memref: *const mlir.Value, - indices: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - buf.appendSliceAssumeCapacity(&.{ value, memref }); - buf.appendSliceAssumeCapacity(indices); - - return mlir.Operation.make(ctx, "memref.atomic_rmw", .{ - .operands = .{ .flat = buf.constSlice() }, - .results = .{ .flat = &.{value.type_()} }, - .attributes = &.{ - .named(ctx, "kind", .int(ctx, .i64, @backingInt(kind))), - }, - .location = location, - }); -} - -// ============================================================================= -// Casts / metadata-only ops -// ============================================================================= - -/// memref.cast — change layout/element-shape compatibility (e.g. unranked ↔ -/// ranked, dynamic ↔ static). Element type and total memory must match. -pub fn cast( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.cast", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// memref.memory_space_cast — change the memory-space attribute on a memref. -pub fn memory_space_cast( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.memory_space_cast", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// memref.assume_alignment — pure, returns the same memref with an alignment -/// hint attached. -pub fn assume_alignment( - ctx: *mlir.Context, - src: *const mlir.Value, - alignment: i32, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.assume_alignment", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{src.type_()} }, - .attributes = &.{ - .named(ctx, "alignment", .int(ctx, .i32, alignment)), - }, - .location = location, - }); -} - -// ============================================================================= -// Shape introspection -// ============================================================================= - -/// memref.dim — returns the size of dim `index` of `memref` as `index`. -pub fn dim( - ctx: *mlir.Context, - memref: *const mlir.Value, - index: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.dim", .{ - .operands = .{ .flat = &.{ memref, index } }, - .results = .{ .flat = &.{.index(ctx)} }, - .location = location, - }); -} - -/// memref.rank — returns rank of `memref` as `index`. -pub fn rank( - ctx: *mlir.Context, - memref: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.rank", .{ - .operands = .{ .flat = &.{memref} }, - .results = .{ .flat = &.{.index(ctx)} }, - .location = location, - }); -} - -/// memref.extract_aligned_pointer_as_index — get the aligned base pointer of -/// a memref as an `index`. -pub fn extract_aligned_pointer_as_index( - ctx: *mlir.Context, - src: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.extract_aligned_pointer_as_index", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{.index(ctx)} }, - .location = location, - }); -} - -// ============================================================================= -// View / SubView / Reinterpret -// ============================================================================= - -/// memref.view — type-changing view of a flat 1-D `memref` as a higher- -/// rank memref. `byte_shift` is an `index` byte offset; `dynamic_sizes` -/// supplies one operand per dynamic dim of `result_type`. -pub fn view( - ctx: *mlir.Context, - source: *const mlir.Value, - byte_shift: *const mlir.Value, - dynamic_sizes: []const *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - buf.appendSliceAssumeCapacity(&.{ source, byte_shift }); - buf.appendSliceAssumeCapacity(dynamic_sizes); - - return mlir.Operation.make(ctx, "memref.view", .{ - .operands = .{ .flat = buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// Mixed static/dynamic offsets-sizes-strides — `static_*` slots holding -/// `kDynamic` consume one entry from the corresponding `dynamic_*` slice. -pub const SubViewArgs = struct { - static_offsets: []const i64, - static_sizes: []const i64, - static_strides: []const i64, - dynamic_offsets: []const *const mlir.Value = &.{}, - dynamic_sizes: []const *const mlir.Value = &.{}, - dynamic_strides: []const *const mlir.Value = &.{}, -}; - -/// memref.subview — strided view into a source memref. The result type must -/// be precomputed by the caller (the dialect provides `inferResultType`, -/// but we don't expose that yet — pass an explicit `result_type`). -pub fn subview( - ctx: *mlir.Context, - source: *const mlir.Value, - args: SubViewArgs, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; - operands_buf.appendAssumeCapacity(source); - operands_buf.appendSliceAssumeCapacity(args.dynamic_offsets); - operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); - operands_buf.appendSliceAssumeCapacity(args.dynamic_strides); - - const seg_sizes = [4]i32{ - 1, - @intCast(args.dynamic_offsets.len), - @intCast(args.dynamic_sizes.len), - @intCast(args.dynamic_strides.len), - }; - - return mlir.Operation.make(ctx, "memref.subview", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "static_offsets", .denseArray(ctx, .i64, args.static_offsets)), - .named(ctx, "static_sizes", .denseArray(ctx, .i64, args.static_sizes)), - .named(ctx, "static_strides", .denseArray(ctx, .i64, args.static_strides)), - }, - .location = location, - }); -} - -pub const ReinterpretCastArgs = struct { - static_offsets: []const i64, - static_sizes: []const i64, - static_strides: []const i64, - dynamic_offsets: []const *const mlir.Value = &.{}, - dynamic_sizes: []const *const mlir.Value = &.{}, - dynamic_strides: []const *const mlir.Value = &.{}, -}; - -/// memref.reinterpret_cast — changes offset/sizes/strides of a memref without -/// touching the underlying memory. -pub fn reinterpret_cast( - ctx: *mlir.Context, - source: *const mlir.Value, - args: ReinterpretCastArgs, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; - operands_buf.appendAssumeCapacity(source); - operands_buf.appendSliceAssumeCapacity(args.dynamic_offsets); - operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); - operands_buf.appendSliceAssumeCapacity(args.dynamic_strides); - - const seg_sizes = [4]i32{ - 1, - @intCast(args.dynamic_offsets.len), - @intCast(args.dynamic_sizes.len), - @intCast(args.dynamic_strides.len), - }; - - return mlir.Operation.make(ctx, "memref.reinterpret_cast", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "static_offsets", .denseArray(ctx, .i64, args.static_offsets)), - .named(ctx, "static_sizes", .denseArray(ctx, .i64, args.static_sizes)), - .named(ctx, "static_strides", .denseArray(ctx, .i64, args.static_strides)), - }, - .location = location, - }); -} - -// ============================================================================= -// Reshape — expand_shape / collapse_shape / reshape -// ============================================================================= - -/// memref.reshape — reshape a memref using a separate `shape` memref of `index` -/// elements. `result_type` carries the new shape. -pub fn reshape( - ctx: *mlir.Context, - source: *const mlir.Value, - shape: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.reshape", .{ - .operands = .{ .flat = &.{ source, shape } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// `reassociation[i]` lists the source dimensions that map into result dim i -/// (for collapse_shape) or the result dimensions that come from source dim i -/// (for expand_shape). Each inner slice is stored as an `IntegerAttr` array. -pub const ReassociationArgs = struct { - /// Index groupings. - reassociation: []const []const i64, - /// For expand_shape only: per-result-dim sizes, with `kDynamic` for slots - /// resolved from `dynamic_output_shape`. Pass &.{} for collapse_shape. - static_output_shape: []const i64 = &.{}, - /// For expand_shape only: the dynamic dim values referenced by `kDynamic` - /// entries in `static_output_shape`. - dynamic_output_shape: []const *const mlir.Value = &.{}, -}; - -fn buildReassociationAttr(ctx: *mlir.Context, reassociation: []const []const i64) *const mlir.Attribute { - var outer: stdx.BoundedArray(*const mlir.Attribute, 32) = .empty; - var inner_buf: stdx.BoundedArray(*const mlir.Attribute, 32) = .empty; - for (reassociation) |group| { - inner_buf = .empty; - for (group) |idx| { - inner_buf.appendAssumeCapacity(.int(ctx, .i64, idx)); - } - outer.appendAssumeCapacity(.array(ctx, inner_buf.constSlice())); - } - return .array(ctx, outer.constSlice()); -} - -/// memref.expand_shape — split source dimensions into multiple result -/// dimensions. `static_output_shape` must have one entry per result dim; -/// dynamic entries are filled from `dynamic_output_shape` in order. -pub fn expand_shape( - ctx: *mlir.Context, - source: *const mlir.Value, - args: ReassociationArgs, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; - operands_buf.appendAssumeCapacity(source); - operands_buf.appendSliceAssumeCapacity(args.dynamic_output_shape); - - return mlir.Operation.make(ctx, "memref.expand_shape", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "reassociation", buildReassociationAttr(ctx, args.reassociation)), - .named(ctx, "static_output_shape", .denseArray(ctx, .i64, args.static_output_shape)), - }, - .location = location, - }); -} - -/// memref.collapse_shape — merge consecutive source dimensions into single -/// result dimensions. Reassociation map is canonical. -pub fn collapse_shape( - ctx: *mlir.Context, - source: *const mlir.Value, - reassociation: []const []const i64, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.collapse_shape", .{ - .operands = .{ .flat = &.{source} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "reassociation", buildReassociationAttr(ctx, reassociation)), - }, - .location = location, - }); -} - -// ============================================================================= -// Transpose — metadata-only permutation. Permutation is an AffineMapAttr. -// ============================================================================= - -/// memref.transpose — permute the dimensions of `src` according to -/// `permutation` (which must be a permutation of `(d0, ..., dN-1)`). Build -/// the map with `mlir.AffineMap.get` or parse via -/// `mlir.parseAffineMapAttribute`. -pub fn transpose( - ctx: *mlir.Context, - src: *const mlir.Value, - permutation: *const mlir.Attribute, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.transpose", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "permutation", permutation), - }, - .location = location, - }); -} - -// ============================================================================= -// Copy / Prefetch -// ============================================================================= - -/// memref.copy — copy source memref into target memref (same element type, -/// same total element count). -pub fn copy( - ctx: *mlir.Context, - source: *const mlir.Value, - target: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.copy", .{ - .operands = .{ .flat = &.{ source, target } }, - .location = location, - }); -} - -pub const PrefetchOpts = struct { - /// True for a write prefetch, false for a read prefetch. - is_write: bool = false, - /// 0 = no locality, 3 = extreme locality (keep in cache). - locality_hint: i32 = 3, - /// True for data cache, false for instruction cache. - is_data_cache: bool = true, -}; - -/// memref.prefetch — non-binding hint to bring `memref[indices]` into cache. -pub fn prefetch( - ctx: *mlir.Context, - memref: *const mlir.Value, - indices: []const *const mlir.Value, - opts: PrefetchOpts, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - buf.appendAssumeCapacity(memref); - buf.appendSliceAssumeCapacity(indices); - - return mlir.Operation.make(ctx, "memref.prefetch", .{ - .operands = .{ .flat = buf.constSlice() }, - .attributes = &.{ - .named(ctx, "isWrite", .boolean(ctx, opts.is_write)), - .named(ctx, "localityHint", .int(ctx, .i32, opts.locality_hint)), - .named(ctx, "isDataCache", .boolean(ctx, opts.is_data_cache)), - }, - .location = location, - }); -} - -// ============================================================================= -// Globals -// ============================================================================= - -pub const GlobalArgs = struct { - pub const Visibility = enum { public, private, nested }; - - name: []const u8, - type_: *const mlir.Type, - /// Optional dense initial value. `null` ⇒ uninitialised global. - initial_value: ?*const mlir.Attribute = null, - visibility: Visibility = .public, - constant: bool = false, - alignment: ?i64 = null, -}; - -/// memref.global — declare a module-scope memref global (matches the symbol -/// referenced by `memref.get_global`). -pub fn global(ctx: *mlir.Context, args: GlobalArgs, location: *const mlir.Location) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "sym_name", .string(ctx, args.name)), - .named(ctx, "sym_visibility", .string(ctx, @tagName(args.visibility))), - .named(ctx, "type", .typeAttr(args.type_)), - }); - if (args.initial_value) |iv| { - attrs.appendAssumeCapacity(.named(ctx, "initial_value", iv)); - } - if (args.constant) { - attrs.appendAssumeCapacity(.named(ctx, "constant", .unit(ctx))); - } - if (args.alignment) |a| { - attrs.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); - } - - return mlir.Operation.make(ctx, "memref.global", .{ - .attributes = attrs.constSlice(), - .verify = false, - .location = location, - }); -} - -/// memref.get_global — fetch the value of a `memref.global` by symbol name. -pub fn get_global( - ctx: *mlir.Context, - name: []const u8, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "memref.get_global", .{ - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "name", .flatSymbolRef(ctx, name)), - }, - .location = location, - }); -} diff --git a/mlir/dialects/memref/test.zig b/mlir/dialects/memref/test.zig deleted file mode 100644 index cddb74c5..00000000 --- a/mlir/dialects/memref/test.zig +++ /dev/null @@ -1,77 +0,0 @@ -const std = @import("std"); - -const memref = @import("mlir/dialects/memref"); -const mlir = @import("mlir"); - -const operation_names: []const []const u8 = &.{ - "memref.alloc", - "memref.alloca", - "memref.assume_alignment", - "memref.atomic_rmw", - "memref.cast", - "memref.collapse_shape", - "memref.copy", - "memref.dealloc", - "memref.dim", - "memref.expand_shape", - "memref.extract_aligned_pointer_as_index", - "memref.get_global", - "memref.global", - "memref.load", - "memref.memory_space_cast", - "memref.prefetch", - "memref.rank", - "memref.realloc", - "memref.reinterpret_cast", - "memref.reshape", - "memref.store", - "memref.subview", - "memref.transpose", - "memref.view", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(memref); - inline for (comptime std.meta.declarations(memref)) |decl| { - const T = @field(memref, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - memref.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} - -test "atomic_rmw kinds print as their IR spelling" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - memref.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - const loc = mlir.Location.unknown(ctx); - const block = mlir.Block.init(&.{ .float(ctx, .f32), try mlir.Type.parse(ctx, "memref<4xf32>"), .int(ctx, .i32), try mlir.Type.parse(ctx, "memref<4xi32>"), .index(ctx) }, &.{ loc, loc, loc, loc, loc }); - defer block.deinit(); - inline for (comptime std.meta.tags(memref.AtomicRMWKind)) |kind| { - const float = switch (kind) { - .addf, .assign, .maximumf, .maxnumf, .minimumf, .minnumf, .mulf => true, - else => false, - }; - const value = block.argument(if (float) 0 else 2); - const ref = block.argument(if (float) 1 else 3); - const op = memref.atomic_rmw(ctx, kind, value, ref, &.{block.argument(4)}, loc); - defer op.deinit(); - var text: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer text.deinit(); - try text.writer.print("{f}", .{op}); - try std.testing.expect(std.mem.indexOf(u8, text.written(), "memref.atomic_rmw " ++ @tagName(kind) ++ " ") != null); - } -} diff --git a/mlir/dialects/mosaic_tpu/BUILD.bazel b/mlir/dialects/mosaic_tpu/BUILD.bazel deleted file mode 100644 index 88e67265..00000000 --- a/mlir/dialects/mosaic_tpu/BUILD.bazel +++ /dev/null @@ -1,38 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -# The types and attributes Mosaic's C API does not expose. -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - "@llvm-project//llvm:Support", - "@llvm-project//mlir:CAPIIR", - "@xla//xla/mosaic:tpu_dialect", - ], -) - -zig_library( - name = "mosaic_tpu", - import_name = "mlir/dialects/mosaic_tpu", - main = "mosaic_tpu.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//stdx", - "@xla//xla/mosaic:tpu_dialect_capi", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":mosaic_tpu", - "//mlir", - ], -) diff --git a/mlir/dialects/mosaic_tpu/capi.cc b/mlir/dialects/mosaic_tpu/capi.cc deleted file mode 100644 index 22662082..00000000 --- a/mlir/dialects/mosaic_tpu/capi.cc +++ /dev/null @@ -1,273 +0,0 @@ -//===- capi.cc - C API for the Mosaic tpu dialect -------------------------===// - -#include "capi.h" - -#include "xla/mosaic/dialect/tpu/tpu_dialect.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/IR/Diagnostics.h" -#include "mlir/IR/Location.h" -#include "llvm/ADT/ArrayRef.h" - -#include -#include - -static mlir::MLIRContext *loadTPU(MlirContext context) { - mlir::MLIRContext *ctx = unwrap(context); - if (ctx) - ctx->getOrLoadDialect(); - return ctx; -} - -template -static MlirAttribute enumAttrGet(MlirContext context, - std::optional value) { - mlir::MLIRContext *ctx = loadTPU(context); - if (!ctx || !value) - return {nullptr}; - return wrap(AttrT::get(ctx, *value)); -} - -template static uint32_t enumAttrGetValue(MlirAttribute attr) { - return static_cast(llvm::cast(unwrap(attr)).getValue()); -} - -static llvm::ArrayRef arrayRef(intptr_t size, const int64_t *data) { - return llvm::ArrayRef(data, static_cast(size)); -} - -bool mlirTypeIsATpuSemaphore(MlirType type) { - return llvm::isa(unwrap(type)); -} - -MlirType mlirTpuSemaphoreTypeGet(MlirContext context) { - mlir::MLIRContext *ctx = loadTPU(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::tpu::SemaphoreType::get(ctx)); -} - -bool mlirTypeIsATpuDMASemaphore(MlirType type) { - return llvm::isa(unwrap(type)); -} - -MlirType mlirTpuDMASemaphoreTypeGet(MlirContext context) { - mlir::MLIRContext *ctx = loadTPU(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::tpu::DMASemaphoreType::get(ctx)); -} - -bool mlirAttributeIsATpuReductionKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirTpuReductionKindAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet( - context, mlir::tpu::symbolizeReductionKind(value)); -} - -uint32_t mlirTpuReductionKindAttrGetValue(MlirAttribute attr) { - return enumAttrGetValue(attr); -} - -bool mlirAttributeIsATpuContractPrecision(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirTpuContractPrecisionAttrGet(MlirContext context, - uint32_t value) { - return enumAttrGet( - context, mlir::tpu::symbolizeContractPrecision(value)); -} - -uint32_t mlirTpuContractPrecisionAttrGetValue(MlirAttribute attr) { - return enumAttrGetValue(attr); -} - -bool mlirAttributeIsATpuRoundingMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirTpuRoundingModeAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet( - context, mlir::tpu::symbolizeRoundingMode(value)); -} - -uint32_t mlirTpuRoundingModeAttrGetValue(MlirAttribute attr) { - return enumAttrGetValue(attr); -} - -bool mlirAttributeIsATpuCoreType(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirTpuCoreTypeAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet( - context, mlir::tpu::symbolizeCoreType(value)); -} - -uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr) { - return enumAttrGetValue(attr); -} - -bool mlirAttributeIsATpuDimensionSemantics(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirTpuDimensionSemanticsAttrGet(MlirContext context, - uint32_t value) { - return enumAttrGet( - context, mlir::tpu::symbolizeDimensionSemantics(value)); -} - -uint32_t mlirTpuDimensionSemanticsAttrGetValue(MlirAttribute attr) { - return enumAttrGetValue(attr); -} - -bool mlirAttributeIsATpuPipelineMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirTpuPipelineModeAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet( - context, mlir::tpu::symbolizePipelineMode(value)); -} - -uint32_t mlirTpuPipelineModeAttrGetValue(MlirAttribute attr) { - return enumAttrGetValue(attr); -} - -bool mlirAttributeIsATpuRevisitMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirTpuRevisitModeAttrGet(MlirContext context, uint32_t value) { - return enumAttrGet( - context, mlir::tpu::symbolizeRevisitMode(value)); -} - -uint32_t mlirTpuRevisitModeAttrGetValue(MlirAttribute attr) { - return enumAttrGetValue(attr); -} - -bool mlirAttributeIsATpuMemorySpace(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, uint32_t value, - int64_t coreType) { - mlir::MLIRContext *ctx = loadTPU(context); - std::optional memorySpace = - mlir::tpu::symbolizeMemorySpace(value); - if (!ctx || !memorySpace) - return {nullptr}; - std::optional core; - if (coreType >= 0) { - if (coreType > UINT32_MAX) - return {nullptr}; - core = mlir::tpu::symbolizeCoreType(static_cast(coreType)); - if (!core) - return {nullptr}; - } - return wrap(mlir::tpu::MemorySpaceAttr::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, - *memorySpace, core)); -} - -uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr) { - return enumAttrGetValue(attr); -} - -int64_t mlirTpuMemorySpaceAttrGetCoreType(MlirAttribute attr) { - std::optional core = - llvm::cast(unwrap(attr)).getCoreType(); - return core ? static_cast(*core) : -1; -} - -bool mlirAttributeIsATpuElementWindow(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirTpuElementWindowAttrGet(MlirContext context, - intptr_t numPadLow, - const int64_t *padLow, - intptr_t numPadHigh, - const int64_t *padHigh) { - mlir::MLIRContext *ctx = loadTPU(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::tpu::ElementWindowAttr::get( - ctx, arrayRef(numPadLow, padLow), arrayRef(numPadHigh, padHigh))); -} - -intptr_t mlirTpuElementWindowAttrGetNumPadLow(MlirAttribute attr) { - return llvm::cast(unwrap(attr)) - .getPadLow() - .size(); -} - -int64_t mlirTpuElementWindowAttrGetPadLow(MlirAttribute attr, intptr_t pos) { - return llvm::cast(unwrap(attr)) - .getPadLow()[pos]; -} - -intptr_t mlirTpuElementWindowAttrGetNumPadHigh(MlirAttribute attr) { - return llvm::cast(unwrap(attr)) - .getPadHigh() - .size(); -} - -int64_t mlirTpuElementWindowAttrGetPadHigh(MlirAttribute attr, intptr_t pos) { - return llvm::cast(unwrap(attr)) - .getPadHigh()[pos]; -} - -bool mlirAttributeIsATpuDotDimensionNumbers(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} - -MlirAttribute mlirTpuDotDimensionNumbersAttrGet( - MlirContext context, intptr_t numLhsContractingDims, - const int64_t *lhsContractingDims, intptr_t numRhsContractingDims, - const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, - const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, - const int64_t *rhsNonContractingDims, intptr_t numOutputDimOrder, - const int64_t *outputDimOrder, intptr_t numLhsBatchDims, - const int64_t *lhsBatchDims, intptr_t numRhsBatchDims, - const int64_t *rhsBatchDims) { - mlir::MLIRContext *ctx = loadTPU(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::tpu::DotDimensionNumbersAttr::get( - ctx, arrayRef(numLhsContractingDims, lhsContractingDims), - arrayRef(numRhsContractingDims, rhsContractingDims), - arrayRef(numLhsNonContractingDims, lhsNonContractingDims), - arrayRef(numRhsNonContractingDims, rhsNonContractingDims), - arrayRef(numOutputDimOrder, outputDimOrder), - arrayRef(numLhsBatchDims, lhsBatchDims), - arrayRef(numRhsBatchDims, rhsBatchDims))); -} - -#define ZML_TPU_DOT_DIMS_GETTERS(Name) \ - intptr_t mlirTpuDotDimensionNumbersAttrGetNum##Name(MlirAttribute attr) { \ - return llvm::cast(unwrap(attr)) \ - .get##Name() \ - .size(); \ - } \ - int64_t mlirTpuDotDimensionNumbersAttrGet##Name(MlirAttribute attr, \ - intptr_t pos) { \ - return llvm::cast(unwrap(attr)) \ - .get##Name()[pos]; \ - } - -ZML_TPU_DOT_DIMS_GETTERS(LhsContractingDims) -ZML_TPU_DOT_DIMS_GETTERS(RhsContractingDims) -ZML_TPU_DOT_DIMS_GETTERS(LhsNonContractingDims) -ZML_TPU_DOT_DIMS_GETTERS(RhsNonContractingDims) -ZML_TPU_DOT_DIMS_GETTERS(OutputDimOrder) -ZML_TPU_DOT_DIMS_GETTERS(LhsBatchDims) -ZML_TPU_DOT_DIMS_GETTERS(RhsBatchDims) - -#undef ZML_TPU_DOT_DIMS_GETTERS diff --git a/mlir/dialects/mosaic_tpu/capi.h b/mlir/dialects/mosaic_tpu/capi.h deleted file mode 100644 index 2e1cc0bb..00000000 --- a/mlir/dialects/mosaic_tpu/capi.h +++ /dev/null @@ -1,145 +0,0 @@ -//===- capi.h - C API for the Mosaic tpu dialect ------------------*- C -*-===// -// -// C entry points for the `tpu` dialect that Mosaic's C API -// (`tpu_dialect.h`) leaves out: the semaphore types and the attributes -// kernels build. - -#ifndef MLIR_DIALECTS_MOSAIC_TPU_CAPI_H -#define MLIR_DIALECTS_MOSAIC_TPU_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// !tpu.semaphore -MLIR_CAPI_EXPORTED bool mlirTypeIsATpuSemaphore(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirTpuSemaphoreTypeGet(MlirContext context); - -// !tpu.dma_semaphore -MLIR_CAPI_EXPORTED bool mlirTypeIsATpuDMASemaphore(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirTpuDMASemaphoreTypeGet(MlirContext context); - -// Enum attributes take and return the C++ enum's integer value (tpu_ops.td); -// `Get` returns a null attribute for a value outside the enum. - -// #tpu.reduction_kind<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuReductionKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTpuReductionKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirTpuReductionKindAttrGetValue(MlirAttribute attr); - -// #tpu.contract_precision<...> -MLIR_CAPI_EXPORTED bool -mlirAttributeIsATpuContractPrecision(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTpuContractPrecisionAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirTpuContractPrecisionAttrGetValue(MlirAttribute attr); - -// #tpu.rounding_mode<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRoundingMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRoundingModeAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirTpuRoundingModeAttrGetValue(MlirAttribute attr); - -// #tpu.core_type<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuCoreType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuCoreTypeAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirTpuCoreTypeAttrGetValue(MlirAttribute attr); - -// #tpu.dimension_semantics<...> -MLIR_CAPI_EXPORTED bool -mlirAttributeIsATpuDimensionSemantics(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTpuDimensionSemanticsAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirTpuDimensionSemanticsAttrGetValue(MlirAttribute attr); - -// #tpu.pipeline_mode<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuPipelineMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuPipelineModeAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirTpuPipelineModeAttrGetValue(MlirAttribute attr); - -// #tpu.revisit_mode<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuRevisitMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuRevisitModeAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirTpuRevisitModeAttrGetValue(MlirAttribute attr); - -// #tpu.memory_space: a negative `coreType` leaves the -// optional core type out, and `GetCoreType` returns -1 when it is absent. -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuMemorySpace(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuMemorySpaceAttrGet(MlirContext context, - uint32_t value, - int64_t coreType); -MLIR_CAPI_EXPORTED uint32_t mlirTpuMemorySpaceAttrGetValue(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t -mlirTpuMemorySpaceAttrGetCoreType(MlirAttribute attr); - -// #tpu.element_window<[pad_low], [pad_high]> -MLIR_CAPI_EXPORTED bool mlirAttributeIsATpuElementWindow(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuElementWindowAttrGet( - MlirContext context, intptr_t numPadLow, const int64_t *padLow, - intptr_t numPadHigh, const int64_t *padHigh); -MLIR_CAPI_EXPORTED intptr_t -mlirTpuElementWindowAttrGetNumPadLow(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuElementWindowAttrGetPadLow(MlirAttribute attr, - intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t -mlirTpuElementWindowAttrGetNumPadHigh(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t -mlirTpuElementWindowAttrGetPadHigh(MlirAttribute attr, intptr_t pos); - -// #tpu.dot_dimension_numbers<...>: the dimension lists in ODS order. -MLIR_CAPI_EXPORTED bool -mlirAttributeIsATpuDotDimensionNumbers(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirTpuDotDimensionNumbersAttrGet( - MlirContext context, intptr_t numLhsContractingDims, - const int64_t *lhsContractingDims, intptr_t numRhsContractingDims, - const int64_t *rhsContractingDims, intptr_t numLhsNonContractingDims, - const int64_t *lhsNonContractingDims, intptr_t numRhsNonContractingDims, - const int64_t *rhsNonContractingDims, intptr_t numOutputDimOrder, - const int64_t *outputDimOrder, intptr_t numLhsBatchDims, - const int64_t *lhsBatchDims, intptr_t numRhsBatchDims, - const int64_t *rhsBatchDims); -MLIR_CAPI_EXPORTED intptr_t -mlirTpuDotDimensionNumbersAttrGetNumLhsContractingDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetLhsContractingDims( - MlirAttribute attr, intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t -mlirTpuDotDimensionNumbersAttrGetNumRhsContractingDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetRhsContractingDims( - MlirAttribute attr, intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t -mlirTpuDotDimensionNumbersAttrGetNumLhsNonContractingDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t -mlirTpuDotDimensionNumbersAttrGetLhsNonContractingDims(MlirAttribute attr, - intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t -mlirTpuDotDimensionNumbersAttrGetNumRhsNonContractingDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t -mlirTpuDotDimensionNumbersAttrGetRhsNonContractingDims(MlirAttribute attr, - intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t -mlirTpuDotDimensionNumbersAttrGetNumOutputDimOrder(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t mlirTpuDotDimensionNumbersAttrGetOutputDimOrder( - MlirAttribute attr, intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t -mlirTpuDotDimensionNumbersAttrGetNumLhsBatchDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t -mlirTpuDotDimensionNumbersAttrGetLhsBatchDims(MlirAttribute attr, intptr_t pos); -MLIR_CAPI_EXPORTED intptr_t -mlirTpuDotDimensionNumbersAttrGetNumRhsBatchDims(MlirAttribute attr); -MLIR_CAPI_EXPORTED int64_t -mlirTpuDotDimensionNumbersAttrGetRhsBatchDims(MlirAttribute attr, intptr_t pos); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_MOSAIC_TPU_CAPI_H diff --git a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig b/mlir/dialects/mosaic_tpu/mosaic_tpu.zig deleted file mode 100644 index 8425e7b4..00000000 --- a/mlir/dialects/mosaic_tpu/mosaic_tpu.zig +++ /dev/null @@ -1,1556 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -/// Mosaic IR version stored as `stable_mosaic.version`, matching -/// `xla/mosaic/dialect/tpu/transforms/serde.h:kVersion`. This is distinct from -/// the custom-call JSON `serialization_format`, which remains 1. -pub const SERDE_VERSION: i32 = 18; - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("tpu"); -} - -/// Register the `mosaic-serde` pass so `PassManager.parse` can pick it up by -/// name. Idempotent — safe to call repeatedly. Required before running the -/// `mosaic-serde{serialize=true}` pipeline that produces the bytecode embedded -/// in `tpu_custom_call`'s backend_config. -pub fn registerMosaicSerdePass() void { - c.mlirTpuRegisterMosaicSerdePass(); -} - -// ============================================================================= -// Types — !tpu.semaphore, !tpu.dma_semaphore, !tpu.float8_exmy<...> -// ============================================================================= - -/// `!tpu.semaphore`. -pub const SemaphoreType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsATpuSemaphore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "semaphore"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirTpuSemaphoreTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// `!tpu.semaphore` element type. -pub fn semaphoreType(ctx: *mlir.Context) *const mlir.Type { - const ty = SemaphoreType.get(ctx, .{}) catch unreachable; - return ty.type_(); -} - -/// `!tpu.dma_semaphore`. -pub const DMASemaphoreType = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirType); - pub const isAFn = c.mlirTypeIsATpuDMASemaphore; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const mnemonic = "dma_semaphore"; - pub fn type_(self: *const Self) *const mlir.Type { - return @ptrCast(self); - } - pub const InitArgs = struct {}; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - _ = args; - const result = c.mlirTpuDMASemaphoreTypeGet(ctx.ptr()); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } -}; - -/// Build `!tpu.dma_semaphore`. -pub fn dmaSemaphoreType(ctx: *mlir.Context) *const mlir.Type { - const ty = DMASemaphoreType.get(ctx, .{}) catch unreachable; - return ty.type_(); -} - -/// `!tpu.float8_exmy` — Mosaic Float8 type. -pub const Float8EXMYType = opaque { - const M = mlir.Methods(Float8EXMYType, c.MlirType); - - pub const isAFn = c.mlirTpuIsAFloat8EXMYType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - pub fn get(ctx: *mlir.Context, underlying: *const mlir.Type) *const Float8EXMYType { - return @ptrCast(c.mlirTpuFloat8EXMYTypeGet(ctx.ptr(), underlying.ptr()).ptr); - } - - pub fn underlyingType(self: *const Float8EXMYType) *const mlir.Type { - return @ptrCast(c.mlirTpuFloat8EXMYTypeGetUnderlyingType(self.ptr()).ptr); - } - - pub fn type_(self: *const Float8EXMYType) *const mlir.Type { - return @ptrCast(self); - } -}; - -pub fn float8ExmyType(ctx: *mlir.Context, underlying: *const mlir.Type) *const mlir.Type { - return @ptrCast(Float8EXMYType.get(ctx, underlying)); -} - -// ============================================================================= -// Attributes -// ============================================================================= - -/// Reduction kind for `tpu.all_reduce`, `tpu.reduce_index`, `tpu.scan`. -pub const ReductionKind = enum(u32) { - sum = 0, - max = 1, - min = 2, - arg_max = 3, - arg_min = 4, - find_first_set = 5, - maxf = 6, - minf = 7, - maxsi = 8, - minsi = 9, - maxui = 10, - minui = 11, - - pub fn attribute(self: ReductionKind, ctx: *mlir.Context) *const mlir.Attribute { - const attr = ReductionKindAttr.get(ctx, .{ .value = self }) catch unreachable; - return attr.attribute(); - } - - fn forInput(self: ReductionKind, input: *const mlir.Value, unsigned_integer: bool) ReductionKind { - if (self != .max and self != .min) return self; - const element_type = input.type_().isA(mlir.ShapedType).?.elementType(); - if (element_type.isFloat()) return if (self == .max) .maxf else .minf; - // Match Mosaic's serde upgrade: scans used unsigned integer extrema, - // while all-reduce used signed integer extrema. - return if (unsigned_integer) - (if (self == .max) .maxui else .minui) - else - (if (self == .max) .maxsi else .minsi); - } -}; - -/// `#tpu.reduction_kind<...>`. -pub const ReductionKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsATpuReductionKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "reduction_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ReductionKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirTpuReductionKindAttrGetValue(self.ptr()))); - } -}; - -/// Matmul precision setting of `tpu.matmul`. -pub const ContractPrecision = enum(u32) { - bf16 = 0, - fp32 = 1, - bf16x3 = 2, - - pub fn attribute(self: ContractPrecision, ctx: *mlir.Context) *const mlir.Attribute { - const attr = ContractPrecisionAttr.get(ctx, .{ .value = self }) catch unreachable; - return attr.attribute(); - } -}; - -/// `#tpu.contract_precision<...>`. -pub const ContractPrecisionAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsATpuContractPrecision; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "contract_precision"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ContractPrecision, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuContractPrecisionAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ContractPrecision { - return @fromBackingInt(@intCast(c.mlirTpuContractPrecisionAttrGetValue(self.ptr()))); - } -}; - -/// Rounding mode of `tpu.fptosi` / `tpu.fptoui` / etc. -pub const RoundingMode = enum(u32) { - towards_zero = 0, - to_nearest_even = 1, - - pub fn attribute(self: RoundingMode, ctx: *mlir.Context) *const mlir.Attribute { - const attr = RoundingModeAttr.get(ctx, .{ .value = self }) catch unreachable; - return attr.attribute(); - } -}; - -/// `#tpu.rounding_mode<...>`. -pub const RoundingModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsATpuRoundingMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "rounding_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: RoundingMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) RoundingMode { - return @fromBackingInt(@intCast(c.mlirTpuRoundingModeAttrGetValue(self.ptr()))); - } -}; - -/// Target core of a kernel, remote DMAs and `tpu.sem_signal`. -pub const CoreType = enum(u32) { - tc = 0, - sc_scalar_subcore = 1, - sc_vector_subcore = 2, - - pub fn attribute(self: CoreType, ctx: *mlir.Context) *const mlir.Attribute { - const attr = CoreTypeAttr.get(ctx, .{ .value = self }) catch unreachable; - return attr.attribute(); - } -}; - -/// `#tpu.core_type<...>`. -pub const CoreTypeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsATpuCoreType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "core_type"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: CoreType, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuCoreTypeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) CoreType { - return @fromBackingInt(@intCast(c.mlirTpuCoreTypeAttrGetValue(self.ptr()))); - } -}; - -/// Grid dimension semantics annotation. -pub const DimensionSemantics = enum(u32) { - parallel = 0, - arbitrary = 1, - core_parallel = 2, - subcore_parallel = 3, - - pub fn attribute(self: DimensionSemantics, ctx: *mlir.Context) *const mlir.Attribute { - const attr = DimensionSemanticsAttr.get(ctx, .{ .value = self }) catch unreachable; - return attr.attribute(); - } -}; - -/// `#tpu.dimension_semantics<...>`. -pub const DimensionSemanticsAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsATpuDimensionSemantics; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "dimension_semantics"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: DimensionSemantics, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuDimensionSemanticsAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) DimensionSemantics { - return @fromBackingInt(@intCast(c.mlirTpuDimensionSemanticsAttrGetValue(self.ptr()))); - } -}; - -/// Buffering mode of a `window_params` entry. -pub const PipelineMode = enum(u32) { - synchronous = 1, - double_buffered = 2, - - pub fn attribute(self: PipelineMode, ctx: *mlir.Context) *const mlir.Attribute { - const attr = PipelineModeAttr.get(ctx, .{ .value = self }) catch unreachable; - return attr.attribute(); - } -}; - -/// `#tpu.pipeline_mode<...>`. -pub const PipelineModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsATpuPipelineMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "pipeline_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: PipelineMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuPipelineModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) PipelineMode { - return @fromBackingInt(@intCast(c.mlirTpuPipelineModeAttrGetValue(self.ptr()))); - } -}; - -/// Block revisit mode of a `window_params` entry. -pub const RevisitMode = enum(u32) { - immediate = 0, - any = 1, - - pub fn attribute(self: RevisitMode, ctx: *mlir.Context) *const mlir.Attribute { - const attr = RevisitModeAttr.get(ctx, .{ .value = self }) catch unreachable; - return attr.attribute(); - } -}; - -/// `#tpu.revisit_mode<...>`. -pub const RevisitModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsATpuRevisitMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "revisit_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: RevisitMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuRevisitModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) RevisitMode { - return @fromBackingInt(@intCast(c.mlirTpuRevisitModeAttrGetValue(self.ptr()))); - } -}; - -/// Memory space of a memref, `#tpu.memory_space<...>`. -pub const MemorySpace = enum(u32) { - any = 0xffff_ffff, - vmem = 0, - smem = 1, - hbm = 2, - cmem = 3, - semaphore_mem = 4, - vmem_shared = 5, - host = 6, - - pub fn attribute(self: MemorySpace, ctx: *mlir.Context) *const mlir.Attribute { - const attr = MemorySpaceAttr.get(ctx, .{ .value = self }) catch unreachable; - return attr.attribute(); - } -}; - -/// `#tpu.memory_space` — passed as the `memory_space` of a memref's type. -pub const MemorySpaceAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsATpuMemorySpace; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "memory_space"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MemorySpace, - coreType: ?CoreType = null, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const core_type: i64 = if (args.coreType) |ct| @backingInt(ct) else -1; - const result = c.mlirTpuMemorySpaceAttrGet(ctx.ptr(), @backingInt(args.value), core_type); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MemorySpace { - return @fromBackingInt(@intCast(c.mlirTpuMemorySpaceAttrGetValue(self.ptr()))); - } - pub fn coreType(self: *const Self) ?CoreType { - const core_type = c.mlirTpuMemorySpaceAttrGetCoreType(self.ptr()); - return if (core_type < 0) null else @fromBackingInt(@intCast(core_type)); - } -}; - -/// `#tpu.element_window<[pad_low], [pad_high]>` — per-arg window padding. -pub const ElementWindowAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsATpuElementWindow; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "element_window"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - padLow: []const i64, - padHigh: []const i64, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuElementWindowAttrGet( - ctx.ptr(), - @intCast(args.padLow.len), - args.padLow.ptr, - @intCast(args.padHigh.len), - args.padHigh.ptr, - ); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn numPadLow(self: *const Self) usize { - return @intCast(c.mlirTpuElementWindowAttrGetNumPadLow(self.ptr())); - } - pub fn padLow(self: *const Self, pos: usize) i64 { - return c.mlirTpuElementWindowAttrGetPadLow(self.ptr(), @intCast(pos)); - } - pub fn numPadHigh(self: *const Self) usize { - return @intCast(c.mlirTpuElementWindowAttrGetNumPadHigh(self.ptr())); - } - pub fn padHigh(self: *const Self, pos: usize) i64 { - return c.mlirTpuElementWindowAttrGetPadHigh(self.ptr(), @intCast(pos)); - } -}; - -/// `#tpu.element_window<[pad_low], [pad_high]>` — per-arg window padding. -pub fn elementWindowAttr( - ctx: *mlir.Context, - pad_low: []const i64, - pad_high: []const i64, -) *const mlir.Attribute { - const attr = ElementWindowAttr.get(ctx, .{ .padLow = pad_low, .padHigh = pad_high }) catch unreachable; - return attr.attribute(); -} - -/// `#tpu.dot_dimension_numbers<...>` — the dimension mapping of `tpu.matmul`. -pub const DotDimensionNumbersAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsATpuDotDimensionNumbers; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "dot_dimension_numbers"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - lhsContractingDims: []const i64, - rhsContractingDims: []const i64, - lhsNonContractingDims: []const i64, - /// Empty when rhs is a 1-D vector. - rhsNonContractingDims: []const i64 = &.{}, - /// Flattened (operand, dim) pairs: operand 0 is lhs, 1 is rhs. - outputDimOrder: []const i64, - lhsBatchDims: []const i64 = &.{}, - rhsBatchDims: []const i64 = &.{}, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirTpuDotDimensionNumbersAttrGet( - ctx.ptr(), - @intCast(args.lhsContractingDims.len), - args.lhsContractingDims.ptr, - @intCast(args.rhsContractingDims.len), - args.rhsContractingDims.ptr, - @intCast(args.lhsNonContractingDims.len), - args.lhsNonContractingDims.ptr, - @intCast(args.rhsNonContractingDims.len), - args.rhsNonContractingDims.ptr, - @intCast(args.outputDimOrder.len), - args.outputDimOrder.ptr, - @intCast(args.lhsBatchDims.len), - args.lhsBatchDims.ptr, - @intCast(args.rhsBatchDims.len), - args.rhsBatchDims.ptr, - ); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn numLhsContractingDims(self: *const Self) usize { - return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsContractingDims(self.ptr())); - } - pub fn lhsContractingDims(self: *const Self, pos: usize) i64 { - return c.mlirTpuDotDimensionNumbersAttrGetLhsContractingDims(self.ptr(), @intCast(pos)); - } - pub fn numRhsContractingDims(self: *const Self) usize { - return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsContractingDims(self.ptr())); - } - pub fn rhsContractingDims(self: *const Self, pos: usize) i64 { - return c.mlirTpuDotDimensionNumbersAttrGetRhsContractingDims(self.ptr(), @intCast(pos)); - } - pub fn numLhsNonContractingDims(self: *const Self) usize { - return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsNonContractingDims(self.ptr())); - } - pub fn lhsNonContractingDims(self: *const Self, pos: usize) i64 { - return c.mlirTpuDotDimensionNumbersAttrGetLhsNonContractingDims(self.ptr(), @intCast(pos)); - } - pub fn numRhsNonContractingDims(self: *const Self) usize { - return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsNonContractingDims(self.ptr())); - } - pub fn rhsNonContractingDims(self: *const Self, pos: usize) i64 { - return c.mlirTpuDotDimensionNumbersAttrGetRhsNonContractingDims(self.ptr(), @intCast(pos)); - } - pub fn numOutputDimOrder(self: *const Self) usize { - return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumOutputDimOrder(self.ptr())); - } - pub fn outputDimOrder(self: *const Self, pos: usize) i64 { - return c.mlirTpuDotDimensionNumbersAttrGetOutputDimOrder(self.ptr(), @intCast(pos)); - } - pub fn numLhsBatchDims(self: *const Self) usize { - return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumLhsBatchDims(self.ptr())); - } - pub fn lhsBatchDims(self: *const Self, pos: usize) i64 { - return c.mlirTpuDotDimensionNumbersAttrGetLhsBatchDims(self.ptr(), @intCast(pos)); - } - pub fn numRhsBatchDims(self: *const Self) usize { - return @intCast(c.mlirTpuDotDimensionNumbersAttrGetNumRhsBatchDims(self.ptr())); - } - pub fn rhsBatchDims(self: *const Self, pos: usize) i64 { - return c.mlirTpuDotDimensionNumbersAttrGetRhsBatchDims(self.ptr(), @intCast(pos)); - } -}; - -// ============================================================================= -// Discardable attributes — tpu.core_type, tpu.dimension_semantics -// ============================================================================= - -/// `tpu.core_type`, on a kernel's `func.func`: the core it runs on. -pub fn coreTypeNamedAttr(ctx: *mlir.Context, value: CoreType) mlir.NamedAttribute { - return .named(ctx, "tpu.core_type", value.attribute(ctx)); -} - -/// `tpu.dimension_semantics`, on a kernel's iteration index argument. -pub fn dimensionSemanticsNamedAttr(ctx: *mlir.Context, value: DimensionSemantics) mlir.NamedAttribute { - return .named(ctx, "tpu.dimension_semantics", value.attribute(ctx)); -} - -// ============================================================================= -// Reductions / scan / sort -// ============================================================================= - -pub fn all_reduce( - ctx: *mlir.Context, - input: *const mlir.Value, - dim: i64, - kind: ReductionKind, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.all_reduce", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "dim", .int(ctx, .i64, dim)), - .named(ctx, "kind", kind.forInput(input, false).attribute(ctx)), - }, - .location = location, - }); -} - -pub fn reduce_index( - ctx: *mlir.Context, - input: *const mlir.Value, - axis: i32, - kind: ReductionKind, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.reduce_index", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "axis", .int(ctx, .i32, axis)), - .named(ctx, "kind", kind.attribute(ctx)), - }, - .location = location, - }); -} - -pub const ScanOpts = struct { - mask: ?*const mlir.Value = null, -}; - -/// Scan the last dimension; an optional rank-one mask spans that dimension. -pub fn scan( - ctx: *mlir.Context, - input: *const mlir.Value, - kind: ReductionKind, - result_type: *const mlir.Type, - opts: ScanOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; - operands_buf.appendAssumeCapacity(input); - if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); - - return mlir.Operation.make(ctx, "tpu.scan", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "kind", kind.forInput(input, true).attribute(ctx)), - .named(ctx, "dimension", .int(ctx, .i64, @as(i64, @intCast(input.type_().isA(mlir.ShapedType).?.rank())) - 1)), - }, - .location = location, - }); -} - -pub const SortOpts = struct { - mask: ?*const mlir.Value = null, - - descending: bool = false, -}; - -pub fn sort( - ctx: *mlir.Context, - keys: *const mlir.Value, - values: *const mlir.Value, - output_mask_type: *const mlir.Type, - sorted_keys_type: *const mlir.Type, - sorted_values_type: *const mlir.Type, - opts: SortOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ keys, values }); - if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); - - return mlir.Operation.make(ctx, "tpu.sort", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{ output_mask_type, sorted_keys_type, sorted_values_type } }, - .attributes = &.{ - .named(ctx, "descending", .boolean(ctx, opts.descending)), - }, - .location = location, - }); -} - -// ============================================================================= -// Memory — load / store / vector_load / vector_store / strided_* -// ============================================================================= - -pub const LoadOpts = struct { - sublane_mask: []const bool, - sublane_stride: i32 = 1, -}; - -/// tpu.load — sublane-aware load into a vreg. -pub fn load( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const *const mlir.Value, - result_type: *const mlir.Type, - opts: LoadOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - operands_buf.appendAssumeCapacity(base); - operands_buf.appendSliceAssumeCapacity(indices); - - return mlir.Operation.make(ctx, "tpu.load", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "sublane_mask", denseBoolArrayAttribute(ctx, opts.sublane_mask)), - .named(ctx, "sublane_stride", .int(ctx, .i32, opts.sublane_stride)), - }, - .location = location, - }); -} - -pub const StoreOpts = struct { - mask: ?*const mlir.Value = null, - - sublane_mask: []const bool, - sublane_stride: i32 = 1, - add: bool = false, -}; - -/// tpu.store — sublane-aware store from a vreg. Optional elementwise mask. -pub fn store( - ctx: *mlir.Context, - value: *const mlir.Value, - base: *const mlir.Value, - indices: []const *const mlir.Value, - opts: StoreOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ value, base }); - operands_buf.appendSliceAssumeCapacity(indices); - const mask_len: i32 = if (opts.mask) |m| blk: { - operands_buf.appendAssumeCapacity(m); - break :blk 1; - } else 0; - - const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; - return mlir.Operation.make(ctx, "tpu.store", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "sublane_mask", denseBoolArrayAttribute(ctx, opts.sublane_mask)), - .named(ctx, "sublane_stride", .int(ctx, .i32, opts.sublane_stride)), - .named(ctx, "add", .boolean(ctx, opts.add)), - }, - .location = location, - }); -} - -pub const VectorLoadOpts = struct { - mask: ?*const mlir.Value = null, - - /// Per-dim load stride. Pass `&.{}` for unit-strided. - strides: []const i32 = &.{}, -}; - -/// tpu.vector_load — multi-dim strided load with optional elementwise mask. -pub fn vector_load( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const *const mlir.Value, - result_type: *const mlir.Type, - opts: VectorLoadOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; - operands_buf.appendAssumeCapacity(base); - operands_buf.appendSliceAssumeCapacity(indices); - const mask_len: i32 = if (opts.mask) |m| blk: { - operands_buf.appendAssumeCapacity(m); - break :blk 1; - } else 0; - - const seg_sizes = [3]i32{ 1, @intCast(indices.len), mask_len }; - return mlir.Operation.make(ctx, "tpu.vector_load", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "strides", .denseArray(ctx, .i32, opts.strides)), - }, - .location = location, - }); -} - -pub const VectorStoreOpts = struct { - mask: ?*const mlir.Value = null, - - strides: []const i32 = &.{}, - add: bool = false, -}; - -pub fn vector_store( - ctx: *mlir.Context, - value: *const mlir.Value, - base: *const mlir.Value, - indices: []const *const mlir.Value, - opts: VectorStoreOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ value, base }); - operands_buf.appendSliceAssumeCapacity(indices); - const mask_len: i32 = if (opts.mask) |m| blk: { - operands_buf.appendAssumeCapacity(m); - break :blk 1; - } else 0; - - const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; - return mlir.Operation.make(ctx, "tpu.vector_store", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "strides", .denseArray(ctx, .i32, opts.strides)), - .named(ctx, "add", .boolean(ctx, opts.add)), - }, - .location = location, - }); -} - -pub fn strided_load( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const *const mlir.Value, - strides: []const i32, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - operands_buf.appendAssumeCapacity(base); - operands_buf.appendSliceAssumeCapacity(indices); - - return mlir.Operation.make(ctx, "tpu.strided_load", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "strides", .denseArray(ctx, .i32, strides)), - }, - .location = location, - }); -} - -pub fn strided_store( - ctx: *mlir.Context, - value: *const mlir.Value, - base: *const mlir.Value, - indices: []const *const mlir.Value, - strides: []const i32, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ value, base }); - operands_buf.appendSliceAssumeCapacity(indices); - - return mlir.Operation.make(ctx, "tpu.strided_store", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .attributes = &.{ - .named(ctx, "strides", .denseArray(ctx, .i32, strides)), - }, - .location = location, - }); -} - -// ============================================================================= -// Compute — matmul, iota, reciprocal, casts -// ============================================================================= - -pub const MatmulOpts = struct { - /// Deprecated when `dimension_numbers` is provided. - transpose_lhs: bool = false, - transpose_rhs: bool = false, - transpose_lhs_hint: bool = false, - /// Optional precision; pass null to omit. - precision: ?ContractPrecision = null, - /// Optional `#tpu.dot_dimension_numbers<...>` attribute. When omitted the - /// canonicalizer derives one. Build with `DotDimensionNumbersAttr` or - /// `dotDimensionNumbersAttr`. - dimension_numbers: ?*const mlir.Attribute = null, -}; - -pub fn matmul( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - acc: *const mlir.Value, - result_type: *const mlir.Type, - opts: MatmulOpts, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "transpose_lhs", .boolean(ctx, opts.transpose_lhs)), - .named(ctx, "transpose_rhs", .boolean(ctx, opts.transpose_rhs)), - .named(ctx, "transpose_lhs_hint", .boolean(ctx, opts.transpose_lhs_hint)), - }); - if (opts.precision) |p| { - attrs.appendAssumeCapacity(.named(ctx, "precision", p.attribute(ctx))); - } - if (opts.dimension_numbers) |dn| { - attrs.appendAssumeCapacity(.named(ctx, "dimension_numbers", dn)); - } - - return mlir.Operation.make(ctx, "tpu.matmul", .{ - .operands = .{ .flat = &.{ lhs, rhs, acc } }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -pub const IotaOpts = struct { - dimensions: ?[]const i32 = null, -}; - -pub fn iota( - ctx: *mlir.Context, - result_type: *const mlir.Type, - opts: IotaOpts, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.dimensions) |d| { - attrs.appendAssumeCapacity(.named(ctx, "dimensions", .denseArray(ctx, .i32, d))); - } - return mlir.Operation.make(ctx, "tpu.iota", .{ - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -pub const ReciprocalOpts = struct { - approx: bool = false, - full_range: bool = false, -}; - -pub fn reciprocal( - ctx: *mlir.Context, - input: *const mlir.Value, - opts: ReciprocalOpts, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.reciprocal", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{input.type_()} }, - .attributes = &.{ - .named(ctx, "approx", .boolean(ctx, opts.approx)), - .named(ctx, "full_range", .boolean(ctx, opts.full_range)), - }, - .location = location, - }); -} - -/// Generic 1-operand → 1-result builder for the various TPU casts. -fn castOp(comptime op_name: []const u8) type { - return struct { - pub fn call( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, - ) *mlir.Operation { - return mlir.Operation.make(ctx, op_name, .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); - } - }; -} - -pub const fptosi = castOp("tpu.fptosi").call; -pub const fptoui = castOp("tpu.fptoui").call; -pub const sitofp = castOp("tpu.sitofp").call; -pub const uitofp = castOp("tpu.uitofp").call; -pub const extf = castOp("tpu.extf").call; -pub const truncf = castOp("tpu.truncf").call; -pub const bitcast = castOp("tpu.bitcast").call; -pub const bitcast_vreg = castOp("tpu.bitcast_vreg").call; -pub const mask_cast = castOp("tpu.mask_cast").call; - -// ============================================================================= -// Shape — reshape / repeat / concatenate / transpose / broadcast_in_sublanes -// ============================================================================= - -pub fn reshape( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.reshape", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn concatenate( - ctx: *mlir.Context, - sources: []const *const mlir.Value, - dimension: i32, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.concatenate", .{ - .operands = .{ .flat = sources }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "dimension", .int(ctx, .i32, dimension)), - }, - .location = location, - }); -} - -pub fn transpose( - ctx: *mlir.Context, - src: *const mlir.Value, - permutation: []const i64, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.transpose", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "permutation", .denseArray(ctx, .i64, permutation)), - }, - .location = location, - }); -} - -pub fn broadcast_in_sublanes( - ctx: *mlir.Context, - src: *const mlir.Value, - lane: i32, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.broadcast_in_sublanes", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "lane", .int(ctx, .i32, lane)), - }, - .location = location, - }); -} - -pub const DynamicRotateOpts = struct { - dimension: i32, - stride: ?i32 = null, - stride_dimension: ?i32 = null, -}; - -pub fn dynamic_rotate( - ctx: *mlir.Context, - value: *const mlir.Value, - amount: *const mlir.Value, - opts: DynamicRotateOpts, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "dimension", .int(ctx, .si32, opts.dimension)), - }); - if (opts.stride) |s| attrs.appendAssumeCapacity(.named(ctx, "stride", .int(ctx, .si32, s))); - if (opts.stride_dimension) |sd| attrs.appendAssumeCapacity(.named(ctx, "stride_dimension", .int(ctx, .si32, sd))); - - return mlir.Operation.make(ctx, "tpu.dynamic_rotate", .{ - .operands = .{ .flat = &.{ value, amount } }, - .results = .{ .flat = &.{value.type_()} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -// ============================================================================= -// Memref-shaped ops — slice / squeeze / reshape / bitcast / reinterpret_cast -// ============================================================================= - -pub const MemRefSliceArgs = struct { - /// `base_idx` count must equal source memref rank. - base_idx: []const *const mlir.Value, - /// Optional dynamic sizes; left empty when target shape is fully static. - dynamic_sizes: []const *const mlir.Value = &.{}, -}; - -pub fn memref_slice( - ctx: *mlir.Context, - mem_ref: *const mlir.Value, - args: MemRefSliceArgs, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; - operands_buf.appendAssumeCapacity(mem_ref); - operands_buf.appendSliceAssumeCapacity(args.base_idx); - operands_buf.appendSliceAssumeCapacity(args.dynamic_sizes); - - const seg_sizes = [3]i32{ 1, @intCast(args.base_idx.len), @intCast(args.dynamic_sizes.len) }; - return mlir.Operation.make(ctx, "tpu.memref_slice", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - }, - .location = location, - }); -} - -pub fn memref_squeeze( - ctx: *mlir.Context, - mem_ref: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.memref_squeeze", .{ - .operands = .{ .flat = &.{mem_ref} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn memref_reshape( - ctx: *mlir.Context, - mem_ref: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.memref_reshape", .{ - .operands = .{ .flat = &.{mem_ref} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn memref_bitcast( - ctx: *mlir.Context, - mem_ref: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.memref_bitcast", .{ - .operands = .{ .flat = &.{mem_ref} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn reinterpret_cast( - ctx: *mlir.Context, - mem_ref: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.reinterpret_cast", .{ - .operands = .{ .flat = &.{mem_ref} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{.named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &.{ 1, 0, 0, 0 }))}, - .location = location, - }); -} - -pub fn assume_layout( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.assume_layout", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn erase_memref_layout( - ctx: *mlir.Context, - mem_ref: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.erase_memref_layout", .{ - .operands = .{ .flat = &.{mem_ref} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -// ============================================================================= -// Mask building — create_mask / create_subelement_mask / mask_cast -// ============================================================================= - -/// `tpu.create_mask` — two equal-length operand groups (low, high) per dim. -pub fn create_mask( - ctx: *mlir.Context, - low_bounds: []const *const mlir.Value, - high_bounds: []const *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - std.debug.assert(low_bounds.len == high_bounds.len); - var operands_buf: stdx.BoundedArray(*const mlir.Value, 32) = .empty; - operands_buf.appendSliceAssumeCapacity(low_bounds); - operands_buf.appendSliceAssumeCapacity(high_bounds); - - return mlir.Operation.make(ctx, "tpu.create_mask", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -// ============================================================================= -// Semaphores / DMA / barriers -// ============================================================================= - -pub fn sem_alloc( - ctx: *mlir.Context, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.sem_alloc", .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn sem_barrier( - ctx: *mlir.Context, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.sem_barrier", .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn sem_read( - ctx: *mlir.Context, - semaphore: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.sem_read", .{ - .operands = .{ .flat = &.{semaphore} }, - .results = .{ .flat = &.{.int(ctx, .i32)} }, - .location = location, - }); -} - -pub fn sem_wait( - ctx: *mlir.Context, - semaphore: *const mlir.Value, - amount: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.sem_wait", .{ - .operands = .{ .flat = &.{ semaphore, amount } }, - .location = location, - }); -} - -pub const SemSignalOpts = struct { - device_id: ?*const mlir.Value = null, - core_id: ?*const mlir.Value = null, - subcore_id: ?*const mlir.Value = null, -}; - -pub fn sem_signal( - ctx: *mlir.Context, - semaphore: *const mlir.Value, - amount: *const mlir.Value, - opts: SemSignalOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ semaphore, amount }); - const dev_len: i32 = if (opts.device_id) |d| blk: { - operands_buf.appendAssumeCapacity(d); - break :blk 1; - } else 0; - const core_len: i32 = if (opts.core_id) |c_| blk: { - operands_buf.appendAssumeCapacity(c_); - break :blk 1; - } else 0; - - const subcore_len: i32 = if (opts.subcore_id) |subcore| blk: { - operands_buf.appendAssumeCapacity(subcore); - break :blk 1; - } else 0; - - const seg_sizes = [5]i32{ 1, 1, dev_len, core_len, subcore_len }; - return mlir.Operation.make(ctx, "tpu.sem_signal", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - }, - .location = location, - }); -} - -pub fn barrier( - ctx: *mlir.Context, - barrier_id: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.barrier", .{ - .operands = .{ .flat = &.{barrier_id} }, - .location = location, - }); -} - -pub const EnqueueDmaOpts = struct { - source_semaphore: ?*const mlir.Value = null, - device_id: ?*const mlir.Value = null, - core_id: ?*const mlir.Value = null, - subcore_id: ?*const mlir.Value = null, - priority: i32 = 0, - strict_ordering: bool = false, -}; - -pub fn enqueue_dma( - ctx: *mlir.Context, - source: *const mlir.Value, - target: *const mlir.Value, - target_semaphore: *const mlir.Value, - opts: EnqueueDmaOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 7) = .empty; - operands_buf.appendAssumeCapacity(source); - const src_sem_len: i32 = if (opts.source_semaphore) |s| blk: { - operands_buf.appendAssumeCapacity(s); - break :blk 1; - } else 0; - operands_buf.appendAssumeCapacity(target); - operands_buf.appendAssumeCapacity(target_semaphore); - const dev_len: i32 = if (opts.device_id) |d| blk: { - operands_buf.appendAssumeCapacity(d); - break :blk 1; - } else 0; - const core_len: i32 = if (opts.core_id) |c_| blk: { - operands_buf.appendAssumeCapacity(c_); - break :blk 1; - } else 0; - - // Source / source_semaphore? / target / target_semaphore / device_id? / core_id? / subcore_id? - const subcore_len: i32 = if (opts.subcore_id) |subcore| blk: { - operands_buf.appendAssumeCapacity(subcore); - break :blk 1; - } else 0; - - const seg_sizes = [7]i32{ 1, src_sem_len, 1, 1, dev_len, core_len, subcore_len }; - return mlir.Operation.make(ctx, "tpu.enqueue_dma", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "priority", .int(ctx, .i32, opts.priority)), - .named(ctx, "strict_ordering", .boolean(ctx, opts.strict_ordering)), - }, - .location = location, - }); -} - -pub const WaitDma2Opts = struct { - device_id: ?*const mlir.Value = null, - core_id: ?*const mlir.Value = null, - strict_ordering: bool = false, -}; - -pub fn wait_dma2( - ctx: *mlir.Context, - semaphore: *const mlir.Value, - src: *const mlir.Value, - dst: *const mlir.Value, - opts: WaitDma2Opts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ semaphore, src, dst }); - const dev_len: i32 = if (opts.device_id) |d| blk: { - operands_buf.appendAssumeCapacity(d); - break :blk 1; - } else 0; - const core_len: i32 = if (opts.core_id) |c_| blk: { - operands_buf.appendAssumeCapacity(c_); - break :blk 1; - } else 0; - - const seg_sizes = [5]i32{ 1, 1, 1, dev_len, core_len }; - return mlir.Operation.make(ctx, "tpu.wait_dma2", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "strict_ordering", .boolean(ctx, opts.strict_ordering)), - }, - .location = location, - }); -} - -// ============================================================================= -// Region-bearing — region / trace / yield -// ============================================================================= - -/// tpu.region — single-block region with an implicit `tpu.yield` terminator. -pub fn region( - ctx: *mlir.Context, - body: *mlir.Block, - result_types: []const *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.region", .{ - .results = .{ .flat = result_types }, - .blocks = &.{body}, - .verify = false, - .location = location, - }); -} - -pub fn trace( - ctx: *mlir.Context, - body: *mlir.Block, - result_types: []const *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.trace", .{ - .results = .{ .flat = result_types }, - .blocks = &.{body}, - .verify = false, - .location = location, - }); -} - -pub fn yield( - ctx: *mlir.Context, - values: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.yield", .{ - .operands = .{ .flat = values }, - .verify = false, - .location = location, - }); -} - -pub fn trace_start( - ctx: *mlir.Context, - level: i32, - message: []const u8, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.trace_start", .{ - .attributes = &.{ - .named(ctx, "level", .int(ctx, .i32, level)), - .named(ctx, "message", .string(ctx, message)), - }, - .verify = false, - .location = location, - }); -} - -pub fn trace_stop(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.trace_stop", .{ - .verify = false, - .location = location, - }); -} - -/// Build a `#tpu.dot_dimension_numbers<...>` attribute. Mirrors -/// `tpu_ops.td:TPU_DotDimensionNumbersAttr`. Pass empty slices for the -/// fields you don't use (most dot operations leave batch dims empty). -pub fn dotDimensionNumbersAttr( - ctx: *mlir.Context, - lhs_contracting: []const i64, - rhs_contracting: []const i64, - lhs_non_contracting: []const i64, - rhs_non_contracting: []const i64, - output_dim_order: []const i64, - lhs_batch: []const i64, - rhs_batch: []const i64, -) *const mlir.Attribute { - const attr = DotDimensionNumbersAttr.get(ctx, .{ - .lhsContractingDims = lhs_contracting, - .rhsContractingDims = rhs_contracting, - .lhsNonContractingDims = lhs_non_contracting, - .rhsNonContractingDims = rhs_non_contracting, - .outputDimOrder = output_dim_order, - .lhsBatchDims = lhs_batch, - .rhsBatchDims = rhs_batch, - }) catch unreachable; - return attr.attribute(); -} - -// ============================================================================= -// Misc — device_id / delay / assume_multiple -// ============================================================================= - -pub fn device_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.device_id", .{ - .results = .{ .flat = &.{.int(ctx, .i32)} }, - .location = location, - }); -} - -pub fn delay( - ctx: *mlir.Context, - cycles: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.delay", .{ - .operands = .{ .flat = &.{cycles} }, - .location = location, - }); -} - -pub fn assume_multiple( - ctx: *mlir.Context, - src: *const mlir.Value, - multiple: i32, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "tpu.assume_multiple", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{src.type_()} }, - .attributes = &.{ - .named(ctx, "multiple", .int(ctx, .i32, multiple)), - }, - .location = location, - }); -} - -// ============================================================================= -// Internal helpers -// ============================================================================= - -/// `tpu.DenseBoolArrayAttr` stores one C `int` (i32) per bool, see `mlirDenseBoolArrayGet` -fn denseBoolArrayAttribute(ctx: *mlir.Context, values: []const bool) *const mlir.Attribute { - var ints: stdx.BoundedArray(i32, 64) = .empty; - for (values) |b| ints.appendAssumeCapacity(@intFromBool(b)); - return .denseArray(ctx, .bool, ints.constSlice()); -} diff --git a/mlir/dialects/mosaic_tpu/test.zig b/mlir/dialects/mosaic_tpu/test.zig deleted file mode 100644 index 20a20f42..00000000 --- a/mlir/dialects/mosaic_tpu/test.zig +++ /dev/null @@ -1,281 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const mosaic_tpu = @import("mlir/dialects/mosaic_tpu"); - -const operation_names: []const []const u8 = &.{ - "tpu.all_reduce", - "tpu.assume_layout", - "tpu.assume_multiple", - "tpu.barrier", - "tpu.bitcast", - "tpu.bitcast_vreg", - "tpu.broadcast_in_sublanes", - "tpu.concatenate", - "tpu.create_mask", - "tpu.delay", - "tpu.device_id", - "tpu.dynamic_rotate", - "tpu.enqueue_dma", - "tpu.erase_memref_layout", - "tpu.extf", - "tpu.fptosi", - "tpu.fptoui", - "tpu.iota", - "tpu.load", - "tpu.mask_cast", - "tpu.matmul", - "tpu.memref_bitcast", - "tpu.memref_reshape", - "tpu.memref_slice", - "tpu.memref_squeeze", - "tpu.reciprocal", - "tpu.reduce_index", - "tpu.region", - "tpu.reinterpret_cast", - "tpu.reshape", - "tpu.scan", - "tpu.sem_alloc", - "tpu.sem_barrier", - "tpu.sem_read", - "tpu.sem_signal", - "tpu.sem_wait", - "tpu.sitofp", - "tpu.sort", - "tpu.store", - "tpu.strided_load", - "tpu.strided_store", - "tpu.trace", - "tpu.trace_start", - "tpu.trace_stop", - "tpu.transpose", - "tpu.truncf", - "tpu.uitofp", - "tpu.vector_load", - "tpu.vector_store", - "tpu.wait_dma2", - "tpu.yield", -}; - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - mosaic_tpu.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -fn expectSegments(op: *const mlir.Operation, expected: []const i32) !void { - const segments = op.attributeByName("operandSegmentSizes").?.isA(mlir.DenseArrayAttribute(.i32)).?; - try std.testing.expectEqual(expected.len, segments.numElements()); - for (expected, 0..) |size, i| try std.testing.expectEqual(size, segments.element(i)); -} - -fn expectPrints(expected: []const u8, value: anytype) !void { - var buf: [256]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(expected, w.buffered()); -} - -fn expectEnumAttr(comptime Attr: type, ctx: *mlir.Context) !void { - const Enum = @FieldType(Attr.InitArgs, "value"); - inline for (@typeInfo(Enum).@"enum".field_names) |name| { - const value: Enum = @field(Enum, name); - const attr = try Attr.get(ctx, .{ .value = value }); - try std.testing.expectEqual(value, attr.value()); - try std.testing.expect(attr.attribute().isA(Attr) != null); - try std.testing.expect(value.attribute(ctx).eql(attr.attribute())); - try expectPrints("#tpu." ++ Attr.mnemonic ++ "<" ++ name ++ ">", attr); - } -} - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(mosaic_tpu); - inline for (comptime std.meta.declarations(mosaic_tpu)) |decl| { - const T = @field(mosaic_tpu, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "DMA and semaphore wrappers satisfy current operand groups" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const args = mlir.Block.init(&.{}, &.{}); - defer args.deinit(); - const source = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); - const target = args.addArgument(try mlir.Type.parse(ctx, "memref<8x128xbf16, #tpu.memory_space>"), loc); - const dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); - const sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); - const amount = args.addArgument(.int(ctx, .i32), loc); - - const dma = mosaic_tpu.enqueue_dma(ctx, source, target, dma_sem, .{}, loc); - defer dma.deinit(); - try expectSegments(dma, &.{ 1, 0, 1, 1, 0, 0, 0 }); - try std.testing.expect(dma.verify()); - const signal = mosaic_tpu.sem_signal(ctx, sem, amount, .{}, loc); - defer signal.deinit(); - try expectSegments(signal, &.{ 1, 1, 0, 0, 0 }); - try std.testing.expect(signal.verify()); - - const sc_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); - const sc_dma_sem = args.addArgument(try mlir.Type.parse(ctx, "memref>"), loc); - const core = args.addArgument(.int(ctx, .i32), loc); - const subcore = args.addArgument(.int(ctx, .i32), loc); - const remote_signal = mosaic_tpu.sem_signal(ctx, sc_sem, amount, .{ .core_id = core, .subcore_id = subcore }, loc); - defer remote_signal.deinit(); - try expectSegments(remote_signal, &.{ 1, 1, 0, 1, 1 }); - try std.testing.expect(remote_signal.operand(3).eql(subcore)); - const remote_dma = mosaic_tpu.enqueue_dma(ctx, source, target, sc_dma_sem, .{ - .source_semaphore = dma_sem, - .core_id = core, - .subcore_id = subcore, - }, loc); - defer remote_dma.deinit(); - try expectSegments(remote_dma, &.{ 1, 1, 1, 1, 0, 1, 1 }); - try std.testing.expect(remote_dma.operand(5).eql(subcore)); -} - -test "matmul and reinterpret cast satisfy current required properties" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const args = mlir.Block.init(&.{}, &.{}); - defer args.deinit(); - const vector_type = mlir.Type.vector(&.{ 8, 8 }, .float(ctx, .f32)); - const lhs = args.addArgument(vector_type, loc); - const rhs = args.addArgument(vector_type, loc); - const acc = args.addArgument(vector_type, loc); - const product = mosaic_tpu.matmul(ctx, lhs, rhs, acc, vector_type, .{}, loc); - defer product.deinit(); - try std.testing.expect(product.verify()); - try std.testing.expect(!product.attributeByName("transpose_lhs_hint").?.isA(mlir.BoolAttribute).?.value()); - - const ref_type = try mlir.Type.parse(ctx, "memref<8x8xf32, #tpu.memory_space>"); - const ref = args.addArgument(ref_type, loc); - const cast = mosaic_tpu.reinterpret_cast(ctx, ref, ref_type, loc); - defer cast.deinit(); - try expectSegments(cast, &.{ 1, 0, 0, 0 }); - try std.testing.expect(cast.verify()); -} - -test "legacy reduction extrema and scan axis preserve their semantics" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const args = mlir.Block.init(&.{}, &.{}); - defer args.deinit(); - const float_type = mlir.Type.vector(&.{ 2, 8 }, .float(ctx, .f32)); - const int_type = mlir.Type.vector(&.{ 2, 8 }, .int(ctx, .i32)); - const floats = args.addArgument(float_type, loc); - const integers = args.addArgument(int_type, loc); - const mask = args.addArgument(mlir.Type.vector(&.{8}, .int(ctx, .i1)), loc); - const float_min = mosaic_tpu.all_reduce(ctx, floats, 1, .min, float_type, loc); - defer float_min.deinit(); - const int_max = mosaic_tpu.all_reduce(ctx, integers, 1, .max, int_type, loc); - defer int_max.deinit(); - try std.testing.expect(float_min.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); - try std.testing.expect(int_max.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); - - const int_scan = mosaic_tpu.scan(ctx, integers, .max, int_type, .{ .mask = mask }, loc); - defer int_scan.deinit(); - try std.testing.expect(int_scan.verify()); - try std.testing.expect(int_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); - try std.testing.expectEqual(@as(i64, 1), int_scan.attributeByName("dimension").?.isA(mlir.IntegerAttribute).?.value(i64)); - const float_scan = mosaic_tpu.scan(ctx, floats, .min, float_type, .{}, loc); - defer float_scan.deinit(); - try std.testing.expect(float_scan.attributeByName("kind").?.eql(try mlir.Attribute.parse(ctx, "#tpu.reduction_kind"))); -} - -test "tpu types are built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - const semaphore = try mosaic_tpu.SemaphoreType.get(ctx, .{}); - try std.testing.expect(mosaic_tpu.semaphoreType(ctx).isA(mosaic_tpu.SemaphoreType) != null); - try std.testing.expect(mosaic_tpu.semaphoreType(ctx).isA(mosaic_tpu.DMASemaphoreType) == null); - try expectPrints("!tpu.semaphore", semaphore); - - const dma_semaphore = try mosaic_tpu.DMASemaphoreType.get(ctx, .{}); - try std.testing.expect(mosaic_tpu.dmaSemaphoreType(ctx).isA(mosaic_tpu.DMASemaphoreType) != null); - try expectPrints("!tpu.dma_semaphore", dma_semaphore); -} - -test "tpu enum attributes are built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - try expectEnumAttr(mosaic_tpu.ReductionKindAttr, ctx); - try expectEnumAttr(mosaic_tpu.ContractPrecisionAttr, ctx); - try expectEnumAttr(mosaic_tpu.RoundingModeAttr, ctx); - try expectEnumAttr(mosaic_tpu.CoreTypeAttr, ctx); - try expectEnumAttr(mosaic_tpu.DimensionSemanticsAttr, ctx); - try expectEnumAttr(mosaic_tpu.PipelineModeAttr, ctx); - try expectEnumAttr(mosaic_tpu.RevisitModeAttr, ctx); - try expectEnumAttr(mosaic_tpu.MemorySpaceAttr, ctx); - - const smem_tc = try mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .smem, .coreType = .sc_scalar_subcore }); - try std.testing.expectEqual(mosaic_tpu.MemorySpace.smem, smem_tc.value()); - try std.testing.expectEqual(mosaic_tpu.CoreType.sc_scalar_subcore, smem_tc.coreType().?); - try expectPrints("#tpu.memory_space", smem_tc); - try std.testing.expectEqual(null, (try mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .hbm })).coreType()); - // The dialect verifier's pairing rules: no core type on hbm, no scalar subcore on vmem. - try std.testing.expectError(error.InvalidMlir, mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .hbm, .coreType = .tc })); - try std.testing.expectError(error.InvalidMlir, mosaic_tpu.MemorySpaceAttr.get(ctx, .{ .value = .vmem, .coreType = .sc_scalar_subcore })); -} - -test "tpu array attributes are built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - const window = try mosaic_tpu.ElementWindowAttr.get(ctx, .{ .padLow = &.{ 0, 2 }, .padHigh = &.{ 1, 3 } }); - try std.testing.expectEqual(2, window.numPadLow()); - try std.testing.expectEqual(2, window.padLow(1)); - try std.testing.expectEqual(2, window.numPadHigh()); - try std.testing.expectEqual(3, window.padHigh(1)); - try expectPrints("#tpu.element_window<[0, 2], [1, 3]>", window); - try std.testing.expect(mosaic_tpu.elementWindowAttr(ctx, &.{ 0, 2 }, &.{ 1, 3 }).eql(window.attribute())); - - const dims = mosaic_tpu.dotDimensionNumbersAttr(ctx, &.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}); - try expectPrints("#tpu.dot_dimension_numbers<[1], [0], [0], [1], [0, 0, 1, 1], [], []>", dims); - - const batched = try mosaic_tpu.DotDimensionNumbersAttr.get(ctx, .{ - .lhsContractingDims = &.{2}, - .rhsContractingDims = &.{1}, - .lhsNonContractingDims = &.{1}, - .outputDimOrder = &.{ 0, 0, 0, 1 }, - .lhsBatchDims = &.{0}, - .rhsBatchDims = &.{0}, - }); - try std.testing.expectEqual(0, batched.numRhsNonContractingDims()); - try std.testing.expectEqual(4, batched.numOutputDimOrder()); - try std.testing.expectEqual(1, batched.outputDimOrder(3)); - try std.testing.expectEqual(0, batched.rhsBatchDims(0)); - try expectPrints("#tpu.dot_dimension_numbers<[2], [1], [1], [], [0, 0, 0, 1], [0], [0]>", batched); -} - -test "kernel attributes carry their tpu names" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - const core_type = mosaic_tpu.coreTypeNamedAttr(ctx, .sc_vector_subcore); - try std.testing.expectEqualStrings("tpu.core_type", core_type.name().str()); - try std.testing.expect(core_type.attribute().eql(mosaic_tpu.CoreType.sc_vector_subcore.attribute(ctx))); - const semantics = mosaic_tpu.dimensionSemanticsNamedAttr(ctx, .arbitrary); - try std.testing.expectEqualStrings("tpu.dimension_semantics", semantics.name().str()); - try std.testing.expect(semantics.attribute().eql(mosaic_tpu.DimensionSemantics.arbitrary.attribute(ctx))); -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - mosaic_tpu.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/nvvm/BUILD.bazel b/mlir/dialects/nvvm/BUILD.bazel deleted file mode 100644 index f6529062..00000000 --- a/mlir/dialects/nvvm/BUILD.bazel +++ /dev/null @@ -1,168 +0,0 @@ -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - ":dialect", - ":headers", - "@llvm-project//mlir:CAPIIR", - ], -) - -zig_library( - name = "nvvm", - srcs = [ - "ops.zig", - ], - import_name = "mlir/dialects/nvvm", - main = "nvvm.zig", - tags = ["manual"], - deps = [ - ":capi", - "//mlir", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":nvvm", - "//mlir", - ], -) - -td_library( - name = "td_files", - srcs = [ - "NVVMAttrs.td", - "NVVMDialect.td", - "NVVMEnums.td", - "NVVMOps.td", - ], - includes = ["/"], - deps = [ - "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - ], -) - -gentbl_cc_library( - name = "dialect_inc_gen", - tbl_outs = [ - ( - [ - "-gen-dialect-decls", - "-dialect=nvvm", - ], - "NVVMDialect.h.inc", - ), - ( - [ - "-gen-dialect-defs", - "-dialect=nvvm", - ], - "NVVMDialect.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "NVVMDialect.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "enums_inc_gen", - tbl_outs = [ - ( - ["-gen-enum-decls"], - "NVVMEnums.h.inc", - ), - ( - ["-gen-enum-defs"], - "NVVMEnums.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "NVVMEnums.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "attrs_inc_gen", - tbl_outs = [ - ( - [ - "-gen-attrdef-decls", - "-attrdefs-dialect=nvvm", - ], - "NVVMAttrs.h.inc", - ), - ( - [ - "-gen-attrdef-defs", - "-attrdefs-dialect=nvvm", - ], - "NVVMAttrs.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "NVVMAttrs.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "ops_inc_gen", - tbl_outs = [ - ( - ["-gen-op-decls"], - "NVVMOps.h.inc", - ), - ( - ["-gen-op-defs"], - "NVVMOps.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "NVVMOps.td", - deps = [":td_files"], -) - -cc_library( - name = "headers", - hdrs = ["NVVMDialect.h"], - deps = [ - ":attrs_inc_gen", - ":dialect_inc_gen", - ":enums_inc_gen", - ":ops_inc_gen", - "@llvm-project//mlir:BytecodeOpInterface", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:LLVMDialect", - "@llvm-project//mlir:SideEffectInterfaces", - ], -) - -cc_library( - name = "dialect", - srcs = [ - "NVVMDialect.cpp", - "NVVMInference.cpp", - ], - deps = [ - ":headers", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:LLVMDialect", - ], -) diff --git a/mlir/dialects/nvvm/NVVMAttrs.td b/mlir/dialects/nvvm/NVVMAttrs.td deleted file mode 100644 index 20c3fa51..00000000 --- a/mlir/dialects/nvvm/NVVMAttrs.td +++ /dev/null @@ -1,52 +0,0 @@ -//===- NVVMAttrs.td - NVVM attributes of the CuTe DSL ------*- tablegen -*-===// -// -// Generated for nvidia-cutlass-dsl 4.8.0 from the DSL compiler -// (`_cutlass_ir.cu13` parser messages, its mnemonics, and probing its -// attribute parser and printer); do not edit by hand. -// -// The enum attributes are NVVMEnums.td. These are the compiler's other nvvm -// attributes: every mnemonic its parser knows is one of the two. -// -//===----------------------------------------------------------------------===// - -#ifndef MLIR_DIALECTS_NVVM_NVVMATTRS_TD -#define MLIR_DIALECTS_NVVM_NVVMATTRS_TD - -include "mlir/dialects/nvvm/NVVMDialect.td" -include "mlir/dialects/nvvm/NVVMEnums.td" - -def NVVM_MMAShapeAttr : NVVM_Attr<"MMAShape", "shape"> { - let summary = "Shape of an MMA operation"; - let description = "`#nvvm.shape`"; - let parameters = (ins "int":$m, "int":$n, "int":$k); - let assemblyFormat = "`<` struct(params) `>`"; -} - -def NVVM_LdStMatrixShapeAttr : NVVM_Attr<"LdStMatrixShape", "ld_st_matrix_shape"> { - let summary = "Matrix shape of ldmatrix, stmatrix and movmatrix"; - let description = "`#nvvm.ld_st_matrix_shape`"; - let parameters = (ins "int":$m, "int":$n); - let assemblyFormat = "`<` struct(params) `>`"; -} - -// Defaulted fields print only when set; the defaults and the verifier's rules -// and messages are the compiler's. -def NVVM_TargetAttr : NVVM_Attr<"Target", "target"> { - let summary = "GPU target of an NVVM module"; - let description = [{ - `#nvvm.target` - }]; - let parameters = (ins - DefaultValuedParameter<"int", "2">:$O, - StringRefParameter<"target triple", "\"nvptx64-nvidia-cuda\"">:$triple, - StringRefParameter<"target chip", "\"sm_75\"">:$chip, - StringRefParameter<"target features", "\"\"">:$features, - OptionalParameter<"::mlir::DictionaryAttr">:$flags, - OptionalParameter<"::mlir::ArrayAttr">:$link, - DefaultValuedParameter<"bool", "false">:$verifyTarget - ); - let assemblyFormat = "(`<` struct(params)^ `>`)?"; - let genVerifyDecl = 1; -} - -#endif // MLIR_DIALECTS_NVVM_NVVMATTRS_TD diff --git a/mlir/dialects/nvvm/NVVMDialect.cpp b/mlir/dialects/nvvm/NVVMDialect.cpp deleted file mode 100644 index 518e825d..00000000 --- a/mlir/dialects/nvvm/NVVMDialect.cpp +++ /dev/null @@ -1,51 +0,0 @@ -//===- NVVMDialect.cpp - NVIDIA's nvvm dialect of the CuTe DSL ------------===// - -#include "mlir/dialects/nvvm/NVVMDialect.h" - -#include "mlir/IR/Builders.h" -#include "mlir/IR/DialectImplementation.h" -#include "llvm/ADT/TypeSwitch.h" - -using namespace mlir; -using namespace mlir::cutlass_compiler::nvvm; - -#include "mlir/dialects/nvvm/NVVMDialect.cpp.inc" - -#include "mlir/dialects/nvvm/NVVMEnums.cpp.inc" - -#define GET_ATTRDEF_CLASSES -#include "mlir/dialects/nvvm/NVVMAttrs.cpp.inc" - -#define GET_OP_CLASSES -#include "mlir/dialects/nvvm/NVVMOps.cpp.inc" - -void NVVMDialect::initialize() { - addAttributes< -#define GET_ATTRDEF_LIST -#include "mlir/dialects/nvvm/NVVMAttrs.cpp.inc" - >(); - addOperations< -#define GET_OP_LIST -#include "mlir/dialects/nvvm/NVVMOps.cpp.inc" - >(); -} - -// The compiler's rules and messages. -LogicalResult TargetAttr::verify(function_ref emitError, - int O, StringRef triple, StringRef chip, - StringRef, DictionaryAttr, ArrayAttr link, - bool) { - if (O < 0 || O > 3) - return emitError() - << "The optimization level must be a number between 0 and 3."; - if (triple.empty()) - return emitError() << "The target triple cannot be empty."; - if (chip.empty()) - return emitError() << "The target chip cannot be empty."; - if (link && !llvm::all_of(link, [](Attribute attr) { - return attr && isa(attr); - })) - return emitError() - << "All the elements in the `link` array must be strings."; - return success(); -} diff --git a/mlir/dialects/nvvm/NVVMDialect.h b/mlir/dialects/nvvm/NVVMDialect.h deleted file mode 100644 index daa1c67e..00000000 --- a/mlir/dialects/nvvm/NVVMDialect.h +++ /dev/null @@ -1,27 +0,0 @@ -//===- NVVMDialect.h - NVIDIA's nvvm dialect of the CuTe DSL ----*- C++ -*-===// - -#ifndef MLIR_DIALECTS_NVVM_NVVMDIALECT_H -#define MLIR_DIALECTS_NVVM_NVVMDIALECT_H - -#include "mlir/Bytecode/BytecodeOpInterface.h" -#include "mlir/Dialect/LLVMIR/LLVMAttrs.h" -#include "mlir/Dialect/LLVMIR/LLVMDialect.h" -#include "mlir/Dialect/LLVMIR/LLVMTypes.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/BuiltinTypes.h" -#include "mlir/IR/DialectImplementation.h" -#include "mlir/IR/OpImplementation.h" -#include "mlir/Interfaces/InferTypeOpInterface.h" -#include "mlir/Interfaces/SideEffectInterfaces.h" - -#include "mlir/dialects/nvvm/NVVMDialect.h.inc" - -#include "mlir/dialects/nvvm/NVVMEnums.h.inc" - -#define GET_ATTRDEF_CLASSES -#include "mlir/dialects/nvvm/NVVMAttrs.h.inc" - -#define GET_OP_CLASSES -#include "mlir/dialects/nvvm/NVVMOps.h.inc" - -#endif // MLIR_DIALECTS_NVVM_NVVMDIALECT_H diff --git a/mlir/dialects/nvvm/NVVMDialect.td b/mlir/dialects/nvvm/NVVMDialect.td deleted file mode 100644 index 03ce7682..00000000 --- a/mlir/dialects/nvvm/NVVMDialect.td +++ /dev/null @@ -1,33 +0,0 @@ -//===- NVVMDialect.td - NVVM dialect of the CuTe DSL -------*- tablegen -*-===// - -#ifndef MLIR_DIALECTS_NVVM_NVVMDIALECT_TD -#define MLIR_DIALECTS_NVVM_NVVMDIALECT_TD - -include "mlir/IR/AttrTypeBase.td" -include "mlir/IR/DialectBase.td" -include "mlir/IR/OpBase.td" - -def NVVM_Dialect : Dialect { - let name = "nvvm"; - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let summary = "NVIDIA's NVVM dialect as the CuTe-DSL compiler registers it"; - let description = [{ - This dialect mirrors the operation, attribute and enum registry of the - `nvvm` dialect embedded in the nvidia-cutlass-dsl 4.8.0 compiler, built on - NVIDIA's own LLVM revision (23.0.0git 271b151610907). It differs from the - upstream NVVM dialect (`mlir::NVVM`) and is not meant to be registered - next to it: both are named `nvvm`. - }]; - let dependentDialects = ["::mlir::LLVM::LLVMDialect"]; - let useDefaultAttributePrinterParser = 1; -} - -class NVVM_Op traits = []> - : Op; - -class NVVM_Attr - : AttrDef { - let mnemonic = attrMnemonic; -} - -#endif // MLIR_DIALECTS_NVVM_NVVMDIALECT_TD diff --git a/mlir/dialects/nvvm/NVVMEnums.td b/mlir/dialects/nvvm/NVVMEnums.td deleted file mode 100644 index fbff0bdc..00000000 --- a/mlir/dialects/nvvm/NVVMEnums.td +++ /dev/null @@ -1,1386 +0,0 @@ -// The nvvm dialect's enums, generated from the nvidia-cutlass-dsl 4.8.0 enum -// bindings (`_nvvm_enum_gen.py`); do not edit. -// -// Case values and spellings are the compiler's. An enum the bindings build -// as a dialect attribute also has its EnumAttr, printed `#` -// or, where the compiler prints it so, `#nvvm`. - -#ifndef MLIR_DIALECTS_NVVM_NVVMENUMS_TD -#define MLIR_DIALECTS_NVVM_NVVMENUMS_TD - -include "mlir/dialects/nvvm/NVVMDialect.td" -include "mlir/IR/EnumAttr.td" - -def NVVM_AtomicOpKindEnum : I32EnumAttr<"AtomicOpKind", "operations supported by atom instruction", [ - I32EnumAttrCase<"AND", 0, "and">, - I32EnumAttrCase<"OR", 1, "or">, - I32EnumAttrCase<"XOR", 2, "xor">, - I32EnumAttrCase<"CAS", 3, "cas">, - I32EnumAttrCase<"EXCH", 4, "exch">, - I32EnumAttrCase<"ADD", 5, "add">, - I32EnumAttrCase<"INC", 6, "inc">, - I32EnumAttrCase<"DEC", 7, "dec">, - I32EnumAttrCase<"MIN", 8, "min">, - I32EnumAttrCase<"MAX", 9, "max">, - I32EnumAttrCase<"UMIN", 10, "umin">, - I32EnumAttrCase<"UMAX", 11, "umax">, - I32EnumAttrCase<"FADD", 12, "fadd">, - I32EnumAttrCase<"SUB", 13, "sub"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_BarrierReductionEnum : I32EnumAttr<"BarrierReduction", "NVVM barrier reduction operation", [ - I32EnumAttrCase<"POPC", 0, "popc">, - I32EnumAttrCase<"AND", 1, "and">, - I32EnumAttrCase<"OR", 2, "or"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_BarrierReduxKindEnum : I32EnumAttr<"BarrierReduxKind", "NVVM barrier redux kind", [ - I32EnumAttrCase<"AND", 0, "and">, - I32EnumAttrCase<"OR", 1, "or">, - I32EnumAttrCase<"POPC", 2, "popc"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_BlockScaleFormatEnum : I32EnumAttr<"BlockScaleFormat", "MMA Block Scale Format", [ - I32EnumAttrCase<"UE8M0", 0, "ue8m0">, - I32EnumAttrCase<"UE4M3", 1, "ue4m3"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_CTAGroupKindEnum : I32EnumAttr<"CTAGroupKind", "NVVM CTA group kind", [ - I32EnumAttrCase<"CTA_1", 0, "cta_1">, - I32EnumAttrCase<"CTA_2", 1, "cta_2"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_CVTPackFloatKindEnum : I32EnumAttr<"CVTPackFloatKind", "NVVM CVT Pack Float kind", [ - I32EnumAttrCase<"F32", 0, "f32">, - I32EnumAttrCase<"F16x2", 1, "f16x2">, - I32EnumAttrCase<"E4M3x2", 2, "e4m3x2">, - I32EnumAttrCase<"E5M2x2", 3, "e5m2x2">, - I32EnumAttrCase<"BF16x2", 4, "bf16x2">, - I32EnumAttrCase<"E2M1x2", 5, "e2m1x2">, - I32EnumAttrCase<"E2M3x2", 6, "e2m3x2">, - I32EnumAttrCase<"E3M2x2", 7, "e3m2x2">, - I32EnumAttrCase<"UE8M0x2", 8, "ue8m0x2"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_CacheEvictionPriorityEnum : I32EnumAttr<"CacheEvictionPriority", "NVVM Cache Eviction Priority", [ - I32EnumAttrCase<"EvictNormal", 0, "evict_normal">, - I32EnumAttrCase<"EvictFirst", 1, "evict_first">, - I32EnumAttrCase<"EvictLast", 2, "evict_last">, - I32EnumAttrCase<"EvictUnchanged", 3, "evict_unchanged">, - I32EnumAttrCase<"NoAllocate", 4, "no_allocate"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_ClusterLaunchControlQueryTypeEnum : I32EnumAttr<"ClusterLaunchControlQueryType", "NVVM ClusterLaunchControlQueryType", [ - I32EnumAttrCase<"IS_CANCELED", 0, "is_canceled">, - I32EnumAttrCase<"GET_FIRST_CTA_ID_X", 1, "get_first_cta_id_x">, - I32EnumAttrCase<"GET_FIRST_CTA_ID_Y", 2, "get_first_cta_id_y">, - I32EnumAttrCase<"GET_FIRST_CTA_ID_Z", 3, "get_first_cta_id_z"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_CompareOpKindEnum : I32EnumAttr<"CompareOpKind", "Comparison operator encoding", [ - I32EnumAttrCase<"EQ", 0, "eq">, - I32EnumAttrCase<"NE", 1, "ne">, - I32EnumAttrCase<"UGT", 2, "ugt">, - I32EnumAttrCase<"UGE", 3, "uge">, - I32EnumAttrCase<"ULT", 4, "ult">, - I32EnumAttrCase<"ULE", 5, "ule">, - I32EnumAttrCase<"SGT", 6, "sgt">, - I32EnumAttrCase<"SGE", 7, "sge">, - I32EnumAttrCase<"SLT", 8, "slt">, - I32EnumAttrCase<"SLE", 9, "sle"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_ConvertFP4TypeEnum : I32EnumAttr<"ConvertFP4Type", "NVVM ConvertFP4Type kind", [ - I32EnumAttrCase<"E2M1", 0, "e2m1"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_ConvertFP8TypeEnum : I32EnumAttr<"ConvertFP8Type", "NVVM ConvertFP8Type kind", [ - I32EnumAttrCase<"E4M3", 0, "e4m3">, - I32EnumAttrCase<"E5M2", 1, "e5m2">, - I32EnumAttrCase<"UE8M0", 2, "ue8m0">, - I32EnumAttrCase<"UE5M3", 3, "ue5m3"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_ConvertScaleKindEnum : I32EnumAttr<"ConvertScaleKind", "NVVM ConvertScale kind", [ - I32EnumAttrCase<"NONE", 0, "none">, - I32EnumAttrCase<"UE8M0", 1, "ue8m0"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_DotAccumulateTypeEnum : I32EnumAttr<"DotAccumulateType", "NVVM DotAccumulateType", [ - I32EnumAttrCase<"SIGNED", 1, "signed">, - I32EnumAttrCase<"UNSIGNED", 0, "unsigned"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_EvictKindEnum : I32EnumAttr<"EvictKind", "NVVM L2 Prefetch Size", [ - I32EnumAttrCase<"NORMAL", 0, "evict_normal">, - I32EnumAttrCase<"FIRST", 1, "evict_first">, - I32EnumAttrCase<"LAST", 2, "evict_last">, - I32EnumAttrCase<"UNCHANGED", 3, "evict_unchanged">, - I32EnumAttrCase<"NOALLOCATE", 4, "evict_no_allocate">, - I32EnumAttrCase<"NONE", 5, "none"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_FPRoundingModeEnum : I32EnumAttr<"FPRoundingMode", "NVVM FPRoundingMode kind", [ - I32EnumAttrCase<"NONE", 0, "none">, - I32EnumAttrCase<"RN", 1, "rn">, - I32EnumAttrCase<"RM", 2, "rm">, - I32EnumAttrCase<"RP", 3, "rp">, - I32EnumAttrCase<"RZ", 4, "rz">, - I32EnumAttrCase<"RNA", 5, "rna">, - I32EnumAttrCase<"RS", 6, "rs"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_GridDepActionKindEnum : I32EnumAttr<"GridDepActionKind", "Action kind for grid dependency control", [ - I32EnumAttrCase<"wait", 0, "wait">, - I32EnumAttrCase<"launch_dependents", 1, "launch_dependents"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_IntegerRoundingModeEnum : I32EnumAttr<"IntegerRoundingMode", "NVVM IntegerRoundingMode kind", [ - I32EnumAttrCase<"NONE", 0, "none">, - I32EnumAttrCase<"RNI", 1, "rni">, - I32EnumAttrCase<"RZI", 2, "rzi">, - I32EnumAttrCase<"RMI", 3, "rmi">, - I32EnumAttrCase<"RPI", 4, "rpi"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_L2PrefetchSizeEnum : I32EnumAttr<"L2PrefetchSize", "NVVM L2 Prefetch Size", [ - I32EnumAttrCase<"NONE", 0, "none">, - I32EnumAttrCase<"RESERVED", 1, "reserved">, - I32EnumAttrCase<"SIZE_64B", 2, "size_64b">, - I32EnumAttrCase<"SIZE_128B", 3, "size_128b">, - I32EnumAttrCase<"SIZE_256B", 4, "size_256b"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_LdStMatrixEltTypeEnum : I32EnumAttr<"LdStMatrixEltType", "Element type for ldmatrix and stmatrix", [ - I32EnumAttrCase<"B16", 0, "b16">, - I32EnumAttrCase<"B8", 1, "b8">, - I32EnumAttrCase<"B8X16_B6X16_P32", 2, "b8x16.b6x16_p32">, - I32EnumAttrCase<"B8X16_B4X16_P64", 3, "b8x16.b4x16_p64"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_LoadCacheModifierExtKindEnum : I32EnumAttr<"LoadCacheModifierExtKind", "NVVM load cache modifier kind(Ext)", [ - I32EnumAttrCase<"CA", 0, "ca">, - I32EnumAttrCase<"CG", 1, "cg">, - I32EnumAttrCase<"CS", 2, "cs">, - I32EnumAttrCase<"LU", 3, "lu">, - I32EnumAttrCase<"CV", 4, "cv">, - I32EnumAttrCase<"NONE", 5, "none"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_LoadCacheModifierKindEnum : I32EnumAttr<"LoadCacheModifierKind", "NVVM load cache modifier kind", [ - I32EnumAttrCase<"CA", 0, "ca">, - I32EnumAttrCase<"CG", 1, "cg">, - I32EnumAttrCase<"CS", 2, "cs">, - I32EnumAttrCase<"LU", 3, "lu">, - I32EnumAttrCase<"CV", 4, "cv"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_LoadShapeEnum : I32EnumAttr<"LoadShape", "shape attribute for ldmatrix", [ - I32EnumAttrCase<"M8N8", 0, "m8n8">, - I32EnumAttrCase<"M8N16", 1, "m8n16">, - I32EnumAttrCase<"M16N16", 3, "m16n16"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_LoadSrcFormatEnum : I32EnumAttr<"LoadSrcFormat", "source format for ldmatrix", [ - I32EnumAttrCase<"B6x16_P32", 0, "b6x16_p32">, - I32EnumAttrCase<"B4x16_P64", 1, "b4x16_p64">, - I32EnumAttrCase<"B8", 2, "b8"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MBarrierLayoutEnum : I32EnumAttr<"MBarrierLayout", "NVVM MBarrier Layout", [ - I32EnumAttrCase<"V0", 0, "v0">, - I32EnumAttrCase<"V1", 1, "v1"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MBarrierPhaseEnum : I32EnumAttr<"MBarrierPhase", "NVVM mbarrier phase type", [ - I32EnumAttrCase<"NONE", 0, "none">, - I32EnumAttrCase<"PRIMARY", 1, "primary">, - I32EnumAttrCase<"CONDITIONAL", 2, "conditional"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MBarrierScopeKindEnum : I32EnumAttr<"MBarrierScopeKind", "NVVM MBarrier scope kind", [ - I32EnumAttrCase<"CTA", 0, "cta">, - I32EnumAttrCase<"CLUSTER", 1, "cluster"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MBarrierSpaceKindEnum : I32EnumAttr<"MBarrierSpaceKind", "NVVM MBarrier space kind", [ - I32EnumAttrCase<"CTA", 0, "cta">, - I32EnumAttrCase<"CLUSTER", 1, "cluster"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MBarrierTxnKindEnum : I32EnumAttr<"MBarrierTxnKind", "NVVM MBarrier Transaction kind", [ - I32EnumAttrCase<"ARRIVE", 0, "arrive">, - I32EnumAttrCase<"ARRIVE_DROP", 1, "arrive_drop">, - I32EnumAttrCase<"ARRIVE_EXPECT_TX", 2, "arrive_expect_tx">, - I32EnumAttrCase<"ARRIVE_DROP_EXPECT_TX", 3, "arrive_drop_expect_tx">, - I32EnumAttrCase<"EXPECT_TX", 4, "expect_tx">, - I32EnumAttrCase<"COMPLETE_TX", 5, "complete_tx"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MBarrierWaitKindEnum : I32EnumAttr<"MBarrierWaitKind", "NVVM MBarrier wait kind", [ - I32EnumAttrCase<"TEST", 0, "test">, - I32EnumAttrCase<"TRY", 1, "try"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MMAB1OpEnum : I32EnumAttr<"MMAB1Op", "MMA binary operations", [ - I32EnumAttrCase<"none", 0, "none">, - I32EnumAttrCase<"xor_popc", 1, "xor_popc">, - I32EnumAttrCase<"and_popc", 2, "and_popc"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MMABlockScaleKindEnum : I32EnumAttr<"MMABlockScaleKind", "Block Scale Kind", [ - I32EnumAttrCase<"MXF8F6F4", 0, "mxf8f6f4">, - I32EnumAttrCase<"MXF4", 1, "mxf4">, - I32EnumAttrCase<"MXF4NVF4", 2, "mxf4nvf4"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MMACtaCountEnum : I32EnumAttr<"MMACtaCount", "MMA CTA count", [ - I32EnumAttrCase<"cta1", 1, "cta1">, - I32EnumAttrCase<"cta2", 2, "cta2"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MMAFragEnum : I32EnumAttr<"MMAFrag", "NVVM MMA frag type", [ - I32EnumAttrCase<"a", 0, "a">, - I32EnumAttrCase<"b", 1, "b">, - I32EnumAttrCase<"c", 2, "c"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MMAIntOverflowEnum : I32EnumAttr<"MMAIntOverflow", "MMA overflow options", [ - I32EnumAttrCase<"satfinite", 1, "satfinite">, - I32EnumAttrCase<"wrapped", 0, "wrapped"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MMAKindEnum : I32EnumAttr<"MMAKind", "MMA operation kind", [ - I32EnumAttrCase<"f8f6f4", 0, "f8f6f4"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MMALayoutEnum : I32EnumAttr<"MMALayout", "NVVM MMA layout", [ - I32EnumAttrCase<"row", 0, "row">, - I32EnumAttrCase<"col", 1, "col"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MMATypesEnum : I32EnumAttr<"MMATypes", "NVVM MMA types", [ - I32EnumAttrCase<"f16", 0, "f16">, - I32EnumAttrCase<"f32", 1, "f32">, - I32EnumAttrCase<"tf32", 2, "tf32">, - I32EnumAttrCase<"bf16", 9, "bf16">, - I32EnumAttrCase<"s8", 4, "s8">, - I32EnumAttrCase<"u8", 3, "u8">, - I32EnumAttrCase<"s32", 5, "s32">, - I32EnumAttrCase<"s4", 8, "s4">, - I32EnumAttrCase<"u4", 7, "u4">, - I32EnumAttrCase<"b1", 6, "b1">, - I32EnumAttrCase<"f64", 10, "f64">, - I32EnumAttrCase<"e4m3", 11, "e4m3">, - I32EnumAttrCase<"e5m2", 12, "e5m2">, - I32EnumAttrCase<"e3m2", 13, "e3m2">, - I32EnumAttrCase<"e2m3", 14, "e2m3">, - I32EnumAttrCase<"e2m1", 15, "e2m1"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MatchSyncKindEnum : I32EnumAttr<"MatchSyncKind", "NVVM match sync kind", [ - I32EnumAttrCase<"any", 0, "any">, - I32EnumAttrCase<"all", 1, "all"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MemOrderKindEnum : I32EnumAttr<"MemOrderKind", "NVVM Memory Ordering kind", [ - I32EnumAttrCase<"WEAK", 0, "weak">, - I32EnumAttrCase<"RELAXED", 1, "relaxed">, - I32EnumAttrCase<"ACQUIRE", 2, "acquire">, - I32EnumAttrCase<"RELEASE", 3, "release">, - I32EnumAttrCase<"ACQ_REL", 4, "acq_rel">, - I32EnumAttrCase<"SC", 5, "sc">, - I32EnumAttrCase<"MMIO", 6, "mmio">, - I32EnumAttrCase<"VOLATILE", 8, "volatile">, - I32EnumAttrCase<"CONSTANT", 7, "constant"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MemScopeKindEnum : I32EnumAttr<"MemScopeKind", "NVVM Memory Scope kind", [ - I32EnumAttrCase<"CTA", 0, "cta">, - I32EnumAttrCase<"CLUSTER", 1, "cluster">, - I32EnumAttrCase<"GPU", 2, "gpu">, - I32EnumAttrCase<"SYS", 3, "sys"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_MulModeEnum : I32EnumAttr<"MulMode", "multiply mode attribute", [ - I32EnumAttrCase<"HI", 0, "hi">, - I32EnumAttrCase<"LO", 1, "lo">, - I32EnumAttrCase<"WIDE", 2, "wide"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_NVVMMemorySpaceEnum : I32EnumAttr<"NVVMMemorySpace", "NVVM Memory Space", [ - I32EnumAttrCase<"Generic", 0, "generic">, - I32EnumAttrCase<"Global", 1, "global">, - I32EnumAttrCase<"Shared", 3, "shared">, - I32EnumAttrCase<"Constant", 4, "constant">, - I32EnumAttrCase<"Local", 5, "local">, - I32EnumAttrCase<"Tensor", 6, "tensor">, - I32EnumAttrCase<"SharedCluster", 7, "shared_cluster"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_PermuteModeEnum : I32EnumAttr<"PermuteMode", "NVVM permute mode", [ - I32EnumAttrCase<"DEFAULT", 0, "default">, - I32EnumAttrCase<"F4E", 1, "f4e">, - I32EnumAttrCase<"B4E", 2, "b4e">, - I32EnumAttrCase<"RC8", 3, "rc8">, - I32EnumAttrCase<"ECL", 4, "ecl">, - I32EnumAttrCase<"ECR", 5, "ecr">, - I32EnumAttrCase<"RC16", 6, "rc16"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_PrefetchCacheLevelEnum : I32EnumAttr<"PrefetchCacheLevel", "NVVM Prefetch Cache Level", [ - I32EnumAttrCase<"L1", 0, "L1">, - I32EnumAttrCase<"L2", 1, "L2"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_ProxyKindEnum : I32EnumAttr<"ProxyKind", "Proxy kind", [ - I32EnumAttrCase<"alias", 0, "alias">, - I32EnumAttrCase<"async_", 1, "async">, - I32EnumAttrCase<"async_global", 2, "async.global">, - I32EnumAttrCase<"async_shared", 3, "async.shared">, - I32EnumAttrCase<"TENSORMAP", 4, "tensormap">, - I32EnumAttrCase<"GENERIC", 5, "generic"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_ReductionKindEnum : I32EnumAttr<"ReductionKind", "NVVM Reduction Kind attribute", [ - I32EnumAttrCase<"ADD", 1, "add">, - I32EnumAttrCase<"AND", 2, "and">, - I32EnumAttrCase<"MAX", 3, "max">, - I32EnumAttrCase<"MIN", 4, "min">, - I32EnumAttrCase<"OR", 5, "or">, - I32EnumAttrCase<"UMAX", 6, "umax">, - I32EnumAttrCase<"UMIN", 7, "umin">, - I32EnumAttrCase<"XOR", 8, "xor">, - I32EnumAttrCase<"FMIN", 9, "fmin">, - I32EnumAttrCase<"FMAX", 10, "fmax"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_ReductionOpEnum : I32EnumAttr<"ReductionOp", "Ops supported by red instruction", [ - I32EnumAttrCase<"AND", 0, "and">, - I32EnumAttrCase<"OR", 1, "or">, - I32EnumAttrCase<"XOR", 2, "xor">, - I32EnumAttrCase<"ADD", 3, "add">, - I32EnumAttrCase<"INC", 4, "inc">, - I32EnumAttrCase<"DEC", 5, "dec">, - I32EnumAttrCase<"MIN", 6, "min">, - I32EnumAttrCase<"MAX", 7, "max"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_ReductionTypeEnum : I32EnumAttr<"ReductionType", "types supported by red instruction", [ - I32EnumAttrCase<"B32", 0, "b32">, - I32EnumAttrCase<"B64", 1, "b64">, - I32EnumAttrCase<"U32", 2, "u32">, - I32EnumAttrCase<"U64", 3, "u64">, - I32EnumAttrCase<"S32", 4, "s32">, - I32EnumAttrCase<"S64", 5, "s64">, - I32EnumAttrCase<"F32", 6, "f32">, - I32EnumAttrCase<"F64", 7, "f64">, - I32EnumAttrCase<"F16", 8, "f16">, - I32EnumAttrCase<"F16X2", 9, "f16x2">, - I32EnumAttrCase<"BF16", 10, "bf16">, - I32EnumAttrCase<"BF16X2", 11, "bf16x2"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SPCompressElemSizeEnum : I32EnumAttr<"SPCompressElemSize", "Sparse tensor compression element size", [ - I32EnumAttrCase<"B4", 4, "b4">, - I32EnumAttrCase<"B8", 8, "b8">, - I32EnumAttrCase<"B16", 16, "b16"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SPCompressFactorTypeEnum : I32EnumAttr<"SPCompressFactorType", "Sparse tensor compression factor type", [ - I32EnumAttrCase<"SP2TO4", 0, "sp2to4"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SPCompressIndexSizeEnum : I32EnumAttr<"SPCompressIndexSize", "Sparse tensor compression index size", [ - I32EnumAttrCase<"B2", 2, "b2">, - I32EnumAttrCase<"B4", 4, "b4"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SPCompressOpKindEnum : I32EnumAttr<"SPCompressOpKind", "spcompress operation kind", [ - I32EnumAttrCase<"MIN", 0, "min">, - I32EnumAttrCase<"MAX", 1, "max"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SPCompressRepFactorEnum : I32EnumAttr<"SPCompressRepFactor", "Sparse tensor compression repetition factor", [ - I32EnumAttrCase<"X1", 1, "x1">, - I32EnumAttrCase<"X2", 2, "x2">, - I32EnumAttrCase<"X4", 4, "x4">, - I32EnumAttrCase<"X8", 8, "x8">, - I32EnumAttrCase<"X16", 16, "x16">, - I32EnumAttrCase<"X32", 32, "x32">, - I32EnumAttrCase<"X64", 64, "x64">, - I32EnumAttrCase<"X128", 128, "x128"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SPDecompressElemSizeEnum : I32EnumAttr<"SPDecompressElemSize", "Sparse tensor decompression element size", [ - I32EnumAttrCase<"B4", 4, "b4">, - I32EnumAttrCase<"B8", 8, "b8">, - I32EnumAttrCase<"B16", 16, "b16"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SPDecompressFactorTypeEnum : I32EnumAttr<"SPDecompressFactorType", "Sparse tensor decompression factor type", [ - I32EnumAttrCase<"SP1TO2", 0, "sp1to2">, - I32EnumAttrCase<"SP1TO4", 1, "sp1to4">, - I32EnumAttrCase<"SP1TO8", 2, "sp1to8">, - I32EnumAttrCase<"SP1TO16", 3, "sp1to16">, - I32EnumAttrCase<"SP2TO4", 4, "sp2to4">, - I32EnumAttrCase<"SP2TO8", 5, "sp2to8">, - I32EnumAttrCase<"SP2TO16", 6, "sp2to16">, - I32EnumAttrCase<"SP4TO8", 7, "sp4to8">, - I32EnumAttrCase<"SP4TO16", 8, "sp4to16">, - I32EnumAttrCase<"SP8TO16", 9, "sp8to16"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SPDecompressIndexSizeEnum : I32EnumAttr<"SPDecompressIndexSize", "Sparse tensor decompression index size", [ - I32EnumAttrCase<"B2", 2, "b2">, - I32EnumAttrCase<"B4", 4, "b4"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SPDecompressRepFactorEnum : I32EnumAttr<"SPDecompressRepFactor", "Sparse tensor decompression repetition factor", [ - I32EnumAttrCase<"X1", 1, "x1">, - I32EnumAttrCase<"X2", 2, "x2">, - I32EnumAttrCase<"X4", 4, "x4">, - I32EnumAttrCase<"X8", 8, "x8">, - I32EnumAttrCase<"X16", 16, "x16">, - I32EnumAttrCase<"X32", 32, "x32">, - I32EnumAttrCase<"X64", 64, "x64"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SaturationModeEnum : I32EnumAttr<"SaturationMode", "NVVM SaturationMode kind", [ - I32EnumAttrCase<"NONE", 0, "none">, - I32EnumAttrCase<"SATFINITE", 1, "satfinite">, - I32EnumAttrCase<"SAT", 2, "sat"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SaturationModeKindEnum : I32EnumAttr<"SaturationModeKind", "NVVM SaturationMode kind", [ - I32EnumAttrCase<"NONE", 0, "none">, - I32EnumAttrCase<"SATFINITE", 1, "satfinite"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_ScaleVecSizeEnum : I32EnumAttr<"ScaleVecSize", "MMA Scale Vector Sizes", [ - I32EnumAttrCase<"X1", 0, "x1">, - I32EnumAttrCase<"X2", 1, "x2">, - I32EnumAttrCase<"X4", 2, "x4"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SetMaxRegisterActionEnum : I32EnumAttr<"SetMaxRegisterAction", "NVVM set max register action", [ - I32EnumAttrCase<"decrease", 1, "decrease">, - I32EnumAttrCase<"increase", 0, "increase"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SharedSpaceEnum : I32EnumAttr<"SharedSpace", "Shared memory space", [ - I32EnumAttrCase<"shared_cta", 0, "cta">, - I32EnumAttrCase<"shared_cluster", 1, "cluster"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_ShflKindEnum : I32EnumAttr<"ShflKind", "NVVM shuffle kind", [ - I32EnumAttrCase<"bfly", 0, "bfly">, - I32EnumAttrCase<"up", 1, "up">, - I32EnumAttrCase<"down", 2, "down">, - I32EnumAttrCase<"idx", 3, "idx"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_SparsityFormatEnum : I32EnumAttr<"SparsityFormat", "MMA Sparsity Format", [ - I32EnumAttrCase<"THREAD", 0, "thread"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_StateSpaceEnum : I32EnumAttr<"StateSpace", "NVVM State Space", [ - I32EnumAttrCase<"GENERIC", 0, "generic">, - I32EnumAttrCase<"GLOBAL", 1, "global">, - I32EnumAttrCase<"SHARED_CTA", 2, "shared_cta">, - I32EnumAttrCase<"SHARED_CLUSTER", 3, "shared_cluster">, - I32EnumAttrCase<"CONSTANT", 4, "constant">, - I32EnumAttrCase<"LOCAL", 5, "local">, - I32EnumAttrCase<"TENSOR", 6, "tensor"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_StoreCacheModifierKindEnum : I32EnumAttr<"StoreCacheModifierKind", "NVVM store cache modifier kind", [ - I32EnumAttrCase<"WB", 0, "wb">, - I32EnumAttrCase<"CG", 1, "cg">, - I32EnumAttrCase<"CS", 2, "cs">, - I32EnumAttrCase<"WT", 3, "wt">, - I32EnumAttrCase<"NONE", 4, "none"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_StoreShapeEnum : I32EnumAttr<"StoreShape", "shape attribute for stmatrix", [ - I32EnumAttrCase<"M8N8", 0, "m8n8">, - I32EnumAttrCase<"M16N8", 2, "m16n8"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_TCBarParamEnum : I32EnumAttr<"TCBarParam", "Cluster MMA Barrier Parameter Type", [ - I32EnumAttrCase<"a1t0", 0, "a1t0">, - I32EnumAttrCase<"a0tx", 1, "a0tx"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_TMALoadModeEnum : I32EnumAttr<"TMALoadMode", "NVVM TMA Load Mode", [ - I32EnumAttrCase<"TILE", 0, "tile">, - I32EnumAttrCase<"IM2COL", 1, "im2col">, - I32EnumAttrCase<"IM2COL_W", 2, "im2col_w">, - I32EnumAttrCase<"IM2COL_W_128", 3, "im2col_w_128">, - I32EnumAttrCase<"TILE_GATHER4", 4, "tile_gather4"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_TMAReduxKindEnum : I32EnumAttr<"TMAReduxKind", "NVVM TMA redux kind", [ - I32EnumAttrCase<"ADD", 0, "add">, - I32EnumAttrCase<"MAX", 2, "max">, - I32EnumAttrCase<"MIN", 1, "min">, - I32EnumAttrCase<"INC", 3, "inc">, - I32EnumAttrCase<"DEC", 4, "dec">, - I32EnumAttrCase<"AND", 5, "and">, - I32EnumAttrCase<"OR", 6, "or">, - I32EnumAttrCase<"XOR", 7, "xor"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_TMAStoreModeEnum : I32EnumAttr<"TMAStoreMode", "NVVM TMA Store Mode", [ - I32EnumAttrCase<"TILE", 0, "tile">, - I32EnumAttrCase<"IM2COL", 1, "im2col">, - I32EnumAttrCase<"TILE_SCATTER4", 2, "tile_scatter4">, - I32EnumAttrCase<"IM2COL_W", 3, "im2col_w"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_Tcgen05CpMulticastEnum : I32EnumAttr<"Tcgen05CpMulticast", "tcgen05 cp multicast", [ - I32EnumAttrCase<"NONE", 0, "none">, - I32EnumAttrCase<"WARPX2_02_13", 1, "warpx2_02_13">, - I32EnumAttrCase<"WARPX2_01_23", 2, "warpx2_01_23">, - I32EnumAttrCase<"WARPX4", 3, "warpx4"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_Tcgen05CpShapeEnum : I32EnumAttr<"Tcgen05CpShape", "tcgen05 cp shapes", [ - I32EnumAttrCase<"SHAPE_128x256b", 0, "shape_128x256b">, - I32EnumAttrCase<"SHAPE_4x256b", 1, "shape_4x256b">, - I32EnumAttrCase<"SHAPE_128x128b", 2, "shape_128x128b">, - I32EnumAttrCase<"SHAPE_64x128b", 3, "shape_64x128b">, - I32EnumAttrCase<"SHAPE_32x128b", 4, "shape_32x128b"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_Tcgen05CpSrcFormatEnum : I32EnumAttr<"Tcgen05CpSrcFormat", "tcgen05 cp source format", [ - I32EnumAttrCase<"B6x16_P32", 0, "b6x16_p32">, - I32EnumAttrCase<"B4x16_P64", 1, "b4x16_p64"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_Tcgen05FenceKindEnum : I32EnumAttr<"Tcgen05FenceKind", "NVVM Tcgen05 fence kind", [ - I32EnumAttrCase<"BEFORE_THREAD_SYNC", 0, "before">, - I32EnumAttrCase<"AFTER_THREAD_SYNC", 1, "after"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_Tcgen05LdStShapeEnum : I32EnumAttr<"Tcgen05LdStShape", "allowed 32-bit signless integer cases: 0, 1, 2, 3, 4", [ - I32EnumAttrCase<"SHAPE_16X64B", 0, "shape_16x64b">, - I32EnumAttrCase<"SHAPE_16X128B", 1, "shape_16x128b">, - I32EnumAttrCase<"SHAPE_16X256B", 2, "shape_16x256b">, - I32EnumAttrCase<"SHAPE_32X32B", 3, "shape_32x32b">, - I32EnumAttrCase<"SHAPE_16X32BX2", 4, "shape_16x32bx2"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_Tcgen05MMABlockScaleEnum : I32EnumAttr<"Tcgen05MMABlockScale", "tcgen05.mma block scale attribute", [ - I32EnumAttrCase<"DEFAULT", 0, "default">, - I32EnumAttrCase<"BLOCK16", 1, "block16">, - I32EnumAttrCase<"BLOCK32", 2, "block32"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_Tcgen05MMACollectorBBufferEnum : I32EnumAttr<"Tcgen05MMACollectorBBuffer", "tcgen05 MMA Collector Buffer B Attribute", [ - I32EnumAttrCase<"B0", 0, "b0">, - I32EnumAttrCase<"B1", 1, "b1">, - I32EnumAttrCase<"B2", 2, "b2">, - I32EnumAttrCase<"B3", 3, "b3"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_Tcgen05MMACollectorOpEnum : I32EnumAttr<"Tcgen05MMACollectorOp", "tcgen05.mma Collector Buffer Operation", [ - I32EnumAttrCase<"DISCARD", 0, "discard">, - I32EnumAttrCase<"LASTUSE", 1, "lastuse">, - I32EnumAttrCase<"FILL", 2, "fill">, - I32EnumAttrCase<"USE", 3, "use"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_Tcgen05MMAKindEnum : I32EnumAttr<"Tcgen05MMAKind", "tcgen05 MMA Supported Types", [ - I32EnumAttrCase<"F16", 0, "f16">, - I32EnumAttrCase<"TF32", 1, "tf32">, - I32EnumAttrCase<"F8F6F4", 2, "f8f6f4">, - I32EnumAttrCase<"I8", 3, "i8">, - I32EnumAttrCase<"MXF8F6F4", 4, "mxf8f6f4">, - I32EnumAttrCase<"MXF4", 5, "mxf4">, - I32EnumAttrCase<"MXF4NVF4", 6, "mxf4nvf4">, - I32EnumAttrCase<"TI16", 7, "ti16"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_Tcgen05WaitKindEnum : I32EnumAttr<"Tcgen05WaitKind", "NVVM Tcgen05 wait kind", [ - I32EnumAttrCase<"LOAD", 0, "load">, - I32EnumAttrCase<"STORE", 1, "store"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_TensormapElemtypeEnum : I32EnumAttr<"TensormapElemtype", "NVVM Tensormap Elemtype", [ - I32EnumAttrCase<"U8", 0, "u8">, - I32EnumAttrCase<"U16", 1, "u16">, - I32EnumAttrCase<"U32", 2, "u32">, - I32EnumAttrCase<"S32", 3, "s32">, - I32EnumAttrCase<"U64", 4, "u64">, - I32EnumAttrCase<"S64", 5, "s64">, - I32EnumAttrCase<"F16", 6, "f16">, - I32EnumAttrCase<"F32", 7, "f32">, - I32EnumAttrCase<"F32_FTZ", 8, "f32.ftz">, - I32EnumAttrCase<"F64", 9, "f64">, - I32EnumAttrCase<"BF16", 10, "bf16">, - I32EnumAttrCase<"TF32", 11, "tf32">, - I32EnumAttrCase<"TF32_FTZ", 12, "tf32.ftz">, - I32EnumAttrCase<"B4x16", 13, "b4x16">, - I32EnumAttrCase<"B4x16P64", 14, "b4x16_p64">, - I32EnumAttrCase<"B6x16P32", 15, "b6x16_p32"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_TensormapFieldEnum : I32EnumAttr<"TensormapField", "NVVM Tensormap Field Kind", [ - I32EnumAttrCase<"GLOBAL_ADDRESS", 0, "global_address">, - I32EnumAttrCase<"RANK", 1, "rank">, - I32EnumAttrCase<"BOX_DIM", 2, "box_dim">, - I32EnumAttrCase<"GLOBAL_DIM", 3, "global_dim">, - I32EnumAttrCase<"GLOBAL_STRIDE", 4, "global_stride">, - I32EnumAttrCase<"ELEMENT_STRIDE", 5, "element_stride">, - I32EnumAttrCase<"ELEMTYPE", 6, "elemtype">, - I32EnumAttrCase<"INTERLEAVE_LAYOUT", 7, "interleave_layout">, - I32EnumAttrCase<"SWIZZLE_MODE", 8, "swizzle_mode">, - I32EnumAttrCase<"SWIZZLE_ATOMICITY", 9, "swizzle_atomicity">, - I32EnumAttrCase<"FILL_MODE", 10, "fill_mode"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_TensormapFillModeEnum : I32EnumAttr<"TensormapFillMode", "NVVM Tensormap Fill Mode", [ - I32EnumAttrCase<"ZERO", 0, "zero">, - I32EnumAttrCase<"OOB_NAN", 1, "oob_nan"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_TensormapInterleaveLayoutEnum : I32EnumAttr<"TensormapInterleaveLayout", "NVVM Tensormap Interleave Layout", [ - I32EnumAttrCase<"NO_INTERLEAVE", 0, "no_interleave">, - I32EnumAttrCase<"B16", 1, "b16">, - I32EnumAttrCase<"B32", 2, "b32"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_TensormapSwizzleAtomicityEnum : I32EnumAttr<"TensormapSwizzleAtomicity", "NVVM Tensormap Swizzle Atomicity", [ - I32EnumAttrCase<"B16", 0, "b16">, - I32EnumAttrCase<"B32", 1, "b32">, - I32EnumAttrCase<"B32_FLIP_8B", 2, "b32_flip_b8">, - I32EnumAttrCase<"B64", 3, "b64"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_TensormapSwizzleModeEnum : I32EnumAttr<"TensormapSwizzleMode", "NVVM Tensormap Swizzle Mode", [ - I32EnumAttrCase<"NO_SWIZZLING", 0, "no_swizzling">, - I32EnumAttrCase<"B32", 1, "b32">, - I32EnumAttrCase<"B64", 2, "b64">, - I32EnumAttrCase<"B128", 3, "b128">, - I32EnumAttrCase<"B96", 4, "b96"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_TmemLayoutEnum : I32EnumAttr<"TmemLayout", "Tensor Memory Layout Enumerated Type", [ - I32EnumAttrCase<"tmem_16dp_128bit", 0, "tmem_16dp_128bit">, - I32EnumAttrCase<"tmem_16dp_256bit", 1, "tmem_16dp_256bit">, - I32EnumAttrCase<"tmem_32dp_32bit", 2, "tmem_32dp_32bit">, - I32EnumAttrCase<"tmem_16dp_64bit", 3, "tmem_16dp_64bit">, - I32EnumAttrCase<"tmem_16dp_32bit_t0_t15", 4, "tmem_16dp_32bit_t0_t15">, - I32EnumAttrCase<"tmem_16dp_32bit_t16_t31", 5, "tmem_16dp_32bit_t16_t31"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_ValidatePatternEnum : I32EnumAttr<"ValidatePattern", "NVVM validate data pattern", [ - I32EnumAttrCase<"NONE", 0, "none">, - I32EnumAttrCase<"PER_16BYTES_80000000", 1, "per_16bytes_80000000">, - I32EnumAttrCase<"PER_16BYTES_8000", 2, "per_16bytes_8000">, - I32EnumAttrCase<"PER_16BYTES_80", 3, "per_16bytes_80">, - I32EnumAttrCase<"PER_16BYTES_8", 4, "per_16bytes_8">, - I32EnumAttrCase<"PER_ELEMENT_FF", 5, "per_element_ff">, - I32EnumAttrCase<"PER_ELEMENT_80000000", 6, "per_element_80000000">, - I32EnumAttrCase<"PER_ELEMENT_8000", 7, "per_element_8000">, - I32EnumAttrCase<"PER_ELEMENT_80", 8, "per_element_80">, - I32EnumAttrCase<"PER_ELEMENT_8", 9, "per_element_8">, - I32EnumAttrCase<"PER_16BYTES_80000000_PAIRED", 10, "per_16bytes_80000000_paired"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_VoteSyncKindEnum : I32EnumAttr<"VoteSyncKind", "NVVM vote sync kind", [ - I32EnumAttrCase<"any", 0, "any">, - I32EnumAttrCase<"all", 1, "all">, - I32EnumAttrCase<"ballot", 2, "ballot">, - I32EnumAttrCase<"uni", 3, "uni"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_WGMMAScaleInEnum : I32EnumAttr<"WGMMAScaleIn", "WGMMA overflow options", [ - I32EnumAttrCase<"one", 1, "one">, - I32EnumAttrCase<"neg", 2, "neg"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_WGMMAScaleOutEnum : I32EnumAttr<"WGMMAScaleOut", "WGMMA input predicate", [ - I32EnumAttrCase<"zero", 0, "zero">, - I32EnumAttrCase<"one", 1, "one"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_WGMMATypesEnum : I32EnumAttr<"WGMMATypes", "NVVM WGMMA types", [ - I32EnumAttrCase<"f16", 0, "f16">, - I32EnumAttrCase<"tf32", 1, "tf32">, - I32EnumAttrCase<"u8", 2, "u8">, - I32EnumAttrCase<"s8", 3, "s8">, - I32EnumAttrCase<"b1", 4, "b1">, - I32EnumAttrCase<"bf16", 5, "bf16">, - I32EnumAttrCase<"e4m3", 6, "e4m3">, - I32EnumAttrCase<"e5m2", 7, "e5m2">, - I32EnumAttrCase<"f32", 8, "f32">, - I32EnumAttrCase<"s32", 9, "s32"> - ]> { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let genSpecializedAttr = 0; -} - -def NVVM_AtomicOpKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_BarrierReductionAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_BarrierReduxKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_BlockScaleFormatAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_CTAGroupKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_CVTPackFloatKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_CacheEvictionPriorityAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_ClusterLaunchControlQueryTypeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_CompareOpKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_ConvertFP4TypeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_ConvertFP8TypeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_ConvertScaleKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_DotAccumulateTypeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_EvictKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_FPRoundingModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_GridDepActionAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_IntegerRoundingModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_L2PrefetchSizeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_LdStMatrixEltTypeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_LoadCacheModifierAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_LoadCacheModifierAttrExt : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_LoadShapeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_LoadSrcFormatAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MBarrierLayoutAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MBarrierScopeKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MBarrierSpaceKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MBarrierTxnKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MBarrierWaitKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MMAB1OpAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MMABlockScaleKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MMACtaCountAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MMAFragAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MMAIntOverflowAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MMAKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MMALayoutAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MMATypesAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MatchSyncKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_MemOrderKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MemScopeKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_MulModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_NVVMMemorySpaceAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_NVVM_MBarrierPhaseAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_PermuteModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_PrefetchCacheLevelAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_ProxyKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_ReductionKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_ReductionOpAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_ReductionTypeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SPCompressElemSizeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SPCompressFactorTypeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SPCompressIndexSizeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SPCompressOpKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SPCompressRepFactorAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SPDecompressElemSizeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SPDecompressFactorTypeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SPDecompressIndexSizeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SPDecompressRepFactorAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SaturationModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SaturationModeKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_ScaleVecSizeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_SetMaxRegisterActionAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_SharedSpaceAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_ShflKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_SparsityFormatAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_StateSpaceAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_StoreCacheModifierAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_StoreShapeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_TCBarParamEnumAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_TMALoadModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_TMAReduxKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_TMAStoreModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_Tcgen05CpMulticastAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_Tcgen05CpShapeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_Tcgen05CpSrcFormatAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_Tcgen05FenceKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_Tcgen05LdStShapeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_Tcgen05MMABlockScaleAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_Tcgen05MMACollectorBBufferAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_Tcgen05MMACollectorOpAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_Tcgen05MMAKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_Tcgen05WaitKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_TensormapElemtypeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_TensormapFieldAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_TensormapFillModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_TensormapInterleaveLayoutAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_TensormapSwizzleAtomicityAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_TensormapSwizzleModeAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_TmemLayoutEnumAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_ValidatePatternAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_VoteSyncKindAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; - let assemblyFormat = "$value"; -} - -def NVVM_WGMMAScaleInAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_WGMMAScaleOutAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -def NVVM_WGMMATypesAttr : EnumAttr { - let cppNamespace = "::mlir::cutlass_compiler::nvvm"; -} - -#endif // MLIR_DIALECTS_NVVM_NVVMENUMS_TD diff --git a/mlir/dialects/nvvm/NVVMInference.cpp b/mlir/dialects/nvvm/NVVMInference.cpp deleted file mode 100644 index 78a9e56f..00000000 --- a/mlir/dialects/nvvm/NVVMInference.cpp +++ /dev/null @@ -1,253 +0,0 @@ -//===- NVVMInference.cpp - result types the compiler infers ---------------===// -// -// Generated from the nvidia-cutlass-dsl 4.8.0 compiler (its inference -// probed by varying operand types); do not edit. An operation without a -// recovered rule fails inference, so the result types written stand. - -#include "mlir/dialects/nvvm/NVVMDialect.h" - -using namespace mlir; -using namespace ::mlir::cutlass_compiler::nvvm; - -LogicalResult AddFOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, - DictionaryAttr attrs, PropertyRef props, - RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(0); - if (group.empty()) - return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult -AddPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult -AtomicRMWOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult -BarrierOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult ClusterLaunchControlQueryCancelOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult -CompareAndSetOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(0); - if (group.empty()) - return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult FAbsOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, - DictionaryAttr attrs, PropertyRef props, - RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(0); - if (group.empty()) - return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult -FMAPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult FmaOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult InternalMmaBlockScaleOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(8); - if (group.empty()) - return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult InternalMmaSparseBlockScaleOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(8); - if (group.empty()) - return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult -LdMatrixOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MBarrierArriveDropExpectTxOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MBarrierArriveDropOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MBarrierArriveExpectTxOp::inferReturnTypes( - MLIRContext *, std::optional, ValueRange operands, - DictionaryAttr attrs, PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult -MBarrierArriveOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult -MBarrierTestWaitOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult -MBarrierTryWaitOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult -MatchSyncOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult MulOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult -MulPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult ReduxOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, - DictionaryAttr attrs, PropertyRef props, - RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult RsqrtOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, - DictionaryAttr attrs, PropertyRef props, - RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(0); - if (group.empty()) - return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult ShflOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, - DictionaryAttr attrs, PropertyRef props, - RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult SubFOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, - DictionaryAttr attrs, PropertyRef props, - RegionRange regions, - SmallVectorImpl &out) { - Adaptor adaptor(operands, attrs, props, regions); - auto group = adaptor.getODSOperands(0); - if (group.empty()) - return failure(); - out.push_back(group.front().getType()); - return success(); -} - -LogicalResult -SubPackedF32x2Op::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} - -LogicalResult -VoteSyncOp::inferReturnTypes(MLIRContext *, std::optional, - ValueRange operands, DictionaryAttr attrs, - PropertyRef props, RegionRange regions, - SmallVectorImpl &out) { - return failure(); -} diff --git a/mlir/dialects/nvvm/NVVMOps.td b/mlir/dialects/nvvm/NVVMOps.td deleted file mode 100644 index d77563d2..00000000 --- a/mlir/dialects/nvvm/NVVMOps.td +++ /dev/null @@ -1,6445 +0,0 @@ -// The nvvm dialect's operations, generated from the nvidia-cutlass-dsl 4.8.0 release; do not edit. -// -// Sources: the operation bindings (`_nvvm_ops_gen.py`: names, classes, -// docstrings, operand/result/attribute structure, inference) and the live -// compiler (`_cutlass_ir` loaded from Python): each operand and result -// constraint carries the verifier's wording and accepts every type class the -// verifier let through when probed with one instance of every type; traits -// and interfaces are the registered operation's; memory effects come from -// loop-invariant code motion (hoisted: Pure) and CSE (merged: at most -// reads). Operations whose effects include writes or could not be told -// apart carry none (unknown effects). The compiler's own verifiers (operand -// combinations, shapes, instruction legality) are left to it. -// -// Operations keep the generic assembly form. -// -// Result types the compiler infers without a recovered rule (written types stand): -// nvvm.add.packed.f32x2 -// nvvm.atomicrmw -// nvvm.barrier -// nvvm.clusterlaunchcontrol.query.cancel -// nvvm.fma -// nvvm.fma.packed.f32x2 -// nvvm.ldmatrix -// nvvm.match.sync -// nvvm.mbarrier.arrive -// nvvm.mbarrier.arrive.expect_tx -// nvvm.mbarrier.arrive_drop -// nvvm.mbarrier.arrive_drop.expect_tx -// nvvm.mbarrier.test.wait -// nvvm.mbarrier.try_wait -// nvvm.mul (`mode = wide` doubles the operand width) -// nvvm.mul.packed.f32x2 -// nvvm.redux.sync -// nvvm.shfl.sync (`return_value_and_is_valid` returns `!llvm.struct<(T, i1)>`) -// nvvm.sub.packed.f32x2 -// nvvm.vote.sync -// -// Constraints not recovered: -// nvvm.convert.bf16x2.to.f8x2 `dstTy`: `fnv8E5M3FNU` has no type here and is not accepted - -#ifndef MLIR_DIALECTS_NVVM_NVVMOPS_TD -#define MLIR_DIALECTS_NVVM_NVVMOPS_TD - -include "mlir/dialects/nvvm/NVVMAttrs.td" -include "mlir/dialects/nvvm/NVVMEnums.td" - -include "mlir/Interfaces/InferTypeOpInterface.td" -include "mlir/Interfaces/SideEffectInterfaces.td" - -// Operand and result constraints, in the compiler's words. -def NVVM_AnyType : Type, "any type">; -def NVVM_FixedLengthVectorOf32Bit : Type($_self)">, "fixed-length vector of 32-bit signless integer values of length 4/8">; -def NVVM_FixedLengthVectorOf8Bit : Type($_self)">, "fixed-length vector of 8-bit signless integer values of length 4 or fixed-length vector of 16-bit signless integer values of length 2">; -def NVVM_FloatingPointOrFixedLengthVector : Type($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "floating-point or fixed-length vector of floating-point values of length 2">; -def NVVM_LLVMDialectCompatibleType : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMFunctionType>($_self)">, "LLVM dialect-compatible type">; -def NVVM_LLVMPointerInAddressSpace0 : Type($_self)">, "LLVM pointer in address space 0">; -def NVVM_LLVMPointerInAddressSpace02 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3">; -def NVVM_LLVMPointerInAddressSpace03 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 7">; -def NVVM_LLVMPointerInAddressSpace04 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 7">; -def NVVM_LLVMPointerInAddressSpace05 : Type($_self)">, "LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 1 or LLVM pointer in address space 7">; -def NVVM_LLVMPointerInAddressSpace1 : Type($_self)">, "LLVM pointer in address space 1">; -def NVVM_LLVMPointerInAddressSpace12 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 0 or LLVM pointer in address space 3 or LLVM pointer in address space 7">; -def NVVM_LLVMPointerInAddressSpace13 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 5 or LLVM pointer in address space 0 or LLVM pointer in address space 4">; -def NVVM_LLVMPointerInAddressSpace14 : Type($_self)">, "LLVM pointer in address space 1 or LLVM pointer in address space 3">; -def NVVM_LLVMPointerInAddressSpace3 : Type($_self)">, "LLVM pointer in address space 3">; -def NVVM_LLVMPointerInAddressSpace32 : Type($_self)">, "LLVM pointer in address space 3 or LLVM pointer in address space 7">; -def NVVM_LLVMPointerInAddressSpace6 : Type($_self)">, "LLVM pointer in address space 6">; -def NVVM_LLVMPointerInAddressSpace62 : Type($_self)">, "LLVM pointer in address space 6 or 64-bit signless integer">; -def NVVM_LLVMPointerInAddressSpace7 : Type($_self)">, "LLVM pointer in address space 7">; -def NVVM_LLVMPointerInAddressSpace72 : Type($_self)">, "LLVM pointer in address space 7 or LLVM pointer in address space 3">; -def NVVM_LLVMPointerType : Type($_self)">, "LLVM pointer type">; -def NVVM_LLVMPointerTypeOrLLVMPointer : Type($_self)">, "LLVM pointer type or LLVM pointer in address space 3">; -def NVVM_LLVMStructureType : Type($_self)">, "LLVM structure type">; -def NVVM_LLVMStructureTypeOr64Bit : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self)">, "LLVM structure type or 64-bit float">; -def NVVM_LLVMTypeWithSize : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self)">, "LLVM type with size">; -def NVVM_T128BitSignlessInteger : Type, "128-bit signless integer">; -def NVVM_T16BitFloatOrBfloat16Type : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "16-bit float or bfloat16 type or 32-bit float or 64-bit float or vector of 16-bit float or bfloat16 type or 32-bit float or 64-bit float values of length 2">; -def NVVM_T16BitSignlessInteger : Type, "16-bit signless integer">; -def NVVM_T16BitSignlessInteger2 : Type, "16-bit signless integer">, BuildableType<"$_builder.getIntegerType(16)">; -def NVVM_T16BitSignlessIntegerOr32 : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "16-bit signless integer or 32-bit signless integer or 64-bit signless integer or 128-bit signless integer or 16-bit float or bfloat16 type or 32-bit float or 64-bit float or 128-bit float or vector of 32-bit float values of length 2/4 or vector of 16-bit float or bfloat16 type values of length 2/4/8">; -def NVVM_T16BitSignlessIntegerOr322 : Type, "16-bit signless integer or 32-bit signless integer">; -def NVVM_T16BitSignlessIntegerOr323 : Type, "16-bit signless integer or 32-bit signless integer or 64-bit signless integer">; -def NVVM_T16BitSignlessIntegerOrVector : Type($_self)">, "16-bit signless integer or vector of 8-bit signless integer values of length 2">; -def NVVM_T1BitSignlessInteger : Type, "1-bit signless integer">; -def NVVM_T1BitSignlessInteger2 : Type, "1-bit signless integer">, BuildableType<"$_builder.getIntegerType(1)">; -def NVVM_T1BitSignlessIntegerOr32 : Type, "1-bit signless integer or 32-bit signless integer">; -def NVVM_T1BitSignlessIntegerOrLLVM : Type($_self)">, "1-bit signless integer or LLVM structure type">; -def NVVM_T32BitFloat : Type($_self)">, "32-bit float">; -def NVVM_T32BitFloat2 : Type($_self)">, "32-bit float">, BuildableType<"::mlir::Float32Type::get($_builder.getContext())">; -def NVVM_T32BitFloatOr64Bit : Type($_self) || ::llvm::isa<::mlir::Float64Type>($_self)">, "32-bit float or 64-bit float">; -def NVVM_T32BitSignlessInteger : Type, "32-bit signless integer">; -def NVVM_T32BitSignlessInteger2 : Type, "32-bit signless integer">, BuildableType<"$_builder.getIntegerType(32)">; -def NVVM_T32BitSignlessIntegerOr1 : Type, "32-bit signless integer or 1-bit signless integer">; -def NVVM_T32BitSignlessIntegerOr32 : Type($_self)">, "32-bit signless integer or 32-bit float">; -def NVVM_T32BitSignlessIntegerOr322 : Type($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self)">, "32-bit signless integer or 32-bit float or LLVM struct type">; -def NVVM_T32BitSignlessIntegerOr64 : Type, "32-bit signless integer or 64-bit signless integer">; -def NVVM_T32BitSignlessIntegerOr642 : Type($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self)">, "32-bit signless integer or 64-bit signless integer or 64-bit float or bfloat16 type or 16-bit float or 32-bit float or vector of 32-bit float or 32-bit signless integer values of length 2/4 or vector of 16-bit float or bfloat16 type values of length 2/4/8">; -def NVVM_T32BitSignlessIntegerOr643 : Type($_self)">, "32-bit signless integer or 64-bit signless integer or vector of 32-bit signless integer or 64-bit signless integer values of length 2/4">; -def NVVM_T32BitSignlessIntegerOrLLVM : Type($_self)">, "32-bit signless integer or LLVM struct type">; -def NVVM_T32BitSignlessIntegerOrVector : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4/8/16/32/64">; -def NVVM_T32BitSignlessIntegerOrVector2 : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4/8/16/32/64/128">; -def NVVM_T32BitSignlessIntegerOrVector3 : Type($_self)">, "32-bit signless integer or vector of 32-bit signless integer values of length 2/4">; -def NVVM_T64BitSignlessInteger : Type, "64-bit signless integer">; -def NVVM_T64BitSignlessInteger2 : Type, "64-bit signless integer">, BuildableType<"$_builder.getIntegerType(64)">; -def NVVM_T64BitSignlessIntegerOr32 : Type, "64-bit signless integer or 32-bit signless integer">; -def NVVM_T8BitSignlessInteger : Type, "8-bit signless integer">, BuildableType<"$_builder.getIntegerType(8)">; -def NVVM_T8BitSignlessInteger2 : Type, "8-bit signless integer">; -def NVVM_T8BitSignlessIntegerOr32 : Type, "8-bit signless integer or 32-bit signless integer">; -def NVVM_VariadicOf16BitSignlessInteger : Type, "variadic of 16-bit signless integer">; -def NVVM_VariadicOf32BitSignlessInteger : Type, "variadic of 32-bit signless integer">; -def NVVM_VariadicOfLLVMDialectCompatibleType : Type($_self) || ::llvm::isa<::mlir::Float16Type>($_self) || ::llvm::isa<::mlir::BFloat16Type>($_self) || ::llvm::isa<::mlir::Float32Type>($_self) || ::llvm::isa<::mlir::Float64Type>($_self) || ::llvm::isa<::mlir::VectorType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMPointerType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMStructType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMArrayType>($_self) || ::llvm::isa<::mlir::LLVM::LLVMFunctionType>($_self)">, "variadic of LLVM dialect-compatible type">; -def NVVM_VectorOf16BitFloatValues : Type($_self)">, "vector of 16-bit float values of length 2">; -def NVVM_VectorOf16BitSignlessInteger : Type($_self)">, "vector of 16-bit signless integer values of length 2">; -def NVVM_VectorOf32BitFloatValues : Type($_self)">, "vector of 32-bit float values of length 2">; -def NVVM_VectorOf32BitFloatValues2 : Type($_self)">, "vector of 32-bit float values">; -def NVVM_VectorOf32BitFloatValues3 : Type($_self)">, "vector of 32-bit float values of length 4">; -def NVVM_VectorOf32BitSignlessInteger : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64/128">; -def NVVM_VectorOf32BitSignlessInteger2 : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64/128 or 32-bit signless integer">; -def NVVM_VectorOf32BitSignlessInteger3 : Type($_self)">, "vector of 32-bit signless integer or 32-bit float values of length 2/4/8/16/32/64/128">; -def NVVM_VectorOf32BitSignlessInteger4 : Type($_self)">, "vector of 32-bit signless integer values of length 2/4/8/16/32/64">; -def NVVM_VectorOf8BitSignlessInteger : Type($_self)">, "vector of 8-bit signless integer values of length 4">; -def NVVM_VectorOf8BitSignlessInteger2 : Type($_self)">, "vector of 8-bit signless integer values of length 2">; -def NVVM_VectorOfAnyTypeValues : Type($_self)">, "vector of any type values">; -def NVVM_VectorOfBfloat16TypeValuesOf : Type($_self)">, "vector of bfloat16 type values of length 2">; - -// Attribute constraints the bindings name but do not describe. -def NVVM_ConvertBF16x2ToF4x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f4E2M1FN type"> { - let storageType = "::mlir::TypeAttr"; - let returnType = "::mlir::TypeAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_ConvertBF16x2ToF6x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f6E2M3FN type or f6E3M2FN type"> { - let storageType = "::mlir::TypeAttr"; - let returnType = "::mlir::TypeAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_ConvertBF16x2ToF8x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float8E8M0FNUType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E4M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float8E5M2Type>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f8E8M0FNU type or f8E4M3FN type or f8E5M2 type or fnv8E5M3FNU type"> { - let storageType = "::mlir::TypeAttr"; - let returnType = "::mlir::TypeAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_ConvertF16x2ToF4x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float4E2M1FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f4E2M1FN type"> { - let storageType = "::mlir::TypeAttr"; - let returnType = "::mlir::TypeAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_ConvertF16x2ToF6x2DstTyAttr : Attr($_self) && (::llvm::isa<::mlir::Float6E2M3FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()) || ::llvm::isa<::mlir::Float6E3M2FNType>(::llvm::cast<::mlir::TypeAttr>($_self).getValue()))">, "type attribute of f6E2M3FN type or f6E3M2FN type"> { - let storageType = "::mlir::TypeAttr"; - let returnType = "::mlir::TypeAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_FmaRndAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::FPRoundingMode>{::mlir::cutlass_compiler::nvvm::FPRoundingMode::NONE, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RN, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RM, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RP, ::mlir::cutlass_compiler::nvvm::FPRoundingMode::RZ}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr>($_self).getValue())">, "NVVM FPRoundingMode kind whose value is one of {none, rm, rn, rp, rz}"> { - let storageType = "::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr"; - let returnType = "::mlir::cutlass_compiler::nvvm::FPRoundingModeAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_MultimemLdReduceOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::WEAK, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::ACQUIRE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {weak, relaxed, acquire}"> { - let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; - let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_MultimemLdReduceOpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::ReductionOp>{::mlir::cutlass_compiler::nvvm::ReductionOp::AND, ::mlir::cutlass_compiler::nvvm::ReductionOp::OR, ::mlir::cutlass_compiler::nvvm::ReductionOp::XOR, ::mlir::cutlass_compiler::nvvm::ReductionOp::ADD, ::mlir::cutlass_compiler::nvvm::ReductionOp::MIN, ::mlir::cutlass_compiler::nvvm::ReductionOp::MAX}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::ReductionOpAttr>($_self).getValue())">, "Ops supported by red instruction whose value is none of {inc, dec}"> { - let storageType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; - let returnType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_MultimemRedOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELEASE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {relaxed, release}"> { - let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; - let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_MultimemRedOpAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::ReductionOp>{::mlir::cutlass_compiler::nvvm::ReductionOp::AND, ::mlir::cutlass_compiler::nvvm::ReductionOp::OR, ::mlir::cutlass_compiler::nvvm::ReductionOp::XOR, ::mlir::cutlass_compiler::nvvm::ReductionOp::ADD, ::mlir::cutlass_compiler::nvvm::ReductionOp::INC, ::mlir::cutlass_compiler::nvvm::ReductionOp::DEC, ::mlir::cutlass_compiler::nvvm::ReductionOp::MIN, ::mlir::cutlass_compiler::nvvm::ReductionOp::MAX}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::ReductionOpAttr>($_self).getValue())">, "ReductionOp attribute"> { - let storageType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; - let returnType = "::mlir::cutlass_compiler::nvvm::ReductionOpAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_MultimemStOrderingAttr : Attr($_self) && ::llvm::is_contained(::llvm::ArrayRef<::mlir::cutlass_compiler::nvvm::MemOrderKind>{::mlir::cutlass_compiler::nvvm::MemOrderKind::WEAK, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELAXED, ::mlir::cutlass_compiler::nvvm::MemOrderKind::RELEASE}, ::llvm::cast<::mlir::cutlass_compiler::nvvm::MemOrderKindAttr>($_self).getValue())">, "NVVM Memory Ordering kind whose value is one of {weak, relaxed, release}"> { - let storageType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; - let returnType = "::mlir::cutlass_compiler::nvvm::MemOrderKindAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_LLVMConstantRangeAttr : Attr($_self)">, "A range of two integers, corresponding to LLVM's ConstantRange"> { - let storageType = "::mlir::LLVM::ConstantRangeAttr"; - let returnType = "::mlir::LLVM::ConstantRangeAttr"; - let convertFromStorage = "$_self"; -} - -def NVVM_TensormapFieldValueAttr : Attr($_self)">, "NVVM Tensormap Elemtype or NVVM Tensormap Interleave Layout or NVVM Tensormap Swizzle Mode or NVVM Tensormap Swizzle Atomicity or NVVM Tensormap Fill Mode"> { - let storageType = "::mlir::Attribute"; - let returnType = "::mlir::Attribute"; - let convertFromStorage = "$_self"; -} - -// Operations whose result types are inferred in C++ (NVVMInference.cpp). -// Verification compares them with the inferred ones where the rule is -// recovered; elsewhere the types written stand, as the compiler accepted them. -class NVVM_InferredOp traits = []> - : NVVM_Op]> { - let extraClassDefinition = [{ - ::llvm::LogicalResult $cppClass::refineReturnTypes( - ::mlir::MLIRContext *context, ::std::optional<::mlir::Location> location, - ::mlir::ValueRange operands, ::mlir::DictionaryAttr attributes, - ::mlir::PropertyRef properties, ::mlir::RegionRange regions, - ::llvm::SmallVectorImpl<::mlir::Type> &returnTypes) { - ::llvm::SmallVector<::mlir::Type, 4> inferred; - if (::mlir::failed(inferReturnTypes(context, ::std::nullopt, operands, - attributes, properties, regions, - inferred))) - return ::mlir::success(); - if (!isCompatibleReturnTypes(inferred, returnTypes)) - return ::mlir::emitOptionalError( - location, "'", getOperationName(), "' op inferred type(s) ", - inferred, " are incompatible with return type(s) of operation ", - returnTypes); - return ::mlir::success(); - } - }]; -} - -def NVVM_AddPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"add.packed.bf16x2.f32x2.f32x2"> { - let description = [{ - This Op performs the add operation on a pair of f32x2 vectors. - The inputs and output are of the form v2f32 and v2bf16 respectively. - The inputs are converted to a scalar of packed i64 where the 0th element of the - vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero - is the only rounding mode supported for this operation. - - Examples: - ```mlir - // Round to zero (only rounding mode supported) - %0 = nvvm.add .packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> - ``` - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB - ); - let results = (outs - NVVM_VectorOfBfloat16TypeValuesOf:$res - ); -} -def NVVM_AddPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"add.packed.f16x2.f32x2.f32x2"> { - let description = [{ - This Op performs the add operation on a pair of f32x2 vectors. - The inputs and output are of the form v2f32 and v2f16 respectively. - The inputs are converted to a scalar of packed i64 where the 0th element of the - vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero - is the only rounding mode supported for this operation. - - Examples: - ```mlir - // Round to zero (only rounding mode supported) - %0 = nvvm.add .packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> - ``` - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB - ); - let results = (outs - NVVM_VectorOf16BitFloatValues:$res - ); -} -def NVVM_AddPackedF32x2Op : NVVM_InferredOp<"add.packed.f32x2"> { - let description = [{ - This Op performs addition of floating-point vectors. - The inputs and outputs are of the form . The `rnd` attribute - specifies the rounding mode. The `ftz` attribute, when set, flushes the - subnormal inputs and results to sign-preserving zero. - This Op is supported from SM100 and onwards. - [For more information, see PTX ISA] - (TODO: Add public PTX link when available) - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues2:$srcA, - NVVM_VectorOf32BitFloatValues2:$srcB, - OptionalAttr:$rnd, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_VectorOf32BitFloatValues2:$res - ); -} -def NVVM_AddPacked_F32x2_BF16x2_F32x2Op : NVVM_Op<"add.packed.f32x2.bf16x2.f32x2"> { - let description = [{ - This Op performs the add operation on a pair of floating-point values. - The inputs are of the form v2bf16 and v2f32. The first input `srcA` is - converted to a scalar of packed i32 where the 0th element of the vector forms - the low-16 bits and the 1st element occupies the high-16 bits. Similarly, - the second input `srcB` is converted to a scalar of packed i64 where the 0th - element of the vector forms the low-32 bits and the 1st element occupies the - high-32 bits. The `rnd` attribute specifies the rounding mode. - - The rounding mode can be one of the following: - - rn: round to nearest even - - rz: round toward zero - - rm: round toward negative infinity - - rp: round toward positive infinity - - Examples: - ```mlir - // Round to nearest even (default) - %0 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB : vector<2xbf16>, vector<2xf32> -> vector<2xf32> - - // Round toward negative infinity - %1 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> - - // Round toward zero with flush-to-zero - %2 = nvvm.add .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> - ``` - }]; - let arguments = (ins - NVVM_VectorOfBfloat16TypeValuesOf:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB, - OptionalAttr:$rnd - ); - let results = (outs - NVVM_VectorOf32BitFloatValues:$res - ); -} -def NVVM_AddPacked_F32x2_F16x2_F32x2Op : NVVM_Op<"add.packed.f32x2.f16x2.f32x2"> { - let description = [{ - This Op performs the add operation on a pair of floating-point values. - The inputs are of the form v2f16 and v2f32. The first input `srcA` is - converted to a scalar of packed i32 where the 0th element of the vector forms - the low-16 bits and the 1st element occupies the high-16 bits. Similarly, - the second input `srcB` is converted to a scalar of packed i64 where the 0th - element of the vector forms the low-32 bits and the 1st element occupies the - high-32 bits. The `rnd` attribute specifies the rounding mode. - - The rounding mode can be one of the following: - - rn: round to nearest even - - rz: round toward zero - - rm: round toward negative infinity - - rp: round toward positive infinity - - Examples: - ```mlir - // Round to nearest even (default) - %0 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB : vector<2xf16>, vector<2xf32> -> vector<2xf32> - - // Round toward negative infinity - %1 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> - - // Round toward zero with flush-to-zero - %2 = nvvm.add .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> - ``` - }]; - let arguments = (ins - NVVM_VectorOf16BitFloatValues:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB, - OptionalAttr:$rnd - ); - let results = (outs - NVVM_VectorOf32BitFloatValues:$res - ); -} -def NVVM_AddFOp : NVVM_InferredOp<"addf", [Pure]> { - let description = [{ - The `nvvm.addf` operation performs floating point addition of two floating - point operands of the same type. - - The rounding mode is specified by the `rnd` attribute, saturation mode by - the `sat` attribute, and flush-to-zero by the `ftz` attribute. - - For more information, see PTX ISA: - - [floating point addition](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-add) - - [half-precision floating point addition](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-add) - }]; - let arguments = (ins - NVVM_T16BitFloatOrBfloat16Type:$lhs, - NVVM_T16BitFloatOrBfloat16Type:$rhs, - OptionalAttr:$rnd, - OptionalAttr:$sat, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_T16BitFloatOrBfloat16Type:$res - ); -} -def NVVM_ApplyPriorityAsyncBulkOp : NVVM_Op<"applypriority.async.bulk"> { - let description = [{ - Sets the eviction priority to evict normal for cache lines within the specified - region of L2 cache that are previously present and marked as evict last. - Other L2 cache lines in the specified region are not affected. - - Arguments: - - `ptr`: Global memory pointer. - - `size`: Address range of length. - - `evict`: Eviction priority. - - Example: - ```mlir - nvvm.applypriority.async.bulk %ptr, %size evict_priority = evict_normal - ``` - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace1:$ptr, - NVVM_T32BitSignlessInteger:$size, - OptionalAttr:$evict - ); - let results = (outs); -} -def NVVM_ApplyPriorityAsyncBulkTensorOp : NVVM_Op<"applypriority.async.bulk.tensor", [AttrSizedOperandSegments]> { - let description = [{ - Sets the eviction priority to evict normal for cache lines within the - specified region of L2 cache that are previously present and marked as - evict last. Other L2 cache lines in the specified region are not affected. - - Arguments: - - `tmaDesc`: The TMA descriptor. - - `box/coordinates`: The coordinates of the tensor. - - `im2col`: The im2col arguments. - - `mode`: Tensor load mode. - - `evict`: Eviction priority. - - Example: - ```mlir - nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr - nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0, %d1, %d2] im2col [ %im2col0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr - nvvm.applypriority.async.bulk.tensor %tma_desc, box [ %d0, %d1, %d2] im2col [ %im2w_halo, %im2w_offset] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr - ``` - }]; - let arguments = (ins - NVVM_LLVMPointerType:$tmaDesc, - Variadic:$coordinates, - Variadic:$im2colArgs, - OptionalAttr:$mode, - OptionalAttr:$evict - ); - let results = (outs); -} -def NVVM_ApplyPriorityAsyncBulkTensorOverrideOp : NVVM_Op<"applypriority.async.bulk.tensor.override", [AttrSizedOperandSegments]> { - let description = [{ - Sets the eviction priority to evict normal for cache lines within the - specified region of L2 cache that are previously present and marked as - evict last. Other L2 cache lines in the specified region are not affected. - It supports a way to override the global address of the tensor. - - Arguments: - - `tmaDesc`: The TMA descriptor. - - `overrideAdrr`: The global address to override. - - `box/coordinates`: The coordinates of the tensor. - - `tensorSize`: The size of the tensor. - - `lowerStride`: The lower stride of the tensor. - - `upperStride`: The upper stride of the tensor. - - `im2col`: The im2col arguments. - - `mode`: Tensor load mode. - - `evict`: Eviction priority. - - Example: - ```mlir - nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr - nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0, %d1, %d2] im2col [ %im2col0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr - nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0] tensor_size [ %ts0] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr - nvvm.applypriority.async.bulk.tensor.override %tma_desc, %gbl_addr, box [ %d0, %d1] tensor_size [ %ts0, %ts1] lower_stride [ %lstrd0] upper_stride [ %ustrd] evict_priority = evict_normal {mode = #nvvm.tma_load_mode} : !llvm.ptr - ``` - }]; - let arguments = (ins - NVVM_LLVMPointerType:$tmaDesc, - NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, - Variadic:$coordinates, - Variadic:$tensorSize, - Variadic:$lowerStride, - Optional:$upperStride, - Variadic:$im2colArgs, - OptionalAttr:$mode, - OptionalAttr:$evict - ); - let results = (outs); -} -def NVVM_AtomicRMWOp : NVVM_InferredOp<"atomicrmw"> { - let description = [{ - Atomically loads the original value at location `a` into destination register `res`, - performs a reduction operation with operand `b` and the value in location `a`, and - stores the result of the specified operation at location `a`, overwriting the - original value. Operand `a` specifies a location in the specified state space. - If no state space is given, perform the memory accesses using `Generic Addressing`. - For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-atom) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace12:$ptr, - NVVM_T16BitSignlessIntegerOr32:$a, - Optional:$b, - NVVM_AtomicOpKindAttr:$op, - OptionalAttr:$memOrder, - OptionalAttr:$syncscope - ); - let results = (outs - NVVM_T16BitSignlessIntegerOr32:$res - ); -} -def NVVM_SyncWarpOp : NVVM_Op<"bar.warp.sync"> { - let description = [{ - The `nvvm.bar.warp.sync` operation performs barrier synchronization for threads - within a warp. - - This operation causes the executing thread to wait until all threads corresponding - to the `mask` operand have executed a `bar.warp.sync` with the same mask value - before resuming execution. - - The `mask` operand specifies the threads participating in the barrier, where each - bit position corresponds to the thread's lane ID within the warp. Only threads with - their corresponding bit set in the mask participate in the barrier synchronization. - - **Important constraints**: - - The behavior is undefined if the executing thread is not included in the mask - (i.e., the bit corresponding to the thread's lane ID is not set) - - For compute capability sm_6x or below, all threads in the mask must execute - the same `bar.warp.sync` instruction in convergence - - This operation also guarantees memory ordering among participating threads. - Threads within the warp that wish to communicate via memory can store to memory, - execute `bar.warp.sync`, and then safely read values stored by other threads - in the warp. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-warp-sync) - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$mask - ); - let results = (outs); -} -def NVVM_BarrierOp : NVVM_InferredOp<"barrier", [AttrSizedOperandSegments]> { - let description = [{ - The `nvvm.barrier` operation performs barrier synchronization and communication - within a CTA (Cooperative Thread Array). It causes executing threads to wait for - all non-exited threads participating in the barrier to arrive. - - The operation takes two optional operands: - - - `barrierId`: Specifies a logical barrier resource with value 0 through 15. - Each CTA instance has sixteen barriers numbered 0..15. Defaults to 0 if not specified. - - `numberOfThreads`: Specifies the number of threads participating in the barrier. - When specified, the value must be a multiple of the warp size. If not specified, - all threads in the CTA participate in the barrier. - - `reductionOp`: specifies the reduction operation (`popc`, `and`, `or`). - - `reductionPredicate`: specifies the predicate to be used with the - `reductionOp`. - - The barrier operation guarantees that when the barrier completes, prior memory - accesses requested by participating threads are performed relative to all threads - participating in the barrier. It also ensures that no new memory access is - requested by participating threads before the barrier completes. - - When a barrier completes, the waiting threads are restarted without delay, and - the barrier is reinitialized so that it can be immediately reused. - - This operation generates an aligned barrier, indicating that all threads in the CTA - will execute the same barrier instruction. Behavior is undefined if all threads in the - CTA do not reach this instruction. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) - }]; - let arguments = (ins - Optional:$barrierId, - Optional:$numberOfThreads, - Optional:$reductionPredicate, - OptionalAttr:$reductionOp - ); - let results = (outs - Optional:$res - ); -} -def NVVM_BarrierArriveOp : NVVM_Op<"barrier.arrive"> { - let description = [{ - Thread that executes this op announces their arrival at the barrier with - given id and continue their execution. - - The default barrier id is 0 that is similar to `nvvm.barrier` Op. When - `barrierId` is not present, the default barrier id is used. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) - }]; - let arguments = (ins - Optional:$barrierId, - NVVM_T32BitSignlessInteger:$numberOfThreads - ); - let results = (outs); -} -def NVVM_BarrierCTAArriveOp : NVVM_Op<"barrier.cta.arrive"> { - let description = [{ - The `nvvm.barrier.cta.arrive` is used by the threads within the CTA for synchronization and communication. - The `barrierId` specifies a logical barrier resource and the `threadCount` specifies the number of threads - participating in the barrier. The `barrier.cta.arrive` does not cause the executing thread to wait for - threads of other participating warps. The `aligned` attribute, when set to 1, generates the .aligned - version of the PTX instruction. - [For more information, see PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$barrierId, - NVVM_T32BitSignlessInteger:$threadCount, - OptionalAttr:$aligned - ); - let results = (outs); -} -def NVVM_BarrierCTAReduxOp : NVVM_Op<"barrier.cta.red"> { - let description = [{ - The `nvvm.barrier.cta.red` is used to perform a reduction operation across threads in the CTA. - The `pred` from all the participating threads are combined using the specified reduction operator. - AND, OR, POPC (population-count) are available as reduction operators. The `barrierId` specifies - a logical barrier resource and the `threadCount` specifies the number of threads participating - in the barrier. When the `threadCount` is not specified, all threads in the CTA participate in - the reduction operation. The `aligned` attribute, when set to 1, generates the .aligned version - of the PTX instruction. - [For more information, see PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) - }]; - let arguments = (ins - NVVM_T1BitSignlessInteger:$pred, - NVVM_T32BitSignlessInteger:$barrierId, - Optional:$threadCount, - NVVM_BarrierReduxKindAttr:$kind, - OptionalAttr:$aligned - ); - let results = (outs - NVVM_T32BitSignlessInteger2:$res - ); -} -def NVVM_BarrierCTASyncOp : NVVM_Op<"barrier.cta.sync"> { - let description = [{ - The `nvvm.barrier.cta.sync` is used by the threads within the CTA for synchronization and communication. - The `barrierId` specifies a logical barrier resource and the `threadCount` specifies the number of - threads participating in the barrier. When the `threadCount` is not specified, all threads in the CTA - participate in the barrier. The `aligned` attribute, when set to 1, generates the .aligned version of - the PTX instruction. - [For more information, see PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar-barrier) - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$barrierId, - Optional:$threadCount, - OptionalAttr:$aligned - ); - let results = (outs); -} -def NVVM_Barrier0Op : NVVM_Op<"barrier0"> { - let description = [{ - The `nvvm.barrier0` operation is a convenience operation that performs barrier - synchronization and communication within a CTA (Cooperative Thread Array) using - barrier ID 0. It is functionally equivalent to `nvvm.barrier` or `nvvm.barrier id=0`. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-bar) - }]; - let arguments = (ins); - let results = (outs); -} -def NVVM_Breakpoint : NVVM_Op<"breakpoint"> { - let description = [{ - Breakpoint suspends execution of the program for debugging. - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-brkpt) - }]; - let arguments = (ins); - let results = (outs); -} -def NVVM_ClmadOp : NVVM_Op<"clmad", [Pure]> { - let description = [{ - Performs a carryless multiplication of `a` and `b`, followed by a carryless - addition of `c`. - When `hi` is true, the result is the upper 64 bits of the product, with the - addition of `c`. Otherwise, the result is the lower 64 bits of the product, - with the addition of `c`. - }]; - let arguments = (ins - NVVM_T64BitSignlessInteger:$a, - NVVM_T64BitSignlessInteger:$b, - NVVM_T64BitSignlessInteger:$c, - OptionalAttr:$high - ); - let results = (outs - NVVM_T64BitSignlessInteger2:$res - ); -} -def NVVM_ClusterArriveOp : NVVM_Op<"cluster.arrive"> { - let description = [{ - The `cluster.arrive` can be used by the threads within the cluster for synchronization and - communication. The `cluster.arrive` instruction marks the warps' arrival at the barrier - without causing the executing thread to wait for other participating threads. - - The `aligned` attribute, when provided, generates the .aligned version of the PTX instruction. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) - }]; - let arguments = (ins - UnitAttr:$aligned - ); - let results = (outs); -} -def NVVM_ClusterArriveRelaxedOp : NVVM_Op<"cluster.arrive.relaxed"> { - let description = [{ - The `cluster.arrive` can be used by the threads within the cluster for synchronization and - communication. The `cluster.arrive` instruction marks the warps' arrival at the barrier - without causing the executing thread to wait for other participating threads. - - The `aligned` attribute, when provided, generates the .aligned version of the PTX instruction. - The .relaxed qualifier on `cluster.arrive` specifies that there are no memory - ordering and visibility guarantees provided for the memory accesses performed prior to - `cluster.arrive`. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) - }]; - let arguments = (ins - UnitAttr:$aligned - ); - let results = (outs); -} -def NVVM_ClusterWaitOp : NVVM_Op<"cluster.wait"> { - let description = [{ - The `cluster.wait` causes the executing thread to wait for all non-exited threads - of the cluster to perform `cluster.arrive`. The `aligned` attribute, when provided, - generates the .aligned version of the PTX instruction. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-barrier-cluster) - }]; - let arguments = (ins - UnitAttr:$aligned - ); - let results = (outs); -} -def NVVM_ClusterLaunchControlQueryCancelOp : NVVM_InferredOp<"clusterlaunchcontrol.query.cancel"> { - let description = [{ - `clusterlaunchcontrol.query.cancel` queries the response of a - `clusterlaunchcontrol.try.cancel` operation specified by operand - `try_cancel_response`. - - Operand `query_type` specifies the type of query to perform and can be one - of the following: - - `is_canceled` : Returns true if the try cancel request succeeded, - and false otherwise. - - `get_first_cta_id_{x/y/z}` : Returns the x, y, or z coordinate of the - first CTA in the canceled cluster. Behaviour is defined only if the try - cancel request succeeded. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-query-cancel) - }]; - let arguments = (ins - NVVM_T128BitSignlessInteger:$try_cancel_response, - NVVM_ClusterLaunchControlQueryTypeAttr:$query_type - ); - let results = (outs - NVVM_T1BitSignlessIntegerOr32:$res - ); -} -def NVVM_ClusterLaunchControlTryCancelOp : NVVM_Op<"clusterlaunchcontrol.try.cancel"> { - let description = [{ - `clusterlaunchcontrol.try.cancel` requests atomically canceling the launch - of a cluster that has not started running yet. It asynchronously writes an - opaque response to shared memory indicating whether the operation succeeded - or failed. - - Operand `smemAddress` specifies the naturally aligned address of the - 16-byte wide shared memory location where the request's response is written. - - Operand `mbarrier` specifies the mbarrier object used to track the - completion of the asynchronous operation. - - If `multicast` is specified, the response is asynchronously written to the - corresponding local shared memory location (specifed by `addr`) of each CTA - in the requesting cluster. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-clusterlaunchcontrol-try-cancel) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace3:$smemAddress, - NVVM_LLVMPointerInAddressSpace3:$mbarrier, - UnitAttr:$multicast - ); - let results = (outs); -} -def NVVM_CompareAndSetOp : NVVM_InferredOp<"compare.and.set"> { - let description = [{ - Compare two values and return boolean mask of 1's or 0's. Return value - will be of the same type as the operands (vector<4xi8> or vector<2xi16>). - - Arguments: - - `a`: The first value. - - `b`: The second value. - - `op`: The comparison operator. (eq, ne, ugt, uge, ult, ule, sgt, sge, slt, sle) - - Example: - ```mlir - %res = nvvm.compare.and.set %a, %b : vector<4xi8> - %res = nvvm.compare.and.set %a, %b : vector<2xi16> - ``` - }]; - let arguments = (ins - NVVM_FixedLengthVectorOf8Bit:$a, - NVVM_FixedLengthVectorOf8Bit:$b, - NVVM_CompareOpKindAttr:$op - ); - let results = (outs - NVVM_FixedLengthVectorOf8Bit:$result - ); -} -def NVVM_ConvertPackIntegerOp : NVVM_Op<"convert.and.pack.integer"> { - let description = [{ - This Op converts the given integer inputs to the specified integer type - with saturation and packs the result into the lower bits of the result. - - The result `dst` is represented as an i32 type. - - The optional operand `srcC` must be specified when the converted types add - up to less than 32 bits. The remaining bits of the result are filled with - the lower bits of `srcC`. - - The `convert_type` attribute specifies the integer type to convert to - before packing. The `signed` attribute can be specified to indicate that - the `convert_type` is signed. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cvt-pack) - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$srcA, - NVVM_T32BitSignlessInteger:$srcB, - Optional:$srcC, - UnitAttr:$is_signed, - TypeAttr:$convert_type - ); - let results = (outs - NVVM_T32BitSignlessInteger2:$dst - ); -} -def NVVM_ConvertBF16x2ToF4x2Op : NVVM_Op<"convert.bf16x2.to.f4x2"> { - let description = [{ - This Op converts each of the given BF16 inputs in an bf16x2 vector to the specified fp4 type. - The result `dst` is returned as an i8 type where the converted values are - packed such that the value converted from the first element of `a` is - stored in the lower 4 bits of `dst` and the value converted from the second - element of `a` is stored in the upper 4 bits of `dst`. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction. - The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. - }]; - let arguments = (ins - NVVM_VectorOfBfloat16TypeValuesOf:$src, - Optional:$scale_factor, - UnitAttr:$isPZO, - OptionalAttr:$scale_factor_kind, - OptionalAttr:$rnd, - OptionalAttr:$relu, - NVVM_ConvertBF16x2ToF4x2DstTyAttr:$dstTy - ); - let results = (outs - NVVM_T8BitSignlessInteger:$dst - ); -} -def NVVM_ConvertBF16x2ToF6x2Op : NVVM_Op<"convert.bf16x2.to.f6x2"> { - let description = [{ - This Op converts each of the given BF16 inputs in an bf16x2 vector to the specified fp6 type. The result `dst` is - represented either as an i16 type or as a vector of two i8 types. - If `dst` is returned as an i16 type, the converted values are packed such - that the value converted from the first element of `a` is stored in the - lower 8 bits of `dst` with 2 MSB bits padded with zeros and the value - converted from the second element of `a` is stored in the upper 8 bits of - `dst` with 2 MSB bits padded with zeros. - If `dst` is returned as a vector type, each converted value is stored as an - i8 element in the vector with 2 MSB bits padded with zeros. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction. The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. - }]; - let arguments = (ins - NVVM_VectorOfBfloat16TypeValuesOf:$src, - Optional:$scale_factor, - UnitAttr:$isPZO, - OptionalAttr:$scale_factor_kind, - OptionalAttr:$rnd, - OptionalAttr:$relu, - NVVM_ConvertBF16x2ToF6x2DstTyAttr:$dstTy - ); - let results = (outs - NVVM_T16BitSignlessIntegerOrVector:$dst - ); -} -def NVVM_ConvertBF16x2ToF8x2Op : NVVM_Op<"convert.bf16x2.to.f8x2"> { - let description = [{ - This Op converts the given bf16 inputs in a bf16x2 vector to the specified - f8 type. The result `dst` is represented either as a packed i16 type or as - a vector of two i8 types. - If `dst` is returned as an i16 type, the converted values are packed such - that the value converted from the first element of `a` is stored in the - lower 8 bits of `dst` and the value converted from the second element of - `a` is stored in the upper 8 bits of `dst`. - If `dst` is returned as a vector type, each converted value is stored as an - i8 element in the vector. - The `rnd` and `sat` attributes specify the rounding and saturation modes - respectively. - - The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. - - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_VectorOfBfloat16TypeValuesOf:$src, - Optional:$scale_factor, - UnitAttr:$isPZO, - OptionalAttr:$scale_factor_kind, - OptionalAttr:$rnd, - OptionalAttr:$sat, - OptionalAttr:$relu, - NVVM_ConvertBF16x2ToF8x2DstTyAttr:$dstTy - ); - let results = (outs - NVVM_T16BitSignlessIntegerOrVector:$dst - ); -} -def NVVM_ConvertBF16x2ToS2F6x2Op : NVVM_Op<"convert.bf16x2.to.s2f6x2"> { - let description = [{ - This Op converts each of the given BF16 inputs in a bf16x2 vector to the - S2F6x2 type. The result `dst` can be either a packed i16 type or a vector - of two i8 types. - If `dst` is returned as an i16 type, the converted values are packed such - that the value converted from the first element of `a` is stored in the - lower 8 bits of `dst` and the value converted from the second element of - `a` is stored in the upper 8 bits of `dst`. - If `dst` is returned as a vector type, each converted value is stored as an - i8 element in the vector. - The `relu` attribute, when set, lowers to the '.relu' variant - of the cvt instruction. - The optional scaling-factors for each of the inputs are provided through - the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported - as the type of the scale-factor currently. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_VectorOfBfloat16TypeValuesOf:$src, - Optional:$scaleFactor, - OptionalAttr:$relu - ); - let results = (outs - NVVM_T16BitSignlessIntegerOrVector:$dst - ); -} -def NVVM_ConvertF16x2ToF4x2Op : NVVM_Op<"convert.f16x2.to.f4x2"> { - let description = [{ - This Op converts each of the given F16 inputs in an f16x2 vector to the specified fp4 type. - The result `dst` is returned as an i8 type where the converted values are - packed such that the value converted from the first element of `a` is - stored in the lower 4 bits of `dst` and the value converted from the second - element of `a` is stored in the upper 4 bits of `dst`. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction. - The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. - }]; - let arguments = (ins - NVVM_VectorOf16BitFloatValues:$src, - Optional:$scale_factor, - UnitAttr:$isPZO, - OptionalAttr:$scale_factor_kind, - OptionalAttr:$rnd, - OptionalAttr:$relu, - NVVM_ConvertF16x2ToF4x2DstTyAttr:$dstTy - ); - let results = (outs - NVVM_T8BitSignlessInteger:$dst - ); -} -def NVVM_ConvertF16x2ToF6x2Op : NVVM_Op<"convert.f16x2.to.f6x2"> { - let description = [{ - This Op converts each of the given F16 inputs in an f16x2 vector to the specified fp6 type. The result `dst` is - represented either as an i16 type or as a vector of two i8 types. - If `dst` is returned as an i16 type, the converted values are packed such - that the value converted from the first element of `a` is stored in the - lower 8 bits of `dst` with 2 MSB bits padded with zeros and the value - converted from the second element of `a` is stored in the upper 8 bits of - `dst` with 2 MSB bits padded with zeros. - If `dst` is returned as a vector type, each converted value is stored as an - i8 element in the vector with 2 MSB bits padded with zeros. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction. The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. - }]; - let arguments = (ins - NVVM_VectorOf16BitFloatValues:$src, - Optional:$scale_factor, - UnitAttr:$isPZO, - OptionalAttr:$scale_factor_kind, - OptionalAttr:$rnd, - OptionalAttr:$relu, - NVVM_ConvertF16x2ToF6x2DstTyAttr:$dstTy - ); - let results = (outs - NVVM_T16BitSignlessIntegerOrVector:$dst - ); -} -def NVVM_ConvertF16x2ToF8x2Op : NVVM_Op<"convert.f16x2.to.f8x2"> { - let description = [{ - This Op converts the given f16 inputs in an f16x2 vector to the specified - f8 type. - The result `dst` is represented as an i16 type or as a vector - of two i8 types. - If `dst` is returned as an i16 type, the converted values from `a` - are packed such that the value converted from the first element of `a` - is stored in the upper 8 bits of `dst` and the value converted from the - second element of `a` is stored in the lower 8 bits of `dst`. - If `dst` is returned as a vector type, each converted value is stored as an - i8 element in the vector. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction. - The `rnd` and `sat` attributes specify the rounding and saturation modes - respectively. - The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_VectorOf16BitFloatValues:$a, - Optional:$scale_factor, - UnitAttr:$isPZO, - OptionalAttr:$scale_factor_kind, - OptionalAttr:$rnd, - OptionalAttr:$sat, - OptionalAttr:$relu, - TypeAttr:$dstTy - ); - let results = (outs - NVVM_T16BitSignlessIntegerOrVector:$dst - ); -} -def NVVM_ConvertF32x2ToBF16x2Op : NVVM_Op<"convert.f32x2.to.bf16x2"> { - let description = [{ - Converts two F32 values to packed bf16x2 format with - the specified rounding mode. The `src_hi` and `src_lo` parameters - correspond to operands `a` and `b` in the PTX ISA, respectively. - - The `random_bits` parameter is required for stochastic rounding and - provides the [random bits](https://docs.nvidia.com/cuda/parallel-thread-execution/#cvt-rs-rbits-layout-bf16) to be used for the conversion. - - The `relu` attribute clamps negative results to 0. - - The `sat` attribute determines saturation behavior. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_T32BitFloat:$src_hi, - NVVM_T32BitFloat:$src_lo, - Optional:$random_bits, - OptionalAttr:$rnd, - OptionalAttr:$sat, - OptionalAttr:$relu - ); - let results = (outs - NVVM_VectorOfBfloat16TypeValuesOf:$dst - ); -} -def NVVM_ConvertF32x2ToF16x2Op : NVVM_Op<"convert.f32x2.to.f16x2"> { - let description = [{ - Converts two F32 values to packed f16x2 format with - the specified rounding mode. The `src_hi` and `src_lo` parameters - correspond to operands `a` and `b` in the PTX ISA, respectively. - - The `random_bits` parameter is required for stochastic rounding and - provides the [random bits](https://docs.nvidia.com/cuda/parallel-thread-execution/#cvt-rs-rbits-layout-f16) to be used for the conversion. - - The `relu` attribute clamps negative results to 0. - - The `sat` attribute determines saturation behavior. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_T32BitFloat:$src_hi, - NVVM_T32BitFloat:$src_lo, - Optional:$random_bits, - OptionalAttr:$rnd, - OptionalAttr:$sat, - OptionalAttr:$relu - ); - let results = (outs - NVVM_VectorOf16BitFloatValues:$dst - ); -} -def NVVM_ConvertF32x2ToF4x2Op : NVVM_Op<"convert.f32x2.to.f4x2"> { - let description = [{ - This Op converts each of the given float inputs to the specified fp4 type. - - The result `dst` is returned as an i8 type where the converted values are - packed such that the value converted from `a` is stored in the upper 4 bits - of `dst` and the value converted from `b` is stored in the lower 4 bits of - `dst`. - - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction. - - The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. - - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_T32BitFloat:$a, - NVVM_T32BitFloat:$b, - Optional:$scale_factor, - UnitAttr:$isPZO, - OptionalAttr:$scale_factor_kind, - OptionalAttr:$rnd, - OptionalAttr:$relu, - TypeAttr:$dstType - ); - let results = (outs - NVVM_T8BitSignlessInteger:$dst - ); -} -def NVVM_ConvertF32x2ToF6x2Op : NVVM_Op<"convert.f32x2.to.f6x2"> { - let description = [{ - This Op converts each of the given float inputs to the specified fp6 type. - The result `dst` is represented either as an i16 type or as a vector - of two i8 types. - If `dst` is returned as an i16 type, the converted values are packed such - that the value converted from `a` is stored in the upper 8 bits of `dst` - with 2 MSB bits padded with zeros and the value converted from `b` is - stored in the lower 8 bits of `dst` with 2 MSB bits padded with zeros. - If `dst` is returned as a vector type, each converted value is stored as an - i8 element in the vector. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction. - - The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. - - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_T32BitFloat:$a, - NVVM_T32BitFloat:$b, - Optional:$scale_factor, - UnitAttr:$isPZO, - OptionalAttr:$scale_factor_kind, - OptionalAttr:$rnd, - OptionalAttr:$relu, - TypeAttr:$dstTy - ); - let results = (outs - NVVM_T16BitSignlessIntegerOrVector:$dst - ); -} -def NVVM_ConvertF32x2ToF8x2Op : NVVM_Op<"convert.f32x2.to.f8x2"> { - let description = [{ - This Op converts each of the given float inputs to the specified fp8 type. - The result `dst` is represented as an i16 type or as a vector - of two i8 types. - If `dst` is returned as an i16 type, the converted values are packed such - that the value converted from `a` is stored in the upper 8 bits of `dst` - and the value converted from `b` is stored in the lower 8 bits of `dst`. - If `dst` is returned as a vector type, each converted value is stored as an - i8 element in the vector. - The `rnd` and `sat` attributes specify the rounding and saturation modes respectively. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction. - - The `pzo` (positive zero only), when provided, forces negative zero to be positive zero. - - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_T32BitFloat:$a, - NVVM_T32BitFloat:$b, - Optional:$scale_factor, - UnitAttr:$isPZO, - OptionalAttr:$scale_factor_kind, - OptionalAttr:$rnd, - OptionalAttr:$sat, - OptionalAttr:$relu, - TypeAttr:$dstTy - ); - let results = (outs - NVVM_T16BitSignlessIntegerOrVector:$dst - ); -} -def NVVM_ConvertF32x2ToS2F6x2Op : NVVM_Op<"convert.f32x2.to.s2f6x2"> { - let description = [{ - This Op converts each of the given f32 inputs to the - S2F6x2 type. The result `dst` can be either a packed i16 type or a vector - of two i8 types. - If `dst` is returned as an i16 type, the converted values are packed such - that the value converted from `a` is stored in the upper 8 bits of `dst` - and the value converted from `b` is stored in the lower 8 bits of `dst`. - If `dst` is returned as a vector type, each converted value is stored as an - i8 element in the vector. - The `relu` attribute, when set, lowers to the '.relu' variant - of the cvt instruction. - The optional scaling-factors for each of the inputs are provided through - the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported - as the type of the scale-factor currently. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_T32BitFloat:$a, - NVVM_T32BitFloat:$b, - Optional:$scaleFactor, - OptionalAttr:$relu - ); - let results = (outs - NVVM_T16BitSignlessIntegerOrVector:$dst - ); -} -def NVVM_ConvertF32x4ToF4x4Op : NVVM_Op<"convert.f32x4.to.f4x4"> { - let description = [{ - Converts a vector<4xf32> to packed f4x4 format using - stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is - provided by the `rbits` parameter. The `dstTy` attribute specifies the - target floating-point format. The `relu` attribute clamps negative results to 0. - - Note: These operations always use RS rounding mode and SATFINITE saturation mode. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues3:$src, - NVVM_T32BitSignlessInteger:$rbits, - OptionalAttr:$relu, - TypeAttr:$dstTy - ); - let results = (outs - NVVM_T16BitSignlessInteger2:$dst - ); -} -def NVVM_ConvertF32x4ToF6x4Op : NVVM_Op<"convert.f32x4.to.f6x4"> { - let description = [{ - Converts a vector<4xf32> to packed f6x4 format using - stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is - provided by the `rbits` parameter. The `dstTy` attribute specifies the - target floating-point format. The `relu` attribute clamps negative results to 0. - - Note: These operations always use RS rounding mode and SATFINITE saturation mode. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues3:$src, - NVVM_T32BitSignlessInteger:$rbits, - OptionalAttr:$relu, - TypeAttr:$dstTy - ); - let results = (outs - NVVM_VectorOf8BitSignlessInteger:$dst - ); -} -def NVVM_ConvertF32x4ToF8x4Op : NVVM_Op<"convert.f32x4.to.f8x4"> { - let description = [{ - Converts a vector<4xf32> to packed f8x4 format using - stochastic rounding (.rs) mode with SATFINITE saturation. Randomness is - provided by the `rbits` parameter. The `dstTy` attribute specifies the - target floating-point format. The `relu` attribute clamps negative results to 0. - - Note: These operations always use RS rounding mode and SATFINITE saturation mode. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues3:$src, - NVVM_T32BitSignlessInteger:$rbits, - OptionalAttr:$relu, - TypeAttr:$dstTy - ); - let results = (outs - NVVM_VectorOf8BitSignlessInteger:$dst - ); -} -def NVVM_ConvertF4x2ToF16x2Op : NVVM_Op<"convert.f4x2.to.f16x2"> { - let description = [{ - This Op converts the given f4 inputs in a packed i8 to f16. - - The result `dst` is represented as a vector of f16 elements. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction." - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_T8BitSignlessInteger2:$src, - OptionalAttr:$relu, - TypeAttr:$srcType - ); - let results = (outs - NVVM_VectorOf16BitFloatValues:$dst - ); -} -def NVVM_ConvertF6x2ToF16x2Op : NVVM_Op<"convert.f6x2.to.f16x2"> { - let description = [{ - This Op converts the given f6 inputs in a i8x2 vector to f16. - - The result `dst` is represented as a vector of f16 elements. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction." - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_VectorOf8BitSignlessInteger2:$src, - OptionalAttr:$relu, - TypeAttr:$srcType - ); - let results = (outs - NVVM_VectorOf16BitFloatValues:$dst - ); -} -def NVVM_ConvertF8x2ToBF16x2Op : NVVM_Op<"convert.f8x2.to.bf16x2"> { - let description = [{ - This Op converts the given f8 inputs in a i8x2 vector to bf16. - - The result `dst` is represented as a vector of bf16 elements. - The `sat` attribute specifies the saturation mode. - - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_VectorOf8BitSignlessInteger2:$src, - Optional:$scaleFactor, - OptionalAttr:$sat, - TypeAttr:$srcType - ); - let results = (outs - NVVM_VectorOfBfloat16TypeValuesOf:$dst - ); -} -def NVVM_ConvertF8x2ToF16x2Op : NVVM_Op<"convert.f8x2.to.f16x2"> { - let description = [{ - This Op converts the given f8 inputs in a i8x2 vector to f16. - - The result `dst` is represented as a vector of f16 elements. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction." - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_VectorOf8BitSignlessInteger2:$src, - OptionalAttr:$relu, - TypeAttr:$srcType - ); - let results = (outs - NVVM_VectorOf16BitFloatValues:$dst - ); -} -def NVVM_ConvertFloatToIntegerOp : NVVM_Op<"convert.float.to.integer"> { - let description = [{ - This Op converts the given float input to the specified integer type. - - The `rnd` attribute specifies the rounding mode. - The `sat` atttribute, when set, limits the result to the range of the - integer type. - The `ftz` attribute, when set, converts denormal inputs to zero. - The `signed` attribute, when set, converts the input to a signed integer. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_T32BitFloat:$src, - OptionalAttr:$rnd, - UnitAttr:$sat, - UnitAttr:$ftz, - UnitAttr:$is_signed - ); - let results = (outs - NVVM_T8BitSignlessIntegerOr32:$dst - ); -} -def NVVM_ConvertFloatToTF32Op : NVVM_Op<"convert.float.to.tf32"> { - let description = [{ - This Op converts the given f32 input to tf32. - The result `res` is represented as an i32 type. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction. The `rnd` and `sat` attributes specify the - the rounding and saturation modes respectively. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_T32BitFloat:$src, - OptionalAttr:$rnd, - OptionalAttr:$sat, - OptionalAttr:$relu - ); - let results = (outs - NVVM_T32BitSignlessInteger2:$res - ); -} -def NVVM_ConvertS2F6x2ToBF16x2Op : NVVM_Op<"convert.s2f6x2.to.bf16x2"> { - let description = [{ - This Op converts a pair of s2f6x2 inputs to bf16x2 type. The result `dst` - is represented as a vector of two bf16 elements. - - The `relu` attribute, when set, lowers to the '.relu' variant - of the cvt instruction. - - The optional scaling-factors for each of the inputs are provided through - the operand `scaleFactor` as a packed i16 type. Only `ue8m0` is supported - as the type of the scale-factor currently. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_VectorOf8BitSignlessInteger2:$src, - Optional:$scaleFactor, - OptionalAttr:$sat, - OptionalAttr:$relu - ); - let results = (outs - NVVM_VectorOfBfloat16TypeValuesOf:$dst - ); -} -def NVVM_CosOp : NVVM_Op<"cos", [Pure]> { - let description = [{ - Computes a fast approximation of the cosine of the input value (in - radians). The `ftz` attribute, when set, flushes subnormal inputs - and results to sign-preserving zero. - }]; - let arguments = (ins - NVVM_T32BitFloat:$src, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_T32BitFloat2:$res - ); -} -def NVVM_CpAsyncBulkCommitGroupOp : NVVM_Op<"cp.async.bulk.commit.group"> { - let description = [{ - This Op commits all prior initiated but uncommitted cp.async.bulk - instructions into a cp.async.bulk-group. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-commit-group) - }]; - let arguments = (ins); - let results = (outs); -} -def NVVM_CpAsyncBulkSharedCTAToGlobalOp : NVVM_Op<"cp.async.bulk.global.shared.cta", [AttrSizedOperandSegments]> { - let description = [{ - Initiates an asynchronous copy operation from Shared CTA memory to - global memory. The 32-bit operand `size` specifies the amount of - memory to be copied, in terms of number of bytes. `size` must be a - multiple of 16. The `l2CacheHint` operand is optional, and it is used - to specify cache eviction policy that may be used during the memory - access. The `byteMask` operand is optional. The i-th bit in the 16-bit - wide `byteMask` specifies whether the i-th byte of each 16-byte wide - chunk of source data is copied to the destination. If the bit is set, - the byte is copied. - - Example: - ```mlir - nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size - : !llvm.ptr<1>, !llvm.ptr<3> - - // with l2_cache_hint - nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size l2_cache_hint = %ch - : !llvm.ptr<1>, !llvm.ptr<3> - - // with byte_mask - nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size byte_mask = %mask - : !llvm.ptr<1>, !llvm.ptr<3> - - // with both l2_cache_hint and byte_mask - nvvm.cp.async.bulk.global.shared.cta %dst, %src, %size l2_cache_hint = %ch byte_mask = %mask - : !llvm.ptr<1>, !llvm.ptr<3> - ``` - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace1:$dstMem, - NVVM_LLVMPointerInAddressSpace3:$srcMem, - NVVM_T32BitSignlessInteger:$size, - Optional:$l2CacheHint, - Optional:$byteMask - ); - let results = (outs); -} -def NVVM_CpAsyncBulkPrefetchOp : NVVM_Op<"cp.async.bulk.prefetch"> { - let description = [{ - Initiates an asynchronous prefetch of data from the location - specified by `srcMem` to the L2 cache. - - The `l2CacheHint` operand is optional, and it is used to specify cache - eviction policy that may be used during the memory access. - The `evict` operand is optional, and it is used to specify cache eviction - priority. The `evict` operand is mutually exclusive with the `l2CacheHint` - operand. - - Example: - ```mlir - nvvm.cp.async.bulk.prefetch %src, %size : !llvm.ptr<1> - - // with l2_cache_hint - nvvm.cp.async.bulk.prefetch %src, %size l2_cache_hint = %ch : !llvm.ptr<1> - ``` - Example: - ```mlir - // with evict_priority - nvvm.cp.async.bulk.prefetch %src, %size evict_priority = evict_last : !llvm.ptr<1> - ``` - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-prefetch) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace1:$srcMem, - NVVM_T32BitSignlessInteger:$size, - Optional:$l2CacheHint, - OptionalAttr:$evict - ); - let results = (outs); -} -def NVVM_CpAsyncBulkGlobalToSharedClusterOp : NVVM_Op<"cp.async.bulk.shared.cluster.global", [AttrSizedOperandSegments]> { - let description = [{ - Initiates an asynchronous copy operation from global memory to shared - memory or shared_cluster memory. - - The `multicastMask` operand is optional and can be used only when the - destination is shared::cluster memory. When it is present, this Op copies - data from global memory to shared memory of multiple CTAs in the cluster. - Operand `multicastMask` specifies the destination CTAs in the cluster such - that each bit position in the 16-bit `multicastMask` operand corresponds to - the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. - - The `l2CacheHint` operand is optional, and it is used to specify cache - eviction policy that may be used during the memory access. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace32:$dstMem, - NVVM_LLVMPointerInAddressSpace1:$srcMem, - NVVM_LLVMPointerInAddressSpace3:$mbar, - NVVM_T32BitSignlessInteger:$size, - Optional:$multicastMask, - Optional:$l2CacheHint, - OptionalAttr:$validatePattern - ); - let results = (outs); -} -def NVVM_CpAsyncBulkSharedCTAToSharedClusterOp : NVVM_Op<"cp.async.bulk.shared.cluster.shared.cta"> { - let description = [{ - Initiates an asynchronous copy operation from Shared CTA memory to Shared - cluster memory. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace7:$dstMem, - NVVM_LLVMPointerInAddressSpace3:$srcMem, - NVVM_LLVMPointerInAddressSpace3:$mbar, - NVVM_T32BitSignlessInteger:$size - ); - let results = (outs); -} -def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOp : NVVM_Op<"cp.async.bulk.tensor.global.shared.cta", [AttrSizedOperandSegments]> { - let description = [{ - Initiates an asynchronous copy of the tensor data from shared::cta - memory to global memory. This Op supports all the store modes specified in - `TMAStoreMode`. - - The `l2CacheHint` operand is optional, and it is used to specify cache - eviction policy that may be used during the memory access. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-cp-async-bulk-tensor) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, - NVVM_LLVMPointerInAddressSpace3:$srcMem, - Variadic:$coordinates, - Optional:$l2CacheHint, - Optional:$predicate, - OptionalAttr:$mode - ); - let results = (outs); -} -def NVVM_CpAsyncBulkTensorSharedCTAToGlobalOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.global.shared.cta.override", [AttrSizedOperandSegments]> { - let description = [{ - Initiates an asynchronous bulk tensor operation that transfers data from shared - memory to global memory overriding one or many of the fields: addr, size, and stride. - - The operation: - - Reads data from shared memory location specified by `srcMem` - - Uses a TMA descriptor `tmaDesc` to control the data access pattern - - Overrides the default destination address with `overrideAdrr` - - Specifies box coordinates via `coordinates` array - - Can specify tensor size via `tensorSize` (optional) - - Controls memory access patterns with `lowerStride` and `upperStride` (optional) - - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) - - Supports different store modes (`TILE`, `IM2COL`, `IM2COL_W`, and `TILE_SCATTER4`) via the `mode` attribute - - Examples: - - // Basic usage with 1D box coordinates (TILE mode) - ```mlir - nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0] { - mode = #nvvm.tma_store_mode - } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> - ``` - - // With L2 cache hint - ```mlir - nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] - l2_cache_hint = %ch { - mode = #nvvm.tma_store_mode - } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> - ``` - - // IM2COL mode - ```mlir - nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] { - mode = #nvvm.tma_store_mode - } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> - ``` - - // With tensor size - ```mlir - nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, - box[%d0] tensor_size[%ts0] { - mode = #nvvm.tma_store_mode - } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> - ``` - - // With stride - ```mlir - nvvm.cp.async.bulk.tensor.global.shared.cta.override %tma_desc, %src, %override_addr, - box[%d0, %d1] tensor_size[%ts0, %ts1] lower_stride[%lstrd0] upper_stride[%ustrd] { - mode = #nvvm.tma_store_mode - } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> - ``` - }]; - let arguments = (ins - NVVM_LLVMPointerType:$tmaDesc, - NVVM_LLVMPointerInAddressSpace3:$srcMem, - NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, - Variadic:$coordinates, - Variadic:$tensorSize, - Variadic:$lowerStride, - Optional:$upperStride, - Optional:$l2CacheHint, - OptionalAttr:$mode - ); - let results = (outs); -} -def NVVM_CpAsyncBulkTensorPrefetchOp : NVVM_Op<"cp.async.bulk.tensor.prefetch", [AttrSizedOperandSegments]> { - let description = [{ - Initiates an asynchronous prefetch operation on the tensor data from global - memory to L2 cache. This Op supports all the load modes specified in - `TMALoadMode`. - - The `l2CacheHint` operand is optional, and it is used to specify cache - eviction policy that may be used during the memory access. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-prefetch-tensor) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, - Variadic:$coordinates, - Variadic:$im2colOffsets, - Optional:$l2CacheHint, - OptionalAttr:$mode, - OptionalAttr:$evict - ); - let results = (outs); -} -def NVVM_CpAsyncBulkTensorPrefetchOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.prefetch.override", [AttrSizedOperandSegments]> { - let description = [{ - Initiates an asynchronous bulk tensor operation that prefetches data from - global memory to L2 cache overriding one or many of the fields: addr, size, and stride. - - The operation: - - Reads data from global memory location specified by `overrideAdrr` - - Uses a TMA descriptor `tmaDesc` to control the data access pattern - - Specifies box coordinates via `coordinates` array - - Can specify tensor size via `tensorSize` (optional) - - Controls memory access patterns with `lowerStride` and `upperStride` (optional) - - Can specify im2col offsets via `im2colOffsets` (optional) - - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) - - Supports an optional cache eviction priority with `evict` (optional) - - Supports different load modes (`TILE`/`IM2COL`/`IM2COL_W`/`IM2COL_W_128`/`TILE_GATHER4`) via the `mode` attribute (default = `TILE`) - - Examples: - - // Basic prefetch operations - ```mlir - nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] : !llvm.ptr, !llvm.ptr<1> - nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] : !llvm.ptr, !llvm.ptr<1> - ``` - - // With L2 cache hint - ```mlir - nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> - nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> - ``` - - // With evict_priority - ```mlir - nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] evict_priority = evict_last : !llvm.ptr, !llvm.ptr<1> - nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] evict_priority = evict_last : !llvm.ptr, !llvm.ptr<1> - ``` - - // With tensor size specification - ```mlir - nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0] tensor_size[%ts0] : !llvm.ptr, !llvm.ptr<1> - ``` - - // With tensor size and stride specifications - ```mlir - nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1] - tensor_size[%ts0, %ts1] lower_stride[%ls0] upper_stride[%us0] : !llvm.ptr, !llvm.ptr<1> - ``` - - // With im2col offsets - ```mlir - nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2] - im2col[%off0] : !llvm.ptr, !llvm.ptr<1> - nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2, %d3] - im2col[%off0, %off1] : !llvm.ptr, !llvm.ptr<1> - ``` - - // Dim stride with cache hint - ```mlir - nvvm.cp.async.bulk.tensor.prefetch.override %tma_desc, %override_addr, box[%d0, %d1, %d2, %d3, %d4] - tensor_size[%ts0, %ts1, %ts2, %ts3, %ts4] - lower_stride[%ls0, %ls1, %ls2, %ls3] - upper_stride[%us0] - l2_cache_hint = %ch : !llvm.ptr, !llvm.ptr<1> - ``` - }]; - let arguments = (ins - NVVM_LLVMPointerType:$tmaDesc, - NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, - Variadic:$coordinates, - Variadic:$tensorSize, - Variadic:$lowerStride, - Optional:$upperStride, - Variadic:$im2colOffsets, - Optional:$l2CacheHint, - OptionalAttr:$evict, - OptionalAttr:$mode - ); - let results = (outs); -} -def NVVM_CpAsyncBulkTensorReduceOp : NVVM_Op<"cp.async.bulk.tensor.reduce", [AttrSizedOperandSegments]> { - let description = [{ - Initiates an asynchronous reduction operation of tensor data in - global memory with tensor data in shared memory. - - The `mode` attribute indicates whether the copy mode is tile or im2col. - The `redOp` attribute specifies the reduction operations applied. - The supported reduction operations are: - {add, min, max, inc, dec, and, or, xor} - - The `l2CacheHint` operand is optional, and it is used to specify cache - eviction policy that may be used during the memory access. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-reduce-async-bulk-tensor) - }]; - let arguments = (ins - NVVM_LLVMPointerType:$tmaDescriptor, - NVVM_LLVMPointerInAddressSpace3:$srcMem, - Variadic:$coordinates, - Optional:$l2CacheHint, - NVVM_TMAReduxKindAttr:$redKind, - OptionalAttr:$mode - ); - let results = (outs); -} -def NVVM_CpAsyncBulkTensorReduceOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.reduce.override", [AttrSizedOperandSegments]> { - let description = [{ - Initiates an asynchronous bulk tensor operation that performs a reduction operation - overriding one or many of the fields: addr, size, and stride. - - The operation: - - Reads data from shared memory location specified by `srcMem` - - Uses a TMA descriptor `tmaDesc` to control the data access pattern - - Overrides the default destination address with `overrideAdrr` - - Specifies box coordinates via `coordinates` array - - Can specify tensor size via `tensorSize` (optional) - - Controls memory access patterns with `lowerStride` and `upperStride` (optional) - - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) - - Supports different store modes (`TILE` or `IM2COL`) via the `mode` attribute - - Performs a reduction operation specified by the `redKind` attribute (supports ADD, MIN, MAX, - INC, DEC, AND, OR, XOR) that combines the source data stored at the destination location - - Examples: - - // Override address with ADD reduction (TILE mode) - ```mlir - nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0] { - redKind = #nvvm.tma_redux_kind, - mode = #nvvm.tma_store_mode - } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> - ``` - - // With L2 cache hint - ```mlir - nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0, %d1] - l2_cache_hint = %ch { - redKind = #nvvm.tma_redux_kind, - mode = #nvvm.tma_store_mode - } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> - ``` - - // IM2COL mode with XOR reduction - ```mlir - nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, box[%d0, %d1, %d2] { - redKind = #nvvm.tma_redux_kind, - mode = #nvvm.tma_store_mode - } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> - ``` - - // With tensor size (dim variant) - ```mlir - nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, - box[%d0] tensor_size[%ts0] { - redKind = #nvvm.tma_redux_kind, - mode = #nvvm.tma_store_mode - } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> - ``` - - // With stride information (dim_stride variant) - ```mlir - nvvm.cp.async.bulk.tensor.reduce.override %tma_desc, %src, %override_addr, - box[%d0, %d1] tensor_size[%ts0, %ts1] lower_stride[%lstrd0] upper_stride[%ustrd] { - redKind = #nvvm.tma_redux_kind, - mode = #nvvm.tma_store_mode - } : !llvm.ptr, !llvm.ptr<3>, !llvm.ptr<1> - ``` - }]; - let arguments = (ins - NVVM_LLVMPointerType:$tmaDesc, - NVVM_LLVMPointerInAddressSpace3:$srcMem, - NVVM_LLVMPointerInAddressSpace1:$overrideAdrr, - Variadic:$coordinates, - Variadic:$tensorSize, - Variadic:$lowerStride, - Optional:$upperStride, - Optional:$l2CacheHint, - NVVM_TMAReduxKindAttr:$redKind, - OptionalAttr:$mode - ); - let results = (outs); -} -def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOp : NVVM_Op<"cp.async.bulk.tensor.shared.cluster.global", [AttrSizedOperandSegments]> { - let description = [{ - Initiates an asynchronous copy operation on the tensor data from global - memory to shared::cluster (or) shared::cta memory. This Op supports all - the load modes specified in `TMALoadMode`. - - The `multicastMask` operand is optional. When it is present, the Op copies - data from global memory to shared memory of multiple CTAs in the cluster. - Operand `multicastMask` specifies the destination CTAs in the cluster such - that each bit position in the 16-bit `multicastMask` operand corresponds to - the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. - - The `l2CacheHint` operand is optional, and it is used to specify cache - eviction policy that may be used during the memory access. - - When the `isCTAOnly` attribute is set to true, the destination is - shared::cta only. Hence, `multicastMask` and `CTAGroup` are not applicable - when `isCTAOnly` is true. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-tensor) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace32:$dstMem, - NVVM_LLVMPointerInAddressSpace0:$tmaDescriptor, - Variadic:$coordinates, - NVVM_LLVMPointerInAddressSpace3:$mbar, - Variadic:$im2colOffsets, - Optional:$multicastMask, - Optional:$l2CacheHint, - Optional:$predicate, - OptionalAttr:$validatePattern, - OptionalAttr:$mode, - OptionalAttr:$isCTAOnly, - OptionalAttr:$group - ); - let results = (outs); -} -def NVVM_CpAsyncBulkTensorGlobalToSharedClusterOverrideAddrOp : NVVM_Op<"cp.async.bulk.tensor.shared.cluster.global.override", [AttrSizedOperandSegments]> { - let description = [{ - Initiates an asynchronous bulk tensor operation that transfers data from global - memory to shared::cluster or shared::cta memory overriding one or many of the fields: addr, size, and stride. - - The operation: - - Writes data to shared::cluster or shared::cta memory location specified by `dstMem` - - Uses a TMA descriptor `tmaDesc` to specify the properties of the tensor copy operation - - Uses `mbar` to specify the memory barrier for the operation - - Overrides the default source address with `overrideAddr` - - Specifies box coordinates via `coordinates` array - - Specifies im2col arguments via `im2colArgs` (optional) - - Can specify tensor size via `tensorSize` (optional) - - Controls memory access patterns with `lowerStride` and `upperStride` (optional) - - Supports multicast mask via `multicastMask` (optional) - - Supports an optional L2 cache hint with `l2CacheHint` to support cache eviction policy (optional) - - Supports different load modes (`TILE`/`IM2COL`/`IM2COL_W`/`IM2COL_W_128`/`TILE_GATHER4`) via the `mode` attribute (default = `TILE`) - - Supports different group kinds (`CTA_1` or `CTA_2`) via the `group` attribute (optional) - - Supports different validate load data patterns via the `validatePattern` attribute (optional) - - Examples: - // Basic usage with box coordinates - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] { - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> - ``` - - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] { - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> - ``` - - // With L2 cache hint - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] - l2_cache_hint = %ch { - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> - ``` - - // With multicast mask (i16) - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] - multicast_mask = %mc { - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i16 - ``` - - // With multicast mask (i32) - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] - multicast_mask = %mc32 { - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i32 - ``` - - // With both multicast mask and L2 cache hint - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] - multicast_mask = %mc - l2_cache_hint = %ch { - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i16 - ``` - - // With CTA_2 group - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] - l2_cache_hint = %ch { - group = #nvvm.cta_group, - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> - ``` - - // With tensor size and strides - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0] - tensor_size[%ts0] { - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> - ``` - - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] - tensor_size[%ts0, %ts1] - lower_stride[%lstrd0] - upper_stride[%ustrd] { - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> - ``` - - // Using IM2COL mode with im2col arguments - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] - im2col[%im2col0] { - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> - ``` - - // Using IM2COL_W mode - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] - im2col[%wHalo, %wOffset] { - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> - ``` - - // Using IM2COL_W_128 mode - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1, %d2] - im2col[%wHalo, %wOffset] { - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1> - ``` - - // Comprehensive example with all optional parameters - ```mlir - nvvm.cp.async.bulk.tensor.shared.cluster.global.override %dst, %tma_desc, %mbar, %override_addr, box[%d0, %d1] - tensor_size[%ts0, %ts1] - lower_stride[%lstrd0] - upper_stride[%ustrd] - multicast_mask = %mc32 - l2_cache_hint = %ch { - group = #nvvm.cta_group, - mode = #nvvm.tma_load_mode - } : !llvm.ptr<3>, !llvm.ptr, !llvm.ptr<1>, i32 - ``` - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace72:$dstMem, - NVVM_LLVMPointerType:$tmaDesc, - NVVM_LLVMPointerInAddressSpace3:$mbar, - NVVM_LLVMPointerInAddressSpace1:$overrideAddr, - Variadic:$coordinates, - Variadic:$im2colArgs, - Variadic:$tensorSize, - Variadic:$lowerStride, - Optional:$upperStride, - Optional:$multicastMask, - Optional:$l2CacheHint, - OptionalAttr:$group, - OptionalAttr:$validatePattern, - OptionalAttr:$mode - ); - let results = (outs); -} -def NVVM_CpAsyncBulkWaitGroupOp : NVVM_Op<"cp.async.bulk.wait_group"> { - let description = [{ - Op waits for completion of the most recent bulk async-groups. - - The `$group` operand tells waiting has to be done until for $group or fewer - of the most recent bulk async-groups. If `$group` is 0, the op wait until - all the most recent bulk async-groups have completed. - - The `$read` indicates that the waiting has to be done until all the bulk - async operations in the specified bulk async-group have completed reading - from their source locations. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cp-async-bulk-wait-group) - }]; - let arguments = (ins - I32Attr:$group, - UnitAttr:$read - ); - let results = (outs); -} -def NVVM_CpAsyncCommitGroupOp : NVVM_Op<"cp.async.commit.group"> { - let arguments = (ins); - let results = (outs); -} -def NVVM_CpAsyncMBarrierArriveOp : NVVM_Op<"cp.async.mbarrier.arrive"> { - let description = [{ - The `cp.async.mbarrier.arrive` Op makes the *mbarrier object* track - all prior cp.async operations initiated by the executing thread. - The `addr` operand specifies the address of the *mbarrier object* - in generic or shared::cta address space. When it is generic, the - underlying memory should fall within the shared::cta space; - otherwise the behavior is undefined. The `noinc` attr impacts - how the mbarrier's state is updated. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-cp-async-mbarrier-arrive) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace02:$addr, - OptionalAttr:$noinc - ); - let results = (outs); -} -def NVVM_CpAsyncOp : NVVM_Op<"cp.async.shared.global"> { - let arguments = (ins - NVVM_LLVMPointerInAddressSpace3:$dst, - NVVM_LLVMPointerInAddressSpace1:$src, - Optional:$cpSize, - I32Attr:$size, - NVVM_LoadCacheModifierAttr:$modifier - ); - let results = (outs); -} -def NVVM_CpAsyncWaitGroupOp : NVVM_Op<"cp.async.wait.group"> { - let arguments = (ins - I32Attr:$n - ); - let results = (outs); -} -def NVVM_CvtPackFloatOp : NVVM_Op<"cvt.packfloat"> { - let description = [{ - This Op converts the given floating point input (`srcA`) to another - packed floating point type (`res`). The operand `from` represents the - source type and `to` represents the destination type. - The result is merged with the given `srcC`. Both the input and result - are represented as 'i32' type. - - Below table describes the format of the result: - ``` - |--------------|-------------------------------------------------------| - | Dst Types | Description | - |--------------|-------------------------------------------------------| - | f16x2 | The result contains two fp16 values, in the low | - | bf16x2 | and high bits. | - |--------------|-------------------------------------------------------| - | e4m3x2 | The result contains two fp8 values in the low or | - | e5m2x2 | high 16-bits of `srcC` depending on `ExtractHi`. | - | ue8m0x2 | When `ExtractHi` is 0, the result is packed in | - | | bits `[7:0]` and `[15:8]` of `srcC` and the high | - | | 16-bits of srcC remain unchanged. | - | | When `ExtractHi` is 1, the result is packed in | - | | bits `[23:16]` and `[31:24]` of `srcC` and the low | - | | 16-bits of srcC remain unchanged. | - |--------------|-------------------------------------------------------| - | e2m3x2 | The result contains two 6-bit floating point values | - | e3m2x2 | with the two MSB's padded with zeroes. The 16-bits | - | | of the result are then packed in the same format | - | | described above. | - |--------------|-------------------------------------------------------| - | e2m1x2 | The result contains two 4-bit floating point values | - | | packed in bits `[7:0]` or `[23:16]` of `srcC` | - | | depending on `ExtractHi`. The other bits of `srcC` | - | | remain unchanged. | - |--------------|-------------------------------------------------------| - ``` - - Below table describes the format of the source operand: - ``` - |--------------|-------------------------------------------------------| - | Src Types | Description | - |--------------|-------------------------------------------------------| - | f16x2 | The two fp16 values are taken from the low and high | - | bf16x2 | high bits of operand `srcA`. | - |--------------|-------------------------------------------------------| - | e4m3x2 | The two fp8 values are taken from the low or high | - | e5m2x2 | 16-bits of `srcA` depending on `ExtractHi`. | - | ue8m0x2 | When `ExtractHi` is 0, the inputs are taken from | - | | bits `[7:0]` and `[15:8]` of `srcA`. | - | | When `ExtractHi` is 1, the inputs are sourced from | - | | bits `[23:16]` and `[31:24]` of `srcA`. | - |--------------|-------------------------------------------------------| - | e2m3x2 | Similar to the row above. | - | e3m2x2 | | - |--------------|-------------------------------------------------------| - | e2m1x2 | The two 4-bit floating point input values are taken | - | | from bits `[7:0]` or `[23:16]` of `srcA` depending | - | | on `ExtractHi`. | - |--------------|-------------------------------------------------------| - ``` - The `relu` attribute, when set, lowers to the '.relu' variant of the - cvt instruction. The `rnd` and `sat` attributes specify the rounding - and saturation respectively. The ue8m0x2, e2m3x2, e3m2x2 and e2m1x2 - types, either as source or destination types, are supported only in SM100a. - [For more information, see PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$srcA, - NVVM_T32BitSignlessInteger:$srcC, - NVVM_CVTPackFloatKindAttr:$from, - NVVM_CVTPackFloatKindAttr:$to, - OptionalAttr:$rnd, - OptionalAttr:$sat, - OptionalAttr:$relu, - OptionalAttr:$extractHi - ); - let results = (outs - NVVM_T32BitSignlessInteger2:$res - ); -} -def NVVM_CvtPackFloatF32Op : NVVM_Op<"cvt.packfloat.f32"> { - let description = [{ - This Op converts two given F32 inputs (in `srcA` and `srcB`) to a - packed floating point type requested in `to`, and the - result is merged with the given `srcC`. - `extractHi`= 0: the result is packed into the low 16-bits of `srcC`. - (and the high-16 bits of `srcC` remain as is) - `extractHi`= 1: the result is packed into the high 16-bits of `srcC`. - (and the low-16 bits of `srcC` remain as is) - The result `res` is represented as an i32 type. - The `relu` attribute, when set, lowers to the '.relu' variant of - the cvt instruction. The `rnd` and `sat` attributes specify the - the rounding and saturation modes respectively. The ue8m0x2, - e2m3x2, e3m2x2 and e2m1x2 types are supported only in SM100a. - [For more information, see PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - NVVM_T32BitFloat:$srcA, - NVVM_T32BitFloat:$srcB, - NVVM_T32BitSignlessInteger:$srcC, - NVVM_CVTPackFloatKindAttr:$to, - OptionalAttr:$rnd, - OptionalAttr:$sat, - OptionalAttr:$relu, - OptionalAttr:$extractHi - ); - let results = (outs - NVVM_T32BitSignlessInteger2:$res - ); -} -def NVVM_CvtToF4x8PackedOp : NVVM_Op<"cvt.to.f4x8.packed"> { - let description = [{ - Fused conversion + packing: performs 4 pairwise conversions to f4 and - packs the 4 byte-sized results into a single i32 using inline PTX - `{ .reg .b8; 4x cvt; mov.b32 }`. - - This avoids the NVPTX backend's lack of a B8 register class, which - causes 3x `prmt.b32` instructions instead of a single `mov.b32` when - packing 4x i8 values via `BUILD_VECTOR v4i8`. - - For f32 source: takes 8 f32 values (4 high-low pairs). - For f16 source: takes 4 i32 values (packed f16x2). - For bf16 source: takes 4 i32 values (packed bf16x2). - - The `srcType` attribute specifies the source element type (f32, f16, - or bf16) and is used to select the correct PTX cvt instruction suffix. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-cvt) - }]; - let arguments = (ins - Variadic:$srcs, - OptionalAttr:$rnd, - TypeAttr:$srcType, - TypeAttr:$dstType - ); - let results = (outs - NVVM_T32BitSignlessInteger2:$dst - ); -} -def NVVM_DotAccumulate2WayOp : NVVM_Op<"dot.accumulate.2way"> { - let description = [{ - Performs a two-way 16-bit to 8-bit dot-product which is accumulated in a - 32-bit result. - Operand `a` is a vector of two 16-bit elements and operand `b` a vector - of four 8-bit elements between which the dot product is computed. - - The `a_type` and `b_type` attributes specify the type of the elements in `a` - and `b` respectively. - If `a_type` or `b_type` is `s`, then the elements in the corresponding - vector are sign-extended to 32-bit before the dot product is computed. - If `a_type` or `b_type` is `u`, then the elements in the corresponding - vector are zero-extended to 32-bit instead. - - The `b_hi` boolean attribute specifies which two bytes of `b` are used for - the dot product. If `b_hi` is true, then the dot product is computed - between `a` and elements at indices 2 and 3 of `b`. If `b_hi` is false, - then the dot product is computed between `a` and elements at indices 0 and - 1 of `b`. - - Operand `c` is a 32-bit integer to which the result is accumulated. It is - treated as holding a signed integer if any of `a_type` or `b_type` is - signed. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-dp2a) - }]; - let arguments = (ins - NVVM_VectorOf16BitSignlessInteger:$a, - NVVM_VectorOf8BitSignlessInteger:$b, - NVVM_T32BitSignlessInteger:$c, - NVVM_DotAccumulateTypeAttr:$a_type, - NVVM_DotAccumulateTypeAttr:$b_type, - BoolAttr:$b_hi - ); - let results = (outs - NVVM_T32BitSignlessInteger2:$res - ); -} -def NVVM_DotAccumulate4WayOp : NVVM_Op<"dot.accumulate.4way"> { - let description = [{ - Performs a four-way byte dot-product which is accumulated in a 32-bit - result. - Operand `a` and `b` are vectors of 4 bytes between which the dot product is - computed. - - The `a_type` and `b_type` attributes specify the type of the elements in `a` - and `b` respectively. - If `a_type` or `b_type` is `signed`, then the elements in the corresponding - vector are sign-extended to 32-bit before the dot product is computed. - If `a_type` or `b_type` is `unsigned`, then the elements in the - corresponding vector are zero-extended to 32-bit instead. - - Operand `c` is a 32-bit integer to which the result is accumulated. It is - treated as holding a signed integer if any of `a_type` or `b_type` is `s8`. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-dp4a) - }]; - let arguments = (ins - NVVM_VectorOf8BitSignlessInteger:$a, - NVVM_VectorOf8BitSignlessInteger:$b, - NVVM_T32BitSignlessInteger:$c, - NVVM_DotAccumulateTypeAttr:$a_type, - NVVM_DotAccumulateTypeAttr:$b_type - ); - let results = (outs - NVVM_T32BitSignlessInteger2:$res - ); -} -def NVVM_ElectSyncOp : NVVM_Op<"elect.sync"> { - let description = [{ - The `elect.sync` instruction elects one predicated active leader - thread from among a set of threads specified in the `membermask`. - When the `membermask` is not provided explicitly, a default value - of `0xFFFFFFFF` is used. The predicate result is set to `True` for - the leader thread, and `False` for all other threads. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-elect-sync) - }]; - let arguments = (ins - Optional:$membermask - ); - let results = (outs - NVVM_T1BitSignlessInteger2:$pred - ); -} -def NVVM_Ex2Op : NVVM_Op<"ex2", [Pure]> { - let description = [{ - Computes a fast approximation of 2 raised to the power of the input - value. The `ftz` attribute, when set, flushes subnormal inputs and - results to sign-preserving zero. - }]; - let arguments = (ins - NVVM_T32BitFloat:$src, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_T32BitFloat2:$res - ); -} -def NVVM_Exit : NVVM_Op<"exit"> { - let description = [{ - Ends execution of a thread. - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#control-flow-instructions-exit) - }]; - let arguments = (ins); - let results = (outs); -} -def NVVM_FAbsOp : NVVM_InferredOp<"fabs"> { - let description = [{ - Returns the absolute value of the floating point operand. The Op also has an - optional `ftz` attribute which flushes subnormal inputs and returns to - sign-preserving zero - - [For more information, see PTX ISA] - (http://sw-mobile-docs/CUDA/GPGPU/parallel-thread-execution/index.html#floating-point-instructions-abs) - }]; - let arguments = (ins - NVVM_FloatingPointOrFixedLengthVector:$value, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_FloatingPointOrFixedLengthVector:$res - ); -} -def NVVM_FenceMbarrierInitOp : NVVM_Op<"fence.mbarrier.init"> { - let description = [{ - Fence operation that applies on the prior nvvm.mbarrier.init - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) - }]; - let arguments = (ins); - let results = (outs); -} -def NVVM_FenceProxyOp : NVVM_Op<"fence.proxy"> { - let description = [{ - Fence operation with proxy to establish an ordering between memory accesses - that may happen through different proxies. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) - }]; - let arguments = (ins - NVVM_ProxyKindAttr:$kind, - OptionalAttr:$space - ); - let results = (outs); -} -def NVVM_FenceProxyAcquireOp : NVVM_Op<"fence.proxy.acquire"> { - let description = [{ - `fence.proxy.acquire` is a uni-directional fence used to establish ordering - between a prior memory access performed via the generic proxy and a - subsequent memory access performed via the tensormap proxy - - The address operand `addr` and the operand `size` together specify the - memory range `[addr, addr+size)` on which the ordering guarantees on the - memory accesses across the proxies is to be provided. The only supported - value for the `size` operand is 128 and must be an immediate. Generic Addressing - is used unconditionally, and the address specified by the operand `addr` must - fall within the `.global` state space. Otherwise, the behavior is undefined - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace0:$addr, - NVVM_T32BitSignlessInteger:$size, - NVVM_MemScopeKindAttr:$scope, - OptionalAttr:$fromProxy, - OptionalAttr:$toProxy - ); - let results = (outs); -} -def NVVM_FenceProxyReleaseOp : NVVM_Op<"fence.proxy.release"> { - let description = [{ - `fence.proxy.release` is a uni-directional fence used to establish ordering - between a prior memory access performed via the generic proxy and a - subsequent memory access performed via the tensormap proxy. `fence.proxy.release` - operation can form a release sequence that synchronizes with an acquire - sequence that contains the fence.proxy.acquire proxy fence operation - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) - }]; - let arguments = (ins - NVVM_MemScopeKindAttr:$scope, - OptionalAttr:$fromProxy, - OptionalAttr:$toProxy - ); - let results = (outs); -} -def NVVM_FenceProxySyncRestrictOp : NVVM_Op<"fence.proxy.sync_restrict"> { - let description = [{ - The `nvvm.fence.proxy.sync_restrict` Op used to establish - ordering between a prior memory access performed between proxies. Currently, - the ordering is only supported between async and generic proxies. `sync_restrict` - restricts `acquire` memory semantics to `shared_cluster` and `release` memory - semantics to `shared_cta` with cluster scope. - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) - }]; - let arguments = (ins - NVVM_MemOrderKindAttr:$order, - OptionalAttr:$fromProxy, - OptionalAttr:$toProxy - ); - let results = (outs); -} -def NVVM_FenceScClusterOp : NVVM_Op<"fence.sc.cluster"> { - let arguments = (ins); - let results = (outs); -} -def NVVM_FenceSyncRestrictOp : NVVM_Op<"fence.sync_restrict"> { - let description = [{ - The `nvvm.fence.sync_restrict` Op restricts the class of memory - operations for which the fence instruction provides the memory ordering guarantees. - `sync_restrict` restricts `acquire` memory semantics to `shared_cluster` and - `release` memory semantics to `shared_cta` with cluster scope. - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-membar) - }]; - let arguments = (ins - NVVM_MemOrderKindAttr:$order - ); - let results = (outs); -} -def NVVM_FmaOp : NVVM_InferredOp<"fma"> { - let description = [{ - The `nvvm.fma` operation performs floating point fused multiply-add of - three operands of the same type. - - The rounding mode is specified by the `rnd` attribute, saturation mode by - the `sat` attribute, flush-to-zero by the `ftz` attribute, and ReLU by the - `relu` attribute. - - Out-of-bounds (OOB) behavior is controlled by the `oob` attribute. `oob` - clamps the result to 0 if either of the operands is `OOB NaN` (see [Tensors](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensors)). - - For more information, see PTX ISA: - - [floating point fused multiply-add](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-fma) - - [half-precision floating point fused multiply-add](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-fma) - }]; - let arguments = (ins - NVVM_T16BitFloatOrBfloat16Type:$a, - NVVM_T16BitFloatOrBfloat16Type:$b, - NVVM_T16BitFloatOrBfloat16Type:$c, - NVVM_FmaRndAttr:$rnd, - OptionalAttr:$sat, - OptionalAttr:$ftz, - OptionalAttr:$relu, - OptionalAttr:$oob - ); - let results = (outs - NVVM_T16BitFloatOrBfloat16Type:$res - ); -} -def NVVM_FMAPackedF32x2Op : NVVM_InferredOp<"fma.packed.f32x2"> { - let description = [{ - This Op performs FMA of floating-point vectors. - The inputs and outputs are of the form . The `rnd` attribute - specifies the rounding mode. The `ftz` attribute, when set, flushes the - subnormal inputs and results to sign-preserving zero. - This Op is supported from SM100 and onwards. - [For more information, see PTX ISA] - (TODO: Add public PTX link when available) - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues2:$srcA, - NVVM_VectorOf32BitFloatValues2:$srcB, - NVVM_VectorOf32BitFloatValues2:$srcC, - OptionalAttr:$rnd, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_VectorOf32BitFloatValues2:$res - ); -} -def NVVM_FMAPacked_F32x2_BF16x2_F32x2_F32x2Op : NVVM_Op<"fma.packed.f32x2.bf16x2.f32x2.f32x2"> { - let description = [{ - This Op performs FMA of a pair of floating-point values. - The first operand is a vector<2xbf16>, and the second and third operands - are vector<2xf32>. The result is a vector<2xf32>. The first input is converted to - a scalar of packed i32 where the 0th element of the vector forms the low-16 bits - and the 1st element occupies the high-16 bits. The second and third inputs are - converted to a scalar of packed i64 where the 0th element of the vector forms the - low-32 bits and the 1st element occupies the high-32 bits. - - The rounding mode can be one of the following: - - rn: round to nearest even - - rz: round toward zero - - rm: round toward negative infinity - - rp: round toward positive infinity - - Examples: - ```mlir - // Round to nearest even (default) - %0 = nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2 %srcA, %srcB, %srcC : vector<2xbf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> - - // Round toward negative infinity - %1 = nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2 %srcA, %srcB, %srcC {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> - ``` - }]; - let arguments = (ins - NVVM_VectorOfBfloat16TypeValuesOf:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB, - NVVM_VectorOf32BitFloatValues:$srcC, - OptionalAttr:$rnd - ); - let results = (outs - NVVM_VectorOf32BitFloatValues:$res - ); -} -def NVVM_FMAPacked_F32x2_F16x2_F32x2_F32x2Op : NVVM_Op<"fma.packed.f32x2.f16x2.f32x2.f32x2"> { - let description = [{ - This Op performs FMA of a pair of floating-point values. - The first operand is a vector<2xf16>, and the second and third operands - are vector<2xf32>. The result is a vector<2xf32>. The first input is converted to - a scalar of packed i32 where the 0th element of the vector forms the low-16 bits - and the 1st element occupies the high-16 bits. The second and third inputs are - converted to a scalar of packed i64 where the 0th element of the vector forms the - low-32 bits and the 1st element occupies the high-32 bits. - - The rounding mode can be one of the following: - - rn: round to nearest even - - rz: round toward zero - - rm: round toward negative infinity - - rp: round toward positive infinity - - Examples: - ```mlir - // Round to nearest even (default) - %0 = nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2 %srcA, %srcB, %srcC : vector<2xf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> - - // Round toward negative infinity - %1 = nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2 %srcA, %srcB, %srcC {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32>, vector<2xf32> -> vector<2xf32> - ``` - }]; - let arguments = (ins - NVVM_VectorOf16BitFloatValues:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB, - NVVM_VectorOf32BitFloatValues:$srcC, - OptionalAttr:$rnd - ); - let results = (outs - NVVM_VectorOf32BitFloatValues:$res - ); -} -def NVVM_MaxOp : NVVM_Op<"fmax"> { - let description = [{ - `nvvm.fmax` Op calculates maximum of given values. if `nan` attributes is - used, then the result is a canonical nan if any of the input is nan. - if `abs` attribute is used, the magnitude of destination operation d is - the maximum of absolute values of input arguments and for two operand - version, `xorsign` is enabled by default which sets the sign bit of the - result by XOR'ing the sign bits of the inputs. `ftz` modifiers flushes - the subnormal inputs and the results to sign preserving zero - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-max) - }]; - let arguments = (ins - NVVM_T32BitFloat:$a, - NVVM_T32BitFloat:$b, - Optional:$c, - UnitAttr:$ftz, - UnitAttr:$nan, - UnitAttr:$abs - ); - let results = (outs - NVVM_T32BitFloat2:$res - ); -} -def NVVM_MinOp : NVVM_Op<"fmin"> { - let description = [{ - `nvvm.fmin` Op calculates minimum of given values. if `nan` attributes is - used, then the result is a canonical nan if any of the input is nan. - if `abs` attribute is used, the magnitude of destination operation d is - the minimum of absolute values of input arguments and for two operand - version, `xorsign` is enabled by default which sets the sign bit of the - result by XOR'ing the sign bits of the inputs. `ftz` modifiers flushes - the subnormal inputs and the results to sign preserving zero. - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-min) - }]; - let arguments = (ins - NVVM_T32BitFloat:$a, - NVVM_T32BitFloat:$b, - Optional:$c, - UnitAttr:$ftz, - UnitAttr:$nan, - UnitAttr:$abs - ); - let results = (outs - NVVM_T32BitFloat2:$res - ); -} -def NVVM_GriddepcontrolOp : NVVM_Op<"griddepcontrol"> { - let description = [{ - If the $kind attribute is set to `wait`, it causes the - executing thread to wait until all prerequisite grids in flight - have completed and all the memory operations from the prerequisite grids - are performed and made visible to the current grid. - - When the $kind is launch_dependents, it signals that specific dependents - the runtime system designated to react to this instruction can be scheduled - as soon as all other CTAs in the grid issue the same instruction or have - completed. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-griddepcontrol) - }]; - let arguments = (ins - NVVM_GridDepActionAttr:$kind - ); - let results = (outs); -} -def NVVM_InlinePtxOp : NVVM_Op<"inline_ptx", [AttrSizedOperandSegments]> { - let description = [{ - This op allows using PTX directly within the NVVM - dialect, while greatly simplifying llvm.inline_asm generation. It - automatically handles register size selection and sets the correct - read/write access for each operand. The operation leverages the - `BasicPtxBuilderInterface` to abstract away low-level details of - PTX assembly formatting. - - The `predicate` attribute is used to specify a predicate for the - PTX instruction. - - Example 1: Read-only Parameters - ```mlir - nvvm.inline_ptx "mbarrier.init.b64 [$0], $1;" (%barrier_gen, %count) : !llvm.ptr, i32 - - // Lowers to: - llvm.inline_asm has_side_effects asm_dialect = att - "mbarrier.init.b64 [$0], $1;", "l,r" %arg0, %arg2 : (!llvm.ptr, i32) -> () - ``` - - Example 2: Read-only and Write-only Parameters - ```mlir - %0 = nvvm.inline_ptx "ex2.approx.ftz.f32 $0, $1;" (%input) : f32 -> f32 - - // Lowers to: - %0 = llvm.inline_asm has_side_effects asm_dialect = att - "ex2.approx.ftz.f32 $0, $1;", "=f,f" %arg0 : (f32) -> f32 - ``` - - Example 3: Predicate Usage - ```mlir - nvvm.inline_ptx "mbarrier.init.b64 [$0], $1;" (%barrier_gen, %count), - predicate = %pred : !llvm.ptr, i32, i1 - - // Lowers to: - llvm.inline_asm has_side_effects asm_dialect = att - "@$2 mbarrier.init.b64 [$0], $1;", "l,r,b" %arg0, %arg2, %arg3 - : (!llvm.ptr, i32, i1) -> () - ``` - }]; - let arguments = (ins - Variadic:$readOnlyArgs, - Variadic:$readWriteArgs, - Optional:$predicate, - StrAttr:$ptxCode - ); - let results = (outs - Optional:$writeOnlyArgs - ); -} -def NVVM_LdMatrixOp : NVVM_InferredOp<"ldmatrix"> { - let arguments = (ins - NVVM_LLVMPointerType:$ptr, - I32Attr:$num, - NVVM_MMALayoutAttr:$layout, - OptionalAttr:$shape, - OptionalAttr:$srcFormat - ); - let results = (outs - NVVM_AnyType:$res - ); -} -def NVVM_ExtLoadOp : NVVM_Op<"load.ext"> { - let description = [{ - The `load.ext` operation is used to read from memory in generic, - global or shared address space. This Op explicitly supports - various cache properties and memory orderings associated with - the load operation. - - The `l2CacheHint` operand is optional, and it is used to specify cache - eviction policy that may be used during the memory access. - [For more information, see PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-ld) - }]; - let arguments = (ins - NVVM_LLVMPointerType:$addr, - Optional:$l2CacheHint, - OptionalAttr:$order, - OptionalAttr:$scope, - OptionalAttr:$prefetch, - OptionalAttr:$evict, - OptionalAttr:$cacheModifier, - OptionalAttr:$sharedSpace, - OptionalAttr:$unified - ); - let results = (outs - NVVM_LLVMTypeWithSize:$res - ); -} -def NVVM_Log2Op : NVVM_Op<"log2", [Pure]> { - let description = [{ - Computes a fast approximation of the base-2 logarithm of the input - value. The `ftz` attribute, when set, flushes subnormal inputs and - results to sign-preserving zero. - }]; - let arguments = (ins - NVVM_T32BitFloat:$src, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_T32BitFloat2:$res - ); -} -def NVVM_MapaOp : NVVM_Op<"mapa"> { - let arguments = (ins - NVVM_LLVMPointerInAddressSpace02:$a, - NVVM_T32BitSignlessInteger:$b - ); - let results = (outs - NVVM_LLVMPointerInAddressSpace03:$res - ); -} -def NVVM_MatchSyncOp : NVVM_InferredOp<"match.sync"> { - let description = [{ - The `match.sync` op performs broadcast and compare of operand `val` across - all non-exited threads in `thread_mask` and returns a mask depending on the - kind and an optional predicate. - - The matching operation kinds are: - - `any`: Returns a mask corresponding to the non-exited threads in the - `thread_mask` that have the same value of operand `val`. - - `all`: Returns a mask and a predicate. If all non-exited threads in the - `thread_mask` have the same value of operand `val`, the predicate is set to - true and the mask corresponds to the non-exited threads in the - `thread_mask`. Otherwise, the predicate is set to false and the mask is 0. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-match-sync) - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$thread_mask, - NVVM_T32BitSignlessIntegerOr64:$val, - NVVM_MatchSyncKindAttr:$kind - ); - let results = (outs - NVVM_T32BitSignlessIntegerOrLLVM:$res - ); -} -def NVVM_MBarrierArriveOp : NVVM_InferredOp<"mbarrier.arrive", [AttrSizedOperandSegments]> { - let description = [{ - The `nvvm.mbarrier.arrive` operation performs an arrive-on operation on the - *mbarrier object* at the specified address. Uses the default `.release.cta` semantics. - This release pattern establishes memory ordering for operations occurring in program - order before this arrive instruction by making operations from the current thread - visible to subsequent operations in other threads within the CTA. When other threads - perform corresponding acquire operations (like 'mbarrier.test.wait'), they synchronize - with this release pattern. - - This operation causes the executing thread to signal its arrival at the barrier. - - - `res`: When the `space` is not shared_cluster, this operation returns an - opaque 64-bit value capturing the phase of the *mbarrier object* prior to - the arrive-on operation. The contents of this return value are - implementation-specific. An *mbarrier object* located in the shared_cluster - space cannot return a value. - - The operation takes the following operands: - - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` - must be a pointer to generic or shared_cta or shared_cluster memory. When it - is generic, the underlying address must be within the shared_cta memory space; - otherwise the behavior is undefined. - - `count`: This specifies the amount by which the pending arrival count is - decremented. If the `count` argument is not specified, the pending arrival - count is decremented by 1. - - `scope`: This specifies the set of threads that directly observe the memory - synchronizing effect of the `mbarrier.arrive` operation. - - `space`: This indicates the memory space where the mbarrier object resides. - - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics - and does not provide any ordering or visibility guarantees. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace04:$addr, - Optional:$count, - Optional:$multicastMask, - OptionalAttr:$scope, - OptionalAttr:$relaxed - ); - let results = (outs - Optional:$res - ); -} -def NVVM_MBarrierArriveExpectTxOp : NVVM_InferredOp<"mbarrier.arrive.expect_tx", [AttrSizedOperandSegments]> { - let description = [{ - The `nvvm.mbarrier.arrive.expect_tx` operation performs an expect-tx operation - followed by an arrive-on operation on the *mbarrier object*. Uses the default - `.release.cta` semantics. This release pattern establishes memory ordering for - operations occurring in program order before this arrive instruction by making - operations from the current thread visible to subsequent operations in other - threads within the CTA. When other threads perform corresponding acquire operations - (like 'mbarrier.test.wait'), they synchronize with this release pattern. - - This operation first performs an expect-tx operation with the specified transaction - count, then performs an arrive-on operation with an implicit count of 1. The - expect-tx operation increases the expect-count of the *mbarrier object* by the - specified value (i.e. `txcount`), setting the current phase to expect and track - the completion of additional asynchronous transactions. - - The operation takes the following operands: - - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic - addressing, but the address must still be in the shared memory space. - - `txcount`: An unsigned integer specifying the expected transaction count - for the expect-tx operation. This represents the number of asynchronous transactions - expected to complete before the barrier phase completes. - - `scope`: This specifies the set of threads that directly observe the memory - synchronizing effect of the `mbarrier.test.wait` operation. - - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics - and does not provide any ordering or visibility guarantees. - - `predicate`: Optional predicate for conditional execution used only when lowering to - inline-ptx. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace04:$addr, - NVVM_T32BitSignlessInteger:$txcount, - Optional:$multicastMask, - Optional:$predicate, - OptionalAttr:$scope, - OptionalAttr:$relaxed - ); - let results = (outs - Optional:$res - ); -} -def NVVM_MBarrierArriveNocompleteOp : NVVM_Op<"mbarrier.arrive.nocomplete"> { - let description = [{ - The `nvvm.mbarrier.arrive.nocomplete` operation performs an arrive-on operation - on the *mbarrier object* with the guarantee that it will not cause the barrier to - complete its current phase. Uses the default `.release.cta` semantics. This release - pattern establishes memory ordering for operations occurring in program order before - this arrive instruction by making operations from the current thread visible to - subsequent operations in other threads within the CTA. When other threads perform - corresponding acquire operations (like 'mbarrier.test.wait'), they synchronize with - this release pattern. - - This operation causes the executing thread to signal its arrival at the barrier - with a specified count, but ensures that the barrier phase will not complete as - a result of this operation. The operation returns an opaque value that - captures the phase of the *mbarrier object* prior to the arrive-on operation. - - The operation takes the following operands: - - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` - must be a pointer to generic or shared::cta memory. When it is generic, the - underlying address must be within the shared::cta memory space; otherwise - the behavior is undefined. - - `count`: Integer specifying the count argument to the arrive-on operation. - Must be in the valid range as specified in the *mbarrier object* contents. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace02:$addr, - NVVM_T32BitSignlessInteger:$count - ); - let results = (outs - NVVM_T64BitSignlessInteger2:$res - ); -} -def NVVM_MBarrierArriveDropOp : NVVM_InferredOp<"mbarrier.arrive_drop", [AttrSizedOperandSegments]> { - let description = [{ - The `nvvm.mbarrier.arrive_drop` operation decrements the expected arrival - count of the *mbarrier object* by `count` and then performs an arrive-on - operation. When `count` is not specified, it defaults to 1. The decrement - of the expected arrival count applies to all the subsequent phases of the - *mbarrier object*. The remaining semantics are identical to those of the - `nvvm.mbarrier.arrive` operation. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace04:$addr, - Optional:$count, - Optional:$multicastMask, - OptionalAttr:$scope, - OptionalAttr:$relaxed - ); - let results = (outs - Optional:$res - ); -} -def NVVM_MBarrierArriveDropExpectTxOp : NVVM_InferredOp<"mbarrier.arrive_drop.expect_tx"> { - let description = [{ - The `nvvm.mbarrier.arrive_drop.expect_tx` operation is similar to the - `nvvm.mbarrier.arrive.expect_tx` operation except that it performs an - `arrive_drop` operation instead of only an `arrive` operation. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace04:$addr, - NVVM_T32BitSignlessInteger:$txcount, - Optional:$multicastMask, - OptionalAttr:$scope, - OptionalAttr:$relaxed - ); - let results = (outs - Optional:$res - ); -} -def NVVM_MBarrierArriveDropNocompleteOp : NVVM_Op<"mbarrier.arrive_drop.nocomplete"> { - let description = [{ - The `nvvm.mbarrier.arrive_drop.nocomplete` operation decrements the expected - arrival count of the *mbarrier object* by the amount `count` and then performs - an arrive-on operation on the *mbarrier object* with the guarantee that it - will not cause the barrier to complete its current phase. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-arrive-drop) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace02:$addr, - NVVM_T32BitSignlessInteger:$count - ); - let results = (outs - NVVM_T64BitSignlessInteger2:$res - ); -} -def NVVM_MBarrierCheckLayoutOp : NVVM_Op<"mbarrier.check.layout"> { - let description = [{ - The `nvvm.mbarrier.check.layout` operation returns `true` if the *mbarrier object* at `addr` - uses the given `layout`; otherwise it returns `false`. - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace3:$addr, - NVVM_MBarrierLayoutAttr:$layout - ); - let results = (outs - NVVM_T1BitSignlessInteger2:$res - ); -} -def NVVM_MBarrierCompleteTxOp : NVVM_Op<"mbarrier.complete_tx", [AttrSizedOperandSegments]> { - let description = [{ - The `nvvm.mbarrier.complete_tx` operation decrements the transaction - count of the *mbarrier object* at `addr` by `txcount`. It also signals - the completion of asynchronous transactions that were tracked by the - current phase. The `scope` specifies the set of threads that can directly - observe the memory synchronizing effect of the `mbarrier.complete_tx` - operation. `CTA` and `CLUSTER` are the only allowed values for `scope`. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-complete-tx) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace32:$addr, - NVVM_T32BitSignlessInteger:$txcount, - Optional:$report, - Optional:$multicastMask, - OptionalAttr:$scope - ); - let results = (outs); -} -def NVVM_MBarrierExpectTxOp : NVVM_Op<"mbarrier.expect_tx"> { - let description = [{ - The `nvvm.mbarrier.expect_tx` operation increases the transaction count - of the mbarrier located at `addr` by `txcount` amount. The `scope` - specifies the set of threads that can directly observe the memory - synchronizing effect of the `mbarrier.expect_tx` operation. `CTA` - and `CLUSTER` are the only allowed values for `scope`. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-expect-tx) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace32:$addr, - NVVM_T32BitSignlessInteger:$txcount, - Optional:$multicastMask, - OptionalAttr:$scope - ); - let results = (outs); -} -def NVVM_MBarrierInitOp : NVVM_Op<"mbarrier.init"> { - let description = [{ - The `nvvm.mbarrier.init` operation initializes an *mbarrier object* at the specified - memory location. - - This operation initializes the *mbarrier object* with the following state: - - Current phase: 0 - - Expected arrival count: `count` - - Pending arrival count: `count` - - Transaction count (tx-count): 0 - - The operation takes the following operands: - - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` - must be a pointer to generic or shared::cta memory. When it is generic, the - underlying address must be within the shared::cta memory space; otherwise - the behavior is undefined. - - `count`: Integer specifying the number of threads that will participate in barrier - synchronization. Must be in the range [1, 2²⁰ - 1]. - - `predicate`: Optional predicate for conditional execution. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-init) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace02:$addr, - NVVM_T32BitSignlessInteger:$count, - Optional:$predicate, - OptionalAttr:$layout - ); - let results = (outs); -} -def NVVM_MBarrierInvalOp : NVVM_Op<"mbarrier.inval"> { - let description = [{ - The `nvvm.mbarrier.inval` operation invalidates an *mbarrier object* at the - specified memory location. - - This operation marks the *mbarrier object* as invalid, making it safe to repurpose - the memory location for other uses or to reinitialize it as a new *mbarrier object*. - It is undefined behavior if the *mbarrier object* is already invalid. - - The operation takes the following operand: - - `addr`: A pointer to the memory location of the *mbarrier object*. The `addr` - must be a pointer to generic or shared::cta memory. When it is generic, the - underlying address must be within the shared::cta memory space; otherwise - the behavior is undefined. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-inval) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace02:$addr - ); - let results = (outs); -} -def NVVM_MBarrierTestWaitOp : NVVM_InferredOp<"mbarrier.test.wait"> { - let description = [{ - The `nvvm.mbarrier.test.wait` operation performs a non-blocking test for the - completion of a specific phase of an *mbarrier object*. It uses the default - `.acquire.cta` semantics. This acquire pattern establishes memory ordering for - operations occurring in program order after this wait instruction by making - operations from other threads in the CTA visible to subsequent operations in the current - thread. When this wait completes, it synchronizes with the corresponding release - pattern from the `mbarrier.arrive` operation, establishing memory ordering within - the CTA. - - This operation tests whether the mbarrier phase specified by the state operand - has completed. It is a non-blocking instruction that immediately returns the - completion status without suspending the executing thread. - - The operation takes the following operands: - - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic - addressing, but the address must still be in the shared memory space. - - `stateOrPhase`: This argument represents a `state` when it is a 64-bit value - and represents a `phase` when it is a 32-bit value. The `state` is an opaque - value returned by a previous `mbarrier.arrive` operation on the same - *mbarrier object* during the current or immediately preceding phase. - The `phase` is an integer specifying the phase parity (0 or 1). - Even phases have parity 0, odd phases have parity 1. - - `scope`: This specifies the set of threads that directly observe the memory - synchronizing effect of the `mbarrier.test.wait` operation. - - `relaxed`: When set to true, the `arrive` operation has relaxed memory semantics - and does not provide any ordering or visibility guarantees. - - The operation returns a boolean value indicating whether the specified phase - has completed: - - `true`: The immediately preceding phase has completed - - `false`: The phase is still incomplete (current phase) - - The primary-phase variants the result is an LLVM struct whose first - field (`res`) is the `i1` completion flag. An optional second field - (`isReportSeen`, `i1`) indicates whether a report was observed, and an - optional third field (`report`, `i32`) carries the report payload. - - **Memory ordering guarantees**: When this wait returns true, the following - ordering guarantees hold: - - 1. All memory accesses (except async operations) requested prior to - `mbarrier.arrive` having release semantics by participating CTA threads - are visible to the executing thread. - 2. All `cp.async` operations requested prior to `cp.async.mbarrier.arrive` - by participating CTA threads are visible to the executing thread. - 3. All `cp.async.bulk` operations using the same *mbarrier object* requested - prior to `mbarrier.arrive` having release semantics by participating CTA - threads are visible to the executing thread. - 4. Memory accesses requested after this wait are not visible to memory - accesses performed prior to `mbarrier.arrive` by other participating - threads. - 5. No ordering guarantee exists for memory accesses by the same thread - between `mbarrier.arrive` and this wait. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace02:$addr, - NVVM_T64BitSignlessIntegerOr32:$stateOrPhase, - OptionalAttr:$scope, - OptionalAttr:$relaxed, - OptionalAttr:$mbarrierPhase - ); - let results = (outs - NVVM_T1BitSignlessIntegerOrLLVM:$res - ); -} -def NVVM_MBarrierTryWaitOp : NVVM_InferredOp<"mbarrier.try_wait"> { - let description = [{ - The `nvvm.mbarrier.try_wait` operation checks whether the specified - *mbarrier object* at `addr` has completed the given phase. Note that - unlike the `nvvm.mbarrier.test.wait` operation, the try_wait operation - is a potentially-blocking one. If the phase is not yet complete, the - calling thread may be suspended. A suspended thread resumes execution - once the phase completes or when a system-defined timeout occurs. - Optionally, the `ticks` operand can be used to provide a custom timeout - (in nanoseconds), overriding the system-defined one. The semantics of - this operation and its operands are otherwise similar to those of the - `nvvm.mbarrier.test.wait` Op. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) - - The primary-phase variants the result is an LLVM struct whose first - field (`res`) is the `i1` completion flag. An optional second field - (`isReportSeen`, `i1`) indicates whether a report was observed, and an - optional third field (`report`, `i32`) carries the report payload. - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace02:$addr, - NVVM_T64BitSignlessIntegerOr32:$stateOrPhase, - Optional:$ticks, - OptionalAttr:$scope, - OptionalAttr:$relaxed, - OptionalAttr:$mbarrierPhase - ); - let results = (outs - NVVM_T1BitSignlessIntegerOrLLVM:$res - ); -} -def NVVM_MBarrierTryWaitParityOp : NVVM_Op<"mbarrier.try_wait.parity"> { - let description = [{ - The `nvvm.mbarrier.try_wait.parity` operation performs a potentially-blocking - test for the completion of a specific phase of an *mbarrier object* using phase - parity. It uses the default `.acquire.cta` semantics. This acquire pattern - establishes memory ordering for operations occurring in program order after this - wait instruction by making operations from other threads in the CTA visible to subsequent - operations in the current thread. When this wait completes, it synchronizes with - the corresponding release pattern from the `mbarrier.arrive` operation, establishing - memory ordering within the CTA. - - This operation waits for the completion of the mbarrier phase indicated by the - phase parity. While it uses the underlying PTX `mbarrier.try_wait.parity` - instruction, this MLIR operation generates a loop that enforces the test to - complete before continuing execution, ensuring the barrier phase is actually - completed rather than potentially timing out. - - The operation takes the following operands: - - `addr`: A pointer to the memory location of the *mbarrier object*. Uses generic - addressing, but the address must still be in the shared memory space. - - `phase`: An integer specifying the phase parity (0 or 1). Even phases - have parity 0, odd phases have parity 1. - - `ticks`: An unsigned integer specifying the suspend time hint in - nanoseconds. This may be used instead of the system-dependent time limit. - - **Memory ordering guarantees**: When this wait returns true, the following - ordering guarantees hold: - - 1. All memory accesses (except async operations) requested prior to - `mbarrier.arrive` having release semantics by participating CTA threads - are visible to the executing thread. - 2. All `cp.async` operations requested prior to `cp.async.mbarrier.arrive` - by participating CTA threads are visible to the executing thread. - 3. All `cp.async.bulk` operations using the same *mbarrier object* requested - prior to `mbarrier.arrive` having release semantics by participating CTA - threads are visible to the executing thread. - 4. Memory accesses requested after this wait are not visible to memory - accesses performed prior to `mbarrier.arrive` by other participating - threads. - 5. No ordering guarantee exists for memory accesses by the same thread - between `mbarrier.arrive` and this wait. - - **Implementation behavior**: - This operation generates a PTX loop that repeatedly calls the underlying - `mbarrier.try_wait.parity` instruction until the barrier phase completes. - Unlike the raw PTX instruction which may return without completion after a - timeout, this MLIR operation guarantees completion by continuing to loop until - the specified phase is reached. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-try-wait) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace02:$addr, - NVVM_T32BitSignlessInteger:$phase, - NVVM_T32BitSignlessInteger:$ticks, - OptionalAttr:$useIntrinsic - ); - let results = (outs); -} -def NVVM_MBarrierTryWaitParityTimeLimitOp : NVVM_Op<"mbarrier.try_wait.parity.timelimit"> { - let description = [{ - The `mbarrier.try_wait.parity.timelimit` is a potentially blocking - instruction which tests for the completion of the phase. If - the phase is not complete, the executing thread may be suspended. - Suspended thread resumes execution when the specified phase completes - or a system-dependent time limit elapses. The 32-bit `timeLimit` operand - specifies the time limit, in nanoseconds, that may be used instead of - the system-dependent limit. The `addr` operand specifies the address - of the mbarrier in generic or shared memory space. The `phase` operand - is the integer parity of either the current phase or the immediately - preceding phase of the mbarrier object. The `order` denotes the - ordering semantics for the memory consistency model. This Op only - supports 'acquire' or 'relaxed' ordering. - [For more information refer PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) - }]; - let arguments = (ins - NVVM_LLVMPointerTypeOrLLVMPointer:$addr, - NVVM_T32BitSignlessInteger:$phase, - NVVM_T32BitSignlessInteger:$timeLimit, - OptionalAttr:$scope, - OptionalAttr:$order - ); - let results = (outs - NVVM_T1BitSignlessInteger2:$res - ); -} -def NVVM_MBarrierTryWaitTimeLimitOp : NVVM_Op<"mbarrier.try_wait.timelimit"> { - let description = [{ - The `mbarrier.try_wait.timelimit` is a potentially blocking - instruction which tests for the completion of the phase. If - the phase is not complete, the executing thread may be suspended. - Suspended thread resumes execution when the specified phase completes - or a system-dependent time limit elapses. The 32-bit `timeLimit` operand - specifies the time limit, in nanoseconds, that may be used instead of - the system-dependent limit. The `addr` operand specifies the address - of the mbarrier in generic or shared memory space. The `state` operand - is the captured state of the mbarrier object from prior operations. - The `order` denotes the ordering semantics for the memory consistency - model. This Op only supports 'acquire' or 'relaxed' ordering. - [For more information refer PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) - }]; - let arguments = (ins - NVVM_LLVMPointerTypeOrLLVMPointer:$addr, - NVVM_T64BitSignlessInteger:$state, - NVVM_T32BitSignlessInteger:$timeLimit, - OptionalAttr:$scope, - OptionalAttr:$order - ); - let results = (outs - NVVM_T1BitSignlessInteger2:$res - ); -} -def NVVM_MBarrierTransactionOp : NVVM_Op<"mbarrier.txn"> { - let description = [{ - The `mbarrier.txn` Op exposes the transactions that can be done on an MBarrier object. - - Operands: - - `addr`: Address of the mbarrier in generic or shared memory space - - `count`: Transaction count for the given kind - - - `multicast`: (Optional) Multicast mask for cluster space operations - - - Attributes: - - `kind`: One of the transaction kinds from MBarrierTxnKind - - `space`: Memory space (CTA or CLUSTER) where MBarrier object is located - - `scope`: Scope of synchronization (CTA or CLUSTER) - - `order`: Memory ordering semantics (release or relaxed only) - - This Op supports MBarrier objects in both shared::CTA and shared::CLUSTER spaces. - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-mbarrier) - }]; - let arguments = (ins - NVVM_LLVMPointerTypeOrLLVMPointer:$addr, - NVVM_T32BitSignlessInteger:$count, - Optional:$multicast, - NVVM_MBarrierTxnKindAttr:$kind, - OptionalAttr:$space, - OptionalAttr:$scope, - OptionalAttr:$order - ); - let results = (outs); -} -def NVVM_MBarrierTransactionCTASpaceOp : NVVM_Op<"mbarrier.txn.cta"> { - let description = [{ - The `mbarrier.txn.cta` Op exposes the transactions that can be done - using an MBarrier object. The `addr` operand specifies the address of - the mbarrier in generic or shared memory space. The `kind` is one of - the various transaction kinds supported in `MBarrierTxnKind`. The - `count` argument is the transaction count for a given `kind`. The - `order` denotes the ordering semantics for the memory consistency - model. This Op only supports 'release' or 'relaxed' ordering. This - Op supports only MBarrier objects located in shared::CTA space (but - not in shared::CLUSTER space) and hence the 'CTASpace' suffix in the - naming. This Op returns an opaque 64-bit value capturing the phase of - the mbarrier object prior to the transaction. - [For more information on 'order/space/scope/kind-of-transactions' refer PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier) - }]; - let arguments = (ins - NVVM_LLVMPointerTypeOrLLVMPointer:$addr, - NVVM_T32BitSignlessInteger:$count, - NVVM_MBarrierTxnKindAttr:$kind, - OptionalAttr:$scope, - OptionalAttr:$order - ); - let results = (outs - NVVM_T64BitSignlessInteger2:$res - ); -} -def NVVM_MBarrierWaitOp : NVVM_Op<"mbarrier.wait"> { - let description = [{ - The `mbarrier.wait` Op exposes the try/test wait operations that - can be performed on MBarrier object. The `addr` operand specifies - the address of the mbarrier in generic or shared memory space. - The `state` operand is the captured state of the mbarrier object - from prior operations. The `kind` is one of TEST_WAIT or TRY_WAIT. - The `order` denotes the ordering semantics for the memory consistency - model. This Op only supports 'acquire' or 'relaxed' ordering. - [For more information refer PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) - }]; - let arguments = (ins - NVVM_LLVMPointerTypeOrLLVMPointer:$addr, - NVVM_T64BitSignlessInteger:$state, - NVVM_MBarrierWaitKindAttr:$kind, - OptionalAttr:$scope, - OptionalAttr:$order - ); - let results = (outs - NVVM_T1BitSignlessInteger2:$res - ); -} -def NVVM_MBarrierWaitParityOp : NVVM_Op<"mbarrier.wait.parity"> { - let description = [{ - The `mbarrier.wait` Op exposes the try/test wait operations that - can be performed on MBarrier object. The `addr` operand specifies - the address of the mbarrier in generic or shared memory space. - The `phase` operand is the integer parity of either the current - phase or the immediately preceding phase of the mbarrier object. - The `kind` is one of TEST_WAIT or TRY_WAIT. - The `order` denotes the ordering semantics for the memory consistency - model. This Op only supports 'acquire' or 'relaxed' ordering. - [For more information refer PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-mbarrier-test-wait-mbarrier-try-wait) - }]; - let arguments = (ins - NVVM_LLVMPointerTypeOrLLVMPointer:$addr, - NVVM_T32BitSignlessInteger:$phase, - NVVM_MBarrierWaitKindAttr:$kind, - OptionalAttr:$scope, - OptionalAttr:$order - ); - let results = (outs - NVVM_T1BitSignlessInteger2:$res - ); -} -def NVVM_MembarOp : NVVM_Op<"memory.barrier"> { - let description = [{ - `membar` operation guarantees that prior memory accesses requested by this - thread are performed at the specified `scope`, before later memory - operations requested by this thread following the membar instruction. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-membar) - }]; - let arguments = (ins - NVVM_MemScopeKindAttr:$scope - ); - let results = (outs); -} -def NVVM_MmaBlockScaleOp : NVVM_Op<"mma.block_scale", [AttrSizedOperandSegments]> { - let description = [{ - The `nvvm.mma.block_scale` operation collectively performs the operation - `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. - - A, B, C and D are dense matrices and SF_A and SF_B are scaling factors. - Dimensions of SF_A and SF_B are based on scale vector sizes (x1, x2, x4), - and the data type must be either ue8m0 or ue4m3. - - All the threads in the warp must execute the same `mma.block_scale` operation. - - This operation follows the same design pattern as `nvvm.mma.sync`, with additional - scaling operands for both A and B matrices. - - Example: - ```mlir - %d = nvvm.mma.block_scale A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] - scaleA[%scaleAData, %byteIdA, %threadIdA] - scaleB[%scaleBData, %byteIdB, %threadIdB] - {shape = #nvvm.shape, - multiplicandAPtxType = #nvvm.mma_type, - multiplicandBPtxType = #nvvm.mma_type, - scaleVecSize = #nvvm.scale_vec_size, - blockScaleFormat = #nvvm.block_scale_format, - kind = #nvvm.block_scale_kind} - : (vector<4xf16>, vector<2xf16>, vector<2xf32>) -> !llvm.struct<(f32, f32)> - ``` - }]; - let arguments = (ins - Variadic:$operandA, - Variadic:$operandB, - Variadic:$operandC, - NVVM_T32BitSignlessInteger:$scaleAData, - NVVM_T16BitSignlessInteger:$byteIdA, - NVVM_T16BitSignlessInteger:$threadIdA, - NVVM_T32BitSignlessInteger:$scaleBData, - NVVM_T16BitSignlessInteger:$byteIdB, - NVVM_T16BitSignlessInteger:$threadIdB, - NVVM_MMAShapeAttr:$shape, - OptionalAttr:$multiplicandAPtxType, - OptionalAttr:$multiplicandBPtxType, - NVVM_ScaleVecSizeAttr:$scaleVecSize, - NVVM_BlockScaleFormatAttr:$blockScaleFormat, - NVVM_MMABlockScaleKindAttr:$kind - ); - let results = (outs - NVVM_LLVMStructureType:$res - ); -} -def NVVM_InternalMmaBlockScaleOp : NVVM_InferredOp<"mma.block_scale.internal"> { - let description = [{ - The `nvvm.mma.blockscale.internal` operation collectively performs the operation - `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. - - A, B, C and D are dense matrices and SF_A and SF_B are scaling factor. - Dimensions of SF_A and SF_B are based on scale vector sizes (X1, X2, X4), - the data type of must be either ue8m0 or ue4m3 - - Example: - ```mlir - - %0 = nvvm.mma.block_scale.internal %operandA, %scaleAData, %byteIdA, %threadIdA, - %operandB, %scaleBData, %byteIdB, %threadIdB, - %operandC - { - layoutA = #nvvm.mma_layout, - layoutB = #nvvm.mma_layout, - shape = #nvvm.shape, - aType = #nvvm.mma_type, - bType = #nvvm.mma_type, - cType = #nvvm.mma_type, - scaleVecSize = #nvvm.scale_vec_size, - blockScaleFormat = #nvvm.block_scale_format - kind = #nvvm.block_scale_kind - } - ``` - }]; - let arguments = (ins - NVVM_VectorOfAnyTypeValues:$operandA, - NVVM_T32BitSignlessInteger:$scaleAData, - NVVM_T16BitSignlessInteger:$byteIdA, - NVVM_T16BitSignlessInteger:$threadIdA, - NVVM_VectorOfAnyTypeValues:$operandB, - NVVM_T32BitSignlessInteger:$scaleBData, - NVVM_T16BitSignlessInteger:$byteIdB, - NVVM_T16BitSignlessInteger:$threadIdB, - NVVM_VectorOfAnyTypeValues:$operandC, - NVVM_MMAShapeAttr:$shape, - NVVM_MMALayoutAttr:$layoutA, - NVVM_MMALayoutAttr:$layoutB, - NVVM_MMATypesAttr:$aType, - NVVM_MMATypesAttr:$bType, - NVVM_MMATypesAttr:$cType, - NVVM_ScaleVecSizeAttr:$scaleVecSize, - NVVM_BlockScaleFormatAttr:$blockScaleFormat, - OptionalAttr:$kind - ); - let results = (outs - NVVM_VectorOfAnyTypeValues:$res - ); -} -def NVVM_MmaSpBlockScaleOp : NVVM_Op<"mma.sp.block_scale", [AttrSizedOperandSegments]> { - let description = [{ - The `nvvm.mma.sp.block_scale` operation collectively performs the operation - `D = matmul(A_sparse * SF_A, B * SF_B) + C` using all threads in a warp. - - A is a sparse matrix, and B, C and D are dense matrices. - SF_A and SF_B are scaling factors. - Dimensions of SF_A and SF_B are based on scale vector sizes (x1, x2, x4), - and the data type must be either ue8m0 or ue4m3. - - This operation is similar to `nvvm.mma.block_scale` but with structured sparsity - in the A operand. The sparsity follows the 2:4 structured sparse pattern - where 2 out of every 4 elements are non-zero. - - All the threads in the warp must execute the same `mma.sp.block_scale` operation. - - The `sparseMetadata` operand provides the sparsity indices that indicate - which elements in the A operand are non-zero. The `sparsitySelector` - controls how the indices are distributed among threads in the warp and - should typically be 0 or 1. - - This operation follows the same design pattern as `nvvm.mma.sp.sync`, with additional - scaling operands for both A and B matrices. Note that sparse block scale operations - always use ordered metadata (sm_90+). - - Example: - ```mlir - %d = nvvm.mma.sp.block_scale A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] - sparseMetadata[%meta] selector[%sel] - scaleA[%scaleAData, %byteIdA, %threadIdA] - scaleB[%scaleBData, %byteIdB, %threadIdB] - {shape = #nvvm.shape, - multiplicandAPtxType = #nvvm.mma_type, - multiplicandBPtxType = #nvvm.mma_type, - scaleVecSize = #nvvm.scale_vec_size, - blockScaleFormat = #nvvm.block_scale_format, - kind = #nvvm.block_scale_kind} - : (vector<2xf16>, vector<2xf16>, vector<2xf32>) -> !llvm.struct<(f32, f32)> - ``` - }]; - let arguments = (ins - Variadic:$operandA, - Variadic:$operandB, - Variadic:$operandC, - NVVM_T32BitSignlessInteger:$sparseMetadata, - NVVM_T32BitSignlessInteger:$sparsitySelector, - NVVM_T32BitSignlessInteger:$scaleAData, - NVVM_T16BitSignlessInteger:$byteIdA, - NVVM_T16BitSignlessInteger:$threadIdA, - NVVM_T32BitSignlessInteger:$scaleBData, - NVVM_T16BitSignlessInteger:$byteIdB, - NVVM_T16BitSignlessInteger:$threadIdB, - NVVM_MMAShapeAttr:$shape, - OptionalAttr:$multiplicandAPtxType, - OptionalAttr:$multiplicandBPtxType, - NVVM_ScaleVecSizeAttr:$scaleVecSize, - NVVM_BlockScaleFormatAttr:$blockScaleFormat, - NVVM_MMABlockScaleKindAttr:$kind, - UnitAttr:$orderedMetadata - ); - let results = (outs - NVVM_LLVMStructureType:$res - ); -} -def NVVM_InternalMmaSparseBlockScaleOp : NVVM_InferredOp<"mma.sp.block_scale.internal"> { - let description = [{ - The `nvvm.mma.sparse.blockscale` operation collectively performs the - operation `D = matmul(A * SF_A, B * SF_B) + C` using all threads in a warp. - - Where A is structued sparse with atleast 50% zeros and B, C and D are - dense and SF_A and SF_B are scaling factor. Dimensions of SF_A and SF_B - are based on scale vector sizes (X1, X2, X4), the data type of - must be either ue8m0 or ue4m3 - - Example: - ```mlir - - %selector = llvm.mlir.constant(0) - %0 = nvvm.mma.sparse.block_scale - %operandA, %scaleAData, %byteIdA, %threadIdA, - %operandB, %scaleBData, %byteIdB, %threadIdB, - %operandC, %spMetadata, %selector - { - layoutA = #nvvm.mma_layout, - layoutB = #nvvm.mma_layout, - shape = #nvvm.shape, - aType = #nvvm.mma_type, - bType = #nvvm.mma_type, - cType = #nvvm.mma_type, - scaleVecSize = #nvvm.scale_vec_size, - blockScaleFormat = #nvvm.block_scale_format, - sparityFormat = #nvvm.sparsity_format - } - ``` - }]; - let arguments = (ins - NVVM_VectorOfAnyTypeValues:$operandA, - NVVM_T32BitSignlessInteger:$scaleAData, - NVVM_T16BitSignlessInteger:$byteIdA, - NVVM_T16BitSignlessInteger:$threadIdA, - NVVM_VectorOfAnyTypeValues:$operandB, - NVVM_T32BitSignlessInteger:$scaleBData, - NVVM_T16BitSignlessInteger:$byteIdB, - NVVM_T16BitSignlessInteger:$threadIdB, - NVVM_VectorOfAnyTypeValues:$operandC, - NVVM_T32BitSignlessInteger:$sp_metadata, - NVVM_T32BitSignlessInteger:$selector, - NVVM_MMAShapeAttr:$shape, - OptionalAttr:$b1Op, - OptionalAttr:$intOverflowBehavior, - NVVM_MMALayoutAttr:$layoutA, - NVVM_MMALayoutAttr:$layoutB, - NVVM_MMATypesAttr:$aType, - NVVM_MMATypesAttr:$bType, - NVVM_MMATypesAttr:$cType, - NVVM_ScaleVecSizeAttr:$scaleVecSize, - NVVM_BlockScaleFormatAttr:$blockScaleFormat, - NVVM_SparsityFormatAttr:$sparsityFormat - ); - let results = (outs - NVVM_VectorOfAnyTypeValues:$res - ); -} -def NVVM_MmaSpOp : NVVM_Op<"mma.sp.sync", [AttrSizedOperandSegments]> { - let description = [{ - The `nvvm.mma.sp.sync` operation collectively performs the sparse operation - `D = matmul(A_sparse, B) + C` using all threads in a warp. - - This operation is similar to `nvvm.mma.sync` but with structured sparsity - in the A operand. The sparsity follows the 2:4 structured sparse pattern - where 2 out of every 4 elements are non-zero. - - All the threads in the warp must execute the same `mma.sp.sync` operation. - - The `sparseMetadata` operand provides the sparsity indices that indicate - which elements in the A operand are non-zero. The `sparsitySelector` - controls how the indices are distributed among threads in the warp and - should typically be 0 or 1. - - The optional `orderedMetadata` attribute specifies the metadata ordering: - - Absence (default): Uses standard sparse metadata ordering - - Presence: Uses ordered metadata (PTX ISA 8.5+, sm_90+) - - The optional `kind` attribute specifies mixed-precision modes for FP8 operations: - - `f8f6f4`: Enables e3m2, e2m3, e2m1 FP8 types and f16 accumulator (PTX ISA 8.7+, sm_90+) - - Only valid with ordered metadata and m16n8k64 shape - - The shapes, layouts, and data types follow the same constraints as the - regular `nvvm.mma.sync` operation, but the A operand contains only the - non-zero elements in compressed format. - - Example: - ```mlir - %d = nvvm.mma.sp.sync A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] - sparseMetadata[%meta] selector[%sel] - {shape = {k = 32 : i32, m = 16 : i32, n = 8 : i32}} - : (vector<2xf16>, vector<2xf16>, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> - - // With ordered metadata: - %d = nvvm.mma.sp.sync A[%a0, %a1] B[%b0, %b1] C[%c0, %c1] - sparseMetadata[%meta] selector[%sel] - {orderedMetadata, shape = {k = 32 : i32, m = 16 : i32, n = 8 : i32}} - : (vector<2xf16>, vector<2xf16>, vector<2xf16>) -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> - ``` - }]; - let arguments = (ins - Variadic:$operandA, - Variadic:$operandB, - Variadic:$operandC, - NVVM_T32BitSignlessInteger:$sparseMetadata, - NVVM_T32BitSignlessInteger:$sparsitySelector, - NVVM_MMAShapeAttr:$shape, - OptionalAttr:$intOverflowBehavior, - OptionalAttr:$multiplicandAPtxType, - OptionalAttr:$multiplicandBPtxType, - UnitAttr:$orderedMetadata, - OptionalAttr:$kind - ); - let results = (outs - NVVM_LLVMStructureType:$res - ); -} -def NVVM_MmaOp : NVVM_Op<"mma.sync", [AttrSizedOperandSegments]> { - let description = [{ - The `nvvm.mma.sync` operation collectively performs the operation - `D = matmul(A, B) + C` using all threads in a warp. - - All the threads in the warp must execute the same `mma.sync` operation. - - For each possible multiplicand PTX data type, there are one or more possible - instruction shapes given as "mMnNkK". The below table describes the posssibilities - as well as the types required for the operands. Note that the data type for - C (the accumulator) and D (the result) can vary independently when there are - multiple possibilities in the "C/D Type" column. - - When an optional attribute cannot be immediately inferred from the types of - the operands and the result during parsing or validation, an error will be - raised. - - `b1Op` is only relevant when the binary (b1) type is given to - `multiplicandDataType`. It specifies how the multiply-and-acumulate is - performed and is either `xor_popc` or `and_poc`. The default is `xor_popc`. - - `intOverflowBehavior` is only relevant when the `multiplicandType` attribute - is one of `u8, s8, u4, s4`, this attribute describes how overflow is handled - in the accumulator. When the attribute is `satfinite`, the accumulator values - are clamped in the int32 range on overflow. This is the default behavior. - Alternatively, accumulator behavior `wrapped` can also be specified, in - which case overflow wraps from one end of the range to the other. - - `layoutA` and `layoutB` are required and should generally be set to - `#nvvm.mma_layout` and `#nvvm.mma_layout` respectively, but other - combinations are possible for certain layouts according to the table below. - - ``` - | A/B Type | Shape | ALayout | BLayout | A Type | B Type | C/D Type | - |----------|-----------|---------|---------|----------|----------|-------------------| - | f64 | .m8n8k4 | row | col | 1x f64 | 1x f64 | 2x f64 | - | f16 | .m8n8k4 | row/col | row/col | 2x f16x2 | 2x f16x2 | 4x f16x2 or 8xf32 | - | | .m16n8k8 | row | col | 2x f16x2 | 1x f16x2 | 2x f16x2 or 4 f32 | - | | .m16n8k16 | row | col | 4x f16x2 | 2x f16x2 | 2x f16x2 or 4 f32 | - | bf16 | .m16n8k8 | row | col | 2x i32 | 1x i32 | 4x f32 | - | | .m16n8k16 | row | col | 4x i32 | 2x i32 | 4x f32 | - | tf32 | .m16n8k4 | row | col | 2x i32 | 1x i32 | 4x f32 | - | | .m16n8k8 | row | col | 4x i32 | 2x i32 | 2x f16x2 or 4 f32 | - | u8/s8 | .m8n8k16 | row | col | 1x i32 | 1x i32 | 2x i32 | - | | .m16n8k16 | row | col | 2x i32 | 1x i32 | 4x i32 | - | | .m16n8k32 | row | col | 4x i32 | 2x i32 | 4x i32 | - | u4/s4 | .m8n8k32 | row | col | 1x i32 | 1x i32 | 2x i32 | - | | m16n8k32 | row | col | 2x i32 | 1x i32 | 4x i32 | - | | m16n8k64 | row | col | 4x i32 | 2x i32 | 4x i32 | - | b1 | m8n8k128 | row | col | 1x i32 | 1x i32 | 2x i32 | - | | m16n8k128 | row | col | 2x i32 | 1x i32 | 4x i32 | - ``` - - Example: - ```mlir - - %128 = nvvm.mma.sync A[%120, %121, %122, %123] - B[%124, %125] - C[%126, %127] - {layoutA = #nvvm.mma_layout, - layoutB = #nvvm.mma_layout, - shape = {k = 16 : i32, m = 16 : i32, n = 8 : i32}} - : (vector<2xf16>, vector<2xf16>, vector<2xf16>) - -> !llvm.struct<(vector<2xf16>, vector<2xf16>)> - ``` - }]; - let arguments = (ins - Variadic:$operandA, - Variadic:$operandB, - Variadic:$operandC, - NVVM_MMAShapeAttr:$shape, - OptionalAttr:$b1Op, - OptionalAttr:$intOverflowBehavior, - NVVM_MMALayoutAttr:$layoutA, - NVVM_MMALayoutAttr:$layoutB, - OptionalAttr:$multiplicandAPtxType, - OptionalAttr:$multiplicandBPtxType - ); - let results = (outs - NVVM_LLVMStructureType:$res - ); -} -def NVVM_MmaSmemDescOp : NVVM_Op<"mma_smem_desc", [MemoryEffects<[MemRead]>]> { - let description = [{ - The `nvvm.mma_smem_desc` constructs a Shared Memory descriptor for use as the MMA Shared Memory operand. - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$pointer, - NVVM_T32BitSignlessInteger:$ldm, - NVVM_T32BitSignlessInteger:$stride, - NVVM_T8BitSignlessInteger2:$baseOffset, - NVVM_T8BitSignlessInteger2:$swizzle, - OptionalAttr:$mmaDescVersion - ); - let results = (outs - NVVM_T64BitSignlessInteger2:$res - ); -} -def NVVM_MovMatrixOp : NVVM_Op<"movmatrix"> { - let description = [{ - Moves a row-major matrix across all threads in a warp, reading elements - from source `$src`, and writing the transposed elements to destination - `$dst`. - - The `shape` attribute indicates the dimensions of the matrix being - transposed. Each matrix element holds 16-bit data as indicated by the - `eltType` attribute. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#warp-level-matrix-movmatrix-instruction) - - Example: - ```mlir - %dst = nvvm.movmatrix %src {shape = #nvvm.ld_st_matrix_shape, - eltType = #nvvm.ld_st_matrix_elt_type} : i32 - ``` - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$src, - NVVM_LdStMatrixShapeAttr:$shape, - OptionalAttr:$layout, - NVVM_LdStMatrixEltTypeAttr:$eltType - ); - let results = (outs - NVVM_T32BitSignlessInteger2:$dst - ); -} -def NVVM_MulOp : NVVM_InferredOp<"mul"> { - let description = [{ - nvvm.mul Op multiplies two values with a given `type` and `mode`. - The type of the operation represents the types of the `a` and `b` operands. - If `.hi` or `.lo` is specified, then `res` is the same size as `a` and `b`, - and either the upper or lower half of the result is written to the - destination register. If `.wide` is specified, then `res` is twice as - wide as `a` and `b` to receive the full result of the multiplication. The - `.wide` suffix is supported only for 16- and 32-bit integer types. - For more information, refer [PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#integer-arithmetic-instructions-mul) - }]; - let arguments = (ins - NVVM_T16BitSignlessIntegerOr323:$a, - NVVM_T16BitSignlessIntegerOr323:$b, - NVVM_MulModeAttr:$mode, - OptionalAttr:$isSigned - ); - let results = (outs - NVVM_T16BitSignlessIntegerOr323:$res - ); -} -def NVVM_MulPacked_BF16x2_BF16x2_F16x2Op : NVVM_Op<"mul.packed.bf16x2.bf16x2.f16x2"> { - let description = [{ - This Op performs the multiplication operation on a pair of "bf16"x2 and "f16"x2 vectors. - The inputs are of the form v2bf16 and v2f16, and the output is of the form v2bf16. - The inputs are converted to a scalar of packed i32 where the 0th element of the - vector forms the low-16 bits and the 1st element occupies the high-16 bits. - - Examples: - ```mlir - %0 = nvvm.mul.packed.bf16x2.bf16x2.f16x2 %srcA, %srcB : vector<2xbf16>, vector<2xf16> -> vector<2xbf16> - ``` - }]; - let arguments = (ins - NVVM_VectorOfBfloat16TypeValuesOf:$srcA, - NVVM_VectorOf16BitFloatValues:$srcB - ); - let results = (outs - NVVM_VectorOfBfloat16TypeValuesOf:$res - ); -} -def NVVM_MulPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"mul.packed.bf16x2.f32x2.f32x2"> { - let description = [{ - This Op performs the multiplication operation on a pair of f32x2 vectors. - The inputs are of the form v2f32 and the output is of the form v2bf16. - The inputs are converted to a scalar of packed i64 where the 0th element of the - vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero - is the only rounding mode supported for this operation. - - Examples: - ```mlir - // Round to zero (only rounding mode supported) - %0 = nvvm.mul.packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> - ``` - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB - ); - let results = (outs - NVVM_VectorOfBfloat16TypeValuesOf:$res - ); -} -def NVVM_MulPacked_F16x2_F16x2_BF16x2Op : NVVM_Op<"mul.packed.f16x2.f16x2.bf16x2"> { - let description = [{ - This Op performs the multiplication operation on a pair of "f16"x2 and "bf16"x2 vectors. - The inputs are of the form v2f16 and v2bf16, and the output is of the form v2f16. - The inputs are converted to a scalar of packed i32 where the 0th element of the - vector forms the low-16 bits and the 1st element occupies the high-16 bits. - - Examples: - ```mlir - %0 = nvvm.mul.packed.f16x2.f16x2.bf16x2 %srcA, %srcB : vector<2xf16>, vector<2xbf16> -> vector<2xf16> - ``` - }]; - let arguments = (ins - NVVM_VectorOf16BitFloatValues:$srcA, - NVVM_VectorOfBfloat16TypeValuesOf:$srcB - ); - let results = (outs - NVVM_VectorOf16BitFloatValues:$res - ); -} -def NVVM_MulPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"mul.packed.f16x2.f32x2.f32x2"> { - let description = [{ - This Op performs the multiplication operation on a pair of f32x2 vectors. - The inputs are of the form v2f32 and the output is of the form v2f16. - The inputs are converted to a scalar of packed i64 where the 0th element of the - vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero - is the only rounding mode supported for this operation. - - Examples: - ```mlir - // Round to zero (only rounding mode supported) - %0 = nvvm.mul.packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> - ``` - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB - ); - let results = (outs - NVVM_VectorOf16BitFloatValues:$res - ); -} -def NVVM_MulPackedF32x2Op : NVVM_InferredOp<"mul.packed.f32x2"> { - let description = [{ - This Op multiplies floating-point vectors. - The inputs and outputs are of the form . The `rnd` attribute - specifies the rounding mode. The `ftz` attribute, when set, flushes the - subnormal inputs and results to sign-preserving zero. - This Op is supported from SM100 and onwards. - [For more information, see PTX ISA] - (TODO: Add public PTX link when available) - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues2:$srcA, - NVVM_VectorOf32BitFloatValues2:$srcB, - OptionalAttr:$rnd, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_VectorOf32BitFloatValues2:$res - ); -} -def NVVM_MultimemLdReduceOp : NVVM_Op<"multimem.ld.reduce"> { - let description = [{ - The `nvvm.multimem.ld.reduce` operation performs a multimem load-reduce - instruction. It reads from a multimem address and reduces partial - results across participating threads. - - The `ordering` attribute specifies the memory ordering semantics (weak, - relaxed, acquire). - - The `op` attribute specifies the reduction operation (and, or, - xor, add, min, max). - - The `dataType` attribute specifies the PTX data type: - - Integer: b32, s32, u32, b64, s64, u64 - - Floating-point: f16, bf16, f32, f64 - - FP8: e5m2, e4m3 - - The optional `accPrec` attribute specifies the accumulator precision type - for sub-single-precision types (f16/bf16/f8E5M2/f8E4M3FN): f16 or f32. - - Example: - ```mlir - %res = nvvm.multimem.ld.reduce %addr - {ordering = #nvvm.mem_order, - op = #nvvm.red_op, - dataType = f32} - : !llvm.ptr<1> -> f32 - ``` - - See PTX ISA: - https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace1:$addr, - NVVM_MultimemLdReduceOrderingAttr:$ordering, - NVVM_MultimemLdReduceOpAttr:$op, - TypeAttr:$dataType, - OptionalAttr:$accPrec - ); - let results = (outs - NVVM_AnyType:$res - ); -} -def NVVM_MultimemRedOp : NVVM_Op<"multimem.red"> { - let description = [{ - The `nvvm.multimem.red` operation performs a multimem atomic reduction, - applying a reduction operation between a source value and the value at a - multimem address. - - The `ordering` attribute specifies the memory ordering semantics. - The `scope` attribute specifies the scope of the operation. - The `op` attribute specifies the reduction operation. - The `dataType` attribute specifies the PTX data type (b32/s32/u32/b64/s64/ - u64 for integer types, f16/bf16/f32/f64 for floating-point types). - - Example: - ```mlir - nvvm.multimem.red %val, %addr - {ordering = #nvvm.mem_order, - scope = #nvvm.mem_scope, - op = #nvvm.red_op, - dataType = f32} - : f32, !llvm.ptr<1> - ``` - - See PTX ISA: - https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem - }]; - let arguments = (ins - NVVM_AnyType:$val, - NVVM_AnyType:$addr, - NVVM_MultimemRedOrderingAttr:$ordering, - NVVM_MemScopeKindAttr:$scope, - NVVM_MultimemRedOpAttr:$op, - TypeAttr:$dataType - ); - let results = (outs); -} -def NVVM_MultimemStOp : NVVM_Op<"multimem.st"> { - let description = [{ - The `nvvm.multimem.st` operation performs a multimem store instruction, - broadcasting a value to a multimem address. - - The `ordering` attribute specifies the memory ordering semantics. - The `scope` attribute specifies the scope of the operation. - The `dataType` attribute specifies the PTX data type (b32/s32/u32/b64/s64/ - u64 for integer types, f16/bf16/f32/f64 for floating-point, e5m2/e4m3 for - FP8 types). - - Example: - ```mlir - nvvm.multimem.st %val, %addr - {ordering = #nvvm.mem_order, - scope = #nvvm.mem_scope, - dataType = f32} - : f32, !llvm.ptr<1> - ``` - - See PTX ISA: - https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-multimem - }]; - let arguments = (ins - NVVM_AnyType:$val, - NVVM_LLVMPointerInAddressSpace1:$addr, - NVVM_MultimemStOrderingAttr:$ordering, - NVVM_MemScopeKindAttr:$scope, - TypeAttr:$dataType - ); - let results = (outs); -} -def NVVM_NanosleepOp : NVVM_Op<"nanosleep"> { - let description = [{ - The op suspends the thread for a sleep duration approximately close to the - delay `$duration`, specified in nanoseconds. - - The sleep duration is approximated, but guaranteed to be in the - interval [0, 2*t]. The maximum sleep duration is 1 millisecond. - The implementation may reduce the sleep duration for individual threads - within a warp such that all sleeping threads in the warp wake up together. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-nanosleep) - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$duration - ); - let results = (outs); -} -def NVVM_PMEventOp : NVVM_Op<"pmevent"> { - let description = [{ - Triggers one or more of a fixed number of performance monitor events, with - event index or mask specified by immediate operand. - - Without `mask` it triggers a single performance monitor event indexed by - immediate operand a, in the range 0..15. - - With `mask` it triggers one or more of the performance monitor events. Each - bit in the 16-bit immediate operand controls an event. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#miscellaneous-instructions-pmevent) - }]; - let arguments = (ins - OptionalAttr:$maskedEventId, - OptionalAttr:$eventId - ); - let results = (outs); -} -def NVVM_PrefetchOp : NVVM_Op<"prefetch"> { - let description = [{ - Prefetches the cache line containing the address given by `addr`. The - operand may be a global, local, or generic pointer. When `tensormap` is - specified, the operand may instead be a constant or generic pointer. If the - address maps to shared memory, the operation has no effect. - - At most one of `cacheLevel` or `tensormap` may be present. The `cacheLevel` - attribute selects the target cache level. When combined with `uniform`, the - prefetch is performed to the uniform cache, in which case `addr` must be a - generic pointer. - - When `tensormap` is used, the line containing `addr` is brought from the - constant or parameter state space for later use by `cp.async.bulk.tensor`. - If `in_param_space` is specified, the generic pointer is interpreted as - referring to the parameter state space. - - `uniform` can be specified after the `cacheLevel` to indicate that the - prefetch is performed to the specified uniform cache level. If `uniform` is - specified, `addr` must be a generic address pointer and no operation is - performed if `addr` maps to a `const`, `local`, or `shared` memory location. - - The `evictPriority` attribute is optional and specifies the cache eviction - priority when `cacheLevel` is L2. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-prefetch-prefetchu) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace13:$addr, - Optional:$predicate, - OptionalAttr:$cacheLevel, - OptionalAttr:$evictPriority, - UnitAttr:$tensormap, - UnitAttr:$uniform, - UnitAttr:$in_param_space - ); - let results = (outs); -} -def NVVM_PermuteOp : NVVM_Op<"prmt", [Pure]> { - let description = [{ - The `nvvm.prmt` operation constructs a permutation of the - bytes of the first one or two operands, selecting based on - the 2 least significant bits of the final operand. - - The bytes in the first one or two source operands are numbered. - The first source operand (%lo) is numbered {b3, b2, b1, b0}, - in the case of the '``default``', '``f4e``' and '``b4e``' variants, - the second source operand (%hi) is numbered {b7, b6, b5, b4}. - - Modes: - - `default`: Index mode - each nibble in `selector` selects a byte from the 8-byte pool - - `f4e` : Forward 4 extract - extracts 4 contiguous bytes starting from position in `selector` - - `b4e` : Backward 4 extract - extracts 4 contiguous bytes in reverse order - - `rc8` : Replicate 8 - replicates the lower 8 bits across the 32-bit result - - `ecl` : Edge clamp left - clamps out-of-range indices to the leftmost valid byte - - `ecr` : Edge clamp right - clamps out-of-range indices to the rightmost valid byte - - `rc16` : Replicate 16 - replicates the lower 16 bits across the 32-bit result - - Depending on the 2 least significant bits of the %selector operand, the result - of the permutation is defined as follows: - - +------------+----------------+--------------+ - | Mode | %selector[1:0] | Output | - +------------+----------------+--------------+ - | '``f4e``' | 0 | {3, 2, 1, 0} | - | +----------------+--------------+ - | | 1 | {4, 3, 2, 1} | - | +----------------+--------------+ - | | 2 | {5, 4, 3, 2} | - | +----------------+--------------+ - | | 3 | {6, 5, 4, 3} | - +------------+----------------+--------------+ - | '``b4e``' | 0 | {5, 6, 7, 0} | - | +----------------+--------------+ - | | 1 | {6, 7, 0, 1} | - | +----------------+--------------+ - | | 2 | {7, 0, 1, 2} | - | +----------------+--------------+ - | | 3 | {0, 1, 2, 3} | - +------------+----------------+--------------+ - | '``rc8``' | 0 | {0, 0, 0, 0} | - | +----------------+--------------+ - | | 1 | {1, 1, 1, 1} | - | +----------------+--------------+ - | | 2 | {2, 2, 2, 2} | - | +----------------+--------------+ - | | 3 | {3, 3, 3, 3} | - +------------+----------------+--------------+ - | '``ecl``' | 0 | {3, 2, 1, 0} | - | +----------------+--------------+ - | | 1 | {3, 2, 1, 1} | - | +----------------+--------------+ - | | 2 | {3, 2, 2, 2} | - | +----------------+--------------+ - | | 3 | {3, 3, 3, 3} | - +------------+----------------+--------------+ - | '``ecr``' | 0 | {0, 0, 0, 0} | - | +----------------+--------------+ - | | 1 | {1, 1, 1, 0} | - | +----------------+--------------+ - | | 2 | {2, 2, 1, 0} | - | +----------------+--------------+ - | | 3 | {3, 2, 1, 0} | - +------------+----------------+--------------+ - | '``rc16``' | 0 | {1, 0, 1, 0} | - | +----------------+--------------+ - | | 1 | {3, 2, 3, 2} | - | +----------------+--------------+ - | | 2 | {1, 0, 1, 0} | - | +----------------+--------------+ - | | 3 | {3, 2, 3, 2} | - +------------+----------------+--------------+ - - [For more information, see PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-prmt) - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$lo, - Optional:$hi, - NVVM_T32BitSignlessInteger:$selector, - NVVM_PermuteModeAttr:$mode - ); - let results = (outs - NVVM_T32BitSignlessInteger2:$res - ); -} -def NVVM_RcpApproxFtzF32Op : NVVM_Op<"rcp.approx.ftz.f", [Pure]> { - let arguments = (ins - NVVM_T32BitFloat:$arg - ); - let results = (outs - NVVM_T32BitFloat2:$res - ); -} -def NVVM_AggrSmemSize : NVVM_Op<"read.ptx.sreg.aggr.smem.size", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClockOp : NVVM_Op<"read.ptx.sreg.clock"> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_Clock64Op : NVVM_Op<"read.ptx.sreg.clock64"> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_BlockInClusterIdXOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.x", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_BlockInClusterIdYOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.y", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_BlockInClusterIdZOp : NVVM_Op<"read.ptx.sreg.cluster.ctaid.z", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClusterId : NVVM_Op<"read.ptx.sreg.cluster.ctarank", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClusterDimBlocksXOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.x", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClusterDimBlocksYOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.y", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClusterDimBlocksZOp : NVVM_Op<"read.ptx.sreg.cluster.nctaid.z", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClusterDim : NVVM_Op<"read.ptx.sreg.cluster.nctarank", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClusterIdXOp : NVVM_Op<"read.ptx.sreg.clusterid.x", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClusterIdYOp : NVVM_Op<"read.ptx.sreg.clusterid.y", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClusterIdZOp : NVVM_Op<"read.ptx.sreg.clusterid.z", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_BlockIdXOp : NVVM_Op<"read.ptx.sreg.ctaid.x", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_BlockIdYOp : NVVM_Op<"read.ptx.sreg.ctaid.y", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_BlockIdZOp : NVVM_Op<"read.ptx.sreg.ctaid.z", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_DynamicSmemSize : NVVM_Op<"read.ptx.sreg.dynamic.smem.size", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg0Op : NVVM_Op<"read.ptx.sreg.envreg0", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg1Op : NVVM_Op<"read.ptx.sreg.envreg1", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg10Op : NVVM_Op<"read.ptx.sreg.envreg10", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg11Op : NVVM_Op<"read.ptx.sreg.envreg11", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg12Op : NVVM_Op<"read.ptx.sreg.envreg12", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg13Op : NVVM_Op<"read.ptx.sreg.envreg13", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg14Op : NVVM_Op<"read.ptx.sreg.envreg14", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg15Op : NVVM_Op<"read.ptx.sreg.envreg15", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg16Op : NVVM_Op<"read.ptx.sreg.envreg16", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg17Op : NVVM_Op<"read.ptx.sreg.envreg17", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg18Op : NVVM_Op<"read.ptx.sreg.envreg18", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg19Op : NVVM_Op<"read.ptx.sreg.envreg19", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg2Op : NVVM_Op<"read.ptx.sreg.envreg2", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg20Op : NVVM_Op<"read.ptx.sreg.envreg20", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg21Op : NVVM_Op<"read.ptx.sreg.envreg21", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg22Op : NVVM_Op<"read.ptx.sreg.envreg22", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg23Op : NVVM_Op<"read.ptx.sreg.envreg23", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg24Op : NVVM_Op<"read.ptx.sreg.envreg24", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg25Op : NVVM_Op<"read.ptx.sreg.envreg25", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg26Op : NVVM_Op<"read.ptx.sreg.envreg26", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg27Op : NVVM_Op<"read.ptx.sreg.envreg27", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg28Op : NVVM_Op<"read.ptx.sreg.envreg28", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg29Op : NVVM_Op<"read.ptx.sreg.envreg29", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg3Op : NVVM_Op<"read.ptx.sreg.envreg3", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg30Op : NVVM_Op<"read.ptx.sreg.envreg30", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg31Op : NVVM_Op<"read.ptx.sreg.envreg31", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg4Op : NVVM_Op<"read.ptx.sreg.envreg4", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg5Op : NVVM_Op<"read.ptx.sreg.envreg5", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg6Op : NVVM_Op<"read.ptx.sreg.envreg6", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg7Op : NVVM_Op<"read.ptx.sreg.envreg7", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg8Op : NVVM_Op<"read.ptx.sreg.envreg8", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_EnvReg9Op : NVVM_Op<"read.ptx.sreg.envreg9", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_GlobalTimerOp : NVVM_Op<"read.ptx.sreg.globaltimer"> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_GlobalTimerLoOp : NVVM_Op<"read.ptx.sreg.globaltimer.lo"> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_GridIdOp : NVVM_Op<"read.ptx.sreg.gridid", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_LaneIdOp : NVVM_Op<"read.ptx.sreg.laneid", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_LaneMaskEqOp : NVVM_Op<"read.ptx.sreg.lanemask.eq", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_LaneMaskGeOp : NVVM_Op<"read.ptx.sreg.lanemask.ge", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_LaneMaskGtOp : NVVM_Op<"read.ptx.sreg.lanemask.gt", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_LaneMaskLeOp : NVVM_Op<"read.ptx.sreg.lanemask.le", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_LaneMaskLtOp : NVVM_Op<"read.ptx.sreg.lanemask.lt", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClusterDimXOp : NVVM_Op<"read.ptx.sreg.nclusterid.x", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClusterDimYOp : NVVM_Op<"read.ptx.sreg.nclusterid.y", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ClusterDimZOp : NVVM_Op<"read.ptx.sreg.nclusterid.z", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_GridDimXOp : NVVM_Op<"read.ptx.sreg.nctaid.x", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_GridDimYOp : NVVM_Op<"read.ptx.sreg.nctaid.y", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_GridDimZOp : NVVM_Op<"read.ptx.sreg.nctaid.z", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_SmDimOp : NVVM_Op<"read.ptx.sreg.nsmid", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_BlockDimXOp : NVVM_Op<"read.ptx.sreg.ntid.x", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_BlockDimYOp : NVVM_Op<"read.ptx.sreg.ntid.y", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_BlockDimZOp : NVVM_Op<"read.ptx.sreg.ntid.z", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_WarpDimOp : NVVM_Op<"read.ptx.sreg.nwarpid", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_SmIdOp : NVVM_Op<"read.ptx.sreg.smid", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ThreadIdXOp : NVVM_Op<"read.ptx.sreg.tid.x", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ThreadIdYOp : NVVM_Op<"read.ptx.sreg.tid.y", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_ThreadIdZOp : NVVM_Op<"read.ptx.sreg.tid.z", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_TotalSmemSize : NVVM_Op<"read.ptx.sreg.total.smem.size", [Pure]> { - let arguments = (ins); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_WarpIdOp : NVVM_Op<"read.ptx.sreg.warpid", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_WarpSizeOp : NVVM_Op<"read.ptx.sreg.warpsize", [Pure]> { - let arguments = (ins - OptionalAttr:$range - ); - let results = (outs - NVVM_LLVMDialectCompatibleType:$res - ); -} -def NVVM_RedOp : NVVM_Op<"red"> { - let description = [{ - Performs a reduction operation with operand `b` and the value in - location `a`, and stores the result of the specified operation at - location `a`. Operand `a` specifies a location in the specified state space. - If no state space is given, perform the memory accesses - using Generic Addressing - - [For more information, see PTX ISA] - (http://sw-mobile-docs/CUDA/GPGPU/parallel-thread-execution/index.html#parallel-synchronization-and-communication-instructions-red) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace05:$a, - NVVM_T32BitSignlessIntegerOr642:$b, - Optional:$cacheHint, - OptionalAttr:$memOrder, - OptionalAttr:$memScope, - NVVM_ReductionOpAttr:$op, - NVVM_ReductionTypeAttr:$type - ); - let results = (outs); -} -def NVVM_ReduxOp : NVVM_InferredOp<"redux.sync"> { - let description = [{ - `redux.sync` performs a reduction operation `kind` of the 32 bit source - register across all non-exited threads in the membermask. - - The `abs` and `nan` attributes can be used in the case of f32 input type, - where the `abs` attribute causes the absolute value of the input to be used - in the reduction operation, and the `nan` attribute causes the reduction - operation to return NaN if any of the inputs to participating threads are - NaN. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-redux-sync) - }]; - let arguments = (ins - NVVM_T32BitSignlessIntegerOr32:$val, - NVVM_T32BitSignlessInteger:$mask_and_clamp, - NVVM_ReductionKindAttr:$kind, - OptionalAttr:$abs, - OptionalAttr:$nan - ); - let results = (outs - NVVM_T32BitSignlessIntegerOr32:$res - ); -} -def NVVM_RsqrtOp : NVVM_InferredOp<"rsqrt", [Pure]> { - let description = [{ - Computes an approximation of the reciprocal of the square root of the - input value: `d = 1 / sqrt(a)`. Supports both f32 and f64. The maximum - relative error for the f32 form over the entire positive finite range - is 2^-22.9. - - The `ftz` attribute, when set, flushes subnormal inputs and results to - sign-preserving zero. For f64 inputs, `ftz=true` selects a coarser - approximation that uses only the upper 32 bits of the input (the lower - 32 bits of the result are zeroed). - - For more information, see PTX ISA: - [rsqrt](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-rsqrt) - }]; - let arguments = (ins - NVVM_T32BitFloatOr64Bit:$src, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_T32BitFloatOr64Bit:$res - ); -} -def NVVM_SetMaxRegisterOp : NVVM_Op<"setmaxregister"> { - let arguments = (ins - I32Attr:$regCount, - NVVM_SetMaxRegisterActionAttr:$action - ); - let results = (outs); -} -def NVVM_ShflOp : NVVM_InferredOp<"shfl.sync"> { - let description = [{ - The `shfl.sync` Op implements data shuffle within threads of a warp. - The `thread_mask` denotes the threads participating in the Op where - the bit position corresponds to a particular thread's laneid. - The `offset` specifies a source lane or source lane offset - (depending on `kind`). The `val` is the input value to be copied from - the source. The `mask_and_clamp` contains two packed values specifying - a mask for logically splitting warps into sub-segments and an upper bound - for clamping the source lane index. - - The `return_value_and_is_valid` unit attribute can be specified to indicate - that the return value is a two-element struct, where the first element is - the result value and the second element is a predicate indicating if the - computed source lane index is valid. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-shfl-sync) - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$thread_mask, - NVVM_T32BitSignlessIntegerOr32:$val, - NVVM_T32BitSignlessInteger:$offset, - NVVM_T32BitSignlessInteger:$mask_and_clamp, - NVVM_ShflKindAttr:$kind, - UnitAttr:$return_value_and_is_valid - ); - let results = (outs - NVVM_T32BitSignlessIntegerOr322:$res - ); -} -def NVVM_SinOp : NVVM_Op<"sin", [Pure]> { - let description = [{ - Computes a fast approximation of the sine of the input value (in radians). - The `ftz` attribute, when set, flushes subnormal inputs and results to - sign-preserving zero. - - For more information, see PTX ISA: - [sin](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-sin) - }]; - let arguments = (ins - NVVM_T32BitFloat:$src, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_T32BitFloat2:$res - ); -} -def NVVM_SPCompressOp : NVVM_Op<"spcompress"> { - let description = [{ - Compresses elements from a dense vector `data` into a structured sparse vector `compressed_data`. - The metadata information is stored in `metadata`. - - Arguments: - - `spDesc`: Sparsity descriptor specifying the selection criteria for elements - - `data`: Input dense vector containing elements to be compressed - - `compFactor`: Compression factor attribute (only `sp2to4` is allowed, resulting in 2:4 compression) - - `indexSize`: Size of metadata index for each selected element (can only be `b2` or `b4` given 2:4 compression) - - `elemSize`: Size in bits for each element from input vector `data` - - `repFactor`: Repeat factor for the operation - - Results: - - `metadata`: Vector storing indices for chosen elements - - `compressed_data`: Output compressed vector containing selected elements - - Examples: - // Example 1: Basic compression with 4-bit elements, 2-bit indices, repeat factor of 1 - ```mlir - %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { - compFactor = #nvvm.spcomp_factor, - indexSize = #nvvm.spcomp_index_size, - elemSize = #nvvm.spcomp_elem_size, - repFactor = #nvvm.spcomp_rep_factor - } : i32, vector<2xi32> -> i32, i32 - ``` - - // Example 2: Compression with 8-bit elements, 4-bit indices, repeat factor of 2 - ```mlir - %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { - compFactor = #nvvm.spcomp_factor, - indexSize = #nvvm.spcomp_index_size, - elemSize = #nvvm.spcomp_elem_size, - repFactor = #nvvm.spcomp_rep_factor - } : i32, vector<4xi32> -> i32, vector<2xi32> - ``` - - // Example 3: Compression with 16-bit elements, 4-bit indices, larger repeat factor - ```mlir - %metadata, %compressed_data = nvvm.spcompress %spdesc, %data { - compFactor = #nvvm.spcomp_factor, - indexSize = #nvvm.spcomp_index_size, - elemSize = #nvvm.spcomp_elem_size, - repFactor = #nvvm.spcomp_rep_factor - } : i32, vector<16xi32> -> vector<2xi32>, vector<8xi32> - ``` - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$spDesc, - NVVM_VectorOf32BitSignlessInteger:$data, - NVVM_SPCompressFactorTypeAttr:$compFactor, - NVVM_SPCompressIndexSizeAttr:$indexSize, - NVVM_SPCompressElemSizeAttr:$elemSize, - NVVM_SPCompressRepFactorAttr:$repFactor - ); - let results = (outs - NVVM_T32BitSignlessIntegerOrVector:$metadata, - NVVM_T32BitSignlessIntegerOrVector:$compressed_data - ); -} -def NVVM_SPDecompressOp : NVVM_Op<"spdecompress"> { - let description = [{ - Decompresses elements from a structured sparse vector `compressed_data` into a dense vector `data`. - The metadata information is stored in `metadata`. - - Arguments: - - `metadata`: Vector storing indices for chosen elements - - `compressed_data`: Compressed vector containing selected elements - - `compFactor`: Decompression factor attribute (sp1to2, sp1to4, sp1to8, sp1to16, sp2to4, sp2to8, sp2to16, sp4to8, sp4to16, sp8to16) - - `indexSize`: Size of metadata index for each selected element (b2 or b4) - - `elemSize`: Size in bits for each element from output vector `data` (b4, b8, b16) - - `repFactor`: Repeat factor for the operation (x1, x2, x4, x8, x16, x32, x64) - - Results: - - `data`: Output dense vector containing decompressed elements - - Examples: - // Example 1: Decompression with 4-bit elements, 2-bit indices, repeat factor of 2 - ```mlir - %res = nvvm.spdecompress %metadata, %compdata { - compFactor = #nvvm.spdecomp_factor, - indexSize = #nvvm.spdecomp_index_size, - elemSize = #nvvm.spdecomp_elem_size, - repFactor = #nvvm.spdecomp_rep_factor - } : i32, i32 -> i32 - ``` - - // Example 2: Decompression with 8-bit elements, 4-bit indices, repeat factor of 2 - ```mlir - %res = nvvm.spdecompress %metadata, %compdata { - compFactor = #nvvm.spdecomp_factor, - indexSize = #nvvm.spdecomp_index_size, - elemSize = #nvvm.spdecomp_elem_size, - repFactor = #nvvm.spdecomp_rep_factor - } : i32, i32 -> vector<2 x i32> - ``` - - // Example 3: Decompression with 16-bit elements, 4-bit indices, repeat factor of 64 - ```mlir - %res = nvvm.spdecompress %metadata, %compdata { - compFactor = #nvvm.spdecomp_factor, - indexSize = #nvvm.spdecomp_index_size, - elemSize = #nvvm.spdecomp_elem_size, - repFactor = #nvvm.spdecomp_rep_factor - } : vector<16 x i32>, vector<64 x i32> -> vector<128 x i32> - ``` - }]; - let arguments = (ins - NVVM_T32BitSignlessIntegerOrVector:$metadata, - NVVM_T32BitSignlessIntegerOrVector:$compressed_data, - NVVM_SPDecompressFactorTypeAttr:$compFactor, - NVVM_SPDecompressIndexSizeAttr:$indexSize, - NVVM_SPDecompressElemSizeAttr:$elemSize, - NVVM_SPDecompressRepFactorAttr:$repFactor - ); - let results = (outs - NVVM_VectorOf32BitSignlessInteger2:$data - ); -} -def NVVM_BulkStoreOp : NVVM_Op<"st.bulk"> { - let description = [{ - Initializes a region of shared memory at the address given by `addr`. - The `size` operand specifies the number of bytes to initialize and must be - a multiple of 8. - The `initVal` operand specifies the value to initialize the memory to. The - only supported value is 0. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-st-bulk) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace02:$addr, - NVVM_T64BitSignlessInteger:$size, - OptionalAttr:$initVal - ); - let results = (outs); -} -def NVVM_StMatrixOp : NVVM_Op<"stmatrix"> { - let description = [{ - Collectively store one or more matrices across all threads in a warp to the - location indicated by the address operand $ptr in shared memory. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#warp-level-matrix-store-instruction-stmatrix) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace3:$ptr, - Variadic:$sources, - NVVM_MMALayoutAttr:$layout, - OptionalAttr:$shape - ); - let results = (outs); -} -def NVVM_AsyncStoreOp : NVVM_Op<"store.async"> { - let arguments = (ins - NVVM_LLVMPointerInAddressSpace7:$addr, - NVVM_T32BitSignlessIntegerOr643:$value, - NVVM_LLVMPointerInAddressSpace7:$mbarrier - ); - let results = (outs); -} -def NVVM_ExtStoreOp : NVVM_Op<"store.ext"> { - let description = [{ - The `store.ext` operation is used to write to memory in generic, - global or shared address space. This Op explicitly supports various - cache properties and memory orderings associated with the store operation. - - The `l2CacheHint` operand is optional, and it is used to specify cache - eviction policy that may be used during the memory access. - [For more information, see PTX ISA] - (https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#data-movement-and-conversion-instructions-st) - }]; - let arguments = (ins - NVVM_LLVMTypeWithSize:$value, - NVVM_LLVMPointerType:$addr, - Optional:$l2CacheHint, - OptionalAttr:$order, - OptionalAttr:$scope, - OptionalAttr:$evict, - OptionalAttr:$cacheModifier, - OptionalAttr:$sharedSpace - ); - let results = (outs); -} -def NVVM_SubPacked_BF16x2_F32x2_F32x2Op : NVVM_Op<"sub.packed.bf16x2.f32x2.f32x2"> { - let description = [{ - This Op performs the sub operation on a pair of f32x2 vectors. - The inputs and output are of the form v2f32 and v2bf16 respectively. - The inputs are converted to a scalar of packed i64 where the 0th element of the - vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero - is the only rounding mode supported for this operation. - - Examples: - ```mlir - // Round to zero (only rounding mode supported) - %0 = nvvm.sub .packed.bf16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xbf16> - ``` - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB - ); - let results = (outs - NVVM_VectorOfBfloat16TypeValuesOf:$res - ); -} -def NVVM_SubPacked_F16x2_F32x2_F32x2Op : NVVM_Op<"sub.packed.f16x2.f32x2.f32x2"> { - let description = [{ - This Op performs the sub operation on a pair of f32x2 vectors. - The inputs and output are of the form v2f32 and v2f16 respectively. - The inputs are converted to a scalar of packed i64 where the 0th element of the - vector forms the low-32 bits and the 1st element occupies the high-32 bits. The round to zero - is the only rounding mode supported for this operation. - - Examples: - ```mlir - // Round to zero (only rounding mode supported) - %0 = nvvm.sub .packed.f16x2.f32x2.f32x2 %srcA, %srcB : vector<2xf32>, vector<2xf32> -> vector<2xf16> - ``` - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB - ); - let results = (outs - NVVM_VectorOf16BitFloatValues:$res - ); -} -def NVVM_SubPackedF32x2Op : NVVM_InferredOp<"sub.packed.f32x2"> { - let description = [{ - This Op performs subtraction of floating-point vectors. - The inputs and outputs are of the form . The `rnd` attribute - specifies the rounding mode. The `ftz` attribute, when set, flushes the - subnormal inputs and results to sign-preserving zero. - This Op is supported from SM100 and onwards. - [For more information, see PTX ISA] - (TODO: Add public PTX link when available) - }]; - let arguments = (ins - NVVM_VectorOf32BitFloatValues2:$srcA, - NVVM_VectorOf32BitFloatValues2:$srcB, - OptionalAttr:$rnd, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_VectorOf32BitFloatValues2:$res - ); -} -def NVVM_SubPacked_F32x2_BF16x2_F32x2Op : NVVM_Op<"sub.packed.f32x2.bf16x2.f32x2"> { - let description = [{ - This Op performs the sub operation on a pair of floating-point values. - The inputs are of the form v2bf16 and v2f32. The first input `srcA` is - converted to a scalar of packed i32 where the 0th element of the vector forms - the low-16 bits and the 1st element occupies the high-16 bits. Similarly, - the second input `srcB` is converted to a scalar of packed i64 where the 0th - element of the vector forms the low-32 bits and the 1st element occupies the - high-32 bits. The `rnd` attribute specifies the rounding mode. - - The rounding mode can be one of the following: - - rn: round to nearest even - - rz: round toward zero - - rm: round toward negative infinity - - rp: round toward positive infinity - - Examples: - ```mlir - // Round to nearest even (default) - %0 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB : vector<2xbf16>, vector<2xf32> -> vector<2xf32> - - // Round toward negative infinity - %1 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> - - // Round toward zero with flush-to-zero - %2 = nvvm.sub .packed.f32x2.bf16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xbf16>, vector<2xf32> -> vector<2xf32> - ``` - }]; - let arguments = (ins - NVVM_VectorOfBfloat16TypeValuesOf:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB, - OptionalAttr:$rnd - ); - let results = (outs - NVVM_VectorOf32BitFloatValues:$res - ); -} -def NVVM_SubPacked_F32x2_F16x2_F32x2Op : NVVM_Op<"sub.packed.f32x2.f16x2.f32x2"> { - let description = [{ - This Op performs the sub operation on a pair of floating-point values. - The inputs are of the form v2f16 and v2f32. The first input `srcA` is - converted to a scalar of packed i32 where the 0th element of the vector forms - the low-16 bits and the 1st element occupies the high-16 bits. Similarly, - the second input `srcB` is converted to a scalar of packed i64 where the 0th - element of the vector forms the low-32 bits and the 1st element occupies the - high-32 bits. The `rnd` attribute specifies the rounding mode. - - The rounding mode can be one of the following: - - rn: round to nearest even - - rz: round toward zero - - rm: round toward negative infinity - - rp: round toward positive infinity - - Examples: - ```mlir - // Round to nearest even (default) - %0 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB : vector<2xf16>, vector<2xf32> -> vector<2xf32> - - // Round toward negative infinity - %1 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> - - // Round toward zero with flush-to-zero - %2 = nvvm.sub .packed.f32x2.f16x2.f32x2 %srcA, %srcB {rnd = #nvvm.fp_rnd_mode} : vector<2xf16>, vector<2xf32> -> vector<2xf32> - ``` - }]; - let arguments = (ins - NVVM_VectorOf16BitFloatValues:$srcA, - NVVM_VectorOf32BitFloatValues:$srcB, - OptionalAttr:$rnd - ); - let results = (outs - NVVM_VectorOf32BitFloatValues:$res - ); -} -def NVVM_SubFOp : NVVM_InferredOp<"subf", [Pure]> { - let description = [{ - The `nvvm.subf` operation performs floating point subtraction of two - operands. - - It supports the same type combinations and modifiers as `nvvm.addf`. - This is equivalent to `nvvm.addf(lhs, -rhs)`. - - For more information, see PTX ISA: - - [floating point subtraction](https://docs.nvidia.com/cuda/parallel-thread-execution/#floating-point-instructions-sub) - - [half-precision floating point subtraction](https://docs.nvidia.com/cuda/parallel-thread-execution/#half-precision-floating-point-instructions-sub) - }]; - let arguments = (ins - NVVM_T16BitFloatOrBfloat16Type:$lhs, - NVVM_T16BitFloatOrBfloat16Type:$rhs, - OptionalAttr:$rnd, - OptionalAttr:$sat, - OptionalAttr:$ftz - ); - let results = (outs - NVVM_T16BitFloatOrBfloat16Type:$res - ); -} -def NVVM_Tcgen05AllocOp : NVVM_Op<"tcgen05.alloc"> { - let description = [{ - The `tcgen05.alloc` Op allocates tensor core memory for - the amount specified by `nCols` and writes the destination - address to the `addr` argument. The `nCols` operand specifies the - number of columns to be allocated and it must be a power-of-two. - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) - }]; - let arguments = (ins - NVVM_LLVMPointerTypeOrLLVMPointer:$addr, - NVVM_T32BitSignlessInteger:$nCols, - UnitAttr:$isExclusive, - OptionalAttr:$group - ); - let results = (outs); -} -def NVVM_Tcgen05CommitOp : NVVM_Op<"tcgen05.commit"> { - let description = [{ - The `tcgen05.commit` makes the *mbarrier object*, specified by - the operand `addr`, track the completion of all the prior - async-tcgen05 operations initiated by the executing thread. - The multicast variants allow signaling on the *mbarrier objects* - of multiple CTAs within the cluster. Operand `multicastMask`, - when present, specifies the destination CTAs in the cluster such - that each bit position in the 16-bit or 32-bit `multicastMask` operand - corresponds to the `nvvm.read.ptx.sreg.ctaid` of the destination CTA. - Operand `smem_a_read` when present, specifies track only shared memory - read of "Matrix A for the prior tcgen05.mma Ops" - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen-async-sync-operations-commit) - }]; - let arguments = (ins - NVVM_LLVMPointerTypeOrLLVMPointer:$addr, - Optional:$multicastMask, - UnitAttr:$smem_a_read, - OptionalAttr:$group - ); - let results = (outs); -} -def NVVM_Tcgen05CpOp : NVVM_Op<"tcgen05.cp"> { - let description = [{ - Instruction tcgen05.cp initiates an asynchronous copy operation from - shared memory to the location specified by the address operand `taddr` - in the Tensor Memory. The 64-bit register operand `smem_desc` specifies - the matrix descriptor representing the source matrix in the shared memory - that needs to be copied. - - Example: - ```mlir - nvvm.tcgen05.cp %taddr, %smem_desc { - group = #nvvm.cta_group, - shape = #nvvm.tcgen05_cp_shape, - multicast = #nvvm.tcgen05_cp_multicast, - srcFormat = #nvvm.tcgen05_cp_src_fmt - } - ``` - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensorcore-5th-generation-instructions-tcgen05-cp) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$taddr, - NVVM_T64BitSignlessInteger:$smem_desc, - NVVM_Tcgen05CpShapeAttr:$shape, - OptionalAttr:$group, - OptionalAttr:$multicast, - OptionalAttr:$srcFormat - ); - let results = (outs); -} -def NVVM_Tcgen05DeallocOp : NVVM_Op<"tcgen05.dealloc"> { - let description = [{ - The `tcgen05.dealloc` Op de-allocates the tensor core memory - specified by `tmemAddr`, which must be from a previous tensor - memory allocation. The `nCols` operand specifies the number - of columns to be de-allocated, and it must be a power-of-two. - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$taddr, - NVVM_T32BitSignlessInteger:$nCols, - UnitAttr:$isExclusive, - OptionalAttr:$group - ); - let results = (outs); -} -def NVVM_Tcgen05FenceOp : NVVM_Op<"tcgen05.fence"> { - let description = [{ - The `tcgen05.fence` orders all prior async tcgen05 operations - with respect to the subsequent tcgen05 and execution ordering operations. - The `tcgen05.fence` orders all subsequent async tcgen05 operations - with respect to the prior tcgen05 and execution ordering operations. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tensorcore-5th-generation-instructions-tcgen05-fence) - }]; - let arguments = (ins - NVVM_Tcgen05FenceKindAttr:$kind - ); - let results = (outs); -} -def NVVM_Tcgen05LdOp : NVVM_Op<"tcgen05.ld"> { - let description = [{ - Instruction `tcgen05.ld` asynchronously loads data from the Tensor Memory at - the location specified by the 32-bit address operand `tmemAddr` into the - destination register `res`, collectively across all threads of the warps. - - The `shape` and the `num` attribute together determines the total - dimension of the data which is loaded from the Tensor Memory. The `shape` - attribute indicates the base dimension of data to be accessed as described - in the Data Movement Shape. The `num` attribute indicates the repeat - factor on the base dimension resulting in the total dimension of the data - that is accessed. - - The shape `16x32bx2` performs two accesses into Tensor Memory of the shape - `16x32b`. The base address of the first access is specified by `tmemAddr` - and the base address of the second access is specified by - `tmemAddr + offset`, where `offset` is an immediate argument. - - The unit attribute `pack` can be used to pack two 16-bit - elements from adjacent columns into a single 32-bit element during the load. - - The following table describes the size of the vector for various combinations - of `num` and `shape` attributes: - ``` - |=====================================================================| - | num/shape | 16x32bx2/16x64b/32x32b | 16x128b | 16x256b | - |=====================================================================| - | x1 | 1 | 2 | 4 | - | x2 | 2 | 4 | 8 | - | x4 | 4 | 8 | 16 | - | x8 | 8 | 16 | 32 | - | x16 | 16 | 32 | 64 | - | x32 | 32 | 64 | 128 | - | x64 | 64 | 128 | NA | - | x128 | 128 | NA | NA | - |=====================================================================| - ``` - - Example: - ```mlir - nvvm.tcgen05.ld %tmemAddr, %offset pack { - shape = #nvvm.tcgen05_ldst_shape, - } : <2xi32> - ``` - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$tmemAddr, - Optional:$offset, - UnitAttr:$pack, - NVVM_Tcgen05LdStShapeAttr:$shape - ); - let results = (outs - NVVM_T32BitSignlessIntegerOrVector2:$res - ); -} -def NVVM_Tcgen05LdRedOp : NVVM_Op<"tcgen05.ld.red"> { - let description = [{ - Instruction `tcgen05.ld.red` asynchronously loads data from the Tensor - Memory at the location specified by the 32-bit address operand `addr` into - the destination register `data`, collectively across all threads of the - warp. The operation also performs reduction operation specified by `op` on - the loaded data across columns in each lane and stored into `redVal` - - The `shape` and the `num` attribute together determines the total - dimension of the data which is loaded from the Tensor Memory. The `shape` - attribute indicates the base dimension of data to be accessed as described - in the Data Movement Shape. The `num` attribute indicates the repeat - factor on the base dimension resulting in the total dimension of the data - that is accessed. - - The shape `16x32bx2` performs two accesses into Tensor Memory of the shape - `16x32b`. The base address of the first access is specified by `addr` - and the base address of the second access is specified by - `addr + offset`, where `offset` is an immediate argument. - - `isSigned` attribute is used to specify whether the integer reduction value is is signed, it is to false by default - - The following table describes the size of the vector for various combinations - of `num` and `shape` attributes: - ``` - |=============================================| - | num/shape | 16x32bx2/32x32b | - |=============================================| - | x2 | 2 | - | x4 | 4 | - | x8 | 8 | - | x16 | 16 | - | x32 | 32 | - | x64 | 64 | - | x128 | 128 | - |=============================================| - ``` - - Example: - ```mlir - // u32 - %data, %redval = nvvm.tcgen05.ld,red %addr, %offset { - shape = #nvvm.tcgen05_ldst_shape, - } : <2xi32>, i32 - - // s32 - %data, %redval = nvvm.tcgen05.ld,red %addr, %offset { - shape = #nvvm.tcgen05_ldst_shape, - isSigned = true - } : <2xi32>, i32 - - // f32 - %data, %redval = nvvm.tcgen05.ld,red %addr { - shape = #nvvm.tcgen05_ldst_shape, - } : <2xf32>, f32 - ``` - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-ld) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$addr, - Optional:$offset, - NVVM_Tcgen05LdStShapeAttr:$shape, - NVVM_ReductionKindAttr:$op, - UnitAttr:$abs, - UnitAttr:$nan, - OptionalAttr:$isSigned - ); - let results = (outs - NVVM_VectorOf32BitSignlessInteger3:$data, - NVVM_T32BitSignlessIntegerOr32:$redVal - ); -} -def NVVM_Tcgen05LdRedSpcompressOp : NVVM_Op<"tcgen05.ld.red.spcompress"> { - let description = [{ - The `tcgen05.ld.red.spcompress` performs in-line sparse compression when loading data from Tensor Memory. - The operation uses a combination of `abs` and `opKind` parameters to select values for compression. - The NaN detection mode is always used for the compression operation, while the .NAN flag only applies - to the reduction portion. - - Arguments: - - `tmemAddr`: Tensor memory address to load from - - `abs`: Whether to use absolute value comparison (used with opKind for value selection) - - `opKind`: Comparison operator kind (min/max) for selecting values (used with abs for value selection) - - `shape`: Pattern used to construct the Matrix (used with repFactor for Matrix construction) - - `repFactor`: Repeat factor, number of times the pattern has to be repeated to form the Matrix (used with shape for Matrix construction) - - `compFactor`: Compression factor (2:4 sparsity, choose 2 pair of elems from every 4 pairs of elems) - - `nan`: Whether to use NaN detection mode - - Results: - - `metadata`: Vector storing sparse metadata - - `compressedData`: Vector storing compressed data - - `redVal`: Reduction value - - Examples: - ```mlir - // MIN operation with x4 repetition factor - %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { - shape = #nvvm.tcgen05_ldst_shape, - opKind = #nvvm.spcomp_op_kind, - compFactor = #nvvm.spcomp_factor, - repFactor = #nvvm.spcomp_rep_factor - } : i32, vector<2xi32> - - // Max operation with absolute value comparison - %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { - abs, - shape = #nvvm.tcgen05_ldst_shape, - opKind = #nvvm.spcomp_op_kind, - compFactor = #nvvm.spcomp_factor, - repFactor = #nvvm.spcomp_rep_factor - } : i32, vector<4xi32> - - // MIN operation with NaN detection for reduction - %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { - nan, - shape = #nvvm.tcgen05_ldst_shape, - opKind = #nvvm.spcomp_op_kind, - compFactor = #nvvm.spcomp_factor, - repFactor = #nvvm.spcomp_rep_factor - } : i32, vector<8xi32> - - // MAX operation with abs and nan options - %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { - abs, - nan, - shape = #nvvm.tcgen05_ldst_shape, - opKind = #nvvm.spcomp_op_kind, - compFactor = #nvvm.spcomp_factor, - repFactor = #nvvm.spcomp_rep_factor - } : vector<2xi32>, vector<32xi32> - - // MIN operation with maximum repetition factor - %metadata, %compressed_data, %red_val = nvvm.tcgen05.ld.red.spcompress %taddr { - shape = #nvvm.tcgen05_ldst_shape, - opKind = #nvvm.spcomp_op_kind, - compFactor = #nvvm.spcomp_factor, - repFactor = #nvvm.spcomp_rep_factor - } : vector<4xi32>, vector<64xi32> - ``` - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$tmemAddr, - UnitAttr:$abs, - UnitAttr:$nan, - NVVM_Tcgen05LdStShapeAttr:$shape, - NVVM_SPCompressOpKindAttr:$opKind, - NVVM_SPCompressFactorTypeAttr:$compFactor, - NVVM_SPCompressRepFactorAttr:$repFactor - ); - let results = (outs - NVVM_T32BitSignlessIntegerOrVector3:$metadata, - NVVM_VectorOf32BitSignlessInteger4:$compressedData, - NVVM_T32BitSignlessInteger:$redVal - ); -} -def NVVM_Tcgen05LdSpcompressOp : NVVM_Op<"tcgen05.ld.spcompress"> { - let description = [{ - The `tcgen05.ld.spcompress` performs an in-line sparse compression when loading data from Tensor Memory. - The operation uses a combination of `abs` and `opKind` parameters to select values for compression. - - With a 2-to-4 sparsity factor (specified by `compFactor`), half as much data is written to the output - as is read from Tensor Memory. The sequence of sparse metadata is written to the `metadata` result, - while the compressed data is written to `compressedData`. - - Arguments: - - `tmemAddr`: Tensor memory address to load from - - `abs`: Whether to use absolute value comparison (used with opKind for value selection) - - `opKind`: Comparison operator kind (min/max) for selecting values (used with abs for value selection) - - `shape`: Pattern used to construct the Matrix (used with repFactor for Matrix construction) - - `repFactor`: Repeat factor, number of times the pattern has to be repeated to form the Matrix (used with shape for Matrix construction) - - `compFactor`: Compression factor (2:4 sparsity, choose 2 pair of elems from every 4 pairs of elems) - - Results: - - `metadata`: Vector storing sparse metadata - - `compressedData`: Vector storing compressed data - - Example: - ```mlir - // MIN operation with x4 repetition factor - %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { - shape = #nvvm.tcgen05_ldst_shape, - opKind = #nvvm.spcomp_op_kind, - compFactor = #nvvm.spcomp_factor, - repFactor = #nvvm.spcomp_rep_factor - } : i32, vector<2xi32> - - // MAX operation with abs option - %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { - abs, - shape = #nvvm.tcgen05_ldst_shape, - opKind = #nvvm.spcomp_op_kind, - compFactor = #nvvm.spcomp_factor, - repFactor = #nvvm.spcomp_rep_factor - } : i32, vector<4xi32> - - // MIN operation with x64 repetition factor - %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { - shape = #nvvm.tcgen05_ldst_shape, - opKind = #nvvm.spcomp_op_kind, - compFactor = #nvvm.spcomp_factor, - repFactor = #nvvm.spcomp_rep_factor - } : vector<2xi32>, vector<32xi32> - - // MAX operation with abs and x128 repetition factor - %metadata, %compressed_data = nvvm.tcgen05.ld.spcompress %taddr { - abs, - shape = #nvvm.tcgen05_ldst_shape, - opKind = #nvvm.spcomp_op_kind, - compFactor = #nvvm.spcomp_factor, - repFactor = #nvvm.spcomp_rep_factor - } : vector<4xi32>, vector<64xi32> - ``` - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$tmemAddr, - UnitAttr:$abs, - NVVM_Tcgen05LdStShapeAttr:$shape, - NVVM_SPCompressOpKindAttr:$opKind, - NVVM_SPCompressFactorTypeAttr:$compFactor, - NVVM_SPCompressRepFactorAttr:$repFactor - ); - let results = (outs - NVVM_T32BitSignlessIntegerOrVector3:$metadata, - NVVM_VectorOf32BitSignlessInteger4:$compressedData - ); -} -def NVVM_Tcgen05MMAOp : NVVM_Op<"tcgen05.mma", [AttrSizedOperandSegments]> { - let description = [{ - The `tcgen05.mma` operation is an asynchronous tensor core instruction that - performs matrix multiplication, accumulation in a single fused operation. It - targets 5th-generation tensor cores, providing developers with fine-grained - control over execution and scheduling. - - ``` - D = A * B + (D * 2^ -scaleInputD) // if `scaleInputD` is provided - D = A * B // if `enableInputD` is false - D = A * B + D // otherwise - ``` - - where: - - A is an `M x K` matrix in tensor memory or described using shared memory descriptor - - B is a `K x N` matrix described using shared memory descriptor - - D is an `M x N` accumulator matrix in tensor memory - - The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op - - - idesc is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) - - Optional Operands: - - `scaleInputD` is an Immediate value operand used for scaling D matrix by 2 ^ (-scaleInputD). The valid range is [0, 15] - - - `disableOutputLane` is a vector mask for selective output - * vector<4 x i32> when ctaGroup is CTA_1 - * vector<8 x i32> when ctaGroup is CTA_2 - - Required Attributes: - - `kind` is a Tcgen05MMAKind attribute - - - `ctaGroup` specifies CTA group configuration - * cta_1: MMA will be performed on the current thread's CTA - * cta_2: MMA will be performed on the current thread and it's peer CTA - - Default Attributes: - - collectorOp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer - - - `aShift` shifts the rows of the A matrix down by one row and can only be - applied if A is in tensor memory - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$matrixD, - NVVM_LLVMPointerInAddressSpace62:$matrixA, - NVVM_T64BitSignlessInteger:$matrixB, - NVVM_T32BitSignlessInteger:$idesc, - NVVM_T1BitSignlessInteger:$enableInputD, - Optional:$scaleInputD, - Optional:$disableOutputLane, - NVVM_Tcgen05MMAKindAttr:$kind, - NVVM_CTAGroupKindAttr:$ctaGroup, - OptionalAttr:$collectorOp, - OptionalAttr:$collectorOpB, - UnitAttr:$aShift - ); - let results = (outs); -} -def NVVM_Tcgen05MMABlockScaleOp : NVVM_Op<"tcgen05.mma.block_scale"> { - let description = [{ - The `tcgen05.mma.block_scale` operation is an asynchronous tensor core instruction - that performs matrix multiplication, accumulation with block scaling in a - single fused operation. It targets 5th-generation tensor cores, providing - developers with fine-grained control over execution and scheduling. - - ``` - D = (A * scale_a) * (B * scale_b)` // if `enableInputD` is false - D = (A * scale_a) * (B * scale_b) + D` - ``` - - where: - - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor - - B is a K x N matrix described using shared memory descriptor - - D is an M x N accumulator matrix in tensor memory - - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively - - The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op - - - `idesc` is a 32 bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) - - Required Attributes: - - `kind` is a Tcgen05MMAKind attribute restricted to mxf8f6f4, mxf4, or mxf4nvf4 - - - `ctaGroup` specifies CTA group configuration - * cta_1: MMA will be performed on the current thread's CTA - * cta_2: MMA will be performed on the current thread and it's peer CTA - - Default Attributes: - - collectorOp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) - }]; - let arguments = (ins - NVVM_AnyType:$matrixD, - NVVM_AnyType:$matrixA, - NVVM_AnyType:$matrixB, - NVVM_AnyType:$idesc, - NVVM_AnyType:$enableInputD, - NVVM_AnyType:$scaleA, - NVVM_AnyType:$scaleB, - NVVM_Tcgen05MMAKindAttr:$kind, - NVVM_CTAGroupKindAttr:$ctaGroup, - OptionalAttr:$blockScale, - OptionalAttr:$collectorOp, - OptionalAttr:$collectorOpB - ); - let results = (outs); -} -def NVVM_Tcgen05MMABlockScaleDecompressBOp : NVVM_Op<"tcgen05.mma.block_scale.decompress_b"> { - let description = [{ - The `tcgen05.mma.block_scale.decompress_b` operation is an asynchronous tensor core - instruction that decompresses the B matrix and performs matrix - multiplication, accumulation with block scaling in a single fused operation. - It targets 5th-generation tensor cores, providing developers with - fine-grained control over execution and scheduling. - - ``` - D = (A * scale_a) * (B * scale_b)` // if `enableInputD` is false - D = (A * scale_a) * (B * scale_b) + D` - ``` - - where: - - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor - - B is a `K x N` matrix described using shared memory descriptor - - D is an `M x N` accumulator matrix in tensor memory - - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively - - The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op - - - `idesc` is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) - - - `decompressBMetadata` is a decompress metadata of B matrix - - Required Attributes: - - `ctaGroup` specifies CTA group configuration - * cta_1: MMA will be performed on the current thread's CTA - * cta_2: MMA will be performed on the current thread and it's peer CTA - - Default Attributes: - - collectorOpA is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer - - collectorOpB is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) - }]; - let arguments = (ins - NVVM_AnyType:$matrixD, - NVVM_AnyType:$matrixA, - NVVM_AnyType:$matrixB, - NVVM_AnyType:$idesc, - NVVM_AnyType:$enableInputD, - NVVM_AnyType:$scaleA, - NVVM_AnyType:$scaleB, - NVVM_AnyType:$decompressBMetadata, - NVVM_Tcgen05MMAKindAttr:$kind, - NVVM_CTAGroupKindAttr:$ctaGroup, - OptionalAttr:$blockScale, - OptionalAttr:$collectorOpA, - OptionalAttr:$collectorOpB - ); - let results = (outs); -} -def NVVM_Tcgen05MMADecompressBOp : NVVM_Op<"tcgen05.mma.decompress_b"> { - let description = [{ - The `tcgen05.mma.decompress_b` operation is an asynchronous tensor core - instruction that decompresses the B matrix and performs matrix - multiplication, accumulation in a single fused operation. It targets - 5th-generation tensor cores, providing developers with fine-grained - control over execution and scheduling. - - ``` - D = A * B // if `enableInputD` is false - D = A * B + D // otherwise - ``` - - where: - - A is an `M x K` matrix in tensor memory or described using shared memory descriptor - - B is a `K x N` matrix described using shared memory descriptor - - D is an `M x N` accumulator matrix in tensor memory - - The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op - - - `idesc` is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) - - - `decompressBMetadata` is a decompress metadata of B matrix - - Optional Operands: - - `disableOutputLane` is a vector mask for selective output - * vector<4 x i32> when ctaGroup is CTA_1 - * vector<8 x i32> when ctaGroup is CTA_2 - - Required Attributes: - - `ctaGroup` specifies CTA group configuration - * cta_1: MMA will be performed on the current thread's CTA - * cta_2: MMA will be performed on the current thread and it's peer CTA - - Default Attributes: - - collectorOAp is a Tcgen05MMACollectorOp attribute with matrix A as the collector buffer - - collectorOpB is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma) - }]; - let arguments = (ins - NVVM_AnyType:$matrixD, - NVVM_AnyType:$matrixA, - NVVM_AnyType:$matrixB, - NVVM_AnyType:$idesc, - NVVM_AnyType:$enableInputD, - NVVM_AnyType:$decompressBMetadata, - Optional:$disableOutputLane, - NVVM_Tcgen05MMAKindAttr:$kind, - NVVM_CTAGroupKindAttr:$ctaGroup, - OptionalAttr:$collectorOpA, - OptionalAttr:$collectorOpB - ); - let results = (outs); -} -def NVVM_Tcgen05MMASparseOp : NVVM_Op<"tcgen05.mma.sp", [AttrSizedOperandSegments]> { - let description = [{ - The `tcgen05.mma.sp` operation is an asynchronous tensor core instruction - that performs matrix multiplication, accumulation with sparse `A` matrix in - a single fused operation. It targets 5th-generation tensor cores, providing - developers with fine-grained control over execution and scheduling. - - ``` - D = A * B + (D * 2^ -scaleInputD) // if `scaleInputD` is provided - D = A * B // if `enableInputD` is false - D = A * B + D // otherwise - ``` - - where: - - A is an `M x (K / 2)` matrix in tensor memory or described using shared memory descriptor - - B is a `K x N` matrix described using shared memory descriptor - - D is an `M x N` accumulator matrix in tensor memory - - sparseMetadata located in tensor memory specifies the mapping of the `K / 2` - non-zero elements to the K elements before performing the MMA operation - - Other attributes and operands are similar to that of tcgen05.mma Op - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-sp) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$matrixD, - NVVM_LLVMPointerInAddressSpace62:$matrixA, - NVVM_T64BitSignlessInteger:$matrixB, - NVVM_T32BitSignlessInteger:$idesc, - NVVM_T1BitSignlessInteger:$enableInputD, - NVVM_LLVMPointerInAddressSpace6:$sparseMetadata, - Optional:$scaleInputD, - Optional:$disableOutputLane, - NVVM_Tcgen05MMAKindAttr:$kind, - NVVM_CTAGroupKindAttr:$ctaGroup, - OptionalAttr:$collectorOp, - OptionalAttr:$collectorOpB, - UnitAttr:$aShift - ); - let results = (outs); -} -def NVVM_Tcgen05MMASparseBlockScaleOp : NVVM_Op<"tcgen05.mma.sp.block_scale"> { - let description = [{ - The `tcgen05.mma.sp.block_scale` operation is an asynchronous tensor core - instruction that performs matrix multiplication, accumulation with block - scaling, and sparse `A` matrix in a single fused operation. It targets - 5th-generation tensor cores, providing developers with fine-grained control - over execution, and scheduling. - - ``` - D = (A * scale_a) * (B * scale_b) // if `enableInputD` is specified - D = (A * scale_a) * (B * scale_b) + D // otherwise - ``` - - where: - - A is an M x (K / 2) matrix in tensor memory or described using shared memory descriptor - - B is a K x N matrix described using shared memory descriptor - - D is an M x N accumulator matrix in tensor memory - - `scale_a` and `scale_b` are matrices in tensor memory used to scale `A` and `B` respectively - - Other attributes and operands are similar to that of tcgen05.mma.block_scale Op - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-sp) - }]; - let arguments = (ins - NVVM_AnyType:$matrixD, - NVVM_AnyType:$matrixA, - NVVM_AnyType:$matrixB, - NVVM_AnyType:$idesc, - NVVM_AnyType:$enableInputD, - NVVM_AnyType:$sparseMetadata, - NVVM_AnyType:$scaleA, - NVVM_AnyType:$scaleB, - NVVM_Tcgen05MMAKindAttr:$kind, - NVVM_CTAGroupKindAttr:$ctaGroup, - OptionalAttr:$blockScale, - OptionalAttr:$collectorOp, - OptionalAttr:$collectorOpB - ); - let results = (outs); -} -def NVVM_Tcgen05MMAWsOp : NVVM_Op<"tcgen05.mma.ws"> { - let description = [{ - The `tcgen05.mma.ws` operation is an asynchronous tensor core instruction - that performs weight stationary convolution matrix multiplication, accumulation - in a single fused operation. It targets 5th-generation tensor cores, providing - developers with fine-grained control over execution, and scheduling. - - ``` - D = A * B` // if `enableInputD` is false - D = A * B + D` // otherwise - ``` - - where: - - A is an `M x K` matrix in tensor memory or described using shared memory descriptor - - B is a `K x N` matrix described using shared memory descriptor - - D is an `M x N` accumulator matrix in tensor memory - - The `shared memory descriptor` can be generated using `tcgen05.mma_smem_desc` Op - - - idesc is a 32-bit value representing the [Instruction Descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instruction-descriptor) - - Optional Operands: - - zeroColMask is a 64 bit value representing the [Zero-column mask descriptor](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-zero-column-mask-descriptor) - - Required Attributes: - - `kind` is a Tcgen05MMAKind attribute - - Default Valued Attributes: - - collectorBBuffer specifies collector buffer for matrix B: b0 (default), b1, b2, b3 - - - collectorOp is a Tcgen05MMACollectorOp attribute with matrix B as the collector buffer - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-ws) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$matrixD, - NVVM_LLVMPointerInAddressSpace62:$matrixA, - NVVM_T64BitSignlessInteger:$matrixB, - NVVM_T32BitSignlessInteger:$idesc, - NVVM_T1BitSignlessInteger:$enableInputD, - Optional:$zeroColMask, - NVVM_Tcgen05MMAKindAttr:$kind, - OptionalAttr:$collectorBBuffer, - OptionalAttr:$collectorOp - ); - let results = (outs); -} -def NVVM_Tcgen05MMAWsSparseOp : NVVM_Op<"tcgen05.mma.ws.sp"> { - let description = [{ - The `tcgen05.mma.ws.sp` operation is an asynchronous tensor core instruction - that performs weight stationary convolution matrix multiplication, accumulation - with sparse `A` matrix in a single fused operation. It targets 5th-generation - tensor cores, providing developers with fine-grained control over execution, - and scheduling. - - ``` - D = A * B` // if `enableInputD` is false - D = A * B + D` // otherwise - ``` - - where: - - A is an M x (K / 2) matrix in memory or descriptor format - - B is a K x N matrix - - D is an M x N accumulator matrix - - sparseMetadata located in tensor memory specifies the mapping of the `K / 2` - non-zero elements to the K elements before performing the MMA operation - - Other attributes and operands are similar to that of tcgen05.mma.ws Op - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-mma-instructions-mma-ws-sp) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$matrixD, - NVVM_LLVMPointerInAddressSpace62:$matrixA, - NVVM_T64BitSignlessInteger:$matrixB, - NVVM_T32BitSignlessInteger:$idesc, - NVVM_T1BitSignlessInteger:$enableInputD, - NVVM_LLVMPointerInAddressSpace6:$sparseMetadata, - Optional:$zeroColMask, - NVVM_Tcgen05MMAKindAttr:$kind, - OptionalAttr:$collectorBBuffer, - OptionalAttr:$collectorOp - ); - let results = (outs); -} -def NVVM_Tcgen05MmaSmemDescOp : NVVM_Op<"tcgen05.mma_smem_desc", [MemoryEffects<[MemRead]>]> { - let description = [{ - The `nvvm.tcgen05_mma_smem_desc` constructs a Shared Memory descriptor - for tcgen05.mma. This descriptor is a 64-bit value which describes the - properties of multiplicand matrix in shared memory including its location - in the shared memory of the current CTA. - - ``` - +-----------+------+------------------------------------------------------+ - | Bit-field | Size | Description | - +-----------+------+------------------------------------------------------+ - | 0-13 | 14 | Matrix start address | - | 14-15 | 2 | Reserved | - | 16-29 | 14 | Leading dim relative-offset (or) absolute-address | - | 30-31 | 2 | Reserved | - | 32-45 | 14 | Stride dimension byte offset | - | 46-48 | 3 | Fixed constant value of 0b001 | - | 49-51 | 3 | Matrix base offset | - | 52 | 1 | Leading dimension stride mode: | - | | | 0: byte offset relative | - | | | 1: byte address absolute | - | 53-60 | 8 | Fixed constant value of 0xb00000000 | - | 61-63 | 3 | Swizzling mode: | - | | | 0: No swizzling | - | | | 1: 128-Byte with 32B atomic swizzling | - | | | 2: 128-Byte swizzling | - | | | 4: 64-Byte swizzling | - | | | 6: 32-Byte swizzling | - | | | (Values 3, 5 and 7 are invalid) | - +-----------+------+------------------------------------------------------+ - ``` - - Example: - ```mlir - %desc = nvvm.tcgen05.mma_smem_desc (%startAddr, %leadingDimOffset, %strideDimOffset, - %baseOffset, %leadingDimMode, %swizzleMode) : (i32, i32, i32, i8, i1, i8) -> i64 - ``` - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-shared-memory-descriptor) - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$startAddr, - NVVM_T32BitSignlessInteger:$leadingDimOffset, - NVVM_T32BitSignlessInteger:$strideDimOffset, - NVVM_T8BitSignlessInteger2:$baseOffset, - NVVM_T1BitSignlessInteger:$leadingDimMode, - NVVM_T8BitSignlessInteger2:$swizzleMode - ); - let results = (outs - NVVM_T64BitSignlessInteger2:$res - ); -} -def NVVM_Tcgen05RelinquishAllocPermitOp : NVVM_Op<"tcgen05.relinquish_alloc_permit"> { - let description = [{ - The `tcgen05.relinquish_alloc_permit` Op specifies that the CTA - of the executing thread is relinquishing the right to allocate - Tensor Memory. So, it is illegal for a CTA to perform `tcgen05.alloc` - after any of its constituent threads execute `tcgen05.relinquish_alloc_permit`. - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-memory-alloc-manage-instructions) - }]; - let arguments = (ins - OptionalAttr:$group - ); - let results = (outs); -} -def NVVM_Tcgen05ShiftOp : NVVM_Op<"tcgen05.shift"> { - let description = [{ - The `tcgen05.shift` is an asynchronous instruction which initiates - the shifting of 32-byte elements downwards across all the rows, - except the last, by one row. The operand `taddr` specifies the base - address of the matrix in Tensor Memory whose rows must be down shifted. - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-shift) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$taddr, - OptionalAttr:$group - ); - let results = (outs); -} -def NVVM_Tcgen05StOp : NVVM_Op<"tcgen05.st"> { - let description = [{ - Instruction `tcgen05.st` asynchronously stores data from the source register `r` - into the Tensor Memory at the location specified by the 32-bit address operand - `tmemAddr`, collectively across all threads of the warps. - - The `shape` and the `num` attribute together determines the total dimension of - the data which is stored to the Tensor Memory. The `shape` indicates the base - dimension of data to be accessed. The `num` attribute indicates the repeat - factor on the base dimension resulting in the total dimension of the data that - is accessed. - - The shape `16x32bx2` performs two accesses into Tensor Memory of the shape - `16x32b`. The base address of the first access is specified by `tmemAddr` - and the base address of the second access is specified by - `tmemAddr + offset`, where `offset` is an immediate argument. - - The unit attribute `unpack` can be used to unpack a 32-bit element - in the register into two 16-bit elements and store them in adjacent columns. - - The following table describes the size of the vector for various combinations - of `num` and `shape` attributes: - ``` - |=====================================================================| - | num/shape | 16x32bx2/16x64b/32x32b | 16x128b | 16x256b | - |=====================================================================| - | x1 | 1 | 2 | 4 | - | x2 | 2 | 4 | 8 | - | x4 | 4 | 8 | 16 | - | x8 | 8 | 16 | 32 | - | x16 | 16 | 32 | 64 | - | x32 | 32 | 64 | 128 | - | x64 | 64 | 128 | NA | - | x128 | 128 | NA | NA | - |=====================================================================| - ``` - - Example: - ```mlir - nvvm.tcgen05.st %tmemAddr, %val, %offset unpack { - shape = #nvvm.tcgen05_ldst_shape, - } : <2xi32> - ``` - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-st) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace6:$tmemAddr, - NVVM_T32BitSignlessIntegerOrVector2:$val, - Optional:$offset, - UnitAttr:$unpack, - NVVM_Tcgen05LdStShapeAttr:$shape - ); - let results = (outs); -} -def NVVM_Tcgen05WaitOp : NVVM_Op<"tcgen05.wait"> { - let description = [{ - The `tcgen05.wait` causes the executing thread to block until - all prior `tcgen05.ld` operations issued by the executing thread - have completed. Similarly, the `tcgen05.wait` causes the executing - thread to block until all prior `tcgen05.st` operations issued by the - executing thread have completed. - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#tcgen05-instructions-tcgen05-wait) - }]; - let arguments = (ins - NVVM_Tcgen05WaitKindAttr:$kind - ); - let results = (outs); -} -def NVVM_Tcgen05MmaSmemDescOpV2 : NVVM_Op<"tcgen05_mma_smem_desc_v2", [MemoryEffects<[MemRead]>]> { - let description = [{ - The `nvvm.tcgen05_mma_smem_desc_v2` constructs a Shared Memory descriptor for use as the MMA - Shared Memory operand. It configures the memory layout according to the provided parameters. - - The descriptor contains the following bitfields: - - Bits 63-61 (3 bits): Swizzle descriptor type - 0 = SWIZZLE_NONE matrix descriptor - 1 = SWIZZLE_128B_ATOM_32B - 2 = SWIZZLE_128B matrix descriptor - 3 = RESERVED - 4 = SWIZZLE_64B descriptor - 5 = RESERVED - 6 = SWIZZLE_32B descriptor - 7 = RESERVED - - Bits 60-54 (7 bits): Reserved - - Bit 53 (1 bit): UTCQMMA.LUTB Matrix B K_Segment start offset in K=128 (48B) block - 0: K[0:63] = GDESC.start_address + 0B - 1: K[64:127] = GDESC.start_address + 24B - - Bit 52 (1 bit): Matrix A or B leading dimension mode - 0 = BYTE_OFFSET (legacy) - 1 = NEXT_BUFFER_ADDR (circular buffer) - - Bits 51-49 (3 bits): Matrix A or B base offset (valid only for matrix descriptor 1, 2 or 3, 4) - - Bits 48-46 (3 bits): Matrix A or B descriptor version, needs to be set to 1 - - Bits 45-32 (14 bits): Matrix A or B stride dimension byte offset, 16 byte aligned, 4 LSBs not included - - Bit 31 (1 bit): Reserved - - Bits 30-16 (15 bits): Matrix A or B leading dimension byte offset or next buffer start address - If LEADING_DIM_MODE == BYTE_OFFSET: Matrix A or B leading dimension byte offset - If LEADING_DIM_MODE == NEXT_BUFFER_ADDR: Matrix A or B next buffer start address - Offset or address is 16 byte aligned, 4 LSBs not included - - Bit 15 (1 bit): Reserved - - Bits 14-0 (15 bits): Matrix A or B start address, 16 byte aligned, 4 LSBs not included - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$startAddress, - NVVM_T32BitSignlessInteger:$leadingDimOffset, - NVVM_T32BitSignlessInteger:$strideDimOffset, - NVVM_T8BitSignlessInteger2:$descriptorVersion, - NVVM_T8BitSignlessInteger2:$baseOffset, - NVVM_T1BitSignlessInteger:$leadingDimMode, - NVVM_T1BitSignlessInteger:$kSegmentOffset, - NVVM_T8BitSignlessInteger2:$swizzleType - ); - let results = (outs - NVVM_T64BitSignlessInteger2:$res - ); -} -def NVVM_TensormapCpFenceProxyOp : NVVM_Op<"tensormap.cp_fenceproxy"> { - let description = [{ - The `tensormap.cp_fenceproxy` is a fused copy and fence operation. - Instructions perform the following operations in order: - - Copies data of size specified by the `size` argument, in bytes, from the - location specified by the address operand `src` in shared memory to the - location specified by the address operand `dst` in the global memory, in - the generic proxy. - - Establishes a uni-directional proxy release pattern on the ordering from - the copy operation to the subsequent access performed in the tensormap - proxy on the address `dst` - - The valid value of `size` operand is 128 and must be an immediate. - - The operands `src` and `dst` specify non-generic addresses in shared::cta and - global state space respectively - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace1:$dst, - NVVM_LLVMPointerInAddressSpace3:$src, - NVVM_T32BitSignlessInteger:$size, - NVVM_MemScopeKindAttr:$scope - ); - let results = (outs); -} -def NVVM_TensormapReplaceOp : NVVM_Op<"tensormap.replace"> { - let description = [{ - The `nvvm.tensormap.replace` replaces the specified field of the tensor-map - object at the location specified by `addr` with a new value (specified by - `new_value` or `new_value_attr`). - - The `field` argument specifies the field of the tensor-map object to - replace. - - `new_value` is an `i32`/`i64` argument that specifies the new value to - replace the `field` with for the `global_address`, `rank`, `box_dim`, - `global_dim`, `global_stride`, and `element_stride` fields. It must be an - `i64` for the `global_address` and `global_stride` fields and `i32` for the - remaining fields. - - For `rank`, `new_value` must be one less than the desired tensor rank as - this field uses zero-based numbering. - - `new_value_attr` is an attribute that specifies the new value to replace - the `field` with for the `elemtype`, `interleave_layout`, `swizzle_mode`, - `swizzle_atomicity`, and `fill_mode` fields. It takes the place of - `new_value` for these fields. It must be a valid attribute corresponding to - the `field` type. - - The ordinal `ord` is an immediate integer argument that specifies the - ordinal of the `field` across the tensor which needs to be replaced and is - required only for the `box_dim`, `global_dim`, `global_stride`, and - `element_stride` fields. - - [For more information, see PTX ISA.](https://docs.nvidia.com/cuda/parallel-thread-execution/#data-movement-and-conversion-instructions-tensormap-replace) - }]; - let arguments = (ins - NVVM_LLVMPointerInAddressSpace14:$addr, - Optional:$new_value, - NVVM_TensormapFieldAttr:$field, - OptionalAttr:$ord, - OptionalAttr:$new_value_attr - ); - let results = (outs); -} -def NVVM_TraceMarkOp : NVVM_Op<"trace.mark", [AttrSizedOperandSegments]> { - let description = [{ - The `nvvm.trace.mark` operation introduces trace events for profiling - programs. - - It takes `event_type`, a 12-bit value in the range 0-4095, which denotes - the type of the event. Additionally, it takes `domain` and `event`, which - are strings that identify the domain and event name of the trace event. - Finally, it takes an optional 32-bit or 64-bit payload, an optional - payload descriptor (required when payload is provided), and a predicate to - conditionally execute the trace event. - - Examples: - - ```mlir - // Basic trace event - nvvm.trace.mark event_type = 42, domain = "myDomain", event = "myEvent" - - // With 32-bit payload - nvvm.trace.mark event_type = 42, payload = %p, - domain = "myDomain", event = "myEvent", payload_descriptor = "counter" - : i32 - - // Conditional trace event - nvvm.trace.mark event_type = 42, predicate = %pred, - domain = "myDomain", event = "myEvent" - ``` - }]; - let arguments = (ins - Optional:$payload, - Optional:$predicate, - I32Attr:$eventType, - StrAttr:$domain, - StrAttr:$event, - OptionalAttr:$payloadDescriptor - ); - let results = (outs); -} -def NVVM_VoteSyncOp : NVVM_InferredOp<"vote.sync"> { - let description = [{ - The `vote.sync` op will cause executing thread to wait until all non-exited - threads corresponding to membermask have executed `vote.sync` with the same - qualifiers and same membermask value before resuming execution. - - The vote operation kinds are: - - `any`: True if source predicate is True for some thread in membermask. - - `all`: True if source predicate is True for all non-exited threads in - membermask. - - `uni`: True if source predicate has the same value in all non-exited - threads in membermask. - - `ballot`: In the ballot form, the destination result is a 32 bit integer. - In this form, the predicate from each thread in membermask are copied into - the corresponding bit position of the result, where the bit position - corresponds to the thread's lane id. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/#parallel-synchronization-and-communication-instructions-vote-sync) - }]; - let arguments = (ins - NVVM_T32BitSignlessInteger:$mask, - NVVM_T1BitSignlessInteger:$pred, - NVVM_VoteSyncKindAttr:$kind - ); - let results = (outs - NVVM_T32BitSignlessIntegerOr1:$res - ); -} -def NVVM_WgmmaGroupSyncAlignedOp : NVVM_Op<"wgmma.commit.group.sync.aligned"> { - let description = [{ - Commits all prior uncommitted warpgroup level matrix multiplication operations. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-commit-group) - }]; - let arguments = (ins); - let results = (outs); -} -def NVVM_WgmmaFenceAlignedOp : NVVM_Op<"wgmma.fence.aligned"> { - let description = [{ - Enforce an ordering of register accesses between warpgroup level matrix - multiplication and other operations. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-fence) - }]; - let arguments = (ins); - let results = (outs); -} -def NVVM_WgmmaMmaAsyncOp : NVVM_Op<"wgmma.mma_async"> { - let description = [{ - The warpgroup (128 threads) level matrix multiply and accumulate operation - has either of the following forms, where matrix D is called accumulator: - D = A * B + D - D = A * B, where the input from accumulator D is disabled. - - Supported shapes: - ``` - |--------------|--------------|------------|--------------|---------------| - | | | | |f16+=e4m3*e4m3 | - | | | | |f16+=e5m2*e5m2 | - |f32+=tf32*tf32|f16+=f16 *f16 | s32+=s8*s8 |s32 += b1 * b1|f16+=e5m2*e4m3 | - | |f32+=f16 *f16 | s32+=u8*u8 | |f16+=e4m3*e5m2 | - | |f32+=bf16*bf16| s32+=u8*u8 | |f16+=e4m3*e5m2 | - | |f32+=bf16*bf16| s32+=s8*u8 | |f32+=e4m3*e4m3 | - | | | s32+=u8*s8 | |f32+=e5m2*e5m2 | - | | | | |f32+=e4m3*e5m2 | - | | | | |f32+=e4m3*e5m2 | - |--------------|--------------|------------|--------------|---------------| - | .m64n8k8 | .m64n8k16 | .m64n8k32 | .m64n8k256 | .m64n8k32 | - | .m64n16k8 | .m64n16k16 | .m64n16k32 | .m64n16k256 | .m64n16k32 | - | .m64n24k8 | .m64n24k16 | .m64n24k32 | .m64n24k256 | .m64n24k32 | - | .m64n32k8 | .m64n32k16 | .m64n32k32 | .m64n32k256 | .m64n32k32 | - | .m64n40k8 | .m64n40k16 | .m64n48k32 | .m64n48k256 | .m64n40k32 | - | .m64n48k8 | .m64n48k16 | .m64n64k32 | .m64n64k256 | .m64n48k32 | - | .m64n56k8 | .m64n56k16 | .m64n80k32 | .m64n80k256 | .m64n56k32 | - | .m64n64k8 | .m64n64k16 | .m64n96k32 | .m64n96k256 | .m64n64k32 | - | .m64n72k8 | .m64n72k16 | .m64n112k32| .m64n112k256 | .m64n72k32 | - | .m64n80k8 | .m64n80k16 | .m64n128k32| .m64n128k256 | .m64n80k32 | - | .m64n88k8 | .m64n88k16 | .m64n144k32| .m64n144k256 | .m64n88k32 | - | .m64n96k8 | .m64n96k16 | .m64n160k32| .m64n160k256 | .m64n96k32 | - | .m64n104k8 | .m64n104k16 | .m64n176k32| .m64n176k256 | .m64n104k32 | - | .m64n112k8 | .m64n112k16 | .m64n192k32| .m64n192k256 | .m64n112k32 | - | .m64n120k8 | .m64n120k16 | .m64n208k32| .m64n208k256 | .m64n120k32 | - | .m64n128k8 | .m64n128k16 | .m64n224k32| .m64n224k256 | .m64n128k32 | - | .m64n136k8 | .m64n136k16 | .m64n240k32| .m64n240k256 | .m64n136k32 | - | .m64n144k8 | .m64n144k16 | .m64n256k32| .m64n256k256 | .m64n144k32 | - | .m64n152k8 | .m64n152k16 | | | .m64n152k32 | - | .m64n160k8 | .m64n160k16 | | | .m64n160k32 | - | .m64n168k8 | .m64n168k16 | | | .m64n168k32 | - | .m64n176k8 | .m64n176k16 | | | .m64n176k32 | - | .m64n184k8 | .m64n184k16 | | | .m64n184k32 | - | .m64n192k8 | .m64n192k16 | | | .m64n192k32 | - | .m64n200k8 | .m64n200k16 | | | .m64n200k32 | - | .m64n208k8 | .m64n208k16 | | | .m64n208k32 | - | .m64n216k8 | .m64n216k16 | | | .m64n216k32 | - | .m64n224k8 | .m64n224k16 | | | .m64n224k32 | - | .m64n232k8 | .m64n232k16 | | | .m64n232k32 | - | .m64n240k8 | .m64n240k16 | | | .m64n240k32 | - | .m64n248k8 | .m64n248k16 | | | .m64n248k32 | - | .m64n256k8 | .m64n256k16 | | | .m64n256k32 | - |--------------|--------------|------------|--------------|---------------| - ``` - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions) - }]; - let arguments = (ins - NVVM_LLVMStructureType:$inouts, - NVVM_T64BitSignlessInteger:$descriptorA, - NVVM_T64BitSignlessInteger:$descriptorB, - NVVM_MMAShapeAttr:$shape, - NVVM_WGMMATypesAttr:$typeA, - NVVM_WGMMATypesAttr:$typeB, - NVVM_WGMMATypesAttr:$typeD, - NVVM_WGMMAScaleOutAttr:$scaleD, - NVVM_WGMMAScaleInAttr:$scaleA, - NVVM_WGMMAScaleInAttr:$scaleB, - NVVM_MMALayoutAttr:$layoutA, - NVVM_MMALayoutAttr:$layoutB, - OptionalAttr:$satfinite - ); - let results = (outs - NVVM_LLVMStructureType:$results_ - ); -} -def NVVM_WgmmaWaitGroupSyncOp : NVVM_Op<"wgmma.wait.group.sync.aligned"> { - let description = [{ - Signal the completion of a preceding warpgroup operation. - - [For more information, see PTX ISA](https://docs.nvidia.com/cuda/parallel-thread-execution/index.html#asynchronous-warpgroup-level-matrix-instructions-wgmma-wait-group) - }]; - let arguments = (ins - I64Attr:$group - ); - let results = (outs); -} -def NVVM_WMMALoadOp : NVVM_Op<"wmma.load"> { - let arguments = (ins - NVVM_LLVMPointerType:$ptr, - NVVM_T32BitSignlessInteger:$stride, - I32Attr:$m, - I32Attr:$n, - I32Attr:$k, - NVVM_MMALayoutAttr:$layout, - NVVM_MMATypesAttr:$eltype, - NVVM_MMAFragAttr:$frag - ); - let results = (outs - NVVM_LLVMStructureTypeOr64Bit:$res - ); -} -def NVVM_WMMAMmaOp : NVVM_Op<"wmma.mma"> { - let arguments = (ins - Variadic:$args, - I32Attr:$m, - I32Attr:$n, - I32Attr:$k, - NVVM_MMALayoutAttr:$layoutA, - NVVM_MMALayoutAttr:$layoutB, - NVVM_MMATypesAttr:$eltypeA, - NVVM_MMATypesAttr:$eltypeB - ); - let results = (outs - NVVM_LLVMStructureType:$res - ); -} -def NVVM_WMMAStoreOp : NVVM_Op<"wmma.store"> { - let arguments = (ins - NVVM_LLVMPointerType:$ptr, - Variadic:$args, - NVVM_T32BitSignlessInteger:$stride, - I32Attr:$m, - I32Attr:$n, - I32Attr:$k, - NVVM_MMALayoutAttr:$layout, - NVVM_MMATypesAttr:$eltype - ); - let results = (outs); -} - -#endif // MLIR_DIALECTS_NVVM_NVVMOPS_TD diff --git a/mlir/dialects/nvvm/capi.cc b/mlir/dialects/nvvm/capi.cc deleted file mode 100644 index ff71af81..00000000 --- a/mlir/dialects/nvvm/capi.cc +++ /dev/null @@ -1,1964 +0,0 @@ -//===- capi.cc - C API for CuTe's nvvm dialect ----------------------------===// - -#include "capi.h" - -#include "mlir/dialects/nvvm/NVVMDialect.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" -#include "mlir/CAPI/Support.h" - -// Registration - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(CuteNVVM, cute_nvvm, - mlir::cutlass_compiler::nvvm::NVVMDialect) - -// Attributes - -namespace nvvm = mlir::cutlass_compiler::nvvm; - -// Unwraps `handle` into `out`: false when it is of another context or not a -// T, or when it is null and not optional. -template -static bool unwrapAs(mlir::MLIRContext *ctx, H handle, bool optional, T &out) { - auto value = unwrap(handle); - if (!value) - return optional; - if (value.getContext() != ctx) - return false; - out = llvm::dyn_cast(value); - return static_cast(out); -} - -bool mlirAttributeIsACuteNVVMAtomicOpKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeAtomicOpKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::AtomicOpKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMAtomicOpKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMBarrierReduction(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduction(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::BarrierReductionAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMBarrierReductionAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMBarrierReduxKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeBarrierReduxKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::BarrierReduxKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMBarrierReduxKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMBlockScaleFormat(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeBlockScaleFormat(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::BlockScaleFormatAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMBlockScaleFormatAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMCTAGroupKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeCTAGroupKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::CTAGroupKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMCTAGroupKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMCVTPackFloatKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeCVTPackFloatKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::CVTPackFloatKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMCVTPackFloatKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMCacheEvictionPriority(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeCacheEvictionPriority(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::CacheEvictionPriorityAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMCacheEvictionPriorityAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMClusterLaunchControlQueryType(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute -mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeClusterLaunchControlQueryType( - value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::ClusterLaunchControlQueryTypeAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVVMCompareOpKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeCompareOpKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::CompareOpKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMCompareOpKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMConvertFP4Type(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP4Type(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::ConvertFP4TypeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMConvertFP4TypeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMConvertFP8Type(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeConvertFP8Type(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::ConvertFP8TypeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMConvertFP8TypeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMConvertScaleKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeConvertScaleKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::ConvertScaleKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMConvertScaleKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMDotAccumulateType(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeDotAccumulateType(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::DotAccumulateTypeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMDotAccumulateTypeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMEvictKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMEvictKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeEvictKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::EvictKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMEvictKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMFPRoundingMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeFPRoundingMode(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::FPRoundingModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMFPRoundingModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMGridDepActionKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeGridDepActionKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::GridDepActionKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMGridDepActionKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMIntegerRoundingMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeIntegerRoundingMode(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::IntegerRoundingModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMIntegerRoundingModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVML2PrefetchSize(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeL2PrefetchSize(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::L2PrefetchSizeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVML2PrefetchSizeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMLdStMatrixEltType(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeLdStMatrixEltType(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::LdStMatrixEltTypeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMLoadCacheModifierExtKind(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierExtKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::LoadCacheModifierExtKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMLoadCacheModifierKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeLoadCacheModifierKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::LoadCacheModifierKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMLoadCacheModifierKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMLoadShape(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMLoadShapeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeLoadShape(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::LoadShapeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMLoadShapeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMLoadSrcFormat(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeLoadSrcFormat(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::LoadSrcFormatAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMLoadSrcFormatAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMBarrierLayout(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierLayout(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MBarrierLayoutAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMBarrierLayoutAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMBarrierPhase(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierPhase(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MBarrierPhaseAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMBarrierPhaseAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMBarrierScopeKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierScopeKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MBarrierScopeKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMBarrierScopeKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMBarrierSpaceKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierSpaceKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MBarrierSpaceKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMBarrierSpaceKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMBarrierTxnKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierTxnKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MBarrierTxnKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMBarrierTxnKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMBarrierWaitKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeMBarrierWaitKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MBarrierWaitKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMBarrierWaitKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMMAB1Op(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMMAB1OpAttrGet(MlirContext context, uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAB1Op(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MMAB1OpAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMMAB1OpAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMMABlockScaleKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeMMABlockScaleKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MMABlockScaleKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMMABlockScaleKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMMACtaCount(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMACtaCount(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MMACtaCountAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMMACtaCountAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMMAFrag(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMMAFragAttrGet(MlirContext context, uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAFrag(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MMAFragAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMMAFragAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMMAIntOverflow(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeMMAIntOverflow(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MMAIntOverflowAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMMAIntOverflowAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMMAKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMMAKindAttrGet(MlirContext context, uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMAKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MMAKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMMAKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMMALayout(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMMALayoutAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMALayout(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MMALayoutAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMMALayoutAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMMATypes(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMMATypesAttrGet(MlirContext context, uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMMATypes(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MMATypesAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMMATypesAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMatchSyncKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeMatchSyncKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MatchSyncKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMatchSyncKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMemOrderKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeMemOrderKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MemOrderKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMemOrderKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMemScopeKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeMemScopeKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MemScopeKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMemScopeKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMMulMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMulModeAttrGet(MlirContext context, uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeMulMode(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::MulModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMMulModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMNVVMMemorySpace(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeNVVMMemorySpace(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::NVVMMemorySpaceAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMNVVMMemorySpaceAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMPermuteMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMPermuteModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizePermuteMode(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::PermuteModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMPermuteModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMPrefetchCacheLevel(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizePrefetchCacheLevel(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::PrefetchCacheLevelAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMPrefetchCacheLevelAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMProxyKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMProxyKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeProxyKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::ProxyKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMProxyKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMReductionKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMReductionKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeReductionKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::ReductionKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMReductionKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMReductionOp(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMReductionOpAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeReductionOp(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::ReductionOpAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMReductionOpAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMReductionType(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMReductionTypeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeReductionType(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::ReductionTypeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMReductionTypeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSPCompressElemSize(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressElemSize(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SPCompressElemSizeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSPCompressElemSizeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSPCompressFactorType(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressFactorType(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SPCompressFactorTypeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSPCompressFactorTypeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSPCompressIndexSize(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressIndexSize(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SPCompressIndexSizeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSPCompressIndexSizeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSPCompressOpKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressOpKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SPCompressOpKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSPCompressOpKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSPCompressRepFactor(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSPCompressRepFactor(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SPCompressRepFactorAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSPCompressRepFactorAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSPDecompressElemSize(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressElemSize(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SPDecompressElemSizeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSPDecompressElemSizeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSPDecompressFactorType(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressFactorType(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SPDecompressFactorTypeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSPDecompressIndexSize(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressIndexSize(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SPDecompressIndexSizeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSPDecompressRepFactor(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSPDecompressRepFactor(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SPDecompressRepFactorAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSPDecompressRepFactorAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSaturationMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSaturationModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSaturationMode(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SaturationModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSaturationModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSaturationModeKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSaturationModeKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SaturationModeKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSaturationModeKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMScaleVecSize(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeScaleVecSize(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::ScaleVecSizeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMScaleVecSizeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSetMaxRegisterAction(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSetMaxRegisterAction(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SetMaxRegisterActionAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSetMaxRegisterActionAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSharedSpace(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeSharedSpace(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SharedSpaceAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSharedSpaceAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMShflKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMShflKindAttrGet(MlirContext context, uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeShflKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::ShflKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMShflKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMSparsityFormat(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeSparsityFormat(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::SparsityFormatAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMSparsityFormatAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMStateSpace(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMStateSpaceAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeStateSpace(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::StateSpaceAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMStateSpaceAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMStoreCacheModifierKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeStoreCacheModifierKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::StoreCacheModifierKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMStoreCacheModifierKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMStoreShape(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMStoreShapeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeStoreShape(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::StoreShapeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMStoreShapeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTCBarParam(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTCBarParamAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTCBarParam(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::TCBarParamAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTCBarParamAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTMALoadMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTMALoadMode(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::TMALoadModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTMALoadModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTMAReduxKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTMAReduxKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::TMAReduxKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTMAReduxKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTMAStoreMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTMAStoreMode(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::TMAStoreModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTMAStoreModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTcgen05CpMulticast(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpMulticast(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::Tcgen05CpMulticastAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTcgen05CpMulticastAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTcgen05CpShape(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpShape(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::Tcgen05CpShapeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTcgen05CpShapeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTcgen05CpSrcFormat(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05CpSrcFormat(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::Tcgen05CpSrcFormatAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTcgen05FenceKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05FenceKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::Tcgen05FenceKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTcgen05FenceKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTcgen05LdStShape(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05LdStShape(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::Tcgen05LdStShapeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTcgen05LdStShapeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTcgen05MMABlockScale(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMABlockScale(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::Tcgen05MMABlockScaleAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorBBuffer( - value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::Tcgen05MMACollectorBBufferAttr::get(ctx, *valueValue)); -} -uint32_t -mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsACuteNVVMTcgen05MMACollectorOp(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMACollectorOp(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::Tcgen05MMACollectorOpAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTcgen05MMAKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05MMAKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::Tcgen05MMAKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTcgen05MMAKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTcgen05WaitKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTcgen05WaitKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::Tcgen05WaitKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTcgen05WaitKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTensormapElemtype(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTensormapElemtype(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::TensormapElemtypeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTensormapElemtypeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTensormapField(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTensormapField(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::TensormapFieldAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTensormapFieldAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTensormapFillMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTensormapFillMode(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::TensormapFillModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTensormapFillModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTensormapInterleaveLayout(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTensormapInterleaveLayout(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::TensormapInterleaveLayoutAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleAtomicity(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::TensormapSwizzleAtomicityAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTensormapSwizzleMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeTensormapSwizzleMode(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::TensormapSwizzleModeAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTensormapSwizzleModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMTmemLayout(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeTmemLayout(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::TmemLayoutAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMTmemLayoutAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMValidatePattern(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMValidatePatternAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeValidatePattern(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::ValidatePatternAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMValidatePatternAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMVoteSyncKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeVoteSyncKind(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::VoteSyncKindAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMVoteSyncKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMWGMMAScaleIn(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleIn(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::WGMMAScaleInAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMWGMMAScaleInAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMWGMMAScaleOut(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = - ::mlir::cutlass_compiler::nvvm::symbolizeWGMMAScaleOut(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::WGMMAScaleOutAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMWGMMAScaleOutAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMWGMMATypes(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - auto valueValue = ::mlir::cutlass_compiler::nvvm::symbolizeWGMMATypes(value); - if (!valueValue) - return {nullptr}; - return wrap(nvvm::WGMMATypesAttr::get(ctx, *valueValue)); -} -uint32_t mlirCuteNVVMWGMMATypesAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsACuteNVVMLdStMatrixShape(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, - int n) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(nvvm::LdStMatrixShapeAttr::get(ctx, m, n)); -} -int mlirCuteNVVMLdStMatrixShapeAttrGetM(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getM(); -} -int mlirCuteNVVMLdStMatrixShapeAttrGetN(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getN(); -} - -bool mlirAttributeIsACuteNVVMMMAShape(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, - int k) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(nvvm::MMAShapeAttr::get(ctx, m, n, k)); -} -int mlirCuteNVVMMMAShapeAttrGetM(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getM(); -} -int mlirCuteNVVMMMAShapeAttrGetN(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getN(); -} -int mlirCuteNVVMMMAShapeAttrGetK(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getK(); -} - -bool mlirAttributeIsACuteNVVMTarget(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} -MlirAttribute mlirCuteNVVMTargetAttrGet(MlirContext context, int O, - MlirStringRef triple, - MlirStringRef chip, - MlirStringRef features, - MlirAttribute flags, MlirAttribute link, - bool verifyTarget) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - ctx->getOrLoadDialect(); - ::mlir::DictionaryAttr flagsValue; - if (!unwrapAs(ctx, flags, true, flagsValue)) - return {nullptr}; - ::mlir::ArrayAttr linkValue; - if (!unwrapAs(ctx, link, true, linkValue)) - return {nullptr}; - return wrap(nvvm::TargetAttr::getChecked( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, ctx, O, - unwrap(triple), unwrap(chip), unwrap(features), flagsValue, linkValue, - verifyTarget)); -} -int mlirCuteNVVMTargetAttrGetO(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getO(); -} -MlirStringRef mlirCuteNVVMTargetAttrGetTriple(MlirAttribute attr) { - return wrap(llvm::cast(unwrap(attr)).getTriple()); -} -MlirStringRef mlirCuteNVVMTargetAttrGetChip(MlirAttribute attr) { - return wrap(llvm::cast(unwrap(attr)).getChip()); -} -MlirStringRef mlirCuteNVVMTargetAttrGetFeatures(MlirAttribute attr) { - return wrap(llvm::cast(unwrap(attr)).getFeatures()); -} -MlirAttribute mlirCuteNVVMTargetAttrGetFlags(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getFlags())); -} -MlirAttribute mlirCuteNVVMTargetAttrGetLink(MlirAttribute attr) { - return wrap(static_cast( - llvm::cast(unwrap(attr)).getLink())); -} -bool mlirCuteNVVMTargetAttrGetVerifyTarget(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getVerifyTarget(); -} diff --git a/mlir/dialects/nvvm/capi.h b/mlir/dialects/nvvm/capi.h deleted file mode 100644 index ee604770..00000000 --- a/mlir/dialects/nvvm/capi.h +++ /dev/null @@ -1,785 +0,0 @@ -//===- capi.h - C API for CuTe's nvvm dialect ---------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_NVVM_CAPI_H -#define MLIR_DIALECTS_NVVM_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Registration - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(CuteNVVM, cute_nvvm); - -// Attributes - -// Constructors return null for invalid parameters or handles of another -// context; optional parameters take null handles. Enums are their integer -// values (the dialect's *Enums.td). Getters require a value accepted by the -// matching IsA. - -// `#nvvm.atomic_op`: operations supported by atom instruction -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMAtomicOpKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMAtomicOpKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMAtomicOpKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.reduction`: NVVM barrier reduction operation -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMBarrierReduction(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMBarrierReductionAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMBarrierReductionAttrGetValue(MlirAttribute attr); - -// `#nvvm.barrier_redux_kind`: NVVM barrier redux kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMBarrierReduxKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMBarrierReduxKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMBarrierReduxKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.block_scale_format`: MMA Block Scale Format -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMBlockScaleFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMBlockScaleFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMBlockScaleFormatAttrGetValue(MlirAttribute attr); - -// `#nvvm.cta_group`: NVVM CTA group kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMCTAGroupKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMCTAGroupKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMCTAGroupKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.packfloat_type`: NVVM CVT Pack Float kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMCVTPackFloatKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMCVTPackFloatKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMCVTPackFloatKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.cache_eviction_priority`: NVVM Cache Eviction Priority -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMCacheEvictionPriority(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMCacheEvictionPriorityAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMCacheEvictionPriorityAttrGetValue(MlirAttribute attr); - -// `#nvvm.cluster_launch_control_query_type`: NVVM ClusterLaunchControlQueryType -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMClusterLaunchControlQueryType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.op`: Comparison operator encoding -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMCompareOpKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMCompareOpKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMCompareOpKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.convert_fp4_type`: NVVM ConvertFP4Type kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMConvertFP4Type(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMConvertFP4TypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMConvertFP4TypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.convert_fp8_type`: NVVM ConvertFP8Type kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMConvertFP8Type(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMConvertFP8TypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMConvertFP8TypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.convert_scale_kind`: NVVM ConvertScale kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMConvertScaleKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMConvertScaleKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMConvertScaleKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.dot_accumulate_type`: NVVM DotAccumulateType -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMDotAccumulateType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMDotAccumulateTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMDotAccumulateTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.evict_kind`: NVVM L2 Prefetch Size -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMEvictKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMEvictKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMEvictKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.fp_rnd_mode`: NVVM FPRoundingMode kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMFPRoundingMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMFPRoundingModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMFPRoundingModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.grid_dep_action`: Action kind for grid dependency control -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMGridDepActionKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMGridDepActionKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMGridDepActionKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.int_rnd_mode`: NVVM IntegerRoundingMode kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMIntegerRoundingMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMIntegerRoundingModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMIntegerRoundingModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.l2_prefetch`: NVVM L2 Prefetch Size -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVML2PrefetchSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVML2PrefetchSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVML2PrefetchSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.ld_st_matrix_elt_type`: Element type for ldmatrix and stmatrix -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMLdStMatrixEltType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMLdStMatrixEltTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.load_cache_modifier_ext`: NVVM load cache modifier kind(Ext) -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMLoadCacheModifierExtKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMLoadCacheModifierExtKindAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.load_cache_modifier`: NVVM load cache modifier kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMLoadCacheModifierKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMLoadCacheModifierKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMLoadCacheModifierKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.load_shape`: shape attribute for ldmatrix -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMLoadShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMLoadShapeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMLoadShapeAttrGetValue(MlirAttribute attr); - -// `#nvvm.load_src_format`: source format for ldmatrix -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMLoadSrcFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMLoadSrcFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMLoadSrcFormatAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbarrier_layout`: NVVM MBarrier Layout -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMMBarrierLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMBarrierLayoutAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMBarrierLayoutAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbarrier_phase`: NVVM mbarrier phase type -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMMBarrierPhase(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMBarrierPhaseAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMBarrierPhaseAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbar_scope`: NVVM MBarrier scope kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMMBarrierScopeKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMBarrierScopeKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMBarrierScopeKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbar_space`: NVVM MBarrier space kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMMBarrierSpaceKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMBarrierSpaceKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMBarrierSpaceKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbar_txn_kind`: NVVM MBarrier Transaction kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMMBarrierTxnKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMBarrierTxnKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMBarrierTxnKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mbar_wait`: NVVM MBarrier wait kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMMBarrierWaitKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMBarrierWaitKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMBarrierWaitKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_b1op`: MMA binary operations -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAB1Op(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAB1OpAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAB1OpAttrGetValue(MlirAttribute attr); - -// `#nvvm.block_scale_kind`: Block Scale Kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMMMABlockScaleKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMMABlockScaleKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMMABlockScaleKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_cta_count`: MMA CTA count -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMACtaCount(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMMACtaCountAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMMACtaCountAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_frag`: NVVM MMA frag type -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAFrag(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAFragAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAFragAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_int_overflow`: MMA overflow options -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMMMAIntOverflow(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMMAIntOverflowAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMMAIntOverflowAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_kind`: MMA operation kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMMAKindAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMMAKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_layout`: NVVM MMA layout -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMALayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMMALayoutAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMMALayoutAttrGetValue(MlirAttribute attr); - -// `#nvvm.mma_type`: NVVM MMA types -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMATypes(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMMATypesAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMMATypesAttrGetValue(MlirAttribute attr); - -// `#nvvm.match_sync_kind`: NVVM match sync kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMMatchSyncKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMatchSyncKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMatchSyncKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mem_order`: NVVM Memory Ordering kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMMemOrderKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMemOrderKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMemOrderKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mem_scope`: NVVM Memory Scope kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMMemScopeKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMemScopeKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMMemScopeKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.mul_mode`: multiply mode attribute -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMulMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMMulModeAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirCuteNVVMMulModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.memory_space`: NVVM Memory Space -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMNVVMMemorySpace(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMNVVMMemorySpaceAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMNVVMMemorySpaceAttrGetValue(MlirAttribute attr); - -// `#nvvm.permute_mode`: NVVM permute mode -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMPermuteMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMPermuteModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMPermuteModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.prefetch_cache_level`: NVVM Prefetch Cache Level -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMPrefetchCacheLevel(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMPrefetchCacheLevelAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMPrefetchCacheLevelAttrGetValue(MlirAttribute attr); - -// `#nvvm.proxy_kind`: Proxy kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMProxyKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMProxyKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMProxyKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.reduction_kind`: NVVM Reduction Kind attribute -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMReductionKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMReductionKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMReductionKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.red_op`: Ops supported by red instruction -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMReductionOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMReductionOpAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMReductionOpAttrGetValue(MlirAttribute attr); - -// `#nvvm.red_type`: types supported by red instruction -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMReductionType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMReductionTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMReductionTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spcomp_elem_size`: Sparse tensor compression element size -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSPCompressElemSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSPCompressElemSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSPCompressElemSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spcomp_factor`: Sparse tensor compression factor type -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSPCompressFactorType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSPCompressFactorTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSPCompressFactorTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spcomp_index_size`: Sparse tensor compression index size -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSPCompressIndexSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSPCompressIndexSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSPCompressIndexSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spcomp_op_kind`: spcompress operation kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSPCompressOpKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSPCompressOpKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSPCompressOpKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.spcomp_rep_factor`: Sparse tensor compression repetition factor -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSPCompressRepFactor(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSPCompressRepFactorAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSPCompressRepFactorAttrGetValue(MlirAttribute attr); - -// `#nvvm.spdecomp_elem_size`: Sparse tensor decompression element size -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSPDecompressElemSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSPDecompressElemSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSPDecompressElemSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spdecomp_factor`: Sparse tensor decompression factor type -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSPDecompressFactorType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSPDecompressFactorTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spdecomp_index_size`: Sparse tensor decompression index size -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSPDecompressIndexSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSPDecompressIndexSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.spdecomp_rep_factor`: Sparse tensor decompression repetition factor -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSPDecompressRepFactor(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSPDecompressRepFactorAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSPDecompressRepFactorAttrGetValue(MlirAttribute attr); - -// `#nvvm.sat_mode`: NVVM SaturationMode kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSaturationMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSaturationModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSaturationModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.sat`: NVVM SaturationMode kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSaturationModeKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSaturationModeKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSaturationModeKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.scale_vec_size`: MMA Scale Vector Sizes -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMScaleVecSize(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMScaleVecSizeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMScaleVecSizeAttrGetValue(MlirAttribute attr); - -// `#nvvm.action`: NVVM set max register action -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSetMaxRegisterAction(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSetMaxRegisterActionAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSetMaxRegisterActionAttrGetValue(MlirAttribute attr); - -// `#nvvm.shared_space`: Shared memory space -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMSharedSpace(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSharedSpaceAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSharedSpaceAttrGetValue(MlirAttribute attr); - -// `#nvvm.shfl_kind`: NVVM shuffle kind -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMShflKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMShflKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMShflKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.sparsity_format`: MMA Sparsity Format -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMSparsityFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMSparsityFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMSparsityFormatAttrGetValue(MlirAttribute attr); - -// `#nvvm.state_space`: NVVM State Space -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStateSpace(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMStateSpaceAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMStateSpaceAttrGetValue(MlirAttribute attr); - -// `#nvvm.store_cache_modifier`: NVVM store cache modifier kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMStoreCacheModifierKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMStoreCacheModifierKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMStoreCacheModifierKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.store_shape`: shape attribute for stmatrix -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMStoreShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMStoreShapeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMStoreShapeAttrGetValue(MlirAttribute attr); - -// `#nvvm.TCBarParam`: Cluster MMA Barrier Parameter Type -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTCBarParam(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTCBarParamAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTCBarParamAttrGetValue(MlirAttribute attr); - -// `#nvvm.tma_load_mode`: NVVM TMA Load Mode -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTMALoadMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTMALoadModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTMALoadModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tma_redux_kind`: NVVM TMA redux kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTMAReduxKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTMAReduxKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTMAReduxKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.tma_store_mode`: NVVM TMA Store Mode -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTMAStoreMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTMAStoreModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTMAStoreModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_cp_multicast`: tcgen05 cp multicast -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTcgen05CpMulticast(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTcgen05CpMulticastAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTcgen05CpMulticastAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_cp_shape`: tcgen05 cp shapes -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTcgen05CpShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTcgen05CpShapeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTcgen05CpShapeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_cp_src_fmt`: tcgen05 cp source format -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTcgen05CpSrcFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTcgen05CpSrcFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_fence`: NVVM Tcgen05 fence kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTcgen05FenceKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTcgen05FenceKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTcgen05FenceKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_ldst_shape`: allowed 32-bit signless integer cases: 0, 1, 2, -// 3, 4 -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTcgen05LdStShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTcgen05LdStShapeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTcgen05LdStShapeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_mma_block_scale`: tcgen05.mma block scale attribute -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTcgen05MMABlockScale(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTcgen05MMABlockScaleAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_mma_collectorb`: tcgen05 MMA Collector Buffer B Attribute -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_mma_collectorop`: tcgen05.mma Collector Buffer Operation -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTcgen05MMACollectorOp(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTcgen05MMACollectorOpAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_mma_kind`: tcgen05 MMA Supported Types -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTcgen05MMAKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTcgen05MMAKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTcgen05MMAKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.tcgen05_wait`: NVVM Tcgen05 wait kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTcgen05WaitKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTcgen05WaitKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTcgen05WaitKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_elemtype`: NVVM Tensormap Elemtype -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTensormapElemtype(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTensormapElemtypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTensormapElemtypeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_field`: NVVM Tensormap Field Kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTensormapField(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTensormapFieldAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTensormapFieldAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_fill_mode`: NVVM Tensormap Fill Mode -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTensormapFillMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTensormapFillModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTensormapFillModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_interleave_layout`: NVVM Tensormap Interleave Layout -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTensormapInterleaveLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapInterleaveLayoutAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_swizzle_atomicity`: NVVM Tensormap Swizzle Atomicity -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTensormapSwizzleAtomicityAttrGet( - MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(MlirAttribute attr); - -// `#nvvm.tensormap_swizzle_mode`: NVVM Tensormap Swizzle Mode -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMTensormapSwizzleMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTensormapSwizzleModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTensormapSwizzleModeAttrGetValue(MlirAttribute attr); - -// `#nvvm.TmemLayout`: Tensor Memory Layout Enumerated Type -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTmemLayout(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTmemLayoutAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMTmemLayoutAttrGetValue(MlirAttribute attr); - -// `#nvvm.validate_pattern`: NVVM validate data pattern -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMValidatePattern(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMValidatePatternAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMValidatePatternAttrGetValue(MlirAttribute attr); - -// `#nvvm.vote_sync_kind`: NVVM vote sync kind -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMVoteSyncKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMVoteSyncKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMVoteSyncKindAttrGetValue(MlirAttribute attr); - -// `#nvvm.wgmma_scale_in`: WGMMA overflow options -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMWGMMAScaleIn(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMWGMMAScaleInAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMWGMMAScaleInAttrGetValue(MlirAttribute attr); - -// `#nvvm.wgmma_scale_out`: WGMMA input predicate -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMWGMMAScaleOut(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMWGMMAScaleOutAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMWGMMAScaleOutAttrGetValue(MlirAttribute attr); - -// `#nvvm.wgmma_type`: NVVM WGMMA types -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMWGMMATypes(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMWGMMATypesAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirCuteNVVMWGMMATypesAttrGetValue(MlirAttribute attr); - -// `#nvvm.ld_st_matrix_shape`: Matrix shape of ldmatrix, stmatrix and movmatrix -MLIR_CAPI_EXPORTED bool -mlirAttributeIsACuteNVVMLdStMatrixShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMLdStMatrixShapeAttrGet(MlirContext context, int m, int n); -MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetM(MlirAttribute attr); -MLIR_CAPI_EXPORTED int mlirCuteNVVMLdStMatrixShapeAttrGetN(MlirAttribute attr); - -// `#nvvm.shape`: Shape of an MMA operation -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMMMAShape(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMMMAShapeAttrGet(MlirContext context, int m, int n, int k); -MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetM(MlirAttribute attr); -MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetN(MlirAttribute attr); -MLIR_CAPI_EXPORTED int mlirCuteNVVMMMAShapeAttrGetK(MlirAttribute attr); - -// `#nvvm.target`: GPU target of an NVVM module -MLIR_CAPI_EXPORTED bool mlirAttributeIsACuteNVVMTarget(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirCuteNVVMTargetAttrGet( - MlirContext context, int O, MlirStringRef triple, MlirStringRef chip, - MlirStringRef features, MlirAttribute flags, MlirAttribute link, - bool verifyTarget); -MLIR_CAPI_EXPORTED int mlirCuteNVVMTargetAttrGetO(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirCuteNVVMTargetAttrGetTriple(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirCuteNVVMTargetAttrGetChip(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirCuteNVVMTargetAttrGetFeatures(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTargetAttrGetFlags(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirCuteNVVMTargetAttrGetLink(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool -mlirCuteNVVMTargetAttrGetVerifyTarget(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_NVVM_CAPI_H diff --git a/mlir/dialects/nvvm/nvvm.zig b/mlir/dialects/nvvm/nvvm.zig deleted file mode 100644 index 14a0e8e2..00000000 --- a/mlir/dialects/nvvm/nvvm.zig +++ /dev/null @@ -1,311 +0,0 @@ -//! Builders for the NVVM operations CuTe kernels emit, over the bindings of -//! NVIDIA's `nvvm` dialect (`mlir/dialects/nvvm`, `ops`): the CuTe-DSL -//! compiler's own NVVM, which the upstream dialect differs from. Each builder -//! fixes the attributes the kernels do not choose. - -const std = @import("std"); - -const mlir = @import("mlir"); - -pub const ops = @import("ops.zig"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("cute_nvvm"); -} - -fn enumAttribute(comptime T: type, ctx: *mlir.Context, value: @FieldType(T.InitArgs, "value")) *const mlir.Attribute { - return (T.get(ctx, .{ .value = value }) catch unreachable).attribute(); -} - -/// `#nvvm.cta_group<...>`: CTAs cooperating on a tcgen05 operation. -pub const CtaGroup = ops.CTAGroupKind; -/// `#nvvm.tcgen05_fence<...>`: `tcgen05.fence::{before,after}_thread_sync`. -pub const Tcgen05FenceKind = ops.Tcgen05FenceKind; -/// `#nvvm.tcgen05_wait<...>`: `tcgen05.wait::{ld,st}`. -pub const Tcgen05WaitKind = ops.Tcgen05WaitKind; -/// `#nvvm.tcgen05_mma_kind<...>`: input types of `tcgen05.mma`. -pub const Tcgen05MmaKind = ops.Tcgen05MMAKind; -/// `#nvvm.tcgen05_ldst_shape<...>`: lane/bit shape of `tcgen05.ld` / `tcgen05.st`. -pub const Tcgen05LdStShape = ops.Tcgen05LdStShape; -/// `#nvvm.mem_scope<...>`: scope of a memory operation. -pub const MemScope = ops.MemScopeKind; -/// `#nvvm.mbar_wait<...>`: `test` (non-blocking) or `try` (may suspend) wait. -pub const MBarrierWaitKind = ops.MBarrierWaitKind; -/// `#nvvm.mbar_scope<...>`: scope of an mbarrier wait. -pub const MBarrierScope = ops.MBarrierScopeKind; -/// `#nvvm.proxy_kind<...>`: memory proxy of `fence.proxy`. -pub const ProxyKind = ops.ProxyKind; -/// `#nvvm.shared_space<...>`: `shared::cta` or `shared::cluster`. -pub const SharedSpace = ops.SharedSpace; - -// ============================================================================= -// Kernel attributes -// ============================================================================= - -/// `nvvm.reqntid`: the CTA shape (x, y, z) a kernel is always launched with. -pub fn reqntid(ctx: *mlir.Context, dims: [3]i32) mlir.NamedAttribute { - return .named(ctx, "nvvm.reqntid", .denseArray(ctx, .i32, &dims)); -} - -/// `nvvm.minctasm`: the number of CTAs a kernel must fit on one SM at once. -pub fn minctasm(ctx: *mlir.Context, ctas: i32) mlir.NamedAttribute { - return .named(ctx, "nvvm.minctasm", .int(ctx, .i32, ctas)); -} - -// ============================================================================= -// Special registers and thread synchronization -// ============================================================================= - -/// nvvm.read.ptx.sreg. — an i32 special register, e.g. `tid.x`, `ctaid.y`, -/// `cluster.ctarank`. -pub fn read_sreg(ctx: *mlir.Context, comptime name: []const u8, location: *const mlir.Location) *mlir.Operation { - const builder = comptime blk: { - var fn_name: [name.len]u8 = name[0..name.len].*; - std.mem.replaceScalar(u8, &fn_name, '.', '_'); - break :blk "read_ptx_sreg_" ++ fn_name; - }; - return @field(ops, builder)(ctx, .int(ctx, .i32), .{}, location); -} - -/// nvvm.barrier — `bar.sync 0` over the whole CTA. -pub fn barrier(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return ops.barrier(ctx, .{}, location); -} - -/// nvvm.bar.warp.sync — `bar.warp.sync mask` (i32 lane mask, -1 for the full warp). -pub fn bar_warp_sync(ctx: *mlir.Context, mask: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.bar_warp_sync(ctx, mask, location); -} - -/// nvvm.elect.sync — i1, true in one elected lane of the warp. -pub fn elect_sync(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return ops.elect_sync(ctx, .int(ctx, .i1), .{}, location); -} - -// ============================================================================= -// mbarrier -// ============================================================================= - -/// nvvm.mbarrier.init — initialize the mbarrier at `ptr` (`!llvm.ptr<3>`) to expect -/// `count` (i32) arrivals per phase. -pub fn mbarrier_init(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.mbarrier_init(ctx, ptr, count, .{ .layout = enumAttribute(ops.MBarrierLayoutAttr, ctx, .v0) }, location); -} - -/// nvvm.mbarrier.arrive — arrive `count` (i32) times (release semantics). -pub fn mbarrier_arrive(ctx: *mlir.Context, ptr: *const mlir.Value, count: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { - return ops.mbarrier_arrive( - ctx, - ptr, - .{ .count = count, .scope = enumAttribute(ops.MemScopeKindAttr, ctx, scope), .relaxed = .boolean(ctx, false) }, - location, - ); -} - -/// nvvm.mbarrier.arrive.expect_tx — arrive and expect `bytes` (i32) of asynchronous -/// transactions (SM90+). -pub fn mbarrier_arrive_expect_tx(ctx: *mlir.Context, ptr: *const mlir.Value, bytes: *const mlir.Value, scope: MemScope, location: *const mlir.Location) *mlir.Operation { - return ops.mbarrier_arrive_expect_tx( - ctx, - ptr, - bytes, - .{ .scope = enumAttribute(ops.MemScopeKindAttr, ctx, scope), .relaxed = .boolean(ctx, false) }, - location, - ); -} - -/// nvvm.mbarrier.wait.parity — i1, whether phase `parity` (i32) has completed -/// (`test`: never suspends; `try`: may suspend for a system-dependent time). -pub fn mbarrier_wait_parity( - ctx: *mlir.Context, - ptr: *const mlir.Value, - parity: *const mlir.Value, - kind: MBarrierWaitKind, - scope: MBarrierScope, - location: *const mlir.Location, -) *mlir.Operation { - return ops.mbarrier_wait_parity( - ctx, - ptr, - parity, - .int(ctx, .i1), - enumAttribute(ops.MBarrierWaitKindAttr, ctx, kind), - .{ .scope = enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope) }, - location, - ); -} - -/// nvvm.mbarrier.try_wait.parity — wait (looping in PTX) until phase `parity` (i32) -/// of the mbarrier completes, with CTA acquire semantics. `suspend_time` (i32) -/// is a suspend-time hint in nanoseconds per try. -pub fn mbarrier_try_wait_parity( - ctx: *mlir.Context, - ptr: *const mlir.Value, - parity: *const mlir.Value, - suspend_time: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return ops.mbarrier_try_wait_parity(ctx, ptr, parity, suspend_time, .{ .useIntrinsic = .boolean(ctx, false) }, location); -} - -/// One potentially suspending parity test with acquire semantics. A false -/// result means the timeout expired; callers that need completion must retry. -pub fn mbarrier_try_wait_parity_timelimit( - ctx: *mlir.Context, - ptr: *const mlir.Value, - parity: *const mlir.Value, - suspend_time: *const mlir.Value, - scope: MBarrierScope, - location: *const mlir.Location, -) *mlir.Operation { - return ops.mbarrier_try_wait_parity_timelimit( - ctx, - ptr, - parity, - suspend_time, - .int(ctx, .i1), - .{ .scope = enumAttribute(ops.MBarrierScopeKindAttr, ctx, scope), .order = enumAttribute(ops.MemOrderKindAttr, ctx, .acquire) }, - location, - ); -} - -/// nvvm.fence.mbarrier.init — make mbarrier initialization visible to the cluster. -pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return ops.fence_mbarrier_init(ctx, location); -} - -/// nvvm.fence.proxy — order memory accesses across proxies. -pub fn fence_proxy(ctx: *mlir.Context, kind: ProxyKind, space: SharedSpace, location: *const mlir.Location) *mlir.Operation { - return ops.fence_proxy(ctx, enumAttribute(ops.ProxyKindAttr, ctx, kind), .{ .space = enumAttribute(ops.SharedSpaceAttr, ctx, space) }, location); -} - -// ============================================================================= -// Asynchronous copies -// ============================================================================= - -/// nvvm.cp.async.commit.group — commit the pending `cp.async` copies as a group. -pub fn cp_async_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return ops.cp_async_commit_group(ctx, location); -} - -/// nvvm.cp.async.wait.group — wait until at most `pending` `cp.async` groups are in flight. -pub fn cp_async_wait_group(ctx: *mlir.Context, pending: u32, location: *const mlir.Location) *mlir.Operation { - return ops.cp_async_wait_group(ctx, .int(ctx, .i32, pending), location); -} - -/// nvvm.cp.async.bulk.commit.group — commit the pending bulk (TMA) copies as a group. -pub fn cp_async_bulk_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return ops.cp_async_bulk_commit_group(ctx, location); -} - -/// nvvm.cp.async.bulk.wait_group — wait until at most `pending` bulk groups are in -/// flight; with `read`, only until their sources have been read. -pub fn cp_async_bulk_wait_group(ctx: *mlir.Context, pending: u32, read: bool, location: *const mlir.Location) *mlir.Operation { - return ops.cp_async_bulk_wait_group(ctx, .int(ctx, .i32, pending), .{ .read = if (read) .unit(ctx) else null }, location); -} - -// ============================================================================= -// tcgen05 (SM100 tensor cores and tensor memory) -// ============================================================================= - -/// nvvm.tcgen05.alloc — allocate `columns` (i32) tensor-memory columns and write the -/// address to `holder` (`!llvm.ptr<3>`). -pub fn tcgen05_alloc(ctx: *mlir.Context, holder: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_alloc(ctx, holder, columns, .{}, location); -} - -/// nvvm.tcgen05.dealloc — free `columns` (i32) tensor-memory columns at `taddr` -/// (`!llvm.ptr<6>`). -pub fn tcgen05_dealloc(ctx: *mlir.Context, taddr: *const mlir.Value, columns: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_dealloc(ctx, taddr, columns, .{}, location); -} - -/// nvvm.tcgen05.fence — order tcgen05 operations around a thread synchronization. -pub fn tcgen05_fence(ctx: *mlir.Context, kind: Tcgen05FenceKind, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_fence(ctx, enumAttribute(ops.Tcgen05FenceKindAttr, ctx, kind), location); -} - -/// nvvm.tcgen05.wait — wait for the issued tensor-memory loads or stores. -pub fn tcgen05_wait(ctx: *mlir.Context, kind: Tcgen05WaitKind, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_wait(ctx, enumAttribute(ops.Tcgen05WaitKindAttr, ctx, kind), location); -} - -/// nvvm.tcgen05.commit — arrive on the mbarrier at `barrier_ptr` when the -/// tcgen05 MMA/copy operations issued by the executing thread with `group` -/// complete. An optional mask selects CTAs in the cluster whose barriers at -/// the same shared-memory offset receive one arrival each. -pub fn tcgen05_commit(ctx: *mlir.Context, barrier_ptr: *const mlir.Value, group: CtaGroup, multicast_mask: ?*const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_commit(ctx, barrier_ptr, .{ .multicastMask = multicast_mask, .group = enumAttribute(ops.CTAGroupKindAttr, ctx, group) }, location); -} - -/// nvvm.tcgen05.mma — D (tensor memory, `!llvm.ptr<6>`) = A (shared-memory -/// descriptor, i64) * B (descriptor, i64) [+ D when `accumulate` (i1)], with the -/// instruction descriptor `idesc` (i32). -pub fn tcgen05_mma( - ctx: *mlir.Context, - d: *const mlir.Value, - a_desc: *const mlir.Value, - b_desc: *const mlir.Value, - idesc: *const mlir.Value, - accumulate: *const mlir.Value, - kind: Tcgen05MmaKind, - cta_group: CtaGroup, - location: *const mlir.Location, -) *mlir.Operation { - return ops.tcgen05_mma( - ctx, - d, - a_desc, - b_desc, - idesc, - accumulate, - enumAttribute(ops.Tcgen05MMAKindAttr, ctx, kind), - enumAttribute(ops.CTAGroupKindAttr, ctx, cta_group), - .{}, - location, - ); -} - -/// nvvm.tcgen05.ld — load `result_type` (an i32 or vector of i32) from tensor memory -/// at `taddr` (`!llvm.ptr<6>`). -pub fn tcgen05_ld(ctx: *mlir.Context, taddr: *const mlir.Value, result_type: *const mlir.Type, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_ld(ctx, taddr, result_type, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), .{}, location); -} - -/// nvvm.tcgen05.st — store `value` (an i32 or vector of i32) to tensor memory at `taddr`. -pub fn tcgen05_st(ctx: *mlir.Context, taddr: *const mlir.Value, value: *const mlir.Value, shape: Tcgen05LdStShape, location: *const mlir.Location) *mlir.Operation { - return ops.tcgen05_st(ctx, taddr, value, enumAttribute(ops.Tcgen05LdStShapeAttr, ctx, shape), .{}, location); -} - -// ============================================================================= -// Clusters -// ============================================================================= - -/// nvvm.mapa — the address of `ptr` in the shared memory of cluster CTA `rank` (i32). -pub fn mapa(ctx: *mlir.Context, ptr: *const mlir.Value, rank: *const mlir.Value, result_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return ops.mapa(ctx, ptr, rank, result_type, location); -} - -/// nvvm.cluster.arrive — arrive on the cluster barrier. -pub fn cluster_arrive(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { - return ops.cluster_arrive(ctx, .{ .aligned = if (aligned) .unit(ctx) else null }, location); -} - -/// nvvm.cluster.wait — wait on the cluster barrier. -pub fn cluster_wait(ctx: *mlir.Context, aligned: bool, location: *const mlir.Location) *mlir.Operation { - return ops.cluster_wait(ctx, .{ .aligned = if (aligned) .unit(ctx) else null }, location); -} - -// ============================================================================= -// Math -// ============================================================================= - -/// nvvm.ex2 — `ex2.approx[.ftz].f32`. -pub fn ex2(ctx: *mlir.Context, value: *const mlir.Value, ftz: bool, location: *const mlir.Location) *mlir.Operation { - return ops.ex2(ctx, value, .float(ctx, .f32), .{ .ftz = .boolean(ctx, ftz) }, location); -} - -/// nvvm.rcp.approx.ftz.f — `rcp.approx.ftz.f32`. -pub fn rcp_approx_ftz(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return ops.rcp_approx_ftz_f(ctx, value, .float(ctx, .f32), location); -} diff --git a/mlir/dialects/nvvm/ops.zig b/mlir/dialects/nvvm/ops.zig deleted file mode 100644 index 88c1c514..00000000 --- a/mlir/dialects/nvvm/ops.zig +++ /dev/null @@ -1,8389 +0,0 @@ -//! Zig bindings of NVIDIA's NVVM dialect of the CuTe DSL (nvidia-cutlass-dsl 4.8.0), generated -//! from the dialect's .td files; do not edit. - -const c = @import("c"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -/// The C dialect handle, `mlirGetDialectHandle__cute_nvvm__`. -pub const dialect_handle = "cute_nvvm"; - -// Enums, generated from the dialect's .td files; values are the compiler's. - -/// operations supported by atom instruction. -pub const AtomicOpKind = enum(u32) { - @"and" = 0, - @"or" = 1, - xor = 2, - cas = 3, - exch = 4, - add = 5, - inc = 6, - dec = 7, - min = 8, - max = 9, - umin = 10, - umax = 11, - fadd = 12, - sub = 13, -}; - -/// NVVM barrier reduction operation. -pub const BarrierReduction = enum(u32) { - popc = 0, - @"and" = 1, - @"or" = 2, -}; - -/// NVVM barrier redux kind. -pub const BarrierReduxKind = enum(u32) { - @"and" = 0, - @"or" = 1, - popc = 2, -}; - -/// MMA Block Scale Format. -pub const BlockScaleFormat = enum(u32) { - ue8m0 = 0, - ue4m3 = 1, -}; - -/// NVVM CTA group kind. -pub const CTAGroupKind = enum(u32) { - cta_1 = 0, - cta_2 = 1, -}; - -/// NVVM CVT Pack Float kind. -pub const CVTPackFloatKind = enum(u32) { - f32 = 0, - f16x2 = 1, - e4m3x2 = 2, - e5m2x2 = 3, - bf16x2 = 4, - e2m1x2 = 5, - e2m3x2 = 6, - e3m2x2 = 7, - ue8m0x2 = 8, -}; - -/// NVVM Cache Eviction Priority. -pub const CacheEvictionPriority = enum(u32) { - evict_normal = 0, - evict_first = 1, - evict_last = 2, - evict_unchanged = 3, - no_allocate = 4, -}; - -/// NVVM ClusterLaunchControlQueryType. -pub const ClusterLaunchControlQueryType = enum(u32) { - is_canceled = 0, - get_first_cta_id_x = 1, - get_first_cta_id_y = 2, - get_first_cta_id_z = 3, -}; - -/// Comparison operator encoding. -pub const CompareOpKind = enum(u32) { - eq = 0, - ne = 1, - ugt = 2, - uge = 3, - ult = 4, - ule = 5, - sgt = 6, - sge = 7, - slt = 8, - sle = 9, -}; - -/// NVVM ConvertFP4Type kind. -pub const ConvertFP4Type = enum(u32) { - e2m1 = 0, -}; - -/// NVVM ConvertFP8Type kind. -pub const ConvertFP8Type = enum(u32) { - e4m3 = 0, - e5m2 = 1, - ue8m0 = 2, - ue5m3 = 3, -}; - -/// NVVM ConvertScale kind. -pub const ConvertScaleKind = enum(u32) { - none = 0, - ue8m0 = 1, -}; - -/// NVVM DotAccumulateType. -pub const DotAccumulateType = enum(u32) { - signed = 1, - unsigned = 0, -}; - -/// NVVM L2 Prefetch Size. -pub const EvictKind = enum(u32) { - evict_normal = 0, - evict_first = 1, - evict_last = 2, - evict_unchanged = 3, - evict_no_allocate = 4, - none = 5, -}; - -/// NVVM FPRoundingMode kind. -pub const FPRoundingMode = enum(u32) { - none = 0, - rn = 1, - rm = 2, - rp = 3, - rz = 4, - rna = 5, - rs = 6, -}; - -/// Action kind for grid dependency control. -pub const GridDepActionKind = enum(u32) { - wait = 0, - launch_dependents = 1, -}; - -/// NVVM IntegerRoundingMode kind. -pub const IntegerRoundingMode = enum(u32) { - none = 0, - rni = 1, - rzi = 2, - rmi = 3, - rpi = 4, -}; - -/// NVVM L2 Prefetch Size. -pub const L2PrefetchSize = enum(u32) { - none = 0, - reserved = 1, - size_64b = 2, - size_128b = 3, - size_256b = 4, -}; - -/// Element type for ldmatrix and stmatrix. -pub const LdStMatrixEltType = enum(u32) { - b16 = 0, - b8 = 1, - @"b8x16.b6x16_p32" = 2, - @"b8x16.b4x16_p64" = 3, -}; - -/// NVVM load cache modifier kind(Ext). -pub const LoadCacheModifierExtKind = enum(u32) { - ca = 0, - cg = 1, - cs = 2, - lu = 3, - cv = 4, - none = 5, -}; - -/// NVVM load cache modifier kind. -pub const LoadCacheModifierKind = enum(u32) { - ca = 0, - cg = 1, - cs = 2, - lu = 3, - cv = 4, -}; - -/// shape attribute for ldmatrix. -pub const LoadShape = enum(u32) { - m8n8 = 0, - m8n16 = 1, - m16n16 = 3, -}; - -/// source format for ldmatrix. -pub const LoadSrcFormat = enum(u32) { - b6x16_p32 = 0, - b4x16_p64 = 1, - b8 = 2, -}; - -/// NVVM MBarrier Layout. -pub const MBarrierLayout = enum(u32) { - v0 = 0, - v1 = 1, -}; - -/// NVVM mbarrier phase type. -pub const MBarrierPhase = enum(u32) { - none = 0, - primary = 1, - conditional = 2, -}; - -/// NVVM MBarrier scope kind. -pub const MBarrierScopeKind = enum(u32) { - cta = 0, - cluster = 1, -}; - -/// NVVM MBarrier space kind. -pub const MBarrierSpaceKind = enum(u32) { - cta = 0, - cluster = 1, -}; - -/// NVVM MBarrier Transaction kind. -pub const MBarrierTxnKind = enum(u32) { - arrive = 0, - arrive_drop = 1, - arrive_expect_tx = 2, - arrive_drop_expect_tx = 3, - expect_tx = 4, - complete_tx = 5, -}; - -/// NVVM MBarrier wait kind. -pub const MBarrierWaitKind = enum(u32) { - @"test" = 0, - @"try" = 1, -}; - -/// MMA binary operations. -pub const MMAB1Op = enum(u32) { - none = 0, - xor_popc = 1, - and_popc = 2, -}; - -/// Block Scale Kind. -pub const MMABlockScaleKind = enum(u32) { - mxf8f6f4 = 0, - mxf4 = 1, - mxf4nvf4 = 2, -}; - -/// MMA CTA count. -pub const MMACtaCount = enum(u32) { - cta1 = 1, - cta2 = 2, -}; - -/// NVVM MMA frag type. -pub const MMAFrag = enum(u32) { - a = 0, - b = 1, - c = 2, -}; - -/// MMA overflow options. -pub const MMAIntOverflow = enum(u32) { - satfinite = 1, - wrapped = 0, -}; - -/// MMA operation kind. -pub const MMAKind = enum(u32) { - f8f6f4 = 0, -}; - -/// NVVM MMA layout. -pub const MMALayout = enum(u32) { - row = 0, - col = 1, -}; - -/// NVVM MMA types. -pub const MMATypes = enum(u32) { - f16 = 0, - f32 = 1, - tf32 = 2, - bf16 = 9, - s8 = 4, - u8 = 3, - s32 = 5, - s4 = 8, - u4 = 7, - b1 = 6, - f64 = 10, - e4m3 = 11, - e5m2 = 12, - e3m2 = 13, - e2m3 = 14, - e2m1 = 15, -}; - -/// NVVM match sync kind. -pub const MatchSyncKind = enum(u32) { - any = 0, - all = 1, -}; - -/// NVVM Memory Ordering kind. -pub const MemOrderKind = enum(u32) { - weak = 0, - relaxed = 1, - acquire = 2, - release = 3, - acq_rel = 4, - sc = 5, - mmio = 6, - @"volatile" = 8, - constant = 7, -}; - -/// NVVM Memory Scope kind. -pub const MemScopeKind = enum(u32) { - cta = 0, - cluster = 1, - gpu = 2, - sys = 3, -}; - -/// multiply mode attribute. -pub const MulMode = enum(u32) { - hi = 0, - lo = 1, - wide = 2, -}; - -/// NVVM Memory Space. -pub const NVVMMemorySpace = enum(u32) { - generic = 0, - global = 1, - shared = 3, - constant = 4, - local = 5, - tensor = 6, - shared_cluster = 7, -}; - -/// NVVM permute mode. -pub const PermuteMode = enum(u32) { - default = 0, - f4e = 1, - b4e = 2, - rc8 = 3, - ecl = 4, - ecr = 5, - rc16 = 6, -}; - -/// NVVM Prefetch Cache Level. -pub const PrefetchCacheLevel = enum(u32) { - L1 = 0, - L2 = 1, -}; - -/// Proxy kind. -pub const ProxyKind = enum(u32) { - alias = 0, - async = 1, - @"async.global" = 2, - @"async.shared" = 3, - tensormap = 4, - generic = 5, -}; - -/// NVVM Reduction Kind attribute. -pub const ReductionKind = enum(u32) { - add = 1, - @"and" = 2, - max = 3, - min = 4, - @"or" = 5, - umax = 6, - umin = 7, - xor = 8, - fmin = 9, - fmax = 10, -}; - -/// Ops supported by red instruction. -pub const ReductionOp = enum(u32) { - @"and" = 0, - @"or" = 1, - xor = 2, - add = 3, - inc = 4, - dec = 5, - min = 6, - max = 7, -}; - -/// types supported by red instruction. -pub const ReductionType = enum(u32) { - b32 = 0, - b64 = 1, - u32 = 2, - u64 = 3, - s32 = 4, - s64 = 5, - f32 = 6, - f64 = 7, - f16 = 8, - f16x2 = 9, - bf16 = 10, - bf16x2 = 11, -}; - -/// Sparse tensor compression element size. -pub const SPCompressElemSize = enum(u32) { - b4 = 4, - b8 = 8, - b16 = 16, -}; - -/// Sparse tensor compression factor type. -pub const SPCompressFactorType = enum(u32) { - sp2to4 = 0, -}; - -/// Sparse tensor compression index size. -pub const SPCompressIndexSize = enum(u32) { - b2 = 2, - b4 = 4, -}; - -/// spcompress operation kind. -pub const SPCompressOpKind = enum(u32) { - min = 0, - max = 1, -}; - -/// Sparse tensor compression repetition factor. -pub const SPCompressRepFactor = enum(u32) { - x1 = 1, - x2 = 2, - x4 = 4, - x8 = 8, - x16 = 16, - x32 = 32, - x64 = 64, - x128 = 128, -}; - -/// Sparse tensor decompression element size. -pub const SPDecompressElemSize = enum(u32) { - b4 = 4, - b8 = 8, - b16 = 16, -}; - -/// Sparse tensor decompression factor type. -pub const SPDecompressFactorType = enum(u32) { - sp1to2 = 0, - sp1to4 = 1, - sp1to8 = 2, - sp1to16 = 3, - sp2to4 = 4, - sp2to8 = 5, - sp2to16 = 6, - sp4to8 = 7, - sp4to16 = 8, - sp8to16 = 9, -}; - -/// Sparse tensor decompression index size. -pub const SPDecompressIndexSize = enum(u32) { - b2 = 2, - b4 = 4, -}; - -/// Sparse tensor decompression repetition factor. -pub const SPDecompressRepFactor = enum(u32) { - x1 = 1, - x2 = 2, - x4 = 4, - x8 = 8, - x16 = 16, - x32 = 32, - x64 = 64, -}; - -/// NVVM SaturationMode kind. -pub const SaturationMode = enum(u32) { - none = 0, - satfinite = 1, - sat = 2, -}; - -/// NVVM SaturationMode kind. -pub const SaturationModeKind = enum(u32) { - none = 0, - satfinite = 1, -}; - -/// MMA Scale Vector Sizes. -pub const ScaleVecSize = enum(u32) { - x1 = 0, - x2 = 1, - x4 = 2, -}; - -/// NVVM set max register action. -pub const SetMaxRegisterAction = enum(u32) { - decrease = 1, - increase = 0, -}; - -/// Shared memory space. -pub const SharedSpace = enum(u32) { - cta = 0, - cluster = 1, -}; - -/// NVVM shuffle kind. -pub const ShflKind = enum(u32) { - bfly = 0, - up = 1, - down = 2, - idx = 3, -}; - -/// MMA Sparsity Format. -pub const SparsityFormat = enum(u32) { - thread = 0, -}; - -/// NVVM State Space. -pub const StateSpace = enum(u32) { - generic = 0, - global = 1, - shared_cta = 2, - shared_cluster = 3, - constant = 4, - local = 5, - tensor = 6, -}; - -/// NVVM store cache modifier kind. -pub const StoreCacheModifierKind = enum(u32) { - wb = 0, - cg = 1, - cs = 2, - wt = 3, - none = 4, -}; - -/// shape attribute for stmatrix. -pub const StoreShape = enum(u32) { - m8n8 = 0, - m16n8 = 2, -}; - -/// Cluster MMA Barrier Parameter Type. -pub const TCBarParam = enum(u32) { - a1t0 = 0, - a0tx = 1, -}; - -/// NVVM TMA Load Mode. -pub const TMALoadMode = enum(u32) { - tile = 0, - im2col = 1, - im2col_w = 2, - im2col_w_128 = 3, - tile_gather4 = 4, -}; - -/// NVVM TMA redux kind. -pub const TMAReduxKind = enum(u32) { - add = 0, - max = 2, - min = 1, - inc = 3, - dec = 4, - @"and" = 5, - @"or" = 6, - xor = 7, -}; - -/// NVVM TMA Store Mode. -pub const TMAStoreMode = enum(u32) { - tile = 0, - im2col = 1, - tile_scatter4 = 2, - im2col_w = 3, -}; - -/// tcgen05 cp multicast. -pub const Tcgen05CpMulticast = enum(u32) { - none = 0, - warpx2_02_13 = 1, - warpx2_01_23 = 2, - warpx4 = 3, -}; - -/// tcgen05 cp shapes. -pub const Tcgen05CpShape = enum(u32) { - shape_128x256b = 0, - shape_4x256b = 1, - shape_128x128b = 2, - shape_64x128b = 3, - shape_32x128b = 4, -}; - -/// tcgen05 cp source format. -pub const Tcgen05CpSrcFormat = enum(u32) { - b6x16_p32 = 0, - b4x16_p64 = 1, -}; - -/// NVVM Tcgen05 fence kind. -pub const Tcgen05FenceKind = enum(u32) { - before = 0, - after = 1, -}; - -/// allowed 32-bit signless integer cases: 0, 1, 2, 3, 4. -pub const Tcgen05LdStShape = enum(u32) { - shape_16x64b = 0, - shape_16x128b = 1, - shape_16x256b = 2, - shape_32x32b = 3, - shape_16x32bx2 = 4, -}; - -/// tcgen05.mma block scale attribute. -pub const Tcgen05MMABlockScale = enum(u32) { - default = 0, - block16 = 1, - block32 = 2, -}; - -/// tcgen05 MMA Collector Buffer B Attribute. -pub const Tcgen05MMACollectorBBuffer = enum(u32) { - b0 = 0, - b1 = 1, - b2 = 2, - b3 = 3, -}; - -/// tcgen05.mma Collector Buffer Operation. -pub const Tcgen05MMACollectorOp = enum(u32) { - discard = 0, - lastuse = 1, - fill = 2, - use = 3, -}; - -/// tcgen05 MMA Supported Types. -pub const Tcgen05MMAKind = enum(u32) { - f16 = 0, - tf32 = 1, - f8f6f4 = 2, - i8 = 3, - mxf8f6f4 = 4, - mxf4 = 5, - mxf4nvf4 = 6, - ti16 = 7, -}; - -/// NVVM Tcgen05 wait kind. -pub const Tcgen05WaitKind = enum(u32) { - load = 0, - store = 1, -}; - -/// NVVM Tensormap Elemtype. -pub const TensormapElemtype = enum(u32) { - u8 = 0, - u16 = 1, - u32 = 2, - s32 = 3, - u64 = 4, - s64 = 5, - f16 = 6, - f32 = 7, - @"f32.ftz" = 8, - f64 = 9, - bf16 = 10, - tf32 = 11, - @"tf32.ftz" = 12, - b4x16 = 13, - b4x16_p64 = 14, - b6x16_p32 = 15, -}; - -/// NVVM Tensormap Field Kind. -pub const TensormapField = enum(u32) { - global_address = 0, - rank = 1, - box_dim = 2, - global_dim = 3, - global_stride = 4, - element_stride = 5, - elemtype = 6, - interleave_layout = 7, - swizzle_mode = 8, - swizzle_atomicity = 9, - fill_mode = 10, -}; - -/// NVVM Tensormap Fill Mode. -pub const TensormapFillMode = enum(u32) { - zero = 0, - oob_nan = 1, -}; - -/// NVVM Tensormap Interleave Layout. -pub const TensormapInterleaveLayout = enum(u32) { - no_interleave = 0, - b16 = 1, - b32 = 2, -}; - -/// NVVM Tensormap Swizzle Atomicity. -pub const TensormapSwizzleAtomicity = enum(u32) { - b16 = 0, - b32 = 1, - b32_flip_b8 = 2, - b64 = 3, -}; - -/// NVVM Tensormap Swizzle Mode. -pub const TensormapSwizzleMode = enum(u32) { - no_swizzling = 0, - b32 = 1, - b64 = 2, - b128 = 3, - b96 = 4, -}; - -/// Tensor Memory Layout Enumerated Type. -pub const TmemLayout = enum(u32) { - tmem_16dp_128bit = 0, - tmem_16dp_256bit = 1, - tmem_32dp_32bit = 2, - tmem_16dp_64bit = 3, - tmem_16dp_32bit_t0_t15 = 4, - tmem_16dp_32bit_t16_t31 = 5, -}; - -/// NVVM validate data pattern. -pub const ValidatePattern = enum(u32) { - none = 0, - per_16bytes_80000000 = 1, - per_16bytes_8000 = 2, - per_16bytes_80 = 3, - per_16bytes_8 = 4, - per_element_ff = 5, - per_element_80000000 = 6, - per_element_8000 = 7, - per_element_80 = 8, - per_element_8 = 9, - per_16bytes_80000000_paired = 10, -}; - -/// NVVM vote sync kind. -pub const VoteSyncKind = enum(u32) { - any = 0, - all = 1, - ballot = 2, - uni = 3, -}; - -/// WGMMA overflow options. -pub const WGMMAScaleIn = enum(u32) { - one = 1, - neg = 2, -}; - -/// WGMMA input predicate. -pub const WGMMAScaleOut = enum(u32) { - zero = 0, - one = 1, -}; - -/// NVVM WGMMA types. -pub const WGMMATypes = enum(u32) { - f16 = 0, - tf32 = 1, - u8 = 2, - s8 = 3, - b1 = 4, - bf16 = 5, - e4m3 = 6, - e5m2 = 7, - f32 = 8, - s32 = 9, -}; - -// Types and attributes, generated from the dialect's .td files. - -/// `#nvvm.atomic_op`: operations supported by atom instruction -pub const AtomicOpKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMAtomicOpKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "atomic_op"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: AtomicOpKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMAtomicOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) AtomicOpKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMAtomicOpKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.reduction`: NVVM barrier reduction operation -pub const BarrierReductionAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMBarrierReduction; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "reduction"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: BarrierReduction, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMBarrierReductionAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) BarrierReduction { - return @fromBackingInt(@intCast(c.mlirCuteNVVMBarrierReductionAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.barrier_redux_kind`: NVVM barrier redux kind -pub const BarrierReduxKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMBarrierReduxKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "barrier_redux_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: BarrierReduxKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMBarrierReduxKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) BarrierReduxKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMBarrierReduxKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.block_scale_format`: MMA Block Scale Format -pub const BlockScaleFormatAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMBlockScaleFormat; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "block_scale_format"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: BlockScaleFormat, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMBlockScaleFormatAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) BlockScaleFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVVMBlockScaleFormatAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.cta_group`: NVVM CTA group kind -pub const CTAGroupKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMCTAGroupKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "cta_group"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: CTAGroupKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMCTAGroupKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) CTAGroupKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMCTAGroupKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.packfloat_type`: NVVM CVT Pack Float kind -pub const CVTPackFloatKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMCVTPackFloatKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "packfloat_type"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: CVTPackFloatKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMCVTPackFloatKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) CVTPackFloatKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMCVTPackFloatKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.cache_eviction_priority`: NVVM Cache Eviction Priority -pub const CacheEvictionPriorityAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMCacheEvictionPriority; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "cache_eviction_priority"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: CacheEvictionPriority, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMCacheEvictionPriorityAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) CacheEvictionPriority { - return @fromBackingInt(@intCast(c.mlirCuteNVVMCacheEvictionPriorityAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.cluster_launch_control_query_type`: NVVM ClusterLaunchControlQueryType -pub const ClusterLaunchControlQueryTypeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMClusterLaunchControlQueryType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "cluster_launch_control_query_type"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ClusterLaunchControlQueryType, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ClusterLaunchControlQueryType { - return @fromBackingInt(@intCast(c.mlirCuteNVVMClusterLaunchControlQueryTypeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.op`: Comparison operator encoding -pub const CompareOpKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMCompareOpKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "op"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: CompareOpKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMCompareOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) CompareOpKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMCompareOpKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.convert_fp4_type`: NVVM ConvertFP4Type kind -pub const ConvertFP4TypeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMConvertFP4Type; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "convert_fp4_type"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ConvertFP4Type, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMConvertFP4TypeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ConvertFP4Type { - return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertFP4TypeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.convert_fp8_type`: NVVM ConvertFP8Type kind -pub const ConvertFP8TypeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMConvertFP8Type; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "convert_fp8_type"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ConvertFP8Type, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMConvertFP8TypeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ConvertFP8Type { - return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertFP8TypeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.convert_scale_kind`: NVVM ConvertScale kind -pub const ConvertScaleKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMConvertScaleKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "convert_scale_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ConvertScaleKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMConvertScaleKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ConvertScaleKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMConvertScaleKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.dot_accumulate_type`: NVVM DotAccumulateType -pub const DotAccumulateTypeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMDotAccumulateType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "dot_accumulate_type"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: DotAccumulateType, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMDotAccumulateTypeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) DotAccumulateType { - return @fromBackingInt(@intCast(c.mlirCuteNVVMDotAccumulateTypeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.evict_kind`: NVVM L2 Prefetch Size -pub const EvictKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMEvictKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "evict_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: EvictKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMEvictKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) EvictKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMEvictKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.fp_rnd_mode`: NVVM FPRoundingMode kind -pub const FPRoundingModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMFPRoundingMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "fp_rnd_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: FPRoundingMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMFPRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) FPRoundingMode { - return @fromBackingInt(@intCast(c.mlirCuteNVVMFPRoundingModeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.grid_dep_action`: Action kind for grid dependency control -pub const GridDepActionKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMGridDepActionKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "grid_dep_action"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: GridDepActionKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMGridDepActionKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) GridDepActionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMGridDepActionKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.int_rnd_mode`: NVVM IntegerRoundingMode kind -pub const IntegerRoundingModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMIntegerRoundingMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "int_rnd_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: IntegerRoundingMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMIntegerRoundingModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) IntegerRoundingMode { - return @fromBackingInt(@intCast(c.mlirCuteNVVMIntegerRoundingModeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.l2_prefetch`: NVVM L2 Prefetch Size -pub const L2PrefetchSizeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVML2PrefetchSize; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "l2_prefetch"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: L2PrefetchSize, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVML2PrefetchSizeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) L2PrefetchSize { - return @fromBackingInt(@intCast(c.mlirCuteNVVML2PrefetchSizeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.ld_st_matrix_elt_type`: Element type for ldmatrix and stmatrix -pub const LdStMatrixEltTypeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMLdStMatrixEltType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "ld_st_matrix_elt_type"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: LdStMatrixEltType, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMLdStMatrixEltTypeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) LdStMatrixEltType { - return @fromBackingInt(@intCast(c.mlirCuteNVVMLdStMatrixEltTypeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.load_cache_modifier_ext`: NVVM load cache modifier kind(Ext) -pub const LoadCacheModifierExtKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMLoadCacheModifierExtKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "load_cache_modifier_ext"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: LoadCacheModifierExtKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMLoadCacheModifierExtKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) LoadCacheModifierExtKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadCacheModifierExtKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.load_cache_modifier`: NVVM load cache modifier kind -pub const LoadCacheModifierKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMLoadCacheModifierKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "load_cache_modifier"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: LoadCacheModifierKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMLoadCacheModifierKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) LoadCacheModifierKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadCacheModifierKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.load_shape`: shape attribute for ldmatrix -pub const LoadShapeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMLoadShape; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "load_shape"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: LoadShape, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMLoadShapeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) LoadShape { - return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadShapeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.load_src_format`: source format for ldmatrix -pub const LoadSrcFormatAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMLoadSrcFormat; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "load_src_format"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: LoadSrcFormat, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMLoadSrcFormatAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) LoadSrcFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVVMLoadSrcFormatAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mbarrier_layout`: NVVM MBarrier Layout -pub const MBarrierLayoutAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierLayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mbarrier_layout"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MBarrierLayout, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MBarrierLayout { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierLayoutAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mbarrier_phase`: NVVM mbarrier phase type -pub const MBarrierPhaseAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierPhase; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mbarrier_phase"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MBarrierPhase, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierPhaseAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MBarrierPhase { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierPhaseAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mbar_scope`: NVVM MBarrier scope kind -pub const MBarrierScopeKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierScopeKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mbar_scope"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MBarrierScopeKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierScopeKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MBarrierScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierScopeKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mbar_space`: NVVM MBarrier space kind -pub const MBarrierSpaceKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierSpaceKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mbar_space"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MBarrierSpaceKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierSpaceKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MBarrierSpaceKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierSpaceKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mbar_txn_kind`: NVVM MBarrier Transaction kind -pub const MBarrierTxnKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierTxnKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mbar_txn_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MBarrierTxnKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierTxnKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MBarrierTxnKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierTxnKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mbar_wait`: NVVM MBarrier wait kind -pub const MBarrierWaitKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMBarrierWaitKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mbar_wait"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MBarrierWaitKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMBarrierWaitKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MBarrierWaitKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMBarrierWaitKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mma_b1op`: MMA binary operations -pub const MMAB1OpAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMMAB1Op; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mma_b1op"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MMAB1Op, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMAB1OpAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MMAB1Op { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAB1OpAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.block_scale_kind`: Block Scale Kind -pub const MMABlockScaleKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMMABlockScaleKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "block_scale_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MMABlockScaleKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMABlockScaleKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MMABlockScaleKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMMABlockScaleKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mma_cta_count`: MMA CTA count -pub const MMACtaCountAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMMACtaCount; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mma_cta_count"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MMACtaCount, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMACtaCountAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MMACtaCount { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMMACtaCountAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mma_frag`: NVVM MMA frag type -pub const MMAFragAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMMAFrag; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mma_frag"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MMAFrag, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMAFragAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MMAFrag { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAFragAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mma_int_overflow`: MMA overflow options -pub const MMAIntOverflowAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMMAIntOverflow; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mma_int_overflow"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MMAIntOverflow, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMAIntOverflowAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MMAIntOverflow { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAIntOverflowAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mma_kind`: MMA operation kind -pub const MMAKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMMAKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mma_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MMAKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMAKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MMAKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMMAKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mma_layout`: NVVM MMA layout -pub const MMALayoutAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMMALayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mma_layout"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MMALayout, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMALayoutAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MMALayout { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMMALayoutAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mma_type`: NVVM MMA types -pub const MMATypesAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMMATypes; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mma_type"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MMATypes, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMATypesAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MMATypes { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMMATypesAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.match_sync_kind`: NVVM match sync kind -pub const MatchSyncKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMatchSyncKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "match_sync_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MatchSyncKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMatchSyncKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MatchSyncKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMatchSyncKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mem_order`: NVVM Memory Ordering kind -pub const MemOrderKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMemOrderKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mem_order"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MemOrderKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMemOrderKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MemOrderKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMemOrderKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mem_scope`: NVVM Memory Scope kind -pub const MemScopeKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMemScopeKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mem_scope"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MemScopeKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMemScopeKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MemScopeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMemScopeKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.mul_mode`: multiply mode attribute -pub const MulModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMulMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "mul_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: MulMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMulModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) MulMode { - return @fromBackingInt(@intCast(c.mlirCuteNVVMMulModeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.memory_space`: NVVM Memory Space -pub const NVVMMemorySpaceAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMNVVMMemorySpace; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "memory_space"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: NVVMMemorySpace, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMNVVMMemorySpaceAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) NVVMMemorySpace { - return @fromBackingInt(@intCast(c.mlirCuteNVVMNVVMMemorySpaceAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.permute_mode`: NVVM permute mode -pub const PermuteModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMPermuteMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "permute_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: PermuteMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMPermuteModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) PermuteMode { - return @fromBackingInt(@intCast(c.mlirCuteNVVMPermuteModeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.prefetch_cache_level`: NVVM Prefetch Cache Level -pub const PrefetchCacheLevelAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMPrefetchCacheLevel; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "prefetch_cache_level"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: PrefetchCacheLevel, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMPrefetchCacheLevelAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) PrefetchCacheLevel { - return @fromBackingInt(@intCast(c.mlirCuteNVVMPrefetchCacheLevelAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.proxy_kind`: Proxy kind -pub const ProxyKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMProxyKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "proxy_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ProxyKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMProxyKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ProxyKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMProxyKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.reduction_kind`: NVVM Reduction Kind attribute -pub const ReductionKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMReductionKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "reduction_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ReductionKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMReductionKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ReductionKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.red_op`: Ops supported by red instruction -pub const ReductionOpAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMReductionOp; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "red_op"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ReductionOp, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMReductionOpAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ReductionOp { - return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionOpAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.red_type`: types supported by red instruction -pub const ReductionTypeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMReductionType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "red_type"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ReductionType, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMReductionTypeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ReductionType { - return @fromBackingInt(@intCast(c.mlirCuteNVVMReductionTypeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.spcomp_elem_size`: Sparse tensor compression element size -pub const SPCompressElemSizeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressElemSize; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "spcomp_elem_size"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SPCompressElemSize, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPCompressElemSizeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SPCompressElemSize { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressElemSizeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.spcomp_factor`: Sparse tensor compression factor type -pub const SPCompressFactorTypeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressFactorType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "spcomp_factor"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SPCompressFactorType, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPCompressFactorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SPCompressFactorType { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressFactorTypeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.spcomp_index_size`: Sparse tensor compression index size -pub const SPCompressIndexSizeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressIndexSize; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "spcomp_index_size"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SPCompressIndexSize, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPCompressIndexSizeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SPCompressIndexSize { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressIndexSizeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.spcomp_op_kind`: spcompress operation kind -pub const SPCompressOpKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressOpKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "spcomp_op_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SPCompressOpKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPCompressOpKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SPCompressOpKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressOpKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.spcomp_rep_factor`: Sparse tensor compression repetition factor -pub const SPCompressRepFactorAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSPCompressRepFactor; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "spcomp_rep_factor"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SPCompressRepFactor, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPCompressRepFactorAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SPCompressRepFactor { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSPCompressRepFactorAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.spdecomp_elem_size`: Sparse tensor decompression element size -pub const SPDecompressElemSizeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressElemSize; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "spdecomp_elem_size"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SPDecompressElemSize, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPDecompressElemSizeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SPDecompressElemSize { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressElemSizeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.spdecomp_factor`: Sparse tensor decompression factor type -pub const SPDecompressFactorTypeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressFactorType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "spdecomp_factor"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SPDecompressFactorType, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPDecompressFactorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SPDecompressFactorType { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressFactorTypeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.spdecomp_index_size`: Sparse tensor decompression index size -pub const SPDecompressIndexSizeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressIndexSize; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "spdecomp_index_size"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SPDecompressIndexSize, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPDecompressIndexSizeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SPDecompressIndexSize { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressIndexSizeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.spdecomp_rep_factor`: Sparse tensor decompression repetition factor -pub const SPDecompressRepFactorAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSPDecompressRepFactor; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "spdecomp_rep_factor"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SPDecompressRepFactor, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSPDecompressRepFactorAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SPDecompressRepFactor { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSPDecompressRepFactorAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.sat_mode`: NVVM SaturationMode kind -pub const SaturationModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSaturationMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "sat_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SaturationMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSaturationModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SaturationMode { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSaturationModeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.sat`: NVVM SaturationMode kind -pub const SaturationModeKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSaturationModeKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "sat"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SaturationModeKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSaturationModeKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SaturationModeKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSaturationModeKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.scale_vec_size`: MMA Scale Vector Sizes -pub const ScaleVecSizeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMScaleVecSize; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "scale_vec_size"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ScaleVecSize, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMScaleVecSizeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ScaleVecSize { - return @fromBackingInt(@intCast(c.mlirCuteNVVMScaleVecSizeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.action`: NVVM set max register action -pub const SetMaxRegisterActionAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSetMaxRegisterAction; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "action"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SetMaxRegisterAction, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSetMaxRegisterActionAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SetMaxRegisterAction { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSetMaxRegisterActionAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.shared_space`: Shared memory space -pub const SharedSpaceAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSharedSpace; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "shared_space"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SharedSpace, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSharedSpaceAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SharedSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSharedSpaceAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.shfl_kind`: NVVM shuffle kind -pub const ShflKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMShflKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "shfl_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ShflKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMShflKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ShflKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMShflKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.sparsity_format`: MMA Sparsity Format -pub const SparsityFormatAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMSparsityFormat; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "sparsity_format"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: SparsityFormat, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMSparsityFormatAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) SparsityFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVVMSparsityFormatAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.state_space`: NVVM State Space -pub const StateSpaceAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMStateSpace; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "state_space"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: StateSpace, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMStateSpaceAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) StateSpace { - return @fromBackingInt(@intCast(c.mlirCuteNVVMStateSpaceAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.store_cache_modifier`: NVVM store cache modifier kind -pub const StoreCacheModifierKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMStoreCacheModifierKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "store_cache_modifier"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: StoreCacheModifierKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMStoreCacheModifierKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) StoreCacheModifierKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMStoreCacheModifierKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.store_shape`: shape attribute for stmatrix -pub const StoreShapeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMStoreShape; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "store_shape"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: StoreShape, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMStoreShapeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) StoreShape { - return @fromBackingInt(@intCast(c.mlirCuteNVVMStoreShapeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.TCBarParam`: Cluster MMA Barrier Parameter Type -pub const TCBarParamAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTCBarParam; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "TCBarParam"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TCBarParam, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTCBarParamAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TCBarParam { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTCBarParamAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tma_load_mode`: NVVM TMA Load Mode -pub const TMALoadModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTMALoadMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tma_load_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TMALoadMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTMALoadModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TMALoadMode { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTMALoadModeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tma_redux_kind`: NVVM TMA redux kind -pub const TMAReduxKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTMAReduxKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tma_redux_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TMAReduxKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTMAReduxKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TMAReduxKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTMAReduxKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tma_store_mode`: NVVM TMA Store Mode -pub const TMAStoreModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTMAStoreMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tma_store_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TMAStoreMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTMAStoreModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TMAStoreMode { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTMAStoreModeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tcgen05_cp_multicast`: tcgen05 cp multicast -pub const Tcgen05CpMulticastAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpMulticast; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tcgen05_cp_multicast"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Tcgen05CpMulticast, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05CpMulticastAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Tcgen05CpMulticast { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpMulticastAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tcgen05_cp_shape`: tcgen05 cp shapes -pub const Tcgen05CpShapeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpShape; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tcgen05_cp_shape"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Tcgen05CpShape, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05CpShapeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Tcgen05CpShape { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpShapeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tcgen05_cp_src_fmt`: tcgen05 cp source format -pub const Tcgen05CpSrcFormatAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05CpSrcFormat; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tcgen05_cp_src_fmt"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Tcgen05CpSrcFormat, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05CpSrcFormatAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Tcgen05CpSrcFormat { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05CpSrcFormatAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tcgen05_fence`: NVVM Tcgen05 fence kind -pub const Tcgen05FenceKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05FenceKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tcgen05_fence"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Tcgen05FenceKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05FenceKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Tcgen05FenceKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05FenceKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tcgen05_ldst_shape`: allowed 32-bit signless integer cases: 0, 1, 2, 3, 4 -pub const Tcgen05LdStShapeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05LdStShape; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tcgen05_ldst_shape"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Tcgen05LdStShape, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05LdStShapeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Tcgen05LdStShape { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05LdStShapeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tcgen05_mma_block_scale`: tcgen05.mma block scale attribute -pub const Tcgen05MMABlockScaleAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMABlockScale; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tcgen05_mma_block_scale"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Tcgen05MMABlockScale, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05MMABlockScaleAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Tcgen05MMABlockScale { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMABlockScaleAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tcgen05_mma_collectorb`: tcgen05 MMA Collector Buffer B Attribute -pub const Tcgen05MMACollectorBBufferAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMACollectorBBuffer; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tcgen05_mma_collectorb"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Tcgen05MMACollectorBBuffer, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Tcgen05MMACollectorBBuffer { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMACollectorBBufferAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tcgen05_mma_collectorop`: tcgen05.mma Collector Buffer Operation -pub const Tcgen05MMACollectorOpAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMACollectorOp; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tcgen05_mma_collectorop"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Tcgen05MMACollectorOp, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05MMACollectorOpAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Tcgen05MMACollectorOp { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMACollectorOpAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tcgen05_mma_kind`: tcgen05 MMA Supported Types -pub const Tcgen05MMAKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05MMAKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tcgen05_mma_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Tcgen05MMAKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05MMAKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Tcgen05MMAKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05MMAKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tcgen05_wait`: NVVM Tcgen05 wait kind -pub const Tcgen05WaitKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTcgen05WaitKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tcgen05_wait"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: Tcgen05WaitKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTcgen05WaitKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) Tcgen05WaitKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTcgen05WaitKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tensormap_elemtype`: NVVM Tensormap Elemtype -pub const TensormapElemtypeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapElemtype; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tensormap_elemtype"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TensormapElemtype, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapElemtypeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TensormapElemtype { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapElemtypeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tensormap_field`: NVVM Tensormap Field Kind -pub const TensormapFieldAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapField; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tensormap_field"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TensormapField, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapFieldAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TensormapField { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapFieldAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tensormap_fill_mode`: NVVM Tensormap Fill Mode -pub const TensormapFillModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapFillMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tensormap_fill_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TensormapFillMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapFillModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TensormapFillMode { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapFillModeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tensormap_interleave_layout`: NVVM Tensormap Interleave Layout -pub const TensormapInterleaveLayoutAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapInterleaveLayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tensormap_interleave_layout"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TensormapInterleaveLayout, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapInterleaveLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TensormapInterleaveLayout { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapInterleaveLayoutAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tensormap_swizzle_atomicity`: NVVM Tensormap Swizzle Atomicity -pub const TensormapSwizzleAtomicityAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapSwizzleAtomicity; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tensormap_swizzle_atomicity"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TensormapSwizzleAtomicity, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TensormapSwizzleAtomicity { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapSwizzleAtomicityAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.tensormap_swizzle_mode`: NVVM Tensormap Swizzle Mode -pub const TensormapSwizzleModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTensormapSwizzleMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "tensormap_swizzle_mode"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TensormapSwizzleMode, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTensormapSwizzleModeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TensormapSwizzleMode { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTensormapSwizzleModeAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.TmemLayout`: Tensor Memory Layout Enumerated Type -pub const TmemLayoutAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTmemLayout; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "TmemLayout"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: TmemLayout, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTmemLayoutAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) TmemLayout { - return @fromBackingInt(@intCast(c.mlirCuteNVVMTmemLayoutAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.validate_pattern`: NVVM validate data pattern -pub const ValidatePatternAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMValidatePattern; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "validate_pattern"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: ValidatePattern, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMValidatePatternAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) ValidatePattern { - return @fromBackingInt(@intCast(c.mlirCuteNVVMValidatePatternAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.vote_sync_kind`: NVVM vote sync kind -pub const VoteSyncKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMVoteSyncKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "vote_sync_kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: VoteSyncKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMVoteSyncKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) VoteSyncKind { - return @fromBackingInt(@intCast(c.mlirCuteNVVMVoteSyncKindAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.wgmma_scale_in`: WGMMA overflow options -pub const WGMMAScaleInAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMAScaleIn; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "wgmma_scale_in"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: WGMMAScaleIn, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMWGMMAScaleInAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) WGMMAScaleIn { - return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMAScaleInAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.wgmma_scale_out`: WGMMA input predicate -pub const WGMMAScaleOutAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMAScaleOut; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "wgmma_scale_out"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: WGMMAScaleOut, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMWGMMAScaleOutAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) WGMMAScaleOut { - return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMAScaleOutAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.wgmma_type`: NVVM WGMMA types -pub const WGMMATypesAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMWGMMATypes; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "wgmma_type"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: WGMMATypes, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMWGMMATypesAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) WGMMATypes { - return @fromBackingInt(@intCast(c.mlirCuteNVVMWGMMATypesAttrGetValue(self.ptr()))); - } -}; - -/// `#nvvm.ld_st_matrix_shape`: Matrix shape of ldmatrix, stmatrix and movmatrix -pub const LdStMatrixShapeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMLdStMatrixShape; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "ld_st_matrix_shape"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - m: c_int, - n: c_int, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMLdStMatrixShapeAttrGet(ctx.ptr(), args.m, args.n); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn m(self: *const Self) c_int { - return c.mlirCuteNVVMLdStMatrixShapeAttrGetM(self.ptr()); - } - pub fn n(self: *const Self) c_int { - return c.mlirCuteNVVMLdStMatrixShapeAttrGetN(self.ptr()); - } -}; - -/// `#nvvm.shape`: Shape of an MMA operation -pub const MMAShapeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMMMAShape; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "shape"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - m: c_int, - n: c_int, - k: c_int, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMMMAShapeAttrGet(ctx.ptr(), args.m, args.n, args.k); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn m(self: *const Self) c_int { - return c.mlirCuteNVVMMMAShapeAttrGetM(self.ptr()); - } - pub fn n(self: *const Self) c_int { - return c.mlirCuteNVVMMMAShapeAttrGetN(self.ptr()); - } - pub fn k(self: *const Self) c_int { - return c.mlirCuteNVVMMMAShapeAttrGetK(self.ptr()); - } -}; - -/// `#nvvm.target`: GPU target of an NVVM module -pub const TargetAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsACuteNVVMTarget; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "target"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - O: c_int = 2, - triple: []const u8 = "nvptx64-nvidia-cuda", - chip: []const u8 = "sm_75", - features: []const u8 = "", - flags: ?*const mlir.Attribute = null, - link: ?*const mlir.Attribute = null, - verifyTarget: bool = false, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirCuteNVVMTargetAttrGet(ctx.ptr(), args.O, mlir.stringRef(args.triple), mlir.stringRef(args.chip), mlir.stringRef(args.features), if (args.flags) |v| v.ptr() else .{ .ptr = null }, if (args.link) |v| v.ptr() else .{ .ptr = null }, args.verifyTarget); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn o(self: *const Self) c_int { - return c.mlirCuteNVVMTargetAttrGetO(self.ptr()); - } - pub fn triple(self: *const Self) []const u8 { - const s = c.mlirCuteNVVMTargetAttrGetTriple(self.ptr()); - return if (s.length == 0) "" else mlir.string(s); - } - pub fn chip(self: *const Self) []const u8 { - const s = c.mlirCuteNVVMTargetAttrGetChip(self.ptr()); - return if (s.length == 0) "" else mlir.string(s); - } - pub fn features(self: *const Self) []const u8 { - const s = c.mlirCuteNVVMTargetAttrGetFeatures(self.ptr()); - return if (s.length == 0) "" else mlir.string(s); - } - pub fn flags(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVVMTargetAttrGetFlags(self.ptr()).ptr); - } - pub fn link(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirCuteNVVMTargetAttrGetLink(self.ptr()).ptr); - } - pub fn verifyTarget(self: *const Self) bool { - return c.mlirCuteNVVMTargetAttrGetVerifyTarget(self.ptr()); - } -}; - -// Operation builders, generated from the operations' .td. -/// `nvvm.add.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn add_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.add.packed.bf16x2.f32x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.add.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn add_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.add.packed.f16x2.f32x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.add.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn add_packed_f32x2( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.add.packed.f32x2.bf16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn add_packed_f32x2_bf16x2_f32x2( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2.bf16x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.add.packed.f32x2.f16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn add_packed_f32x2_f16x2_f32x2( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - return mlir.Operation.make(ctx, "nvvm.add.packed.f32x2.f16x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.addf`. Result types are explicit; attributes use mlir.Attribute. -pub fn addf( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - sat: ?*const mlir.Attribute = null, - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.addf", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.applypriority.async.bulk`. Result types are explicit; attributes use mlir.Attribute. -pub fn applypriority_async_bulk( - ctx: *mlir.Context, - ptr: *const mlir.Value, - size: *const mlir.Value, - opts: struct { - evict: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); - return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk", .{ - .operands = .{ .flat = &.{ ptr, size } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.applypriority.async.bulk.tensor`. Result types are explicit; attributes use mlir.Attribute. -pub fn applypriority_async_bulk_tensor( - ctx: *mlir.Context, - tmaDesc: *const mlir.Value, - coordinates: []const *const mlir.Value, - im2colArgs: []const *const mlir.Value, - opts: struct { - mode: ?*const mlir.Attribute = null, - evict: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); - return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk.tensor", .{ - .operands = .{ .variadic = &.{ - &.{tmaDesc}, - coordinates, - im2colArgs, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.applypriority.async.bulk.tensor.override`. Result types are explicit; attributes use mlir.Attribute. -pub fn applypriority_async_bulk_tensor_override( - ctx: *mlir.Context, - tmaDesc: *const mlir.Value, - overrideAdrr: *const mlir.Value, - coordinates: []const *const mlir.Value, - tensorSize: []const *const mlir.Value, - lowerStride: []const *const mlir.Value, - im2colArgs: []const *const mlir.Value, - opts: struct { - upperStride: ?*const mlir.Value = null, - mode: ?*const mlir.Attribute = null, - evict: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); - return mlir.Operation.make(ctx, "nvvm.applypriority.async.bulk.tensor.override", .{ - .operands = .{ .variadic = &.{ - &.{tmaDesc}, - &.{overrideAdrr}, - coordinates, - tensorSize, - lowerStride, - if (opts.upperStride) |value| &.{value} else &.{}, - im2colArgs, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.atomicrmw`. Result types are explicit; attributes use mlir.Attribute. -pub fn atomicrmw( - ctx: *mlir.Context, - ptr: *const mlir.Value, - a: *const mlir.Value, - res_type: *const mlir.Type, - op: *const mlir.Attribute, - opts: struct { - b: ?*const mlir.Value = null, - memOrder: ?*const mlir.Attribute = null, - syncscope: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "op", op)); - if (opts.memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); - if (opts.syncscope) |value| attributes.appendAssumeCapacity(.named(ctx, "syncscope", value)); - return mlir.Operation.make(ctx, "nvvm.atomicrmw", .{ - .operands = .{ .flat = if (opts.b) |value| &.{ ptr, a, value } else &.{ ptr, a } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.bar.warp.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn bar_warp_sync(ctx: *mlir.Context, mask: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.bar.warp.sync", .{ - .operands = .{ .flat = &.{mask} }, - .location = location, - }); -} - -/// `nvvm.barrier`. Result types are explicit; attributes use mlir.Attribute. -pub fn barrier( - ctx: *mlir.Context, - opts: struct { - barrierId: ?*const mlir.Value = null, - numberOfThreads: ?*const mlir.Value = null, - reductionPredicate: ?*const mlir.Value = null, - res_type: ?*const mlir.Type = null, - reductionOp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (opts.res_type) |value| results.appendAssumeCapacity(value); - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.reductionOp) |value| attributes.appendAssumeCapacity(.named(ctx, "reductionOp", value)); - return mlir.Operation.make(ctx, "nvvm.barrier", .{ - .operands = .{ .variadic = &.{ - if (opts.barrierId) |value| &.{value} else &.{}, - if (opts.numberOfThreads) |value| &.{value} else &.{}, - if (opts.reductionPredicate) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = results.constSlice() }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.barrier.arrive`. Result types are explicit; attributes use mlir.Attribute. -pub fn barrier_arrive( - ctx: *mlir.Context, - numberOfThreads: *const mlir.Value, - opts: struct { - barrierId: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.barrier.arrive", .{ - .operands = .{ .flat = if (opts.barrierId) |value| &.{ value, numberOfThreads } else &.{numberOfThreads} }, - .location = location, - }); -} - -/// `nvvm.barrier.cta.arrive`. Result types are explicit; attributes use mlir.Attribute. -pub fn barrier_cta_arrive( - ctx: *mlir.Context, - barrierId: *const mlir.Value, - threadCount: *const mlir.Value, - opts: struct { - aligned: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); - return mlir.Operation.make(ctx, "nvvm.barrier.cta.arrive", .{ - .operands = .{ .flat = &.{ barrierId, threadCount } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.barrier.cta.red`. Result types are explicit; attributes use mlir.Attribute. -pub fn barrier_cta_red( - ctx: *mlir.Context, - pred: *const mlir.Value, - barrierId: *const mlir.Value, - res_type: *const mlir.Type, - kind: *const mlir.Attribute, - opts: struct { - threadCount: ?*const mlir.Value = null, - aligned: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); - return mlir.Operation.make(ctx, "nvvm.barrier.cta.red", .{ - .operands = .{ .flat = if (opts.threadCount) |value| &.{ pred, barrierId, value } else &.{ pred, barrierId } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.barrier.cta.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn barrier_cta_sync( - ctx: *mlir.Context, - barrierId: *const mlir.Value, - opts: struct { - threadCount: ?*const mlir.Value = null, - aligned: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); - return mlir.Operation.make(ctx, "nvvm.barrier.cta.sync", .{ - .operands = .{ .flat = if (opts.threadCount) |value| &.{ barrierId, value } else &.{barrierId} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.barrier0`. Result types are explicit; attributes use mlir.Attribute. -pub fn barrier0(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.barrier0", .{ - .location = location, - }); -} - -/// `nvvm.breakpoint`. Result types are explicit; attributes use mlir.Attribute. -pub fn breakpoint(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.breakpoint", .{ - .location = location, - }); -} - -/// `nvvm.clmad`. Result types are explicit; attributes use mlir.Attribute. -pub fn clmad( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c_: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - high: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.high) |value| attributes.appendAssumeCapacity(.named(ctx, "high", value)); - return mlir.Operation.make(ctx, "nvvm.clmad", .{ - .operands = .{ .flat = &.{ a, b, c_ } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cluster.arrive`. Result types are explicit; attributes use mlir.Attribute. -pub fn cluster_arrive( - ctx: *mlir.Context, - opts: struct { - aligned: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); - return mlir.Operation.make(ctx, "nvvm.cluster.arrive", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cluster.arrive.relaxed`. Result types are explicit; attributes use mlir.Attribute. -pub fn cluster_arrive_relaxed( - ctx: *mlir.Context, - opts: struct { - aligned: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); - return mlir.Operation.make(ctx, "nvvm.cluster.arrive.relaxed", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cluster.wait`. Result types are explicit; attributes use mlir.Attribute. -pub fn cluster_wait( - ctx: *mlir.Context, - opts: struct { - aligned: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aligned) |value| attributes.appendAssumeCapacity(.named(ctx, "aligned", value)); - return mlir.Operation.make(ctx, "nvvm.cluster.wait", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.clusterlaunchcontrol.query.cancel`. Result types are explicit; attributes use mlir.Attribute. -pub fn clusterlaunchcontrol_query_cancel(ctx: *mlir.Context, try_cancel_response: *const mlir.Value, res_type: *const mlir.Type, query_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.clusterlaunchcontrol.query.cancel", .{ - .operands = .{ .flat = &.{try_cancel_response} }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "query_type", query_type), - }, - .location = location, - }); -} - -/// `nvvm.clusterlaunchcontrol.try.cancel`. Result types are explicit; attributes use mlir.Attribute. -pub fn clusterlaunchcontrol_try_cancel( - ctx: *mlir.Context, - smemAddress: *const mlir.Value, - mbarrier: *const mlir.Value, - opts: struct { - multicast: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); - return mlir.Operation.make(ctx, "nvvm.clusterlaunchcontrol.try.cancel", .{ - .operands = .{ .flat = &.{ smemAddress, mbarrier } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.compare.and.set`. Result types are explicit; attributes use mlir.Attribute. -pub fn compare_and_set(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, result_type: *const mlir.Type, op: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.compare.and.set", .{ - .operands = .{ .flat = &.{ a, b } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "op", op), - }, - .location = location, - }); -} - -/// `nvvm.convert.and.pack.integer`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_and_pack_integer( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - dst_type: *const mlir.Type, - convert_type: *const mlir.Attribute, - opts: struct { - srcC: ?*const mlir.Value = null, - is_signed: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); - attributes.appendAssumeCapacity(.named(ctx, "convert_type", convert_type)); - return mlir.Operation.make(ctx, "nvvm.convert.and.pack.integer", .{ - .operands = .{ .flat = if (opts.srcC) |value| &.{ srcA, srcB, value } else &.{ srcA, srcB } }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.bf16x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_bf16x2_to_f4x2( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - dstTy: *const mlir.Attribute, - opts: struct { - scale_factor: ?*const mlir.Value = null, - isPZO: ?*const mlir.Attribute = null, - scale_factor_kind: ?*const mlir.Attribute = null, - rnd: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); - return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f4x2", .{ - .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.bf16x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_bf16x2_to_f6x2( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - dstTy: *const mlir.Attribute, - opts: struct { - scale_factor: ?*const mlir.Value = null, - isPZO: ?*const mlir.Attribute = null, - scale_factor_kind: ?*const mlir.Attribute = null, - rnd: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); - return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f6x2", .{ - .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.bf16x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_bf16x2_to_f8x2( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - dstTy: *const mlir.Attribute, - opts: struct { - scale_factor: ?*const mlir.Value = null, - isPZO: ?*const mlir.Attribute = null, - scale_factor_kind: ?*const mlir.Attribute = null, - rnd: ?*const mlir.Attribute = null, - sat: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); - return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.f8x2", .{ - .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.bf16x2.to.s2f6x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_bf16x2_to_s2f6x2( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - opts: struct { - scaleFactor: ?*const mlir.Value = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - return mlir.Operation.make(ctx, "nvvm.convert.bf16x2.to.s2f6x2", .{ - .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f16x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f16x2_to_f4x2( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - dstTy: *const mlir.Attribute, - opts: struct { - scale_factor: ?*const mlir.Value = null, - isPZO: ?*const mlir.Attribute = null, - scale_factor_kind: ?*const mlir.Attribute = null, - rnd: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); - return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f4x2", .{ - .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f16x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f16x2_to_f6x2( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - dstTy: *const mlir.Attribute, - opts: struct { - scale_factor: ?*const mlir.Value = null, - isPZO: ?*const mlir.Attribute = null, - scale_factor_kind: ?*const mlir.Attribute = null, - rnd: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); - return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f6x2", .{ - .operands = .{ .flat = if (opts.scale_factor) |value| &.{ src, value } else &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f16x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f16x2_to_f8x2( - ctx: *mlir.Context, - a: *const mlir.Value, - dst_type: *const mlir.Type, - dstTy: *const mlir.Attribute, - opts: struct { - scale_factor: ?*const mlir.Value = null, - isPZO: ?*const mlir.Attribute = null, - scale_factor_kind: ?*const mlir.Attribute = null, - rnd: ?*const mlir.Attribute = null, - sat: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); - return mlir.Operation.make(ctx, "nvvm.convert.f16x2.to.f8x2", .{ - .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, value } else &.{a} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f32x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_bf16x2( - ctx: *mlir.Context, - src_hi: *const mlir.Value, - src_lo: *const mlir.Value, - dst_type: *const mlir.Type, - opts: struct { - random_bits: ?*const mlir.Value = null, - rnd: ?*const mlir.Attribute = null, - sat: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.bf16x2", .{ - .operands = .{ .flat = if (opts.random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f32x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_f16x2( - ctx: *mlir.Context, - src_hi: *const mlir.Value, - src_lo: *const mlir.Value, - dst_type: *const mlir.Type, - opts: struct { - random_bits: ?*const mlir.Value = null, - rnd: ?*const mlir.Attribute = null, - sat: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f16x2", .{ - .operands = .{ .flat = if (opts.random_bits) |value| &.{ src_hi, src_lo, value } else &.{ src_hi, src_lo } }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f32x2.to.f4x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_f4x2( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - dst_type: *const mlir.Type, - dstType: *const mlir.Attribute, - opts: struct { - scale_factor: ?*const mlir.Value = null, - isPZO: ?*const mlir.Attribute = null, - scale_factor_kind: ?*const mlir.Attribute = null, - rnd: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstType", dstType)); - return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f4x2", .{ - .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f32x2.to.f6x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_f6x2( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - dst_type: *const mlir.Type, - dstTy: *const mlir.Attribute, - opts: struct { - scale_factor: ?*const mlir.Value = null, - isPZO: ?*const mlir.Attribute = null, - scale_factor_kind: ?*const mlir.Attribute = null, - rnd: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); - return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f6x2", .{ - .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f32x2.to.f8x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_f8x2( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - dst_type: *const mlir.Type, - dstTy: *const mlir.Attribute, - opts: struct { - scale_factor: ?*const mlir.Value = null, - isPZO: ?*const mlir.Attribute = null, - scale_factor_kind: ?*const mlir.Attribute = null, - rnd: ?*const mlir.Attribute = null, - sat: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - if (opts.isPZO) |value| attributes.appendAssumeCapacity(.named(ctx, "isPZO", value)); - if (opts.scale_factor_kind) |value| attributes.appendAssumeCapacity(.named(ctx, "scale_factor_kind", value)); - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); - return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.f8x2", .{ - .operands = .{ .flat = if (opts.scale_factor) |value| &.{ a, b, value } else &.{ a, b } }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f32x2.to.s2f6x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x2_to_s2f6x2( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - dst_type: *const mlir.Type, - opts: struct { - scaleFactor: ?*const mlir.Value = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - return mlir.Operation.make(ctx, "nvvm.convert.f32x2.to.s2f6x2", .{ - .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ a, b, value } else &.{ a, b } }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f32x4.to.f4x4`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x4_to_f4x4( - ctx: *mlir.Context, - src: *const mlir.Value, - rbits: *const mlir.Value, - dst_type: *const mlir.Type, - dstTy: *const mlir.Attribute, - opts: struct { - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); - return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f4x4", .{ - .operands = .{ .flat = &.{ src, rbits } }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f32x4.to.f6x4`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x4_to_f6x4( - ctx: *mlir.Context, - src: *const mlir.Value, - rbits: *const mlir.Value, - dst_type: *const mlir.Type, - dstTy: *const mlir.Attribute, - opts: struct { - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); - return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f6x4", .{ - .operands = .{ .flat = &.{ src, rbits } }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f32x4.to.f8x4`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f32x4_to_f8x4( - ctx: *mlir.Context, - src: *const mlir.Value, - rbits: *const mlir.Value, - dst_type: *const mlir.Type, - dstTy: *const mlir.Attribute, - opts: struct { - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "dstTy", dstTy)); - return mlir.Operation.make(ctx, "nvvm.convert.f32x4.to.f8x4", .{ - .operands = .{ .flat = &.{ src, rbits } }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f4x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f4x2_to_f16x2( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - srcType: *const mlir.Attribute, - opts: struct { - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); - return mlir.Operation.make(ctx, "nvvm.convert.f4x2.to.f16x2", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f6x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f6x2_to_f16x2( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - srcType: *const mlir.Attribute, - opts: struct { - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); - return mlir.Operation.make(ctx, "nvvm.convert.f6x2.to.f16x2", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f8x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f8x2_to_bf16x2( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - srcType: *const mlir.Attribute, - opts: struct { - scaleFactor: ?*const mlir.Value = null, - sat: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); - return mlir.Operation.make(ctx, "nvvm.convert.f8x2.to.bf16x2", .{ - .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.f8x2.to.f16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_f8x2_to_f16x2( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - srcType: *const mlir.Attribute, - opts: struct { - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); - return mlir.Operation.make(ctx, "nvvm.convert.f8x2.to.f16x2", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.float.to.integer`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_float_to_integer( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - sat: ?*const mlir.Attribute = null, - ftz: ?*const mlir.Attribute = null, - is_signed: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - if (opts.is_signed) |value| attributes.appendAssumeCapacity(.named(ctx, "is_signed", value)); - return mlir.Operation.make(ctx, "nvvm.convert.float.to.integer", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.float.to.tf32`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_float_to_tf32( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - sat: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - return mlir.Operation.make(ctx, "nvvm.convert.float.to.tf32", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.convert.s2f6x2.to.bf16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn convert_s2f6x2_to_bf16x2( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - opts: struct { - scaleFactor: ?*const mlir.Value = null, - sat: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - return mlir.Operation.make(ctx, "nvvm.convert.s2f6x2.to.bf16x2", .{ - .operands = .{ .flat = if (opts.scaleFactor) |value| &.{ src, value } else &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cos`. Result types are explicit; attributes use mlir.Attribute. -pub fn cos( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.cos", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.commit.group`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.commit.group", .{ - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.global.shared.cta`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_global_shared_cta( - ctx: *mlir.Context, - dstMem: *const mlir.Value, - srcMem: *const mlir.Value, - size: *const mlir.Value, - opts: struct { - l2CacheHint: ?*const mlir.Value = null, - byteMask: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.global.shared.cta", .{ - .operands = .{ .variadic = &.{ - &.{dstMem}, - &.{srcMem}, - &.{size}, - if (opts.l2CacheHint) |value| &.{value} else &.{}, - if (opts.byteMask) |value| &.{value} else &.{}, - } }, - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.prefetch`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_prefetch( - ctx: *mlir.Context, - srcMem: *const mlir.Value, - size: *const mlir.Value, - opts: struct { - l2CacheHint: ?*const mlir.Value = null, - evict: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.prefetch", .{ - .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ srcMem, size, value } else &.{ srcMem, size } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.shared.cluster.global`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_shared_cluster_global( - ctx: *mlir.Context, - dstMem: *const mlir.Value, - srcMem: *const mlir.Value, - mbar: *const mlir.Value, - size: *const mlir.Value, - opts: struct { - multicastMask: ?*const mlir.Value = null, - l2CacheHint: ?*const mlir.Value = null, - validatePattern: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.shared.cluster.global", .{ - .operands = .{ .variadic = &.{ - &.{dstMem}, - &.{srcMem}, - &.{mbar}, - &.{size}, - if (opts.multicastMask) |value| &.{value} else &.{}, - if (opts.l2CacheHint) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.shared.cluster.shared.cta`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_shared_cluster_shared_cta(ctx: *mlir.Context, dstMem: *const mlir.Value, srcMem: *const mlir.Value, mbar: *const mlir.Value, size: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.shared.cluster.shared.cta", .{ - .operands = .{ .flat = &.{ dstMem, srcMem, mbar, size } }, - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.tensor.global.shared.cta`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_global_shared_cta( - ctx: *mlir.Context, - tmaDescriptor: *const mlir.Value, - srcMem: *const mlir.Value, - coordinates: []const *const mlir.Value, - opts: struct { - l2CacheHint: ?*const mlir.Value = null, - predicate: ?*const mlir.Value = null, - mode: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.global.shared.cta", .{ - .operands = .{ .variadic = &.{ - &.{tmaDescriptor}, - &.{srcMem}, - coordinates, - if (opts.l2CacheHint) |value| &.{value} else &.{}, - if (opts.predicate) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.tensor.global.shared.cta.override`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_global_shared_cta_override( - ctx: *mlir.Context, - tmaDesc: *const mlir.Value, - srcMem: *const mlir.Value, - overrideAdrr: *const mlir.Value, - coordinates: []const *const mlir.Value, - tensorSize: []const *const mlir.Value, - lowerStride: []const *const mlir.Value, - opts: struct { - upperStride: ?*const mlir.Value = null, - l2CacheHint: ?*const mlir.Value = null, - mode: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.global.shared.cta.override", .{ - .operands = .{ .variadic = &.{ - &.{tmaDesc}, - &.{srcMem}, - &.{overrideAdrr}, - coordinates, - tensorSize, - lowerStride, - if (opts.upperStride) |value| &.{value} else &.{}, - if (opts.l2CacheHint) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.tensor.prefetch`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_prefetch( - ctx: *mlir.Context, - tmaDescriptor: *const mlir.Value, - coordinates: []const *const mlir.Value, - im2colOffsets: []const *const mlir.Value, - opts: struct { - l2CacheHint: ?*const mlir.Value = null, - mode: ?*const mlir.Attribute = null, - evict: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.prefetch", .{ - .operands = .{ .variadic = &.{ - &.{tmaDescriptor}, - coordinates, - im2colOffsets, - if (opts.l2CacheHint) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.tensor.prefetch.override`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_prefetch_override( - ctx: *mlir.Context, - tmaDesc: *const mlir.Value, - overrideAdrr: *const mlir.Value, - coordinates: []const *const mlir.Value, - tensorSize: []const *const mlir.Value, - lowerStride: []const *const mlir.Value, - im2colOffsets: []const *const mlir.Value, - opts: struct { - upperStride: ?*const mlir.Value = null, - l2CacheHint: ?*const mlir.Value = null, - evict: ?*const mlir.Attribute = null, - mode: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.prefetch.override", .{ - .operands = .{ .variadic = &.{ - &.{tmaDesc}, - &.{overrideAdrr}, - coordinates, - tensorSize, - lowerStride, - if (opts.upperStride) |value| &.{value} else &.{}, - im2colOffsets, - if (opts.l2CacheHint) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.tensor.reduce`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_reduce( - ctx: *mlir.Context, - tmaDescriptor: *const mlir.Value, - srcMem: *const mlir.Value, - coordinates: []const *const mlir.Value, - redKind: *const mlir.Attribute, - opts: struct { - l2CacheHint: ?*const mlir.Value = null, - mode: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "redKind", redKind)); - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.reduce", .{ - .operands = .{ .variadic = &.{ - &.{tmaDescriptor}, - &.{srcMem}, - coordinates, - if (opts.l2CacheHint) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.tensor.reduce.override`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_reduce_override( - ctx: *mlir.Context, - tmaDesc: *const mlir.Value, - srcMem: *const mlir.Value, - overrideAdrr: *const mlir.Value, - coordinates: []const *const mlir.Value, - tensorSize: []const *const mlir.Value, - lowerStride: []const *const mlir.Value, - redKind: *const mlir.Attribute, - opts: struct { - upperStride: ?*const mlir.Value = null, - l2CacheHint: ?*const mlir.Value = null, - mode: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "redKind", redKind)); - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.reduce.override", .{ - .operands = .{ .variadic = &.{ - &.{tmaDesc}, - &.{srcMem}, - &.{overrideAdrr}, - coordinates, - tensorSize, - lowerStride, - if (opts.upperStride) |value| &.{value} else &.{}, - if (opts.l2CacheHint) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.tensor.shared.cluster.global`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_shared_cluster_global( - ctx: *mlir.Context, - dstMem: *const mlir.Value, - tmaDescriptor: *const mlir.Value, - coordinates: []const *const mlir.Value, - mbar: *const mlir.Value, - im2colOffsets: []const *const mlir.Value, - opts: struct { - multicastMask: ?*const mlir.Value = null, - l2CacheHint: ?*const mlir.Value = null, - predicate: ?*const mlir.Value = null, - validatePattern: ?*const mlir.Attribute = null, - mode: ?*const mlir.Attribute = null, - isCTAOnly: ?*const mlir.Attribute = null, - group: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - if (opts.isCTAOnly) |value| attributes.appendAssumeCapacity(.named(ctx, "isCTAOnly", value)); - if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.shared.cluster.global", .{ - .operands = .{ .variadic = &.{ - &.{dstMem}, - &.{tmaDescriptor}, - coordinates, - &.{mbar}, - im2colOffsets, - if (opts.multicastMask) |value| &.{value} else &.{}, - if (opts.l2CacheHint) |value| &.{value} else &.{}, - if (opts.predicate) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.tensor.shared.cluster.global.override`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_tensor_shared_cluster_global_override( - ctx: *mlir.Context, - dstMem: *const mlir.Value, - tmaDesc: *const mlir.Value, - mbar: *const mlir.Value, - overrideAddr: *const mlir.Value, - coordinates: []const *const mlir.Value, - im2colArgs: []const *const mlir.Value, - tensorSize: []const *const mlir.Value, - lowerStride: []const *const mlir.Value, - opts: struct { - upperStride: ?*const mlir.Value = null, - multicastMask: ?*const mlir.Value = null, - l2CacheHint: ?*const mlir.Value = null, - group: ?*const mlir.Attribute = null, - validatePattern: ?*const mlir.Attribute = null, - mode: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); - if (opts.validatePattern) |value| attributes.appendAssumeCapacity(.named(ctx, "validatePattern", value)); - if (opts.mode) |value| attributes.appendAssumeCapacity(.named(ctx, "mode", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.tensor.shared.cluster.global.override", .{ - .operands = .{ .variadic = &.{ - &.{dstMem}, - &.{tmaDesc}, - &.{mbar}, - &.{overrideAddr}, - coordinates, - im2colArgs, - tensorSize, - lowerStride, - if (opts.upperStride) |value| &.{value} else &.{}, - if (opts.multicastMask) |value| &.{value} else &.{}, - if (opts.l2CacheHint) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.bulk.wait_group`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_bulk_wait_group( - ctx: *mlir.Context, - group: *const mlir.Attribute, - opts: struct { - read: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "group", group)); - if (opts.read) |value| attributes.appendAssumeCapacity(.named(ctx, "read", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.bulk.wait_group", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.commit.group`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_commit_group(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.cp.async.commit.group", .{ - .location = location, - }); -} - -/// `nvvm.cp.async.mbarrier.arrive`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_mbarrier_arrive( - ctx: *mlir.Context, - addr: *const mlir.Value, - opts: struct { - noinc: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.noinc) |value| attributes.appendAssumeCapacity(.named(ctx, "noinc", value)); - return mlir.Operation.make(ctx, "nvvm.cp.async.mbarrier.arrive", .{ - .operands = .{ .flat = &.{addr} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cp.async.shared.global`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_shared_global( - ctx: *mlir.Context, - dst: *const mlir.Value, - src: *const mlir.Value, - size: *const mlir.Attribute, - modifier: *const mlir.Attribute, - opts: struct { - cpSize: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.cp.async.shared.global", .{ - .operands = .{ .flat = if (opts.cpSize) |value| &.{ dst, src, value } else &.{ dst, src } }, - .attributes = &.{ - .named(ctx, "size", size), - .named(ctx, "modifier", modifier), - }, - .location = location, - }); -} - -/// `nvvm.cp.async.wait.group`. Result types are explicit; attributes use mlir.Attribute. -pub fn cp_async_wait_group(ctx: *mlir.Context, n: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.cp.async.wait.group", .{ - .attributes = &.{ - .named(ctx, "n", n), - }, - .location = location, - }); -} - -/// `nvvm.cvt.packfloat`. Result types are explicit; attributes use mlir.Attribute. -pub fn cvt_packfloat( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcC: *const mlir.Value, - res_type: *const mlir.Type, - from: *const mlir.Attribute, - to: *const mlir.Attribute, - opts: struct { - rnd: ?*const mlir.Attribute = null, - sat: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - extractHi: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "from", from)); - attributes.appendAssumeCapacity(.named(ctx, "to", to)); - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - if (opts.extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); - return mlir.Operation.make(ctx, "nvvm.cvt.packfloat", .{ - .operands = .{ .flat = &.{ srcA, srcC } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cvt.packfloat.f32`. Result types are explicit; attributes use mlir.Attribute. -pub fn cvt_packfloat_f32( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - srcC: *const mlir.Value, - res_type: *const mlir.Type, - to: *const mlir.Attribute, - opts: struct { - rnd: ?*const mlir.Attribute = null, - sat: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - extractHi: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "to", to)); - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - if (opts.extractHi) |value| attributes.appendAssumeCapacity(.named(ctx, "extractHi", value)); - return mlir.Operation.make(ctx, "nvvm.cvt.packfloat.f32", .{ - .operands = .{ .flat = &.{ srcA, srcB, srcC } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.cvt.to.f4x8.packed`. Result types are explicit; attributes use mlir.Attribute. -pub fn cvt_to_f4x8_packed( - ctx: *mlir.Context, - srcs: []const *const mlir.Value, - dst_type: *const mlir.Type, - srcType: *const mlir.Attribute, - dstType: *const mlir.Attribute, - opts: struct { - rnd: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - attributes.appendAssumeCapacity(.named(ctx, "srcType", srcType)); - attributes.appendAssumeCapacity(.named(ctx, "dstType", dstType)); - return mlir.Operation.make(ctx, "nvvm.cvt.to.f4x8.packed", .{ - .operands = .{ .flat = srcs }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.dot.accumulate.2way`. Result types are explicit; attributes use mlir.Attribute. -pub fn dot_accumulate_2way(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, b_hi: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.dot.accumulate.2way", .{ - .operands = .{ .flat = &.{ a, b, c_ } }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "a_type", a_type), - .named(ctx, "b_type", b_type), - .named(ctx, "b_hi", b_hi), - }, - .location = location, - }); -} - -/// `nvvm.dot.accumulate.4way`. Result types are explicit; attributes use mlir.Attribute. -pub fn dot_accumulate_4way(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, c_: *const mlir.Value, res_type: *const mlir.Type, a_type: *const mlir.Attribute, b_type: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.dot.accumulate.4way", .{ - .operands = .{ .flat = &.{ a, b, c_ } }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "a_type", a_type), - .named(ctx, "b_type", b_type), - }, - .location = location, - }); -} - -/// `nvvm.elect.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn elect_sync( - ctx: *mlir.Context, - pred_type: *const mlir.Type, - opts: struct { - membermask: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.elect.sync", .{ - .operands = .{ .flat = if (opts.membermask) |value| &.{value} else &.{} }, - .results = .{ .flat = &.{pred_type} }, - .location = location, - }); -} - -/// `nvvm.ex2`. Result types are explicit; attributes use mlir.Attribute. -pub fn ex2( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.ex2", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.exit`. Result types are explicit; attributes use mlir.Attribute. -pub fn exit(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.exit", .{ - .location = location, - }); -} - -/// `nvvm.fabs`. Result types are explicit; attributes use mlir.Attribute. -pub fn fabs( - ctx: *mlir.Context, - value_: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.fabs", .{ - .operands = .{ .flat = &.{value_} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.fence.mbarrier.init`. Result types are explicit; attributes use mlir.Attribute. -pub fn fence_mbarrier_init(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.fence.mbarrier.init", .{ - .location = location, - }); -} - -/// `nvvm.fence.proxy`. Result types are explicit; attributes use mlir.Attribute. -pub fn fence_proxy( - ctx: *mlir.Context, - kind: *const mlir.Attribute, - opts: struct { - space: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); - return mlir.Operation.make(ctx, "nvvm.fence.proxy", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.fence.proxy.acquire`. Result types are explicit; attributes use mlir.Attribute. -pub fn fence_proxy_acquire( - ctx: *mlir.Context, - addr: *const mlir.Value, - size: *const mlir.Value, - scope: *const mlir.Attribute, - opts: struct { - fromProxy: ?*const mlir.Attribute = null, - toProxy: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scope", scope)); - if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); - if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); - return mlir.Operation.make(ctx, "nvvm.fence.proxy.acquire", .{ - .operands = .{ .flat = &.{ addr, size } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.fence.proxy.release`. Result types are explicit; attributes use mlir.Attribute. -pub fn fence_proxy_release( - ctx: *mlir.Context, - scope: *const mlir.Attribute, - opts: struct { - fromProxy: ?*const mlir.Attribute = null, - toProxy: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scope", scope)); - if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); - if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); - return mlir.Operation.make(ctx, "nvvm.fence.proxy.release", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.fence.proxy.sync_restrict`. Result types are explicit; attributes use mlir.Attribute. -pub fn fence_proxy_sync_restrict( - ctx: *mlir.Context, - order: *const mlir.Attribute, - opts: struct { - fromProxy: ?*const mlir.Attribute = null, - toProxy: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "order", order)); - if (opts.fromProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "fromProxy", value)); - if (opts.toProxy) |value| attributes.appendAssumeCapacity(.named(ctx, "toProxy", value)); - return mlir.Operation.make(ctx, "nvvm.fence.proxy.sync_restrict", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.fence.sc.cluster`. Result types are explicit; attributes use mlir.Attribute. -pub fn fence_sc_cluster(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.fence.sc.cluster", .{ - .location = location, - }); -} - -/// `nvvm.fence.sync_restrict`. Result types are explicit; attributes use mlir.Attribute. -pub fn fence_sync_restrict(ctx: *mlir.Context, order: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.fence.sync_restrict", .{ - .attributes = &.{ - .named(ctx, "order", order), - }, - .location = location, - }); -} - -/// `nvvm.fma`. Result types are explicit; attributes use mlir.Attribute. -pub fn fma( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c_: *const mlir.Value, - res_type: *const mlir.Type, - rnd: *const mlir.Attribute, - opts: struct { - sat: ?*const mlir.Attribute = null, - ftz: ?*const mlir.Attribute = null, - relu: ?*const mlir.Attribute = null, - oob: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "rnd", rnd)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - if (opts.relu) |value| attributes.appendAssumeCapacity(.named(ctx, "relu", value)); - if (opts.oob) |value| attributes.appendAssumeCapacity(.named(ctx, "oob", value)); - return mlir.Operation.make(ctx, "nvvm.fma", .{ - .operands = .{ .flat = &.{ a, b, c_ } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.fma.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn fma_packed_f32x2( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - srcC: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB, srcC } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn fma_packed_f32x2_bf16x2_f32x2_f32x2( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - srcC: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB, srcC } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn fma_packed_f32x2_f16x2_f32x2_f32x2( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - srcC: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - return mlir.Operation.make(ctx, "nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB, srcC } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.fmax`. Result types are explicit; attributes use mlir.Attribute. -pub fn fmax( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - c_: ?*const mlir.Value = null, - ftz: ?*const mlir.Attribute = null, - nan: ?*const mlir.Attribute = null, - abs: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); - if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); - return mlir.Operation.make(ctx, "nvvm.fmax", .{ - .operands = .{ .flat = if (opts.c_) |value| &.{ a, b, value } else &.{ a, b } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.fmin`. Result types are explicit; attributes use mlir.Attribute. -pub fn fmin( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - c_: ?*const mlir.Value = null, - ftz: ?*const mlir.Attribute = null, - nan: ?*const mlir.Attribute = null, - abs: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); - if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); - return mlir.Operation.make(ctx, "nvvm.fmin", .{ - .operands = .{ .flat = if (opts.c_) |value| &.{ a, b, value } else &.{ a, b } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.griddepcontrol`. Result types are explicit; attributes use mlir.Attribute. -pub fn griddepcontrol(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.griddepcontrol", .{ - .attributes = &.{ - .named(ctx, "kind", kind), - }, - .location = location, - }); -} - -/// `nvvm.inline_ptx`. Result types are explicit; attributes use mlir.Attribute. -pub fn inline_ptx( - ctx: *mlir.Context, - readOnlyArgs: []const *const mlir.Value, - readWriteArgs: []const *const mlir.Value, - ptxCode: *const mlir.Attribute, - opts: struct { - predicate: ?*const mlir.Value = null, - writeOnlyArgs_type: ?*const mlir.Type = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (opts.writeOnlyArgs_type) |value| results.appendAssumeCapacity(value); - return mlir.Operation.make(ctx, "nvvm.inline_ptx", .{ - .operands = .{ .variadic = &.{ - readOnlyArgs, - readWriteArgs, - if (opts.predicate) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = results.constSlice() }, - .attributes = &.{ - .named(ctx, "ptxCode", ptxCode), - }, - .location = location, - }); -} - -/// `nvvm.ldmatrix`. Result types are explicit; attributes use mlir.Attribute. -pub fn ldmatrix( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - num: *const mlir.Attribute, - layout: *const mlir.Attribute, - opts: struct { - shape: ?*const mlir.Attribute = null, - srcFormat: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "num", num)); - attributes.appendAssumeCapacity(.named(ctx, "layout", layout)); - if (opts.shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); - if (opts.srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); - return mlir.Operation.make(ctx, "nvvm.ldmatrix", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.load.ext`. Result types are explicit; attributes use mlir.Attribute. -pub fn load_ext( - ctx: *mlir.Context, - addr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - l2CacheHint: ?*const mlir.Value = null, - order: ?*const mlir.Attribute = null, - scope: ?*const mlir.Attribute = null, - prefetch_: ?*const mlir.Attribute = null, - evict: ?*const mlir.Attribute = null, - cacheModifier: ?*const mlir.Attribute = null, - sharedSpace: ?*const mlir.Attribute = null, - unified: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; - if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.prefetch_) |value| attributes.appendAssumeCapacity(.named(ctx, "prefetch", value)); - if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); - if (opts.cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); - if (opts.sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); - if (opts.unified) |value| attributes.appendAssumeCapacity(.named(ctx, "unified", value)); - return mlir.Operation.make(ctx, "nvvm.load.ext", .{ - .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ addr, value } else &.{addr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.log2`. Result types are explicit; attributes use mlir.Attribute. -pub fn log2( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.log2", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mapa`. Result types are explicit; attributes use mlir.Attribute. -pub fn mapa(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mapa", .{ - .operands = .{ .flat = &.{ a, b } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.match.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn match_sync(ctx: *mlir.Context, thread_mask: *const mlir.Value, val: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.match.sync", .{ - .operands = .{ .flat = &.{ thread_mask, val } }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "kind", kind), - }, - .location = location, - }); -} - -/// `nvvm.mbarrier.arrive`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_arrive( - ctx: *mlir.Context, - addr: *const mlir.Value, - opts: struct { - count: ?*const mlir.Value = null, - multicastMask: ?*const mlir.Value = null, - res_type: ?*const mlir.Type = null, - scope: ?*const mlir.Attribute = null, - relaxed: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (opts.res_type) |value| results.appendAssumeCapacity(value); - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive", .{ - .operands = .{ .variadic = &.{ - &.{addr}, - if (opts.count) |value| &.{value} else &.{}, - if (opts.multicastMask) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = results.constSlice() }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.arrive.expect_tx`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_arrive_expect_tx( - ctx: *mlir.Context, - addr: *const mlir.Value, - txcount: *const mlir.Value, - opts: struct { - multicastMask: ?*const mlir.Value = null, - predicate: ?*const mlir.Value = null, - res_type: ?*const mlir.Type = null, - scope: ?*const mlir.Attribute = null, - relaxed: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (opts.res_type) |value| results.appendAssumeCapacity(value); - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive.expect_tx", .{ - .operands = .{ .variadic = &.{ - &.{addr}, - &.{txcount}, - if (opts.multicastMask) |value| &.{value} else &.{}, - if (opts.predicate) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = results.constSlice() }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.arrive.nocomplete`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_arrive_nocomplete(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive.nocomplete", .{ - .operands = .{ .flat = &.{ addr, count } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.mbarrier.arrive_drop`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_arrive_drop( - ctx: *mlir.Context, - addr: *const mlir.Value, - opts: struct { - count: ?*const mlir.Value = null, - multicastMask: ?*const mlir.Value = null, - res_type: ?*const mlir.Type = null, - scope: ?*const mlir.Attribute = null, - relaxed: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (opts.res_type) |value| results.appendAssumeCapacity(value); - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop", .{ - .operands = .{ .variadic = &.{ - &.{addr}, - if (opts.count) |value| &.{value} else &.{}, - if (opts.multicastMask) |value| &.{value} else &.{}, - } }, - .results = .{ .flat = results.constSlice() }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.arrive_drop.expect_tx`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_arrive_drop_expect_tx( - ctx: *mlir.Context, - addr: *const mlir.Value, - txcount: *const mlir.Value, - opts: struct { - multicastMask: ?*const mlir.Value = null, - res_type: ?*const mlir.Type = null, - scope: ?*const mlir.Attribute = null, - relaxed: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var results: stdx.BoundedArray(*const mlir.Type, 256) = .empty; - if (opts.res_type) |value| results.appendAssumeCapacity(value); - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop.expect_tx", .{ - .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, - .results = .{ .flat = results.constSlice() }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.arrive_drop.nocomplete`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_arrive_drop_nocomplete(ctx: *mlir.Context, addr: *const mlir.Value, count: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mbarrier.arrive_drop.nocomplete", .{ - .operands = .{ .flat = &.{ addr, count } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.mbarrier.check.layout`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_check_layout(ctx: *mlir.Context, addr: *const mlir.Value, res_type: *const mlir.Type, layout: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mbarrier.check.layout", .{ - .operands = .{ .flat = &.{addr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "layout", layout), - }, - .location = location, - }); -} - -/// `nvvm.mbarrier.complete_tx`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_complete_tx( - ctx: *mlir.Context, - addr: *const mlir.Value, - txcount: *const mlir.Value, - opts: struct { - report: ?*const mlir.Value = null, - multicastMask: ?*const mlir.Value = null, - scope: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.complete_tx", .{ - .operands = .{ .variadic = &.{ - &.{addr}, - &.{txcount}, - if (opts.report) |value| &.{value} else &.{}, - if (opts.multicastMask) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.expect_tx`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_expect_tx( - ctx: *mlir.Context, - addr: *const mlir.Value, - txcount: *const mlir.Value, - opts: struct { - multicastMask: ?*const mlir.Value = null, - scope: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.expect_tx", .{ - .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, txcount, value } else &.{ addr, txcount } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.init`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_init( - ctx: *mlir.Context, - addr: *const mlir.Value, - count: *const mlir.Value, - opts: struct { - predicate: ?*const mlir.Value = null, - layout: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.init", .{ - .operands = .{ .flat = if (opts.predicate) |value| &.{ addr, count, value } else &.{ addr, count } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.inval`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_inval(ctx: *mlir.Context, addr: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mbarrier.inval", .{ - .operands = .{ .flat = &.{addr} }, - .location = location, - }); -} - -/// `nvvm.mbarrier.test.wait`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_test_wait( - ctx: *mlir.Context, - addr: *const mlir.Value, - stateOrPhase: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - scope: ?*const mlir.Attribute = null, - relaxed: ?*const mlir.Attribute = null, - mbarrierPhase: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); - if (opts.mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.test.wait", .{ - .operands = .{ .flat = &.{ addr, stateOrPhase } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.try_wait`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_try_wait( - ctx: *mlir.Context, - addr: *const mlir.Value, - stateOrPhase: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - ticks: ?*const mlir.Value = null, - scope: ?*const mlir.Attribute = null, - relaxed: ?*const mlir.Attribute = null, - mbarrierPhase: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.relaxed) |value| attributes.appendAssumeCapacity(.named(ctx, "relaxed", value)); - if (opts.mbarrierPhase) |value| attributes.appendAssumeCapacity(.named(ctx, "mbarrierPhase", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait", .{ - .operands = .{ .flat = if (opts.ticks) |value| &.{ addr, stateOrPhase, value } else &.{ addr, stateOrPhase } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.try_wait.parity`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_try_wait_parity( - ctx: *mlir.Context, - addr: *const mlir.Value, - phase: *const mlir.Value, - ticks: *const mlir.Value, - opts: struct { - useIntrinsic: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.useIntrinsic) |value| attributes.appendAssumeCapacity(.named(ctx, "useIntrinsic", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity", .{ - .operands = .{ .flat = &.{ addr, phase, ticks } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.try_wait.parity.timelimit`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_try_wait_parity_timelimit( - ctx: *mlir.Context, - addr: *const mlir.Value, - phase: *const mlir.Value, - timeLimit: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - scope: ?*const mlir.Attribute = null, - order: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.parity.timelimit", .{ - .operands = .{ .flat = &.{ addr, phase, timeLimit } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.try_wait.timelimit`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_try_wait_timelimit( - ctx: *mlir.Context, - addr: *const mlir.Value, - state: *const mlir.Value, - timeLimit: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - scope: ?*const mlir.Attribute = null, - order: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.try_wait.timelimit", .{ - .operands = .{ .flat = &.{ addr, state, timeLimit } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.txn`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_txn( - ctx: *mlir.Context, - addr: *const mlir.Value, - count: *const mlir.Value, - kind: *const mlir.Attribute, - opts: struct { - multicast: ?*const mlir.Value = null, - space: ?*const mlir.Attribute = null, - scope: ?*const mlir.Attribute = null, - order: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.space) |value| attributes.appendAssumeCapacity(.named(ctx, "space", value)); - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.txn", .{ - .operands = .{ .flat = if (opts.multicast) |value| &.{ addr, count, value } else &.{ addr, count } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.txn.cta`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_txn_cta( - ctx: *mlir.Context, - addr: *const mlir.Value, - count: *const mlir.Value, - res_type: *const mlir.Type, - kind: *const mlir.Attribute, - opts: struct { - scope: ?*const mlir.Attribute = null, - order: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.txn.cta", .{ - .operands = .{ .flat = &.{ addr, count } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.wait`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_wait( - ctx: *mlir.Context, - addr: *const mlir.Value, - state: *const mlir.Value, - res_type: *const mlir.Type, - kind: *const mlir.Attribute, - opts: struct { - scope: ?*const mlir.Attribute = null, - order: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.wait", .{ - .operands = .{ .flat = &.{ addr, state } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mbarrier.wait.parity`. Result types are explicit; attributes use mlir.Attribute. -pub fn mbarrier_wait_parity( - ctx: *mlir.Context, - addr: *const mlir.Value, - phase: *const mlir.Value, - res_type: *const mlir.Type, - kind: *const mlir.Attribute, - opts: struct { - scope: ?*const mlir.Attribute = null, - order: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); - return mlir.Operation.make(ctx, "nvvm.mbarrier.wait.parity", .{ - .operands = .{ .flat = &.{ addr, phase } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.memory.barrier`. Result types are explicit; attributes use mlir.Attribute. -pub fn memory_barrier(ctx: *mlir.Context, scope: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.memory.barrier", .{ - .attributes = &.{ - .named(ctx, "scope", scope), - }, - .location = location, - }); -} - -/// `nvvm.mma.block_scale`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_block_scale( - ctx: *mlir.Context, - operandA: []const *const mlir.Value, - operandB: []const *const mlir.Value, - operandC: []const *const mlir.Value, - scaleAData: *const mlir.Value, - byteIdA: *const mlir.Value, - threadIdA: *const mlir.Value, - scaleBData: *const mlir.Value, - byteIdB: *const mlir.Value, - threadIdB: *const mlir.Value, - res_type: *const mlir.Type, - shape: *const mlir.Attribute, - scaleVecSize: *const mlir.Attribute, - blockScaleFormat: *const mlir.Attribute, - kind: *const mlir.Attribute, - opts: struct { - multiplicandAPtxType: ?*const mlir.Attribute = null, - multiplicandBPtxType: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); - if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); - attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); - attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - return mlir.Operation.make(ctx, "nvvm.mma.block_scale", .{ - .operands = .{ .variadic = &.{ - operandA, - operandB, - operandC, - &.{scaleAData}, - &.{byteIdA}, - &.{threadIdA}, - &.{scaleBData}, - &.{byteIdB}, - &.{threadIdB}, - } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mma.block_scale.internal`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_block_scale_internal( - ctx: *mlir.Context, - operandA: *const mlir.Value, - scaleAData: *const mlir.Value, - byteIdA: *const mlir.Value, - threadIdA: *const mlir.Value, - operandB: *const mlir.Value, - scaleBData: *const mlir.Value, - byteIdB: *const mlir.Value, - threadIdB: *const mlir.Value, - operandC: *const mlir.Value, - res_type: *const mlir.Type, - shape: *const mlir.Attribute, - layoutA: *const mlir.Attribute, - layoutB: *const mlir.Attribute, - aType: *const mlir.Attribute, - bType: *const mlir.Attribute, - cType: *const mlir.Attribute, - scaleVecSize: *const mlir.Attribute, - blockScaleFormat: *const mlir.Attribute, - opts: struct { - kind: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); - attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); - attributes.appendAssumeCapacity(.named(ctx, "aType", aType)); - attributes.appendAssumeCapacity(.named(ctx, "bType", bType)); - attributes.appendAssumeCapacity(.named(ctx, "cType", cType)); - attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); - attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); - if (opts.kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); - return mlir.Operation.make(ctx, "nvvm.mma.block_scale.internal", .{ - .operands = .{ .flat = &.{ operandA, scaleAData, byteIdA, threadIdA, operandB, scaleBData, byteIdB, threadIdB, operandC } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mma.sp.block_scale`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_sp_block_scale( - ctx: *mlir.Context, - operandA: []const *const mlir.Value, - operandB: []const *const mlir.Value, - operandC: []const *const mlir.Value, - sparseMetadata: *const mlir.Value, - sparsitySelector: *const mlir.Value, - scaleAData: *const mlir.Value, - byteIdA: *const mlir.Value, - threadIdA: *const mlir.Value, - scaleBData: *const mlir.Value, - byteIdB: *const mlir.Value, - threadIdB: *const mlir.Value, - res_type: *const mlir.Type, - shape: *const mlir.Attribute, - scaleVecSize: *const mlir.Attribute, - blockScaleFormat: *const mlir.Attribute, - kind: *const mlir.Attribute, - opts: struct { - multiplicandAPtxType: ?*const mlir.Attribute = null, - multiplicandBPtxType: ?*const mlir.Attribute = null, - orderedMetadata: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); - if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); - attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); - attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); - return mlir.Operation.make(ctx, "nvvm.mma.sp.block_scale", .{ - .operands = .{ .variadic = &.{ - operandA, - operandB, - operandC, - &.{sparseMetadata}, - &.{sparsitySelector}, - &.{scaleAData}, - &.{byteIdA}, - &.{threadIdA}, - &.{scaleBData}, - &.{byteIdB}, - &.{threadIdB}, - } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mma.sp.block_scale.internal`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_sp_block_scale_internal( - ctx: *mlir.Context, - operandA: *const mlir.Value, - scaleAData: *const mlir.Value, - byteIdA: *const mlir.Value, - threadIdA: *const mlir.Value, - operandB: *const mlir.Value, - scaleBData: *const mlir.Value, - byteIdB: *const mlir.Value, - threadIdB: *const mlir.Value, - operandC: *const mlir.Value, - sp_metadata: *const mlir.Value, - selector: *const mlir.Value, - res_type: *const mlir.Type, - shape: *const mlir.Attribute, - layoutA: *const mlir.Attribute, - layoutB: *const mlir.Attribute, - aType: *const mlir.Attribute, - bType: *const mlir.Attribute, - cType: *const mlir.Attribute, - scaleVecSize: *const mlir.Attribute, - blockScaleFormat: *const mlir.Attribute, - sparsityFormat: *const mlir.Attribute, - opts: struct { - b1Op: ?*const mlir.Attribute = null, - intOverflowBehavior: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 11) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (opts.b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); - if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); - attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); - attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); - attributes.appendAssumeCapacity(.named(ctx, "aType", aType)); - attributes.appendAssumeCapacity(.named(ctx, "bType", bType)); - attributes.appendAssumeCapacity(.named(ctx, "cType", cType)); - attributes.appendAssumeCapacity(.named(ctx, "scaleVecSize", scaleVecSize)); - attributes.appendAssumeCapacity(.named(ctx, "blockScaleFormat", blockScaleFormat)); - attributes.appendAssumeCapacity(.named(ctx, "sparsityFormat", sparsityFormat)); - return mlir.Operation.make(ctx, "nvvm.mma.sp.block_scale.internal", .{ - .operands = .{ .flat = &.{ operandA, scaleAData, byteIdA, threadIdA, operandB, scaleBData, byteIdB, threadIdB, operandC, sp_metadata, selector } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mma.sp.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_sp_sync( - ctx: *mlir.Context, - operandA: []const *const mlir.Value, - operandB: []const *const mlir.Value, - operandC: []const *const mlir.Value, - sparseMetadata: *const mlir.Value, - sparsitySelector: *const mlir.Value, - res_type: *const mlir.Type, - shape: *const mlir.Attribute, - opts: struct { - intOverflowBehavior: ?*const mlir.Attribute = null, - multiplicandAPtxType: ?*const mlir.Attribute = null, - multiplicandBPtxType: ?*const mlir.Attribute = null, - orderedMetadata: ?*const mlir.Attribute = null, - kind: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); - if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); - if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); - if (opts.orderedMetadata) |value| attributes.appendAssumeCapacity(.named(ctx, "orderedMetadata", value)); - if (opts.kind) |value| attributes.appendAssumeCapacity(.named(ctx, "kind", value)); - return mlir.Operation.make(ctx, "nvvm.mma.sp.sync", .{ - .operands = .{ .variadic = &.{ - operandA, - operandB, - operandC, - &.{sparseMetadata}, - &.{sparsitySelector}, - } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mma.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_sync( - ctx: *mlir.Context, - operandA: []const *const mlir.Value, - operandB: []const *const mlir.Value, - operandC: []const *const mlir.Value, - res_type: *const mlir.Type, - shape: *const mlir.Attribute, - layoutA: *const mlir.Attribute, - layoutB: *const mlir.Attribute, - opts: struct { - b1Op: ?*const mlir.Attribute = null, - intOverflowBehavior: ?*const mlir.Attribute = null, - multiplicandAPtxType: ?*const mlir.Attribute = null, - multiplicandBPtxType: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (opts.b1Op) |value| attributes.appendAssumeCapacity(.named(ctx, "b1Op", value)); - if (opts.intOverflowBehavior) |value| attributes.appendAssumeCapacity(.named(ctx, "intOverflowBehavior", value)); - attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); - attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); - if (opts.multiplicandAPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandAPtxType", value)); - if (opts.multiplicandBPtxType) |value| attributes.appendAssumeCapacity(.named(ctx, "multiplicandBPtxType", value)); - return mlir.Operation.make(ctx, "nvvm.mma.sync", .{ - .operands = .{ .variadic = &.{ - operandA, - operandB, - operandC, - } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn mma_smem_desc( - ctx: *mlir.Context, - pointer: *const mlir.Value, - ldm: *const mlir.Value, - stride: *const mlir.Value, - baseOffset: *const mlir.Value, - swizzle: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - mmaDescVersion: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.mmaDescVersion) |value| attributes.appendAssumeCapacity(.named(ctx, "mmaDescVersion", value)); - return mlir.Operation.make(ctx, "nvvm.mma_smem_desc", .{ - .operands = .{ .flat = &.{ pointer, ldm, stride, baseOffset, swizzle } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.movmatrix`. Result types are explicit; attributes use mlir.Attribute. -pub fn movmatrix( - ctx: *mlir.Context, - src: *const mlir.Value, - dst_type: *const mlir.Type, - shape: *const mlir.Attribute, - eltType: *const mlir.Attribute, - opts: struct { - layout: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (opts.layout) |value| attributes.appendAssumeCapacity(.named(ctx, "layout", value)); - attributes.appendAssumeCapacity(.named(ctx, "eltType", eltType)); - return mlir.Operation.make(ctx, "nvvm.movmatrix", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{dst_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mul`. Result types are explicit; attributes use mlir.Attribute. -pub fn mul( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - res_type: *const mlir.Type, - mode: *const mlir.Attribute, - opts: struct { - isSigned: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "mode", mode)); - if (opts.isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); - return mlir.Operation.make(ctx, "nvvm.mul", .{ - .operands = .{ .flat = &.{ a, b } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.mul.packed.bf16x2.bf16x2.f16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn mul_packed_bf16x2_bf16x2_f16x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mul.packed.bf16x2.bf16x2.f16x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.mul.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn mul_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mul.packed.bf16x2.f32x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.mul.packed.f16x2.f16x2.bf16x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn mul_packed_f16x2_f16x2_bf16x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mul.packed.f16x2.f16x2.bf16x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.mul.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn mul_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.mul.packed.f16x2.f32x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.mul.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn mul_packed_f32x2( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.mul.packed.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.multimem.ld.reduce`. Result types are explicit; attributes use mlir.Attribute. -pub fn multimem_ld_reduce( - ctx: *mlir.Context, - addr: *const mlir.Value, - res_type: *const mlir.Type, - ordering: *const mlir.Attribute, - op: *const mlir.Attribute, - dataType: *const mlir.Attribute, - opts: struct { - accPrec: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "ordering", ordering)); - attributes.appendAssumeCapacity(.named(ctx, "op", op)); - attributes.appendAssumeCapacity(.named(ctx, "dataType", dataType)); - if (opts.accPrec) |value| attributes.appendAssumeCapacity(.named(ctx, "accPrec", value)); - return mlir.Operation.make(ctx, "nvvm.multimem.ld.reduce", .{ - .operands = .{ .flat = &.{addr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.multimem.red`. Result types are explicit; attributes use mlir.Attribute. -pub fn multimem_red(ctx: *mlir.Context, val: *const mlir.Value, addr: *const mlir.Value, ordering: *const mlir.Attribute, scope: *const mlir.Attribute, op: *const mlir.Attribute, dataType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.multimem.red", .{ - .operands = .{ .flat = &.{ val, addr } }, - .attributes = &.{ - .named(ctx, "ordering", ordering), - .named(ctx, "scope", scope), - .named(ctx, "op", op), - .named(ctx, "dataType", dataType), - }, - .location = location, - }); -} - -/// `nvvm.multimem.st`. Result types are explicit; attributes use mlir.Attribute. -pub fn multimem_st(ctx: *mlir.Context, val: *const mlir.Value, addr: *const mlir.Value, ordering: *const mlir.Attribute, scope: *const mlir.Attribute, dataType: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.multimem.st", .{ - .operands = .{ .flat = &.{ val, addr } }, - .attributes = &.{ - .named(ctx, "ordering", ordering), - .named(ctx, "scope", scope), - .named(ctx, "dataType", dataType), - }, - .location = location, - }); -} - -/// `nvvm.nanosleep`. Result types are explicit; attributes use mlir.Attribute. -pub fn nanosleep(ctx: *mlir.Context, duration: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.nanosleep", .{ - .operands = .{ .flat = &.{duration} }, - .location = location, - }); -} - -/// `nvvm.pmevent`. Result types are explicit; attributes use mlir.Attribute. -pub fn pmevent( - ctx: *mlir.Context, - opts: struct { - maskedEventId: ?*const mlir.Attribute = null, - eventId: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.maskedEventId) |value| attributes.appendAssumeCapacity(.named(ctx, "maskedEventId", value)); - if (opts.eventId) |value| attributes.appendAssumeCapacity(.named(ctx, "eventId", value)); - return mlir.Operation.make(ctx, "nvvm.pmevent", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.prefetch`. Result types are explicit; attributes use mlir.Attribute. -pub fn prefetch( - ctx: *mlir.Context, - addr: *const mlir.Value, - opts: struct { - predicate: ?*const mlir.Value = null, - cacheLevel: ?*const mlir.Attribute = null, - evictPriority: ?*const mlir.Attribute = null, - tensormap: ?*const mlir.Attribute = null, - uniform: ?*const mlir.Attribute = null, - in_param_space: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.cacheLevel) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheLevel", value)); - if (opts.evictPriority) |value| attributes.appendAssumeCapacity(.named(ctx, "evictPriority", value)); - if (opts.tensormap) |value| attributes.appendAssumeCapacity(.named(ctx, "tensormap", value)); - if (opts.uniform) |value| attributes.appendAssumeCapacity(.named(ctx, "uniform", value)); - if (opts.in_param_space) |value| attributes.appendAssumeCapacity(.named(ctx, "in_param_space", value)); - return mlir.Operation.make(ctx, "nvvm.prefetch", .{ - .operands = .{ .flat = if (opts.predicate) |value| &.{ addr, value } else &.{addr} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.prmt`. Result types are explicit; attributes use mlir.Attribute. -pub fn prmt( - ctx: *mlir.Context, - lo: *const mlir.Value, - selector: *const mlir.Value, - res_type: *const mlir.Type, - mode: *const mlir.Attribute, - opts: struct { - hi: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.prmt", .{ - .operands = .{ .flat = if (opts.hi) |value| &.{ lo, value, selector } else &.{ lo, selector } }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "mode", mode), - }, - .location = location, - }); -} - -/// `nvvm.rcp.approx.ftz.f`. Result types are explicit; attributes use mlir.Attribute. -pub fn rcp_approx_ftz_f(ctx: *mlir.Context, arg: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.rcp.approx.ftz.f", .{ - .operands = .{ .flat = &.{arg} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.aggr.smem.size`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_aggr_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.aggr.smem.size", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.clock`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_clock(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clock", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.clock64`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_clock64(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clock64", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.cluster.ctaid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_ctaid_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.cluster.ctaid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_ctaid_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.cluster.ctaid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_ctaid_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctaid.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.cluster.ctarank`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_ctarank( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.ctarank", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.cluster.nctaid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_nctaid_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.cluster.nctaid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_nctaid_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.cluster.nctaid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_nctaid_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctaid.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.cluster.nctarank`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_cluster_nctarank( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.cluster.nctarank", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.clusterid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_clusterid_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.clusterid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_clusterid_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.clusterid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_clusterid_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.clusterid.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.ctaid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ctaid_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.ctaid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ctaid_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.ctaid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ctaid_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ctaid.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.dynamic.smem.size`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_dynamic_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.dynamic.smem.size", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg0`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg0(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg0", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg1`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg1(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg1", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg10`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg10(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg10", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg11`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg11(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg11", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg12`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg12(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg12", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg13`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg13(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg13", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg14`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg14(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg14", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg15`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg15(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg15", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg16`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg16(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg16", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg17`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg17(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg17", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg18`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg18(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg18", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg19`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg19(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg19", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg2`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg2(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg2", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg20`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg20(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg20", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg21`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg21(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg21", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg22`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg22(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg22", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg23`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg23(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg23", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg24`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg24(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg24", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg25`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg25(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg25", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg26`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg26(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg26", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg27`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg27(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg27", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg28`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg28(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg28", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg29`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg29(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg29", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg3`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg3(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg3", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg30`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg30(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg30", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg31`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg31(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg31", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg4`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg4(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg4", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg5`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg5(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg5", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg6`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg6(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg6", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg7`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg7(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg7", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg8`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg8(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg8", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.envreg9`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_envreg9(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.envreg9", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.globaltimer`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_globaltimer(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.globaltimer", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.globaltimer.lo`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_globaltimer_lo(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.globaltimer.lo", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.gridid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_gridid( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.gridid", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.laneid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_laneid( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.laneid", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.lanemask.eq`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_lanemask_eq(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.eq", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.lanemask.ge`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_lanemask_ge(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.ge", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.lanemask.gt`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_lanemask_gt(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.gt", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.lanemask.le`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_lanemask_le(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.le", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.lanemask.lt`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_lanemask_lt(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.lanemask.lt", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.nclusterid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nclusterid_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.nclusterid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nclusterid_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.nclusterid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nclusterid_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nclusterid.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.nctaid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nctaid_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.nctaid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nctaid_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.nctaid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nctaid_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nctaid.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.nsmid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nsmid( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nsmid", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.ntid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ntid_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.ntid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ntid_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.ntid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_ntid_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.ntid.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.nwarpid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_nwarpid( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.nwarpid", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.smid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_smid( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.smid", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.tid.x`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_tid_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.tid.y`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_tid_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.tid.z`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_tid_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.tid.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.total.smem.size`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_total_smem_size(ctx: *mlir.Context, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.total.smem.size", .{ - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.warpid`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_warpid( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.warpid", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.read.ptx.sreg.warpsize`. Result types are explicit; attributes use mlir.Attribute. -pub fn read_ptx_sreg_warpsize( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |value| attributes.appendAssumeCapacity(.named(ctx, "range", value)); - return mlir.Operation.make(ctx, "nvvm.read.ptx.sreg.warpsize", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.red`. Result types are explicit; attributes use mlir.Attribute. -pub fn red( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - op: *const mlir.Attribute, - type_: *const mlir.Attribute, - opts: struct { - cacheHint: ?*const mlir.Value = null, - memOrder: ?*const mlir.Attribute = null, - memScope: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.memOrder) |value| attributes.appendAssumeCapacity(.named(ctx, "memOrder", value)); - if (opts.memScope) |value| attributes.appendAssumeCapacity(.named(ctx, "memScope", value)); - attributes.appendAssumeCapacity(.named(ctx, "op", op)); - attributes.appendAssumeCapacity(.named(ctx, "type", type_)); - return mlir.Operation.make(ctx, "nvvm.red", .{ - .operands = .{ .flat = if (opts.cacheHint) |value| &.{ a, b, value } else &.{ a, b } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.redux.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn redux_sync( - ctx: *mlir.Context, - val: *const mlir.Value, - mask_and_clamp: *const mlir.Value, - res_type: *const mlir.Type, - kind: *const mlir.Attribute, - opts: struct { - abs: ?*const mlir.Attribute = null, - nan: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); - if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); - return mlir.Operation.make(ctx, "nvvm.redux.sync", .{ - .operands = .{ .flat = &.{ val, mask_and_clamp } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.rsqrt`. Result types are explicit; attributes use mlir.Attribute. -pub fn rsqrt( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.rsqrt", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.setmaxregister`. Result types are explicit; attributes use mlir.Attribute. -pub fn setmaxregister(ctx: *mlir.Context, regCount: *const mlir.Attribute, action: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.setmaxregister", .{ - .attributes = &.{ - .named(ctx, "regCount", regCount), - .named(ctx, "action", action), - }, - .location = location, - }); -} - -/// `nvvm.shfl.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn shfl_sync( - ctx: *mlir.Context, - thread_mask: *const mlir.Value, - val: *const mlir.Value, - offset: *const mlir.Value, - mask_and_clamp: *const mlir.Value, - res_type: *const mlir.Type, - kind: *const mlir.Attribute, - opts: struct { - return_value_and_is_valid: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.return_value_and_is_valid) |value| attributes.appendAssumeCapacity(.named(ctx, "return_value_and_is_valid", value)); - return mlir.Operation.make(ctx, "nvvm.shfl.sync", .{ - .operands = .{ .flat = &.{ thread_mask, val, offset, mask_and_clamp } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.sin`. Result types are explicit; attributes use mlir.Attribute. -pub fn sin( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.sin", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.spcompress`. Result types are explicit; attributes use mlir.Attribute. -pub fn spcompress(ctx: *mlir.Context, spDesc: *const mlir.Value, data: *const mlir.Value, metadata_type: *const mlir.Type, compressed_data_type: *const mlir.Type, compFactor: *const mlir.Attribute, indexSize: *const mlir.Attribute, elemSize: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.spcompress", .{ - .operands = .{ .flat = &.{ spDesc, data } }, - .results = .{ .flat = &.{ metadata_type, compressed_data_type } }, - .attributes = &.{ - .named(ctx, "compFactor", compFactor), - .named(ctx, "indexSize", indexSize), - .named(ctx, "elemSize", elemSize), - .named(ctx, "repFactor", repFactor), - }, - .location = location, - }); -} - -/// `nvvm.spdecompress`. Result types are explicit; attributes use mlir.Attribute. -pub fn spdecompress(ctx: *mlir.Context, metadata: *const mlir.Value, compressed_data: *const mlir.Value, data_type: *const mlir.Type, compFactor: *const mlir.Attribute, indexSize: *const mlir.Attribute, elemSize: *const mlir.Attribute, repFactor: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.spdecompress", .{ - .operands = .{ .flat = &.{ metadata, compressed_data } }, - .results = .{ .flat = &.{data_type} }, - .attributes = &.{ - .named(ctx, "compFactor", compFactor), - .named(ctx, "indexSize", indexSize), - .named(ctx, "elemSize", elemSize), - .named(ctx, "repFactor", repFactor), - }, - .location = location, - }); -} - -/// `nvvm.st.bulk`. Result types are explicit; attributes use mlir.Attribute. -pub fn st_bulk( - ctx: *mlir.Context, - addr: *const mlir.Value, - size: *const mlir.Value, - opts: struct { - initVal: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.initVal) |value| attributes.appendAssumeCapacity(.named(ctx, "initVal", value)); - return mlir.Operation.make(ctx, "nvvm.st.bulk", .{ - .operands = .{ .flat = &.{ addr, size } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.stmatrix`. Result types are explicit; attributes use mlir.Attribute. -pub fn stmatrix( - ctx: *mlir.Context, - ptr: *const mlir.Value, - sources: []const *const mlir.Value, - layout: *const mlir.Attribute, - opts: struct { - shape: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; - operands.appendAssumeCapacity(ptr); - operands.appendSliceAssumeCapacity(sources); - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "layout", layout)); - if (opts.shape) |value| attributes.appendAssumeCapacity(.named(ctx, "shape", value)); - return mlir.Operation.make(ctx, "nvvm.stmatrix", .{ - .operands = .{ .flat = operands.constSlice() }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.store.async`. Result types are explicit; attributes use mlir.Attribute. -pub fn store_async(ctx: *mlir.Context, addr: *const mlir.Value, value_: *const mlir.Value, mbarrier: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.store.async", .{ - .operands = .{ .flat = &.{ addr, value_, mbarrier } }, - .location = location, - }); -} - -/// `nvvm.store.ext`. Result types are explicit; attributes use mlir.Attribute. -pub fn store_ext( - ctx: *mlir.Context, - value_: *const mlir.Value, - addr: *const mlir.Value, - opts: struct { - l2CacheHint: ?*const mlir.Value = null, - order: ?*const mlir.Attribute = null, - scope: ?*const mlir.Attribute = null, - evict: ?*const mlir.Attribute = null, - cacheModifier: ?*const mlir.Attribute = null, - sharedSpace: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.order) |value| attributes.appendAssumeCapacity(.named(ctx, "order", value)); - if (opts.scope) |value| attributes.appendAssumeCapacity(.named(ctx, "scope", value)); - if (opts.evict) |value| attributes.appendAssumeCapacity(.named(ctx, "evict", value)); - if (opts.cacheModifier) |value| attributes.appendAssumeCapacity(.named(ctx, "cacheModifier", value)); - if (opts.sharedSpace) |value| attributes.appendAssumeCapacity(.named(ctx, "sharedSpace", value)); - return mlir.Operation.make(ctx, "nvvm.store.ext", .{ - .operands = .{ .flat = if (opts.l2CacheHint) |value| &.{ value_, addr, value } else &.{ value_, addr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.sub.packed.bf16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn sub_packed_bf16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.sub.packed.bf16x2.f32x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.sub.packed.f16x2.f32x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn sub_packed_f16x2_f32x2_f32x2(ctx: *mlir.Context, srcA: *const mlir.Value, srcB: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.sub.packed.f16x2.f32x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.sub.packed.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn sub_packed_f32x2( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.sub.packed.f32x2.bf16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn sub_packed_f32x2_bf16x2_f32x2( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2.bf16x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.sub.packed.f32x2.f16x2.f32x2`. Result types are explicit; attributes use mlir.Attribute. -pub fn sub_packed_f32x2_f16x2_f32x2( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - return mlir.Operation.make(ctx, "nvvm.sub.packed.f32x2.f16x2.f32x2", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.subf`. Result types are explicit; attributes use mlir.Attribute. -pub fn subf( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - rnd: ?*const mlir.Attribute = null, - sat: ?*const mlir.Attribute = null, - ftz: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.rnd) |value| attributes.appendAssumeCapacity(.named(ctx, "rnd", value)); - if (opts.sat) |value| attributes.appendAssumeCapacity(.named(ctx, "sat", value)); - if (opts.ftz) |value| attributes.appendAssumeCapacity(.named(ctx, "ftz", value)); - return mlir.Operation.make(ctx, "nvvm.subf", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.alloc`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_alloc( - ctx: *mlir.Context, - addr: *const mlir.Value, - nCols: *const mlir.Value, - opts: struct { - isExclusive: ?*const mlir.Attribute = null, - group: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); - if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.alloc", .{ - .operands = .{ .flat = &.{ addr, nCols } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.commit`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_commit( - ctx: *mlir.Context, - addr: *const mlir.Value, - opts: struct { - multicastMask: ?*const mlir.Value = null, - smem_a_read: ?*const mlir.Attribute = null, - group: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.smem_a_read) |value| attributes.appendAssumeCapacity(.named(ctx, "smem_a_read", value)); - if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.commit", .{ - .operands = .{ .flat = if (opts.multicastMask) |value| &.{ addr, value } else &.{addr} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.cp`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_cp( - ctx: *mlir.Context, - taddr: *const mlir.Value, - smem_desc: *const mlir.Value, - shape: *const mlir.Attribute, - opts: struct { - group: ?*const mlir.Attribute = null, - multicast: ?*const mlir.Attribute = null, - srcFormat: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); - if (opts.multicast) |value| attributes.appendAssumeCapacity(.named(ctx, "multicast", value)); - if (opts.srcFormat) |value| attributes.appendAssumeCapacity(.named(ctx, "srcFormat", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.cp", .{ - .operands = .{ .flat = &.{ taddr, smem_desc } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.dealloc`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_dealloc( - ctx: *mlir.Context, - taddr: *const mlir.Value, - nCols: *const mlir.Value, - opts: struct { - isExclusive: ?*const mlir.Attribute = null, - group: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.isExclusive) |value| attributes.appendAssumeCapacity(.named(ctx, "isExclusive", value)); - if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.dealloc", .{ - .operands = .{ .flat = &.{ taddr, nCols } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.fence`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_fence(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05.fence", .{ - .attributes = &.{ - .named(ctx, "kind", kind), - }, - .location = location, - }); -} - -/// `nvvm.tcgen05.ld`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_ld( - ctx: *mlir.Context, - tmemAddr: *const mlir.Value, - res_type: *const mlir.Type, - shape: *const mlir.Attribute, - opts: struct { - offset: ?*const mlir.Value = null, - pack: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.pack) |value| attributes.appendAssumeCapacity(.named(ctx, "pack", value)); - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.ld", .{ - .operands = .{ .flat = if (opts.offset) |value| &.{ tmemAddr, value } else &.{tmemAddr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.ld.red`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_ld_red( - ctx: *mlir.Context, - addr: *const mlir.Value, - data_type: *const mlir.Type, - redVal_type: *const mlir.Type, - shape: *const mlir.Attribute, - op: *const mlir.Attribute, - opts: struct { - offset: ?*const mlir.Value = null, - abs: ?*const mlir.Attribute = null, - nan: ?*const mlir.Attribute = null, - isSigned: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - attributes.appendAssumeCapacity(.named(ctx, "op", op)); - if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); - if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); - if (opts.isSigned) |value| attributes.appendAssumeCapacity(.named(ctx, "isSigned", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.red", .{ - .operands = .{ .flat = if (opts.offset) |value| &.{ addr, value } else &.{addr} }, - .results = .{ .flat = &.{ data_type, redVal_type } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.ld.red.spcompress`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_ld_red_spcompress( - ctx: *mlir.Context, - tmemAddr: *const mlir.Value, - metadata_type: *const mlir.Type, - compressedData_type: *const mlir.Type, - redVal_type: *const mlir.Type, - shape: *const mlir.Attribute, - opKind: *const mlir.Attribute, - compFactor: *const mlir.Attribute, - repFactor: *const mlir.Attribute, - opts: struct { - abs: ?*const mlir.Attribute = null, - nan: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); - if (opts.nan) |value| attributes.appendAssumeCapacity(.named(ctx, "nan", value)); - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - attributes.appendAssumeCapacity(.named(ctx, "opKind", opKind)); - attributes.appendAssumeCapacity(.named(ctx, "compFactor", compFactor)); - attributes.appendAssumeCapacity(.named(ctx, "repFactor", repFactor)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.red.spcompress", .{ - .operands = .{ .flat = &.{tmemAddr} }, - .results = .{ .flat = &.{ metadata_type, compressedData_type, redVal_type } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.ld.spcompress`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_ld_spcompress( - ctx: *mlir.Context, - tmemAddr: *const mlir.Value, - metadata_type: *const mlir.Type, - compressedData_type: *const mlir.Type, - shape: *const mlir.Attribute, - opKind: *const mlir.Attribute, - compFactor: *const mlir.Attribute, - repFactor: *const mlir.Attribute, - opts: struct { - abs: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.abs) |value| attributes.appendAssumeCapacity(.named(ctx, "abs", value)); - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - attributes.appendAssumeCapacity(.named(ctx, "opKind", opKind)); - attributes.appendAssumeCapacity(.named(ctx, "compFactor", compFactor)); - attributes.appendAssumeCapacity(.named(ctx, "repFactor", repFactor)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.ld.spcompress", .{ - .operands = .{ .flat = &.{tmemAddr} }, - .results = .{ .flat = &.{ metadata_type, compressedData_type } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.mma`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma( - ctx: *mlir.Context, - matrixD: *const mlir.Value, - matrixA: *const mlir.Value, - matrixB: *const mlir.Value, - idesc: *const mlir.Value, - enableInputD: *const mlir.Value, - kind: *const mlir.Attribute, - ctaGroup: *const mlir.Attribute, - opts: struct { - scaleInputD: ?*const mlir.Value = null, - disableOutputLane: ?*const mlir.Value = null, - collectorOp: ?*const mlir.Attribute = null, - collectorOpB: ?*const mlir.Attribute = null, - aShift: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); - if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); - if (opts.aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.mma", .{ - .operands = .{ .variadic = &.{ - &.{matrixD}, - &.{matrixA}, - &.{matrixB}, - &.{idesc}, - &.{enableInputD}, - if (opts.scaleInputD) |value| &.{value} else &.{}, - if (opts.disableOutputLane) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.mma.block_scale`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_block_scale( - ctx: *mlir.Context, - matrixD: *const mlir.Value, - matrixA: *const mlir.Value, - matrixB: *const mlir.Value, - idesc: *const mlir.Value, - enableInputD: *const mlir.Value, - scaleA: *const mlir.Value, - scaleB: *const mlir.Value, - kind: *const mlir.Attribute, - ctaGroup: *const mlir.Attribute, - opts: struct { - blockScale: ?*const mlir.Attribute = null, - collectorOp: ?*const mlir.Attribute = null, - collectorOpB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); - if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); - if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.block_scale", .{ - .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, scaleA, scaleB } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.mma.block_scale.decompress_b`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_block_scale_decompress_b( - ctx: *mlir.Context, - matrixD: *const mlir.Value, - matrixA: *const mlir.Value, - matrixB: *const mlir.Value, - idesc: *const mlir.Value, - enableInputD: *const mlir.Value, - scaleA: *const mlir.Value, - scaleB: *const mlir.Value, - decompressBMetadata: *const mlir.Value, - kind: *const mlir.Attribute, - ctaGroup: *const mlir.Attribute, - opts: struct { - blockScale: ?*const mlir.Attribute = null, - collectorOpA: ?*const mlir.Attribute = null, - collectorOpB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); - if (opts.collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); - if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.block_scale.decompress_b", .{ - .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, scaleA, scaleB, decompressBMetadata } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.mma.decompress_b`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_decompress_b( - ctx: *mlir.Context, - matrixD: *const mlir.Value, - matrixA: *const mlir.Value, - matrixB: *const mlir.Value, - idesc: *const mlir.Value, - enableInputD: *const mlir.Value, - decompressBMetadata: *const mlir.Value, - kind: *const mlir.Attribute, - ctaGroup: *const mlir.Attribute, - opts: struct { - disableOutputLane: ?*const mlir.Value = null, - collectorOpA: ?*const mlir.Attribute = null, - collectorOpB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (opts.collectorOpA) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpA", value)); - if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.decompress_b", .{ - .operands = .{ .flat = if (opts.disableOutputLane) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, decompressBMetadata } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.mma.sp`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_sp( - ctx: *mlir.Context, - matrixD: *const mlir.Value, - matrixA: *const mlir.Value, - matrixB: *const mlir.Value, - idesc: *const mlir.Value, - enableInputD: *const mlir.Value, - sparseMetadata: *const mlir.Value, - kind: *const mlir.Attribute, - ctaGroup: *const mlir.Attribute, - opts: struct { - scaleInputD: ?*const mlir.Value = null, - disableOutputLane: ?*const mlir.Value = null, - collectorOp: ?*const mlir.Attribute = null, - collectorOpB: ?*const mlir.Attribute = null, - aShift: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); - if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); - if (opts.aShift) |value| attributes.appendAssumeCapacity(.named(ctx, "aShift", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.sp", .{ - .operands = .{ .variadic = &.{ - &.{matrixD}, - &.{matrixA}, - &.{matrixB}, - &.{idesc}, - &.{enableInputD}, - &.{sparseMetadata}, - if (opts.scaleInputD) |value| &.{value} else &.{}, - if (opts.disableOutputLane) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.mma.sp.block_scale`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_sp_block_scale( - ctx: *mlir.Context, - matrixD: *const mlir.Value, - matrixA: *const mlir.Value, - matrixB: *const mlir.Value, - idesc: *const mlir.Value, - enableInputD: *const mlir.Value, - sparseMetadata: *const mlir.Value, - scaleA: *const mlir.Value, - scaleB: *const mlir.Value, - kind: *const mlir.Attribute, - ctaGroup: *const mlir.Attribute, - opts: struct { - blockScale: ?*const mlir.Attribute = null, - collectorOp: ?*const mlir.Attribute = null, - collectorOpB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - attributes.appendAssumeCapacity(.named(ctx, "ctaGroup", ctaGroup)); - if (opts.blockScale) |value| attributes.appendAssumeCapacity(.named(ctx, "blockScale", value)); - if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); - if (opts.collectorOpB) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOpB", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.sp.block_scale", .{ - .operands = .{ .flat = &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata, scaleA, scaleB } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.mma.ws`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_ws( - ctx: *mlir.Context, - matrixD: *const mlir.Value, - matrixA: *const mlir.Value, - matrixB: *const mlir.Value, - idesc: *const mlir.Value, - enableInputD: *const mlir.Value, - kind: *const mlir.Attribute, - opts: struct { - zeroColMask: ?*const mlir.Value = null, - collectorBBuffer: ?*const mlir.Attribute = null, - collectorOp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); - if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.ws", .{ - .operands = .{ .flat = if (opts.zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.mma.ws.sp`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_ws_sp( - ctx: *mlir.Context, - matrixD: *const mlir.Value, - matrixA: *const mlir.Value, - matrixB: *const mlir.Value, - idesc: *const mlir.Value, - enableInputD: *const mlir.Value, - sparseMetadata: *const mlir.Value, - kind: *const mlir.Attribute, - opts: struct { - zeroColMask: ?*const mlir.Value = null, - collectorBBuffer: ?*const mlir.Attribute = null, - collectorOp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "kind", kind)); - if (opts.collectorBBuffer) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorBBuffer", value)); - if (opts.collectorOp) |value| attributes.appendAssumeCapacity(.named(ctx, "collectorOp", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.mma.ws.sp", .{ - .operands = .{ .flat = if (opts.zeroColMask) |value| &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata, value } else &.{ matrixD, matrixA, matrixB, idesc, enableInputD, sparseMetadata } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.mma_smem_desc`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_smem_desc(ctx: *mlir.Context, startAddr: *const mlir.Value, leadingDimOffset: *const mlir.Value, strideDimOffset: *const mlir.Value, baseOffset: *const mlir.Value, leadingDimMode: *const mlir.Value, swizzleMode: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05.mma_smem_desc", .{ - .operands = .{ .flat = &.{ startAddr, leadingDimOffset, strideDimOffset, baseOffset, leadingDimMode, swizzleMode } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.tcgen05.relinquish_alloc_permit`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_relinquish_alloc_permit( - ctx: *mlir.Context, - opts: struct { - group: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.relinquish_alloc_permit", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.shift`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_shift( - ctx: *mlir.Context, - taddr: *const mlir.Value, - opts: struct { - group: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.group) |value| attributes.appendAssumeCapacity(.named(ctx, "group", value)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.shift", .{ - .operands = .{ .flat = &.{taddr} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.st`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_st( - ctx: *mlir.Context, - tmemAddr: *const mlir.Value, - val: *const mlir.Value, - shape: *const mlir.Attribute, - opts: struct { - offset: ?*const mlir.Value = null, - unpack: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.unpack) |value| attributes.appendAssumeCapacity(.named(ctx, "unpack", value)); - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - return mlir.Operation.make(ctx, "nvvm.tcgen05.st", .{ - .operands = .{ .flat = if (opts.offset) |value| &.{ tmemAddr, val, value } else &.{ tmemAddr, val } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.tcgen05.wait`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_wait(ctx: *mlir.Context, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05.wait", .{ - .attributes = &.{ - .named(ctx, "kind", kind), - }, - .location = location, - }); -} - -/// `nvvm.tcgen05_mma_smem_desc_v2`. Result types are explicit; attributes use mlir.Attribute. -pub fn tcgen05_mma_smem_desc_v2(ctx: *mlir.Context, startAddress: *const mlir.Value, leadingDimOffset: *const mlir.Value, strideDimOffset: *const mlir.Value, descriptorVersion: *const mlir.Value, baseOffset: *const mlir.Value, leadingDimMode: *const mlir.Value, kSegmentOffset: *const mlir.Value, swizzleType: *const mlir.Value, res_type: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tcgen05_mma_smem_desc_v2", .{ - .operands = .{ .flat = &.{ startAddress, leadingDimOffset, strideDimOffset, descriptorVersion, baseOffset, leadingDimMode, kSegmentOffset, swizzleType } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `nvvm.tensormap.cp_fenceproxy`. Result types are explicit; attributes use mlir.Attribute. -pub fn tensormap_cp_fenceproxy(ctx: *mlir.Context, dst: *const mlir.Value, src: *const mlir.Value, size: *const mlir.Value, scope: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.tensormap.cp_fenceproxy", .{ - .operands = .{ .flat = &.{ dst, src, size } }, - .attributes = &.{ - .named(ctx, "scope", scope), - }, - .location = location, - }); -} - -/// `nvvm.tensormap.replace`. Result types are explicit; attributes use mlir.Attribute. -pub fn tensormap_replace( - ctx: *mlir.Context, - addr: *const mlir.Value, - field: *const mlir.Attribute, - opts: struct { - new_value: ?*const mlir.Value = null, - ord: ?*const mlir.Attribute = null, - new_value_attr: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "field", field)); - if (opts.ord) |value| attributes.appendAssumeCapacity(.named(ctx, "ord", value)); - if (opts.new_value_attr) |value| attributes.appendAssumeCapacity(.named(ctx, "new_value_attr", value)); - return mlir.Operation.make(ctx, "nvvm.tensormap.replace", .{ - .operands = .{ .flat = if (opts.new_value) |value| &.{ addr, value } else &.{addr} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.trace.mark`. Result types are explicit; attributes use mlir.Attribute. -pub fn trace_mark( - ctx: *mlir.Context, - eventType: *const mlir.Attribute, - domain: *const mlir.Attribute, - event: *const mlir.Attribute, - opts: struct { - payload: ?*const mlir.Value = null, - predicate: ?*const mlir.Value = null, - payloadDescriptor: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "eventType", eventType)); - attributes.appendAssumeCapacity(.named(ctx, "domain", domain)); - attributes.appendAssumeCapacity(.named(ctx, "event", event)); - if (opts.payloadDescriptor) |value| attributes.appendAssumeCapacity(.named(ctx, "payloadDescriptor", value)); - return mlir.Operation.make(ctx, "nvvm.trace.mark", .{ - .operands = .{ .variadic = &.{ - if (opts.payload) |value| &.{value} else &.{}, - if (opts.predicate) |value| &.{value} else &.{}, - } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.vote.sync`. Result types are explicit; attributes use mlir.Attribute. -pub fn vote_sync(ctx: *mlir.Context, mask: *const mlir.Value, pred: *const mlir.Value, res_type: *const mlir.Type, kind: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.vote.sync", .{ - .operands = .{ .flat = &.{ mask, pred } }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "kind", kind), - }, - .location = location, - }); -} - -/// `nvvm.wgmma.commit.group.sync.aligned`. Result types are explicit; attributes use mlir.Attribute. -pub fn wgmma_commit_group_sync_aligned(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.wgmma.commit.group.sync.aligned", .{ - .location = location, - }); -} - -/// `nvvm.wgmma.fence.aligned`. Result types are explicit; attributes use mlir.Attribute. -pub fn wgmma_fence_aligned(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.wgmma.fence.aligned", .{ - .location = location, - }); -} - -/// `nvvm.wgmma.mma_async`. Result types are explicit; attributes use mlir.Attribute. -pub fn wgmma_mma_async( - ctx: *mlir.Context, - inouts: *const mlir.Value, - descriptorA: *const mlir.Value, - descriptorB: *const mlir.Value, - results__type: *const mlir.Type, - shape: *const mlir.Attribute, - typeA: *const mlir.Attribute, - typeB: *const mlir.Attribute, - typeD: *const mlir.Attribute, - scaleD: *const mlir.Attribute, - scaleA: *const mlir.Attribute, - scaleB: *const mlir.Attribute, - layoutA: *const mlir.Attribute, - layoutB: *const mlir.Attribute, - opts: struct { - satfinite: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "shape", shape)); - attributes.appendAssumeCapacity(.named(ctx, "typeA", typeA)); - attributes.appendAssumeCapacity(.named(ctx, "typeB", typeB)); - attributes.appendAssumeCapacity(.named(ctx, "typeD", typeD)); - attributes.appendAssumeCapacity(.named(ctx, "scaleD", scaleD)); - attributes.appendAssumeCapacity(.named(ctx, "scaleA", scaleA)); - attributes.appendAssumeCapacity(.named(ctx, "scaleB", scaleB)); - attributes.appendAssumeCapacity(.named(ctx, "layoutA", layoutA)); - attributes.appendAssumeCapacity(.named(ctx, "layoutB", layoutB)); - if (opts.satfinite) |value| attributes.appendAssumeCapacity(.named(ctx, "satfinite", value)); - return mlir.Operation.make(ctx, "nvvm.wgmma.mma_async", .{ - .operands = .{ .flat = &.{ inouts, descriptorA, descriptorB } }, - .results = .{ .flat = &.{results__type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `nvvm.wgmma.wait.group.sync.aligned`. Result types are explicit; attributes use mlir.Attribute. -pub fn wgmma_wait_group_sync_aligned(ctx: *mlir.Context, group: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.wgmma.wait.group.sync.aligned", .{ - .attributes = &.{ - .named(ctx, "group", group), - }, - .location = location, - }); -} - -/// `nvvm.wmma.load`. Result types are explicit; attributes use mlir.Attribute. -pub fn wmma_load(ctx: *mlir.Context, ptr: *const mlir.Value, stride: *const mlir.Value, res_type: *const mlir.Type, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layout: *const mlir.Attribute, eltype: *const mlir.Attribute, frag: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.wmma.load", .{ - .operands = .{ .flat = &.{ ptr, stride } }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "m", m), - .named(ctx, "n", n), - .named(ctx, "k", k), - .named(ctx, "layout", layout), - .named(ctx, "eltype", eltype), - .named(ctx, "frag", frag), - }, - .location = location, - }); -} - -/// `nvvm.wmma.mma`. Result types are explicit; attributes use mlir.Attribute. -pub fn wmma_mma(ctx: *mlir.Context, args: []const *const mlir.Value, res_type: *const mlir.Type, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layoutA: *const mlir.Attribute, layoutB: *const mlir.Attribute, eltypeA: *const mlir.Attribute, eltypeB: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "nvvm.wmma.mma", .{ - .operands = .{ .flat = args }, - .results = .{ .flat = &.{res_type} }, - .attributes = &.{ - .named(ctx, "m", m), - .named(ctx, "n", n), - .named(ctx, "k", k), - .named(ctx, "layoutA", layoutA), - .named(ctx, "layoutB", layoutB), - .named(ctx, "eltypeA", eltypeA), - .named(ctx, "eltypeB", eltypeB), - }, - .location = location, - }); -} - -/// `nvvm.wmma.store`. Result types are explicit; attributes use mlir.Attribute. -pub fn wmma_store(ctx: *mlir.Context, ptr: *const mlir.Value, args: []const *const mlir.Value, stride: *const mlir.Value, m: *const mlir.Attribute, n: *const mlir.Attribute, k: *const mlir.Attribute, layout: *const mlir.Attribute, eltype: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 256) = .empty; - operands.appendAssumeCapacity(ptr); - operands.appendSliceAssumeCapacity(args); - operands.appendAssumeCapacity(stride); - return mlir.Operation.make(ctx, "nvvm.wmma.store", .{ - .operands = .{ .flat = operands.constSlice() }, - .attributes = &.{ - .named(ctx, "m", m), - .named(ctx, "n", n), - .named(ctx, "k", k), - .named(ctx, "layout", layout), - .named(ctx, "eltype", eltype), - }, - .location = location, - }); -} diff --git a/mlir/dialects/nvvm/test.zig b/mlir/dialects/nvvm/test.zig deleted file mode 100644 index b908f6f4..00000000 --- a/mlir/dialects/nvvm/test.zig +++ /dev/null @@ -1,481 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const nvvm = @import("mlir/dialects/nvvm"); - -const operation_names: []const []const u8 = &.{ - "nvvm.add.packed.bf16x2.f32x2.f32x2", - "nvvm.add.packed.f16x2.f32x2.f32x2", - "nvvm.add.packed.f32x2", - "nvvm.add.packed.f32x2.bf16x2.f32x2", - "nvvm.add.packed.f32x2.f16x2.f32x2", - "nvvm.addf", - "nvvm.applypriority.async.bulk", - "nvvm.applypriority.async.bulk.tensor", - "nvvm.applypriority.async.bulk.tensor.override", - "nvvm.atomicrmw", - "nvvm.bar.warp.sync", - "nvvm.barrier", - "nvvm.barrier.arrive", - "nvvm.barrier.cta.arrive", - "nvvm.barrier.cta.red", - "nvvm.barrier.cta.sync", - "nvvm.barrier0", - "nvvm.breakpoint", - "nvvm.clmad", - "nvvm.cluster.arrive", - "nvvm.cluster.arrive.relaxed", - "nvvm.cluster.wait", - "nvvm.clusterlaunchcontrol.query.cancel", - "nvvm.clusterlaunchcontrol.try.cancel", - "nvvm.compare.and.set", - "nvvm.convert.and.pack.integer", - "nvvm.convert.bf16x2.to.f4x2", - "nvvm.convert.bf16x2.to.f6x2", - "nvvm.convert.bf16x2.to.f8x2", - "nvvm.convert.bf16x2.to.s2f6x2", - "nvvm.convert.f16x2.to.f4x2", - "nvvm.convert.f16x2.to.f6x2", - "nvvm.convert.f16x2.to.f8x2", - "nvvm.convert.f32x2.to.bf16x2", - "nvvm.convert.f32x2.to.f16x2", - "nvvm.convert.f32x2.to.f4x2", - "nvvm.convert.f32x2.to.f6x2", - "nvvm.convert.f32x2.to.f8x2", - "nvvm.convert.f32x2.to.s2f6x2", - "nvvm.convert.f32x4.to.f4x4", - "nvvm.convert.f32x4.to.f6x4", - "nvvm.convert.f32x4.to.f8x4", - "nvvm.convert.f4x2.to.f16x2", - "nvvm.convert.f6x2.to.f16x2", - "nvvm.convert.f8x2.to.bf16x2", - "nvvm.convert.f8x2.to.f16x2", - "nvvm.convert.float.to.integer", - "nvvm.convert.float.to.tf32", - "nvvm.convert.s2f6x2.to.bf16x2", - "nvvm.cos", - "nvvm.cp.async.bulk.commit.group", - "nvvm.cp.async.bulk.global.shared.cta", - "nvvm.cp.async.bulk.prefetch", - "nvvm.cp.async.bulk.shared.cluster.global", - "nvvm.cp.async.bulk.shared.cluster.shared.cta", - "nvvm.cp.async.bulk.tensor.global.shared.cta", - "nvvm.cp.async.bulk.tensor.global.shared.cta.override", - "nvvm.cp.async.bulk.tensor.prefetch", - "nvvm.cp.async.bulk.tensor.prefetch.override", - "nvvm.cp.async.bulk.tensor.reduce", - "nvvm.cp.async.bulk.tensor.reduce.override", - "nvvm.cp.async.bulk.tensor.shared.cluster.global", - "nvvm.cp.async.bulk.tensor.shared.cluster.global.override", - "nvvm.cp.async.bulk.wait_group", - "nvvm.cp.async.commit.group", - "nvvm.cp.async.mbarrier.arrive", - "nvvm.cp.async.shared.global", - "nvvm.cp.async.wait.group", - "nvvm.cvt.packfloat", - "nvvm.cvt.packfloat.f32", - "nvvm.cvt.to.f4x8.packed", - "nvvm.dot.accumulate.2way", - "nvvm.dot.accumulate.4way", - "nvvm.elect.sync", - "nvvm.ex2", - "nvvm.exit", - "nvvm.fabs", - "nvvm.fence.mbarrier.init", - "nvvm.fence.proxy", - "nvvm.fence.proxy.acquire", - "nvvm.fence.proxy.release", - "nvvm.fence.proxy.sync_restrict", - "nvvm.fence.sc.cluster", - "nvvm.fence.sync_restrict", - "nvvm.fma", - "nvvm.fma.packed.f32x2", - "nvvm.fma.packed.f32x2.bf16x2.f32x2.f32x2", - "nvvm.fma.packed.f32x2.f16x2.f32x2.f32x2", - "nvvm.fmax", - "nvvm.fmin", - "nvvm.griddepcontrol", - "nvvm.inline_ptx", - "nvvm.ldmatrix", - "nvvm.load.ext", - "nvvm.log2", - "nvvm.mapa", - "nvvm.match.sync", - "nvvm.mbarrier.arrive", - "nvvm.mbarrier.arrive.expect_tx", - "nvvm.mbarrier.arrive.nocomplete", - "nvvm.mbarrier.arrive_drop", - "nvvm.mbarrier.arrive_drop.expect_tx", - "nvvm.mbarrier.arrive_drop.nocomplete", - "nvvm.mbarrier.check.layout", - "nvvm.mbarrier.complete_tx", - "nvvm.mbarrier.expect_tx", - "nvvm.mbarrier.init", - "nvvm.mbarrier.inval", - "nvvm.mbarrier.test.wait", - "nvvm.mbarrier.try_wait", - "nvvm.mbarrier.try_wait.parity", - "nvvm.mbarrier.try_wait.parity.timelimit", - "nvvm.mbarrier.try_wait.timelimit", - "nvvm.mbarrier.txn", - "nvvm.mbarrier.txn.cta", - "nvvm.mbarrier.wait", - "nvvm.mbarrier.wait.parity", - "nvvm.memory.barrier", - "nvvm.mma.block_scale", - "nvvm.mma.block_scale.internal", - "nvvm.mma.sp.block_scale", - "nvvm.mma.sp.block_scale.internal", - "nvvm.mma.sp.sync", - "nvvm.mma.sync", - "nvvm.mma_smem_desc", - "nvvm.movmatrix", - "nvvm.mul", - "nvvm.mul.packed.bf16x2.bf16x2.f16x2", - "nvvm.mul.packed.bf16x2.f32x2.f32x2", - "nvvm.mul.packed.f16x2.f16x2.bf16x2", - "nvvm.mul.packed.f16x2.f32x2.f32x2", - "nvvm.mul.packed.f32x2", - "nvvm.multimem.ld.reduce", - "nvvm.multimem.red", - "nvvm.multimem.st", - "nvvm.nanosleep", - "nvvm.pmevent", - "nvvm.prefetch", - "nvvm.prmt", - "nvvm.rcp.approx.ftz.f", - "nvvm.read.ptx.sreg.aggr.smem.size", - "nvvm.read.ptx.sreg.clock", - "nvvm.read.ptx.sreg.clock64", - "nvvm.read.ptx.sreg.cluster.ctaid.x", - "nvvm.read.ptx.sreg.cluster.ctaid.y", - "nvvm.read.ptx.sreg.cluster.ctaid.z", - "nvvm.read.ptx.sreg.cluster.ctarank", - "nvvm.read.ptx.sreg.cluster.nctaid.x", - "nvvm.read.ptx.sreg.cluster.nctaid.y", - "nvvm.read.ptx.sreg.cluster.nctaid.z", - "nvvm.read.ptx.sreg.cluster.nctarank", - "nvvm.read.ptx.sreg.clusterid.x", - "nvvm.read.ptx.sreg.clusterid.y", - "nvvm.read.ptx.sreg.clusterid.z", - "nvvm.read.ptx.sreg.ctaid.x", - "nvvm.read.ptx.sreg.ctaid.y", - "nvvm.read.ptx.sreg.ctaid.z", - "nvvm.read.ptx.sreg.dynamic.smem.size", - "nvvm.read.ptx.sreg.envreg0", - "nvvm.read.ptx.sreg.envreg1", - "nvvm.read.ptx.sreg.envreg10", - "nvvm.read.ptx.sreg.envreg11", - "nvvm.read.ptx.sreg.envreg12", - "nvvm.read.ptx.sreg.envreg13", - "nvvm.read.ptx.sreg.envreg14", - "nvvm.read.ptx.sreg.envreg15", - "nvvm.read.ptx.sreg.envreg16", - "nvvm.read.ptx.sreg.envreg17", - "nvvm.read.ptx.sreg.envreg18", - "nvvm.read.ptx.sreg.envreg19", - "nvvm.read.ptx.sreg.envreg2", - "nvvm.read.ptx.sreg.envreg20", - "nvvm.read.ptx.sreg.envreg21", - "nvvm.read.ptx.sreg.envreg22", - "nvvm.read.ptx.sreg.envreg23", - "nvvm.read.ptx.sreg.envreg24", - "nvvm.read.ptx.sreg.envreg25", - "nvvm.read.ptx.sreg.envreg26", - "nvvm.read.ptx.sreg.envreg27", - "nvvm.read.ptx.sreg.envreg28", - "nvvm.read.ptx.sreg.envreg29", - "nvvm.read.ptx.sreg.envreg3", - "nvvm.read.ptx.sreg.envreg30", - "nvvm.read.ptx.sreg.envreg31", - "nvvm.read.ptx.sreg.envreg4", - "nvvm.read.ptx.sreg.envreg5", - "nvvm.read.ptx.sreg.envreg6", - "nvvm.read.ptx.sreg.envreg7", - "nvvm.read.ptx.sreg.envreg8", - "nvvm.read.ptx.sreg.envreg9", - "nvvm.read.ptx.sreg.globaltimer", - "nvvm.read.ptx.sreg.globaltimer.lo", - "nvvm.read.ptx.sreg.gridid", - "nvvm.read.ptx.sreg.laneid", - "nvvm.read.ptx.sreg.lanemask.eq", - "nvvm.read.ptx.sreg.lanemask.ge", - "nvvm.read.ptx.sreg.lanemask.gt", - "nvvm.read.ptx.sreg.lanemask.le", - "nvvm.read.ptx.sreg.lanemask.lt", - "nvvm.read.ptx.sreg.nclusterid.x", - "nvvm.read.ptx.sreg.nclusterid.y", - "nvvm.read.ptx.sreg.nclusterid.z", - "nvvm.read.ptx.sreg.nctaid.x", - "nvvm.read.ptx.sreg.nctaid.y", - "nvvm.read.ptx.sreg.nctaid.z", - "nvvm.read.ptx.sreg.nsmid", - "nvvm.read.ptx.sreg.ntid.x", - "nvvm.read.ptx.sreg.ntid.y", - "nvvm.read.ptx.sreg.ntid.z", - "nvvm.read.ptx.sreg.nwarpid", - "nvvm.read.ptx.sreg.smid", - "nvvm.read.ptx.sreg.tid.x", - "nvvm.read.ptx.sreg.tid.y", - "nvvm.read.ptx.sreg.tid.z", - "nvvm.read.ptx.sreg.total.smem.size", - "nvvm.read.ptx.sreg.warpid", - "nvvm.read.ptx.sreg.warpsize", - "nvvm.red", - "nvvm.redux.sync", - "nvvm.rsqrt", - "nvvm.setmaxregister", - "nvvm.shfl.sync", - "nvvm.sin", - "nvvm.spcompress", - "nvvm.spdecompress", - "nvvm.st.bulk", - "nvvm.stmatrix", - "nvvm.store.async", - "nvvm.store.ext", - "nvvm.sub.packed.bf16x2.f32x2.f32x2", - "nvvm.sub.packed.f16x2.f32x2.f32x2", - "nvvm.sub.packed.f32x2", - "nvvm.sub.packed.f32x2.bf16x2.f32x2", - "nvvm.sub.packed.f32x2.f16x2.f32x2", - "nvvm.subf", - "nvvm.tcgen05.alloc", - "nvvm.tcgen05.commit", - "nvvm.tcgen05.cp", - "nvvm.tcgen05.dealloc", - "nvvm.tcgen05.fence", - "nvvm.tcgen05.ld", - "nvvm.tcgen05.ld.red", - "nvvm.tcgen05.ld.red.spcompress", - "nvvm.tcgen05.ld.spcompress", - "nvvm.tcgen05.mma", - "nvvm.tcgen05.mma.block_scale", - "nvvm.tcgen05.mma.block_scale.decompress_b", - "nvvm.tcgen05.mma.decompress_b", - "nvvm.tcgen05.mma.sp", - "nvvm.tcgen05.mma.sp.block_scale", - "nvvm.tcgen05.mma.ws", - "nvvm.tcgen05.mma.ws.sp", - "nvvm.tcgen05.mma_smem_desc", - "nvvm.tcgen05.relinquish_alloc_permit", - "nvvm.tcgen05.shift", - "nvvm.tcgen05.st", - "nvvm.tcgen05.wait", - "nvvm.tcgen05_mma_smem_desc_v2", - "nvvm.tensormap.cp_fenceproxy", - "nvvm.tensormap.replace", - "nvvm.trace.mark", - "nvvm.vote.sync", - "nvvm.wgmma.commit.group.sync.aligned", - "nvvm.wgmma.fence.aligned", - "nvvm.wgmma.mma_async", - "nvvm.wgmma.wait.group.sync.aligned", - "nvvm.wmma.load", - "nvvm.wmma.mma", - "nvvm.wmma.store", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(nvvm); - std.testing.refAllDecls(nvvm.ops); - inline for (comptime std.meta.declarations(nvvm.ops)) |decl| { - const T = @field(nvvm.ops, decl); - if (@TypeOf(T) == type) std.testing.refAllDecls(T); - } -} - -test "every enum attribute prints as the compiler does" { - @setEvalBranchQuota(1_000_000); - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - nvvm.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - inline for (.{ - .{ nvvm.ops.AtomicOpKindAttr, "#nvvm" }, - .{ nvvm.ops.BarrierReductionAttr, "#nvvm.reduction<{s}>" }, - .{ nvvm.ops.BarrierReduxKindAttr, "#nvvm.barrier_redux_kind<{s}>" }, - .{ nvvm.ops.BlockScaleFormatAttr, "#nvvm.block_scale_format<{s}>" }, - .{ nvvm.ops.CTAGroupKindAttr, "#nvvm.cta_group<{s}>" }, - .{ nvvm.ops.CVTPackFloatKindAttr, "#nvvm.packfloat_type<{s}>" }, - .{ nvvm.ops.CacheEvictionPriorityAttr, "#nvvm" }, - .{ nvvm.ops.ClusterLaunchControlQueryTypeAttr, "#nvvm" }, - .{ nvvm.ops.CompareOpKindAttr, "#nvvm.op<{s}>" }, - .{ nvvm.ops.ConvertFP4TypeAttr, "#nvvm.convert_fp4_type<{s}>" }, - .{ nvvm.ops.ConvertFP8TypeAttr, "#nvvm.convert_fp8_type<{s}>" }, - .{ nvvm.ops.ConvertScaleKindAttr, "#nvvm.convert_scale_kind<{s}>" }, - .{ nvvm.ops.DotAccumulateTypeAttr, "#nvvm.dot_accumulate_type<{s}>" }, - .{ nvvm.ops.EvictKindAttr, "#nvvm.evict_kind<{s}>" }, - .{ nvvm.ops.FPRoundingModeAttr, "#nvvm.fp_rnd_mode<{s}>" }, - .{ nvvm.ops.GridDepActionKindAttr, "#nvvm" }, - .{ nvvm.ops.IntegerRoundingModeAttr, "#nvvm.int_rnd_mode<{s}>" }, - .{ nvvm.ops.L2PrefetchSizeAttr, "#nvvm.l2_prefetch<{s}>" }, - .{ nvvm.ops.LdStMatrixEltTypeAttr, "#nvvm.ld_st_matrix_elt_type<{s}>" }, - .{ nvvm.ops.LoadCacheModifierExtKindAttr, "#nvvm.load_cache_modifier_ext<{s}>" }, - .{ nvvm.ops.LoadCacheModifierKindAttr, "#nvvm" }, - .{ nvvm.ops.LoadShapeAttr, "#nvvm.load_shape<{s}>" }, - .{ nvvm.ops.LoadSrcFormatAttr, "#nvvm.load_src_format<{s}>" }, - .{ nvvm.ops.MBarrierLayoutAttr, "#nvvm.mbarrier_layout<{s}>" }, - .{ nvvm.ops.MBarrierPhaseAttr, "#nvvm.mbarrier_phase<{s}>" }, - .{ nvvm.ops.MBarrierScopeKindAttr, "#nvvm.mbar_scope<{s}>" }, - .{ nvvm.ops.MBarrierSpaceKindAttr, "#nvvm.mbar_space<{s}>" }, - .{ nvvm.ops.MBarrierTxnKindAttr, "#nvvm.mbar_txn_kind<{s}>" }, - .{ nvvm.ops.MBarrierWaitKindAttr, "#nvvm.mbar_wait<{s}>" }, - .{ nvvm.ops.MMAB1OpAttr, "#nvvm.mma_b1op<{s}>" }, - .{ nvvm.ops.MMABlockScaleKindAttr, "#nvvm.block_scale_kind<{s}>" }, - .{ nvvm.ops.MMACtaCountAttr, "#nvvm.mma_cta_count<{s}>" }, - .{ nvvm.ops.MMAFragAttr, "#nvvm.mma_frag<{s}>" }, - .{ nvvm.ops.MMAIntOverflowAttr, "#nvvm.mma_int_overflow<{s}>" }, - .{ nvvm.ops.MMAKindAttr, "#nvvm.mma_kind<{s}>" }, - .{ nvvm.ops.MMALayoutAttr, "#nvvm.mma_layout<{s}>" }, - .{ nvvm.ops.MMATypesAttr, "#nvvm.mma_type<{s}>" }, - .{ nvvm.ops.MatchSyncKindAttr, "#nvvm" }, - .{ nvvm.ops.MemOrderKindAttr, "#nvvm.mem_order<{s}>" }, - .{ nvvm.ops.MemScopeKindAttr, "#nvvm.mem_scope<{s}>" }, - .{ nvvm.ops.MulModeAttr, "#nvvm" }, - .{ nvvm.ops.NVVMMemorySpaceAttr, "#nvvm.memory_space<{s}>" }, - .{ nvvm.ops.PermuteModeAttr, "#nvvm.permute_mode<{s}>" }, - .{ nvvm.ops.PrefetchCacheLevelAttr, "#nvvm" }, - .{ nvvm.ops.ProxyKindAttr, "#nvvm.proxy_kind<{s}>" }, - .{ nvvm.ops.ReductionKindAttr, "#nvvm" }, - .{ nvvm.ops.ReductionOpAttr, "#nvvm.red_op<{s}>" }, - .{ nvvm.ops.ReductionTypeAttr, "#nvvm.red_type<{s}>" }, - .{ nvvm.ops.SPCompressElemSizeAttr, "#nvvm.spcomp_elem_size<{s}>" }, - .{ nvvm.ops.SPCompressFactorTypeAttr, "#nvvm.spcomp_factor<{s}>" }, - .{ nvvm.ops.SPCompressIndexSizeAttr, "#nvvm.spcomp_index_size<{s}>" }, - .{ nvvm.ops.SPCompressOpKindAttr, "#nvvm.spcomp_op_kind<{s}>" }, - .{ nvvm.ops.SPCompressRepFactorAttr, "#nvvm.spcomp_rep_factor<{s}>" }, - .{ nvvm.ops.SPDecompressElemSizeAttr, "#nvvm.spdecomp_elem_size<{s}>" }, - .{ nvvm.ops.SPDecompressFactorTypeAttr, "#nvvm.spdecomp_factor<{s}>" }, - .{ nvvm.ops.SPDecompressIndexSizeAttr, "#nvvm.spdecomp_index_size<{s}>" }, - .{ nvvm.ops.SPDecompressRepFactorAttr, "#nvvm.spdecomp_rep_factor<{s}>" }, - .{ nvvm.ops.SaturationModeAttr, "#nvvm.sat_mode<{s}>" }, - .{ nvvm.ops.SaturationModeKindAttr, "#nvvm.sat<{s}>" }, - .{ nvvm.ops.ScaleVecSizeAttr, "#nvvm.scale_vec_size<{s}>" }, - .{ nvvm.ops.SetMaxRegisterActionAttr, "#nvvm" }, - .{ nvvm.ops.SharedSpaceAttr, "#nvvm.shared_space<{s}>" }, - .{ nvvm.ops.ShflKindAttr, "#nvvm" }, - .{ nvvm.ops.SparsityFormatAttr, "#nvvm.sparsity_format<{s}>" }, - .{ nvvm.ops.StateSpaceAttr, "#nvvm.state_space<{s}>" }, - .{ nvvm.ops.StoreCacheModifierKindAttr, "#nvvm.store_cache_modifier<{s}>" }, - .{ nvvm.ops.StoreShapeAttr, "#nvvm.store_shape<{s}>" }, - .{ nvvm.ops.TCBarParamAttr, "#nvvm.TCBarParam<{s}>" }, - .{ nvvm.ops.TMALoadModeAttr, "#nvvm.tma_load_mode<{s}>" }, - .{ nvvm.ops.TMAReduxKindAttr, "#nvvm.tma_redux_kind<{s}>" }, - .{ nvvm.ops.TMAStoreModeAttr, "#nvvm.tma_store_mode<{s}>" }, - .{ nvvm.ops.Tcgen05CpMulticastAttr, "#nvvm.tcgen05_cp_multicast<{s}>" }, - .{ nvvm.ops.Tcgen05CpShapeAttr, "#nvvm.tcgen05_cp_shape<{s}>" }, - .{ nvvm.ops.Tcgen05CpSrcFormatAttr, "#nvvm.tcgen05_cp_src_fmt<{s}>" }, - .{ nvvm.ops.Tcgen05FenceKindAttr, "#nvvm.tcgen05_fence<{s}>" }, - .{ nvvm.ops.Tcgen05LdStShapeAttr, "#nvvm.tcgen05_ldst_shape<{s}>" }, - .{ nvvm.ops.Tcgen05MMABlockScaleAttr, "#nvvm.tcgen05_mma_block_scale<{s}>" }, - .{ nvvm.ops.Tcgen05MMACollectorBBufferAttr, "#nvvm.tcgen05_mma_collectorb<{s}>" }, - .{ nvvm.ops.Tcgen05MMACollectorOpAttr, "#nvvm.tcgen05_mma_collectorop<{s}>" }, - .{ nvvm.ops.Tcgen05MMAKindAttr, "#nvvm.tcgen05_mma_kind<{s}>" }, - .{ nvvm.ops.Tcgen05WaitKindAttr, "#nvvm.tcgen05_wait<{s}>" }, - .{ nvvm.ops.TensormapElemtypeAttr, "#nvvm.tensormap_elemtype<{s}>" }, - .{ nvvm.ops.TensormapFieldAttr, "#nvvm" }, - .{ nvvm.ops.TensormapFillModeAttr, "#nvvm.tensormap_fill_mode<{s}>" }, - .{ nvvm.ops.TensormapInterleaveLayoutAttr, "#nvvm.tensormap_interleave_layout<{s}>" }, - .{ nvvm.ops.TensormapSwizzleAtomicityAttr, "#nvvm.tensormap_swizzle_atomicity<{s}>" }, - .{ nvvm.ops.TensormapSwizzleModeAttr, "#nvvm.tensormap_swizzle_mode<{s}>" }, - .{ nvvm.ops.TmemLayoutAttr, "#nvvm.TmemLayout<{s}>" }, - .{ nvvm.ops.ValidatePatternAttr, "#nvvm.validate_pattern<{s}>" }, - .{ nvvm.ops.VoteSyncKindAttr, "#nvvm" }, - .{ nvvm.ops.WGMMAScaleInAttr, "#nvvm.wgmma_scale_in<{s}>" }, - .{ nvvm.ops.WGMMAScaleOutAttr, "#nvvm.wgmma_scale_out<{s}>" }, - .{ nvvm.ops.WGMMATypesAttr, "#nvvm.wgmma_type<{s}>" }, - }) |example| { - const T = example[0]; - const E = @FieldType(T.InitArgs, "value"); - inline for (@typeInfo(E).@"enum".field_names) |name| { - const attr = try T.get(ctx, .{ .value = @field(E, name) }); - try std.testing.expectEqual(@field(E, name), attr.value()); - var buf: [256]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{attr}); - try std.testing.expectEqualStrings(std.fmt.comptimePrint(example[1], .{name}), w.buffered()); - } - } -} - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - nvvm.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -test "every bound operation is registered" { - const ctx = try testContext(); - defer ctx.deinit(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} - -test "nvvm types and attributes print as the compiler does" { - @setEvalBranchQuota(100_000); - const ctx = try testContext(); - defer ctx.deinit(); - // The compiler's printed forms, which these must print byte for byte. - inline for (.{ - .{ nvvm.ops.MMAShapeAttr, "#nvvm.shape" }, - .{ nvvm.ops.LdStMatrixShapeAttr, "#nvvm.ld_st_matrix_shape" }, - .{ nvvm.ops.TargetAttr, "#nvvm.target" }, - .{ nvvm.ops.TargetAttr, "#nvvm.target" }, - .{ nvvm.ops.TargetAttr, "#nvvm.target" }, - }) |example| { - const T = example[0]; - const is_type = @hasDecl(T, "type_"); - const parsed = if (is_type) try mlir.Type.parse(ctx, example[1]) else try mlir.Attribute.parse(ctx, example[1]); - const value = parsed.isA(T) orelse return error.TestUnexpectedResult; - try std.testing.expect((try rebuildEnums(T, ctx, value)).eql(value)); - var buf: [512]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(example[1], w.buffered()); - } -} - -test "reqntid and minctasm are kernel attributes of the nvvm dialect" { - const ctx = try testContext(); - defer ctx.deinit(); - var buf: [128]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ nvvm.reqntid(ctx, .{ 128, 1, 1 }), nvvm.minctasm(ctx, 1) })}); - try std.testing.expectEqualStrings("{nvvm.minctasm = 1 : i32, nvvm.reqntid = array}", w.buffered()); -} - -test "nvvm target rejects what the compiler rejects" { - const ctx = try testContext(); - defer ctx.deinit(); - try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .O = 4 })); - try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .chip = "" })); - const not_strings = try mlir.Attribute.parse(ctx, "[\"a\", 2]"); - try std.testing.expectError(error.InvalidMlir, nvvm.ops.TargetAttr.get(ctx, .{ .link = not_strings })); -} - -/// `rebuild` for parameters that are slices of enums. -fn rebuildEnums(comptime T: type, ctx: *mlir.Context, value: *const T) !*const T { - var args: T.InitArgs = undefined; - var buffers: [4][16]u32 = undefined; - inline for (@typeInfo(T.InitArgs).@"struct".field_names, 0..) |field, i| { - const name = [_]u8{comptime std.ascii.toUpper(field[0])} ++ field[1..]; - const noun = [_]u8{comptime std.ascii.toLower(field[0])} ++ field[1..]; - const accessor = if (@hasDecl(T, noun ++ "_")) noun ++ "_" else noun; - const info = @typeInfo(@FieldType(T.InitArgs, field)); - if (info == .pointer and info.pointer.size == .slice and @typeInfo(info.pointer.child) == .@"enum") { - const E = info.pointer.child; - const count = @field(T, "num" ++ name)(value); - const elements: []E = @ptrCast(buffers[i][0..count]); - for (elements, 0..) |*element, pos| element.* = @field(T, noun[0 .. noun.len - 1])(value, pos); - @field(args, field) = elements; - } else { - @field(args, field) = @field(T, accessor)(value); - } - } - return T.get(ctx, args); -} diff --git a/mlir/dialects/rocdl/BUILD.bazel b/mlir/dialects/rocdl/BUILD.bazel deleted file mode 100644 index ef0d7583..00000000 --- a/mlir/dialects/rocdl/BUILD.bazel +++ /dev/null @@ -1,46 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:CAPIROCDL", - "@llvm-project//mlir:ROCDLDialect", - ], -) - -zig_library( - name = "rocdl", - srcs = [ - "annotations.zig", - "attributes.zig", - "enums.zig", - "ops.zig", - "target.zig", - "types.zig", - ], - import_name = "mlir/dialects/rocdl", - main = "rocdl.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//mlir/dialects/llvm", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":capi", - ":rocdl", - "//mlir", - ], -) diff --git a/mlir/dialects/rocdl/README.md b/mlir/dialects/rocdl/README.md deleted file mode 100644 index 500e4657..00000000 --- a/mlir/dialects/rocdl/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# ROCDL - -Depend on `//mlir/dialects/rocdl` and import `mlir/dialects/rocdl` for LLVM's -AMDGPU intrinsic dialect. The package links LLVM's `ROCDLDialect` and -`CAPIROCDL`. `fly_rocdl` is a separate dialect and package. - -The pinned LLVM revision supplies 374 operations, 15 attributes, and 12 enums. - -- `ops` provides every operation builder. Names replace dots with underscores, - such as `ops.raw_buffer_load` and `ops.mfma_f32_16x16x4f32`. Arguments are - operands, explicit result types, required attributes, an `opts` struct when - there are optional attributes, and the location, preceded by the MLIR context. - Attribute names match ODS. Optional and default-valued attributes are fields - of `opts` with `null` defaults; pass `.{}` to omit them and let MLIR supply - its defaults. Builders verify the resulting operation. -- Enum types and attribute wrappers are available at the module root and in - `enums`/`attributes`. Attribute constructors return `error.InvalidMlir` for - invalid enum values or flag bits. Bit enums support `combine` and `contains`. -- `TargetAttr` constructs and reads `#rocdl.target`, including optimization - level, triple, chip, features, ABI, flags, and link files. Defaults match LLVM. - Invalid fields and attributes from another context are rejected. -- `annotations` supplies function, module, and memory annotations. Apply - function annotations to `llvm.func`, and `xnack`/`sramecc` to LLVM-compatible - modules. A missing module option preserves the target default; `false` - explicitly disables the option. -- `types` supplies every builtin vector type ROCDL's TableGen records use, - named or inline, and LLVM pointers in ROCDL address spaces. ROCDL defines no independent - MLIR types. -- `registerDialect` registers ROCDL; loading it also loads its LLVM dependency. - The existing `cvtPkF32Fp8` convenience builder remains available. - -```zig -const rocdl = @import("mlir/dialects/rocdl"); - -const policy = try rocdl.Gfx942CachePolicyAttr.get(ctx, .{ .value = .combine(&.{ .sc0, .nt }) }); -const load = rocdl.ops.raw_buffer_load( - ctx, resource, offset, scalar_offset, - mlir.Type.float(ctx, .f32), .{ .aux = policy.attribute() }, location, -); -``` - -The `:capi` target exposes attribute constructors, predicates, and getters -through `capi.h`; dialect registration comes from upstream -`mlir-c/Dialect/ROCDL.h`. Declarations and implementations are grouped by -introspection (a test helper), attributes, and target configuration. Getters require a handle -accepted by the matching predicate. Contexts own all returned attributes and -strings. There are no additional C operation wrappers: Zig builders use MLIR's -standard operation API. - -Tests compare the generated operation list with the loaded dialect's registered -operation count, check every operation name, round-trip every named enum value, -reject invalid enum/target inputs, and verify and reparse representative barrier, -scheduling, buffer, and matrix operations in an LLVM function. The tests require -no AMD GPU or ROCm runtime. diff --git a/mlir/dialects/rocdl/REGENERATING.md b/mlir/dialects/rocdl/REGENERATING.md deleted file mode 100644 index 526cb17b..00000000 --- a/mlir/dialects/rocdl/REGENERATING.md +++ /dev/null @@ -1,62 +0,0 @@ -# Regenerating the ROCDL bindings - -These are instructions for an agent. They describe what the generated ROCDL -bindings must contain after an LLVM bump, where to derive them from, and how to -check the result. Scripts and intermediate files are not committed; only the -generated bindings are. - -## Sources - -Use the LLVM revision Bazel pins, and read TableGen's resolved records rather -than the `.td` text: multiclasses instantiate many ROCDL operations. - -```sh -bazel build @llvm-project//llvm:llvm-tblgen -llvm_project="$(bazel info output_base)/external/+llvm+llvm-project" -bazel-bin/external/+llvm+llvm-project/llvm/llvm-tblgen --dump-json \ - -I "$llvm_project/mlir/include" -I "$llvm_project/llvm/include" \ - "$llvm_project/mlir/include/mlir/Dialect/LLVMIR/ROCDLOps.td" \ - -o /tmp/rocdl-records.json -``` - -Every `Op`, `AttrDef` and `EnumInfo` record whose dialect is `ROCDL_Dialect` -belongs to the bindings. - -## What each file must contain - -Names, builder parameters, `opts`, wrappers, C API names and comments follow -`mlir/dialects/README.md`. Specific to ROCDL: - -- `ops.zig`: one builder per operation, listed in `test.zig`'s - `operation_names`. Follow argument constraints through `Arg` wrappers. -- `enums.zig`: one enum per ROCDL enum, with its bit width and values. Unlike - the README rule, a case whose IR spelling is not an identifier or is a - keyword takes the C++ symbol (`volatile` → `volatile_op`). Bit enums are - non-exhaustive and provide `combine` and `contains`. -- `attributes.zig`: one `Attr` wrapper per enum attribute, and - `enum_attributes` listing them. -- `types.zig`: `AddressSpace`, `pointerType`, and a `v` constructor - for every vector type the records use, named or inline. -- `capi.h`/`capi.cc`: `mlirROCDLGetOperationCount` under `// Introspection` - and, per enum attribute, the predicate, the getter (null for out-of-range - values) and `GetValue` under `// Attributes`. The `// Target` sections are - maintained by hand: in the - pinned revision `ROCDLTargetAttr::getChecked` delegates to `Base::get`, so the - C wrapper verifies target inputs first. - -Each generated Zig file starts with the "Generated from LLVM ROCDL TableGen -records" header. `target.zig`, `annotations.zig` and `rocdl.zig` are maintained -by hand; update their aliases and helpers when LLVM's surface changes. If the -dialect gains variadic operands, regions, successors, its own types, an enum -without an attribute, or new target fields or discardable attributes, extend -the bindings and this file. - -## Checks - -- Format and run the conformance checks of `mlir/dialects/README.md`. -- `bazel test //mlir/dialects/rocdl:test //dsl/fly:test //zml:test`. Beyond the - README's tests, the operation list is compared to - `mlirROCDLGetOperationCount`, and attribute tests exercise every named enum - case, named bit groups and combined bitfields. Add fixtures when a change - introduces a new operand layout or attribute representation. -- A regeneration without an LLVM change is byte-identical. diff --git a/mlir/dialects/rocdl/annotations.zig b/mlir/dialects/rocdl/annotations.zig deleted file mode 100644 index 85128473..00000000 --- a/mlir/dialects/rocdl/annotations.zig +++ /dev/null @@ -1,56 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); - -/// Discardable ROCDL attributes for an `llvm.func` operation. -pub const FunctionOpts = struct { - kernel: bool = false, - reqdWorkGroupSize: ?[3]i32 = null, - flatWorkGroupSize: ?[]const u8 = null, - maxFlatWorkGroupSize: ?i32 = null, - wavesPerEu: ?WavesPerEu = null, - unsafeFpAtomics: ?bool = null, - - pub const WavesPerEu = struct { - min: u32, - max: ?u32 = null, - }; -}; - -pub fn annotateFunction(ctx: *mlir.Context, op: *mlir.Operation, opts: FunctionOpts) void { - if (opts.kernel) op.setAttributeByName("rocdl.kernel", .unit(ctx)); - if (opts.reqdWorkGroupSize) |size| op.setAttributeByName("rocdl.reqd_work_group_size", .denseArray(ctx, .i32, &size)); - if (opts.flatWorkGroupSize) |size| op.setAttributeByName("rocdl.flat_work_group_size", .string(ctx, size)); - if (opts.maxFlatWorkGroupSize) |size| op.setAttributeByName("rocdl.max_flat_work_group_size", .int(ctx, .i32, size)); - if (opts.wavesPerEu) |waves| { - var buffer: [24]u8 = undefined; - const value = if (waves.max) |max| std.fmt.bufPrint(&buffer, "{d},{d}", .{ waves.min, max }) else std.fmt.bufPrint(&buffer, "{d}", .{waves.min}); - op.setAttributeByName("rocdl.waves_per_eu", .string(ctx, value catch unreachable)); - } - if (opts.unsafeFpAtomics) |enabled| op.setAttributeByName("rocdl.unsafe_fp_atomics", .boolean(ctx, enabled)); -} - -/// A missing flag preserves the target's default; false explicitly disables it. -pub const ModuleOpts = struct { - xnack: ?bool = null, - sramecc: ?bool = null, -}; - -pub fn annotateModule(ctx: *mlir.Context, op: *mlir.Operation, opts: ModuleOpts) void { - if (opts.xnack) |enabled| op.setAttributeByName("rocdl.xnack", .boolean(ctx, enabled)); - if (opts.sramecc) |enabled| op.setAttributeByName("rocdl.sramecc", .boolean(ctx, enabled)); -} - -pub const MemoryOpts = struct { - lastUse: bool = false, - noRemoteMemory: bool = false, - noFineGrainedMemory: bool = false, - ignoreDenormalMode: bool = false, -}; - -pub fn annotateMemory(ctx: *mlir.Context, op: *mlir.Operation, opts: MemoryOpts) void { - if (opts.lastUse) op.setAttributeByName("rocdl.last_use", .unit(ctx)); - if (opts.noRemoteMemory) op.setAttributeByName("rocdl.no_remote_memory", .unit(ctx)); - if (opts.noFineGrainedMemory) op.setAttributeByName("rocdl.no_fine_grained_memory", .unit(ctx)); - if (opts.ignoreDenormalMode) op.setAttributeByName("rocdl.ignore_denormal_mode", .unit(ctx)); -} diff --git a/mlir/dialects/rocdl/attributes.zig b/mlir/dialects/rocdl/attributes.zig deleted file mode 100644 index 9e1b9863..00000000 --- a/mlir/dialects/rocdl/attributes.zig +++ /dev/null @@ -1,417 +0,0 @@ -// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -const c = @import("c"); -const mlir = @import("mlir"); -const enums = @import("enums.zig"); - -pub const TargetAttr = @import("target.zig").TargetAttr; - -/// `#rocdl.buffer_oob_mode`. -pub const BufferOOBModeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.BufferOOBMode; - pub const isAFn = c.mlirAttributeIsAROCDLBufferOOBMode; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLBufferOOBModeAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLBufferOOBModeAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.buffer_oob_mode_flag`. -pub const BufferOOBModeModuleFlagAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.BufferOOBMode; - pub const isAFn = c.mlirAttributeIsAROCDLBufferOOBModeModuleFlag; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLBufferOOBModeModuleFlagAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLBufferOOBModeModuleFlagAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.gfx12_atomic_cache_policy`. -pub const Gfx12AtomicCachePolicyAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.Gfx12AtomicCachePolicy; - pub const isAFn = c.mlirAttributeIsAROCDLGfx12AtomicCachePolicy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLGfx12AtomicCachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLGfx12AtomicCachePolicyAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.gfx12_cache_policy`. -pub const Gfx12CachePolicyAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.Gfx12CachePolicy; - pub const isAFn = c.mlirAttributeIsAROCDLGfx12CachePolicy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLGfx12CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLGfx12CachePolicyAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.gfx942_cache_policy`. -pub const Gfx942CachePolicyAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.Gfx942CachePolicy; - pub const isAFn = c.mlirAttributeIsAROCDLGfx942CachePolicy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLGfx942CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLGfx942CachePolicyAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.mfma_neg_modifier`. -pub const MFMANegModifierAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.MFMANegModifier; - pub const isAFn = c.mlirAttributeIsAROCDLMFMANegModifier; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLMFMANegModifierAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLMFMANegModifierAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.mfma_perm_b`. -pub const MFMAPermBAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.MFMAPermB; - pub const isAFn = c.mlirAttributeIsAROCDLMFMAPermB; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLMFMAPermBAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLMFMAPermBAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.matrix_format`. -pub const MatrixFormatAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.MatrixFormat; - pub const isAFn = c.mlirAttributeIsAROCDLMatrixFormat; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLMatrixFormatAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLMatrixFormatAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.pre_gfx12_cache_policy`. -pub const PreGfx12CachePolicyAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.PreGfx12CachePolicy; - pub const isAFn = c.mlirAttributeIsAROCDLPreGfx12CachePolicy; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLPreGfx12CachePolicyAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLPreGfx12CachePolicyAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.sched_group_mask`. -pub const SchedGroupMaskAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.SchedGroupMask; - pub const isAFn = c.mlirAttributeIsAROCDLSchedGroupMask; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLSchedGroupMaskAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLSchedGroupMaskAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.tbuffer_oob_mode_flag`. -pub const TBufferOOBModeModuleFlagAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.BufferOOBMode; - pub const isAFn = c.mlirAttributeIsAROCDLTBufferOOBModeModuleFlag; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLTBufferOOBModeModuleFlagAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.wmma_c_modifier`. -pub const WMMACModifierAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.WMMACModifier; - pub const isAFn = c.mlirAttributeIsAROCDLWMMACModifier; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLWMMACModifierAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLWMMACModifierAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.wmma_matrix_scale`. -pub const WMMAMatrixScaleAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.WMMAMatrixScale; - pub const isAFn = c.mlirAttributeIsAROCDLWMMAMatrixScale; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLWMMAMatrixScaleAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLWMMAMatrixScaleAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -/// `#rocdl.wmma_matrix_scale_format`. -pub const WMMAMatrixScaleFormatAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const Value = enums.WMMAMatrixScaleFormat; - pub const isAFn = c.mlirAttributeIsAROCDLWMMAMatrixScaleFormat; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - value: Value, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLWMMAMatrixScaleFormatAttrGet(ctx.ptr(), @backingInt(args.value)).ptr orelse return error.InvalidMlir); - } - - pub fn value(self: *const Self) Value { - return @fromBackingInt(@intCast(c.mlirROCDLWMMAMatrixScaleFormatAttrGetValue(self.ptr()))); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -pub const enum_attributes = .{ - BufferOOBModeAttr, - BufferOOBModeModuleFlagAttr, - Gfx12AtomicCachePolicyAttr, - Gfx12CachePolicyAttr, - Gfx942CachePolicyAttr, - MFMANegModifierAttr, - MFMAPermBAttr, - MatrixFormatAttr, - PreGfx12CachePolicyAttr, - SchedGroupMaskAttr, - TBufferOOBModeModuleFlagAttr, - WMMACModifierAttr, - WMMAMatrixScaleAttr, - WMMAMatrixScaleFormatAttr, -}; diff --git a/mlir/dialects/rocdl/capi.cc b/mlir/dialects/rocdl/capi.cc deleted file mode 100644 index 7a01bd56..00000000 --- a/mlir/dialects/rocdl/capi.cc +++ /dev/null @@ -1,440 +0,0 @@ -//===- capi.cc - C API for the rocdl dialect ------------------------------===// - -#include "capi.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Support.h" -#include "mlir/Dialect/LLVMIR/ROCDLDialect.h" -#include "mlir/IR/Diagnostics.h" - -// Introspection - -intptr_t mlirROCDLGetOperationCount(MlirContext context) { - auto *ctx = unwrap(context); - if (!ctx) - return 0; - ctx->getOrLoadDialect(); - intptr_t count = 0; - for (auto op : ctx->getRegisteredOperations()) - if (op.getDialectNamespace() == "rocdl") - ++count; - return count; -} - -// Attributes - -bool mlirAttributeIsAROCDLBufferOOBMode(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirROCDLBufferOOBModeAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::BufferOOBModeAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLBufferOOBModeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAROCDLBufferOOBModeModuleFlag(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} - -MlirAttribute mlirROCDLBufferOOBModeModuleFlagAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::BufferOOBModeModuleFlagAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsAROCDLGfx12AtomicCachePolicy(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} - -MlirAttribute mlirROCDLGfx12AtomicCachePolicyAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = mlir::ROCDL::symbolizeGfx12AtomicCachePolicy( - static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::Gfx12AtomicCachePolicyAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLGfx12AtomicCachePolicyAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsAROCDLGfx12CachePolicy(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirROCDLGfx12CachePolicyAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizeGfx12CachePolicy(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::Gfx12CachePolicyAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLGfx12CachePolicyAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAROCDLGfx942CachePolicy(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} - -MlirAttribute mlirROCDLGfx942CachePolicyAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizeGfx942CachePolicy(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::Gfx942CachePolicyAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLGfx942CachePolicyAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAROCDLMFMANegModifier(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirROCDLMFMANegModifierAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizeMFMANegModifier(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::MFMANegModifierAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLMFMANegModifierAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAROCDLMFMAPermB(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirROCDLMFMAPermBAttrGet(MlirContext context, uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = mlir::ROCDL::symbolizeMFMAPermB(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::MFMAPermBAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLMFMAPermBAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAROCDLMatrixFormat(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirROCDLMatrixFormatAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizeMatrixFormat(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::MatrixFormatAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLMatrixFormatAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAROCDLPreGfx12CachePolicy(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} - -MlirAttribute mlirROCDLPreGfx12CachePolicyAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizePreGfx12CachePolicy(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::PreGfx12CachePolicyAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLPreGfx12CachePolicyAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsAROCDLSchedGroupMask(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirROCDLSchedGroupMaskAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizeSchedGroupMask(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::SchedGroupMaskAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLSchedGroupMaskAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAROCDLTBufferOOBModeModuleFlag(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} - -MlirAttribute mlirROCDLTBufferOOBModeModuleFlagAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizeBufferOOBMode(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::TBufferOOBModeModuleFlagAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -bool mlirAttributeIsAROCDLWMMACModifier(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirROCDLWMMACModifierAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - if (value > 65535) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizeWMMACModifier(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::WMMACModifierAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLWMMACModifierAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAROCDLWMMAMatrixScale(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirROCDLWMMAMatrixScaleAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizeWMMAMatrixScale(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::WMMAMatrixScaleAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLWMMAMatrixScaleAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAROCDLWMMAMatrixScaleFormat(MlirAttribute attr) { - return llvm::isa_and_nonnull( - unwrap(attr)); -} - -MlirAttribute mlirROCDLWMMAMatrixScaleFormatAttrGet(MlirContext context, - uint32_t value) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - auto symbol = - mlir::ROCDL::symbolizeWMMAMatrixScaleFormat(static_cast(value)); - if (!symbol) - return {nullptr}; - ctx->getOrLoadDialect(); - return wrap(mlir::ROCDL::WMMAMatrixScaleFormatAttr::get(ctx, *symbol)); -} - -uint32_t mlirROCDLWMMAMatrixScaleFormatAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)) - .getValue()); -} - -// Target - -bool mlirAttributeIsAROCDLTarget(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirROCDLTargetAttrGet(MlirContext context, int optLevel, - MlirStringRef triple, MlirStringRef chip, - MlirStringRef features, MlirStringRef abi, - MlirAttribute flags, MlirAttribute link) { - auto *ctx = unwrap(context); - if (!ctx) - return {nullptr}; - for (auto s : {triple, chip, features, abi}) - if (!s.data && s.length) - return {nullptr}; - auto flagsValue = - llvm::dyn_cast_if_present(unwrap(flags)); - auto linkValue = llvm::dyn_cast_if_present(unwrap(link)); - if (unwrap(flags) && (!flagsValue || flagsValue.getContext() != ctx)) - return {nullptr}; - if (unwrap(link) && (!linkValue || linkValue.getContext() != ctx)) - return {nullptr}; - ctx->getOrLoadDialect(); - // The upstream custom getChecked builder delegates to unchecked Base::get. - // Verify first so invalid C API inputs return null instead of asserting. - if (mlir::failed(mlir::ROCDL::ROCDLTargetAttr::verifyInvariants( - [&] { return mlir::emitError(mlir::UnknownLoc::get(ctx)); }, optLevel, - unwrap(triple), unwrap(chip), unwrap(features), unwrap(abi), - flagsValue, linkValue))) - return {nullptr}; - return wrap(mlir::ROCDL::ROCDLTargetAttr::get( - ctx, optLevel, unwrap(triple), unwrap(chip), unwrap(features), - unwrap(abi), flagsValue, linkValue)); -} - -int mlirROCDLTargetAttrGetOptLevel(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).getO(); -} - -MlirStringRef mlirROCDLTargetAttrGetTriple(MlirAttribute attr) { - return wrap( - llvm::cast(unwrap(attr)).getTriple()); -} - -MlirStringRef mlirROCDLTargetAttrGetChip(MlirAttribute attr) { - return wrap(llvm::cast(unwrap(attr)).getChip()); -} - -MlirStringRef mlirROCDLTargetAttrGetFeatures(MlirAttribute attr) { - return wrap( - llvm::cast(unwrap(attr)).getFeatures()); -} - -MlirStringRef mlirROCDLTargetAttrGetAbi(MlirAttribute attr) { - return wrap(llvm::cast(unwrap(attr)).getAbi()); -} - -MlirAttribute mlirROCDLTargetAttrGetFlags(MlirAttribute attr) { - return wrap( - llvm::cast(unwrap(attr)).getFlags()); -} - -MlirAttribute mlirROCDLTargetAttrGetLink(MlirAttribute attr) { - return wrap(llvm::cast(unwrap(attr)).getLink()); -} - -bool mlirROCDLTargetAttrHasFlag(MlirAttribute attr, MlirStringRef flag) { - return llvm::cast(unwrap(attr)) - .hasFlag(unwrap(flag)); -} - -bool mlirROCDLTargetAttrHasWave64(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).hasWave64(); -} - -bool mlirROCDLTargetAttrHasFastMath(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).hasFastMath(); -} - -bool mlirROCDLTargetAttrHasDaz(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).hasDaz(); -} - -bool mlirROCDLTargetAttrHasFiniteOnly(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).hasFiniteOnly(); -} - -bool mlirROCDLTargetAttrHasUnsafeMath(MlirAttribute attr) { - return llvm::cast(unwrap(attr)).hasUnsafeMath(); -} - -bool mlirROCDLTargetAttrHasCorrectSqrt(MlirAttribute attr) { - return llvm::cast(unwrap(attr)) - .hasCorrectSqrt(); -} diff --git a/mlir/dialects/rocdl/capi.h b/mlir/dialects/rocdl/capi.h deleted file mode 100644 index b73a948d..00000000 --- a/mlir/dialects/rocdl/capi.h +++ /dev/null @@ -1,140 +0,0 @@ -//===- capi.h - C API for the rocdl dialect -----------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_ROCDL_CAPI_H -#define MLIR_DIALECTS_ROCDL_CAPI_H - -#include "mlir-c/Dialect/ROCDL.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Introspection - -MLIR_CAPI_EXPORTED intptr_t mlirROCDLGetOperationCount(MlirContext context); - -// Attributes - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLBufferOOBMode(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLBufferOOBModeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLBufferOOBModeAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool -mlirAttributeIsAROCDLBufferOOBModeModuleFlag(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLBufferOOBModeModuleFlagAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool -mlirAttributeIsAROCDLGfx12AtomicCachePolicy(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLGfx12AtomicCachePolicyAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLGfx12AtomicCachePolicyAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool -mlirAttributeIsAROCDLGfx12CachePolicy(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLGfx12CachePolicyAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLGfx12CachePolicyAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool -mlirAttributeIsAROCDLGfx942CachePolicy(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLGfx942CachePolicyAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLGfx942CachePolicyAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool -mlirAttributeIsAROCDLMFMANegModifier(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLMFMANegModifierAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLMFMANegModifierAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLMFMAPermB(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLMFMAPermBAttrGet(MlirContext context, - uint32_t value); -MLIR_CAPI_EXPORTED uint32_t mlirROCDLMFMAPermBAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLMatrixFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLMatrixFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLMatrixFormatAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool -mlirAttributeIsAROCDLPreGfx12CachePolicy(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLPreGfx12CachePolicyAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLPreGfx12CachePolicyAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLSchedGroupMask(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLSchedGroupMaskAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLSchedGroupMaskAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool -mlirAttributeIsAROCDLTBufferOOBModeModuleFlag(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLTBufferOOBModeModuleFlagAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLTBufferOOBModeModuleFlagAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLWMMACModifier(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLWMMACModifierAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLWMMACModifierAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool -mlirAttributeIsAROCDLWMMAMatrixScale(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLWMMAMatrixScaleAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLWMMAMatrixScaleAttrGetValue(MlirAttribute attr); - -MLIR_CAPI_EXPORTED bool -mlirAttributeIsAROCDLWMMAMatrixScaleFormat(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLWMMAMatrixScaleFormatAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirROCDLWMMAMatrixScaleFormatAttrGetValue(MlirAttribute attr); - -// Target - -MLIR_CAPI_EXPORTED bool mlirAttributeIsAROCDLTarget(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLTargetAttrGet( - MlirContext context, int optLevel, MlirStringRef triple, MlirStringRef chip, - MlirStringRef features, MlirStringRef abi, MlirAttribute flags, - MlirAttribute link); -MLIR_CAPI_EXPORTED int mlirROCDLTargetAttrGetOptLevel(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirROCDLTargetAttrGetTriple(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirROCDLTargetAttrGetChip(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef -mlirROCDLTargetAttrGetFeatures(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirStringRef mlirROCDLTargetAttrGetAbi(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirROCDLTargetAttrGetFlags(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute mlirROCDLTargetAttrGetLink(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFlag(MlirAttribute attr, - MlirStringRef flag); -MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasWave64(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFastMath(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasDaz(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasFiniteOnly(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasUnsafeMath(MlirAttribute attr); -MLIR_CAPI_EXPORTED bool mlirROCDLTargetAttrHasCorrectSqrt(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_ROCDL_CAPI_H diff --git a/mlir/dialects/rocdl/enums.zig b/mlir/dialects/rocdl/enums.zig deleted file mode 100644 index fde24ff7..00000000 --- a/mlir/dialects/rocdl/enums.zig +++ /dev/null @@ -1,208 +0,0 @@ -// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -/// ROCDL buffer out-of-bounds mode -pub const BufferOOBMode = enum(u32) { - any = 0, - relaxed = 1, - strict = 2, -}; - -/// gfx12 atomic cache policy bits -pub const Gfx12AtomicCachePolicy = enum(u32) { - none = 0, - keep_return = 1, - nt = 2, - cascade = 4, - scope_se = 8, - scope_dev = 16, - scope_sys = 24, - nv = 32, - volatile_op = 2147483648, - _, - - pub fn combine(values: []const Gfx12AtomicCachePolicy) Gfx12AtomicCachePolicy { - var bits: u32 = 0; - for (values) |value| bits |= @backingInt(value); - return @fromBackingInt(bits); - } - - pub fn contains(self: Gfx12AtomicCachePolicy, value: Gfx12AtomicCachePolicy) bool { - return @backingInt(self) & @backingInt(value) == @backingInt(value); - } -}; - -/// gfx12 cache policy bits -pub const Gfx12CachePolicy = enum(u32) { - none = 0, - nt = 1, - ht = 2, - lu = 3, - nt_rt = 4, - rt_nt = 5, - nt_ht = 6, - nt_wb = 7, - scope_se = 8, - scope_dev = 16, - scope_sys = 24, - nv = 32, - swz = 64, - scal = 2048, - volatile_op = 2147483648, - _, - - pub fn combine(values: []const Gfx12CachePolicy) Gfx12CachePolicy { - var bits: u32 = 0; - for (values) |value| bits |= @backingInt(value); - return @fromBackingInt(bits); - } - - pub fn contains(self: Gfx12CachePolicy, value: Gfx12CachePolicy) bool { - return @backingInt(self) & @backingInt(value) == @backingInt(value); - } -}; - -/// gfx942 buffer cache policy bits -pub const Gfx942CachePolicy = enum(u32) { - none = 0, - sc0 = 1, - nt = 2, - swz = 8, - sc1 = 16, - volatile_op = 2147483648, - _, - - pub fn combine(values: []const Gfx942CachePolicy) Gfx942CachePolicy { - var bits: u32 = 0; - for (values) |value| bits |= @backingInt(value); - return @fromBackingInt(bits); - } - - pub fn contains(self: Gfx942CachePolicy, value: Gfx942CachePolicy) bool { - return @backingInt(self) & @backingInt(value) == @backingInt(value); - } -}; - -/// negation modifier bitfield for gfx94x double-precision MFMA -pub const MFMANegModifier = enum(u32) { - none = 0, - neg_a = 1, - neg_b = 2, - neg_c = 4, - _, - - pub fn combine(values: []const MFMANegModifier) MFMANegModifier { - var bits: u32 = 0; - for (values) |value| bits |= @backingInt(value); - return @fromBackingInt(bits); - } - - pub fn contains(self: MFMANegModifier, value: MFMANegModifier) bool { - return @backingInt(self) & @backingInt(value) == @backingInt(value); - } -}; - -/// permutations of the lanes storing B in an MFMA -pub const MFMAPermB = enum(u32) { - none = 0, - bcast_first_32 = 1, - bcast_second_32 = 2, - rotate_16_right = 3, - bcast_first_16 = 4, - bcast_second_16 = 5, - bcast_third_16 = 6, - bcast_fourth_16 = 7, -}; - -/// matrix operand formats selected by scaled MFMA/WMMA format fields -pub const MatrixFormat = enum(u32) { - fp8_e4m3 = 0, - fp8_e5m2 = 1, - fp6_e2m3 = 2, - fp6_e3m2 = 3, - fp4_e2m1 = 4, -}; - -/// pre-gfx12 buffer cache policy bits -pub const PreGfx12CachePolicy = enum(u32) { - none = 0, - glc = 1, - slc = 2, - dlc = 4, - swz = 8, - scc = 16, - all = 23, - volatile_op = 2147483648, - _, - - pub fn combine(values: []const PreGfx12CachePolicy) PreGfx12CachePolicy { - var bits: u32 = 0; - for (values) |value| bits |= @backingInt(value); - return @fromBackingInt(bits); - } - - pub fn contains(self: PreGfx12CachePolicy, value: PreGfx12CachePolicy) bool { - return @backingInt(self) & @backingInt(value) == @backingInt(value); - } -}; - -/// instruction type mask for scheduling barriers -pub const SchedGroupMask = enum(u32) { - none = 0, - non_mem_non_sideeffect = 1, - valu = 2, - salu = 4, - mfma_wmma = 8, - all_vmem = 16, - vmem_read = 32, - vmem_write = 64, - all_ds = 128, - ds_read = 256, - ds_write = 512, - transcendental = 1024, - ldsdma = 2048, - all = 4095, - _, - - pub fn combine(values: []const SchedGroupMask) SchedGroupMask { - var bits: u32 = 0; - for (values) |value| bits |= @backingInt(value); - return @fromBackingInt(bits); - } - - pub fn contains(self: SchedGroupMask, value: SchedGroupMask) bool { - return @backingInt(self) & @backingInt(value) == @backingInt(value); - } -}; - -/// WMMA C operand modifiers -pub const WMMACModifier = enum(u16) { - none = 0, - neg = 1, - abs = 2, - neg_abs = 3, - _, - - pub fn combine(values: []const WMMACModifier) WMMACModifier { - var bits: u16 = 0; - for (values) |value| bits |= @backingInt(value); - return @fromBackingInt(bits); - } - - pub fn contains(self: WMMACModifier, value: WMMACModifier) bool { - return @backingInt(self) & @backingInt(value) == @backingInt(value); - } -}; - -/// matrix scale row selector -pub const WMMAMatrixScale = enum(u32) { - row0 = 0, - row1 = 1, -}; - -/// matrix scale exponent formats -pub const WMMAMatrixScaleFormat = enum(u32) { - e8 = 0, - e5m3 = 1, - e4m3 = 2, -}; diff --git a/mlir/dialects/rocdl/ops.zig b/mlir/dialects/rocdl/ops.zig deleted file mode 100644 index b18a33cf..00000000 --- a/mlir/dialects/rocdl/ops.zig +++ /dev/null @@ -1,8600 +0,0 @@ -// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -/// `rocdl.asyncmark`. Mark the end of a group of asynchronous operations -pub fn asyncmark( - ctx: *mlir.Context, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.asyncmark", .{ - .location = location, - }); -} - -/// `rocdl.ballot`. Vote across thread group -pub fn ballot( - ctx: *mlir.Context, - pred: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.ballot", .{ - .operands = .{ .flat = &.{pred} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.barrier`. -pub fn barrier( - ctx: *mlir.Context, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.barrier", .{ - .location = location, - }); -} - -/// `rocdl.cluster.id.x`. -pub fn cluster_id_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.cluster.id.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cluster.id.y`. -pub fn cluster_id_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.cluster.id.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cluster.id.z`. -pub fn cluster_id_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.cluster.id.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cluster.load.async.to.lds.b128`. -pub fn cluster_load_async_to_lds_b128( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - mask: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - cpol: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.cpol) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b128", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cluster.load.async.to.lds.b32`. -pub fn cluster_load_async_to_lds_b32( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - mask: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - cpol: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.cpol) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b32", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cluster.load.async.to.lds.b64`. -pub fn cluster_load_async_to_lds_b64( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - mask: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - cpol: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.cpol) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b64", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cluster.load.async.to.lds.b8`. -pub fn cluster_load_async_to_lds_b8( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - mask: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - cpol: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.cpol) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "cpol", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.cluster.load.async.to.lds.b8", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr, mask } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cluster.workgroup.id.x`. -pub fn cluster_workgroup_id_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cluster.workgroup.id.y`. -pub fn cluster_workgroup_id_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cluster.workgroup.id.z`. -pub fn cluster_workgroup_id_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.cluster.workgroup.id.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cos`. -pub fn cos( - ctx: *mlir.Context, - arg: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cos", .{ - .operands = .{ .flat = &.{arg} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.f32.bf8`. Convert bf8 to f32 -pub fn cvt_f32_bf8( - ctx: *mlir.Context, - srcA: *const mlir.Value, - res_type: *const mlir.Type, - byteSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.f32.bf8", .{ - .operands = .{ .flat = &.{srcA} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.f32.fp8`. Convert fp8 to f32 -pub fn cvt_f32_fp8( - ctx: *mlir.Context, - srcA: *const mlir.Value, - res_type: *const mlir.Type, - byteSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.f32.fp8", .{ - .operands = .{ .flat = &.{srcA} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.pk.bf8.f32`. Convert two f32's to bf8 -pub fn cvt_pk_bf8_f32( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - old: *const mlir.Value, - res_type: *const mlir.Type, - wordSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.pk.bf8.f32", .{ - .operands = .{ .flat = &.{ srcA, srcB, old } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.pk.f32.bf8`. Convert packed bf8 to packed f32 -pub fn cvt_pk_f32_bf8( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - wordSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.pk.f32.bf8", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.pk.f32.fp8`. Convert packed fp8 to packed f32 -pub fn cvt_pk_f32_fp8( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - wordSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.pk.f32.fp8", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.pk.fp8.f32`. Convert two f32's to fp8 -pub fn cvt_pk_fp8_f32( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - old: *const mlir.Value, - res_type: *const mlir.Type, - wordSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "wordSel", wordSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.pk.fp8.f32", .{ - .operands = .{ .flat = &.{ srcA, srcB, old } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.pkrtz`. Convert two f32 input into a vector<2xf16> -pub fn cvt_pkrtz( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.pkrtz", .{ - .operands = .{ .flat = &.{ srcA, srcB } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk16.bf16.bf6`. Scales 16 bf6 and converts them to 16 bf16. -pub fn cvt_scale_pk16_bf16_bf6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.bf16.bf6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk16.bf16.fp6`. Scales 16 fp6 and converts them to 16 bf16. -pub fn cvt_scale_pk16_bf16_fp6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.bf16.fp6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk16.f16.bf6`. Scales 16 bf6 and converts them to 16 f16. -pub fn cvt_scale_pk16_f16_bf6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f16.bf6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk16.f16.fp6`. Scales 16 fp6 and converts them to 16 f16. -pub fn cvt_scale_pk16_f16_fp6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f16.fp6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk16.f32.bf6`. Scales 16 bf6 and converts them to 16 f32. -pub fn cvt_scale_pk16_f32_bf6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f32.bf6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk16.f32.fp6`. Scales 16 fp6 and converts them to 16 f32. -pub fn cvt_scale_pk16_f32_fp6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk16.f32.fp6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk8.bf16.bf8`. Scales 8 bf8 and converts them to 8 bf16. -pub fn cvt_scale_pk8_bf16_bf8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.bf8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk8.bf16.fp4`. Scales 8 fp4 and converts them to 8 bf16. -pub fn cvt_scale_pk8_bf16_fp4( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.fp4", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk8.bf16.fp8`. Scales 8 fp8 and converts them to 8 bf16. -pub fn cvt_scale_pk8_bf16_fp8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.bf16.fp8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk8.f16.bf8`. Scales 8 bf8 and converts them to 8 f16. -pub fn cvt_scale_pk8_f16_bf8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.bf8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk8.f16.fp4`. Scales 8 fp4 and converts them to 8 f16. -pub fn cvt_scale_pk8_f16_fp4( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.fp4", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk8.f16.fp8`. Scales 8 fp8 and converts them to 8 f16. -pub fn cvt_scale_pk8_f16_fp8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f16.fp8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk8.f32.bf8`. Scales 8 bf8 and converts them to 8 f32. -pub fn cvt_scale_pk8_f32_bf8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.bf8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk8.f32.fp4`. Scales 8 fp4 and converts them to 8 f32. -pub fn cvt_scale_pk8_f32_fp4( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.fp4", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scale.pk8.f32.fp8`. Scales 8 fp8 and converts them to 8 f32. -pub fn cvt_scale_pk8_f32_fp8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - scaleSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "scaleSel", scaleSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scale.pk8.f32.fp8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.2xpk16.bf6.f32`. Scale and convert two vector<16xf32> to 32 packed bf6 -pub fn cvt_scalef32_2xpk16_bf6_f32( - ctx: *mlir.Context, - src0: *const mlir.Value, - src1: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.2xpk16.bf6.f32", .{ - .operands = .{ .flat = &.{ src0, src1, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.2xpk16.fp6.f32`. Scale and convert two vector<16xf32> to 32 packed fp6 -pub fn cvt_scalef32_2xpk16_fp6_f32( - ctx: *mlir.Context, - src0: *const mlir.Value, - src1: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.2xpk16.fp6.f32", .{ - .operands = .{ .flat = &.{ src0, src1, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.f16.bf8`. Scaled convert bf8 from packed vector to f16, updating tied result -pub fn cvt_scalef32_f16_bf8( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcSelIndex: *const mlir.Attribute, - dstLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); - attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f16.bf8", .{ - .operands = .{ .flat = &.{ oldVdst, src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.f16.fp8`. Scaled convert fp8 from packed vector to f16, updating tied result -pub fn cvt_scalef32_f16_fp8( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcSelIndex: *const mlir.Attribute, - dstLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); - attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f16.fp8", .{ - .operands = .{ .flat = &.{ oldVdst, src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.f32.bf8`. Scaled convert bf8 from packed vector to f32 -pub fn cvt_scalef32_f32_bf8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f32.bf8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.f32.fp8`. Scaled convert fp8 from packed vector to f32 -pub fn cvt_scalef32_f32_fp8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.f32.fp8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.bf16.bf8`. Scaled convert two bf8to two bf16 -pub fn cvt_scalef32_pk_bf16_bf8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.bf8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.bf16.fp4`. Scale and convert two packed fp4 to packed bf16 -pub fn cvt_scalef32_pk_bf16_fp4( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.fp4", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.bf16.fp8`. Scaled convert two fp8to two bf16 -pub fn cvt_scalef32_pk_bf16_fp8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf16.fp8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.bf8.bf16`. Scaled convert two bf16to two bf8, updating packed vector -pub fn cvt_scalef32_pk_bf8_bf16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.bf16", .{ - .operands = .{ .flat = &.{ oldVdst, src0, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.bf8.f16`. Scaled convert two f16to two bf8, updating packed vector -pub fn cvt_scalef32_pk_bf8_f16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.f16", .{ - .operands = .{ .flat = &.{ oldVdst, src0, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.bf8.f32`. Scaled convert two f32 to two bf8, updating packed vector -pub fn cvt_scalef32_pk_bf8_f32( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - src1: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.bf8.f32", .{ - .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.f16.bf8`. Scaled convert two bf8to two f16 -pub fn cvt_scalef32_pk_f16_bf8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.bf8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.f16.fp4`. Scale and convert two packed fp4 to packed f16 -pub fn cvt_scalef32_pk_f16_fp4( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.fp4", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.f16.fp8`. Scaled convert two fp8to two f16 -pub fn cvt_scalef32_pk_f16_fp8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f16.fp8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.f32.bf8`. Scaled convert two bf8to two f32 -pub fn cvt_scalef32_pk_f32_bf8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.bf8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.f32.fp4`. Scale and convert two packed fp4 to packed f32 -pub fn cvt_scalef32_pk_f32_fp4( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcSelIndex", srcSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.fp4", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.f32.fp8`. Scaled convert two fp8to two f32 -pub fn cvt_scalef32_pk_f32_fp8( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - srcLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "srcLoHiSel", srcLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.f32.fp8", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.fp4.bf16`. Scale and convert two bf16 to packed fp4, updating tied vector -pub fn cvt_scalef32_pk_fp4_bf16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.bf16", .{ - .operands = .{ .flat = &.{ oldVdst, src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.fp4.f16`. Scale and convert two f16 to packed fp4, updating tied vector -pub fn cvt_scalef32_pk_fp4_f16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.f16", .{ - .operands = .{ .flat = &.{ oldVdst, src, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.fp4.f32`. Scale and convert two f32 values to two packed fp4, updating tied vector -pub fn cvt_scalef32_pk_fp4_f32( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - src1: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp4.f32", .{ - .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.fp8.bf16`. Scaled convert two bf16to two fp8, updating packed vector -pub fn cvt_scalef32_pk_fp8_bf16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.bf16", .{ - .operands = .{ .flat = &.{ oldVdst, src0, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.fp8.f16`. Scaled convert two f16to two fp8, updating packed vector -pub fn cvt_scalef32_pk_fp8_f16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.f16", .{ - .operands = .{ .flat = &.{ oldVdst, src0, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk.fp8.f32`. Scaled convert two f32 to two fp8, updating packed vector -pub fn cvt_scalef32_pk_fp8_f32( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - src1: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstLoHiSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstLoHiSel", dstLoHiSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk.fp8.f32", .{ - .operands = .{ .flat = &.{ oldVdst, src0, src1, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk16.bf6.bf16`. Scale and convert packed bf16 to packed bf6 -pub fn cvt_scalef32_pk16_bf6_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.bf16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk16.bf6.f16`. Scale and convert packed f16 to packed bf6 -pub fn cvt_scalef32_pk16_bf6_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.f16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk16.bf6.f32`. Scale and convert packed f32 to packed bf6 -pub fn cvt_scalef32_pk16_bf6_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.bf6.f32", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk16.fp6.bf16`. Scale and convert packed bf16 to packed fp6 -pub fn cvt_scalef32_pk16_fp6_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.bf16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk16.fp6.f16`. Scale and convert packed f16 to packed fp6 -pub fn cvt_scalef32_pk16_fp6_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.f16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk16.fp6.f32`. Scale and convert packed f32 to packed fp6 -pub fn cvt_scalef32_pk16_fp6_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk16.fp6.f32", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk32.bf16.bf6`. Scale and convert packed bf6 to packed bf16 -pub fn cvt_scalef32_pk32_bf16_bf6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf16.bf6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk32.bf16.fp6`. Scale and convert packed fp6 to packed bf16 -pub fn cvt_scalef32_pk32_bf16_fp6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf16.fp6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk32.bf6.bf16`. Scale and convert packed bf16 to packed bf6 -pub fn cvt_scalef32_pk32_bf6_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf6.bf16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk32.bf6.f16`. Scale and convert packed f16 to packed bf6 -pub fn cvt_scalef32_pk32_bf6_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.bf6.f16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk32.f16.bf6`. Scale and convert packed bf6 to packed f16 -pub fn cvt_scalef32_pk32_f16_bf6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f16.bf6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk32.f16.fp6`. Scale and convert packed fp6 to packed f16 -pub fn cvt_scalef32_pk32_f16_fp6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f16.fp6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk32.f32.bf6`. Scale and convert packed bf6 to packed f32 -pub fn cvt_scalef32_pk32_f32_bf6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f32.bf6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk32.f32.fp6`. Scale and convert packed fp6 to packed f32 -pub fn cvt_scalef32_pk32_f32_fp6( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.f32.fp6", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk32.fp6.bf16`. Scale and convert packed bf16 to packed fp6 -pub fn cvt_scalef32_pk32_fp6_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.fp6.bf16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk32.fp6.f16`. Scale and convert packed f16 to packed fp6 -pub fn cvt_scalef32_pk32_fp6_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk32.fp6.f16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk8.bf8.bf16`. Scale and convert packed bf16 to packed bf8 -pub fn cvt_scalef32_pk8_bf8_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.bf16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk8.bf8.f16`. Scale and convert packed f16 to packed bf8 -pub fn cvt_scalef32_pk8_bf8_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.f16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk8.bf8.f32`. Scale and convert packed f32 to packed bf8 -pub fn cvt_scalef32_pk8_bf8_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.bf8.f32", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk8.fp4.bf16`. Scale and convert packed bf16 to packed fp4 -pub fn cvt_scalef32_pk8_fp4_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.bf16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk8.fp4.f16`. Scale and convert packed f16 to packed fp4 -pub fn cvt_scalef32_pk8_fp4_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.f16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk8.fp4.f32`. Scale and convert packed f32 to packed fp4 -pub fn cvt_scalef32_pk8_fp4_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp4.f32", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk8.fp8.bf16`. Scale and convert packed bf16 to packed fp8 -pub fn cvt_scalef32_pk8_fp8_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.bf16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk8.fp8.f16`. Scale and convert packed f16 to packed fp8 -pub fn cvt_scalef32_pk8_fp8_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.f16", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.pk8.fp8.f32`. Scale and convert packed f32 to packed fp8 -pub fn cvt_scalef32_pk8_fp8_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.pk8.fp8.f32", .{ - .operands = .{ .flat = &.{ src, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.bf8.bf16`. Scaled convert bf16to bf8 with stochiastic rounding, updating packed vector -pub fn cvt_scalef32_sr_bf8_bf16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.bf16", .{ - .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.bf8.f16`. Scaled convert f16to bf8 with stochiastic rounding, updating packed vector -pub fn cvt_scalef32_sr_bf8_f16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.f16", .{ - .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.bf8.f32`. Scaled convert f32to bf8 with stochiastic rounding, updating packed vector -pub fn cvt_scalef32_sr_bf8_f32( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.bf8.f32", .{ - .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.fp8.bf16`. Scaled convert bf16to fp8 with stochiastic rounding, updating packed vector -pub fn cvt_scalef32_sr_fp8_bf16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.bf16", .{ - .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.fp8.f16`. Scaled convert f16to fp8 with stochiastic rounding, updating packed vector -pub fn cvt_scalef32_sr_fp8_f16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.f16", .{ - .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.fp8.f32`. Scaled convert f32to fp8 with stochiastic rounding, updating packed vector -pub fn cvt_scalef32_sr_fp8_f32( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src0: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.fp8.f32", .{ - .operands = .{ .flat = &.{ oldVdst, src0, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk.fp4.bf16`. Scale and convert two bf16 to packed fp4 with stochiastic rounding, updating tied vector -pub fn cvt_scalef32_sr_pk_fp4_bf16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.bf16", .{ - .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk.fp4.f16`. Scale and convert two f16 to packed fp4 with stochiastic rounding, updating tied vector -pub fn cvt_scalef32_sr_pk_fp4_f16( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.f16", .{ - .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk.fp4.f32`. Scale and convert two f32 to packed fp4 with stochiastic rounding, updating tied vector -pub fn cvt_scalef32_sr_pk_fp4_f32( - ctx: *mlir.Context, - oldVdst: *const mlir.Value, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - dstSelIndex: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dstSelIndex", dstSelIndex)); - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk.fp4.f32", .{ - .operands = .{ .flat = &.{ oldVdst, src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk16.bf6.bf16`. Scale and convert packed bf16 to packed bf6 with stochastic rounding -pub fn cvt_scalef32_sr_pk16_bf6_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.bf16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk16.bf6.f16`. Scale and convert packed f16 to packed bf6 with stochastic rounding -pub fn cvt_scalef32_sr_pk16_bf6_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.f16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk16.bf6.f32`. Scale and convert packed f32 to packed bf6 with stochastic rounding -pub fn cvt_scalef32_sr_pk16_bf6_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.bf6.f32", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk16.fp6.bf16`. Scale and convert packed bf16 to packed fp6 with stochastic rounding -pub fn cvt_scalef32_sr_pk16_fp6_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.bf16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk16.fp6.f16`. Scale and convert packed f16 to packed fp6 with stochastic rounding -pub fn cvt_scalef32_sr_pk16_fp6_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.f16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk16.fp6.f32`. Scale and convert packed f32 to packed fp6 with stochastic rounding -pub fn cvt_scalef32_sr_pk16_fp6_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk16.fp6.f32", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk32.bf6.bf16`. Scale and convert packed bf16 to packed bf6 with stochiastic rounding -pub fn cvt_scalef32_sr_pk32_bf6_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.bf16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk32.bf6.f16`. Scale and convert packed f16 to packed bf6 with stochiastic rounding -pub fn cvt_scalef32_sr_pk32_bf6_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.f16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk32.bf6.f32`. Scale and convert packed f32 to packed bf6 with stochiastic rounding -pub fn cvt_scalef32_sr_pk32_bf6_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.bf6.f32", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk32.fp6.bf16`. Scale and convert packed bf16 to packed fp6 with stochiastic rounding -pub fn cvt_scalef32_sr_pk32_fp6_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.bf16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk32.fp6.f16`. Scale and convert packed f16 to packed fp6 with stochiastic rounding -pub fn cvt_scalef32_sr_pk32_fp6_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.f16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk32.fp6.f32`. Scale and convert packed f32 to packed fp6 with stochiastic rounding -pub fn cvt_scalef32_sr_pk32_fp6_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk32.fp6.f32", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk8.bf8.bf16`. Scale and convert packed bf16 to packed bf8 with stochastic rounding -pub fn cvt_scalef32_sr_pk8_bf8_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.bf16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk8.bf8.f16`. Scale and convert packed f16 to packed bf8 with stochastic rounding -pub fn cvt_scalef32_sr_pk8_bf8_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.f16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk8.bf8.f32`. Scale and convert packed f32 to packed bf8 with stochastic rounding -pub fn cvt_scalef32_sr_pk8_bf8_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.bf8.f32", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk8.fp4.bf16`. Scale and convert packed bf16 to packed fp4 with stochastic rounding -pub fn cvt_scalef32_sr_pk8_fp4_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.bf16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk8.fp4.f16`. Scale and convert packed f16 to packed fp4 with stochastic rounding -pub fn cvt_scalef32_sr_pk8_fp4_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.f16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk8.fp4.f32`. Scale and convert packed f32 to packed fp4 with stochastic rounding -pub fn cvt_scalef32_sr_pk8_fp4_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp4.f32", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk8.fp8.bf16`. Scale and convert packed bf16 to packed fp8 with stochastic rounding -pub fn cvt_scalef32_sr_pk8_fp8_bf16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.bf16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk8.fp8.f16`. Scale and convert packed f16 to packed fp8 with stochastic rounding -pub fn cvt_scalef32_sr_pk8_fp8_f16( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.f16", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.scalef32.sr.pk8.fp8.f32`. Scale and convert packed f32 to packed fp8 with stochastic rounding -pub fn cvt_scalef32_sr_pk8_fp8_f32( - ctx: *mlir.Context, - src: *const mlir.Value, - seed: *const mlir.Value, - scale: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.cvt.scalef32.sr.pk8.fp8.f32", .{ - .operands = .{ .flat = &.{ src, seed, scale } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.cvt.sr.bf8.f32`. Convert f32 to bf8, stochiastic rounding -pub fn cvt_sr_bf8_f32( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - old: *const mlir.Value, - res_type: *const mlir.Type, - byteSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.sr.bf8.f32", .{ - .operands = .{ .flat = &.{ srcA, srcB, old } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.cvt.sr.fp8.f32`. Convert f32 to fp8, stochiastic rounding -pub fn cvt_sr_fp8_f32( - ctx: *mlir.Context, - srcA: *const mlir.Value, - srcB: *const mlir.Value, - old: *const mlir.Value, - res_type: *const mlir.Type, - byteSel: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "byteSel", byteSel)); - return mlir.Operation.make(ctx, "rocdl.cvt.sr.fp8.f32", .{ - .operands = .{ .flat = &.{ srcA, srcB, old } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.dot4.f32.bf8.bf8`. -pub fn dot4_f32_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.dot4.f32.bf8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.dot4.f32.bf8.fp8`. -pub fn dot4_f32_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.dot4.f32.bf8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.dot4.f32.fp8.bf8`. -pub fn dot4_f32_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.dot4.f32.fp8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.dot4.f32.fp8.fp8`. -pub fn dot4_f32_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.dot4.f32.fp8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.ds.atomic.async.barrier.arrive.b64`. -pub fn ds_atomic_async_barrier_arrive_b64( - ctx: *mlir.Context, - barrierPtr: *const mlir.Value, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.ds.atomic.async.barrier.arrive.b64", .{ - .operands = .{ .flat = &.{barrierPtr} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.ds.atomic.barrier.arrive.rtn.b64`. -pub fn ds_atomic_barrier_arrive_rtn_b64( - ctx: *mlir.Context, - barrierPtr: *const mlir.Value, - val: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.ds.atomic.barrier.arrive.rtn.b64", .{ - .operands = .{ .flat = &.{ barrierPtr, val } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.ds.load.tr16.b128`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). -pub fn ds_load_tr16_b128( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.ds.load.tr16.b128", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.ds.load.tr4.b64`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). -pub fn ds_load_tr4_b64( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.ds.load.tr4.b64", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.ds.load.tr6.b96`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). -pub fn ds_load_tr6_b96( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.ds.load.tr6.b96", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.ds.load.tr8.b64`. Loads and transposes a matrix from ds memory to registers (available in gfx1250+). -pub fn ds_load_tr8_b64( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.ds.load.tr8.b64", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.ds.read.tr16.b64`. -pub fn ds_read_tr16_b64( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.ds.read.tr16.b64", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.ds.read.tr4.b64`. -pub fn ds_read_tr4_b64( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.ds.read.tr4.b64", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.ds.read.tr6.b96`. -pub fn ds_read_tr6_b96( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.ds.read.tr6.b96", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.ds.read.tr8.b64`. -pub fn ds_read_tr8_b64( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.ds.read.tr8.b64", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.ds_bpermute`. -pub fn ds_bpermute( - ctx: *mlir.Context, - index: *const mlir.Value, - src: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.ds_bpermute", .{ - .operands = .{ .flat = &.{ index, src } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.ds_swizzle`. -pub fn ds_swizzle( - ctx: *mlir.Context, - src: *const mlir.Value, - offset: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.ds_swizzle", .{ - .operands = .{ .flat = &.{ src, offset } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.exp`. -pub fn exp( - ctx: *mlir.Context, - arg: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.exp", .{ - .operands = .{ .flat = &.{arg} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.exp2`. -pub fn exp2( - ctx: *mlir.Context, - arg: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.exp2", .{ - .operands = .{ .flat = &.{arg} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.fdot2`. -pub fn fdot2( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.fdot2", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.fdot2.bf16.bf16`. -pub fn fdot2_bf16_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.fdot2.bf16.bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.fdot2.f16.f16`. -pub fn fdot2_f16_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.fdot2.f16.f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.fdot2.f32.bf16`. -pub fn fdot2_f32_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.fdot2.f32.bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.flat.prefetch`. -pub fn flat_prefetch( - ctx: *mlir.Context, - ptr: *const mlir.Value, - opts: struct { - cachePolicy: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.cachePolicy) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.flat.prefetch", .{ - .operands = .{ .flat = &.{ptr} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.fmed3`. Median of three float/half values -pub fn fmed3( - ctx: *mlir.Context, - src0: *const mlir.Value, - src1: *const mlir.Value, - src2: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.fmed3", .{ - .operands = .{ .flat = &.{ src0, src1, src2 } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.global.load.async.lds`. Version of rocdl.load.async.to.lds specialized to global pointers -pub fn global_load_async_lds( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - size: *const mlir.Attribute, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "size", size)); - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.load.async.lds", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.load.async.to.lds.b128`. -pub fn global_load_async_to_lds_b128( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b128", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.load.async.to.lds.b32`. -pub fn global_load_async_to_lds_b32( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b32", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.load.async.to.lds.b64`. -pub fn global_load_async_to_lds_b64( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b64", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.load.async.to.lds.b8`. -pub fn global_load_async_to_lds_b8( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.load.async.to.lds.b8", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.load.lds`. -pub fn global_load_lds( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - size: *const mlir.Attribute, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "size", size)); - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.load.lds", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.load.tr.b128`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). -pub fn global_load_tr_b128( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.load.tr.b128", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.load.tr.b64`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). -pub fn global_load_tr_b64( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.load.tr.b64", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.load.tr4.b64`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). -pub fn global_load_tr4_b64( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.load.tr4.b64", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.load.tr6.b96`. Loads and transposes a matrix from global memory to registers (available in gfx1250+). -pub fn global_load_tr6_b96( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.load.tr6.b96", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.prefetch`. -pub fn global_prefetch( - ctx: *mlir.Context, - ptr: *const mlir.Value, - opts: struct { - cachePolicy: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.cachePolicy) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.prefetch", .{ - .operands = .{ .flat = &.{ptr} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.store.async.from.lds.b128`. -pub fn global_store_async_from_lds_b128( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b128", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.store.async.from.lds.b32`. -pub fn global_store_async_from_lds_b32( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b32", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.store.async.from.lds.b64`. -pub fn global_store_async_from_lds_b64( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b64", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.global.store.async.from.lds.b8`. -pub fn global_store_async_from_lds_b8( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.global.store.async.from.lds.b8", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.iglp.opt`. -pub fn iglp_opt( - ctx: *mlir.Context, - variant: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "variant", variant)); - return mlir.Operation.make(ctx, "rocdl.iglp.opt", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.load.async.to.lds`. Gathering load to LDS that requires explicit async memory tracking -pub fn load_async_to_lds( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - size: *const mlir.Attribute, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "size", size)); - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.load.async.to.lds", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.load.to.lds`. -pub fn load_to_lds( - ctx: *mlir.Context, - globalPtr: *const mlir.Value, - ldsPtr: *const mlir.Value, - size: *const mlir.Attribute, - offset: *const mlir.Attribute, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 6) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "size", size)); - attributes.appendAssumeCapacity(.named(ctx, "offset", offset)); - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.load.to.lds", .{ - .operands = .{ .flat = &.{ globalPtr, ldsPtr } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.log`. -pub fn log( - ctx: *mlir.Context, - arg: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.log", .{ - .operands = .{ .flat = &.{arg} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.make.buffer.rsrc`. -pub fn make_buffer_rsrc( - ctx: *mlir.Context, - base: *const mlir.Value, - stride: *const mlir.Value, - numRecords: *const mlir.Value, - flags: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.make.buffer.rsrc", .{ - .operands = .{ .flat = &.{ base, stride, numRecords, flags } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.mbcnt.hi`. -pub fn mbcnt_hi( - ctx: *mlir.Context, - in0: *const mlir.Value, - in1: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - arg_attrs: ?*const mlir.Attribute = null, - res_attrs: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.arg_attrs) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", attribute)); - } - if (opts.res_attrs) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "res_attrs", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.mbcnt.hi", .{ - .operands = .{ .flat = &.{ in0, in1 } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mbcnt.lo`. -pub fn mbcnt_lo( - ctx: *mlir.Context, - in0: *const mlir.Value, - in1: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - arg_attrs: ?*const mlir.Attribute = null, - res_attrs: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.arg_attrs) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "arg_attrs", attribute)); - } - if (opts.res_attrs) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "res_attrs", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.mbcnt.lo", .{ - .operands = .{ .flat = &.{ in0, in1 } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x16bf16.1k`. -pub fn mfma_f32_16x16x16bf16_1k( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x16bf16.1k", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x16f16`. -pub fn mfma_f32_16x16x16f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x16f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x1f32`. -pub fn mfma_f32_16x16x1f32( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x1f32", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x2bf16`. -pub fn mfma_f32_16x16x2bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x2bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x32.bf16`. -pub fn mfma_f32_16x16x32_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x32.bf8.bf8`. -pub fn mfma_f32_16x16x32_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x32.bf8.fp8`. -pub fn mfma_f32_16x16x32_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.bf8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x32.f16`. -pub fn mfma_f32_16x16x32_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x32.fp8.bf8`. -pub fn mfma_f32_16x16x32_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.fp8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x32.fp8.fp8`. -pub fn mfma_f32_16x16x32_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x32.fp8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x4bf16.1k`. -pub fn mfma_f32_16x16x4bf16_1k( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4bf16.1k", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x4f16`. -pub fn mfma_f32_16x16x4f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x4f32`. -pub fn mfma_f32_16x16x4f32( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x4f32", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x8.xf32`. -pub fn mfma_f32_16x16x8_xf32( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x8.xf32", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.16x16x8bf16`. -pub fn mfma_f32_16x16x8bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.16x16x8bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x16.bf16`. -pub fn mfma_f32_32x32x16_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x16.bf8.bf8`. -pub fn mfma_f32_32x32x16_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x16.bf8.fp8`. -pub fn mfma_f32_32x32x16_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.bf8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x16.f16`. -pub fn mfma_f32_32x32x16_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x16.fp8.bf8`. -pub fn mfma_f32_32x32x16_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.fp8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x16.fp8.fp8`. -pub fn mfma_f32_32x32x16_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x16.fp8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x1f32`. -pub fn mfma_f32_32x32x1f32( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x1f32", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x2bf16`. -pub fn mfma_f32_32x32x2bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x2bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x2f32`. -pub fn mfma_f32_32x32x2f32( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x2f32", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x4.xf32`. -pub fn mfma_f32_32x32x4_xf32( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4.xf32", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x4bf16`. -pub fn mfma_f32_32x32x4bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x4bf16.1k`. -pub fn mfma_f32_32x32x4bf16_1k( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4bf16.1k", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x4f16`. -pub fn mfma_f32_32x32x4f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x4f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x8bf16.1k`. -pub fn mfma_f32_32x32x8bf16_1k( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x8bf16.1k", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.32x32x8f16`. -pub fn mfma_f32_32x32x8f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.32x32x8f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.4x4x1f32`. -pub fn mfma_f32_4x4x1f32( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x1f32", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.4x4x2bf16`. -pub fn mfma_f32_4x4x2bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x2bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.4x4x4bf16.1k`. -pub fn mfma_f32_4x4x4bf16_1k( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x4bf16.1k", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f32.4x4x4f16`. -pub fn mfma_f32_4x4x4f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f32.4x4x4f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f64.16x16x4f64`. -pub fn mfma_f64_16x16x4f64( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f64.16x16x4f64", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.f64.4x4x4f64`. -pub fn mfma_f64_4x4x4f64( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.f64.4x4x4f64", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.i32.16x16x16i8`. -pub fn mfma_i32_16x16x16i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x16i8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.i32.16x16x32.i8`. -pub fn mfma_i32_16x16x32_i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x32.i8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.i32.16x16x4i8`. -pub fn mfma_i32_16x16x4i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x4i8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.i32.16x16x64.i8`. -pub fn mfma_i32_16x16x64_i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.i32.16x16x64.i8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.i32.32x32x16.i8`. -pub fn mfma_i32_32x32x16_i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x16.i8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.i32.32x32x32.i8`. -pub fn mfma_i32_32x32x32_i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x32.i8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.i32.32x32x4i8`. -pub fn mfma_i32_32x32x4i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x4i8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.i32.32x32x8i8`. -pub fn mfma_i32_32x32x8i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.i32.32x32x8i8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.i32.4x4x4i8`. -pub fn mfma_i32_4x4x4i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - blgp: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - return mlir.Operation.make(ctx, "rocdl.mfma.i32.4x4x4i8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.scale.f32.16x16x128.f8f6f4`. -pub fn mfma_scale_f32_16x16x128_f8f6f4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - scaleA: *const mlir.Value, - scaleB: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - blgp: *const mlir.Attribute, - opselA: *const mlir.Attribute, - opselB: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - attributes.appendAssumeCapacity(.named(ctx, "opselA", opselA)); - attributes.appendAssumeCapacity(.named(ctx, "opselB", opselB)); - return mlir.Operation.make(ctx, "rocdl.mfma.scale.f32.16x16x128.f8f6f4", .{ - .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.mfma.scale.f32.32x32x64.f8f6f4`. -pub fn mfma_scale_f32_32x32x64_f8f6f4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - scaleA: *const mlir.Value, - scaleB: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - blgp: *const mlir.Attribute, - opselA: *const mlir.Attribute, - opselB: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "blgp", blgp)); - attributes.appendAssumeCapacity(.named(ctx, "opselA", opselA)); - attributes.appendAssumeCapacity(.named(ctx, "opselB", opselB)); - return mlir.Operation.make(ctx, "rocdl.mfma.scale.f32.32x32x64.f8f6f4", .{ - .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.permlane16.swap`. -pub fn permlane16_swap( - ctx: *mlir.Context, - old: *const mlir.Value, - src: *const mlir.Value, - res_type: *const mlir.Type, - fi: *const mlir.Attribute, - boundControl: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); - attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); - return mlir.Operation.make(ctx, "rocdl.permlane16.swap", .{ - .operands = .{ .flat = &.{ old, src } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.permlane16.var`. -pub fn permlane16_var( - ctx: *mlir.Context, - old: *const mlir.Value, - src0: *const mlir.Value, - src1: *const mlir.Value, - res_type: *const mlir.Type, - fi: *const mlir.Attribute, - boundControl: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); - attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); - return mlir.Operation.make(ctx, "rocdl.permlane16.var", .{ - .operands = .{ .flat = &.{ old, src0, src1 } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.permlane32.swap`. -pub fn permlane32_swap( - ctx: *mlir.Context, - old: *const mlir.Value, - src: *const mlir.Value, - res_type: *const mlir.Type, - fi: *const mlir.Attribute, - boundControl: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); - attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); - return mlir.Operation.make(ctx, "rocdl.permlane32.swap", .{ - .operands = .{ .flat = &.{ old, src } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.permlanex16`. -pub fn permlanex16( - ctx: *mlir.Context, - old: *const mlir.Value, - src0: *const mlir.Value, - src1: *const mlir.Value, - src2: *const mlir.Value, - res_type: *const mlir.Type, - fi: *const mlir.Attribute, - boundControl: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); - attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); - return mlir.Operation.make(ctx, "rocdl.permlanex16", .{ - .operands = .{ .flat = &.{ old, src0, src1, src2 } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.permlanex16.var`. -pub fn permlanex16_var( - ctx: *mlir.Context, - old: *const mlir.Value, - src0: *const mlir.Value, - src1: *const mlir.Value, - res_type: *const mlir.Type, - fi: *const mlir.Attribute, - boundControl: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "fi", fi)); - attributes.appendAssumeCapacity(.named(ctx, "boundControl", boundControl)); - return mlir.Operation.make(ctx, "rocdl.permlanex16.var", .{ - .operands = .{ .flat = &.{ old, src0, src1 } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.ptr.s.buffer.load`. -pub fn ptr_s_buffer_load( - ctx: *mlir.Context, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.ptr.s.buffer.load", .{ - .operands = .{ .flat = &.{ rsrc, offset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.buffer.atomic.cmpswap`. -pub fn raw_buffer_atomic_cmpswap( - ctx: *mlir.Context, - src: *const mlir.Value, - cmp: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.cmpswap", .{ - .operands = .{ .flat = &.{ src, cmp, rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.buffer.atomic.fadd`. -pub fn raw_buffer_atomic_fadd( - ctx: *mlir.Context, - vdata: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.fadd", .{ - .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.buffer.atomic.fmax`. -pub fn raw_buffer_atomic_fmax( - ctx: *mlir.Context, - vdata: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.fmax", .{ - .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.buffer.atomic.smax`. -pub fn raw_buffer_atomic_smax( - ctx: *mlir.Context, - vdata: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.smax", .{ - .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.buffer.atomic.umin`. -pub fn raw_buffer_atomic_umin( - ctx: *mlir.Context, - vdata: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.buffer.atomic.umin", .{ - .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.buffer.load`. -pub fn raw_buffer_load( - ctx: *mlir.Context, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.buffer.load", .{ - .operands = .{ .flat = &.{ rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.buffer.store`. -pub fn raw_buffer_store( - ctx: *mlir.Context, - vdata: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - opts: struct { - aux: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.buffer.store", .{ - .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.ptr.buffer.atomic.cmpswap`. -pub fn raw_ptr_buffer_atomic_cmpswap( - ctx: *mlir.Context, - src: *const mlir.Value, - cmp: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.cmpswap", .{ - .operands = .{ .flat = &.{ src, cmp, rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.ptr.buffer.atomic.fadd`. -pub fn raw_ptr_buffer_atomic_fadd( - ctx: *mlir.Context, - vdata: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.fadd", .{ - .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.ptr.buffer.atomic.fmax`. -pub fn raw_ptr_buffer_atomic_fmax( - ctx: *mlir.Context, - vdata: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.fmax", .{ - .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.ptr.buffer.atomic.smax`. -pub fn raw_ptr_buffer_atomic_smax( - ctx: *mlir.Context, - vdata: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.smax", .{ - .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.ptr.buffer.atomic.umin`. -pub fn raw_ptr_buffer_atomic_umin( - ctx: *mlir.Context, - vdata: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.atomic.umin", .{ - .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.ptr.buffer.load`. -pub fn raw_ptr_buffer_load( - ctx: *mlir.Context, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load", .{ - .operands = .{ .flat = &.{ rsrc, offset, soffset } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.ptr.buffer.load.async.lds`. Async variant of raw.ptr.buffer.load.lds -pub fn raw_ptr_buffer_load_async_lds( - ctx: *mlir.Context, - rsrc: *const mlir.Value, - ldsPtr: *const mlir.Value, - size: *const mlir.Value, - voffset: *const mlir.Value, - soffset: *const mlir.Value, - offset: *const mlir.Value, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load.async.lds", .{ - .operands = .{ .flat = &.{ rsrc, ldsPtr, size, voffset, soffset, offset } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.ptr.buffer.load.lds`. -pub fn raw_ptr_buffer_load_lds( - ctx: *mlir.Context, - rsrc: *const mlir.Value, - ldsPtr: *const mlir.Value, - size: *const mlir.Value, - voffset: *const mlir.Value, - soffset: *const mlir.Value, - offset: *const mlir.Value, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.load.lds", .{ - .operands = .{ .flat = &.{ rsrc, ldsPtr, size, voffset, soffset, offset } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.raw.ptr.buffer.store`. -pub fn raw_ptr_buffer_store( - ctx: *mlir.Context, - vdata: *const mlir.Value, - rsrc: *const mlir.Value, - offset: *const mlir.Value, - soffset: *const mlir.Value, - opts: struct { - aux: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.aux) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "aux", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.raw.ptr.buffer.store", .{ - .operands = .{ .flat = &.{ vdata, rsrc, offset, soffset } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.rcp`. -pub fn rcp( - ctx: *mlir.Context, - arg: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.rcp", .{ - .operands = .{ .flat = &.{arg} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.readfirstlane`. Get the value in first active lane. -pub fn readfirstlane( - ctx: *mlir.Context, - src: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.readfirstlane", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.readlane`. Get the value in the specific lane. -pub fn readlane( - ctx: *mlir.Context, - src0: *const mlir.Value, - src1: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.readlane", .{ - .operands = .{ .flat = &.{ src0, src1 } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.rsq`. -pub fn rsq( - ctx: *mlir.Context, - arg: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.rsq", .{ - .operands = .{ .flat = &.{arg} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.s.barrier`. -pub fn s_barrier( - ctx: *mlir.Context, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.s.barrier", .{ - .location = location, - }); -} - -/// `rocdl.s.barrier.init`. -pub fn s_barrier_init( - ctx: *mlir.Context, - ptr: *const mlir.Value, - memberCnt: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "memberCnt", memberCnt)); - return mlir.Operation.make(ctx, "rocdl.s.barrier.init", .{ - .operands = .{ .flat = &.{ptr} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.barrier.join`. -pub fn s_barrier_join( - ctx: *mlir.Context, - ptr: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.s.barrier.join", .{ - .operands = .{ .flat = &.{ptr} }, - .location = location, - }); -} - -/// `rocdl.s.barrier.leave`. -pub fn s_barrier_leave( - ctx: *mlir.Context, - id: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "id", id)); - return mlir.Operation.make(ctx, "rocdl.s.barrier.leave", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.barrier.signal`. -pub fn s_barrier_signal( - ctx: *mlir.Context, - id: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "id", id)); - return mlir.Operation.make(ctx, "rocdl.s.barrier.signal", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.barrier.signal.isfirst`. -pub fn s_barrier_signal_isfirst( - ctx: *mlir.Context, - res_type: *const mlir.Type, - id: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "id", id)); - return mlir.Operation.make(ctx, "rocdl.s.barrier.signal.isfirst", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.barrier.signal.var`. -pub fn s_barrier_signal_var( - ctx: *mlir.Context, - ptr: *const mlir.Value, - memberCnt: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "memberCnt", memberCnt)); - return mlir.Operation.make(ctx, "rocdl.s.barrier.signal.var", .{ - .operands = .{ .flat = &.{ptr} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.barrier.wait`. -pub fn s_barrier_wait( - ctx: *mlir.Context, - id: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "id", id)); - return mlir.Operation.make(ctx, "rocdl.s.barrier.wait", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.get.barrier.state`. -pub fn s_get_barrier_state( - ctx: *mlir.Context, - res_type: *const mlir.Type, - id: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "id", id)); - return mlir.Operation.make(ctx, "rocdl.s.get.barrier.state", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.get.named.barrier.state`. -pub fn s_get_named_barrier_state( - ctx: *mlir.Context, - ptr: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.s.get.named.barrier.state", .{ - .operands = .{ .flat = &.{ptr} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.s.nop`. -pub fn s_nop( - ctx: *mlir.Context, - count: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "count", count)); - return mlir.Operation.make(ctx, "rocdl.s.nop", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.setprio`. -pub fn s_setprio( - ctx: *mlir.Context, - priority: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "priority", priority)); - return mlir.Operation.make(ctx, "rocdl.s.setprio", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.sleep`. -pub fn s_sleep( - ctx: *mlir.Context, - count: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "count", count)); - return mlir.Operation.make(ctx, "rocdl.s.sleep", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.wait.asynccnt`. Wait until ASYNCCNT is less than or equal to `count` -pub fn s_wait_asynccnt( - ctx: *mlir.Context, - count: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "count", count)); - return mlir.Operation.make(ctx, "rocdl.s.wait.asynccnt", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.wait.dscnt`. Wait until DSCNT is less than or equal to `count` -pub fn s_wait_dscnt( - ctx: *mlir.Context, - count: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "count", count)); - return mlir.Operation.make(ctx, "rocdl.s.wait.dscnt", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.wait.expcnt`. Wait until EXPCNT is less than or equal to `count` -pub fn s_wait_expcnt( - ctx: *mlir.Context, - count: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "count", count)); - return mlir.Operation.make(ctx, "rocdl.s.wait.expcnt", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.wait.loadcnt`. Wait until LOADCNT is less than or equal to `count` -pub fn s_wait_loadcnt( - ctx: *mlir.Context, - count: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "count", count)); - return mlir.Operation.make(ctx, "rocdl.s.wait.loadcnt", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.wait.storecnt`. Wait until STORECNT is less than or equal to `count` -pub fn s_wait_storecnt( - ctx: *mlir.Context, - count: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "count", count)); - return mlir.Operation.make(ctx, "rocdl.s.wait.storecnt", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.wait.tensorcnt`. Wait until TENSORCNT is less than or equal to `count` -pub fn s_wait_tensorcnt( - ctx: *mlir.Context, - count: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "count", count)); - return mlir.Operation.make(ctx, "rocdl.s.wait.tensorcnt", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.waitcnt`. -pub fn s_waitcnt( - ctx: *mlir.Context, - bitfield: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "bitfield", bitfield)); - return mlir.Operation.make(ctx, "rocdl.s.waitcnt", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.s.wakeup.barrier`. -pub fn s_wakeup_barrier( - ctx: *mlir.Context, - ptr: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.s.wakeup.barrier", .{ - .operands = .{ .flat = &.{ptr} }, - .location = location, - }); -} - -/// `rocdl.sched.barrier`. -pub fn sched_barrier( - ctx: *mlir.Context, - mask: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "mask", mask)); - return mlir.Operation.make(ctx, "rocdl.sched.barrier", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.sched.group.barrier`. -pub fn sched_group_barrier( - ctx: *mlir.Context, - mask: *const mlir.Attribute, - size: *const mlir.Attribute, - groupId: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "mask", mask)); - attributes.appendAssumeCapacity(.named(ctx, "size", size)); - attributes.appendAssumeCapacity(.named(ctx, "groupId", groupId)); - return mlir.Operation.make(ctx, "rocdl.sched.group.barrier", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.sdot2`. -pub fn sdot2( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.sdot2", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.sdot4`. -pub fn sdot4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.sdot4", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.sdot8`. -pub fn sdot8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.sdot8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.sin`. -pub fn sin( - ctx: *mlir.Context, - arg: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.sin", .{ - .operands = .{ .flat = &.{arg} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x128.bf8.bf8`. -pub fn smfmac_f32_16x16x128_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.bf8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x128.bf8.fp8`. -pub fn smfmac_f32_16x16x128_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.bf8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x128.fp8.bf8`. -pub fn smfmac_f32_16x16x128_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.fp8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x128.fp8.fp8`. -pub fn smfmac_f32_16x16x128_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x128.fp8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x32.bf16`. -pub fn smfmac_f32_16x16x32_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x32.bf16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x32.f16`. -pub fn smfmac_f32_16x16x32_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x32.f16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x64.bf16`. -pub fn smfmac_f32_16x16x64_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x64.bf8.bf8`. -pub fn smfmac_f32_16x16x64_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x64.bf8.fp8`. -pub fn smfmac_f32_16x16x64_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.bf8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x64.f16`. -pub fn smfmac_f32_16x16x64_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.f16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x64.fp8.bf8`. -pub fn smfmac_f32_16x16x64_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.fp8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.16x16x64.fp8.fp8`. -pub fn smfmac_f32_16x16x64_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.16x16x64.fp8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x16.bf16`. -pub fn smfmac_f32_32x32x16_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x16.bf16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x16.f16`. -pub fn smfmac_f32_32x32x16_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x16.f16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x32.bf16`. -pub fn smfmac_f32_32x32x32_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x32.bf8.bf8`. -pub fn smfmac_f32_32x32x32_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x32.bf8.fp8`. -pub fn smfmac_f32_32x32x32_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.bf8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x32.f16`. -pub fn smfmac_f32_32x32x32_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.f16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x32.fp8.bf8`. -pub fn smfmac_f32_32x32x32_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.fp8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x32.fp8.fp8`. -pub fn smfmac_f32_32x32x32_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x32.fp8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x64.bf8.bf8`. -pub fn smfmac_f32_32x32x64_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.bf8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x64.bf8.fp8`. -pub fn smfmac_f32_32x32x64_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.bf8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x64.fp8.bf8`. -pub fn smfmac_f32_32x32x64_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.fp8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.f32.32x32x64.fp8.fp8`. -pub fn smfmac_f32_32x32x64_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.f32.32x32x64.fp8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.i32.16x16x128.i8`. -pub fn smfmac_i32_16x16x128_i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.i32.16x16x128.i8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.i32.16x16x64.i8`. -pub fn smfmac_i32_16x16x64_i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.i32.16x16x64.i8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.i32.32x32x32.i8`. -pub fn smfmac_i32_32x32x32_i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.i32.32x32x32.i8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.smfmac.i32.32x32x64.i8`. -pub fn smfmac_i32_32x32x64_i8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - cbsz: *const mlir.Attribute, - abid: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "cbsz", cbsz)); - attributes.appendAssumeCapacity(.named(ctx, "abid", abid)); - return mlir.Operation.make(ctx, "rocdl.smfmac.i32.32x32x64.i8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.sqrt`. -pub fn sqrt( - ctx: *mlir.Context, - arg: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.sqrt", .{ - .operands = .{ .flat = &.{arg} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.sudot4`. -pub fn sudot4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.sudot4", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.sudot8`. -pub fn sudot8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.sudot8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.bf16.16x16x32.bf16`. -pub fn swmmac_bf16_16x16x32_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.swmmac.bf16.16x16x32.bf16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.swmmac.bf16.16x16x64.bf16`. -pub fn swmmac_bf16_16x16x64_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.bf16.16x16x64.bf16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.bf16f32.16x16x64.bf16`. -pub fn swmmac_bf16f32_16x16x64_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.bf16f32.16x16x64.bf16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.f16.16x16x128.bf8.bf8`. -pub fn swmmac_f16_16x16x128_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.bf8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.f16.16x16x128.bf8.fp8`. -pub fn swmmac_f16_16x16x128_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.bf8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.f16.16x16x128.fp8.bf8`. -pub fn swmmac_f16_16x16x128_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.fp8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.f16.16x16x128.fp8.fp8`. -pub fn swmmac_f16_16x16x128_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x128.fp8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.f16.16x16x32.f16`. -pub fn swmmac_f16_16x16x32_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x32.f16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.swmmac.f16.16x16x64.f16`. -pub fn swmmac_f16_16x16x64_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.f16.16x16x64.f16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x128.bf8.bf8`. -pub fn swmmac_f32_16x16x128_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.bf8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x128.bf8.fp8`. -pub fn swmmac_f32_16x16x128_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.bf8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x128.fp8.bf8`. -pub fn swmmac_f32_16x16x128_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.fp8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x128.fp8.fp8`. -pub fn swmmac_f32_16x16x128_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x128.fp8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x32.bf16`. -pub fn swmmac_f32_16x16x32_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x32.bf8.bf8`. -pub fn swmmac_f32_16x16x32_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x32.bf8.fp8`. -pub fn swmmac_f32_16x16x32_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.bf8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x32.f16`. -pub fn swmmac_f32_16x16x32_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.f16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x32.fp8.bf8`. -pub fn swmmac_f32_16x16x32_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.fp8.bf8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x32.fp8.fp8`. -pub fn swmmac_f32_16x16x32_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x32.fp8.fp8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x64.bf16`. -pub fn swmmac_f32_16x16x64_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x64.bf16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.f32.16x16x64.f16`. -pub fn swmmac_f32_16x16x64_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.f32.16x16x64.f16", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.i32.16x16x128.iu8`. -pub fn swmmac_i32_16x16x128_iu8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x128.iu8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.i32.16x16x32.iu4`. -pub fn swmmac_i32_16x16x32_iu4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x32.iu4", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.i32.16x16x32.iu8`. -pub fn swmmac_i32_16x16x32_iu8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x32.iu8", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.swmmac.i32.16x16x64.iu4`. -pub fn swmmac_i32_16x16x64_iu4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - index: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.swmmac.i32.16x16x64.iu4", .{ - .operands = .{ .flat = &.{ a, b, c, index } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.tanh`. -pub fn tanh( - ctx: *mlir.Context, - arg: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.tanh", .{ - .operands = .{ .flat = &.{arg} }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.tensor.load.to.lds`. Base class for ROCDL tensor load/store to/from LDS. -pub fn tensor_load_to_lds( - ctx: *mlir.Context, - dgroup0: *const mlir.Value, - dgroup1: *const mlir.Value, - dgroup2: *const mlir.Value, - dgroup3: *const mlir.Value, - dgroup4: *const mlir.Value, - opts: struct { - cachePolicy: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.cachePolicy) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.tensor.load.to.lds", .{ - .operands = .{ .flat = &.{ dgroup0, dgroup1, dgroup2, dgroup3, dgroup4 } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.tensor.store.from.lds`. Base class for ROCDL tensor load/store to/from LDS. -pub fn tensor_store_from_lds( - ctx: *mlir.Context, - dgroup0: *const mlir.Value, - dgroup1: *const mlir.Value, - dgroup2: *const mlir.Value, - dgroup3: *const mlir.Value, - dgroup4: *const mlir.Value, - opts: struct { - cachePolicy: ?*const mlir.Attribute = null, - alias_scopes: ?*const mlir.Attribute = null, - noalias_scopes: ?*const mlir.Attribute = null, - tbaa: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - if (opts.cachePolicy) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "cachePolicy", attribute)); - } - if (opts.alias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "alias_scopes", attribute)); - } - if (opts.noalias_scopes) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "noalias_scopes", attribute)); - } - if (opts.tbaa) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "tbaa", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.tensor.store.from.lds", .{ - .operands = .{ .flat = &.{ dgroup0, dgroup1, dgroup2, dgroup3, dgroup4 } }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.udot2`. -pub fn udot2( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.udot2", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.udot4`. -pub fn udot4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.udot4", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.udot8`. -pub fn udot8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.udot8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.update.dpp`. -pub fn update_dpp( - ctx: *mlir.Context, - old: *const mlir.Value, - src: *const mlir.Value, - res_type: *const mlir.Type, - dppCtrl: *const mlir.Attribute, - rowMask: *const mlir.Attribute, - bankMask: *const mlir.Attribute, - boundCtrl: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 4) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "dppCtrl", dppCtrl)); - attributes.appendAssumeCapacity(.named(ctx, "rowMask", rowMask)); - attributes.appendAssumeCapacity(.named(ctx, "bankMask", bankMask)); - attributes.appendAssumeCapacity(.named(ctx, "boundCtrl", boundCtrl)); - return mlir.Operation.make(ctx, "rocdl.update.dpp", .{ - .operands = .{ .flat = &.{ old, src } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wait.asyncmark`. Wait until N or fewer async operation groups are unexecuted -pub fn wait_asyncmark( - ctx: *mlir.Context, - count: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - attributes.appendAssumeCapacity(.named(ctx, "count", count)); - return mlir.Operation.make(ctx, "rocdl.wait.asyncmark", .{ - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wave.barrier`. -pub fn wave_barrier( - ctx: *mlir.Context, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.wave.barrier", .{ - .location = location, - }); -} - -/// `rocdl.wave.id`. -pub fn wave_id( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wave.id", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wavefrontsize`. -pub fn wavefrontsize( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wavefrontsize", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.bf16.16x16x16.bf16`. -pub fn wmma_bf16_16x16x16_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - opsel: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.opsel) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "opsel", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.bf16.16x16x16.bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.bf16.16x16x32.bf16`. -pub fn wmma_bf16_16x16x32_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.bf16.16x16x32.bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.bf16f32.16x16x32.bf16`. -pub fn wmma_bf16f32_16x16x32_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.bf16f32.16x16x32.bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f16.16x16x128.bf8_bf8`. -pub fn wmma_f16_16x16x128_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.bf8_bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f16.16x16x128.bf8_fp8`. -pub fn wmma_f16_16x16x128_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.bf8_fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f16.16x16x128.fp8_bf8`. -pub fn wmma_f16_16x16x128_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.fp8_bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f16.16x16x128.fp8_fp8`. -pub fn wmma_f16_16x16x128_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x128.fp8_fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f16.16x16x16.f16`. -pub fn wmma_f16_16x16x16_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - opsel: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.opsel) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "opsel", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x16.f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f16.16x16x32.f16`. -pub fn wmma_f16_16x16x32_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x32.f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f16.16x16x64.bf8_bf8`. -pub fn wmma_f16_16x16x64_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.bf8_bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f16.16x16x64.bf8_fp8`. -pub fn wmma_f16_16x16x64_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.bf8_fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f16.16x16x64.fp8_bf8`. -pub fn wmma_f16_16x16x64_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.fp8_bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f16.16x16x64.fp8_fp8`. -pub fn wmma_f16_16x16x64_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f16.16x16x64.fp8_fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x128.bf8_bf8`. -pub fn wmma_f32_16x16x128_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.bf8_bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x128.bf8_fp8`. -pub fn wmma_f32_16x16x128_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.bf8_fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x128.fp8_bf8`. -pub fn wmma_f32_16x16x128_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.fp8_bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x128.fp8_fp8`. -pub fn wmma_f32_16x16x128_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x128.fp8_fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x16.bf16`. -pub fn wmma_f32_16x16x16_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x16.bf8_bf8`. -pub fn wmma_f32_16x16x16_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf8_bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x16.bf8_fp8`. -pub fn wmma_f32_16x16x16_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.bf8_fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x16.f16`. -pub fn wmma_f32_16x16x16_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x16.fp8_bf8`. -pub fn wmma_f32_16x16x16_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.fp8_bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x16.fp8_fp8`. -pub fn wmma_f32_16x16x16_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x16.fp8_fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x32.bf16`. -pub fn wmma_f32_16x16x32_bf16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x32.bf16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x32.f16`. -pub fn wmma_f32_16x16x32_f16( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x32.f16", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x4.f32`. -pub fn wmma_f32_16x16x4_f32( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x4.f32", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x64.bf8_bf8`. -pub fn wmma_f32_16x16x64_bf8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.bf8_bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x64.bf8_fp8`. -pub fn wmma_f32_16x16x64_bf8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.bf8_fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x64.fp8_bf8`. -pub fn wmma_f32_16x16x64_fp8_bf8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.fp8_bf8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.f32.16x16x64.fp8_fp8`. -pub fn wmma_f32_16x16x64_fp8_fp8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.f32.16x16x64.fp8_fp8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.i32.16x16x16.iu4`. -pub fn wmma_i32_16x16x16_iu4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x16.iu4", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.i32.16x16x16.iu8`. -pub fn wmma_i32_16x16x16_iu8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x16.iu8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.i32.16x16x32.iu4`. -pub fn wmma_i32_16x16x32_iu4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x32.iu4", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.i32.16x16x64.iu8`. -pub fn wmma_i32_16x16x64_iu8( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - signA: ?*const mlir.Attribute = null, - signB: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - clamp: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - if (opts.signA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signA", attribute)); - } - if (opts.signB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "signB", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - if (opts.clamp) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "clamp", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.i32.16x16x64.iu8", .{ - .operands = .{ .flat = &.{ a, b, c } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.scale.f32.16x16x128.f8f6f4`. -pub fn wmma_scale_f32_16x16x128_f8f6f4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - scaleA: *const mlir.Value, - scaleB: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - fmtA: ?*const mlir.Attribute = null, - fmtB: ?*const mlir.Attribute = null, - modC: ?*const mlir.Attribute = null, - scaleAType: ?*const mlir.Attribute = null, - fmtScaleA: ?*const mlir.Attribute = null, - scaleBType: ?*const mlir.Attribute = null, - fmtScaleB: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; - if (opts.fmtA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtA", attribute)); - } - if (opts.fmtB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtB", attribute)); - } - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.scaleAType) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); - } - if (opts.fmtScaleA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); - } - if (opts.scaleBType) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); - } - if (opts.fmtScaleB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.scale.f32.16x16x128.f8f6f4", .{ - .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.scale.f32.32x16x128.f4`. -pub fn wmma_scale_f32_32x16x128_f4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - scaleA: *const mlir.Value, - scaleB: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - scaleAType: ?*const mlir.Attribute = null, - fmtScaleA: ?*const mlir.Attribute = null, - scaleBType: ?*const mlir.Attribute = null, - fmtScaleB: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.scaleAType) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); - } - if (opts.fmtScaleA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); - } - if (opts.scaleBType) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); - } - if (opts.fmtScaleB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.scale.f32.32x16x128.f4", .{ - .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.scale16.f32.16x16x128.f8f6f4`. -pub fn wmma_scale16_f32_16x16x128_f8f6f4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - scaleA: *const mlir.Value, - scaleB: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - fmtA: ?*const mlir.Attribute = null, - fmtB: ?*const mlir.Attribute = null, - modC: ?*const mlir.Attribute = null, - scaleAType: ?*const mlir.Attribute = null, - fmtScaleA: ?*const mlir.Attribute = null, - scaleBType: ?*const mlir.Attribute = null, - fmtScaleB: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 9) = .empty; - if (opts.fmtA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtA", attribute)); - } - if (opts.fmtB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtB", attribute)); - } - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.scaleAType) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); - } - if (opts.fmtScaleA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); - } - if (opts.scaleBType) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); - } - if (opts.fmtScaleB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.scale16.f32.16x16x128.f8f6f4", .{ - .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.wmma.scale16.f32.32x16x128.f4`. -pub fn wmma_scale16_f32_32x16x128_f4( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c: *const mlir.Value, - scaleA: *const mlir.Value, - scaleB: *const mlir.Value, - res_type: *const mlir.Type, - opts: struct { - modC: ?*const mlir.Attribute = null, - scaleAType: ?*const mlir.Attribute = null, - fmtScaleA: ?*const mlir.Attribute = null, - scaleBType: ?*const mlir.Attribute = null, - fmtScaleB: ?*const mlir.Attribute = null, - reuseA: ?*const mlir.Attribute = null, - reuseB: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 7) = .empty; - if (opts.modC) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "modC", attribute)); - } - if (opts.scaleAType) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "scaleAType", attribute)); - } - if (opts.fmtScaleA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtScaleA", attribute)); - } - if (opts.scaleBType) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "scaleBType", attribute)); - } - if (opts.fmtScaleB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "fmtScaleB", attribute)); - } - if (opts.reuseA) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseA", attribute)); - } - if (opts.reuseB) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "reuseB", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.wmma.scale16.f32.32x16x128.f4", .{ - .operands = .{ .flat = &.{ a, b, c, scaleA, scaleB } }, - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.workgroup.id.x`. -pub fn workgroup_id_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.workgroup.id.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.workgroup.id.y`. -pub fn workgroup_id_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.workgroup.id.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.workgroup.id.z`. -pub fn workgroup_id_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.workgroup.id.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.workitem.id.x`. -pub fn workitem_id_x( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.workitem.id.x", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.workitem.id.y`. -pub fn workitem_id_y( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.workitem.id.y", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} - -/// `rocdl.workitem.id.z`. -pub fn workitem_id_z( - ctx: *mlir.Context, - res_type: *const mlir.Type, - opts: struct { - range: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attributes: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.range) |attribute| { - attributes.appendAssumeCapacity(.named(ctx, "range", attribute)); - } - return mlir.Operation.make(ctx, "rocdl.workitem.id.z", .{ - .results = .{ .flat = &.{res_type} }, - .attributes = attributes.constSlice(), - .location = location, - }); -} diff --git a/mlir/dialects/rocdl/rocdl.zig b/mlir/dialects/rocdl/rocdl.zig deleted file mode 100644 index 456ad494..00000000 --- a/mlir/dialects/rocdl/rocdl.zig +++ /dev/null @@ -1,56 +0,0 @@ -//! LLVM ROCDL operations, AMDGPU attributes, and GPU target configuration. -//! All operation builders from the pinned LLVM revision are available in `ops`. - -const mlir = @import("mlir"); - -pub const types = @import("types.zig"); -pub const annotations = @import("annotations.zig"); -pub const AddressSpace = types.AddressSpace; -pub const pointerType = types.pointerType; - -pub const ops = @import("ops.zig"); -pub const enums = @import("enums.zig"); -pub const attributes = @import("attributes.zig"); - -pub const BufferOOBModeAttr = attributes.BufferOOBModeAttr; -pub const BufferOOBModeModuleFlagAttr = attributes.BufferOOBModeModuleFlagAttr; -pub const Gfx12AtomicCachePolicyAttr = attributes.Gfx12AtomicCachePolicyAttr; -pub const Gfx12CachePolicyAttr = attributes.Gfx12CachePolicyAttr; -pub const Gfx942CachePolicyAttr = attributes.Gfx942CachePolicyAttr; -pub const MFMANegModifierAttr = attributes.MFMANegModifierAttr; -pub const MFMAPermBAttr = attributes.MFMAPermBAttr; -pub const MatrixFormatAttr = attributes.MatrixFormatAttr; -pub const PreGfx12CachePolicyAttr = attributes.PreGfx12CachePolicyAttr; -pub const SchedGroupMaskAttr = attributes.SchedGroupMaskAttr; -pub const TBufferOOBModeModuleFlagAttr = attributes.TBufferOOBModeModuleFlagAttr; -pub const TargetAttr = attributes.TargetAttr; -pub const WMMACModifierAttr = attributes.WMMACModifierAttr; -pub const WMMAMatrixScaleAttr = attributes.WMMAMatrixScaleAttr; -pub const WMMAMatrixScaleFormatAttr = attributes.WMMAMatrixScaleFormatAttr; - -pub const BufferOOBMode = enums.BufferOOBMode; -pub const Gfx12AtomicCachePolicy = enums.Gfx12AtomicCachePolicy; -pub const Gfx12CachePolicy = enums.Gfx12CachePolicy; -pub const Gfx942CachePolicy = enums.Gfx942CachePolicy; -pub const MFMANegModifier = enums.MFMANegModifier; -pub const MFMAPermB = enums.MFMAPermB; -pub const MatrixFormat = enums.MatrixFormat; -pub const PreGfx12CachePolicy = enums.PreGfx12CachePolicy; -pub const SchedGroupMask = enums.SchedGroupMask; -pub const WMMACModifier = enums.WMMACModifier; -pub const WMMAMatrixScale = enums.WMMAMatrixScale; -pub const WMMAMatrixScaleFormat = enums.WMMAMatrixScaleFormat; - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("rocdl"); -} - -/// Converts two packed FP8 values from the selected word to vector<2xf32>. -pub fn cvtPkF32Fp8( - ctx: *mlir.Context, - bits: *const mlir.Value, - high: bool, - location: *const mlir.Location, -) *mlir.Operation { - return ops.cvt_pk_f32_fp8(ctx, bits, .vector(&.{2}, .float(ctx, .f32)), .boolean(ctx, high), location); -} diff --git a/mlir/dialects/rocdl/target.zig b/mlir/dialects/rocdl/target.zig deleted file mode 100644 index 30311dc5..00000000 --- a/mlir/dialects/rocdl/target.zig +++ /dev/null @@ -1,100 +0,0 @@ -const c = @import("c"); -const mlir = @import("mlir"); - -/// GPU compilation target, `#rocdl.target`. -pub const TargetAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAROCDLTarget; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - optLevel: i32 = 2, - triple: []const u8 = "amdgcn-amd-amdhsa", - chip: []const u8 = "gfx900", - features: []const u8 = "", - abi: []const u8 = "600", - flags: ?*const mlir.Attribute = null, - link: ?*const mlir.Attribute = null, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - return @ptrCast(c.mlirROCDLTargetAttrGet( - ctx.ptr(), - args.optLevel, - mlir.stringRef(args.triple), - mlir.stringRef(args.chip), - mlir.stringRef(args.features), - mlir.stringRef(args.abi), - .{ .ptr = if (args.flags) |a| @constCast(a.ptr().ptr) else null }, - .{ .ptr = if (args.link) |a| @constCast(a.ptr().ptr) else null }, - ).ptr orelse return error.InvalidMlir); - } - - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn optLevel(self: *const Self) i32 { - return c.mlirROCDLTargetAttrGetOptLevel(self.ptr()); - } - - pub fn triple(self: *const Self) []const u8 { - return string(c.mlirROCDLTargetAttrGetTriple(self.ptr())); - } - - pub fn chip(self: *const Self) []const u8 { - return string(c.mlirROCDLTargetAttrGetChip(self.ptr())); - } - - pub fn features(self: *const Self) []const u8 { - return string(c.mlirROCDLTargetAttrGetFeatures(self.ptr())); - } - - pub fn abi(self: *const Self) []const u8 { - return string(c.mlirROCDLTargetAttrGetAbi(self.ptr())); - } - - pub fn flags(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirROCDLTargetAttrGetFlags(self.ptr()).ptr); - } - - pub fn link(self: *const Self) ?*const mlir.Attribute { - return @ptrCast(c.mlirROCDLTargetAttrGetLink(self.ptr()).ptr); - } - - pub fn hasFlag(self: *const Self, flag: []const u8) bool { - return c.mlirROCDLTargetAttrHasFlag(self.ptr(), mlir.stringRef(flag)); - } - - pub fn hasWave64(self: *const Self) bool { - return c.mlirROCDLTargetAttrHasWave64(self.ptr()); - } - - pub fn hasFastMath(self: *const Self) bool { - return c.mlirROCDLTargetAttrHasFastMath(self.ptr()); - } - - pub fn hasDaz(self: *const Self) bool { - return c.mlirROCDLTargetAttrHasDaz(self.ptr()); - } - - pub fn hasFiniteOnly(self: *const Self) bool { - return c.mlirROCDLTargetAttrHasFiniteOnly(self.ptr()); - } - - pub fn hasUnsafeMath(self: *const Self) bool { - return c.mlirROCDLTargetAttrHasUnsafeMath(self.ptr()); - } - - pub fn hasCorrectSqrt(self: *const Self) bool { - return c.mlirROCDLTargetAttrHasCorrectSqrt(self.ptr()); - } -}; - -fn string(ref: c.MlirStringRef) []const u8 { - return if (ref.length == 0) "" else mlir.string(ref); -} diff --git a/mlir/dialects/rocdl/test.zig b/mlir/dialects/rocdl/test.zig deleted file mode 100644 index 81f0b646..00000000 --- a/mlir/dialects/rocdl/test.zig +++ /dev/null @@ -1,592 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); -const rocdl = @import("mlir/dialects/rocdl"); - -const operation_names: []const []const u8 = &.{ - "rocdl.asyncmark", - "rocdl.ballot", - "rocdl.barrier", - "rocdl.cluster.id.x", - "rocdl.cluster.id.y", - "rocdl.cluster.id.z", - "rocdl.cluster.load.async.to.lds.b128", - "rocdl.cluster.load.async.to.lds.b32", - "rocdl.cluster.load.async.to.lds.b64", - "rocdl.cluster.load.async.to.lds.b8", - "rocdl.cluster.workgroup.id.x", - "rocdl.cluster.workgroup.id.y", - "rocdl.cluster.workgroup.id.z", - "rocdl.cos", - "rocdl.cvt.f32.bf8", - "rocdl.cvt.f32.fp8", - "rocdl.cvt.pk.bf8.f32", - "rocdl.cvt.pk.f32.bf8", - "rocdl.cvt.pk.f32.fp8", - "rocdl.cvt.pk.fp8.f32", - "rocdl.cvt.pkrtz", - "rocdl.cvt.scale.pk16.bf16.bf6", - "rocdl.cvt.scale.pk16.bf16.fp6", - "rocdl.cvt.scale.pk16.f16.bf6", - "rocdl.cvt.scale.pk16.f16.fp6", - "rocdl.cvt.scale.pk16.f32.bf6", - "rocdl.cvt.scale.pk16.f32.fp6", - "rocdl.cvt.scale.pk8.bf16.bf8", - "rocdl.cvt.scale.pk8.bf16.fp4", - "rocdl.cvt.scale.pk8.bf16.fp8", - "rocdl.cvt.scale.pk8.f16.bf8", - "rocdl.cvt.scale.pk8.f16.fp4", - "rocdl.cvt.scale.pk8.f16.fp8", - "rocdl.cvt.scale.pk8.f32.bf8", - "rocdl.cvt.scale.pk8.f32.fp4", - "rocdl.cvt.scale.pk8.f32.fp8", - "rocdl.cvt.scalef32.2xpk16.bf6.f32", - "rocdl.cvt.scalef32.2xpk16.fp6.f32", - "rocdl.cvt.scalef32.f16.bf8", - "rocdl.cvt.scalef32.f16.fp8", - "rocdl.cvt.scalef32.f32.bf8", - "rocdl.cvt.scalef32.f32.fp8", - "rocdl.cvt.scalef32.pk.bf16.bf8", - "rocdl.cvt.scalef32.pk.bf16.fp4", - "rocdl.cvt.scalef32.pk.bf16.fp8", - "rocdl.cvt.scalef32.pk.bf8.bf16", - "rocdl.cvt.scalef32.pk.bf8.f16", - "rocdl.cvt.scalef32.pk.bf8.f32", - "rocdl.cvt.scalef32.pk.f16.bf8", - "rocdl.cvt.scalef32.pk.f16.fp4", - "rocdl.cvt.scalef32.pk.f16.fp8", - "rocdl.cvt.scalef32.pk.f32.bf8", - "rocdl.cvt.scalef32.pk.f32.fp4", - "rocdl.cvt.scalef32.pk.f32.fp8", - "rocdl.cvt.scalef32.pk.fp4.bf16", - "rocdl.cvt.scalef32.pk.fp4.f16", - "rocdl.cvt.scalef32.pk.fp4.f32", - "rocdl.cvt.scalef32.pk.fp8.bf16", - "rocdl.cvt.scalef32.pk.fp8.f16", - "rocdl.cvt.scalef32.pk.fp8.f32", - "rocdl.cvt.scalef32.pk16.bf6.bf16", - "rocdl.cvt.scalef32.pk16.bf6.f16", - "rocdl.cvt.scalef32.pk16.bf6.f32", - "rocdl.cvt.scalef32.pk16.fp6.bf16", - "rocdl.cvt.scalef32.pk16.fp6.f16", - "rocdl.cvt.scalef32.pk16.fp6.f32", - "rocdl.cvt.scalef32.pk32.bf16.bf6", - "rocdl.cvt.scalef32.pk32.bf16.fp6", - "rocdl.cvt.scalef32.pk32.bf6.bf16", - "rocdl.cvt.scalef32.pk32.bf6.f16", - "rocdl.cvt.scalef32.pk32.f16.bf6", - "rocdl.cvt.scalef32.pk32.f16.fp6", - "rocdl.cvt.scalef32.pk32.f32.bf6", - "rocdl.cvt.scalef32.pk32.f32.fp6", - "rocdl.cvt.scalef32.pk32.fp6.bf16", - "rocdl.cvt.scalef32.pk32.fp6.f16", - "rocdl.cvt.scalef32.pk8.bf8.bf16", - "rocdl.cvt.scalef32.pk8.bf8.f16", - "rocdl.cvt.scalef32.pk8.bf8.f32", - "rocdl.cvt.scalef32.pk8.fp4.bf16", - "rocdl.cvt.scalef32.pk8.fp4.f16", - "rocdl.cvt.scalef32.pk8.fp4.f32", - "rocdl.cvt.scalef32.pk8.fp8.bf16", - "rocdl.cvt.scalef32.pk8.fp8.f16", - "rocdl.cvt.scalef32.pk8.fp8.f32", - "rocdl.cvt.scalef32.sr.bf8.bf16", - "rocdl.cvt.scalef32.sr.bf8.f16", - "rocdl.cvt.scalef32.sr.bf8.f32", - "rocdl.cvt.scalef32.sr.fp8.bf16", - "rocdl.cvt.scalef32.sr.fp8.f16", - "rocdl.cvt.scalef32.sr.fp8.f32", - "rocdl.cvt.scalef32.sr.pk.fp4.bf16", - "rocdl.cvt.scalef32.sr.pk.fp4.f16", - "rocdl.cvt.scalef32.sr.pk.fp4.f32", - "rocdl.cvt.scalef32.sr.pk16.bf6.bf16", - "rocdl.cvt.scalef32.sr.pk16.bf6.f16", - "rocdl.cvt.scalef32.sr.pk16.bf6.f32", - "rocdl.cvt.scalef32.sr.pk16.fp6.bf16", - "rocdl.cvt.scalef32.sr.pk16.fp6.f16", - "rocdl.cvt.scalef32.sr.pk16.fp6.f32", - "rocdl.cvt.scalef32.sr.pk32.bf6.bf16", - "rocdl.cvt.scalef32.sr.pk32.bf6.f16", - "rocdl.cvt.scalef32.sr.pk32.bf6.f32", - "rocdl.cvt.scalef32.sr.pk32.fp6.bf16", - "rocdl.cvt.scalef32.sr.pk32.fp6.f16", - "rocdl.cvt.scalef32.sr.pk32.fp6.f32", - "rocdl.cvt.scalef32.sr.pk8.bf8.bf16", - "rocdl.cvt.scalef32.sr.pk8.bf8.f16", - "rocdl.cvt.scalef32.sr.pk8.bf8.f32", - "rocdl.cvt.scalef32.sr.pk8.fp4.bf16", - "rocdl.cvt.scalef32.sr.pk8.fp4.f16", - "rocdl.cvt.scalef32.sr.pk8.fp4.f32", - "rocdl.cvt.scalef32.sr.pk8.fp8.bf16", - "rocdl.cvt.scalef32.sr.pk8.fp8.f16", - "rocdl.cvt.scalef32.sr.pk8.fp8.f32", - "rocdl.cvt.sr.bf8.f32", - "rocdl.cvt.sr.fp8.f32", - "rocdl.dot4.f32.bf8.bf8", - "rocdl.dot4.f32.bf8.fp8", - "rocdl.dot4.f32.fp8.bf8", - "rocdl.dot4.f32.fp8.fp8", - "rocdl.ds.atomic.async.barrier.arrive.b64", - "rocdl.ds.atomic.barrier.arrive.rtn.b64", - "rocdl.ds.load.tr16.b128", - "rocdl.ds.load.tr4.b64", - "rocdl.ds.load.tr6.b96", - "rocdl.ds.load.tr8.b64", - "rocdl.ds.read.tr16.b64", - "rocdl.ds.read.tr4.b64", - "rocdl.ds.read.tr6.b96", - "rocdl.ds.read.tr8.b64", - "rocdl.ds_bpermute", - "rocdl.ds_swizzle", - "rocdl.exp", - "rocdl.exp2", - "rocdl.fdot2", - "rocdl.fdot2.bf16.bf16", - "rocdl.fdot2.f16.f16", - "rocdl.fdot2.f32.bf16", - "rocdl.flat.prefetch", - "rocdl.fmed3", - "rocdl.global.load.async.lds", - "rocdl.global.load.async.to.lds.b128", - "rocdl.global.load.async.to.lds.b32", - "rocdl.global.load.async.to.lds.b64", - "rocdl.global.load.async.to.lds.b8", - "rocdl.global.load.lds", - "rocdl.global.load.tr.b128", - "rocdl.global.load.tr.b64", - "rocdl.global.load.tr4.b64", - "rocdl.global.load.tr6.b96", - "rocdl.global.prefetch", - "rocdl.global.store.async.from.lds.b128", - "rocdl.global.store.async.from.lds.b32", - "rocdl.global.store.async.from.lds.b64", - "rocdl.global.store.async.from.lds.b8", - "rocdl.iglp.opt", - "rocdl.load.async.to.lds", - "rocdl.load.to.lds", - "rocdl.log", - "rocdl.make.buffer.rsrc", - "rocdl.mbcnt.hi", - "rocdl.mbcnt.lo", - "rocdl.mfma.f32.16x16x16bf16.1k", - "rocdl.mfma.f32.16x16x16f16", - "rocdl.mfma.f32.16x16x1f32", - "rocdl.mfma.f32.16x16x2bf16", - "rocdl.mfma.f32.16x16x32.bf16", - "rocdl.mfma.f32.16x16x32.bf8.bf8", - "rocdl.mfma.f32.16x16x32.bf8.fp8", - "rocdl.mfma.f32.16x16x32.f16", - "rocdl.mfma.f32.16x16x32.fp8.bf8", - "rocdl.mfma.f32.16x16x32.fp8.fp8", - "rocdl.mfma.f32.16x16x4bf16.1k", - "rocdl.mfma.f32.16x16x4f16", - "rocdl.mfma.f32.16x16x4f32", - "rocdl.mfma.f32.16x16x8.xf32", - "rocdl.mfma.f32.16x16x8bf16", - "rocdl.mfma.f32.32x32x16.bf16", - "rocdl.mfma.f32.32x32x16.bf8.bf8", - "rocdl.mfma.f32.32x32x16.bf8.fp8", - "rocdl.mfma.f32.32x32x16.f16", - "rocdl.mfma.f32.32x32x16.fp8.bf8", - "rocdl.mfma.f32.32x32x16.fp8.fp8", - "rocdl.mfma.f32.32x32x1f32", - "rocdl.mfma.f32.32x32x2bf16", - "rocdl.mfma.f32.32x32x2f32", - "rocdl.mfma.f32.32x32x4.xf32", - "rocdl.mfma.f32.32x32x4bf16", - "rocdl.mfma.f32.32x32x4bf16.1k", - "rocdl.mfma.f32.32x32x4f16", - "rocdl.mfma.f32.32x32x8bf16.1k", - "rocdl.mfma.f32.32x32x8f16", - "rocdl.mfma.f32.4x4x1f32", - "rocdl.mfma.f32.4x4x2bf16", - "rocdl.mfma.f32.4x4x4bf16.1k", - "rocdl.mfma.f32.4x4x4f16", - "rocdl.mfma.f64.16x16x4f64", - "rocdl.mfma.f64.4x4x4f64", - "rocdl.mfma.i32.16x16x16i8", - "rocdl.mfma.i32.16x16x32.i8", - "rocdl.mfma.i32.16x16x4i8", - "rocdl.mfma.i32.16x16x64.i8", - "rocdl.mfma.i32.32x32x16.i8", - "rocdl.mfma.i32.32x32x32.i8", - "rocdl.mfma.i32.32x32x4i8", - "rocdl.mfma.i32.32x32x8i8", - "rocdl.mfma.i32.4x4x4i8", - "rocdl.mfma.scale.f32.16x16x128.f8f6f4", - "rocdl.mfma.scale.f32.32x32x64.f8f6f4", - "rocdl.permlane16.swap", - "rocdl.permlane16.var", - "rocdl.permlane32.swap", - "rocdl.permlanex16", - "rocdl.permlanex16.var", - "rocdl.ptr.s.buffer.load", - "rocdl.raw.buffer.atomic.cmpswap", - "rocdl.raw.buffer.atomic.fadd", - "rocdl.raw.buffer.atomic.fmax", - "rocdl.raw.buffer.atomic.smax", - "rocdl.raw.buffer.atomic.umin", - "rocdl.raw.buffer.load", - "rocdl.raw.buffer.store", - "rocdl.raw.ptr.buffer.atomic.cmpswap", - "rocdl.raw.ptr.buffer.atomic.fadd", - "rocdl.raw.ptr.buffer.atomic.fmax", - "rocdl.raw.ptr.buffer.atomic.smax", - "rocdl.raw.ptr.buffer.atomic.umin", - "rocdl.raw.ptr.buffer.load", - "rocdl.raw.ptr.buffer.load.async.lds", - "rocdl.raw.ptr.buffer.load.lds", - "rocdl.raw.ptr.buffer.store", - "rocdl.rcp", - "rocdl.readfirstlane", - "rocdl.readlane", - "rocdl.rsq", - "rocdl.s.barrier", - "rocdl.s.barrier.init", - "rocdl.s.barrier.join", - "rocdl.s.barrier.leave", - "rocdl.s.barrier.signal", - "rocdl.s.barrier.signal.isfirst", - "rocdl.s.barrier.signal.var", - "rocdl.s.barrier.wait", - "rocdl.s.get.barrier.state", - "rocdl.s.get.named.barrier.state", - "rocdl.s.nop", - "rocdl.s.setprio", - "rocdl.s.sleep", - "rocdl.s.wait.asynccnt", - "rocdl.s.wait.dscnt", - "rocdl.s.wait.expcnt", - "rocdl.s.wait.loadcnt", - "rocdl.s.wait.storecnt", - "rocdl.s.wait.tensorcnt", - "rocdl.s.waitcnt", - "rocdl.s.wakeup.barrier", - "rocdl.sched.barrier", - "rocdl.sched.group.barrier", - "rocdl.sdot2", - "rocdl.sdot4", - "rocdl.sdot8", - "rocdl.sin", - "rocdl.smfmac.f32.16x16x128.bf8.bf8", - "rocdl.smfmac.f32.16x16x128.bf8.fp8", - "rocdl.smfmac.f32.16x16x128.fp8.bf8", - "rocdl.smfmac.f32.16x16x128.fp8.fp8", - "rocdl.smfmac.f32.16x16x32.bf16", - "rocdl.smfmac.f32.16x16x32.f16", - "rocdl.smfmac.f32.16x16x64.bf16", - "rocdl.smfmac.f32.16x16x64.bf8.bf8", - "rocdl.smfmac.f32.16x16x64.bf8.fp8", - "rocdl.smfmac.f32.16x16x64.f16", - "rocdl.smfmac.f32.16x16x64.fp8.bf8", - "rocdl.smfmac.f32.16x16x64.fp8.fp8", - "rocdl.smfmac.f32.32x32x16.bf16", - "rocdl.smfmac.f32.32x32x16.f16", - "rocdl.smfmac.f32.32x32x32.bf16", - "rocdl.smfmac.f32.32x32x32.bf8.bf8", - "rocdl.smfmac.f32.32x32x32.bf8.fp8", - "rocdl.smfmac.f32.32x32x32.f16", - "rocdl.smfmac.f32.32x32x32.fp8.bf8", - "rocdl.smfmac.f32.32x32x32.fp8.fp8", - "rocdl.smfmac.f32.32x32x64.bf8.bf8", - "rocdl.smfmac.f32.32x32x64.bf8.fp8", - "rocdl.smfmac.f32.32x32x64.fp8.bf8", - "rocdl.smfmac.f32.32x32x64.fp8.fp8", - "rocdl.smfmac.i32.16x16x128.i8", - "rocdl.smfmac.i32.16x16x64.i8", - "rocdl.smfmac.i32.32x32x32.i8", - "rocdl.smfmac.i32.32x32x64.i8", - "rocdl.sqrt", - "rocdl.sudot4", - "rocdl.sudot8", - "rocdl.swmmac.bf16.16x16x32.bf16", - "rocdl.swmmac.bf16.16x16x64.bf16", - "rocdl.swmmac.bf16f32.16x16x64.bf16", - "rocdl.swmmac.f16.16x16x128.bf8.bf8", - "rocdl.swmmac.f16.16x16x128.bf8.fp8", - "rocdl.swmmac.f16.16x16x128.fp8.bf8", - "rocdl.swmmac.f16.16x16x128.fp8.fp8", - "rocdl.swmmac.f16.16x16x32.f16", - "rocdl.swmmac.f16.16x16x64.f16", - "rocdl.swmmac.f32.16x16x128.bf8.bf8", - "rocdl.swmmac.f32.16x16x128.bf8.fp8", - "rocdl.swmmac.f32.16x16x128.fp8.bf8", - "rocdl.swmmac.f32.16x16x128.fp8.fp8", - "rocdl.swmmac.f32.16x16x32.bf16", - "rocdl.swmmac.f32.16x16x32.bf8.bf8", - "rocdl.swmmac.f32.16x16x32.bf8.fp8", - "rocdl.swmmac.f32.16x16x32.f16", - "rocdl.swmmac.f32.16x16x32.fp8.bf8", - "rocdl.swmmac.f32.16x16x32.fp8.fp8", - "rocdl.swmmac.f32.16x16x64.bf16", - "rocdl.swmmac.f32.16x16x64.f16", - "rocdl.swmmac.i32.16x16x128.iu8", - "rocdl.swmmac.i32.16x16x32.iu4", - "rocdl.swmmac.i32.16x16x32.iu8", - "rocdl.swmmac.i32.16x16x64.iu4", - "rocdl.tanh", - "rocdl.tensor.load.to.lds", - "rocdl.tensor.store.from.lds", - "rocdl.udot2", - "rocdl.udot4", - "rocdl.udot8", - "rocdl.update.dpp", - "rocdl.wait.asyncmark", - "rocdl.wave.barrier", - "rocdl.wave.id", - "rocdl.wavefrontsize", - "rocdl.wmma.bf16.16x16x16.bf16", - "rocdl.wmma.bf16.16x16x32.bf16", - "rocdl.wmma.bf16f32.16x16x32.bf16", - "rocdl.wmma.f16.16x16x128.bf8_bf8", - "rocdl.wmma.f16.16x16x128.bf8_fp8", - "rocdl.wmma.f16.16x16x128.fp8_bf8", - "rocdl.wmma.f16.16x16x128.fp8_fp8", - "rocdl.wmma.f16.16x16x16.f16", - "rocdl.wmma.f16.16x16x32.f16", - "rocdl.wmma.f16.16x16x64.bf8_bf8", - "rocdl.wmma.f16.16x16x64.bf8_fp8", - "rocdl.wmma.f16.16x16x64.fp8_bf8", - "rocdl.wmma.f16.16x16x64.fp8_fp8", - "rocdl.wmma.f32.16x16x128.bf8_bf8", - "rocdl.wmma.f32.16x16x128.bf8_fp8", - "rocdl.wmma.f32.16x16x128.fp8_bf8", - "rocdl.wmma.f32.16x16x128.fp8_fp8", - "rocdl.wmma.f32.16x16x16.bf16", - "rocdl.wmma.f32.16x16x16.bf8_bf8", - "rocdl.wmma.f32.16x16x16.bf8_fp8", - "rocdl.wmma.f32.16x16x16.f16", - "rocdl.wmma.f32.16x16x16.fp8_bf8", - "rocdl.wmma.f32.16x16x16.fp8_fp8", - "rocdl.wmma.f32.16x16x32.bf16", - "rocdl.wmma.f32.16x16x32.f16", - "rocdl.wmma.f32.16x16x4.f32", - "rocdl.wmma.f32.16x16x64.bf8_bf8", - "rocdl.wmma.f32.16x16x64.bf8_fp8", - "rocdl.wmma.f32.16x16x64.fp8_bf8", - "rocdl.wmma.f32.16x16x64.fp8_fp8", - "rocdl.wmma.i32.16x16x16.iu4", - "rocdl.wmma.i32.16x16x16.iu8", - "rocdl.wmma.i32.16x16x32.iu4", - "rocdl.wmma.i32.16x16x64.iu8", - "rocdl.wmma.scale.f32.16x16x128.f8f6f4", - "rocdl.wmma.scale.f32.32x16x128.f4", - "rocdl.wmma.scale16.f32.16x16x128.f8f6f4", - "rocdl.wmma.scale16.f32.32x16x128.f4", - "rocdl.workgroup.id.x", - "rocdl.workgroup.id.y", - "rocdl.workgroup.id.z", - "rocdl.workitem.id.x", - "rocdl.workitem.id.y", - "rocdl.workitem.id.z", -}; - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - rocdl.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -fn expectAttributeRoundTrip(ctx: *mlir.Context, attribute: *const mlir.Attribute) !void { - var buffer: [2048]u8 = undefined; - var writer = std.Io.Writer.fixed(&buffer); - try attribute.format(&writer); - try std.testing.expect(attribute.eql(try mlir.Attribute.parse(ctx, writer.buffered()))); -} - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(rocdl); - std.testing.refAllDecls(rocdl.ops); - std.testing.refAllDecls(rocdl.types); - std.testing.refAllDecls(rocdl.annotations); - inline for (rocdl.attributes.enum_attributes) |A| std.testing.refAllDecls(A); - std.testing.refAllDecls(rocdl.TargetAttr); -} - -test "packed FP8 conversion uses the registered LLVM ROCDL dialect" { - const ctx = try testContext(); - defer ctx.deinit(); - try std.testing.expect(ctx.isRegisteredOperation("rocdl.cvt.pk.f32.fp8")); - - const loc = mlir.Location.unknown(ctx); - const block = mlir.Block.init(&.{.int(ctx, .i32)}, &.{loc}); - defer block.deinit(); - for ([_]bool{ false, true }) |high| { - const op = rocdl.cvtPkF32Fp8(ctx, block.argument(0), high, loc); - defer op.deinit(); - try std.testing.expect(op.verify()); - try std.testing.expect(op.result(0).type_().eql(.vector(&.{2}, .float(ctx, .f32)))); - try std.testing.expect(op.attributeByName("wordSel").?.eql(.boolean(ctx, high))); - } -} - -test "every bound operation is registered" { - const ctx = try testContext(); - defer ctx.deinit(); - try std.testing.expectEqual(operation_names.len, @as(usize, @intCast(c.mlirROCDLGetOperationCount(ctx.ptr())))); - for (operation_names, 0..) |name, i| { - try std.testing.expect(ctx.isRegisteredOperation(name)); - for (operation_names[0..i]) |earlier| try std.testing.expect(!std.mem.eql(u8, name, earlier)); - } - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} - -test "all ROCDL enum attributes construct and round-trip every named value" { - @setEvalBranchQuota(100_000); - const ctx = try testContext(); - defer ctx.deinit(); - inline for (rocdl.attributes.enum_attributes) |A| { - for (std.enums.values(A.Value)) |value| { - const attribute = try A.get(ctx, .{ .value = value }); - try std.testing.expect(attribute.isA(A) != null); - try std.testing.expectEqual(value, attribute.value()); - try expectAttributeRoundTrip(ctx, attribute.attribute()); - } - } - const mask: rocdl.SchedGroupMask = .combine(&.{ .valu, .mfma_wmma, .ds_read }); - try std.testing.expect(mask.contains(.mfma_wmma)); - try std.testing.expect(!mask.contains(.all_vmem)); - const attribute = try rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = mask }); - try std.testing.expectEqual(mask, attribute.value()); - try expectAttributeRoundTrip(ctx, attribute.attribute()); -} - -test "ROCDL enum constructors reject invalid values and flag bits" { - const ctx = try testContext(); - defer ctx.deinit(); - try std.testing.expect(c.mlirROCDLMatrixFormatAttrGet(ctx.ptr(), 5).ptr == null); - try std.testing.expect(c.mlirROCDLWMMACModifierAttrGet(ctx.ptr(), 65536).ptr == null); - try std.testing.expectError(error.InvalidMlir, rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = @fromBackingInt(1 << 20) })); - try std.testing.expectError(error.InvalidMlir, rocdl.PreGfx12CachePolicyAttr.get(ctx, .{ .value = @fromBackingInt(1 << 6) })); -} - -test "target attributes expose all fields and reject invalid options" { - const ctx = try testContext(); - defer ctx.deinit(); - const defaults = try rocdl.TargetAttr.get(ctx, .{}); - try std.testing.expectEqual(@as(i32, 2), defaults.optLevel()); - try std.testing.expectEqualStrings("amdgcn-amd-amdhsa", defaults.triple()); - try std.testing.expectEqualStrings("gfx900", defaults.chip()); - try std.testing.expectEqualStrings("", defaults.features()); - try std.testing.expectEqualStrings("600", defaults.abi()); - try std.testing.expect(defaults.flags() == null and defaults.link() == null); - try std.testing.expect(defaults.hasWave64()); - try std.testing.expect(!defaults.hasFastMath()); - try std.testing.expect(defaults.hasCorrectSqrt()); - try expectAttributeRoundTrip(ctx, defaults.attribute()); - - const flags: *const mlir.Attribute = @ptrCast(mlir.DictionaryAttribute.init(ctx, &.{ - .named(ctx, "no_wave64", .unit(ctx)), - .named(ctx, "fast", .unit(ctx)), - .named(ctx, "daz", .unit(ctx)), - .named(ctx, "finite_only", .unit(ctx)), - .named(ctx, "unsafe_math", .unit(ctx)), - .named(ctx, "unsafe_sqrt", .unit(ctx)), - })); - const link: *const mlir.Attribute = @ptrCast(mlir.ArrayAttribute.init(ctx, &.{.string(ctx, "device.bc")})); - const target = try rocdl.TargetAttr.get(ctx, .{ .optLevel = 3, .chip = "gfx942", .features = "+xnack", .abi = "500", .flags = flags, .link = link }); - try std.testing.expectEqual(@as(i32, 3), target.optLevel()); - try std.testing.expectEqualStrings("gfx942", target.chip()); - try std.testing.expectEqualStrings("+xnack", target.features()); - try std.testing.expectEqualStrings("500", target.abi()); - try std.testing.expect(target.flags().?.eql(flags)); - try std.testing.expect(target.link().?.eql(link)); - try std.testing.expect(target.hasFlag("fast") and target.hasFastMath()); - try std.testing.expect(!target.hasWave64()); - try std.testing.expect(target.hasDaz() and target.hasFiniteOnly()); - try std.testing.expect(target.hasUnsafeMath() and !target.hasCorrectSqrt()); - try expectAttributeRoundTrip(ctx, target.attribute()); - - try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .optLevel = 4 })); - try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .triple = "" })); - try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .chip = "" })); - try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .abi = "700" })); - try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .flags = .unit(ctx) })); - const badLink: *const mlir.Attribute = @ptrCast(mlir.ArrayAttribute.init(ctx, &.{.unit(ctx)})); - try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .link = badLink })); - const other = try testContext(); - defer other.deinit(); - const otherFlags: *const mlir.Attribute = @ptrCast(mlir.DictionaryAttribute.init(other, &.{})); - try std.testing.expectError(error.InvalidMlir, rocdl.TargetAttr.get(ctx, .{ .flags = otherFlags })); -} - -test "ROCDL barriers buffer loads and matrix operations verify and reparse" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc = mlir.Location.unknown(ctx); - const module = try mlir.Module.parse(ctx, - \\module { - \\ llvm.func @test(%rsrc: vector<4xi32>, %offset: i32, %a: f32, %c: vector<4xf32>) { - \\ llvm.return - \\ } - \\} - ); - defer module.deinit(); - const function = module.body().firstOperation().?; - const block = function.region(0).firstBlock().?; - const terminator = block.terminator().?; - const i32Type: *const mlir.Type = .int(ctx, .i32); - const f32Type: *const mlir.Type = .float(ctx, .f32); - const vec4f32 = rocdl.types.v4f32(ctx); - const zero: *const mlir.Attribute = .int(ctx, .i32, 0); - const opList = .{ - rocdl.ops.s_barrier(ctx, loc), - rocdl.ops.s_sleep(ctx, .int(ctx, .i32, 1), loc), - rocdl.ops.workitem_id_x(ctx, i32Type, .{}, loc), - rocdl.ops.sched_barrier(ctx, (try rocdl.SchedGroupMaskAttr.get(ctx, .{ .value = .combine(&.{ .valu, .mfma_wmma }) })).attribute(), loc), - rocdl.ops.raw_buffer_load(ctx, block.argument(0), block.argument(1), block.argument(1), f32Type, .{}, loc), - rocdl.ops.raw_buffer_load( - ctx, - block.argument(0), - block.argument(1), - block.argument(1), - f32Type, - .{ .aux = (try rocdl.Gfx942CachePolicyAttr.get(ctx, .{ .value = .combine(&.{ .sc0, .nt }) })).attribute() }, - loc, - ), - rocdl.ops.mfma_f32_16x16x4f32(ctx, block.argument(2), block.argument(2), block.argument(3), vec4f32, zero, zero, (try rocdl.MFMAPermBAttr.get(ctx, .{ .value = .none })).attribute(), loc), - }; - inline for (opList) |op| { - try std.testing.expect(op.verify()); - block.insertOwnedOperationBefore(terminator, op); - } - rocdl.annotations.annotateFunction(ctx, function, .{ - .kernel = true, - .reqdWorkGroupSize = .{ 64, 1, 1 }, - .flatWorkGroupSize = "64,64", - .maxFlatWorkGroupSize = 64, - .wavesPerEu = .{ .min = 1, .max = 2 }, - .unsafeFpAtomics = true, - }); - rocdl.annotations.annotateMemory(ctx, opList[4], .{ .lastUse = true, .noRemoteMemory = true, .noFineGrainedMemory = true, .ignoreDenormalMode = true }); - try std.testing.expect(opList[4].attributeByName("aux").?.eql(.int(ctx, .i32, 0))); - try std.testing.expect(opList[5].attributeByName("aux").?.isA(rocdl.Gfx942CachePolicyAttr) != null); - try std.testing.expect(function.attributeByName("rocdl.waves_per_eu").?.eql(.string(ctx, "1,2"))); - try std.testing.expect(module.operation().verify()); - var buffer: [16384]u8 = undefined; - var writer = std.Io.Writer.fixed(&buffer); - try module.format(&writer); - const parsed = try mlir.Module.parse(ctx, writer.buffered()); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "ROCDL annotations preserve module flag defaults and explicit false" { - const ctx = try testContext(); - defer ctx.deinit(); - const module = mlir.Module.init(.unknown(ctx)); - defer module.deinit(); - rocdl.annotations.annotateModule(ctx, module.operation(), .{}); - try std.testing.expect(module.operation().attributeByName("rocdl.xnack") == null); - rocdl.annotations.annotateModule(ctx, module.operation(), .{ .xnack = false, .sramecc = true }); - try std.testing.expect(module.operation().attributeByName("rocdl.xnack").?.eql(.boolean(ctx, false))); - try std.testing.expect(module.operation().attributeByName("rocdl.sramecc").?.eql(.boolean(ctx, true))); - try std.testing.expect(module.operation().verify()); - const pointer = rocdl.pointerType(ctx, .barrier); - try std.testing.expect(pointer.eql(try mlir.Type.parse(ctx, "!llvm.ptr<15>"))); - try std.testing.expect(rocdl.pointerType(ctx, .buffer_resource).eql(try mlir.Type.parse(ctx, "!llvm.ptr<8>"))); -} diff --git a/mlir/dialects/rocdl/types.zig b/mlir/dialects/rocdl/types.zig deleted file mode 100644 index a91ede15..00000000 --- a/mlir/dialects/rocdl/types.zig +++ /dev/null @@ -1,147 +0,0 @@ -// Generated from LLVM ROCDL TableGen records; see REGENERATING.md. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - -const mlir = @import("mlir"); -const llvm = @import("mlir/dialects/llvm"); - -/// ROCDL address spaces use LLVM pointer types. -pub const AddressSpace = enum(u32) { - generic = 0, - global = 1, - shared = 3, - constant = 4, - private = 5, - buffer_fat_pointer = 7, - buffer_resource = 8, - buffer_strided_pointer = 9, - barrier = 15, -}; - -pub fn pointerType(ctx: *mlir.Context, space: AddressSpace) *const mlir.Type { - return llvm.pointerType(ctx, space); -} - -/// `vector<16xbf16>`, as used by ROCDL operations. -pub fn v16bf16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{16}, .float(ctx, .bf16)); -} - -/// `vector<16xf16>`, as used by ROCDL operations. -pub fn v16f16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{16}, .float(ctx, .f16)); -} - -/// `vector<16xf32>`, as used by ROCDL operations. -pub fn v16f32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{16}, .float(ctx, .f32)); -} - -/// `vector<16xi32>`, as used by ROCDL operations. -pub fn v16i32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{16}, .int(ctx, .i32)); -} - -/// `vector<2xbf16>`, as used by ROCDL operations. -pub fn v2bf16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{2}, .float(ctx, .bf16)); -} - -/// `vector<2xf16>`, as used by ROCDL operations. -pub fn v2f16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{2}, .float(ctx, .f16)); -} - -/// `vector<2xf32>`, as used by ROCDL operations. -pub fn v2f32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{2}, .float(ctx, .f32)); -} - -/// `vector<2xi16>`, as used by ROCDL operations. -pub fn v2i16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{2}, .int(ctx, .i16)); -} - -/// `vector<2xi32>`, as used by ROCDL operations. -pub fn v2i32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{2}, .int(ctx, .i32)); -} - -/// `vector<32xbf16>`, as used by ROCDL operations. -pub fn v32bf16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{32}, .float(ctx, .bf16)); -} - -/// `vector<32xf16>`, as used by ROCDL operations. -pub fn v32f16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{32}, .float(ctx, .f16)); -} - -/// `vector<32xf32>`, as used by ROCDL operations. -pub fn v32f32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{32}, .float(ctx, .f32)); -} - -/// `vector<32xi32>`, as used by ROCDL operations. -pub fn v32i32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{32}, .int(ctx, .i32)); -} - -/// `vector<3xi32>`, as used by ROCDL operations. -pub fn v3i32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{3}, .int(ctx, .i32)); -} - -/// `vector<4xf16>`, as used by ROCDL operations. -pub fn v4f16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{4}, .float(ctx, .f16)); -} - -/// `vector<4xf32>`, as used by ROCDL operations. -pub fn v4f32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{4}, .float(ctx, .f32)); -} - -/// `vector<4xf64>`, as used by ROCDL operations. -pub fn v4f64(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{4}, .float(ctx, .f64)); -} - -/// `vector<4xi16>`, as used by ROCDL operations. -pub fn v4i16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{4}, .int(ctx, .i16)); -} - -/// `vector<4xi32>`, as used by ROCDL operations. -pub fn v4i32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{4}, .int(ctx, .i32)); -} - -/// `vector<6xi32>`, as used by ROCDL operations. -pub fn v6i32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{6}, .int(ctx, .i32)); -} - -/// `vector<8xbf16>`, as used by ROCDL operations. -pub fn v8bf16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{8}, .float(ctx, .bf16)); -} - -/// `vector<8xf16>`, as used by ROCDL operations. -pub fn v8f16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{8}, .float(ctx, .f16)); -} - -/// `vector<8xf32>`, as used by ROCDL operations. -pub fn v8f32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{8}, .float(ctx, .f32)); -} - -/// `vector<8xi16>`, as used by ROCDL operations. -pub fn v8i16(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{8}, .int(ctx, .i16)); -} - -/// `vector<8xi32>`, as used by ROCDL operations. -pub fn v8i32(ctx: *mlir.Context) *const mlir.Type { - return .vector(&.{8}, .int(ctx, .i32)); -} diff --git a/mlir/dialects/scf/BUILD.bazel b/mlir/dialects/scf/BUILD.bazel deleted file mode 100644 index 1d7a0b3c..00000000 --- a/mlir/dialects/scf/BUILD.bazel +++ /dev/null @@ -1,23 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "scf", - import_name = "mlir/dialects/scf", - main = "scf.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//mlir", - "//stdx", - "@llvm-project//mlir:CAPISCF", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":scf", - "//mlir", - ], -) diff --git a/mlir/dialects/scf/scf.zig b/mlir/dialects/scf/scf.zig deleted file mode 100644 index 1c82a9e0..00000000 --- a/mlir/dialects/scf/scf.zig +++ /dev/null @@ -1,363 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("scf"); -} - -pub const ForOpts = struct { - /// Per td: `scf.for` uses signed comparison by default. Setting this emits - /// the `unsignedCmp` UnitAttr so the induction variable advances with - /// unsigned comparison instead. - unsigned_cmp: bool = false, - num_stages: ?i32 = null, -}; - -/// scf.for — operands are (lower, upper, step, init_args...) as a flat list. -/// Result arity equals init_args arity; result types must match init_args types. -/// -/// The body block must have arguments `[iv_type, *init_arg_types]` and be -/// terminated with `scf.yield`. -pub fn for_( - ctx: *mlir.Context, - lower: *const mlir.Value, - upper: *const mlir.Value, - step: *const mlir.Value, - init_args: []const *const mlir.Value, - body: *mlir.Block, - opts: ForOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; - operands.appendSliceAssumeCapacity(&.{ lower, upper, step }); - operands.appendSliceAssumeCapacity(init_args); - - var result_types: stdx.BoundedArray(*const mlir.Type, 64) = .empty; - for (init_args) |v| result_types.appendAssumeCapacity(v.type_()); - - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (opts.unsigned_cmp) { - attrs.appendAssumeCapacity(.named(ctx, "unsignedCmp", .unit(ctx))); - } - if (opts.num_stages) |n| { - attrs.appendAssumeCapacity(.named(ctx, "tt.num_stages", .int(ctx, .i32, n))); - } - - return mlir.Operation.make(ctx, "scf.for", .{ - .operands = .{ .flat = operands.constSlice() }, - .results = .{ .flat = result_types.constSlice() }, - .blocks = &.{body}, - .attributes = attrs.constSlice(), - // Per-op verify fails with "operand #0 does not dominate this use" when - // the scf.for references outer SSA values that only become reachable - // once it's appended to the parent block. Defer to module-level verify. - .verify = false, - .location = location, - }); -} - -/// scf.if — cond is i1; then/else blocks must be terminated with `scf.yield` -/// (or implicitly for no-result forms). -/// -/// Built via raw OperationState because it has two separate regions. -pub fn if_( - ctx: *mlir.Context, - cond: *const mlir.Value, - result_types: []const *const mlir.Type, - then_block: *mlir.Block, - opts: struct { - else_block: ?*mlir.Block = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - _ = ctx; - var state: mlir.OperationState = .init("scf.if", location); - state.addOperands(&.{cond}); - state.addResults(result_types); - - const then_region = mlir.Region.init(); - then_region.appendOwnedBlock(then_block); - const else_region = mlir.Region.init(); - if (opts.else_block) |b| else_region.appendOwnedBlock(b); - state.addOwnedRegions(&.{ then_region, else_region }); - - return mlir.Operation.init(&state) catch @panic("Failed to create scf.if"); -} - -/// scf.yield — variadic result terminator for scf.for / scf.if regions. -pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "scf.yield", .{ - .operands = .{ .flat = values }, - .verify = false, - .location = location, - }); -} - -/// scf.while — two regions. The `before` region computes the condition and -/// terminates with `scf.condition`. The `after` region runs when the condition -/// is true and terminates with `scf.yield`. The arguments of `before` have the -/// types of `inits`. The trailing operands of `scf.condition` feed both the -/// `after` region's arguments and the `scf.while` op's results. -pub fn while_( - ctx: *mlir.Context, - inits: []const *const mlir.Value, - result_types: []const *const mlir.Type, - before_block: *mlir.Block, - after_block: *mlir.Block, - location: *const mlir.Location, -) *mlir.Operation { - _ = ctx; - var state: mlir.OperationState = .init("scf.while", location); - state.addOperands(inits); - state.addResults(result_types); - - const before_region = mlir.Region.init(); - before_region.appendOwnedBlock(before_block); - const after_region = mlir.Region.init(); - after_region.appendOwnedBlock(after_block); - state.addOwnedRegions(&.{ before_region, after_region }); - - return mlir.Operation.init(&state) catch @panic("Failed to create scf.while"); -} - -/// scf.condition — terminator of the `before` region of scf.while. -/// First operand is i1 (the continuation condition). The remaining operands -/// are forwarded to either the `after` region's arguments (if true) or the -/// scf.while's results (if false). -pub fn condition( - ctx: *mlir.Context, - cond: *const mlir.Value, - args: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 64) = .empty; - buf.appendAssumeCapacity(cond); - buf.appendSliceAssumeCapacity(args); - return mlir.Operation.make(ctx, "scf.condition", .{ - .operands = .{ .flat = buf.constSlice() }, - .verify = false, - .location = location, - }); -} - -pub const ExecuteRegionOpts = struct { - no_inline: bool = false, -}; - -/// scf.execute_region — runs `body` exactly once, returns the yielded values. -/// `body` has no arguments but may span multiple blocks (unlike for/if). -pub fn execute_region( - ctx: *mlir.Context, - result_types: []const *const mlir.Type, - body: *mlir.Block, - opts: ExecuteRegionOpts, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.no_inline) { - attrs.appendAssumeCapacity(.named(ctx, "no_inline", .unit(ctx))); - } - return mlir.Operation.make(ctx, "scf.execute_region", .{ - .results = .{ .flat = result_types }, - .blocks = &.{body}, - .attributes = attrs.constSlice(), - // execute_region has its own region-verify; defer to the module-level - // verify so unresolved SSA uses don't block construction. - .verify = false, - .location = location, - }); -} - -/// scf.parallel — multi-dim parallel for loop with optional reductions. -/// Operands are (lbs, ubs, steps, inits). `body_block` takes one `index` arg -/// per loop dim and must be terminated with `scf.reduce` (or no-op if inits is -/// empty — the parser auto-inserts an empty reduce). -pub fn parallel( - ctx: *mlir.Context, - lbs: []const *const mlir.Value, - ubs: []const *const mlir.Value, - steps: []const *const mlir.Value, - inits: []const *const mlir.Value, - body_block: *mlir.Block, - result_types: []const *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - std.debug.assert(lbs.len == ubs.len); - std.debug.assert(lbs.len == steps.len); - - var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; - operands.appendSliceAssumeCapacity(lbs); - operands.appendSliceAssumeCapacity(ubs); - operands.appendSliceAssumeCapacity(steps); - operands.appendSliceAssumeCapacity(inits); - - const seg_sizes = [4]i32{ - @intCast(lbs.len), - @intCast(ubs.len), - @intCast(steps.len), - @intCast(inits.len), - }; - return mlir.Operation.make(ctx, "scf.parallel", .{ - .operands = .{ .flat = operands.constSlice() }, - .results = .{ .flat = result_types }, - .blocks = &.{body_block}, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - }, - .verify = false, - .location = location, - }); -} - -/// scf.reduce — terminator of `scf.parallel`. Each region is a reduction body -/// taking two args (lhs, rhs) matching the corresponding operand's type, -/// terminated with `scf.reduce.return`. -pub fn reduce( - ctx: *mlir.Context, - operands: []const *const mlir.Value, - reduction_blocks: []const *mlir.Block, - location: *const mlir.Location, -) *mlir.Operation { - _ = ctx; - std.debug.assert(operands.len == reduction_blocks.len); - var state: mlir.OperationState = .init("scf.reduce", location); - state.addOperands(operands); - - var regions: stdx.BoundedArray(*mlir.Region, 16) = .empty; - for (reduction_blocks) |b| { - const r = mlir.Region.init(); - r.appendOwnedBlock(b); - regions.appendAssumeCapacity(r); - } - state.addOwnedRegions(regions.constSlice()); - - return mlir.Operation.init(&state) catch @panic("Failed to create scf.reduce"); -} - -pub fn reduce_return(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "scf.reduce.return", .{ - .operands = .{ .flat = &.{value} }, - .verify = false, - .location = location, - }); -} - -/// scf.forall — multi-dim parallel region, similar to `scf.parallel` but using -/// tensor-based `shared_outs` instead of init values. This wrapper builds the -/// fully-dynamic form (all lb/ub/step are runtime values). -/// -/// `body_block` must have `rank + shared_outs.len` arguments (one index per -/// loop dim, plus one per shared_out) and must be terminated with -/// `scf.forall.in_parallel`. -pub fn forall( - ctx: *mlir.Context, - lbs: []const *const mlir.Value, - ubs: []const *const mlir.Value, - steps: []const *const mlir.Value, - shared_outs: []const *const mlir.Value, - body_block: *mlir.Block, - result_types: []const *const mlir.Type, - opts: struct { - mapping: ?*const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - std.debug.assert(lbs.len == ubs.len); - std.debug.assert(lbs.len == steps.len); - - var operands: stdx.BoundedArray(*const mlir.Value, 64) = .empty; - operands.appendSliceAssumeCapacity(lbs); - operands.appendSliceAssumeCapacity(ubs); - operands.appendSliceAssumeCapacity(steps); - operands.appendSliceAssumeCapacity(shared_outs); - - const seg_sizes = [4]i32{ - @intCast(lbs.len), - @intCast(ubs.len), - @intCast(steps.len), - @intCast(shared_outs.len), - }; - - // Fully-dynamic form: the static arrays use the kDynamic sentinel - // (mlir::ShapedType::kDynamic == INT64_MIN). We build one slot per loop - // dim, all dynamic. This matches how MLIR printers emit - // `scf.forall (%iv) = (%lb) to (%ub) step (%step)` when no static - // constants are folded in. - const k_dynamic: i64 = std.math.minInt(i64); - var sentinel: stdx.BoundedArray(i64, 16) = .empty; - for (0..lbs.len) |_| sentinel.appendAssumeCapacity(k_dynamic); - const sentinel_slice = sentinel.constSlice(); - - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "staticLowerBound", .denseArray(ctx, .i64, sentinel_slice)), - .named(ctx, "staticUpperBound", .denseArray(ctx, .i64, sentinel_slice)), - .named(ctx, "staticStep", .denseArray(ctx, .i64, sentinel_slice)), - }); - if (opts.mapping) |m| attrs.appendAssumeCapacity(.named(ctx, "mapping", m)); - - return mlir.Operation.make(ctx, "scf.forall", .{ - .operands = .{ .flat = operands.constSlice() }, - .results = .{ .flat = result_types }, - .blocks = &.{body_block}, - .attributes = attrs.constSlice(), - .verify = false, - .location = location, - }); -} - -/// scf.forall.in_parallel — terminator for scf.forall. The inner region is -/// a single block containing a flat list of ops (typically -/// tensor.parallel_insert_slice). `body_block` must be built by the caller. -pub fn forall_in_parallel( - ctx: *mlir.Context, - body_block: *mlir.Block, - location: *const mlir.Location, -) *mlir.Operation { - _ = ctx; - var state: mlir.OperationState = .init("scf.forall.in_parallel", location); - const region = mlir.Region.init(); - region.appendOwnedBlock(body_block); - state.addOwnedRegions(&.{region}); - return mlir.Operation.init(&state) catch @panic("Failed to create scf.forall.in_parallel"); -} - -/// scf.index_switch — branches to case_blocks[i] if arg == cases[i], else -/// falls through to default_block. Blocks must be terminated with scf.yield. -/// The op's results have the types yielded by each region. -pub fn index_switch( - ctx: *mlir.Context, - arg: *const mlir.Value, - cases: []const i64, - default_block: *mlir.Block, - case_blocks: []const *mlir.Block, - result_types: []const *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - std.debug.assert(cases.len == case_blocks.len); - var state: mlir.OperationState = .init("scf.index_switch", location); - state.addOperands(&.{arg}); - state.addResults(result_types); - state.addAttributes(&.{ - .named(ctx, "cases", .denseArray(ctx, .i64, cases)), - }); - - // First region is the default, then one per case — this is how - // `VariadicRegion>:$caseRegions` is positioned in the td. - const default_region = mlir.Region.init(); - default_region.appendOwnedBlock(default_block); - - var regions: stdx.BoundedArray(*mlir.Region, 64) = .empty; - regions.appendAssumeCapacity(default_region); - for (case_blocks) |b| { - const r = mlir.Region.init(); - r.appendOwnedBlock(b); - regions.appendAssumeCapacity(r); - } - state.addOwnedRegions(regions.constSlice()); - - return mlir.Operation.init(&state) catch @panic("Failed to create scf.index_switch"); -} diff --git a/mlir/dialects/scf/test.zig b/mlir/dialects/scf/test.zig deleted file mode 100644 index dceac66e..00000000 --- a/mlir/dialects/scf/test.zig +++ /dev/null @@ -1,39 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const scf = @import("mlir/dialects/scf"); - -const operation_names: []const []const u8 = &.{ - "scf.condition", - "scf.execute_region", - "scf.for", - "scf.forall", - "scf.forall.in_parallel", - "scf.if", - "scf.index_switch", - "scf.parallel", - "scf.reduce", - "scf.reduce.return", - "scf.while", - "scf.yield", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(scf); - inline for (comptime std.meta.declarations(scf)) |decl| { - const T = @field(scf, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - scf.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/shardy/BUILD.bazel b/mlir/dialects/shardy/BUILD.bazel deleted file mode 100644 index 4077f902..00000000 --- a/mlir/dialects/shardy/BUILD.bazel +++ /dev/null @@ -1,23 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "shardy", - import_name = "mlir/dialects/shardy", - main = "shardy.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//mlir", - "@shardy//shardy/integrations/c:sdy_capi", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":shardy", - "//mlir", - "//mlir/dialects/func", - ], -) diff --git a/mlir/dialects/shardy/shardy.zig b/mlir/dialects/shardy/shardy.zig deleted file mode 100644 index d73bf81b..00000000 --- a/mlir/dialects/shardy/shardy.zig +++ /dev/null @@ -1,364 +0,0 @@ -//! Bindings for OpenXLA shardy dialect -//! -//! - dialect overview: https://openxla.org/shardy/sdy_dialect -//! - C API: https://github.com/openxla/shardy/blob/main/shardy/integrations/c/attributes.h -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("sdy"); -} - -/// `sdy.sharding`, the sharding of a function argument or result. -pub fn shardingNamedAttr(ctx: *mlir.Context, sharding: *const TensorShardingAttr) mlir.NamedAttribute { - return .named(ctx, "sdy.sharding", sharding.attribute()); -} - -/// `sdy.mesh`. Not verified: it only verifies inside a module. -pub fn mesh(ctx: *mlir.Context, sym_name: []const u8, mesh_: *const MeshAttr, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "sdy.mesh", .{ - .attributes = &.{ - .named(ctx, "sym_name", .string(ctx, sym_name)), - .named(ctx, "mesh", mesh_.attribute()), - }, - .verify = false, - .location = location, - }); -} - -pub fn sharding_constraint(ctx: *mlir.Context, input: *const mlir.Value, sharding: *const TensorShardingAttr, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "sdy.sharding_constraint", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{input.type_()} }, - .attributes = &.{ - .named(ctx, "sharding", sharding.attribute()), - }, - .location = location, - }); -} - -pub fn manual_computation( - ctx: *mlir.Context, - tensors: []const *const mlir.Value, - result_types: []const *const mlir.Type, - body_block: *mlir.Block, - in_shardings: *const TensorShardingPerValueAttr, - out_shardings: *const TensorShardingPerValueAttr, - manual_axes: *const ManualAxesAttr, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "sdy.manual_computation", .{ - .operands = .{ .flat = tensors }, - .results = .{ .flat = result_types }, - .blocks = &.{body_block}, - .attributes = &.{ - .named(ctx, "in_shardings", in_shardings.attribute()), - .named(ctx, "out_shardings", out_shardings.attribute()), - .named(ctx, "manual_axes", manual_axes.attribute()), - }, - .location = location, - }); -} - -/// `sdy.return`. Not verified: a terminator only verifies inside its parent. -pub fn return_(ctx: *mlir.Context, results: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "sdy.return", .{ - .operands = .{ .flat = results }, - .verify = false, - .location = location, - }); -} - -pub const MeshAxisAttr = opaque { - const M = mlir.Methods(MeshAxisAttr, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsAMeshAxisAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - name: []const u8, - size: i64, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const MeshAxisAttr { - return @ptrCast(c.sdyMeshAxisAttrGet(ctx.ptr(), mlir.stringRef(args.name), args.size).ptr); - } - - pub fn attribute(self: *const MeshAxisAttr) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn name(self: *const MeshAxisAttr) []const u8 { - return mlir.string(c.sdyMeshAxisAttrGetName(self.ptr())); - } - - pub fn size(self: *const MeshAxisAttr) i64 { - return c.sdyMeshAxisAttrGetSize(self.ptr()); - } -}; - -pub const MeshAttr = opaque { - const M = mlir.Methods(MeshAttr, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsAMeshAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - axes: []const *const MeshAxisAttr, - device_ids: []const i64 = &.{}, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const MeshAttr { - const axes: []const c.struct_MlirAttribute = @ptrCast(args.axes); - return @ptrCast(c.sdyMeshAttrGet( - ctx.ptr(), - @intCast(axes.len), - axes.ptr, - @intCast(args.device_ids.len), - args.device_ids.ptr, - ).ptr); - } - - pub fn attribute(self: *const MeshAttr) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -pub const SubAxisInfoAttr = opaque { - const M = mlir.Methods(SubAxisInfoAttr, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsASubAxisInfoAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - pre_size: i64, - size: i64, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const SubAxisInfoAttr { - return @ptrCast(c.sdySubAxisInfoAttrGet(ctx.ptr(), args.pre_size, args.size).ptr); - } - - pub fn attribute(self: *const SubAxisInfoAttr) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn preSize(self: *const SubAxisInfoAttr) i64 { - return c.sdySubAxisInfoAttrGetPreSize(self.ptr()); - } - - pub fn size(self: *const SubAxisInfoAttr) i64 { - return c.sdySubAxisInfoAttrGetSize(self.ptr()); - } -}; - -pub const AxisRefAttr = opaque { - const M = mlir.Methods(AxisRefAttr, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsAnAxisRefAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn named(ctx: *mlir.Context, axis_name: []const u8) *const AxisRefAttr { - return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), .{ .ptr = null }).ptr); - } - - pub fn subAxis(ctx: *mlir.Context, axis_name: []const u8, pre_size: i64, size: i64) *const AxisRefAttr { - const sub_axis: *const SubAxisInfoAttr = .get(ctx, .{ .pre_size = pre_size, .size = size }); - return @ptrCast(c.sdyAxisRefAttrGet(ctx.ptr(), mlir.stringRef(axis_name), sub_axis.ptr()).ptr); - } - - pub fn attribute(self: *const AxisRefAttr) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn name(self: *const AxisRefAttr) []const u8 { - return mlir.string(c.sdyAxisRefAttrGetName(self.ptr())); - } - - pub fn subAxisInfo(self: *const AxisRefAttr) ?*const SubAxisInfoAttr { - const attr_ = c.sdyAxisRefAttrGetSubAxisInfo(self.ptr()); - return if (attr_.ptr == null) null else @ptrCast(attr_.ptr); - } -}; - -pub const DimensionShardingAttr = opaque { - const M = mlir.Methods(DimensionShardingAttr, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsADimensionShardingAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - axes: []const *const AxisRefAttr, - is_closed: bool, - priority: ?i64 = null, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const DimensionShardingAttr { - const attrs: []const c.struct_MlirAttribute = @ptrCast(args.axes); - return @ptrCast(c.sdyDimensionShardingAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr, args.is_closed, args.priority orelse -1).ptr); - } - - pub fn closed(ctx: *mlir.Context, axes: []const *const AxisRefAttr) *const DimensionShardingAttr { - return get(ctx, .{ .axes = axes, .is_closed = true }); - } - - pub fn open(ctx: *mlir.Context, axes: []const *const AxisRefAttr) *const DimensionShardingAttr { - return get(ctx, .{ .axes = axes, .is_closed = false }); - } - - pub fn replicated(ctx: *mlir.Context) *const DimensionShardingAttr { - return closed(ctx, &.{}); - } - - pub fn attribute(self: *const DimensionShardingAttr) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn numAxes(self: *const DimensionShardingAttr) usize { - return @intCast(c.sdyDimensionShardingAttrGetAxesSize(self.ptr())); - } - - pub fn axis(self: *const DimensionShardingAttr, index: usize) *const AxisRefAttr { - return @ptrCast(c.sdyDimensionShardingAttrGetAxesElem(self.ptr(), @intCast(index)).ptr); - } - - pub fn isClosed(self: *const DimensionShardingAttr) bool { - return c.sdyDimensionShardingAttrGetIsClosed(self.ptr()); - } - - pub fn priority(self: *const DimensionShardingAttr) ?i64 { - const value = c.sdyDimensionShardingAttrGetPriority(self.ptr()); - return if (value == -1) null else value; - } -}; - -pub const TensorShardingAttr = opaque { - const M = mlir.Methods(TensorShardingAttr, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsATensorShardingAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - mesh: []const u8, - dimensions: []const *const DimensionShardingAttr, - replicated_axes: []const *const AxisRefAttr = &.{}, - unreduced_axes: []const *const AxisRefAttr = &.{}, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const TensorShardingAttr { - comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.Attribute)); - const dimensions: []const c.struct_MlirAttribute = @ptrCast(args.dimensions); - const replicated: []const c.struct_MlirAttribute = @ptrCast(args.replicated_axes); - const unreduced: []const c.struct_MlirAttribute = @ptrCast(args.unreduced_axes); - return @ptrCast(c.sdyTensorShardingAttrGet( - ctx.ptr(), - mlir.Attribute.flatSymbolRef(ctx, args.mesh).ptr(), - @intCast(dimensions.len), - dimensions.ptr, - @intCast(replicated.len), - replicated.ptr, - @intCast(unreduced.len), - unreduced.ptr, - 0, - ).ptr); - } - - pub fn attribute(self: *const TensorShardingAttr) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn meshOrRef(self: *const TensorShardingAttr) *const mlir.Attribute { - return @ptrCast(c.sdyTensorShardingAttrGetMeshOrRef(self.ptr()).ptr); - } - - pub fn numDimensions(self: *const TensorShardingAttr) usize { - return @intCast(c.sdyTensorShardingAttrGetDimShardingsSize(self.ptr())); - } - - pub fn dimension(self: *const TensorShardingAttr, index: usize) *const DimensionShardingAttr { - return @ptrCast(c.sdyTensorShardingAttrGetDimShardingsElem(self.ptr(), @intCast(index)).ptr); - } - - pub fn numReplicatedAxes(self: *const TensorShardingAttr) usize { - return @intCast(c.sdyTensorShardingAttrGetReplicatedAxesSize(self.ptr())); - } - - pub fn replicatedAxis(self: *const TensorShardingAttr, index: usize) *const AxisRefAttr { - return @ptrCast(c.sdyTensorShardingAttrGetReplicatedAxesElem(self.ptr(), @intCast(index)).ptr); - } - - pub fn numUnreducedAxes(self: *const TensorShardingAttr) usize { - return @intCast(c.sdyTensorShardingAttrGetUnreducedAxesSize(self.ptr())); - } - - pub fn unreducedAxis(self: *const TensorShardingAttr, index: usize) *const AxisRefAttr { - return @ptrCast(c.sdyTensorShardingAttrGetUnreducedAxesElem(self.ptr(), @intCast(index)).ptr); - } -}; - -pub const TensorShardingPerValueAttr = opaque { - const M = mlir.Methods(TensorShardingPerValueAttr, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsATensorShardingPerValueAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn get(ctx: *mlir.Context, shardings: []const *const TensorShardingAttr) *const TensorShardingPerValueAttr { - const attrs: []const c.struct_MlirAttribute = @ptrCast(shardings); - return @ptrCast(c.sdyTensorShardingPerValueAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); - } - - pub fn attribute(self: *const TensorShardingPerValueAttr) *const mlir.Attribute { - return @ptrCast(self); - } -}; - -pub const ManualAxesAttr = opaque { - const M = mlir.Methods(ManualAxesAttr, c.MlirAttribute); - - pub const isAFn = c.sdyAttributeIsAManualAxesAttr; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - - pub fn get(ctx: *mlir.Context, axes: []const *const mlir.StringAttribute) *const ManualAxesAttr { - comptime std.debug.assert(@sizeOf(c.struct_MlirAttribute) == @sizeOf(*const mlir.StringAttribute)); - const attrs: []const c.struct_MlirAttribute = @ptrCast(axes); - return @ptrCast(c.sdyManualAxesAttrGet(ctx.ptr(), @intCast(attrs.len), attrs.ptr).ptr); - } - - pub fn attribute(self: *const ManualAxesAttr) *const mlir.Attribute { - return @ptrCast(self); - } - - pub fn numAxes(self: *const ManualAxesAttr) usize { - return @intCast(c.sdyManualAxesAttrGetAxesSize(self.ptr())); - } - - pub fn axis(self: *const ManualAxesAttr, index: usize) []const u8 { - return mlir.string(c.sdyManualAxesAttrGetAxesElem(self.ptr(), @intCast(index))); - } -}; diff --git a/mlir/dialects/shardy/test.zig b/mlir/dialects/shardy/test.zig deleted file mode 100644 index a4a757d2..00000000 --- a/mlir/dialects/shardy/test.zig +++ /dev/null @@ -1,200 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const shardy = @import("mlir/dialects/shardy"); - -const ManualAxesAttr = shardy.ManualAxesAttr; -const TensorShardingAttr = shardy.TensorShardingAttr; -const TensorShardingPerValueAttr = shardy.TensorShardingPerValueAttr; - -const operation_names: []const []const u8 = &.{ - "sdy.manual_computation", - "sdy.mesh", - "sdy.return", - "sdy.sharding_constraint", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(shardy); - inline for (comptime std.meta.declarations(shardy)) |decl| { - const T = @field(shardy, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - shardy.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} - -test TensorShardingAttr { - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - shardy.registerDialect(registry); - func.registerExtensions(registry); - - var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - - var w: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer w.deinit(); - - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttr = .get(ctx, .{ - .mesh = "folded_mesh", - .dimensions = &.{.replicated(ctx)}, - .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, - }); - try sharding.attribute().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{}], replicated={\"link_x\", \"link_y\"}>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttr = .get(ctx, .{ - .mesh = "suggested_mesh", - .dimensions = &.{ - .closed(ctx, &.{.named(ctx, "link_z")}), - .closed(ctx, &.{.named(ctx, "link_x")}), - }, - .replicated_axes = &.{.named(ctx, "link_y")}, - }); - try sharding.attribute().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@suggested_mesh, [{\"link_z\"}, {\"link_x\"}], replicated={\"link_y\"}>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttr = .get(ctx, .{ - .mesh = "fold_mesh", - .dimensions = &.{ .closed(ctx, &.{.named(ctx, "link_x")}), .replicated(ctx) }, - }); - try sharding.attribute().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@fold_mesh, [{\"link_x\"}, {}]>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttr = .get(ctx, .{ - .mesh = "folded_mesh", - .dimensions = &.{.closed(ctx, &.{ .named(ctx, "link_x"), .named(ctx, "link_y") })}, - }); - try sharding.attribute().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@folded_mesh, [{\"link_x\", \"link_y\"}]>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttr = .get(ctx, .{ - .mesh = "strategy_fold", - .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, - .replicated_axes = &.{.named(ctx, "link_y")}, - }); - try sharding.attribute().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@strategy_fold, [{\"link_x\"}], replicated={\"link_y\"}>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttr = .get(ctx, .{ - .mesh = "mix_mesh", - .dimensions = &.{ .open(ctx, &.{}), .replicated(ctx) }, - .replicated_axes = &.{ .named(ctx, "link_x"), .named(ctx, "link_y") }, - }); - try sharding.attribute().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@mix_mesh, [{?}, {}], replicated={\"link_x\", \"link_y\"}>", w.written()); - } - { - defer w.clearRetainingCapacity(); - const sharding: *const TensorShardingAttr = .get(ctx, .{ - .mesh = "3d_mesh", - .dimensions = &.{ - .closed(ctx, &.{.named(ctx, "link_x")}), - .closed(ctx, &.{.named(ctx, "link_y")}), - .closed(ctx, &.{.named(ctx, "link_z")}), - }, - }); - try sharding.attribute().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy.sharding<@\"3d_mesh\", [{\"link_x\"}, {\"link_y\"}, {\"link_z\"}]>", w.written()); - } -} - -test TensorShardingPerValueAttr { - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - shardy.registerDialect(registry); - func.registerExtensions(registry); - - var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - - const shardings: [2]*const TensorShardingAttr = .{ - .get(ctx, .{ - .mesh = "mesh", - .dimensions = &.{.closed(ctx, &.{.named(ctx, "link_x")})}, - }), - .get(ctx, .{ - .mesh = "mesh", - .dimensions = &.{.replicated(ctx)}, - .replicated_axes = &.{.named(ctx, "link_y")}, - }), - }; - - const per_value = TensorShardingPerValueAttr.get(ctx, &shardings); - - var w: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer w.deinit(); - - try per_value.attribute().format(&w.writer); - try std.testing.expectEqualSlices( - u8, - "#sdy.sharding_per_value<[<@mesh, [{\"link_x\"}]>, <@mesh, [{}], replicated={\"link_y\"}>]>", - w.written(), - ); -} - -test ManualAxesAttr { - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - shardy.registerDialect(registry); - func.registerExtensions(registry); - - var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - - const axes: [2]*const mlir.StringAttribute = .{ - .init(ctx, "link_x"), - .init(ctx, "link_y"), - }; - - const manual_axes: *const ManualAxesAttr = .get(ctx, &axes); - - try std.testing.expectEqual(2, manual_axes.numAxes()); - try std.testing.expectEqualStrings("link_x", manual_axes.axis(0)); - try std.testing.expectEqualStrings("link_y", manual_axes.axis(1)); - - var w: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer w.deinit(); - - try manual_axes.attribute().format(&w.writer); - try std.testing.expectEqualSlices(u8, "#sdy", w.written()); -} - -test "sub-axis references build" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - shardy.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - const ref = shardy.AxisRefAttr.subAxis(ctx, "x", 2, 4); - try std.testing.expectEqualStrings("x", ref.name()); - try std.testing.expectEqual(@as(i64, 4), ref.subAxisInfo().?.size()); -} diff --git a/mlir/dialects/stablehlo/BUILD.bazel b/mlir/dialects/stablehlo/BUILD.bazel index d7214000..45e69239 100644 --- a/mlir/dialects/stablehlo/BUILD.bazel +++ b/mlir/dialects/stablehlo/BUILD.bazel @@ -4,10 +4,10 @@ zig_library( name = "stablehlo", import_name = "mlir/dialects/stablehlo", main = "stablehlo.zig", - tags = ["manual"], - visibility = ["//visibility:public"], + visibility = ["//mlir/dialects:__subpackages__"], deps = [ "//mlir", + "//mlir:c", "//stdx", "@stablehlo//:stablehlo_dialect_capi", ], @@ -15,9 +15,5 @@ zig_library( zig_test( name = "test", - main = "test.zig", - deps = [ - ":stablehlo", - "//mlir", - ], + deps = [":stablehlo"], ) diff --git a/mlir/dialects/stablehlo/stablehlo.zig b/mlir/dialects/stablehlo/stablehlo.zig index 385a95b0..a21eec76 100644 --- a/mlir/dialects/stablehlo/stablehlo.zig +++ b/mlir/dialects/stablehlo/stablehlo.zig @@ -5,10 +5,6 @@ const mlir = @import("mlir"); const stringRef = mlir.stringRef; const stdx = @import("stdx"); -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("stablehlo"); -} - pub const abs = unaryOp("stablehlo.abs").call; pub const cosine = unaryOp("stablehlo.cosine").call; pub const sine = unaryOp("stablehlo.sine").call; @@ -97,7 +93,7 @@ pub fn cholesky(ctx: *mlir.Context, value: *const mlir.Value, lower: bool, locat .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "lower", .int(ctx, .i1, @intFromBool(lower))), + .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(lower))), }, .location = location, }); @@ -111,41 +107,13 @@ pub fn clamp(ctx: *mlir.Context, min: *const mlir.Value, value: *const mlir.Valu }); } -pub const CompositeOpts = struct { - name: []const u8, - decomposition: []const u8, - composite_attributes: []const mlir.NamedAttribute = &.{}, - version: i32 = 0, -}; - -pub fn composite( - ctx: *mlir.Context, - inputs: []const *const mlir.Value, - result_types: []const *const mlir.Type, - opts: CompositeOpts, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.composite", .{ - .operands = .{ .flat = inputs }, - .results = .{ .flat = result_types }, - .attributes = &.{ - .named(ctx, "name", .string(ctx, opts.name)), - .named(ctx, "decomposition", .flatSymbolRef(ctx, opts.decomposition)), - .named(ctx, "composite_attributes", .dict(ctx, opts.composite_attributes)), - .named(ctx, "version", .int(ctx, .i32, opts.version)), - }, - .verify = false, - .location = location, - }); -} - pub const DotPrecision = union(enum) { fast, high, highest, algorithm: DotAlgorithm, - pub fn precision(self: DotPrecision) PrecisionAttr.Precision { + pub fn precision(self: DotPrecision) PrecisionAttribute.Precision { return switch (self) { .fast => .DEFAULT, .high => .HIGH, @@ -155,9 +123,9 @@ pub const DotPrecision = union(enum) { }; } - pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, element_type: *const mlir.Type) ?*const mlir.Attribute { + pub fn algorithmAttr(self: DotPrecision, ctx: *mlir.Context, operand_type: mlir.RankedTensorType) ?mlir.Attribute { return switch (self) { - .algorithm => |algo| dotAlgorithmAttr(ctx, algo, element_type), + .algorithm => |algo| algo.asAttr(ctx, operand_type), else => null, }; } @@ -181,12 +149,12 @@ pub const DotAlgorithm = struct { }; }; -pub fn dotAlgorithmAttr(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, element_type: *const mlir.Type) *const mlir.Attribute { +pub fn dotAlgorithmAttribute(ctx: *mlir.Context, dot_algorithm: DotAlgorithm, element_type: *const mlir.Type) *const mlir.Attribute { return @ptrCast(c.stablehloDotAlgorithmGet( ctx.ptr(), element_type.ptr(), element_type.ptr(), - mlir.Type.float(ctx, dot_algorithm.accumulation).ptr(), + mlir.floatType(ctx, dot_algorithm.accumulation).ptr(), dot_algorithm.component_count, dot_algorithm.component_count, dot_algorithm.num_primitive_operations, @@ -210,22 +178,22 @@ pub fn dot_general( }, location: *const mlir.Location, ) *mlir.Operation { - const precisions: [2]*const mlir.Attribute = @splat(precisionAttr(ctx, opts.dot_precision.precision())); + const precisions: [2]*const mlir.Attribute = @splat(precisionAttribute(ctx, opts.dot_precision.precision())); const attributes = [3]mlir.NamedAttribute{ .named( ctx, "dot_dimension_numbers", - dotDimensionNumbersAttr(ctx, .{ + dotDimensionNumbersAttribute(ctx, .{ .lhs_batching_dimensions = opts.lhs_batching_dimensions, .rhs_batching_dimensions = opts.rhs_batching_dimensions, .lhs_contracting_dimensions = opts.lhs_contracting_dimensions, .rhs_contracting_dimensions = opts.rhs_contracting_dimensions, }), ), - .named(ctx, "precision_config", .array(ctx, &precisions)), + .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &precisions)), // keep algorithm as the last attribute so we can omit it when it's not set. .named(ctx, "algorithm", switch (opts.dot_precision) { - .algorithm => |v| dotAlgorithmAttr(ctx, v, lhs.type_()), + .algorithm => |v| dotAlgorithmAttribute(ctx, v, lhs.type_()), else => undefined, }), }; @@ -247,9 +215,9 @@ pub fn constant( ) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.constant", .{ .operands = .{ .flat = &.{} }, - .results = .{ .flat = &.{.rankedTensor(dims, elem_type)} }, + .results = .{ .flat = &.{mlir.rankedTensorType(dims, elem_type)} }, .attributes = &.{ - .named(ctx, "value", .denseElements(.rankedTensor(dims, elem_type), raw_bytes)), + .named(ctx, "value", mlir.denseElementsAttribute(mlir.RankedTensorType.get(dims, elem_type, null).shaped(), raw_bytes)), }, .location = location, }); @@ -266,7 +234,7 @@ pub fn broadcast_in_dim( .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "broadcast_dimensions", .denseArray(ctx, .i64, dims)), + .named(ctx, "broadcast_dimensions", mlir.denseArrayAttribute(ctx, .i64, dims)), }, .location = location, }); @@ -283,7 +251,7 @@ pub fn transpose( .operands = .{ .flat = &.{value} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "permutation", .denseArray(ctx, .i64, opts.permutation)), + .named(ctx, "permutation", mlir.denseArrayAttribute(ctx, .i64, opts.permutation)), }, .location = location, }); @@ -302,9 +270,9 @@ pub fn slice( .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "start_indices", .denseArray(ctx, .i64, start_indices)), - .named(ctx, "limit_indices", .denseArray(ctx, .i64, limit_indices)), - .named(ctx, "strides", .denseArray(ctx, .i64, strides)), + .named(ctx, "start_indices", mlir.denseArrayAttribute(ctx, .i64, start_indices)), + .named(ctx, "limit_indices", mlir.denseArrayAttribute(ctx, .i64, limit_indices)), + .named(ctx, "strides", mlir.denseArrayAttribute(ctx, .i64, strides)), }, .location = location, }); @@ -315,7 +283,7 @@ pub fn concatenate(ctx: *mlir.Context, inputs: []const *const mlir.Value, dimens .operands = .{ .flat = inputs }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "dimension", .int(ctx, .i64, dimension)), + .named(ctx, "dimension", mlir.integerAttribute(ctx, .i64, dimension)), }, .location = location, }); @@ -358,7 +326,7 @@ pub fn gather( .operands = .{ .flat = &.{ value, indices } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "dimension_numbers", gatherDimensionNumbersAttr( + .named(ctx, "dimension_numbers", gatherDimensionNumbersAttribute( ctx, args.offset_dims, args.collapsed_slice_dims, @@ -367,8 +335,8 @@ pub fn gather( args.start_index_map, args.index_vector_dim, )), - .named(ctx, "slice_sizes", .denseArray(ctx, .i64, slice_sizes)), - .named(ctx, "indices_are_sorted", .boolean(ctx, args.indices_are_sorted)), + .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, slice_sizes)), + .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), }, .location = location, }, @@ -385,7 +353,7 @@ pub const ScatterArgs = struct { indices_are_sorted: bool = false, unique_indices: bool = false, - pub fn scatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { + pub fn getScatterDimensionNumbers(self: ScatterArgs, ctx: *mlir.Context) *const mlir.Attribute { return @ptrCast(c.stablehloScatterDimensionNumbersGet( ctx.ptr(), @intCast(self.update_window_dims.len), @@ -419,9 +387,9 @@ pub fn scatter( .operands = .{ .variadic = &.{ inputs, scatter_indices, updates } }, .blocks = &.{update_block}, .attributes = &.{ - .named(ctx, "scatter_dimension_numbers", args.scatterDimensionNumbers(ctx)), - .named(ctx, "indices_are_sorted", .boolean(ctx, args.indices_are_sorted)), - .named(ctx, "unique_indices", .boolean(ctx, args.unique_indices)), + .named(ctx, "scatter_dimension_numbers", args.getScatterDimensionNumbers(ctx)), + .named(ctx, "indices_are_sorted", mlir.boolAttribute(ctx, args.indices_are_sorted)), + .named(ctx, "unique_indices", mlir.boolAttribute(ctx, args.unique_indices)), }, .result_type_inference = true, .location = location, @@ -434,7 +402,7 @@ pub fn iota(ctx: *mlir.Context, dimension: i64, result_type: *const mlir.Type, l .operands = .{ .flat = &.{} }, .results = .{ .flat = &.{result_type} }, .attributes = &.{ - .named(ctx, "iota_dimension", .int(ctx, .i64, dimension)), + .named(ctx, "iota_dimension", mlir.integerAttribute(ctx, .i64, dimension)), }, .location = location, }); @@ -445,7 +413,7 @@ pub fn reverse(ctx: *mlir.Context, operand: *const mlir.Value, dimensions: []con .operands = .{ .flat = &.{operand} }, .results = .{ .flat = &.{operand.type_()} }, .attributes = &.{ - .named(ctx, "dimensions", .denseArray(ctx, .i64, dimensions)), + .named(ctx, "dimensions", mlir.denseArrayAttribute(ctx, .i64, dimensions)), }, .location = location, }); @@ -455,16 +423,16 @@ pub fn compare( ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, - comparison_direction: ComparisonDirectionAttr.Direction, - compare_type: CompareTypeAttr.Type, + comparison_direction: ComparisonDirection.Direction, + compare_type: CompareType.Type, location: *const mlir.Location, ) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.compare", .{ .operands = .{ .flat = &.{ lhs, rhs } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "comparison_direction", comparisonDirectionAttr(ctx, comparison_direction)), - .named(ctx, "compare_type", compareTypeAttr(ctx, compare_type)), + .named(ctx, "comparison_direction", comparisonDirection(ctx, comparison_direction)), + .named(ctx, "compare_type", compareType(ctx, compare_type)), }, .location = location, }); @@ -481,7 +449,7 @@ pub fn dynamic_slice( .operands = .{ .variadic = &.{ &.{operand}, start_indices } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "slice_sizes", .denseArray(ctx, .i64, new_dims)), + .named(ctx, "slice_sizes", mlir.denseArrayAttribute(ctx, .i64, new_dims)), }, .location = location, }); @@ -493,14 +461,14 @@ pub const PadOpts = struct { interior: []const i64, }; -pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, opts: PadOpts, location: *const mlir.Location) *mlir.Operation { +pub fn pad(ctx: *mlir.Context, value: *const mlir.Value, padding_value: *const mlir.Value, location: *const mlir.Location, opts: PadOpts) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.pad", .{ .operands = .{ .flat = &.{ value, padding_value } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "edge_padding_low", .denseArray(ctx, .i64, opts.low)), - .named(ctx, "edge_padding_high", .denseArray(ctx, .i64, opts.high)), - .named(ctx, "interior_padding", .denseArray(ctx, .i64, opts.interior)), + .named(ctx, "edge_padding_low", mlir.denseArrayAttribute(ctx, .i64, opts.low)), + .named(ctx, "edge_padding_high", mlir.denseArrayAttribute(ctx, .i64, opts.high)), + .named(ctx, "interior_padding", mlir.denseArrayAttribute(ctx, .i64, opts.interior)), }, .location = location, }); @@ -510,7 +478,7 @@ pub const TriangularSolveOpts = struct { left_side: bool, lower: bool, unit_diagonal: bool, - transpose_a: TransposeAttr.Type, + transpose_a: TransposeAttribute.Type, }; pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *const mlir.Value, opts: TriangularSolveOpts, location: *const mlir.Location) *mlir.Operation { @@ -518,17 +486,17 @@ pub fn triangular_solve(ctx: *mlir.Context, value: *const mlir.Value, other: *co .operands = .{ .flat = &.{ value, other } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "left_side", .int(ctx, .i1, @intFromBool(opts.left_side))), - .named(ctx, "lower", .int(ctx, .i1, @intFromBool(opts.lower))), - .named(ctx, "unit_diagonal", .int(ctx, .i1, @intFromBool(opts.unit_diagonal))), - .named(ctx, "transpose_a", transposeAttr(ctx, opts.transpose_a)), + .named(ctx, "left_side", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.left_side))), + .named(ctx, "lower", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.lower))), + .named(ctx, "unit_diagonal", mlir.integerAttribute(ctx, .i1, @intFromBool(opts.unit_diagonal))), + .named(ctx, "transpose_a", transposeAttribute(ctx, opts.transpose_a)), }, .location = location, }); } pub const FftOpts = struct { - kind: FftTypeAttr.Type, + kind: FftType.Type, length: []const i64, }; @@ -537,15 +505,15 @@ pub fn fft(ctx: *mlir.Context, value: *const mlir.Value, opts: FftOpts, location .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "fft_type", fftTypeAttr(ctx, opts.kind)), - .named(ctx, "fft_length", .denseArray(ctx, .i64, opts.length)), + .named(ctx, "fft_type", fftType(ctx, opts.kind)), + .named(ctx, "fft_length", mlir.denseArrayAttribute(ctx, .i64, opts.length)), }, .location = location, }); } -pub const RngDistributionAttr = opaque { - const M = mlir.Methods(RngDistributionAttr, c.MlirAttribute); +pub const RngDistribution = opaque { + const M = mlir.Methods(RngDistribution, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsARngDistributionAttr; pub const ptr = M.ptr; @@ -557,32 +525,24 @@ pub const RngDistributionAttr = opaque { NORMAL, }; - pub const InitArgs = struct { - value: Type, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const RngDistributionAttr { - return @ptrCast(c.stablehloRngDistributionAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); + pub fn get(ctx: *mlir.Context, value_: Type) *const RngDistribution { + return @ptrCast(c.stablehloRngDistributionAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); } - pub fn value(self: *const RngDistributionAttr) Type { + pub fn value(self: *const RngDistribution) Type { return std.meta.stringToEnum( Type, mlir.string(c.stablehloRngDistributionAttrGetValue(self.ptr())), ) orelse unreachable; } - - pub fn attribute(self: *const RngDistributionAttr) *const mlir.Attribute { - return @ptrCast(self); - } }; -pub fn rngDistributionAttr(ctx: *mlir.Context, value: RngDistributionAttr.Type) *const mlir.Attribute { - return RngDistributionAttr.get(ctx, .{ .value = value }).attribute(); +pub fn rngDistribution(ctx: *mlir.Context, value: RngDistribution.Type) *const mlir.Attribute { + return @ptrCast(RngDistribution.get(ctx, value)); } -pub const RngAlgorithmAttr = opaque { - const M = mlir.Methods(RngAlgorithmAttr, c.MlirAttribute); +pub const RngAlgorithm = opaque { + const M = mlir.Methods(RngAlgorithm, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsARngAlgorithmAttr; pub const ptr = M.ptr; @@ -595,36 +555,28 @@ pub const RngAlgorithmAttr = opaque { PHILOX, }; - pub const InitArgs = struct { - value: Type, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const RngAlgorithmAttr { - return @ptrCast(c.stablehloRngAlgorithmAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); + pub fn get(ctx: *mlir.Context, value_: Type) *const RngAlgorithm { + return @ptrCast(c.stablehloRngAlgorithmAttrGet(ctx.ptr(), stringRef(@tagName(value_))).ptr); } - pub fn value(self: *const RngAlgorithmAttr) Type { + pub fn value(self: *const RngAlgorithm) Type { return std.meta.stringToEnum( Type, mlir.string(c.stablehloRngAlgorithmAttrGetValue(self.ptr())), ) orelse unreachable; } - - pub fn attribute(self: *const RngAlgorithmAttr) *const mlir.Attribute { - return @ptrCast(self); - } }; -pub fn rngAlgorithmAttr(ctx: *mlir.Context, value: RngAlgorithmAttr.Type) *const mlir.Attribute { - return RngAlgorithmAttr.get(ctx, .{ .value = value }).attribute(); +pub fn rngAlgorithm(ctx: *mlir.Context, value: RngAlgorithm.Type) *const mlir.Attribute { + return @ptrCast(RngAlgorithm.get(ctx, value)); } -pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistributionAttr.Type, location: *const mlir.Location) *mlir.Operation { +pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape: *const mlir.Value, rng_distribution: RngDistribution.Type, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.rng", .{ .operands = .{ .flat = &.{ a, b, shape } }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "rng_distribution", rngDistributionAttr(ctx, rng_distribution)), + .named(ctx, "rng_distribution", rngDistribution(ctx, rng_distribution)), }, .location = location, }); @@ -632,7 +584,7 @@ pub fn rng(ctx: *mlir.Context, a: *const mlir.Value, b: *const mlir.Value, shape pub fn rng_bit_generator( ctx: *mlir.Context, - rng_algorithm: RngAlgorithmAttr.Type, + rng_algorithm: RngAlgorithm.Type, initial_state: *const mlir.Value, res_state_type: *const mlir.Type, res_type: *const mlir.Type, @@ -642,7 +594,7 @@ pub fn rng_bit_generator( .operands = .{ .flat = &.{initial_state} }, .results = .{ .flat = &.{ res_state_type, res_type } }, .attributes = &.{ - .named(ctx, "rng_algorithm", rngAlgorithmAttr(ctx, rng_algorithm)), + .named(ctx, "rng_algorithm", rngAlgorithm(ctx, rng_algorithm)), }, .location = location, }); @@ -653,8 +605,8 @@ pub fn reduce_precision(ctx: *mlir.Context, value: *const mlir.Value, exponent_b .operands = .{ .flat = &.{value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "exponent_bits", .int(ctx, .i32, exponent_bits)), - .named(ctx, "mantissa_bits", .int(ctx, .i32, mantissa_bits)), + .named(ctx, "exponent_bits", mlir.integerAttribute(ctx, .i32, exponent_bits)), + .named(ctx, "mantissa_bits", mlir.integerAttribute(ctx, .i32, mantissa_bits)), }, .location = location, }); @@ -668,126 +620,12 @@ pub fn dynamic_update_slice(ctx: *mlir.Context, operand: *const mlir.Value, upda }); } -pub fn while_( - ctx: *mlir.Context, - operands: []const *const mlir.Value, - result_types: []const *const mlir.Type, - cond_block: *mlir.Block, - body_block: *mlir.Block, - location: *const mlir.Location, -) *mlir.Operation { - _ = ctx; - var state = mlir.OperationState.init("stablehlo.while", location); - state.addOperands(operands); - state.addResults(result_types); - - const cond_region = mlir.Region.init(); - cond_region.appendOwnedBlock(cond_block); - const body_region = mlir.Region.init(); - body_region.appendOwnedBlock(body_block); - state.addOwnedRegions(&.{ cond_region, body_region }); - - return mlir.Operation.init(&state) catch @panic("Failed to create stablehlo.while operation"); -} - -/// `stablehlo.if`. Not verified: its branches may use values defined -/// outside them, which only verify once the operation is in place. -pub fn if_( - ctx: *mlir.Context, - pred: *const mlir.Value, - true_branch_block: *mlir.Block, - false_branch_block: *mlir.Block, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.if", .{ - .operands = .{ .flat = &.{pred} }, - .result_type_inference = true, - .blocks = &.{ true_branch_block, false_branch_block }, - .verify = false, - .location = location, - }); -} - -pub fn reduce( - ctx: *mlir.Context, - inputs: []const *const mlir.Value, - init_values: []const *const mlir.Value, - body_block: *mlir.Block, - dimensions: []const i64, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.reduce", .{ - .operands = .{ .variadic = &.{ inputs, init_values } }, - .result_type_inference = true, - .blocks = &.{body_block}, - .attributes = &.{ - .named(ctx, "dimensions", .denseArray(ctx, .i64, dimensions)), - }, - .location = location, - }); -} - -pub fn reduce_window( - ctx: *mlir.Context, - inputs: []const *const mlir.Value, - init_values: []const *const mlir.Value, - body_block: *mlir.Block, - window_dimensions: []const i64, - opts: struct { - window_strides: ?[]const i64 = null, - base_dilations: ?[]const i64 = null, - window_dilations: ?[]const i64 = null, - /// (low, high) padding of each dimension. - padding: ?[]const [2]i64 = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 5) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "window_dimensions", .denseArray(ctx, .i64, window_dimensions))); - if (opts.window_strides) |v| attrs.appendAssumeCapacity(.named(ctx, "window_strides", .denseArray(ctx, .i64, v))); - if (opts.base_dilations) |v| attrs.appendAssumeCapacity(.named(ctx, "base_dilations", .denseArray(ctx, .i64, v))); - if (opts.window_dilations) |v| attrs.appendAssumeCapacity(.named(ctx, "window_dilations", .denseArray(ctx, .i64, v))); - if (opts.padding) |v| attrs.appendAssumeCapacity(.named(ctx, "padding", .denseElements( - .rankedTensor(&.{ @intCast(v.len), 2 }, .int(ctx, .i64)), - @as([]const i64, @ptrCast(v)), - ))); - return mlir.Operation.make(ctx, "stablehlo.reduce_window", .{ - .operands = .{ .variadic = &.{ inputs, init_values } }, - .result_type_inference = true, - .blocks = &.{body_block}, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -pub fn sort( - ctx: *mlir.Context, - inputs: []const *const mlir.Value, - comparator_block: *mlir.Block, - opts: struct { - dimension: i64 = -1, - is_stable: bool = false, - }, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.sort", .{ - .operands = .{ .variadic = &.{inputs} }, - .result_type_inference = true, - .blocks = &.{comparator_block}, - .attributes = &.{ - .named(ctx, "dimension", .int(ctx, .i64, opts.dimension)), - .named(ctx, "is_stable", .boolean(ctx, opts.is_stable)), - }, - .location = location, - }); -} - pub fn get_tuple_element(ctx: *mlir.Context, tuple_value: *const mlir.Value, index: i64, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.get_tuple_element", .{ .operands = .{ .flat = &.{tuple_value} }, .result_type_inference = true, .attributes = &.{ - .named(ctx, "index", .int(ctx, .i32, index)), + .named(ctx, "index", mlir.integerAttribute(ctx, .i32, index)), }, .location = location, }); @@ -811,7 +649,7 @@ pub const ConvolutionOpts = struct { output_spatial_dimensions: []const i64, feature_group_count: i64, batch_group_count: i64, - precision_config: []const PrecisionAttr.Precision = &.{}, + precision_config: []const PrecisionAttribute.Precision = &.{}, }; pub fn convolution( @@ -824,7 +662,7 @@ pub fn convolution( ) *mlir.Operation { var max_precisions: [2]*const mlir.Attribute = undefined; for (opts.precision_config, 0..) |p, i| { - max_precisions[i] = precisionAttr(ctx, p); + max_precisions[i] = precisionAttribute(ctx, p); } var window_reversal: [3]i32 = undefined; for (opts.window_reversal, 0..) |w, i| { @@ -834,12 +672,12 @@ pub fn convolution( .operands = .{ .flat = &.{ lhs, rhs } }, .results = .{ .flat = &.{res_type} }, .attributes = &.{ - .named(ctx, "window_strides", .denseArray(ctx, .i64, opts.window_strides)), - .named(ctx, "padding", .denseElements(.rankedTensor(opts.pad_shape, .int(ctx, .i64)), opts.pad_value)), - .named(ctx, "lhs_dilation", .denseArray(ctx, .i64, opts.lhs_dilation)), - .named(ctx, "rhs_dilation", .denseArray(ctx, .i64, opts.rhs_dilation)), - .named(ctx, "window_reversal", .denseArray(ctx, .bool, window_reversal[0..opts.window_reversal.len])), - .named(ctx, "dimension_numbers", convDimensionNumbersAttr(ctx, .{ + .named(ctx, "window_strides", mlir.denseArrayAttribute(ctx, .i64, opts.window_strides)), + .named(ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(opts.pad_shape, mlir.integerType(ctx, .i64), null).shaped(), opts.pad_value)), + .named(ctx, "lhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.lhs_dilation)), + .named(ctx, "rhs_dilation", mlir.denseArrayAttribute(ctx, .i64, opts.rhs_dilation)), + .named(ctx, "window_reversal", mlir.denseArrayAttribute(ctx, .bool, window_reversal[0..opts.window_reversal.len])), + .named(ctx, "dimension_numbers", convDimensionNumbersAttribute(ctx, .{ .input_batch_dimension = opts.input_batch_dimension, .input_feature_dimension = opts.input_feature_dimension, .input_spatial_dimensions = opts.input_spatial_dimensions, @@ -850,18 +688,15 @@ pub fn convolution( .output_feature_dimension = opts.output_feature_dimension, .output_spatial_dimensions = opts.output_spatial_dimensions, })), - .named(ctx, "feature_group_count", .int(ctx, .i64, opts.feature_group_count)), - .named(ctx, "batch_group_count", .int(ctx, .i64, opts.batch_group_count)), - .named(ctx, "precision_config", .array(ctx, &max_precisions)), + .named(ctx, "feature_group_count", mlir.integerAttribute(ctx, .i64, opts.feature_group_count)), + .named(ctx, "batch_group_count", mlir.integerAttribute(ctx, .i64, opts.batch_group_count)), + .named(ctx, "precision_config", mlir.arrayAttribute(ctx, &max_precisions)), }, .location = location, }); } pub const CustomCallOpts = struct { - pub const MAX_OPERANDS = 64; - pub const MAX_RESULTS = 16; - pub const ApiVersion = enum(i32) { original = 1, status_returning = 2, @@ -876,24 +711,19 @@ pub const CustomCallOpts = struct { typed_ffi: *const mlir.Attribute, }; - pub const OutputOperandAlias = struct { - output_index: i64, - operand_index: i64, - }; - call_target_name: []const u8, has_side_effect: ?bool = null, api_version: ?ApiVersion = null, backend_config: ?BackendConfig = null, operand_layouts: ?[]const []const usize = null, result_layouts: ?[]const []const usize = null, - output_operand_aliases: ?[]const OutputOperandAlias = null, + output_operand_aliases: ?[]const i64 = null, additional_attributes: []const mlir.NamedAttribute = &.{}, }; pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, opts: CustomCallOpts, location: *const mlir.Location) *mlir.Operation { - const MAX_OPERANDS = CustomCallOpts.MAX_OPERANDS; - const MAX_RESULTS = CustomCallOpts.MAX_RESULTS; + const MAX_OPERANDS = 64; + const MAX_RESULTS = 16; const MINOR_TO_MAJOR = comptime blk: { var ret: [mlir.ShapedType.MAX_RANK]usize = undefined; for (0..ret.len) |i| { @@ -902,52 +732,56 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result break :blk ret; }; - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .empty; + var attrs: stdx.BoundedArray(mlir.NamedAttribute, 32) = .{}; attrs.appendSliceAssumeCapacity(&.{ - .named(ctx, "call_target_name", .string(ctx, opts.call_target_name)), + .named(ctx, "call_target_name", mlir.stringAttribute(ctx, opts.call_target_name)), }); attrs.appendSliceAssumeCapacity(opts.additional_attributes); if (opts.has_side_effect) |has_side_effects| { attrs.appendAssumeCapacity( - .named(ctx, "has_side_effect", .boolean(ctx, has_side_effects)), + .named(ctx, "has_side_effect", mlir.boolAttribute(ctx, has_side_effects)), ); } const backend_config_api_version: ?CustomCallOpts.ApiVersion = if (opts.backend_config) |bc| bc else null; if (opts.api_version orelse backend_config_api_version) |v| { attrs.appendAssumeCapacity( - .named(ctx, "api_version", .int(ctx, .i32, @backingInt(v))), + .named(ctx, "api_version", mlir.integerAttribute(ctx, .i32, @intFromEnum(v))), ); } if (opts.backend_config) |backend_config| { attrs.appendAssumeCapacity( .named(ctx, "backend_config", switch (backend_config) { .typed_ffi => |v| @ptrCast(v), - inline else => |v| .string(ctx, v), + inline else => |v| mlir.stringAttribute(ctx, v), }), ); } if (opts.output_operand_aliases) |output_operand_aliases| { - var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .empty; - for (output_operand_aliases) |alias| { - const output_tuple_indices = if (result_types.len > 1) &[1]i64{alias.output_index} else &.{}; + var buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_RESULTS) = .{}; + for (output_operand_aliases, 0..) |alias, output_index| { + const output_tuple_indices = if (result_types.len > 1) &[1]i64{@intCast(output_index)} else &.{}; buffer.appendAssumeCapacity( - outputOperandAliasAttr(ctx, .{ .operand_index = alias.operand_index, .output_tuple_indices = output_tuple_indices }), + outputOperandAliasAttribute(ctx, .{ .operand_index = alias, .output_tuple_indices = output_tuple_indices }), ); } attrs.appendAssumeCapacity( - .named(ctx, "output_operand_aliases", .array(ctx, buffer.constSlice())), + .named(ctx, "output_operand_aliases", mlir.arrayAttribute(ctx, buffer.constSlice())), ); } - var layouts_buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_OPERANDS) = .empty; + var layouts_buffer: stdx.BoundedArray(*const mlir.Attribute, MAX_OPERANDS) = .{}; if (opts.operand_layouts) |layouts| { for (layouts) |ol| { layouts_buffer.appendAssumeCapacity( - .denseElements( - .rankedTensor(&.{@intCast(ol.len)}, .index(ctx)), + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(ol.len)}, + mlir.indexType(ctx), + null, + ).shaped(), ol, ), ); @@ -957,23 +791,31 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result const shaped_type = input.type_().isA(mlir.ShapedType).?; const ol = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; layouts_buffer.appendAssumeCapacity( - .denseElements( - .rankedTensor(&.{@intCast(ol.len)}, .index(ctx)), + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(ol.len)}, + mlir.indexType(ctx), + null, + ).shaped(), ol, ), ); } } attrs.appendAssumeCapacity( - .named(ctx, "operand_layouts", .array(ctx, layouts_buffer.constSlice())), + .named(ctx, "operand_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), ); layouts_buffer.clear(); if (opts.result_layouts) |layouts| { for (layouts) |rl| { layouts_buffer.appendAssumeCapacity( - .denseElements( - .rankedTensor(&.{@intCast(rl.len)}, .index(ctx)), + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(rl.len)}, + mlir.indexType(ctx), + null, + ).shaped(), rl, ), ); @@ -983,15 +825,19 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result const shaped_type = result_type.isA(mlir.ShapedType).?; const rl = MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - shaped_type.rank() ..]; layouts_buffer.appendAssumeCapacity( - .denseElements( - .rankedTensor(&.{@intCast(rl.len)}, .index(ctx)), + mlir.denseElementsAttribute( + mlir.RankedTensorType.get( + &.{@intCast(rl.len)}, + mlir.indexType(ctx), + null, + ).shaped(), rl, ), ); } } attrs.appendAssumeCapacity( - .named(ctx, "result_layouts", .array(ctx, layouts_buffer.constSlice())), + .named(ctx, "result_layouts", mlir.arrayAttribute(ctx, layouts_buffer.constSlice())), ); return mlir.Operation.make(ctx, "stablehlo.custom_call", .{ @@ -1002,14 +848,6 @@ pub fn custom_call(ctx: *mlir.Context, inputs: []const *const mlir.Value, result }); } -pub fn optimizationBarrier(ctx: *mlir.Context, inputs: []const *const mlir.Value, result_types: []const *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.optimization_barrier", .{ - .operands = .{ .flat = inputs }, - .results = .{ .flat = result_types }, - .location = location, - }); -} - pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { return custom_call(ctx, &.{}, &.{value}, .{ .call_target_name = "CreateBuffer", @@ -1018,40 +856,28 @@ pub fn createBuffer(ctx: *mlir.Context, value: *const mlir.Type, location: *cons } pub fn pin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return custom_call( - ctx, - &.{value}, - &.{@ptrCast(mlir.MemRefType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, - .{ - .call_target_name = "Pin", - .api_version = .typed_ffi, - }, - location, - ); + return custom_call(ctx, &.{}, &.{@ptrCast(mlir.MemRefType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ + .call_target_name = "Pin", + .api_version = .typed_ffi, + }, location); } pub fn unpin(ctx: *mlir.Context, value: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return custom_call( - ctx, - &.{value}, - &.{@ptrCast(mlir.RankedTensorType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, - .{ - .call_target_name = "Unpin", - .api_version = .typed_ffi, - }, - location, - ); + return custom_call(ctx, &.{}, &.{@ptrCast(mlir.RankedTensorType.fromShaped(value.type_().isA(mlir.ShapedType).?))}, .{ + .call_target_name = "Unpin", + .api_version = .typed_ffi, + }, location); } pub fn partition_id(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { return mlir.Operation.make(ctx, "stablehlo.partition_id", .{ - .results = .{ .flat = &.{.rankedTensor(&.{}, .int(ctx, .u32))} }, + .results = .{ .flat = &.{mlir.rankedTensorType(&.{}, mlir.integerType(ctx, .u32))} }, .location = location, }); } -pub const DotDimensionNumbersAttr = opaque { - const M = mlir.Methods(DotDimensionNumbersAttr, c.MlirAttribute); +pub const DotDimensionNumbersAttribute = opaque { + const M = mlir.Methods(DotDimensionNumbersAttribute, mlir.Attribute); pub const isAFn = c.stablehloAttributeIsADotDimensionNumbers; pub const ptr = M.ptr; @@ -1065,7 +891,7 @@ pub const DotDimensionNumbersAttr = opaque { rhs_contracting_dimensions: []const i64, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) *const DotDimensionNumbersAttr { + pub fn init(ctx: *mlir.Context, args: InitArgs) *const DotDimensionNumbersAttribute { return @ptrCast(c.stablehloDotDimensionNumbersGet( ctx.ptr(), @intCast(args.lhs_batching_dimensions.len), @@ -1079,56 +905,52 @@ pub const DotDimensionNumbersAttr = opaque { ).ptr); } - pub fn lhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { + pub fn getLhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { return @intCast(c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsSize(self.ptr())); } - pub fn lhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { + pub fn getLhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetLhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn rhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { + pub fn getRhsBatchingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { return @intCast(c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsSize(self.ptr())); } - pub fn rhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { + pub fn getRhsBatchingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetRhsBatchingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn lhsContractingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { + pub fn getLhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { return @intCast(c.stablehloDotDimensionNumbersGetLhsContractingDimensionsSize(self.ptr())); } - pub fn lhsContractingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { + pub fn getLhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetLhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn rhsContractingDimensionsSize(self: *const DotDimensionNumbersAttr) usize { + pub fn getRhsContractingDimensionsSize(self: *const DotDimensionNumbersAttribute) usize { return @intCast(c.stablehloDotDimensionNumbersGetRhsContractingDimensionsSize(self.ptr())); } - pub fn rhsContractingDimensionsElem(self: *const DotDimensionNumbersAttr, pos: usize) i64 { + pub fn getRhsContractingDimensionsElem(self: *const DotDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloDotDimensionNumbersGetRhsContractingDimensionsElem(self.ptr(), @intCast(pos)); } - - pub fn attribute(self: *const DotDimensionNumbersAttr) *const mlir.Attribute { - return @ptrCast(self); - } }; -pub fn dotDimensionNumbersAttr(ctx: *mlir.Context, args: DotDimensionNumbersAttr.InitArgs) *const mlir.Attribute { - return @ptrCast(DotDimensionNumbersAttr.get(ctx, args)); +pub fn dotDimensionNumbersAttribute(ctx: *mlir.Context, args: DotDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(DotDimensionNumbersAttribute.init(ctx, args)); } -pub const GatherDimensionNumbersAttr = opaque { - const M = mlir.Methods(GatherDimensionNumbersAttr, c.MlirAttribute); +pub const GatherDimensionNumbersAttribute = opaque { + const M = mlir.Methods(GatherDimensionNumbersAttribute, mlir.Attribute); pub const isAFn = c.stablehloAttributeIsAGatherDimensionNumbers; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); + pub const format = M.eql(c.mlirAttributePrint); - pub fn get( + pub fn init( ctx: *mlir.Context, offset_dims: []const i64, collapsed_slice_dims: []const i64, @@ -1136,7 +958,7 @@ pub const GatherDimensionNumbersAttr = opaque { start_indices_batching_dims: []const i64, start_index_map: []const i64, index_vector_dim: i64, - ) *const GatherDimensionNumbersAttr { + ) *const GatherDimensionNumbersAttribute { return @ptrCast(c.stablehloGatherDimensionNumbersGet( ctx.ptr(), @intCast(offset_dims.len), @@ -1153,52 +975,52 @@ pub const GatherDimensionNumbersAttr = opaque { ).ptr); } - pub fn offsetDimsSize(self: *const GatherDimensionNumbersAttr) usize { + pub fn getOffsetDimsSize(self: *const GatherDimensionNumbersAttribute) usize { return @intCast(c.stablehloGatherDimensionNumbersGetOffsetDimsSize(self.ptr())); } - pub fn offsetDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { + pub fn getOffsetDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetOffsetDimsElem(self.ptr(), @intCast(pos)); } - pub fn collapsedSliceDimsSize(self: *const GatherDimensionNumbersAttr) usize { + pub fn getCollapsedSliceDimsSize(self: *const GatherDimensionNumbersAttribute) usize { return @intCast(c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsSize(self.ptr())); } - pub fn collapsedSliceDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { + pub fn getCollapsedSliceDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetCollapsedSliceDimsElem(self.ptr(), @intCast(pos)); } - pub fn startIndexMapSize(self: *const GatherDimensionNumbersAttr) usize { + pub fn getStartIndexMapSize(self: *const GatherDimensionNumbersAttribute) usize { return @intCast(c.stablehloGatherDimensionNumbersGetStartIndexMapSize(self.ptr())); } - pub fn operandBatchingDimsSize(self: *const GatherDimensionNumbersAttr) usize { + pub fn getOperandBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { return @intCast(c.stablehloGatherDimensionNumbersGetOperandBatchingDimsSize(self.ptr())); } - pub fn operandBatchingDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { + pub fn getOperandBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetOperandBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn startIndicesBatchingDimsSize(self: *const GatherDimensionNumbersAttr) usize { + pub fn getStartIndicesBatchingDimsSize(self: *const GatherDimensionNumbersAttribute) usize { return @intCast(c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsSize(self.ptr())); } - pub fn startIndicesBatchingDimsElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { + pub fn getStartIndicesBatchingDimsElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetStartIndicesBatchingDimsElem(self.ptr(), @intCast(pos)); } - pub fn startIndexMapElem(self: *const GatherDimensionNumbersAttr, pos: usize) i64 { + pub fn getStartIndexMapElem(self: *const GatherDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloGatherDimensionNumbersGetStartIndexMapElem(self.ptr(), @intCast(pos)); } - pub fn indexVectorDim(self: *const GatherDimensionNumbersAttr) usize { + pub fn getIndexVectorDim(self: *const GatherDimensionNumbersAttribute) usize { return @intCast(c.stablehloGatherDimensionNumbersGetIndexVectorDim(self.ptr())); } }; -pub fn gatherDimensionNumbersAttr( +pub fn gatherDimensionNumbersAttribute( ctx: *mlir.Context, offset_dims: []const i64, collapsed_slice_dims: []const i64, @@ -1207,7 +1029,7 @@ pub fn gatherDimensionNumbersAttr( start_index_map: []const i64, index_vector_dim: i64, ) *const mlir.Attribute { - return @ptrCast(GatherDimensionNumbersAttr.get( + return @ptrCast(GatherDimensionNumbersAttribute.init( ctx, offset_dims, collapsed_slice_dims, @@ -1218,8 +1040,8 @@ pub fn gatherDimensionNumbersAttr( )); } -pub const ConvDimensionNumbersAttr = opaque { - const M = mlir.Methods(ConvDimensionNumbersAttr, c.MlirAttribute); +pub const ConvDimensionNumbersAttribute = opaque { + const M = mlir.Methods(ConvDimensionNumbersAttribute, mlir.Attribute); pub const isAFn = c.stablehloAttributeIsAConvDimensionNumbers; pub const ptr = M.ptr; @@ -1238,7 +1060,7 @@ pub const ConvDimensionNumbersAttr = opaque { output_spatial_dimensions: []const i64, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) *const ConvDimensionNumbersAttr { + pub fn init(ctx: *mlir.Context, args: InitArgs) *const ConvDimensionNumbersAttribute { return @ptrCast(c.stablehloConvDimensionNumbersGet( ctx.ptr(), args.input_batch_dimension, @@ -1256,65 +1078,61 @@ pub const ConvDimensionNumbersAttr = opaque { ).ptr); } - pub fn inputBatchDimension(self: *const ConvDimensionNumbersAttr) i64 { + pub fn getInputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { return c.stablehloConvDimensionNumbersGetInputBatchDimension(self.ptr()); } - pub fn inputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { + pub fn getInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { return c.stablehloConvDimensionNumbersGetInputFeatureDimension(self.ptr()); } - pub fn inputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { + pub fn getInputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { return @intCast(c.stablehloConvDimensionNumbersGetInputSpatialDimensionsSize(self.ptr())); } - pub fn inputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { + pub fn getInputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloConvDimensionNumbersGetInputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn kernelInputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { + pub fn getKernelInputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { return c.stablehloConvDimensionNumbersGetKernelInputFeatureDimension(self.ptr()); } - pub fn kernelOutputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { + pub fn getKernelOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { return c.stablehloConvDimensionNumbersGetKernelOutputFeatureDimension(self.ptr()); } - pub fn kernelSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { + pub fn getKernelSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { return @intCast(c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsSize(self.ptr())); } - pub fn kernelSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { + pub fn getKernelSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloConvDimensionNumbersGetKernelSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - pub fn outputBatchDimension(self: *const ConvDimensionNumbersAttr) i64 { + pub fn getOutputBatchDimension(self: *const ConvDimensionNumbersAttribute) i64 { return c.stablehloConvDimensionNumbersGetOutputBatchDimension(self.ptr()); } - pub fn outputFeatureDimension(self: *const ConvDimensionNumbersAttr) i64 { + pub fn getOutputFeatureDimension(self: *const ConvDimensionNumbersAttribute) i64 { return c.stablehloConvDimensionNumbersGetOutputFeatureDimension(self.ptr()); } - pub fn outputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttr) usize { + pub fn getOutputSpatialDimensionsSize(self: *const ConvDimensionNumbersAttribute) usize { return @intCast(c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsSize(self.ptr())); } - pub fn outputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttr, pos: usize) i64 { + pub fn getOutputSpatialDimensionsElem(self: *const ConvDimensionNumbersAttribute, pos: usize) i64 { return c.stablehloConvDimensionNumbersGetOutputSpatialDimensionsElem(self.ptr(), @intCast(pos)); } - - pub fn attribute(self: *const ConvDimensionNumbersAttr) *const mlir.Attribute { - return @ptrCast(self); - } }; -pub fn convDimensionNumbersAttr(ctx: *mlir.Context, args: ConvDimensionNumbersAttr.InitArgs) *const mlir.Attribute { - return @ptrCast(ConvDimensionNumbersAttr.get(ctx, args)); +pub fn convDimensionNumbersAttribute(ctx: *mlir.Context, args: ConvDimensionNumbersAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(ConvDimensionNumbersAttribute.init(ctx, args)); } -pub const OutputOperandAliasAttr = opaque { - const M = mlir.Methods(OutputOperandAliasAttr, c.MlirAttribute); +pub const OutputOperandAliasAttribute = opaque { + const M = mlir.Methods(OutputOperandAliasAttribute, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAOutputOperandAlias; pub const ptr = M.ptr; @@ -1327,7 +1145,7 @@ pub const OutputOperandAliasAttr = opaque { operand_tuple_indices: []const i64 = &.{}, }; - pub fn get(ctx: *mlir.Context, args: InitArgs) *const OutputOperandAliasAttr { + pub fn get(ctx: *mlir.Context, args: InitArgs) *const OutputOperandAliasAttribute { return @ptrCast(c.stablehloOutputOperandAliasGet( ctx.ptr(), @intCast(args.output_tuple_indices.len), @@ -1337,18 +1155,14 @@ pub const OutputOperandAliasAttr = opaque { @ptrCast(args.operand_tuple_indices), ).ptr); } - - pub fn attribute(self: *const OutputOperandAliasAttr) *const mlir.Attribute { - return @ptrCast(self); - } }; -pub fn outputOperandAliasAttr(ctx: *mlir.Context, args: OutputOperandAliasAttr.InitArgs) *const mlir.Attribute { - return @ptrCast(OutputOperandAliasAttr.get(ctx, args)); +pub fn outputOperandAliasAttribute(ctx: *mlir.Context, args: OutputOperandAliasAttribute.InitArgs) *const mlir.Attribute { + return @ptrCast(OutputOperandAliasAttribute.get(ctx, args)); } -pub const PrecisionAttr = opaque { - const M = mlir.Methods(PrecisionAttr, c.MlirAttribute); +pub const PrecisionAttribute = opaque { + const M = mlir.Methods(PrecisionAttribute, mlir.Attribute); pub const isAFn = c.stablehloAttributeIsAPrecisionAttr; pub const ptr = M.ptr; @@ -1361,30 +1175,22 @@ pub const PrecisionAttr = opaque { HIGHEST, }; - pub const InitArgs = struct { - value: Precision, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const PrecisionAttr { - return @ptrCast(c.stablehloPrecisionAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); + pub fn init(ctx: *mlir.Context, value: Precision) *const PrecisionAttribute { + return @ptrCast(c.stablehloPrecisionAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); } - pub fn value(self: *const PrecisionAttr) Precision { - const name = mlir.string(c.stablehloPrecisionAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Precision, name) orelse unreachable; - } - - pub fn attribute(self: *const PrecisionAttr) *const mlir.Attribute { - return @ptrCast(self); + pub fn getValue(self: *const PrecisionAttribute) Precision { + const value = mlir.string(c.stablehloPrecisionAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Precision, value) orelse unreachable; } }; -pub fn precisionAttr(ctx: *mlir.Context, precision: PrecisionAttr.Precision) *const mlir.Attribute { - return PrecisionAttr.get(ctx, .{ .value = precision }).attribute(); +pub fn precisionAttribute(ctx: *mlir.Context, precision: PrecisionAttribute.Precision) *const mlir.Attribute { + return @ptrCast(PrecisionAttribute.init(ctx, precision)); } -pub const ComparisonDirectionAttr = opaque { - const M = mlir.Methods(ComparisonDirectionAttr, c.MlirAttribute); +pub const ComparisonDirection = opaque { + const M = mlir.Methods(ComparisonDirection, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAComparisonDirectionAttr; pub const ptr = M.ptr; @@ -1400,30 +1206,22 @@ pub const ComparisonDirectionAttr = opaque { LT, }; - pub const InitArgs = struct { - value: Direction, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const ComparisonDirectionAttr { - return @ptrCast(c.stablehloComparisonDirectionAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); + pub fn init(ctx: *mlir.Context, value: Direction) *const ComparisonDirection { + return @ptrCast(c.stablehloComparisonDirectionAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn value(self: *const ComparisonDirectionAttr) Direction { - const name = mlir.string(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Direction, name) orelse unreachable; - } - - pub fn attribute(self: *const ComparisonDirectionAttr) *const mlir.Attribute { - return @ptrCast(self); + pub fn getValue(self: *const ComparisonDirection) Direction { + const value = mlir.string(c.stablehloComparisonDirectionAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Direction, value) orelse unreachable; } }; -pub fn comparisonDirectionAttr(ctx: *mlir.Context, value: ComparisonDirectionAttr.Direction) *const mlir.Attribute { - return ComparisonDirectionAttr.get(ctx, .{ .value = value }).attribute(); +pub fn comparisonDirection(ctx: *mlir.Context, value: ComparisonDirection.Direction) *const mlir.Attribute { + return @ptrCast(ComparisonDirection.init(ctx, value)); } -pub const CompareTypeAttr = opaque { - const M = mlir.Methods(CompareTypeAttr, c.MlirAttribute); +pub const CompareType = opaque { + const M = mlir.Methods(CompareType, c.MlirAttribute); pub const isAFn = c.stablehloAttributeIsAComparisonTypeAttr; pub const ptr = M.ptr; @@ -1437,30 +1235,22 @@ pub const CompareTypeAttr = opaque { TOTALORDER, }; - pub const InitArgs = struct { - value: Type, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const CompareTypeAttr { - return @ptrCast(c.stablehloComparisonTypeAttrGet(ctx.ptr(), stringRef(@tagName(args.value))).ptr); + pub fn init(ctx: *mlir.Context, value: Type) *const CompareType { + return @ptrCast(c.stablehloComparisonTypeAttrGet(ctx.ptr(), stringRef(@tagName(value))).ptr); } - pub fn value(self: *const CompareTypeAttr) Type { - const name = mlir.string(c.stablehloComparisonTypeAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Type, name) orelse unreachable; - } - - pub fn attribute(self: *const CompareTypeAttr) *const mlir.Attribute { - return @ptrCast(self); + pub fn getValue(self: *const CompareType) Type { + const value = mlir.string(c.stablehloComparisonTypeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub fn compareTypeAttr(ctx: *mlir.Context, value: CompareTypeAttr.Type) *const mlir.Attribute { - return CompareTypeAttr.get(ctx, .{ .value = value }).attribute(); +pub fn compareType(ctx: *mlir.Context, value: CompareType.Type) *const mlir.Attribute { + return @ptrCast(CompareType.init(ctx, value)); } -pub const TransposeAttr = opaque { - const M = mlir.Methods(TransposeAttr, c.MlirAttribute); +pub const TransposeAttribute = opaque { + const M = mlir.Methods(TransposeAttribute, mlir.Attribute); pub const isAFn = c.stablehloAttributeIsATransposeAttr; pub const ptr = M.ptr; @@ -1473,30 +1263,22 @@ pub const TransposeAttr = opaque { ADJOINT, }; - pub const InitArgs = struct { - value: Type, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const TransposeAttr { - return @ptrCast(c.stablehloTransposeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); + pub fn init(ctx: *mlir.Context, value: Type) *const TransposeAttribute { + return @ptrCast(c.stablehloTransposeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn value(self: *const TransposeAttr) Type { - const name = mlir.string(c.stablehloTransposeAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Type, name) orelse unreachable; - } - - pub fn attribute(self: *const TransposeAttr) *const mlir.Attribute { - return @ptrCast(self); + pub fn getValue(self: *const TransposeAttribute) Type { + const value = mlir.string(c.stablehloTransposeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub fn transposeAttr(ctx: *mlir.Context, value: TransposeAttr.Type) *const mlir.Attribute { - return TransposeAttr.get(ctx, .{ .value = value }).attribute(); +pub fn transposeAttribute(ctx: *mlir.Context, value: TransposeAttribute.Type) *const mlir.Attribute { + return @ptrCast(TransposeAttribute.init(ctx, value)); } -pub const FftTypeAttr = opaque { - const M = mlir.Methods(FftTypeAttr, c.MlirAttribute); +pub const FftType = opaque { + const M = mlir.Methods(FftType, mlir.Attribute); pub const isAFn = c.stablehloAttributeIsAFftTypeAttr; pub const ptr = M.ptr; @@ -1510,26 +1292,18 @@ pub const FftTypeAttr = opaque { IRFFT, }; - pub const InitArgs = struct { - value: Type, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) *const FftTypeAttr { - return @ptrCast(c.stablehloFftTypeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(args.value))).ptr); + pub fn init(ctx: *mlir.Context, value: Type) *const FftType { + return @ptrCast(c.stablehloFftTypeAttrGet(ctx.ptr(), mlir.stringRef(@tagName(value))).ptr); } - pub fn value(self: *const FftTypeAttr) Type { - const name = mlir.string(c.stablehloFftTypeAttrGetValue(self.ptr())); - return std.meta.stringToEnum(Type, name) orelse unreachable; - } - - pub fn attribute(self: *const FftTypeAttr) *const mlir.Attribute { - return @ptrCast(self); + pub fn getValue(self: *const FftType) Type { + const value = mlir.string(c.stablehloFftTypeAttrGetValue(self.ptr())); + return std.meta.stringToEnum(Type, value) orelse unreachable; } }; -pub fn fftTypeAttr(ctx: *mlir.Context, value: FftTypeAttr.Type) *const mlir.Attribute { - return FftTypeAttr.get(ctx, .{ .value = value }).attribute(); +pub fn fftType(ctx: *mlir.Context, value: FftType.Type) *const mlir.Attribute { + return @ptrCast(FftType.init(ctx, value)); } fn stringFromStream(buf: []u8, streamFn: anytype, args: anytype) []const u8 { @@ -1591,9 +1365,7 @@ pub fn serializePortableArtifact2(module: *mlir.Module, target_version: []const if (sctx.err) |err| { return err; } - // TODO(Corentin): Find out why this crashes on macOS - //if (c.mlirLogicalResultIsFailure(result)) { - if (@as(c_int, result.value) == 0) { + if (c.mlirLogicalResultIsFailure(result)) { return error.InvalidMlirBytecodeVersion; } } @@ -1613,43 +1385,3 @@ pub fn returns(ctx: *mlir.Context, values: []const *const mlir.Value, location: .location = location, }); } - -pub const ChannelType = enum(i64) { - collective = 0, - send_recv = 1, -}; - -pub fn channelHandleAttr( - ctx: *mlir.Context, - handle: i64, - channel_type: ChannelType, -) *const mlir.Attribute { - return @ptrCast(c.stablehloChannelHandleGet( - ctx.ptr(), - handle, - @backingInt(channel_type), - ).ptr); -} - -pub fn allReduce( - ctx: *mlir.Context, - inputs: []const *const mlir.Value, - result_types: []const *const mlir.Type, - reducer_block: *mlir.Block, - replica_groups: *const mlir.Attribute, - channel_handle: *const mlir.Attribute, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "stablehlo.all_reduce", .{ - .operands = .{ .variadic = &.{inputs} }, - .results = .{ .flat = result_types }, - .blocks = &.{reducer_block}, - .attributes = &.{ - .named(ctx, "replica_groups", replica_groups), - .named(ctx, "channel_handle", channel_handle), - .named(ctx, "use_global_device_ids", .unit(ctx)), - }, - .verify = true, - .location = location, - }); -} diff --git a/mlir/dialects/stablehlo/test.zig b/mlir/dialects/stablehlo/test.zig deleted file mode 100644 index c22b2ebd..00000000 --- a/mlir/dialects/stablehlo/test.zig +++ /dev/null @@ -1,106 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); - -const operation_names: []const []const u8 = &.{ - "stablehlo.abs", - "stablehlo.add", - "stablehlo.all_reduce", - "stablehlo.and", - "stablehlo.atan2", - "stablehlo.bitcast_convert", - "stablehlo.broadcast_in_dim", - "stablehlo.cbrt", - "stablehlo.ceil", - "stablehlo.cholesky", - "stablehlo.clamp", - "stablehlo.compare", - "stablehlo.complex", - "stablehlo.composite", - "stablehlo.concatenate", - "stablehlo.constant", - "stablehlo.convert", - "stablehlo.convolution", - "stablehlo.cosine", - "stablehlo.count_leading_zeros", - "stablehlo.custom_call", - "stablehlo.divide", - "stablehlo.dot_general", - "stablehlo.dynamic_slice", - "stablehlo.dynamic_update_slice", - "stablehlo.exponential", - "stablehlo.exponential_minus_one", - "stablehlo.fft", - "stablehlo.floor", - "stablehlo.gather", - "stablehlo.get_tuple_element", - "stablehlo.if", - "stablehlo.imag", - "stablehlo.iota", - "stablehlo.is_finite", - "stablehlo.log", - "stablehlo.log_plus_one", - "stablehlo.logistic", - "stablehlo.maximum", - "stablehlo.minimum", - "stablehlo.multiply", - "stablehlo.negate", - "stablehlo.not", - "stablehlo.optimization_barrier", - "stablehlo.or", - "stablehlo.pad", - "stablehlo.partition_id", - "stablehlo.popcnt", - "stablehlo.power", - "stablehlo.real", - "stablehlo.reduce", - "stablehlo.reduce_precision", - "stablehlo.reduce_window", - "stablehlo.remainder", - "stablehlo.reshape", - "stablehlo.return", - "stablehlo.reverse", - "stablehlo.rng", - "stablehlo.rng_bit_generator", - "stablehlo.round_nearest_afz", - "stablehlo.round_nearest_even", - "stablehlo.rsqrt", - "stablehlo.scatter", - "stablehlo.select", - "stablehlo.shift_left", - "stablehlo.shift_right_arithmetic", - "stablehlo.shift_right_logical", - "stablehlo.sign", - "stablehlo.sine", - "stablehlo.slice", - "stablehlo.sort", - "stablehlo.sqrt", - "stablehlo.subtract", - "stablehlo.tanh", - "stablehlo.transpose", - "stablehlo.triangular_solve", - "stablehlo.tuple", - "stablehlo.while", - "stablehlo.xor", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(stablehlo); - inline for (comptime std.meta.declarations(stablehlo)) |decl| { - const T = @field(stablehlo, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - stablehlo.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/tcl/BUILD.bazel b/mlir/dialects/tcl/BUILD.bazel deleted file mode 100644 index d57b6f36..00000000 --- a/mlir/dialects/tcl/BUILD.bazel +++ /dev/null @@ -1,172 +0,0 @@ -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - ":dialect", - ":headers", - "@llvm-project//mlir:CAPIIR", - ], -) - -zig_library( - name = "tcl", - import_name = "mlir/dialects/tcl", - main = "tcl.zig", - tags = ["manual"], - deps = [ - ":capi", - "//mlir", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":tcl", - "//mlir", - "//mlir/dialects/func", - ], -) - -# The native subset of the Furiosa XLA plugin's `tcl` dialect -# (xla/backends/furiosa/codegen/tcl/ir): what ZML emits and the plugin's custom -# call parses. Files mirror the plugin's one for one. -td_library( - name = "td_files", - srcs = [ - "tcl_attrs.td", - "tcl_dialect.td", - "tcl_ops.td", - "tcl_types.td", - ], - includes = ["/"], - deps = [ - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - ], -) - -gentbl_cc_library( - name = "dialect_inc_gen", - tbl_outs = [ - ( - ["-gen-dialect-decls"], - "tcl_dialect.h.inc", - ), - ( - ["-gen-dialect-defs"], - "tcl_dialect.cc.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "tcl_dialect.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "types_inc_gen", - tbl_outs = [ - ( - [ - "-gen-typedef-decls", - "-typedefs-dialect=tcl", - ], - "tcl_types.h.inc", - ), - ( - [ - "-gen-typedef-defs", - "-typedefs-dialect=tcl", - ], - "tcl_types.cc.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "tcl_types.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "attrs_inc_gen", - tbl_outs = [ - ( - [ - "-gen-attrdef-decls", - "-attrdefs-dialect=tcl", - ], - "tcl_attrs.h.inc", - ), - ( - [ - "-gen-attrdef-defs", - "-attrdefs-dialect=tcl", - ], - "tcl_attrs.cc.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "tcl_attrs.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "ops_inc_gen", - tbl_outs = [ - ( - ["-gen-op-decls"], - "tcl_ops.h.inc", - ), - ( - ["-gen-op-defs"], - "tcl_ops.cc.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "tcl_ops.td", - deps = [":td_files"], -) - -cc_library( - name = "headers", - hdrs = [ - "tcl_attrs.h", - "tcl_dialect.h", - "tcl_ops.h", - "tcl_types.h", - ], - deps = [ - ":attrs_inc_gen", - ":dialect_inc_gen", - ":ops_inc_gen", - ":types_inc_gen", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:BytecodeOpInterface", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:SideEffectInterfaces", - ], -) - -cc_library( - name = "dialect", - srcs = [ - "tcl_attrs.cc", - "tcl_dialect.cc", - "tcl_ops.cc", - "tcl_types.cc", - ], - textual_hdrs = ["ve_instructions.inc"], - deps = [ - ":headers", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", - ], -) diff --git a/mlir/dialects/tcl/capi.cc b/mlir/dialects/tcl/capi.cc deleted file mode 100644 index 70ee09d6..00000000 --- a/mlir/dialects/tcl/capi.cc +++ /dev/null @@ -1,118 +0,0 @@ -//===- capi.cc - C API for the Furiosa TCL dialect ------------------------===// - -#include "capi.h" - -#include "mlir/dialects/tcl/tcl_ops.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" -#include "mlir/CAPI/Support.h" - -#include - -using namespace mlir; -using namespace xla::furiosa::tcl; - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Tcl, tcl, TclDialect) - -// `getChecked` with its diagnostic silenced: an invalid parameter is a null -// handle, which the caller reports. -template -static auto checked(MlirContext context, Args &&...args) { - MLIRContext *ctx = unwrap(context); - if (!ctx) - return T(); - ctx->getOrLoadDialect(); - ScopedDiagnosticHandler silence(ctx, [](Diagnostic &) { return success(); }); - return T::getChecked([ctx] { return emitError(UnknownLoc::get(ctx)); }, ctx, - std::forward(args)...); -} - -template static T as(MlirAttribute attr) { - return dyn_cast_if_present(unwrap(attr)); -} - -MlirAttribute mlirTclSymbolAttrGet(MlirContext context, MlirStringRef name) { - return wrap(checked(context, unwrap(name))); -} - -MlirAttribute mlirTclTacticAttrGet(MlirContext context, MlirStringRef value) { - return wrap(checked(context, unwrap(value))); -} - -MlirAttribute mlirTclVeOpcodeAttrGet(MlirContext context, MlirStringRef value) { - return wrap(checked(context, unwrap(value))); -} - -MlirAttribute mlirTclReduceModeAttrGet(MlirContext context, - MlirStringRef value) { - return wrap(checked(context, unwrap(value))); -} - -MlirAttribute mlirTclPredicateAttrGet(MlirContext context, - MlirStringRef value) { - return wrap(checked(context, unwrap(value))); -} - -MlirAttribute mlirTclContextAttrGet(MlirContext context, MlirAttribute fields) { - auto dict = as(fields); - return dict ? wrap(checked(context, dict)) - : MlirAttribute{nullptr}; -} - -MlirAttribute mlirTclReadOptionsAttrGet(MlirContext context, - MlirAttribute fields) { - auto dict = as(fields); - return dict ? wrap(checked(context, dict)) - : MlirAttribute{nullptr}; -} - -MlirAttribute mlirTclConfigAttrGet(MlirContext context, MlirAttribute fields) { - auto dict = as(fields); - return dict ? wrap(checked(context, dict)) - : MlirAttribute{nullptr}; -} - -MlirAttribute mlirTclExprAttrGet(MlirContext context, MlirStringRef kind, - MlirAttribute args) { - auto array = as(args); - return array ? wrap(checked(context, unwrap(kind), array)) - : MlirAttribute{nullptr}; -} - -MlirAttribute mlirTclDramMappingAttrGet(MlirContext context, MlirAttribute chip, - MlirAttribute inner, - MlirAttribute original) { - auto c = as(chip), i = as(inner), - o = as(original); - return c && i && o ? wrap(checked(context, c, i, o)) - : MlirAttribute{nullptr}; -} - -MlirType mlirTclLogicalTypeGet(MlirContext context, MlirType element, - MlirAttribute axes) { - auto array = as(axes); - if (!array || mlirTypeIsNull(element)) - return {nullptr}; - return wrap(checked(context, unwrap(element), array)); -} - -bool mlirTypeIsATclLogical(MlirType type) { - return isa_and_nonnull(unwrap(type)); -} - -MlirType mlirTclLogicalTypeGetElementType(MlirType type) { - return wrap(cast(unwrap(type)).getElementType()); -} - -MlirAttribute mlirTclLogicalTypeGetAxes(MlirType type) { - return wrap(cast(unwrap(type)).getAxes()); -} - -MlirType mlirTclMappedTypeGet(MlirContext context, MlirType element, - MlirAttribute mapping) { - auto m = as(mapping); - if (!m || mlirTypeIsNull(element)) - return {nullptr}; - return wrap(checked(context, unwrap(element), m)); -} diff --git a/mlir/dialects/tcl/capi.h b/mlir/dialects/tcl/capi.h deleted file mode 100644 index 31e72c3b..00000000 --- a/mlir/dialects/tcl/capi.h +++ /dev/null @@ -1,53 +0,0 @@ -//===- capi.h - C API for the Furiosa TCL dialect -----------------*- C -*-===// - -#ifndef MLIR_DIALECTS_TCL_CAPI_H -#define MLIR_DIALECTS_TCL_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Tcl, tcl); - -// Constructors verify their parameters and return a null handle when invalid. -MLIR_CAPI_EXPORTED MlirAttribute mlirTclSymbolAttrGet(MlirContext context, - MlirStringRef name); -MLIR_CAPI_EXPORTED MlirAttribute mlirTclExprAttrGet(MlirContext context, - MlirStringRef kind, - MlirAttribute args); -MLIR_CAPI_EXPORTED MlirAttribute mlirTclTacticAttrGet(MlirContext context, - MlirStringRef value); -MLIR_CAPI_EXPORTED MlirAttribute mlirTclVeOpcodeAttrGet(MlirContext context, - MlirStringRef value); -MLIR_CAPI_EXPORTED MlirAttribute mlirTclReduceModeAttrGet(MlirContext context, - MlirStringRef value); -MLIR_CAPI_EXPORTED MlirAttribute mlirTclPredicateAttrGet(MlirContext context, - MlirStringRef value); -MLIR_CAPI_EXPORTED MlirAttribute mlirTclContextAttrGet(MlirContext context, - MlirAttribute fields); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTclReadOptionsAttrGet(MlirContext context, MlirAttribute fields); -MLIR_CAPI_EXPORTED MlirAttribute mlirTclConfigAttrGet(MlirContext context, - MlirAttribute fields); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTclDramMappingAttrGet(MlirContext context, MlirAttribute chip, - MlirAttribute inner, MlirAttribute original); - -MLIR_CAPI_EXPORTED MlirType mlirTclLogicalTypeGet(MlirContext context, - MlirType element, - MlirAttribute axes); -MLIR_CAPI_EXPORTED bool mlirTypeIsATclLogical(MlirType type); -MLIR_CAPI_EXPORTED MlirType mlirTclLogicalTypeGetElementType(MlirType type); -MLIR_CAPI_EXPORTED MlirAttribute mlirTclLogicalTypeGetAxes(MlirType type); - -MLIR_CAPI_EXPORTED MlirType mlirTclMappedTypeGet(MlirContext context, - MlirType element, - MlirAttribute mapping); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_TCL_CAPI_H diff --git a/mlir/dialects/tcl/tcl.zig b/mlir/dialects/tcl/tcl.zig deleted file mode 100644 index a3eb4673..00000000 --- a/mlir/dialects/tcl/tcl.zig +++ /dev/null @@ -1,364 +0,0 @@ -const c = @import("c"); -const mlir = @import("mlir"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("tcl"); -} - -fn stringRef(s: []const u8) c.MlirStringRef { - return .{ .data = s.ptr, .length = s.len }; -} - -fn attribute(a: c.MlirAttribute) mlir.Error!*const mlir.Attribute { - return if (a.ptr) |p| @ptrCast(p) else error.InvalidMlir; -} - -fn type_(t: c.MlirType) mlir.Error!*const mlir.Type { - return if (t.ptr) |p| @ptrCast(p) else error.InvalidMlir; -} - -// ============================================================================= -// Attributes -// ============================================================================= - -pub const Tactic = enum { - einsum_by_dpe, - reduce_by_ve, - interleaving, - elementwise, - tensor_operation, - einsum_by_ve, - filter_compaction, - - pub fn name(self: Tactic) []const u8 { - return switch (self) { - .einsum_by_dpe => "EinsumByDpe", - .reduce_by_ve => "ReduceByVe", - .interleaving => "Interleaving", - .elementwise => "Elementwise", - .tensor_operation => "TensorOperation", - .einsum_by_ve => "EinsumByVe", - .filter_compaction => "FilterCompaction", - }; - } -}; - -pub const ReduceMode = enum { - addi, - addf, - maxi, - maxf, - mini, - minf, - /// A running i32 sum that keeps its axes. - cumsum, - - pub fn name(self: ReduceMode) []const u8 { - return switch (self) { - .addi => "Addi", - .addf => "Addf", - .maxi => "Maxi", - .maxf => "Maxf", - .mini => "Mini", - .minf => "Minf", - .cumsum => "Cumsum", - }; - } -}; - -pub const Predicate = enum { eq, ne, lt, le, gt, ge }; - -/// An axis or another TCL identifier: `#tcl.symbol<"A">`. -pub fn symbolAttr(ctx: *mlir.Context, name: []const u8) mlir.Error!*const mlir.Attribute { - return attribute(c.mlirTclSymbolAttrGet(ctx.ptr(), stringRef(name))); -} - -/// `#tcl.expr`: `add`, `mul`, `stride`, `broadcast`, `arg`, ... -pub fn exprAttr(ctx: *mlir.Context, kind: []const u8, args: []const *const mlir.Attribute) mlir.Error!*const mlir.Attribute { - return attribute(c.mlirTclExprAttrGet(ctx.ptr(), stringRef(kind), mlir.Attribute.array(ctx, args).ptr())); -} - -pub fn tacticAttr(ctx: *mlir.Context, value: Tactic) *const mlir.Attribute { - return attribute(c.mlirTclTacticAttrGet(ctx.ptr(), stringRef(value.name()))) catch unreachable; -} - -/// A VE opcode of the SDK vocabulary (`exp`, `addf`, `to_f16`, ...). -pub fn veOpcodeAttr(ctx: *mlir.Context, opcode: []const u8) mlir.Error!*const mlir.Attribute { - return attribute(c.mlirTclVeOpcodeAttrGet(ctx.ptr(), stringRef(opcode))); -} - -pub fn reduceModeAttr(ctx: *mlir.Context, mode: ReduceMode) *const mlir.Attribute { - return attribute(c.mlirTclReduceModeAttrGet(ctx.ptr(), stringRef(mode.name()))) catch unreachable; -} - -pub fn predicateAttr(ctx: *mlir.Context, value: Predicate) *const mlir.Attribute { - return attribute(c.mlirTclPredicateAttrGet(ctx.ptr(), stringRef(@tagName(value)))) catch unreachable; -} - -/// `@ context(operator = {Chip: ...}, heuristic_hint = {...})`. -pub fn contextAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { - return attribute(c.mlirTclContextAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); -} - -/// `read(x, subtraction=, table_lookup=, broadcast_to=, typecast_to=, pad=)`. -pub fn readOptionsAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { - return attribute(c.mlirTclReadOptionsAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); -} - -/// `@compiler_config({...})` / `@auto_config({...})`, JSON-compatible fields. -pub fn configAttr(ctx: *mlir.Context, fields: []const mlir.NamedAttribute) mlir.Error!*const mlir.Attribute { - return attribute(c.mlirTclConfigAttrGet(ctx.ptr(), mlir.Attribute.dict(ctx, fields).ptr())); -} - -/// `dram(chip | inner)`, with the original axes of a padded mapping. -pub fn dramMappingAttr(ctx: *mlir.Context, chip: []const *const mlir.Attribute, inner: []const *const mlir.Attribute, original: []const *const mlir.Attribute) mlir.Error!*const mlir.Attribute { - return attribute(c.mlirTclDramMappingAttrGet( - ctx.ptr(), - mlir.Attribute.array(ctx, chip).ptr(), - mlir.Attribute.array(ctx, inner).ptr(), - mlir.Attribute.array(ctx, original).ptr(), - )); -} - -/// Discardable TCL attributes of a `func.func`. -pub const FunctionOpts = struct { - /// `tcl.updated_inputs`: per result, the argument it updates in place, or -1. - updated_inputs: ?[]const i64 = null, - /// `tcl.compiler_config`, a `configAttr`. - compiler_config: ?*const mlir.Attribute = null, - /// `tcl.auto_config`, a `configAttr`. - auto_config: ?*const mlir.Attribute = null, - /// `tcl.fusible`: the plugin may compile the function into the unit around it. - fusible: bool = false, -}; - -/// Sets `tcl.axes`, each axis's size or definition by name, and the -/// attributes of `opts` on `function`. -pub fn annotateFunction(ctx: *mlir.Context, function: *mlir.Operation, axes: []const mlir.NamedAttribute, opts: FunctionOpts) void { - if (opts.updated_inputs) |inputs| function.setAttributeByName("tcl.updated_inputs", .denseArray(ctx, .i64, inputs)); - function.setAttributeByName("tcl.axes", .dict(ctx, axes)); - if (opts.compiler_config) |config| function.setAttributeByName("tcl.compiler_config", config); - if (opts.auto_config) |config| function.setAttributeByName("tcl.auto_config", config); - if (opts.fusible) function.setAttributeByName("tcl.fusible", .unit(ctx)); -} - -// ============================================================================= -// Types -// ============================================================================= - -/// `[A, B]/bf16`: a logical tensor over named axes. -pub fn logicalType(ctx: *mlir.Context, element: *const mlir.Type, axes: []const *const mlir.Attribute) mlir.Error!*const mlir.Type { - return (try LogicalType.get(ctx, .{ .elementType = element, .axes = axes })).type_(); -} - -/// `dram(...)/bf16`: a tensor in an explicit DRAM mapping. -pub fn mappedType(ctx: *mlir.Context, element: *const mlir.Type, mapping: *const mlir.Attribute) mlir.Error!*const mlir.Type { - return type_(c.mlirTclMappedTypeGet(ctx.ptr(), element.ptr(), mapping.ptr())); -} - -pub const LogicalType = opaque { - const M = mlir.Methods(LogicalType, c.MlirType); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub const isAFn = c.mlirTypeIsATclLogical; - - pub const InitArgs = struct { - elementType: *const mlir.Type, - axes: []const *const mlir.Attribute, - }; - - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const LogicalType { - return @ptrCast(c.mlirTclLogicalTypeGet(ctx.ptr(), args.elementType.ptr(), mlir.Attribute.array(ctx, args.axes).ptr()).ptr orelse return error.InvalidMlir); - } - - pub fn type_(self: *const LogicalType) *const mlir.Type { - return @ptrCast(self); - } - - pub fn elementType(self: *const LogicalType) *const mlir.Type { - return @ptrCast(c.mlirTclLogicalTypeGetElementType(self.ptr()).ptr); - } - - pub fn axes(self: *const LogicalType) *const mlir.ArrayAttribute { - return @ptrCast(c.mlirTclLogicalTypeGetAxes(self.ptr()).ptr); - } -}; - -// ============================================================================= -// Operations -// ============================================================================= -// -// Builders do not verify: kernel instructions only verify inside a -// `tcl.kernel`, and a kernel or graph operation only once its operands -// dominate it. Verify each graph-level operation after inserting it. - -pub const KernelOpts = struct { - context: ?*const mlir.Attribute = null, -}; - -pub fn kernel(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, tactic: Tactic, body: *mlir.Block, opts: KernelOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: [2]mlir.NamedAttribute = undefined; - attrs[0] = .named(ctx, "tactic", tacticAttr(ctx, tactic)); - if (opts.context) |a| attrs[1] = .named(ctx, "context", a); - return mlir.Operation.make(ctx, "tcl.kernel", .{ - .operands = .{ .flat = inputs }, - .results = .{ .flat = &.{output} }, - .attributes = attrs[0 .. @as(usize, 1) + @intFromBool(opts.context != null)], - .blocks = &.{body}, - .verify = false, - .location = location, - }); -} - -pub const ReadOpts = struct { - options: ?*const mlir.Attribute = null, -}; - -pub fn read(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, opts: ReadOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: [1]mlir.NamedAttribute = undefined; - if (opts.options) |o| attrs[0] = .named(ctx, "options", o); - return mlir.Operation.make(ctx, "tcl.read", .{ - .operands = .{ .flat = inputs }, - .results = .{ .flat = &.{output} }, - .attributes = attrs[0..@intFromBool(opts.options != null)], - .verify = false, - .location = location, - }); -} - -pub fn dpe(ctx: *mlir.Context, lhs: *const mlir.Value, rhs: *const mlir.Value, output: *const mlir.Type, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tcl.dpe", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{output} }, - .verify = false, - .location = location, - }); -} - -pub fn ve(ctx: *mlir.Context, inputs: []const *const mlir.Value, output: *const mlir.Type, opcode: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tcl.ve", .{ - .operands = .{ .flat = inputs }, - .results = .{ .flat = &.{output} }, - .attributes = &.{.named(ctx, "opcode", opcode)}, - .verify = false, - .location = location, - }); -} - -pub fn ve_reduce(ctx: *mlir.Context, input: *const mlir.Value, output: *const mlir.Type, mode: ReduceMode, axes: []const *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tcl.ve_reduce", .{ - .operands = .{ .flat = &.{input} }, - .results = .{ .flat = &.{output} }, - .attributes = &.{ - .named(ctx, "mode", reduceModeAttr(ctx, mode)), - .named(ctx, "axes", .array(ctx, axes)), - }, - .verify = false, - .location = location, - }); -} - -pub fn ve_select(ctx: *mlir.Context, condition: *const mlir.Value, true_value: *const mlir.Value, false_value: *const mlir.Value, output: *const mlir.Type, predicate: Predicate, threshold: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tcl.ve_select", .{ - .operands = .{ .flat = &.{ condition, true_value, false_value } }, - .results = .{ .flat = &.{output} }, - .attributes = &.{ - .named(ctx, "predicate", predicateAttr(ctx, predicate)), - .named(ctx, "threshold", threshold), - }, - .verify = false, - .location = location, - }); -} - -/// `tcl.filter_compaction`: keeps the elements of `input` along `axes` whose -/// `condition` satisfies `predicate` against `threshold`, packed into at most -/// `max_size` (an integer or axis attribute) elements. -pub fn filter_compaction(ctx: *mlir.Context, input: *const mlir.Value, condition: *const mlir.Value, output: *const mlir.Type, predicate: Predicate, threshold: *const mlir.Attribute, axes: []const *const mlir.Attribute, max_size: *const mlir.Attribute, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tcl.filter_compaction", .{ - .operands = .{ .flat = &.{ input, condition } }, - .results = .{ .flat = &.{output} }, - .attributes = &.{ - .named(ctx, "predicate", predicateAttr(ctx, predicate)), - .named(ctx, "threshold", threshold), - .named(ctx, "axes", .array(ctx, axes)), - .named(ctx, "max_size", max_size), - }, - .verify = false, - .location = location, - }); -} - -pub fn write(ctx: *mlir.Context, input: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tcl.write", .{ - .operands = .{ .flat = &.{input} }, - .verify = false, - .location = location, - }); -} - -pub const GraphOp = enum { gather, all_gather, scatter, reshape, transmute, concat, slice, arange, vector, as_logical, as_dram, index_read, index_write, scratchpad, full, reduce_max_i32, sym_expr }; - -pub const GraphOpts = struct { - options: []const mlir.NamedAttribute = &.{}, - context: ?*const mlir.Attribute = null, -}; - -/// `tcl.graph.`: a graph-level TCL operation whose fields go in `options`. -pub fn graph(ctx: *mlir.Context, comptime op: GraphOp, inputs: []const *const mlir.Value, output: *const mlir.Type, opts: GraphOpts, location: *const mlir.Location) *mlir.Operation { - var attrs: [2]mlir.NamedAttribute = undefined; - attrs[0] = .named(ctx, "options", .dict(ctx, opts.options)); - if (opts.context) |a| attrs[1] = .named(ctx, "context", a); - return mlir.Operation.make(ctx, "tcl.graph." ++ @tagName(op), .{ - .operands = .{ .flat = inputs }, - .results = .{ .flat = &.{output} }, - .attributes = attrs[0 .. @as(usize, 1) + @intFromBool(opts.context != null)], - .verify = false, - .location = location, - }); -} - -/// `tcl.graph.copy`, `copy(src=value, dst=input)`: writes `src` into the -/// kernel input `dst` in place. -pub fn copy(ctx: *mlir.Context, src: *const mlir.Value, dst: *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tcl.graph.copy", .{ - .operands = .{ .flat = &.{ src, dst } }, - .results = .{ .flat = &.{} }, - .attributes = &.{.named(ctx, "options", .dict(ctx, &.{}))}, - .verify = false, - .location = location, - }); -} - -pub const LoopOpts = struct { - limit: ?*const mlir.Value = null, - bound: ?*const mlir.Attribute = null, -}; - -/// `tcl.graph.for`: runs `body` up to `limit` (a scalar i32 tensor) or `bound` -/// (an integer or axis attribute) times. The body's arguments are the index, -/// then one accumulator per `inits`; it ends with `tcl.graph.yield`. -pub fn loop(ctx: *mlir.Context, inits: []const *const mlir.Value, result_types: []const *const mlir.Type, body: *mlir.Block, opts: LoopOpts, location: *const mlir.Location) *mlir.Operation { - const limits: []const *const mlir.Value = if (opts.limit) |l| &.{l} else &.{}; - var attrs: [1]mlir.NamedAttribute = undefined; - if (opts.bound) |b| attrs[0] = .named(ctx, "bound", b); - return mlir.Operation.make(ctx, "tcl.graph.for", .{ - .operands = .{ .variadic = &.{ limits, inits } }, - .results = .{ .flat = result_types }, - .attributes = attrs[0..@intFromBool(opts.bound != null)], - .blocks = &.{body}, - .verify = false, - .location = location, - }); -} - -pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "tcl.graph.yield", .{ - .operands = .{ .flat = values }, - .verify = false, - .location = location, - }); -} diff --git a/mlir/dialects/tcl/tcl_attrs.cc b/mlir/dialects/tcl/tcl_attrs.cc deleted file mode 100644 index 5b2fe575..00000000 --- a/mlir/dialects/tcl/tcl_attrs.cc +++ /dev/null @@ -1,266 +0,0 @@ -//===- tcl_attrs.cc - Furiosa TCL attributes ------------------------------===// - -#include "mlir/dialects/tcl/tcl_attrs.h" - -#include "mlir/dialects/tcl/tcl_dialect.h" -#include "mlir/dialects/tcl/tcl_types.h" - -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/StringExtras.h" - -#include -namespace xla::furiosa::tcl { -using namespace mlir; -using Error = llvm::function_ref; -static bool Identifier(llvm::StringRef s) { - if (s.empty() || (!llvm::isAlpha(s.front()) && s.front() != '_')) - return false; - return llvm::all_of(s, [](char c) { return llvm::isAlnum(c) || c == '_'; }); -} -static bool Symbol(llvm::StringRef s) { - auto [base, view] = s.split('.'); - return Identifier(base) && - (view.empty() ? !s.contains('.') : Identifier(view)); -} -static bool Mapping(Attribute a) { - if (!a) - return false; - if (isa(a)) - return IsNumber(a); - if (isa(a)) - return true; - auto e = dyn_cast(a); - return e && - llvm::is_contained( - {"stride", "modulo", "padding", "resize", "pair", "broadcast"}, - e.getKind()) && - llvm::all_of(e.getArgs(), Mapping); -} -static bool Json(Attribute a) { - if (auto n = dyn_cast(a)) - return n.getValue().isFinite(); - if (IsNumber(a) || isa(a)) - return true; - if (auto arr = dyn_cast(a)) - return llvm::all_of(arr, Json); - if (auto dict = dyn_cast(a)) - return llvm::all_of(dict, - [](NamedAttribute f) { return Json(f.getValue()); }); - return false; -} -static LogicalResult Enum(Error error, llvm::StringRef value, - llvm::ArrayRef values) { - return llvm::is_contained(values, value) - ? success() - : error() << "invalid enumerator: " << value; -} - -bool IsNumber(Attribute a) { - if (!a) - return false; - if (auto i = dyn_cast(a)) - return i.getValue().getBitWidth() <= 64; - return isa(a); -} -bool IsExpression(Attribute a) { - if (!a) - return false; - if (isa(a)) - return IsNumber(a); - if (isa(a)) - return true; - if (auto e = dyn_cast(a)) - return e.getKind() != "broadcast" && e.getKind() != "pair" && - e.getKind() != "padding" && e.getKind() != "resize"; - return false; -} -bool IsAxis(Attribute a) { - if (!a) - return false; - auto s = dyn_cast(a); - return s && llvm::isUpper(s.getName().front()); -} -bool IsAxes(Attribute a) { - if (!a) - return false; - auto array = dyn_cast(a); - return array && llvm::all_of(array, IsAxis); -} -LogicalResult VerifyKeys(Error error, DictionaryAttr d, - llvm::ArrayRef keys) { - for (auto f : d) - if (!llvm::is_contained(keys, f.getName().strref())) - return error() << "unknown field: " << f.getName(); - return success(); -} -LogicalResult VerifyAxes(Error error, ArrayAttr axes) { - if (!IsAxes(axes)) - return error() << "axes must be uppercase TCL symbols"; - llvm::SmallDenseSet seen; - for (auto a : axes) - if (!seen.insert(a).second) - return error() << "duplicate logical axis"; - return success(); -} - -// opcode, emitted spelling, arity. Conversion widths are handled below. -static constexpr VeInstructionSpec kVeInstructions[] = { -#define VE(name, text, arity) {name, text, arity}, -#include "mlir/dialects/tcl/ve_instructions.inc" -#undef VE -}; -llvm::ArrayRef VeInstructions() { return kVeInstructions; } - -LogicalResult SymbolAttr::verify(Error error, llvm::StringRef name) { - return Symbol(name) ? success() - : error() << "expected a TCL identifier, optionally with " - "one view suffix"; -} -LogicalResult ExprAttr::verify(Error error, llvm::StringRef kind, - ArrayAttr args) { - if (kind == "broadcast") - return args.empty() ? success() : error() << "broadcast takes no arguments"; - if (kind == "arg") { - if (args.size() != 1 || !isa(args[0]) || !IsNumber(args[0]) || - cast(args[0]).getInt() < 0) - return error() << "arg requires one nonnegative operand index"; - return success(); - } - bool map = llvm::is_contained( - {"stride", "modulo", "padding", "resize", "pair"}, kind); - bool arithmetic = - llvm::is_contained({"add", "sub", "mul", "div", "rem", "exact_div", "eq", - "ne", "lt", "le", "gt", "ge"}, - kind); - if ((!map && !arithmetic) || args.size() != 2) - return error() << "expected a known binary expression"; - for (auto a : args) - if (!(map ? Mapping(a) : IsExpression(a))) - return error() << "invalid expression operand"; - if (llvm::is_contained({"div", "exact_div", "rem", "stride", "modulo"}, kind)) - if (auto n = dyn_cast(args[1]); n && n.getInt() == 0) - return error() << "division by zero"; - return success(); -} -LogicalResult TacticAttr::verify(Error e, llvm::StringRef v) { - return Enum(e, v, - {"EinsumByDpe", "ReduceByVe", "Interleaving", "Elementwise", - "TensorOperation", "EinsumByVe", "FilterCompaction"}); -} -LogicalResult VeOpcodeAttr::verify(Error e, llvm::StringRef v) { - for (const auto &spec : VeInstructions()) - if (spec.name == v) - return success(); - if (v.consume_front("to_f") || v.consume_front("to_i")) { - unsigned width; - if (!v.getAsInteger(10, width) && width <= 31 && std::to_string(width) == v) - return success(); - } - return e() << "unknown VE opcode"; -} -LogicalResult ReduceModeAttr::verify(Error e, llvm::StringRef v) { - return Enum(e, v, {"Addi", "Addf", "Maxi", "Maxf", "Mini", "Minf", "Cumsum"}); -} - -LogicalResult PredicateAttr::verify(Error e, llvm::StringRef v) { - return Enum(e, v, {"eq", "ne", "lt", "le", "gt", "ge"}); -} -LogicalResult ConfigAttr::verify(Error e, DictionaryAttr d) { - return Json(d) ? success() - : e() << "compiler/auto configuration requires " - "JSON-compatible attributes"; -} - -LogicalResult ReadOptionsAttr::verify(Error e, DictionaryAttr d) { - if (failed(VerifyKeys(e, d, - {"subtraction", "table_lookup", "broadcast_to", - "typecast_to", "pad", "slide"}))) - return failure(); - for (auto field : d) { - auto k = field.getName().strref(); - auto v = field.getValue(); - if (k == "subtraction" || k == "table_lookup") { - auto i = dyn_cast(v); - if (!i || !IsNumber(i) || i.getInt() < 1) - return e() << k << " must index an auxiliary read operand (>=1)"; - } else if (k == "broadcast_to") { - if (!IsAxes(v)) - return e() << "broadcast_to requires axes"; - } else if (k == "typecast_to") { - auto t = dyn_cast(v); - if (!t || ElementName(t.getValue()).empty()) - return e() << "invalid typecast element type"; - } else if (k == "slide") { - auto entries = dyn_cast(v); - if (!entries) - return e() << "slide requires an axis dictionary"; - for (auto item : entries) { - auto spec = dyn_cast(item.getValue()); - if (!Symbol(item.getName()) || - !llvm::isUpper(item.getName().strref().front()) || !spec || - failed(VerifyKeys(e, spec, - {"undilated_window", "frame_axis", "window_axis", - "stride", "dilation"})) || - !IsExpression(spec.get("undilated_window")) || - !IsAxis(spec.get("frame_axis")) || !IsAxis(spec.get("window_axis"))) - return e() << "slide requires undilated_window, frame_axis and " - "window_axis"; - for (auto key : {"stride", "dilation"}) - if (auto a = spec.get(key); a && !IsExpression(a)) - return e() << "invalid slide " << key; - } - } else { - auto entries = dyn_cast(v); - if (!entries) - return e() << k << " requires an axis dictionary"; - for (auto item : entries) { - if (!Symbol(item.getName()) || - !llvm::isUpper(item.getName().strref().front())) - return e() << "invalid axis"; - auto p = dyn_cast(item.getValue()); - if (!p || p.size() != 3 || !IsExpression(p[0]) || !IsExpression(p[1]) || - !IsNumber(p[2])) - return e() << "padding requires [left, right, fill]"; - } - } - } - return success(); -} -LogicalResult ContextAttr::verify(Error e, DictionaryAttr d) { - if (failed(VerifyKeys(e, d, {"operator", "heuristic_hint"}))) - return failure(); - if (auto a = d.get("operator")) { - auto layout = dyn_cast(a); - if (!layout || failed(VerifyKeys(e, layout, {"Chip", "Cluster", "Split"}))) - return e() << "invalid operator layout"; - if (!layout.get("Chip")) - return e() << "operator layout requires Chip"; - for (auto f : layout) { - if (f.getName() == "Split") { - if (!IsAxes(f.getValue())) - return e() << "Split requires axes"; - } else if (!IsAxis(f.getValue()) && - !(isa(f.getValue()) && - cast(f.getValue()).getKind() == "broadcast")) - return e() << "Chip/Cluster requires axis or Broadcast"; - } - } - if (auto a = d.get("heuristic_hint")) { - auto hints = dyn_cast(a); - if (!hints) - return e() << "heuristic_hint requires a dictionary"; - for (auto f : hints) - if (!Identifier(f.getName()) || - !(IsNumber(f.getValue()) || IsAxis(f.getValue()))) - return e() << "invalid heuristic hint"; - } - return success(); -} - -LogicalResult DramMappingAttr::verify(Error e, ArrayAttr chip, ArrayAttr inner, - ArrayAttr original) { - if (!llvm::all_of(chip, Mapping) || !llvm::all_of(inner, Mapping)) - return e() << "invalid DRAM mapping"; - return VerifyAxes(e, original); -} -} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_attrs.h b/mlir/dialects/tcl/tcl_attrs.h deleted file mode 100644 index 816583b9..00000000 --- a/mlir/dialects/tcl/tcl_attrs.h +++ /dev/null @@ -1,37 +0,0 @@ -//===- tcl_attrs.h - Furiosa TCL attributes ---------------------*- C++ -*-===// - -#ifndef MLIR_DIALECTS_TCL_TCL_ATTRS_H -#define MLIR_DIALECTS_TCL_TCL_ATTRS_H - -#include "mlir/dialects/tcl/tcl_dialect.h" - -#include "mlir/IR/Attributes.h" -#include "mlir/IR/BuiltinAttributes.h" -#include "mlir/IR/Diagnostics.h" -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/STLFunctionalExtras.h" -#include "llvm/ADT/StringRef.h" -#define GET_ATTRDEF_CLASSES -#include "mlir/dialects/tcl/tcl_attrs.h.inc" -namespace xla::furiosa::tcl { -struct VeInstructionSpec { - llvm::StringRef name; - llvm::StringRef spelling; - unsigned arity; -}; -llvm::ArrayRef VeInstructions(); - -// Schema predicates shared by attribute, type and operation verifiers. -bool IsNumber(mlir::Attribute a); -bool IsExpression(mlir::Attribute a); -bool IsAxis(mlir::Attribute a); -bool IsAxes(mlir::Attribute a); -mlir::LogicalResult -VerifyKeys(llvm::function_ref error, - mlir::DictionaryAttr d, llvm::ArrayRef keys); -mlir::LogicalResult -VerifyAxes(llvm::function_ref error, - mlir::ArrayAttr axes); -} // namespace xla::furiosa::tcl - -#endif // MLIR_DIALECTS_TCL_TCL_ATTRS_H diff --git a/mlir/dialects/tcl/tcl_attrs.td b/mlir/dialects/tcl/tcl_attrs.td deleted file mode 100644 index f85c518e..00000000 --- a/mlir/dialects/tcl/tcl_attrs.td +++ /dev/null @@ -1,74 +0,0 @@ -//===- tcl_attrs.td - Furiosa TCL attributes ---------------*- tablegen -*-===// - -#ifndef MLIR_DIALECTS_TCL_TCL_ATTRS_TD -#define MLIR_DIALECTS_TCL_TCL_ATTRS_TD - -include "mlir/IR/AttrTypeBase.td" -include "mlir/dialects/tcl/tcl_dialect.td" - -def Tcl_Symbol : AttrDef { - let mnemonic = "symbol"; - let parameters = (ins StringRefParameter<>:$name); - let assemblyFormat = "`<` $name `>`"; - let genVerifyDecl = 1; -} -def Tcl_Expr : AttrDef { - let mnemonic = "expr"; - let summary = "Checked symbolic arithmetic or DRAM mapping expression tree"; - let parameters = (ins StringRefParameter<>:$kind, "::mlir::ArrayAttr":$args); - let assemblyFormat = "`<` $kind `,` $args `>`"; - let genVerifyDecl = 1; -} -def Tcl_Tactic : AttrDef { - let mnemonic = "tactic"; - let parameters = (ins StringRefParameter<>:$value); - let assemblyFormat = "`<` $value `>`"; - let genVerifyDecl = 1; -} -def Tcl_VeOpcode : AttrDef { - let mnemonic = "ve_opcode"; - let parameters = (ins StringRefParameter<>:$value); - let assemblyFormat = "`<` $value `>`"; - let genVerifyDecl = 1; -} -def Tcl_ReduceMode : AttrDef { - let mnemonic = "reduce_mode"; - let parameters = (ins StringRefParameter<>:$value); - let assemblyFormat = "`<` $value `>`"; - let genVerifyDecl = 1; -} - -def Tcl_Predicate : AttrDef { - let mnemonic = "predicate"; - let parameters = (ins StringRefParameter<>:$value); - let assemblyFormat = "`<` $value `>`"; - let genVerifyDecl = 1; -} -def Tcl_Context : AttrDef { - let mnemonic = "context"; - let parameters = (ins "::mlir::DictionaryAttr":$fields); - let assemblyFormat = "`<` $fields `>`"; - let genVerifyDecl = 1; -} -def Tcl_ReadOptions : AttrDef { - let mnemonic = "read_options"; - let parameters = (ins "::mlir::DictionaryAttr":$fields); - let assemblyFormat = "`<` $fields `>`"; - let genVerifyDecl = 1; -} - -def Tcl_Config : AttrDef { - let mnemonic = "config"; - let parameters = (ins "::mlir::DictionaryAttr":$fields); - let assemblyFormat = "`<` $fields `>`"; - let genVerifyDecl = 1; -} -def Tcl_DramMapping : AttrDef { - let mnemonic = "dram_mapping"; - let parameters = (ins "::mlir::ArrayAttr":$chip, "::mlir::ArrayAttr":$inner, - "::mlir::ArrayAttr":$original); - let assemblyFormat = "`<` $chip `,` $inner `,` $original `>`"; - let genVerifyDecl = 1; -} - -#endif // MLIR_DIALECTS_TCL_TCL_ATTRS_TD diff --git a/mlir/dialects/tcl/tcl_dialect.cc b/mlir/dialects/tcl/tcl_dialect.cc deleted file mode 100644 index 2cea51a1..00000000 --- a/mlir/dialects/tcl/tcl_dialect.cc +++ /dev/null @@ -1,34 +0,0 @@ -//===- tcl_dialect.cc - Furiosa TCL dialect -------------------------------===// - -#include "mlir/dialects/tcl/tcl_dialect.h" - -#include "mlir/dialects/tcl/tcl_attrs.h" // IWYU pragma: keep -#include "mlir/dialects/tcl/tcl_ops.h" // IWYU pragma: keep -#include "mlir/dialects/tcl/tcl_types.h" // IWYU pragma: keep - -#include "mlir/IR/Builders.h" // IWYU pragma: keep -#include "mlir/IR/DialectImplementation.h" // IWYU pragma: keep -#include "llvm/ADT/TypeSwitch.h" // IWYU pragma: keep - -// Storage definitions must be complete where the dialect registers them. -#include "mlir/dialects/tcl/tcl_dialect.cc.inc" -#define GET_ATTRDEF_CLASSES -#include "mlir/dialects/tcl/tcl_attrs.cc.inc" -#define GET_TYPEDEF_CLASSES -#include "mlir/dialects/tcl/tcl_types.cc.inc" -namespace xla::furiosa::tcl { -void TclDialect::initialize() { - addAttributes< -#define GET_ATTRDEF_LIST -#include "mlir/dialects/tcl/tcl_attrs.cc.inc" - >(); - addTypes< -#define GET_TYPEDEF_LIST -#include "mlir/dialects/tcl/tcl_types.cc.inc" - >(); - addOperations< -#define GET_OP_LIST -#include "mlir/dialects/tcl/tcl_ops.cc.inc" - >(); -} -} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_dialect.h b/mlir/dialects/tcl/tcl_dialect.h deleted file mode 100644 index 97267065..00000000 --- a/mlir/dialects/tcl/tcl_dialect.h +++ /dev/null @@ -1,9 +0,0 @@ -//===- tcl_dialect.h - Furiosa TCL dialect ----------------------*- C++ -*-===// - -#ifndef MLIR_DIALECTS_TCL_TCL_DIALECT_H -#define MLIR_DIALECTS_TCL_TCL_DIALECT_H - -#include "mlir/IR/Dialect.h" // IWYU pragma: keep -#include "mlir/dialects/tcl/tcl_dialect.h.inc" - -#endif // MLIR_DIALECTS_TCL_TCL_DIALECT_H diff --git a/mlir/dialects/tcl/tcl_dialect.td b/mlir/dialects/tcl/tcl_dialect.td deleted file mode 100644 index 7b86a825..00000000 --- a/mlir/dialects/tcl/tcl_dialect.td +++ /dev/null @@ -1,19 +0,0 @@ -//===- tcl_dialect.td - Furiosa TCL dialect ----------------*- tablegen -*-===// - -#ifndef MLIR_DIALECTS_TCL_TCL_DIALECT_TD -#define MLIR_DIALECTS_TCL_TCL_DIALECT_TD - -include "mlir/IR/DialectBase.td" - -// Native TCL: tensors over named axes, `tk.*` tensor operations and graph -// operations, one-to-one with the TCL source the Furiosa compiler takes. The -// subset of xla/backends/furiosa/codegen/tcl/ir that ZML emits. -def Tcl_Dialect : Dialect { - let name = "tcl"; - let cppNamespace = "::xla::furiosa::tcl"; - let summary = "Typed Furiosa Tensor Compute Language graph"; - let useDefaultTypePrinterParser = 1; - let useDefaultAttributePrinterParser = 1; -} - -#endif // MLIR_DIALECTS_TCL_TCL_DIALECT_TD diff --git a/mlir/dialects/tcl/tcl_ops.cc b/mlir/dialects/tcl/tcl_ops.cc deleted file mode 100644 index 9d569f0b..00000000 --- a/mlir/dialects/tcl/tcl_ops.cc +++ /dev/null @@ -1,473 +0,0 @@ -//===- tcl_ops.cc - Furiosa TCL operations --------------------------------===// - -#include "mlir/dialects/tcl/tcl_ops.h" - -#include "mlir/IR/Builders.h" // IWYU pragma: keep -#include "mlir/IR/Matchers.h" -#include "llvm/ADT/DenseSet.h" -#include "llvm/ADT/STLExtras.h" -#define GET_OP_CLASSES -#include "mlir/dialects/tcl/tcl_ops.cc.inc" -namespace xla::furiosa::tcl { -using namespace mlir; -static bool Tensor(Type t) { return isa(t); } -static bool ScalarI32(Type t) { - auto l = dyn_cast(t); - return l && l.getAxes().empty() && l.getElementType().isInteger(32); -} -static bool LoopIndex(Value v) { - auto arg = dyn_cast(v); - return arg && arg.getArgNumber() == 0 && - isa_and_nonnull(arg.getOwner()->getParentOp()); -} -static LogicalResult InKernel(Operation *op) { - if (!op->getParentOfType()) - return op->emitOpError("requires a tcl.kernel region"); - return success(); -} -// Exact graph operation schema. Optional operands occupy documented positions; -// the options dictionary never changes operand ordering. -static LogicalResult VerifyGraph(Operation *op, DictionaryAttr options) { - auto e = [&] { return op->emitOpError(); }; - llvm::StringRef n = op->getName().getStringRef().drop_front(10); - int min = 1, max = 1, results = 1; - llvm::SmallVector fields; - if (n == "all_gather") { - fields = {"axis"}; - } else if (n == "gather" || n == "scatter") { - min = 2; - max = 3; - fields = {"axis", "batch_axis", "table_dram_layout"}; - } else if (n == "concat") { - max = -1; - } else if (n == "slice") { - fields = {"axis", "offset"}; - } else if (n == "arange") { - min = max = 0; - fields = {"start", "end", "step"}; - } else if (n == "vector") { - min = max = 0; - fields = {"values", "repeat_to"}; - } else if (n == "index_read") { - min = max = 2; - } else if (n == "index_write") { - min = max = 3; - } else if (n == "copy") { - min = max = 2; - results = 0; - } else if (n == "scratchpad") { - min = max = 0; - } else if (n == "full") { - min = max = 0; - fields = {"value"}; - } else if (n == "sym_expr") { - min = 0; - max = -1; - fields = {"expr"}; - } - if (op->getNumOperands() < min || (max >= 0 && op->getNumOperands() > max) || - op->getNumResults() != results) - return e() << "incorrect operand or result count"; - if (failed(VerifyKeys(e, options, fields))) - return failure(); - if (n == "gather" && options.get("batch_axis") && op->getNumOperands() == 3) - return e() << "SDK sparse gather cannot also use batch_axis"; - for (Type t : op->getOperandTypes()) - if (!Tensor(t)) - return e() << "expected a native TCL tensor operand"; - for (Type t : op->getResultTypes()) - if (!Tensor(t)) - return e() << "expected a native TCL tensor result"; - for (auto f : options) { - auto k = f.getName().strref(); - auto v = f.getValue(); - bool ok = false; - if (k.ends_with("axis") || k == "repeat_to") - ok = IsAxis(v); - else if (k == "axes") - ok = IsAxes(v); - else if (k == "value") - ok = IsNumber(v); - else if (k == "values") { - auto a = dyn_cast(v); - ok = a && !a.empty() && llvm::all_of(a, IsNumber); - } else if (k == "table_dram_layout") - ok = isa(v) && isa(cast(v).getValue()); - else - ok = IsExpression(v); - if (!ok) - return e() << "invalid field " << k; - } - for (auto k : fields) { - bool required = llvm::is_contained( - {"axis", "offset", "end", "values", "expr", "value"}, k); - if (required && !options.get(k)) - return e() << "missing required field " << k; - } - if (n == "as_logical" && (!isa(op->getOperand(0).getType()) || - !isa(op->getResult(0).getType()))) - return e() << "as_logical requires storage -> logical"; - if (n == "as_dram" && (!isa(op->getOperand(0).getType()) || - !isa(op->getResult(0).getType()))) - return e() << "as_dram requires logical -> mapped"; - if (n == "index_read" || n == "index_write") { - if (!LoopIndex(op->getOperand(1))) - return e() << "requires the index of an enclosing loop"; - auto table = dyn_cast(op->getOperand(0).getType()); - auto slice = dyn_cast( - (n == "index_read" ? op->getResult(0) : op->getOperand(2)).getType()); - if (!table || !slice || table.getAxes().empty() || - table.getElementType() != slice.getElementType() || - table.getAxes().getValue().drop_front() != slice.getAxes().getValue()) - return e() << "slice must be the table without its outermost axis"; - if (n == "index_write" && - (op->getResult(0).getType() != table || !op->getOperand(0).hasOneUse())) - return e() << "writes in place: the table must have no other use"; - } - if (n == "copy" && - (!isa(op->getOperand(1)) || - op->getOperand(0).getType() != op->getOperand(1).getType())) - return e() << "copies a value into a kernel input of its type"; - if ((n == "reduce_max_i32" || n == "sym_expr") && - !ScalarI32(op->getResult(0).getType())) - return e() << "requires a scalar i32 result"; - if (n == "sym_expr") { - for (Value v : op->getOperands()) - if (!ScalarI32(v.getType())) - return e() << "requires scalar i32 operands"; - llvm::SmallVector pending{options.get("expr")}; - while (!pending.empty()) { - auto x = dyn_cast(pending.pop_back_val()); - if (!x) - continue; - if (x.getKind() == "arg") { - if (cast(x.getArgs()[0]).getInt() >= op->getNumOperands()) - return e() << "expression operand out of range"; - continue; - } - pending.append(x.getArgs().begin(), x.getArgs().end()); - } - } - return success(); -} - -LogicalResult KernelOp::verify() { - if (getBody().empty() || getBody().front().empty() || - getBody().front().getNumArguments()) - return emitOpError("requires one body without block arguments"); - auto write = dyn_cast(getBody().front().getTerminator()); - if (!write) - return emitOpError("requires tcl.write terminator"); - for (auto input : getInputs()) - if (!Tensor(input.getType())) - return emitOpError("requires native tensor inputs"); - for (Operation &op : getBody().front()) { - if (!isa(op) && - op.getName().getStringRef() != "arith.constant") - return emitOpError("unsupported kernel instruction ") << op.getName(); - } - WalkResult captured = getBody().walk([&](Operation *op) { - for (Value v : op->getOperands()) { - if (getBody().isAncestor(v.getParentRegion())) - continue; - if (!isa(op) || !llvm::is_contained(getInputs(), v)) { - op->emitOpError( - "external values must be explicit kernel inputs consumed by read"); - return WalkResult::interrupt(); - } - } - return WalkResult::advance(); - }); - if (captured.wasInterrupted()) - return failure(); - return success(); -} -LogicalResult ReadOp::verify() { - if (failed(InKernel(*this))) - return failure(); - if (getInputs().empty() || - !llvm::all_of(getInputs(), [](Value v) { return Tensor(v.getType()); })) - return emitOpError("requires tensor inputs"); - if (getOptions()) - for (auto k : {"subtraction", "table_lookup"}) - if (auto a = getOptions()->getFields().getAs(k); - a && a.getInt() >= getInputs().size()) - return emitOpError("auxiliary read operand index out of bounds"); - if (auto input = dyn_cast(getInputs()[0].getType())) { - SmallVector axes(input.getAxes().begin(), input.getAxes().end()); - Type element = input.getElementType(); - if (getOptions()) { - auto fields = getOptions()->getFields(); - if (auto table = fields.getAs("table_lookup")) { - auto t = dyn_cast(getInputs()[table.getInt()].getType()); - if (!t || t.getAxes().size() != 1) - return emitOpError("lookup table must be a one-axis logical tensor"); - element = t.getElementType(); - } - if (auto cast = fields.getAs("typecast_to")) - element = cast.getValue(); - if (auto pads = fields.getAs("pad")) - for (auto pad : pads) - if (!llvm::is_contained(axes, - SymbolAttr::get(getContext(), pad.getName()))) - return emitOpError("padding axis missing from input"); - // A slide replaces its axis with the window and frame axes. - if (auto slides = fields.getAs("slide")) - for (auto slide : slides) { - auto it = - llvm::find(axes, SymbolAttr::get(getContext(), slide.getName())); - if (it == axes.end()) - return emitOpError("slide axis missing from input"); - auto spec = cast(slide.getValue()); - *it = spec.get("window_axis"); - axes.insert(it + 1, spec.get("frame_axis")); - } - if (auto broadcast = fields.getAs("broadcast_to")) { - for (auto axis : axes) - if (!llvm::is_contained(broadcast, axis)) - return emitOpError("broadcast must retain all read axes"); - axes.assign(broadcast.begin(), broadcast.end()); - } - } - if (getOutput().getType().getAxes().getValue() != - ArrayRef(axes) || - getOutput().getType().getElementType() != element) - return emitOpError("read result disagrees with its modifiers"); - } - return success(); -} -LogicalResult DpeOp::verify() { - if (failed(InKernel(*this))) - return failure(); - auto out = getOutput().getType(); - if (!out.getElementType().isF32() && - !out.getElementType().isSignlessInteger(32)) - return emitOpError("DPE accumulators require f32 or i32"); - for (auto axis : out.getAxes()) - if (!llvm::is_contained(getLhs().getType().getAxes(), axis) && - !llvm::is_contained(getRhs().getType().getAxes(), axis)) - return emitOpError("DPE result axis missing from both inputs"); - return success(); -} -LogicalResult VeOp::verify() { - if (failed(InKernel(*this))) - return failure(); - unsigned arity = 1; - for (auto spec : VeInstructions()) - if (spec.name == getOpcode().getValue()) - arity = spec.arity; - if (getInputs().size() != arity) - return emitOpError("incorrect VE operand count"); - llvm::SmallDenseSet axes; - for (Value v : getInputs()) { - if (auto type = dyn_cast(v.getType())) { - for (auto a : type.getAxes()) - axes.insert(a); - } else if (!isa(v.getType()) || - !matchPattern(v, m_Constant())) - return emitOpError("VE scalar operands must be constants"); - } - auto out = getOutput().getType(); - if (axes.size() != out.getAxes().size() || - !llvm::all_of(out.getAxes(), - [&](Attribute a) { return axes.contains(a); })) - return emitOpError("VE output axes must be the union of operand axes"); - auto code = getOpcode().getValue(); - llvm::StringRef spelling = code; - for (auto spec : VeInstructions()) - if (spec.name == code) - spelling = spec.spelling; - bool fp = spelling.starts_with("to_f") || spelling == "as_f" || - spelling.ends_with("_f") || - llvm::is_contained({"+", "-", "*", "/", "exp", "neg_exp", "sqrt", - "tanh", "sigmoid", "erf", "log", "sin", "cos"}, - spelling); - if (fp ? !out.getElementType().isF32() - : !out.getElementType().isSignlessInteger(32)) - return emitOpError("VE registers produce f32 or i32 according to opcode"); - return success(); -} -LogicalResult VeReduceOp::verify() { - if (failed(InKernel(*this)) || - failed(VerifyAxes([&] { return emitOpError(); }, getAxes()))) - return failure(); - // An integer cumulative sum runs along its axes and keeps them. - if (getMode().getValue() == "Cumsum") { - if (getAxes().empty() || - getInput().getType().getAxes() != getOutput().getType().getAxes() || - !llvm::all_of(getAxes(), - [&](Attribute a) { - return llvm::is_contained( - getInput().getType().getAxes(), a); - }) || - !getInput().getType().getElementType().isInteger(32)) - return emitOpError("cumsum keeps the axes of an i32 value"); - return success(); - } - llvm::SmallVector remaining; - for (Attribute a : getInput().getType().getAxes()) - if (!llvm::is_contained(getAxes(), a)) - remaining.push_back(a); - if (getAxes().empty() || - remaining.size() + getAxes().size() != - getInput().getType().getAxes().size() || - remaining != getOutput().getType().getAxes().getValue()) - return emitOpError("reduction result must remove exactly the reduced axes"); - return success(); -} - -LogicalResult VeSelectOp::verify() { - if (failed(InKernel(*this)) || !IsNumber(getThreshold())) - return emitOpError("requires kernel and numeric threshold"); - llvm::SmallDenseSet axes; - for (Value v : - SmallVector{getCondition(), getTrueValue(), getFalseValue()}) { - if (auto t = dyn_cast(v.getType())) { - for (auto a : t.getAxes()) - axes.insert(a); - } else if (!matchPattern(v, m_Constant())) - return emitOpError("scalar branches require constants"); - } - auto output = getOutput().getType().getAxes(); - if (axes.size() != output.size() || - !llvm::all_of(output, [&](Attribute a) { return axes.contains(a); })) - return emitOpError("select output axes must cover its operands"); - return success(); -} - -LogicalResult FilterCompactionOp::verify() { - if (failed(InKernel(*this)) || - failed(VerifyAxes([&] { return emitOpError(); }, getAxes()))) - return failure(); - auto size = dyn_cast(getMaxSize()); - if (!IsNumber(getThreshold()) || - (!IsAxis(getMaxSize()) && !(size && size.getInt() > 0)) || - getAxes().empty()) - return emitOpError( - "requires numeric threshold, positive maximum size, and axes"); - if (getInput().getType().getAxes() != getCondition().getType().getAxes() || - getInput().getType().getElementType() != - getOutput().getType().getElementType() || - !llvm::all_of(getAxes(), [&](Attribute axis) { - return llvm::is_contained(getInput().getType().getAxes(), axis); - })) - return emitOpError("incompatible compaction input, condition, or axes"); - if (!llvm::all_of(getOutput().getUsers(), - [](Operation *op) { return isa(op); })) - return emitOpError( - "filter_compaction must be the final kernel instruction"); - return success(); -} - -LogicalResult WriteOp::verify() { - auto kernel = dyn_cast((*this)->getParentOp()); - if (!kernel) - return emitOpError("requires a kernel parent"); - auto in = getInput().getType(), out = kernel.getOutput().getType(); - if (in.getAxes().size() != out.getAxes().size()) - return emitOpError("write must preserve the axis set"); - for (auto a : in.getAxes()) - if (!llvm::is_contained(out.getAxes(), a)) - return emitOpError("write must preserve the axis set"); - // TCL write.to implicitly consumes the preceding instruction. - auto previous = (*this)->getPrevNode(); - while (previous && previous->getName().getStringRef() == "arith.constant") - previous = previous->getPrevNode(); - if (getInput().getDefiningOp() != previous) - return emitOpError( - "write must consume the immediately preceding instruction"); - return success(); -} - -LogicalResult GraphGatherOp::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphAllGatherOp::verify() { - if (failed(VerifyGraph(*this, getOptions()))) - return failure(); - auto input = dyn_cast(getInputs()[0].getType()); - auto output = dyn_cast(getOutputs()[0].getType()); - if (!input || !output || input != output) - return emitOpError("all_gather requires matching logical tensor types"); - if (!llvm::is_contained(input.getAxes(), getOptions().get("axis"))) - return emitOpError("collective axis must be present in the input"); - return success(); -} -LogicalResult GraphScatterOp::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphReshapeOp::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphTransmuteOp::verify() { - return VerifyGraph(*this, getOptions()); -} - -LogicalResult GraphConcatOp::verify() { - return VerifyGraph(*this, getOptions()); -} - -LogicalResult GraphSliceOp::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphArangeOp::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphVectorOp::verify() { - return VerifyGraph(*this, getOptions()); -} - -LogicalResult GraphAsLogicalOp::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphAsDramOp::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphCopyOp::verify() { return VerifyGraph(*this, getOptions()); } -LogicalResult GraphIndexReadOp::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphIndexWriteOp::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphScratchpadOp::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphFullOp::verify() { return VerifyGraph(*this, getOptions()); } -LogicalResult GraphReduceMaxI32Op::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphSymExprOp::verify() { - return VerifyGraph(*this, getOptions()); -} -LogicalResult GraphForOp::verify() { - if (!getLimit() == !getBound()) - return emitOpError("requires exactly one of a limit tensor or a bound"); - if (getLimit() && !ScalarI32(getLimit().getType())) - return emitOpError("limit must be a scalar i32 tensor"); - if (auto bound = getBoundAttr()) { - auto n = dyn_cast(bound); - if (!IsAxis(bound) && !(n && IsNumber(n) && n.getInt() > 0)) - return emitOpError("bound must be a positive integer or an axis"); - } - Block &body = getBody().front(); - if (body.getNumArguments() != getInits().size() + 1 || - !ScalarI32(body.getArgument(0).getType())) - return emitOpError("requires a scalar i32 index and one accumulator each"); - auto yield = dyn_cast(body.getTerminator()); - if (!yield) - return emitOpError("requires a tcl.graph.yield terminator"); - if (getOutputs().size() != getInits().size() || - yield.getValues().size() != getInits().size()) - return emitOpError("requires one result and one yield per accumulator"); - for (auto [i, init] : llvm::enumerate(getInits())) { - Type t = init.getType(); - if (!Tensor(t) || body.getArgument(i + 1).getType() != t || - getOutputs()[i].getType() != t || yield.getValues()[i].getType() != t) - return emitOpError("accumulator ") << i << " changes type"; - } - return success(); -} - -} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_ops.h b/mlir/dialects/tcl/tcl_ops.h deleted file mode 100644 index bce6ccae..00000000 --- a/mlir/dialects/tcl/tcl_ops.h +++ /dev/null @@ -1,19 +0,0 @@ -//===- tcl_ops.h - Furiosa TCL operations -----------------------*- C++ -*-===// - -#ifndef MLIR_DIALECTS_TCL_TCL_OPS_H -#define MLIR_DIALECTS_TCL_TCL_OPS_H - -#include "mlir/dialects/tcl/tcl_attrs.h" // IWYU pragma: export -#include "mlir/dialects/tcl/tcl_dialect.h" // IWYU pragma: export -#include "mlir/dialects/tcl/tcl_types.h" // IWYU pragma: export - -#include "mlir/Bytecode/BytecodeOpInterface.h" // IWYU pragma: keep -#include "mlir/IR/BuiltinAttributes.h" // IWYU pragma: keep -#include "mlir/IR/BuiltinTypes.h" // IWYU pragma: keep -#include "mlir/IR/OpDefinition.h" // IWYU pragma: keep -#include "mlir/IR/OpImplementation.h" // IWYU pragma: keep -#include "mlir/Interfaces/SideEffectInterfaces.h" // IWYU pragma: keep -#define GET_OP_CLASSES -#include "mlir/dialects/tcl/tcl_ops.h.inc" - -#endif // MLIR_DIALECTS_TCL_TCL_OPS_H diff --git a/mlir/dialects/tcl/tcl_ops.td b/mlir/dialects/tcl/tcl_ops.td deleted file mode 100644 index 23676ec9..00000000 --- a/mlir/dialects/tcl/tcl_ops.td +++ /dev/null @@ -1,122 +0,0 @@ -//===- tcl_ops.td - Furiosa TCL operations -----------------*- tablegen -*-===// - -#ifndef MLIR_DIALECTS_TCL_TCL_OPS_TD -#define MLIR_DIALECTS_TCL_TCL_OPS_TD - -include "mlir/IR/OpBase.td" -include "mlir/Interfaces/SideEffectInterfaces.td" -include "mlir/dialects/tcl/tcl_attrs.td" -include "mlir/dialects/tcl/tcl_dialect.td" -include "mlir/dialects/tcl/tcl_types.td" - -class Tcl_Op traits = []> - : Op; - -// Options contain typed attributes, never snippets of TCL source. Their exact -// per-operation schema is verified in tcl_ops.cc. Generic assembly is -// intentional: all native constructs support standard MLIR parsing, printing -// and bytecode. -class Tcl_GraphOp traits = [MemoryEffects<[MemRead]>]> - : Tcl_Op { - let arguments = (ins Variadic:$inputs, - DefaultValuedAttr:$options, - OptionalAttr:$context); - let results = (outs Variadic:$outputs); - let hasVerifier = 1; -} -def Tcl_GraphGatherOp : Tcl_GraphOp<"gather">; -def Tcl_GraphAllGatherOp : Tcl_GraphOp<"all_gather">; -def Tcl_GraphScatterOp : Tcl_GraphOp<"scatter">; -def Tcl_GraphReshapeOp : Tcl_GraphOp<"reshape">; -def Tcl_GraphTransmuteOp : Tcl_GraphOp<"transmute">; - -def Tcl_GraphConcatOp : Tcl_GraphOp<"concat">; - -def Tcl_GraphSliceOp : Tcl_GraphOp<"slice">; -def Tcl_GraphArangeOp : Tcl_GraphOp<"arange">; -def Tcl_GraphVectorOp : Tcl_GraphOp<"vector">; - -def Tcl_GraphAsLogicalOp : Tcl_GraphOp<"as_logical">; -def Tcl_GraphAsDramOp : Tcl_GraphOp<"as_dram">; -// `copy(src=value, dst=input)` writes a value into a kernel input in place. -def Tcl_GraphCopyOp - : Tcl_GraphOp<"copy", [MemoryEffects<[MemRead, MemWrite]>]>; - -// Loop-level graph constructs. `x[i]` reads and `acc[i] = v` writes the slice -// of the outermost axis at the loop index; `sym_expr` evaluates an integer -// expression (`#tcl.expr` names its operands) on the device. -def Tcl_GraphIndexReadOp : Tcl_GraphOp<"index_read">; -def Tcl_GraphIndexWriteOp : Tcl_GraphOp<"index_write">; -def Tcl_GraphScratchpadOp : Tcl_GraphOp<"scratchpad">; -def Tcl_GraphFullOp : Tcl_GraphOp<"full">; -def Tcl_GraphReduceMaxI32Op : Tcl_GraphOp<"reduce_max_i32">; -def Tcl_GraphSymExprOp : Tcl_GraphOp<"sym_expr">; - -// `outputs = for i in 0..limit (acc = init, ...): body; yield ...;`. The limit -// is a static bound (integer or axis) or a scalar i32 tensor computed in the -// function. The block arguments are the loop index, then the accumulators. -def Tcl_GraphForOp : Tcl_Op<"graph.for", - [AttrSizedOperandSegments, RecursiveMemoryEffects, SingleBlock]> { - let arguments = (ins Optional:$limit, Variadic:$inits, - OptionalAttr:$bound); - let results = (outs Variadic:$outputs); - let regions = (region SizedRegion<1>:$body); - let hasVerifier = 1; -} -def Tcl_GraphYieldOp : Tcl_Op<"graph.yield", - [Pure, Terminator, HasParent<"GraphForOp">]> { - let arguments = (ins Variadic:$values); -} - -def Tcl_KernelOp : Tcl_Op<"kernel", [MemoryEffects<[MemRead]>, SingleBlock]> { - let arguments = (ins Variadic:$inputs, Tcl_Tactic:$tactic, - OptionalAttr:$context); - let results = (outs Tcl_Logical:$output); - let regions = (region AnyRegion:$body); - let hasVerifier = 1; -} -def Tcl_ReadOp : Tcl_Op<"read", [Pure]> { - let arguments = (ins Variadic:$inputs, - OptionalAttr:$options); - let results = (outs Tcl_Logical:$output); - let hasVerifier = 1; -} -def Tcl_DpeOp : Tcl_Op<"dpe", [Pure]> { - let arguments = (ins Tcl_Logical:$lhs, Tcl_Logical:$rhs); - let results = (outs Tcl_Logical:$output); - let hasVerifier = 1; -} -def Tcl_VeOp : Tcl_Op<"ve", [Pure]> { - let arguments = (ins Variadic:$inputs, Tcl_VeOpcode:$opcode); - let results = (outs Tcl_Logical:$output); - let hasVerifier = 1; -} -def Tcl_VeReduceOp : Tcl_Op<"ve_reduce", [Pure]> { - let arguments = (ins Tcl_Logical:$input, Tcl_ReduceMode:$mode, - ArrayAttr:$axes); - let results = (outs Tcl_Logical:$output); - let hasVerifier = 1; -} - -def Tcl_VeSelectOp : Tcl_Op<"ve_select", [Pure]> { - let arguments = (ins Tcl_Logical:$condition, Tcl_Predicate:$predicate, - AnyAttr:$threshold, AnyType:$true_value, - AnyType:$false_value); - let results = (outs Tcl_Logical:$output); - let hasVerifier = 1; -} - -def Tcl_FilterCompactionOp : Tcl_Op<"filter_compaction", [Pure]> { - let arguments = (ins Tcl_Logical:$input, Tcl_Logical:$condition, - Tcl_Predicate:$predicate, AnyAttr:$threshold, - ArrayAttr:$axes, AnyAttr:$max_size); - let results = (outs Tcl_Logical:$output); - let hasVerifier = 1; -} - -def Tcl_WriteOp : Tcl_Op<"write", [Pure, Terminator]> { - let arguments = (ins Tcl_Logical:$input); - let hasVerifier = 1; -} - -#endif // MLIR_DIALECTS_TCL_TCL_OPS_TD diff --git a/mlir/dialects/tcl/tcl_types.cc b/mlir/dialects/tcl/tcl_types.cc deleted file mode 100644 index 8b7f919a..00000000 --- a/mlir/dialects/tcl/tcl_types.cc +++ /dev/null @@ -1,68 +0,0 @@ -//===- tcl_types.cc - Furiosa TCL types -----------------------------------===// - -#include "mlir/dialects/tcl/tcl_types.h" - -#include "mlir/dialects/tcl/tcl_attrs.h" -#include "mlir/dialects/tcl/tcl_dialect.h" - -#include "llvm/ADT/STLExtras.h" -namespace xla::furiosa::tcl { -using namespace mlir; -using Error = llvm::function_ref; -llvm::StringRef ElementName(Type t) { - if (t.isBF16()) - return "bf16"; - if (t.isF16()) - return "f16"; - if (t.isF32()) - return "f32"; - if (t.isF64()) - return "f64"; - if (isa(t)) - return "f4_e2"; - if (isa(t)) - return "f8_e4"; - if (isa(t)) - return "f8_e5"; - if (auto i = dyn_cast(t)) { - if (i.getWidth() == 1 && i.isSignless()) - return "bool"; - if (i.isUnsigned()) { - switch (i.getWidth()) { - case 8: - return "u8"; - case 16: - return "u16"; - case 32: - return "u32"; - case 64: - return "u64"; - } - } else { - switch (i.getWidth()) { - case 4: - return "i4"; - case 8: - return "i8"; - case 16: - return "i16"; - case 32: - return "i32"; - case 64: - return "i64"; - } - } - } - return {}; -} -LogicalResult LogicalType::verify(Error e, Type element, ArrayAttr axes) { - if (ElementName(element).empty()) - return e() << "unsupported TCL element type"; - return VerifyAxes(e, axes); -} -LogicalResult MappedType::verify(Error e, Type element, - DramMappingAttr mapping) { - return ElementName(element).empty() ? e() << "unsupported TCL element type" - : success(); -} -} // namespace xla::furiosa::tcl diff --git a/mlir/dialects/tcl/tcl_types.h b/mlir/dialects/tcl/tcl_types.h deleted file mode 100644 index b1fdd99d..00000000 --- a/mlir/dialects/tcl/tcl_types.h +++ /dev/null @@ -1,18 +0,0 @@ -//===- tcl_types.h - Furiosa TCL types --------------------------*- C++ -*-===// - -#ifndef MLIR_DIALECTS_TCL_TCL_TYPES_H -#define MLIR_DIALECTS_TCL_TCL_TYPES_H - -#include "mlir/dialects/tcl/tcl_attrs.h" -#include "mlir/dialects/tcl/tcl_dialect.h" - -#include "mlir/IR/BuiltinTypes.h" -#include "llvm/ADT/StringRef.h" -#define GET_TYPEDEF_CLASSES -#include "mlir/dialects/tcl/tcl_types.h.inc" -namespace xla::furiosa::tcl { -// Empty name denotes a type outside the SDK's 17 element types. -llvm::StringRef ElementName(mlir::Type type); -} // namespace xla::furiosa::tcl - -#endif // MLIR_DIALECTS_TCL_TCL_TYPES_H diff --git a/mlir/dialects/tcl/tcl_types.td b/mlir/dialects/tcl/tcl_types.td deleted file mode 100644 index 1cbe5e7f..00000000 --- a/mlir/dialects/tcl/tcl_types.td +++ /dev/null @@ -1,23 +0,0 @@ -//===- tcl_types.td - Furiosa TCL types --------------------*- tablegen -*-===// - -#ifndef MLIR_DIALECTS_TCL_TCL_TYPES_TD -#define MLIR_DIALECTS_TCL_TCL_TYPES_TD - -include "mlir/IR/AttrTypeBase.td" -include "mlir/dialects/tcl/tcl_dialect.td" - -def Tcl_Logical : TypeDef { - let mnemonic = "logical"; - let parameters = (ins "::mlir::Type":$elementType, "::mlir::ArrayAttr":$axes); - let assemblyFormat = "`<` $elementType `,` $axes `>`"; - let genVerifyDecl = 1; -} -def Tcl_Mapped : TypeDef { - let mnemonic = "mapped"; - let parameters = (ins "::mlir::Type":$elementType, - "::xla::furiosa::tcl::DramMappingAttr":$mapping); - let assemblyFormat = "`<` $elementType `,` $mapping `>`"; - let genVerifyDecl = 1; -} - -#endif // MLIR_DIALECTS_TCL_TCL_TYPES_TD diff --git a/mlir/dialects/tcl/test.zig b/mlir/dialects/tcl/test.zig deleted file mode 100644 index 5adc34c0..00000000 --- a/mlir/dialects/tcl/test.zig +++ /dev/null @@ -1,134 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const func = @import("mlir/dialects/func"); -const tcl = @import("mlir/dialects/tcl"); - -const operation_names: []const []const u8 = &.{ - "tcl.dpe", - "tcl.filter_compaction", - "tcl.graph.all_gather", - "tcl.graph.arange", - "tcl.graph.as_dram", - "tcl.graph.as_logical", - "tcl.graph.concat", - "tcl.graph.copy", - "tcl.graph.for", - "tcl.graph.full", - "tcl.graph.gather", - "tcl.graph.index_read", - "tcl.graph.index_write", - "tcl.graph.reduce_max_i32", - "tcl.graph.reshape", - "tcl.graph.scatter", - "tcl.graph.scratchpad", - "tcl.graph.slice", - "tcl.graph.sym_expr", - "tcl.graph.transmute", - "tcl.graph.vector", - "tcl.graph.yield", - "tcl.kernel", - "tcl.read", - "tcl.ve", - "tcl.ve_reduce", - "tcl.ve_select", - "tcl.write", -}; - -test { - std.testing.refAllDecls(tcl); - std.testing.refAllDecls(tcl.LogicalType); -} - -fn testContext() !*mlir.Context { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - inline for (.{ func, tcl }) |dialect| dialect.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - return ctx; -} - -test "every bound operation is registered" { - const ctx = try testContext(); - defer ctx.deinit(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} - -test "types and attributes verify" { - const ctx = try testContext(); - defer ctx.deinit(); - - const a = try tcl.symbolAttr(ctx, "A"); - const t = try tcl.logicalType(ctx, .float(ctx, .bf16), &.{a}); - try std.testing.expect(t.isA(tcl.LogicalType) != null); - try std.testing.expectError(error.InvalidMlir, tcl.symbolAttr(ctx, "not an axis")); - try std.testing.expectError(error.InvalidMlir, tcl.logicalType(ctx, .float(ctx, .bf16), &.{ a, a })); - try std.testing.expectError(error.InvalidMlir, tcl.veOpcodeAttr(ctx, "unknown")); - _ = try tcl.veOpcodeAttr(ctx, "to_f16"); -} - -/// `tcl.kernel(input) { read; filter_compaction along `axis`; write }`. -fn compactionKernel(ctx: *mlir.Context, entry: *mlir.Block, axis: *const mlir.Attribute) !*mlir.Operation { - const loc: *const mlir.Location = .unknown(ctx); - const n = try tcl.symbolAttr(ctx, "N"); - const k = try tcl.symbolAttr(ctx, "K"); - const input = try tcl.logicalType(ctx, .int(ctx, .i32), &.{n}); - const output = try tcl.logicalType(ctx, .int(ctx, .i32), &.{k}); - const body = mlir.Block.init(&.{}, &.{}); - const x = tcl.read(ctx, &.{entry.argument(0)}, input, .{}, loc).appendTo(body); - const compacted = tcl.filter_compaction(ctx, x.result(0), x.result(0), output, .ge, .int(ctx, .i32, 0), &.{axis}, k, loc).appendTo(body); - _ = tcl.write(ctx, compacted.result(0), loc).appendTo(body); - return tcl.kernel(ctx, &.{entry.argument(0)}, output, .filter_compaction, body, .{}, loc).appendTo(entry); -} - -test "filter_compaction verifies inside a kernel" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc: *const mlir.Location = .unknown(ctx); - const n = try tcl.symbolAttr(ctx, "N"); - const entry = mlir.Block.init(&.{try tcl.logicalType(ctx, .int(ctx, .i32), &.{n})}, &.{loc}); - // The kernel verifier walks to its inputs' regions. - const function = func.func(ctx, .{ .name = "compact", .block = entry, .results = &.{}, .location = loc, .verify = false }); - defer function.deinit(); - try std.testing.expect((try compactionKernel(ctx, entry, n)).verify()); - try std.testing.expect(!(try compactionKernel(ctx, entry, try tcl.symbolAttr(ctx, "Missing"))).verify()); -} - -test "function attributes carry their tcl names" { - const ctx = try testContext(); - defer ctx.deinit(); - const loc: *const mlir.Location = .unknown(ctx); - const entry = mlir.Block.init(&.{}, &.{}); - _ = func.returns(ctx, &.{}, loc).appendTo(entry); - const function = func.func(ctx, .{ .name = "f", .block = entry, .location = loc }); - defer function.deinit(); - const config = try tcl.configAttr(ctx, &.{.named(ctx, "lowering_mode", .string(ctx, "Heuristic"))}); - tcl.annotateFunction(ctx, function, &.{.named(ctx, "A", .int(ctx, .i64, 4))}, .{ - .updated_inputs = &.{-1}, - .compiler_config = config, - .auto_config = config, - .fusible = true, - }); - try std.testing.expect(function.verify()); - var text: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer text.deinit(); - try text.writer.print("{f}", .{function}); - try std.testing.expect(std.mem.indexOf(u8, text.written(), "attributes {tcl.auto_config = #tcl.config<{lowering_mode = \"Heuristic\"}>, tcl.axes = {A = 4 : i64}, tcl.compiler_config = #tcl.config<{lowering_mode = \"Heuristic\"}>, tcl.fusible, tcl.updated_inputs = array}") != null); -} - -test "attributes and types print and reparse" { - const ctx = try testContext(); - defer ctx.deinit(); - const a = try tcl.symbolAttr(ctx, "A"); - const t = try tcl.logicalType(ctx, .float(ctx, .bf16), &.{a}); - var type_text: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer type_text.deinit(); - try type_text.writer.print("{f}", .{t}); - try std.testing.expect((try mlir.Type.parse(ctx, type_text.written())).eql(t)); - var attr_text: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer attr_text.deinit(); - try attr_text.writer.print("{f}", .{a}); - try std.testing.expect((try mlir.Attribute.parse(ctx, attr_text.written())).eql(a)); -} diff --git a/mlir/dialects/tcl/ve_instructions.inc b/mlir/dialects/tcl/ve_instructions.inc deleted file mode 100644 index 657eddca..00000000 --- a/mlir/dialects/tcl/ve_instructions.inc +++ /dev/null @@ -1,48 +0,0 @@ -//===- ve_instructions.inc - Furiosa TCL VE instructions ------------------===// -// -// SDK 2026.3 VE opcode vocabulary; conversion widths 0..31 are parametric. - -VE("exp", "exp", 1) -VE("neg_exp", "neg_exp", 1) -VE("sqrt", "sqrt", 1) -VE("tanh", "tanh", 1) -VE("sigmoid", "sigmoid", 1) -VE("erf", "erf", 1) -VE("log", "log", 1) -VE("sin", "sin", 1) -VE("cos", "cos", 1) -VE("negf", "neg_f", 1) -VE("absf", "abs_f", 1) -VE("addi", "+_fxp", 2) -VE("addi_sat", "+_fxp_sat", 2) -VE("subi", "-_fxp", 2) -VE("subi_sat", "-_fxp_sat", 2) -VE("mul_q31", "*_fxp", 2) -VE("muli", "*_i", 2) -VE("shl", "<<", 2) -VE("shl_sat", "<<_sat", 2) -VE("shr_logical", ">>>", 2) -VE("shr_arith", ">>", 2) -VE("shr_arith_round", ">>_round", 2) -VE("bit_andi", "&_i", 2) -VE("bit_ori", "|_i", 2) -VE("bit_xori", "^_i", 2) -VE("mini", "min_i", 2) -VE("maxi", "max_i", 2) -VE("absmini", "|min|_i", 2) -VE("absmaxi", "|max|_i", 2) -VE("addf", "+", 2) -VE("subf", "-", 2) -VE("mulf", "*", 2) -VE("divf", "/", 2) -VE("bit_andf", "&_f", 2) -VE("bit_orf", "|_f", 2) -VE("bit_xorf", "^_f", 2) -VE("minf", "min_f", 2) -VE("maxf", "max_f", 2) -VE("absminf", "|min|_f", 2) -VE("absmaxf", "|max|_f", 2) -VE("reinterpret_as_float", "as_f", 1) -VE("reinterpret_as_int", "as_i", 1) -VE("to_float", "to_f31", 1) -VE("to_int", "to_i31", 1) diff --git a/mlir/dialects/ttir/BUILD.bazel b/mlir/dialects/ttir/BUILD.bazel deleted file mode 100644 index 8ab5598f..00000000 --- a/mlir/dialects/ttir/BUILD.bazel +++ /dev/null @@ -1,39 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - "@llvm-project//mlir:CAPIIR", - "@triton//:TritonDialects", - ], -) - -zig_library( - name = "ttir", - import_name = "mlir/dialects/ttir", - main = "ttir.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//stdx", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":ttir", - "//mlir", - "//mlir/dialects/arith", - "//mlir/dialects/func", - "//mlir/dialects/math", - "//mlir/dialects/scf", - ], -) diff --git a/mlir/dialects/ttir/capi.cc b/mlir/dialects/ttir/capi.cc deleted file mode 100644 index 0e7afb28..00000000 --- a/mlir/dialects/ttir/capi.cc +++ /dev/null @@ -1,167 +0,0 @@ -//===- capi.cc - C API for the Triton tt dialect --------------------------===// - -#include "capi.h" - -#include "triton/Dialect/Triton/IR/Dialect.h" -#include "triton/Dialect/Triton/IR/Types.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/CAPI/Registration.h" - -// Note: the second arg `tt` controls the generated symbol name -// `mlirGetDialectHandle__tt__`, which `mlir.DialectHandle.fromString("tt")` on -// the Zig side resolves at comptime (see `DialectHandle.fromString` in -// mlir/mlir.zig). Must match the dialect's declared name (`let name = "tt"` in -// TritonDialect.td). -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(Triton, tt, mlir::triton::TritonDialect) - -static mlir::MLIRContext *loadTriton(MlirContext context) { - mlir::MLIRContext *ctx = unwrap(context); - if (ctx) - ctx->getOrLoadDialect(); - return ctx; -} - -using mlir::cast; -using mlir::isa; - -MlirType mlirTritonPointerTypeGet(MlirType pointee, int32_t addressSpace) { - return wrap(mlir::triton::PointerType::get( - unwrap(pointee), static_cast(addressSpace))); -} - -MlirType mlirTritonPointerTypeGetPointee(MlirType type) { - return wrap(cast(unwrap(type)).getPointeeType()); -} - -int32_t mlirTritonPointerTypeGetAddressSpace(MlirType type) { - return static_cast( - cast(unwrap(type)).getAddressSpace()); -} - -bool mlirTypeIsATritonPointer(MlirType type) { - return isa(unwrap(type)); -} - -MlirType mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t *shape, - MlirType elementType, - MlirAttribute sharedLayout) { - mlir::Attribute layoutAttr; - if (!mlirAttributeIsNull(sharedLayout)) { - layoutAttr = unwrap(sharedLayout); - } - // TensorDescType::get is a TypeBuilderWithInferredContext: the context is - // inferred from elementType. Signature: (shape, elementType, sharedLayout). - return wrap(mlir::triton::TensorDescType::get( - llvm::ArrayRef(shape, rank), unwrap(elementType), layoutAttr)); -} - -bool mlirTypeIsATritonTensorDesc(MlirType type) { - return isa(unwrap(type)); -} - -MlirAttribute mlirTritonProgramIDDimAttrGet(MlirContext context, - int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::ProgramIDDimAttr::get( - ctx, static_cast(value))); -} - -MlirAttribute mlirTritonCacheModifierAttrGet(MlirContext context, - int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::CacheModifierAttr::get( - ctx, static_cast(value))); -} - -MlirAttribute mlirTritonEvictionPolicyAttrGet(MlirContext context, - int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::EvictionPolicyAttr::get( - ctx, static_cast(value))); -} - -MlirAttribute mlirTritonPaddingOptionAttrGet(MlirContext context, - int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::PaddingOptionAttr::get( - ctx, static_cast(value))); -} - -MlirAttribute mlirTritonRoundingModeAttrGet(MlirContext context, - int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::RoundingModeAttr::get( - ctx, static_cast(value))); -} - -MlirAttribute mlirTritonInputPrecisionAttrGet(MlirContext context, - int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::InputPrecisionAttr::get( - ctx, static_cast(value))); -} - -MlirAttribute mlirTritonRMWOpAttrGet(MlirContext context, int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::RMWOpAttr::get( - ctx, static_cast(value))); -} - -MlirAttribute mlirTritonMemSemanticAttrGet(MlirContext context, int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::MemSemanticAttr::get( - ctx, static_cast(value))); -} - -MlirAttribute mlirTritonMemSyncScopeAttrGet(MlirContext context, - int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::MemSyncScopeAttr::get( - ctx, static_cast(value))); -} - -MlirAttribute mlirTritonPropagateNanAttrGet(MlirContext context, - int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::PropagateNanAttr::get( - ctx, static_cast(value))); -} - -MlirAttribute mlirTritonScaleDotElemTypeAttrGet(MlirContext context, - int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::ScaleDotElemTypeAttr::get( - ctx, static_cast(value))); -} - -MlirAttribute mlirTritonDescriptorReduceKindAttrGet(MlirContext context, - int32_t value) { - mlir::MLIRContext *ctx = loadTriton(context); - if (!ctx) - return {nullptr}; - return wrap(mlir::triton::DescriptorReduceKindAttr::get( - ctx, static_cast(value))); -} diff --git a/mlir/dialects/ttir/capi.h b/mlir/dialects/ttir/capi.h deleted file mode 100644 index efd0aae9..00000000 --- a/mlir/dialects/ttir/capi.h +++ /dev/null @@ -1,59 +0,0 @@ -//===- capi.h - C API for the Triton tt dialect -------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_TTIR_CAPI_H -#define MLIR_DIALECTS_TTIR_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(Triton, tt); - -// !tt.ptr -MLIR_CAPI_EXPORTED MlirType mlirTritonPointerTypeGet(MlirType pointee, - int32_t addressSpace); -MLIR_CAPI_EXPORTED MlirType mlirTritonPointerTypeGetPointee(MlirType type); -MLIR_CAPI_EXPORTED int32_t mlirTritonPointerTypeGetAddressSpace(MlirType type); -MLIR_CAPI_EXPORTED bool mlirTypeIsATritonPointer(MlirType type); - -// !tt.tensordesc> — result of tt.make_tensor_descriptor. -// `shape` is a `rank`-element array of int64_t block dims; `elementType` is -// the scalar element type; `sharedLayout` may be a null MlirAttribute for none. -MLIR_CAPI_EXPORTED MlirType -mlirTritonTensorDescTypeGet(intptr_t rank, const int64_t *shape, - MlirType elementType, MlirAttribute sharedLayout); -MLIR_CAPI_EXPORTED bool mlirTypeIsATritonTensorDesc(MlirType type); - -// I32EnumAttr constructors -MLIR_CAPI_EXPORTED MlirAttribute -mlirTritonProgramIDDimAttrGet(MlirContext context, int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTritonCacheModifierAttrGet(MlirContext context, int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTritonEvictionPolicyAttrGet(MlirContext context, int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTritonPaddingOptionAttrGet(MlirContext context, int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTritonRoundingModeAttrGet(MlirContext context, int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTritonInputPrecisionAttrGet(MlirContext context, int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute mlirTritonRMWOpAttrGet(MlirContext context, - int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTritonMemSemanticAttrGet(MlirContext context, int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTritonMemSyncScopeAttrGet(MlirContext context, int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTritonPropagateNanAttrGet(MlirContext context, int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTritonScaleDotElemTypeAttrGet(MlirContext context, int32_t value); -MLIR_CAPI_EXPORTED MlirAttribute -mlirTritonDescriptorReduceKindAttrGet(MlirContext context, int32_t value); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_TTIR_CAPI_H diff --git a/mlir/dialects/ttir/test.zig b/mlir/dialects/ttir/test.zig deleted file mode 100644 index 04361e4e..00000000 --- a/mlir/dialects/ttir/test.zig +++ /dev/null @@ -1,172 +0,0 @@ -const std = @import("std"); - -const arith = @import("mlir/dialects/arith"); -const func = @import("mlir/dialects/func"); -const math = @import("mlir/dialects/math"); -const mlir = @import("mlir"); -const scf = @import("mlir/dialects/scf"); -const ttir = @import("mlir/dialects/ttir"); - -const operation_names: []const []const u8 = &.{ - "tt.addptr", - "tt.assert", - "tt.atomic_cas", - "tt.atomic_rmw", - "tt.bitcast", - "tt.broadcast", - "tt.call", - "tt.clampf", - "tt.descriptor_gather", - "tt.descriptor_load", - "tt.descriptor_reduce", - "tt.descriptor_scatter", - "tt.descriptor_store", - "tt.dot", - "tt.dot_scaled", - "tt.elementwise_inline_asm", - "tt.expand_dims", - "tt.extern_elementwise", - "tt.fp_to_fp", - "tt.func", - "tt.gather", - "tt.get_num_programs", - "tt.get_program_id", - "tt.histogram", - "tt.int_to_ptr", - "tt.join", - "tt.load", - "tt.make_range", - "tt.make_tensor_descriptor", - "tt.map_elementwise", - "tt.map_elementwise.return", - "tt.mulhiui", - "tt.precise_divf", - "tt.precise_sqrt", - "tt.print", - "tt.ptr_to_int", - "tt.reduce", - "tt.reduce.return", - "tt.reshape", - "tt.return", - "tt.scan", - "tt.scan.return", - "tt.splat", - "tt.split", - "tt.store", - "tt.trans", - "tt.unsplat", -}; - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(ttir); - inline for (comptime std.meta.declarations(ttir)) |decl| { - const T = @field(ttir, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "tt.func round-trip" { - // Build: module { tt.func public @id(%arg0: !tt.ptr) { tt.return } } - // by constructing it with the Zig wrappers, printing to string, and re-parsing. - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - func.registerDialect(registry); - arith.registerDialect(registry); - scf.registerDialect(registry); - math.registerDialect(registry); - ttir.registerDialect(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - - try std.testing.expect(ctx.isRegisteredOperation("tt.func")); - try std.testing.expect(ctx.isRegisteredOperation("tt.return")); - try std.testing.expect(ctx.isRegisteredOperation("tt.load")); - try std.testing.expect(ctx.isRegisteredOperation("tt.store")); - try std.testing.expect(ctx.isRegisteredOperation("tt.get_program_id")); - try std.testing.expect(ctx.isRegisteredOperation("tt.splat")); - try std.testing.expect(ctx.isRegisteredOperation("tt.addptr")); - try std.testing.expect(ctx.isRegisteredOperation("scf.for")); - try std.testing.expect(ctx.isRegisteredOperation("arith.constant")); - - const loc: *const mlir.Location = .unknown(ctx); - const module: *mlir.Module = .init(loc); - defer module.deinit(); - - const f32_ty = mlir.Type.float(ctx, .f32); - const ptr_ty = ttir.pointerType(f32_ty, 1); - - const entry = mlir.Block.init(&.{ ptr_ty, ptr_ty }, &.{ loc, loc }); - - const arg0 = entry.argument(0); - const arg1 = entry.argument(1); - - // %0 = tt.load %arg0 : !tt.ptr - const loaded = ttir.load(ctx, arg0, f32_ty, .none, .normal, false, .{}, loc); - _ = loaded.appendTo(entry); - - // %1 = arith.constant 1.0 : f32 — built with Operation.make to avoid cross-dialect import - const one_attr: *const mlir.Attribute = .float(ctx, .f32, 1.0); - const one = mlir.Operation.make(ctx, "arith.constant", .{ - .results = .{ .flat = &.{f32_ty} }, - .attributes = &.{.named(ctx, "value", one_attr)}, - .location = loc, - }); - _ = one.appendTo(entry); - - // %2 = arith.addf %0, %1 : f32 - const summed = mlir.Operation.make(ctx, "arith.addf", .{ - .operands = .{ .flat = &.{ loaded.result(0), one.result(0) } }, - .result_type_inference = true, - .location = loc, - }); - _ = summed.appendTo(entry); - - // tt.store %arg1, %2 - _ = ttir.store(ctx, arg1, summed.result(0), .none, .normal, .{}, loc).appendTo(entry); - - // tt.return - _ = ttir.return_(ctx, &.{}, loc).appendTo(entry); - - const f = ttir.func(ctx, .{ - .name = "add_one", - .block = entry, - .location = loc, - }); - _ = f.appendTo(module.body()); - - // Verify the operation. - try std.testing.expect(module.operation().verify()); - - // Print IR and re-parse it to confirm it's valid text. - var al: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer al.deinit(); - try al.writer.print("{f}", .{module.operation()}); - - const parsed = try mlir.Module.parse(ctx, al.written()); - defer parsed.deinit(); - try std.testing.expect(parsed.operation().verify()); -} - -test "divisibility is an i32 attribute of the tt dialect" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - var buf: [64]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{mlir.Attribute.dict(ctx, &.{ttir.divisibility(ctx, 16)})}); - try std.testing.expectEqualStrings("{tt.divisibility = 16 : i32}", w.buffered()); -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - ttir.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/ttir/ttir.zig b/mlir/dialects/ttir/ttir.zig deleted file mode 100644 index ee7b5307..00000000 --- a/mlir/dialects/ttir/ttir.zig +++ /dev/null @@ -1,1231 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("tt"); -} - -// ============================================================================= -// Types — !tt.ptr -// ============================================================================= - -pub const PointerType = opaque { - const M = mlir.Methods(PointerType, c.MlirType); - - pub const isAFn = c.mlirTypeIsATritonPointer; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - pointee_type: *const mlir.Type, - address_space: i32, - }; - - pub fn get(args: InitArgs) *const PointerType { - return @ptrCast(c.mlirTritonPointerTypeGet(args.pointee_type.ptr(), args.address_space).ptr); - } - - pub fn pointee(self: *const PointerType) *const mlir.Type { - return @ptrCast(c.mlirTritonPointerTypeGetPointee(self.ptr()).ptr); - } - - pub fn addressSpace(self: *const PointerType) i32 { - return c.mlirTritonPointerTypeGetAddressSpace(self.ptr()); - } - - pub fn type_(self: *const PointerType) *const mlir.Type { - return @ptrCast(self); - } -}; - -/// Convenience alias: `!tt.ptr` as a plain *const mlir.Type. -pub fn pointerType(pointee_type: *const mlir.Type, address_space: i32) *const mlir.Type { - return @ptrCast(PointerType.get(.{ .pointee_type = pointee_type, .address_space = address_space })); -} - -// ----------------------------------------------------------------------------- -// !tt.tensordesc> — TMA descriptor type. -// ----------------------------------------------------------------------------- - -pub const TensorDescType = opaque { - const M = mlir.Methods(TensorDescType, c.MlirType); - - pub const isAFn = c.mlirTypeIsATritonTensorDesc; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - pub const InitArgs = struct { - shape: []const i64, - element_type: *const mlir.Type, - /// May be null for an unlayout'd descriptor. - shared_layout: ?*const mlir.Attribute = null, - }; - - pub fn get(args: InitArgs) *const TensorDescType { - const attr_ptr = if (args.shared_layout) |a| a.ptr() else c.MlirAttribute{ .ptr = null }; - return @ptrCast(c.mlirTritonTensorDescTypeGet( - @intCast(args.shape.len), - args.shape.ptr, - args.element_type.ptr(), - attr_ptr, - ).ptr); - } - - pub fn type_(self: *const TensorDescType) *const mlir.Type { - return @ptrCast(self); - } -}; - -pub fn tensorDescType( - shape: []const i64, - element_type: *const mlir.Type, - opts: struct { - shared_layout: ?*const mlir.Attribute = null, - }, -) *const mlir.Type { - return @ptrCast(TensorDescType.get(.{ .shape = shape, .element_type = element_type, .shared_layout = opts.shared_layout })); -} - -// ============================================================================= -// Enum attributes -// ============================================================================= - -pub const ProgramIDDim = enum(i32) { x = 0, y = 1, z = 2 }; -pub const CacheModifier = enum(i32) { none = 1, ca = 2, cg = 3, wb = 4, cs = 5, wt = 6, cv = 7 }; -pub const EvictionPolicy = enum(i32) { normal = 1, evict_first = 2, evict_last = 3 }; -pub const PaddingOption = enum(i32) { pad_zero = 1, pad_nan = 2 }; -pub const RoundingMode = enum(i32) { rtz = 0, rtne = 1 }; -pub const InputPrecision = enum(i32) { tf32 = 0, tf32x3 = 1, ieee = 2, bf16x3 = 3, bf16x6 = 4 }; - -/// Atomic read-modify-write operations. -pub const RMWOp = enum(i32) { - @"and" = 1, - @"or" = 2, - xor = 3, - add = 4, - fadd = 5, - max = 6, - min = 7, - umax = 8, - umin = 9, - xchg = 10, -}; - -/// Memory ordering semantics. -pub const MemSemantic = enum(i32) { - relaxed = 1, - acquire = 2, - release = 3, - acq_rel = 4, -}; - -/// Memory sync scope. -pub const MemSyncScope = enum(i32) { - gpu = 1, - cta = 2, - sys = 3, -}; - -/// NaN propagation for clampf. 0xFFFF is the exact .td encoding. -pub const PropagateNan = enum(i32) { - none = 0, - all = 0xFFFF, -}; - -/// Packed FP element types for tt.dot_scaled. -pub const ScaleDotElemType = enum(i32) { - e4m3 = 0, - e5m2 = 1, - e2m3 = 2, - e3m2 = 3, - e2m1 = 4, - bf16 = 5, - fp16 = 6, -}; - -/// Reduction kind for tt.descriptor_reduce. -pub const DescriptorReduceKind = enum(i32) { - add = 1, - min = 2, - max = 3, - inc = 4, - dec = 5, - @"and" = 6, - @"or" = 7, - xor = 8, -}; - -pub fn programDimAttr(ctx: *mlir.Context, v: ProgramIDDim) *const mlir.Attribute { - return @ptrCast(c.mlirTritonProgramIDDimAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -pub fn cacheModifierAttr(ctx: *mlir.Context, v: CacheModifier) *const mlir.Attribute { - return @ptrCast(c.mlirTritonCacheModifierAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -pub fn evictionPolicyAttr(ctx: *mlir.Context, v: EvictionPolicy) *const mlir.Attribute { - return @ptrCast(c.mlirTritonEvictionPolicyAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -pub fn paddingOptionAttr(ctx: *mlir.Context, v: PaddingOption) *const mlir.Attribute { - return @ptrCast(c.mlirTritonPaddingOptionAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -pub fn roundingModeAttr(ctx: *mlir.Context, v: RoundingMode) *const mlir.Attribute { - return @ptrCast(c.mlirTritonRoundingModeAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -pub fn inputPrecisionAttr(ctx: *mlir.Context, v: InputPrecision) *const mlir.Attribute { - return @ptrCast(c.mlirTritonInputPrecisionAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -pub fn rmwOpAttr(ctx: *mlir.Context, v: RMWOp) *const mlir.Attribute { - return @ptrCast(c.mlirTritonRMWOpAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -pub fn memSemanticAttr(ctx: *mlir.Context, v: MemSemantic) *const mlir.Attribute { - return @ptrCast(c.mlirTritonMemSemanticAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -pub fn memSyncScopeAttr(ctx: *mlir.Context, v: MemSyncScope) *const mlir.Attribute { - return @ptrCast(c.mlirTritonMemSyncScopeAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -pub fn propagateNanAttr(ctx: *mlir.Context, v: PropagateNan) *const mlir.Attribute { - return @ptrCast(c.mlirTritonPropagateNanAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -pub fn scaleDotElemTypeAttr(ctx: *mlir.Context, v: ScaleDotElemType) *const mlir.Attribute { - return @ptrCast(c.mlirTritonScaleDotElemTypeAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -pub fn descriptorReduceKindAttr(ctx: *mlir.Context, v: DescriptorReduceKind) *const mlir.Attribute { - return @ptrCast(c.mlirTritonDescriptorReduceKindAttrGet(ctx.ptr(), @backingInt(v)).ptr); -} - -/// `tt.divisibility`: an argument attribute promising that the argument (the -/// address, for a pointer) is a multiple of `divisor`. -pub fn divisibility(ctx: *mlir.Context, divisor: u32) mlir.NamedAttribute { - return .named(ctx, "tt.divisibility", .int(ctx, .i32, divisor)); -} - -// ============================================================================= -// Function ops — tt.func / tt.return -// ============================================================================= - -/// Args for building a `tt.func` op. -pub const FuncOpArgs = struct { - pub const Visibility = enum { public, private }; - - name: []const u8, - args: ?[]const *const mlir.Type = null, - args_attributes: ?[]const *const mlir.Attribute = null, - results: ?[]const *const mlir.Type = null, - results_attributes: ?[]const *const mlir.Attribute = null, - block: *mlir.Block, - location: *const mlir.Location, - no_inline: bool = false, - visibility: Visibility = .public, - /// `tt.func` has `HasParent<"ModuleOp">`, so verification before appending - /// to a module always fails. Defer to the module-level verify instead. - verify: bool = false, -}; - -pub fn func(ctx: *mlir.Context, args: FuncOpArgs) *mlir.Operation { - var args_buffer: stdx.BoundedArray(*const mlir.Type, 1024) = .empty; - var results_buffer: stdx.BoundedArray(*const mlir.Type, 32) = .empty; - - var attr_tuples_buffer: stdx.BoundedArray(mlir.NamedAttribute, 16) = .empty; - attr_tuples_buffer.appendSliceAssumeCapacity(&.{ - .named(ctx, "sym_name", .string(ctx, args.name)), - .named(ctx, "sym_visibility", .string(ctx, @tagName(args.visibility))), - .named(ctx, "function_type", .typeAttr(.function( - ctx, - args.args orelse args_: { - for (0..args.block.numArguments()) |i| { - args_buffer.appendAssumeCapacity(args.block.argument(i).type_()); - } - break :args_ args_buffer.constSlice(); - }, - args.results orelse results_: { - const terminator = args.block.terminator() orelse { - @panic("block has no terminator"); - }; - for (0..terminator.numOperands()) |i| { - results_buffer.appendAssumeCapacity(terminator.operand(i).type_()); - } - break :results_ results_buffer.constSlice(); - }, - ))), - }); - - if (args.args_attributes) |args_attributes| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, args_attributes))); - } - if (args.results_attributes) |results_attributes| { - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, results_attributes))); - } - // Always emit noinline even when false — matches Triton's IR convention. - attr_tuples_buffer.appendAssumeCapacity(.named(ctx, "noinline", .boolean(ctx, args.no_inline))); - - return .make(ctx, "tt.func", .{ - .blocks = &.{args.block}, - .attributes = attr_tuples_buffer.constSlice(), - .location = args.location, - .verify = args.verify, - }); -} - -pub fn return_(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return .make(ctx, "tt.return", .{ - .operands = .{ .flat = values }, - .verify = false, - .location = location, - }); -} - -// ============================================================================= -// SPMD ops -// ============================================================================= - -pub fn get_program_id(ctx: *mlir.Context, axis: ProgramIDDim, location: *const mlir.Location) *mlir.Operation { - return .make(ctx, "tt.get_program_id", .{ - .results = .{ .flat = &.{.int(ctx, .i32)} }, - .attributes = &.{ - .named(ctx, "axis", programDimAttr(ctx, axis)), - }, - .location = location, - }); -} - -pub fn get_num_programs(ctx: *mlir.Context, axis: ProgramIDDim, location: *const mlir.Location) *mlir.Operation { - return .make(ctx, "tt.get_num_programs", .{ - .results = .{ .flat = &.{.int(ctx, .i32)} }, - .attributes = &.{ - .named(ctx, "axis", programDimAttr(ctx, axis)), - }, - .location = location, - }); -} - -// ============================================================================= -// Range -// ============================================================================= - -pub fn make_range( - ctx: *mlir.Context, - start: i32, - end: i32, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.make_range", .{ - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "start", .int(ctx, .i32, start)), - .named(ctx, "end", .int(ctx, .i32, end)), - }, - .location = location, - }); -} - -// ============================================================================= -// Shape manipulation -// ============================================================================= - -pub fn splat( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.splat", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn expand_dims( - ctx: *mlir.Context, - src: *const mlir.Value, - axis: i32, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.expand_dims", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "axis", .int(ctx, .i32, axis)), - }, - .location = location, - }); -} - -pub fn broadcast( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.broadcast", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn reshape( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - allow_reorder: bool, - efficient_layout: bool, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - if (allow_reorder) { - attrs.appendAssumeCapacity(.named(ctx, "allow_reorder", .unit(ctx))); - } - if (efficient_layout) { - attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", .unit(ctx))); - } - return .make(ctx, "tt.reshape", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -pub fn trans( - ctx: *mlir.Context, - src: *const mlir.Value, - order: []const i32, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.trans", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "order", .denseArray(ctx, .i32, order)), - }, - .location = location, - }); -} - -// ============================================================================= -// Pointer arithmetic & memory -// ============================================================================= - -/// tt.addptr — result type matches ptr type (TypesMatchWith in TritonOps.td). -pub fn addptr( - ctx: *mlir.Context, - ptr_val: *const mlir.Value, - offset: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.addptr", .{ - .operands = .{ .flat = &.{ ptr_val, offset } }, - .results = .{ .flat = &.{ptr_val.type_()} }, - .location = location, - }); -} - -/// tt.load — AttrSizedOperandSegments: (ptr, [mask], [other]). Uses flat operand list + -/// explicit operandSegmentSizes because Operation.make variadic coercion zeros mixed-length segments. -pub fn load( - ctx: *mlir.Context, - ptr_val: *const mlir.Value, - result_type: *const mlir.Type, - cache: CacheModifier, - evict: EvictionPolicy, - is_volatile: bool, - opts: struct { - mask: ?*const mlir.Value = null, - other: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; - operands_buf.appendAssumeCapacity(ptr_val); - const mask_len: i32 = if (opts.mask) |m| blk: { - operands_buf.appendAssumeCapacity(m); - break :blk 1; - } else 0; - const other_len: i32 = if (opts.other) |o| blk: { - operands_buf.appendAssumeCapacity(o); - break :blk 1; - } else 0; - - const seg_sizes = [3]i32{ 1, mask_len, other_len }; - return .make(ctx, "tt.load", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "cache", cacheModifierAttr(ctx, cache)), - .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), - .named(ctx, "isVolatile", .boolean(ctx, is_volatile)), - }, - .location = location, - }); -} - -/// tt.store — operands: (ptr, value, [mask]). Flat operand list; mask is trailing variadic. -pub fn store( - ctx: *mlir.Context, - ptr_val: *const mlir.Value, - value: *const mlir.Value, - cache: CacheModifier, - evict: EvictionPolicy, - opts: struct { - mask: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; - buf.appendSliceAssumeCapacity(&.{ ptr_val, value }); - if (opts.mask) |m| buf.appendAssumeCapacity(m); - return .make(ctx, "tt.store", .{ - .operands = .{ .flat = buf.constSlice() }, - .attributes = &.{ - .named(ctx, "cache", cacheModifierAttr(ctx, cache)), - .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), - }, - .location = location, - }); -} - -// ============================================================================= -// Compute -// ============================================================================= - -pub fn dot( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c_acc: *const mlir.Value, - result_type: *const mlir.Type, - input_precision_: InputPrecision, - max_num_imprecise_acc: i32, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.dot", .{ - .operands = .{ .flat = &.{ a, b, c_acc } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "inputPrecision", inputPrecisionAttr(ctx, input_precision_)), - .named(ctx, "maxNumImpreciseAcc", .int(ctx, .i32, max_num_imprecise_acc)), - }, - .location = location, - }); -} - -/// tt.reduce — combine_block must have args [elem, elem] matching srcs' element types, -/// terminated with tt.reduce.return. -pub fn reduce( - ctx: *mlir.Context, - srcs: []const *const mlir.Value, - axis: i32, - combine_block: *mlir.Block, - result_types: []const *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - // Single variadic operand group — flat list, no operandSegmentSizes needed. - return .make(ctx, "tt.reduce", .{ - .operands = .{ .flat = srcs }, - .results = .{ .flat = result_types }, - .blocks = &.{combine_block}, - .attributes = &.{ - .named(ctx, "axis", .int(ctx, .i32, axis)), - }, - .location = location, - }); -} - -pub fn reduce_return( - ctx: *mlir.Context, - values: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.reduce.return", .{ - .operands = .{ .flat = values }, - .verify = false, - .location = location, - }); -} - -// ============================================================================= -// Elementwise casts specific to Triton (arith.bitcast doesn't support ptrs) -// ============================================================================= - -/// tt.int_to_ptr — cast int-like to ptr-like (element-wise, same shape). -pub fn int_to_ptr( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.int_to_ptr", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// tt.ptr_to_int — cast ptr-like to int-like (element-wise, same shape). -pub fn ptr_to_int( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.ptr_to_int", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// tt.bitcast — cast between types of the same bit-width (also works on ptrs, -/// unlike arith.bitcast). -pub fn bitcast( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.bitcast", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// tt.fp_to_fp — float to float cast, optionally with a rounding mode. -/// Pass `rounding = null` to omit the rounding attribute. -pub fn fp_to_fp( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - rounding: ?RoundingMode = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 1) = .empty; - if (opts.rounding) |rm| { - attrs.appendAssumeCapacity(.named(ctx, "rounding", roundingModeAttr(ctx, rm))); - } - return .make(ctx, "tt.fp_to_fp", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -// ============================================================================= -// Math ops -// ============================================================================= - -/// tt.clampf — clamp float tensor to [min, max] element-wise. -pub fn clampf( - ctx: *mlir.Context, - x: *const mlir.Value, - min: *const mlir.Value, - max: *const mlir.Value, - propagate_nan: PropagateNan, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.clampf", .{ - .operands = .{ .flat = &.{ x, min, max } }, - .results = .{ .flat = &.{x.type_()} }, - .attributes = &.{ - .named(ctx, "propagateNan", propagateNanAttr(ctx, propagate_nan)), - }, - .location = location, - }); -} - -/// tt.precise_sqrt — precise sqrt (SameOperandsAndResultType). -pub fn precise_sqrt( - ctx: *mlir.Context, - x: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.precise_sqrt", .{ - .operands = .{ .flat = &.{x} }, - .results = .{ .flat = &.{x.type_()} }, - .location = location, - }); -} - -/// tt.precise_divf — precise float div (SameOperandsAndResultType). -pub fn precise_divf( - ctx: *mlir.Context, - x: *const mlir.Value, - y: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.precise_divf", .{ - .operands = .{ .flat = &.{ x, y } }, - .results = .{ .flat = &.{x.type_()} }, - .location = location, - }); -} - -/// tt.mulhiui — most significant N bits of 2N-bit unsigned product. -pub fn mulhiui( - ctx: *mlir.Context, - x: *const mlir.Value, - y: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.mulhiui", .{ - .operands = .{ .flat = &.{ x, y } }, - .results = .{ .flat = &.{x.type_()} }, - .location = location, - }); -} - -// ============================================================================= -// Shape manipulation — more -// ============================================================================= - -/// tt.unsplat — tensor-of-1-element → scalar. -pub fn unsplat( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.unsplat", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// tt.join — join two tensors along a new minor dimension (shape gains +1 dim of size 2). -pub fn join( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.join", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// tt.split — split a tensor along its last dimension (size must be 2). Returns (lhs, rhs). -pub fn split( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.split", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{ result_type, result_type } }, - .location = location, - }); -} - -// ============================================================================= -// Gather / Histogram -// ============================================================================= - -/// tt.gather — gather elements from `src` using `indices` along `axis`. -/// Result shape matches `indices` shape. -pub fn gather( - ctx: *mlir.Context, - src: *const mlir.Value, - indices: *const mlir.Value, - axis: i32, - result_type: *const mlir.Type, - efficient_layout: bool, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "axis", .int(ctx, .i32, axis))); - if (efficient_layout) { - attrs.appendAssumeCapacity(.named(ctx, "efficient_layout", .unit(ctx))); - } - return .make(ctx, "tt.gather", .{ - .operands = .{ .flat = &.{ src, indices } }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// tt.histogram — build a histogram of `src`. Output size = number of bins. -pub fn histogram( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - opts: struct { - mask: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; - buf.appendAssumeCapacity(src); - if (opts.mask) |m| buf.appendAssumeCapacity(m); - return .make(ctx, "tt.histogram", .{ - .operands = .{ .flat = buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -// ============================================================================= -// Scan — parallel scan with combine region (like reduce) -// ============================================================================= - -/// tt.scan — combine_block must already contain the tt.scan.return terminator -/// and have arguments with element types matching `srcs`. -pub fn scan( - ctx: *mlir.Context, - srcs: []const *const mlir.Value, - axis: i32, - reverse: bool, - combine_block: *mlir.Block, - result_types: []const *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.scan", .{ - .operands = .{ .variadic = &.{srcs} }, - .results = .{ .flat = result_types }, - .blocks = &.{combine_block}, - .attributes = &.{ - .named(ctx, "axis", .int(ctx, .i32, axis)), - .named(ctx, "reverse", .boolean(ctx, reverse)), - }, - .location = location, - }); -} - -pub fn scan_return( - ctx: *mlir.Context, - values: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.scan.return", .{ - .operands = .{ .flat = values }, - .verify = false, - .location = location, - }); -} - -// ============================================================================= -// Atomics -// ============================================================================= - -/// tt.atomic_rmw — (ptr, val, [mask]). Result type matches val's type. -pub fn atomic_rmw( - ctx: *mlir.Context, - rmw: RMWOp, - ptr_val: *const mlir.Value, - val: *const mlir.Value, - sem: MemSemantic, - scope: MemSyncScope, - opts: struct { - mask: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 3) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ ptr_val, val }); - if (opts.mask) |m| operands_buf.appendAssumeCapacity(m); - - return .make(ctx, "tt.atomic_rmw", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{val.type_()} }, - .attributes = &.{ - .named(ctx, "atomic_rmw_op", rmwOpAttr(ctx, rmw)), - .named(ctx, "sem", memSemanticAttr(ctx, sem)), - .named(ctx, "scope", memSyncScopeAttr(ctx, scope)), - }, - .location = location, - }); -} - -/// tt.atomic_cas — compare-and-swap. Result type matches val's type. -pub fn atomic_cas( - ctx: *mlir.Context, - ptr_val: *const mlir.Value, - cmp: *const mlir.Value, - val: *const mlir.Value, - sem: MemSemantic, - scope: MemSyncScope, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.atomic_cas", .{ - .operands = .{ .flat = &.{ ptr_val, cmp, val } }, - .results = .{ .flat = &.{val.type_()} }, - .attributes = &.{ - .named(ctx, "sem", memSemanticAttr(ctx, sem)), - .named(ctx, "scope", memSyncScopeAttr(ctx, scope)), - }, - .location = location, - }); -} - -// ============================================================================= -// Debug -// ============================================================================= - -/// tt.assert — device-side assert. `condition` must be i1 or tensor<...xi1>. -pub fn assert_( - ctx: *mlir.Context, - condition: *const mlir.Value, - message: []const u8, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.assert", .{ - .operands = .{ .flat = &.{condition} }, - .attributes = &.{ - .named(ctx, "message", .string(ctx, message)), - }, - .location = location, - }); -} - -// ============================================================================= -// Call -// ============================================================================= - -/// tt.call — direct call to another tt.func in the same module. -/// `opts.arg_attrs` / `opts.res_attrs` propagate hints like tt.divisibility; -/// leave them unset to omit them. -pub fn call( - ctx: *mlir.Context, - callee: []const u8, - operands: []const *const mlir.Value, - result_types: []const *const mlir.Type, - opts: struct { - arg_attrs: ?[]const *const mlir.Attribute = null, - res_attrs: ?[]const *const mlir.Attribute = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 3) = .empty; - attrs.appendAssumeCapacity(.named(ctx, "callee", .flatSymbolRef(ctx, callee))); - if (opts.arg_attrs) |aa| { - attrs.appendAssumeCapacity(.named(ctx, "arg_attrs", .array(ctx, aa))); - } - if (opts.res_attrs) |ra| { - attrs.appendAssumeCapacity(.named(ctx, "res_attrs", .array(ctx, ra))); - } - return .make(ctx, "tt.call", .{ - .operands = .{ .flat = operands }, - .results = .{ .flat = result_types }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -// ============================================================================= -// Scaled dot (microscaling spec). -// ============================================================================= - -/// tt.dot_scaled — dot with optional per-block scale factors. -/// a_scale/b_scale may be null. AttrSizedOperandSegments: (a, b, c, [a_scale], [b_scale]). -pub fn dot_scaled( - ctx: *mlir.Context, - a: *const mlir.Value, - b: *const mlir.Value, - c_acc: *const mlir.Value, - a_elem_type: ScaleDotElemType, - b_elem_type: ScaleDotElemType, - result_type: *const mlir.Type, - fast_math: bool, - lhs_k_pack: bool, - rhs_k_pack: bool, - opts: struct { - a_scale: ?*const mlir.Value = null, - b_scale: ?*const mlir.Value = null, - }, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 5) = .empty; - buf.appendSliceAssumeCapacity(&.{ a, b, c_acc }); - const a_scale_len: i32 = if (opts.a_scale) |s| blk: { - buf.appendAssumeCapacity(s); - break :blk 1; - } else 0; - const b_scale_len: i32 = if (opts.b_scale) |s| blk: { - buf.appendAssumeCapacity(s); - break :blk 1; - } else 0; - - const seg_sizes = [5]i32{ 1, 1, 1, a_scale_len, b_scale_len }; - return .make(ctx, "tt.dot_scaled", .{ - .operands = .{ .flat = buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "a_elem_type", scaleDotElemTypeAttr(ctx, a_elem_type)), - .named(ctx, "b_elem_type", scaleDotElemTypeAttr(ctx, b_elem_type)), - .named(ctx, "fastMath", .boolean(ctx, fast_math)), - .named(ctx, "lhs_k_pack", .boolean(ctx, lhs_k_pack)), - .named(ctx, "rhs_k_pack", .boolean(ctx, rhs_k_pack)), - }, - .location = location, - }); -} - -// ============================================================================= -// Extern elementwise (call library function pointwise) -// ============================================================================= - -pub fn extern_elementwise( - ctx: *mlir.Context, - srcs: []const *const mlir.Value, - result_type: *const mlir.Type, - libname: []const u8, - libpath: []const u8, - symbol: []const u8, - pure: bool, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.extern_elementwise", .{ - .operands = .{ .flat = srcs }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "libname", .string(ctx, libname)), - .named(ctx, "libpath", .string(ctx, libpath)), - .named(ctx, "symbol", .string(ctx, symbol)), - .named(ctx, "pure", .boolean(ctx, pure)), - }, - .location = location, - }); -} - -// ============================================================================= -// Device-side print -// ============================================================================= - -/// tt.print — device-side print. `is_signed` must be `args.len` long and -/// indicates signedness per arg (1 = signed, 0 = unsigned) for printf-style -/// integer formatting. -pub fn print( - ctx: *mlir.Context, - prefix: []const u8, - hex: bool, - args: []const *const mlir.Value, - is_signed: []const i32, - location: *const mlir.Location, -) *mlir.Operation { - std.debug.assert(args.len == is_signed.len); - return .make(ctx, "tt.print", .{ - .operands = .{ .flat = args }, - .attributes = &.{ - .named(ctx, "prefix", .string(ctx, prefix)), - .named(ctx, "hex", .boolean(ctx, hex)), - .named(ctx, "isSigned", .denseArray(ctx, .i32, is_signed)), - }, - .location = location, - }); -} - -// ============================================================================= -// Tensor descriptors (TMA) -// ============================================================================= - -/// tt.make_tensor_descriptor — build a TMA descriptor from a base ptr, runtime -/// shape/strides, and the compile-time block shape baked into `result_type`. -pub fn make_tensor_descriptor( - ctx: *mlir.Context, - base: *const mlir.Value, - shape: []const *const mlir.Value, - strides: []const *const mlir.Value, - padding: PaddingOption, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - buf.appendAssumeCapacity(base); - buf.appendSliceAssumeCapacity(shape); - buf.appendSliceAssumeCapacity(strides); - // SameVariadicOperandSize — flatten all operands. - return .make(ctx, "tt.make_tensor_descriptor", .{ - .operands = .{ .flat = buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "padding", paddingOptionAttr(ctx, padding)), - }, - .location = location, - }); -} - -pub fn descriptor_load( - ctx: *mlir.Context, - desc: *const mlir.Value, - indices: []const *const mlir.Value, - result_type: *const mlir.Type, - cache: CacheModifier, - evict: EvictionPolicy, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - buf.appendAssumeCapacity(desc); - buf.appendSliceAssumeCapacity(indices); - return .make(ctx, "tt.descriptor_load", .{ - .operands = .{ .flat = buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "cache", cacheModifierAttr(ctx, cache)), - .named(ctx, "evict", evictionPolicyAttr(ctx, evict)), - }, - .location = location, - }); -} - -pub fn descriptor_store( - ctx: *mlir.Context, - desc: *const mlir.Value, - src: *const mlir.Value, - indices: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - buf.appendSliceAssumeCapacity(&.{ desc, src }); - buf.appendSliceAssumeCapacity(indices); - return .make(ctx, "tt.descriptor_store", .{ - .operands = .{ .flat = buf.constSlice() }, - .location = location, - }); -} - -pub fn descriptor_reduce( - ctx: *mlir.Context, - kind: DescriptorReduceKind, - desc: *const mlir.Value, - src: *const mlir.Value, - indices: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - var buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - buf.appendSliceAssumeCapacity(&.{ desc, src }); - buf.appendSliceAssumeCapacity(indices); - return .make(ctx, "tt.descriptor_reduce", .{ - .operands = .{ .flat = buf.constSlice() }, - .attributes = &.{ - .named(ctx, "kind", descriptorReduceKindAttr(ctx, kind)), - }, - .location = location, - }); -} - -pub fn descriptor_gather( - ctx: *mlir.Context, - desc: *const mlir.Value, - x_offsets: *const mlir.Value, - y_offset: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.descriptor_gather", .{ - .operands = .{ .flat = &.{ desc, x_offsets, y_offset } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn descriptor_scatter( - ctx: *mlir.Context, - desc: *const mlir.Value, - x_offsets: *const mlir.Value, - y_offset: *const mlir.Value, - src: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.descriptor_scatter", .{ - .operands = .{ .flat = &.{ desc, x_offsets, y_offset, src } }, - .location = location, - }); -} - -// ============================================================================= -// Map elementwise — like a pointwise `map` with a user-defined scalar region. -// ============================================================================= - -/// tt.map_elementwise — `scalar_block` receives `pack * srcs.len` scalar args -/// (packed groups of `pack` elements per src tensor) and must be terminated -/// with `tt.map_elementwise.return`. -pub fn map_elementwise( - ctx: *mlir.Context, - srcs: []const *const mlir.Value, - pack: i32, - scalar_block: *mlir.Block, - result_types: []const *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.map_elementwise", .{ - .operands = .{ .flat = srcs }, - .results = .{ .flat = result_types }, - .blocks = &.{scalar_block}, - .attributes = &.{ - .named(ctx, "pack", .int(ctx, .i32, pack)), - }, - .location = location, - }); -} - -pub fn map_elementwise_return( - ctx: *mlir.Context, - values: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.map_elementwise.return", .{ - .operands = .{ .flat = values }, - .verify = false, - .location = location, - }); -} - -// ============================================================================= -// Inline assembly -// ============================================================================= - -/// tt.elementwise_inline_asm — pointwise inline asm returning one or more -/// tensors. -pub fn elementwise_inline_asm( - ctx: *mlir.Context, - asm_string: []const u8, - constraints: []const u8, - args: []const *const mlir.Value, - result_types: []const *const mlir.Type, - pure: bool, - packed_element: i32, - location: *const mlir.Location, -) *mlir.Operation { - return .make(ctx, "tt.elementwise_inline_asm", .{ - .operands = .{ .flat = args }, - .results = .{ .flat = result_types }, - .attributes = &.{ - .named(ctx, "asm_string", .string(ctx, asm_string)), - .named(ctx, "constraints", .string(ctx, constraints)), - .named(ctx, "pure", .boolean(ctx, pure)), - .named(ctx, "packed_element", .int(ctx, .i32, packed_element)), - }, - .location = location, - }); -} diff --git a/mlir/dialects/ub/BUILD.bazel b/mlir/dialects/ub/BUILD.bazel deleted file mode 100644 index 039ca5ff..00000000 --- a/mlir/dialects/ub/BUILD.bazel +++ /dev/null @@ -1,34 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:UBDialect", - ], -) - -zig_library( - name = "ub", - import_name = "mlir/dialects/ub", - main = "ub.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":ub", - "//mlir", - ], -) diff --git a/mlir/dialects/ub/capi.cc b/mlir/dialects/ub/capi.cc deleted file mode 100644 index cf37a2c0..00000000 --- a/mlir/dialects/ub/capi.cc +++ /dev/null @@ -1,8 +0,0 @@ -//===- capi.cc - C API for the ub dialect ---------------------------------===// - -#include "capi.h" - -#include "mlir/CAPI/Registration.h" -#include "mlir/Dialect/UB/IR/UBOps.h" - -MLIR_DEFINE_CAPI_DIALECT_REGISTRATION(UB, ub, mlir::ub::UBDialect) diff --git a/mlir/dialects/ub/capi.h b/mlir/dialects/ub/capi.h deleted file mode 100644 index 76765ad9..00000000 --- a/mlir/dialects/ub/capi.h +++ /dev/null @@ -1,18 +0,0 @@ -//===- capi.h - C API for the ub dialect --------------------------*- C -*-===// - -#ifndef MLIR_DIALECTS_UB_CAPI_H -#define MLIR_DIALECTS_UB_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -MLIR_DECLARE_CAPI_DIALECT_REGISTRATION(UB, ub); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_UB_CAPI_H diff --git a/mlir/dialects/ub/test.zig b/mlir/dialects/ub/test.zig deleted file mode 100644 index 738d1378..00000000 --- a/mlir/dialects/ub/test.zig +++ /dev/null @@ -1,23 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const ub = @import("mlir/dialects/ub"); - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(ub); - inline for (comptime std.meta.declarations(ub)) |decl| { - const T = @field(ub, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "registerDialect registers ub" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - ub.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - try std.testing.expect(ctx.isRegisteredOperation("ub.poison")); -} diff --git a/mlir/dialects/ub/ub.zig b/mlir/dialects/ub/ub.zig deleted file mode 100644 index b5352257..00000000 --- a/mlir/dialects/ub/ub.zig +++ /dev/null @@ -1,5 +0,0 @@ -const mlir = @import("mlir"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("ub"); -} diff --git a/mlir/dialects/vector/BUILD.bazel b/mlir/dialects/vector/BUILD.bazel deleted file mode 100644 index 579fa599..00000000 --- a/mlir/dialects/vector/BUILD.bazel +++ /dev/null @@ -1,36 +0,0 @@ -load("@rules_cc//cc:defs.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -cc_library( - name = "capi", - srcs = ["capi.cc"], - hdrs = ["capi.h"], - tags = ["manual"], - deps = [ - "@llvm-project//mlir:CAPIIR", - "@llvm-project//mlir:VectorDialect", - ], -) - -zig_library( - name = "vector", - import_name = "mlir/dialects/vector", - main = "vector.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":capi", - "//mlir", - "//stdx", - "@llvm-project//mlir:CAPIVector", - ], -) - -zig_test( - name = "test", - main = "test.zig", - deps = [ - ":vector", - "//mlir", - ], -) diff --git a/mlir/dialects/vector/capi.cc b/mlir/dialects/vector/capi.cc deleted file mode 100644 index f22ad655..00000000 --- a/mlir/dialects/vector/capi.cc +++ /dev/null @@ -1,51 +0,0 @@ -//===- capi.cc - C API for the vector dialect -----------------------------===// - -#include "capi.h" - -#include "mlir/CAPI/IR.h" -#include "mlir/Dialect/Vector/IR/VectorOps.h" - -static mlir::MLIRContext *loadVector(MlirContext context) { - mlir::MLIRContext *ctx = unwrap(context); - if (ctx) - ctx->getOrLoadDialect(); - return ctx; -} - -bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirVectorCombiningKindAttrGet(MlirContext context, - uint32_t value) { - mlir::MLIRContext *ctx = loadVector(context); - std::optional kind = - mlir::vector::symbolizeCombiningKind(value); - if (!ctx || !kind) - return {nullptr}; - return wrap(mlir::vector::CombiningKindAttr::get(ctx, *kind)); -} - -uint32_t mlirVectorCombiningKindAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} - -bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr) { - return llvm::isa_and_nonnull(unwrap(attr)); -} - -MlirAttribute mlirVectorIteratorTypeAttrGet(MlirContext context, - uint32_t value) { - mlir::MLIRContext *ctx = loadVector(context); - std::optional type = - mlir::vector::symbolizeIteratorType(value); - if (!ctx || !type) - return {nullptr}; - return wrap(mlir::vector::IteratorTypeAttr::get(ctx, *type)); -} - -uint32_t mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr) { - return static_cast( - llvm::cast(unwrap(attr)).getValue()); -} diff --git a/mlir/dialects/vector/capi.h b/mlir/dialects/vector/capi.h deleted file mode 100644 index 75e12cbd..00000000 --- a/mlir/dialects/vector/capi.h +++ /dev/null @@ -1,36 +0,0 @@ -//===- capi.h - C API for the vector dialect ----------------------*- C -*-===// -// -// C entry points for the `vector` dialect's enum attributes, which upstream's -// C API leaves out. - -#ifndef MLIR_DIALECTS_VECTOR_CAPI_H -#define MLIR_DIALECTS_VECTOR_CAPI_H - -#include "mlir-c/IR.h" - -#ifdef __cplusplus -extern "C" { -#endif - -// Enum attributes take and return the C++ enum's integer value; `Get` returns -// a null attribute for a value outside the enum. - -// #vector.kind<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorCombiningKind(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirVectorCombiningKindAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirVectorCombiningKindAttrGetValue(MlirAttribute attr); - -// #vector.iterator_type<...> -MLIR_CAPI_EXPORTED bool mlirAttributeIsAVectorIteratorType(MlirAttribute attr); -MLIR_CAPI_EXPORTED MlirAttribute -mlirVectorIteratorTypeAttrGet(MlirContext context, uint32_t value); -MLIR_CAPI_EXPORTED uint32_t -mlirVectorIteratorTypeAttrGetValue(MlirAttribute attr); - -#ifdef __cplusplus -} -#endif - -#endif // MLIR_DIALECTS_VECTOR_CAPI_H diff --git a/mlir/dialects/vector/test.zig b/mlir/dialects/vector/test.zig deleted file mode 100644 index ca737ae7..00000000 --- a/mlir/dialects/vector/test.zig +++ /dev/null @@ -1,86 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const vector = @import("mlir/dialects/vector"); - -const operation_names: []const []const u8 = &.{ - "vector.bitcast", - "vector.broadcast", - "vector.compressstore", - "vector.constant_mask", - "vector.contract", - "vector.create_mask", - "vector.deinterleave", - "vector.expandload", - "vector.extract", - "vector.extract_strided_slice", - "vector.fma", - "vector.from_elements", - "vector.gather", - "vector.insert", - "vector.insert_strided_slice", - "vector.interleave", - "vector.load", - "vector.maskedload", - "vector.maskedstore", - "vector.multi_reduction", - "vector.reduction", - "vector.scatter", - "vector.shape_cast", - "vector.shuffle", - "vector.step", - "vector.store", - "vector.to_elements", - "vector.transfer_read", - "vector.transfer_write", - "vector.transpose", - "vector.type_cast", - "vector.vscale", - "vector.yield", -}; - -fn expectPrints(expected: []const u8, value: anytype) !void { - var buf: [128]u8 = undefined; - var w: std.Io.Writer = .fixed(&buf); - try w.print("{f}", .{value}); - try std.testing.expectEqualStrings(expected, w.buffered()); -} - -test { - @setEvalBranchQuota(100_000); - std.testing.refAllDecls(vector); - inline for (comptime std.meta.declarations(vector)) |decl| { - const T = @field(vector, decl); - if (@TypeOf(T) == type and @typeInfo(T) == .@"opaque") std.testing.refAllDecls(T); - } -} - -test "vector enum attributes are built through the C API" { - const ctx = try mlir.Context.init(.{}); - defer ctx.deinit(); - - inline for (@typeInfo(vector.CombiningKind).@"enum".field_names) |name| { - const value: vector.CombiningKind = @field(vector.CombiningKind, name); - const attr = try vector.CombiningKindAttr.get(ctx, .{ .value = value }); - try std.testing.expectEqual(value, attr.value()); - try std.testing.expect(value.attribute(ctx).isA(vector.CombiningKindAttr) != null); - try expectPrints("#vector.kind<" ++ name ++ ">", attr); - } - inline for (@typeInfo(vector.IteratorType).@"enum".field_names) |name| { - const value: vector.IteratorType = @field(vector.IteratorType, name); - const attr = try vector.IteratorTypeAttr.get(ctx, .{ .value = value }); - try std.testing.expectEqual(value, attr.value()); - try expectPrints("#vector.iterator_type<" ++ name ++ ">", attr); - } -} - -test "every bound operation is registered" { - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - vector.registerDialect(registry); - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - for (operation_names) |name| try std.testing.expect(ctx.isRegisteredOperation(name)); - try std.testing.expect(!ctx.allowUnregisteredDialects()); -} diff --git a/mlir/dialects/vector/vector.zig b/mlir/dialects/vector/vector.zig deleted file mode 100644 index 1696d04b..00000000 --- a/mlir/dialects/vector/vector.zig +++ /dev/null @@ -1,820 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const mlir = @import("mlir"); -const stdx = @import("stdx"); - -pub fn registerDialect(registry: *mlir.DialectRegistry) void { - registry.registerDialect("vector"); -} - -// ============================================================================= -// Enum attributes -// ============================================================================= - -/// Kind of combining function for contractions and reductions. -pub const CombiningKind = enum(u32) { - add = 0, - mul = 1, - minui = 2, - minsi = 3, - minnumf = 4, - maxui = 5, - maxsi = 6, - maxnumf = 7, - @"and" = 8, - @"or" = 9, - xor = 10, - minimumf = 11, - maximumf = 12, - minimumnumf = 13, - maximumnumf = 14, - - pub fn attribute(self: CombiningKind, ctx: *mlir.Context) *const mlir.Attribute { - return (CombiningKindAttr.get(ctx, .{ .value = self }) catch unreachable).attribute(); - } -}; - -/// Iterator type of a `vector.contract` dimension. -pub const IteratorType = enum(u32) { - parallel = 0, - reduction = 1, - - pub fn attribute(self: IteratorType, ctx: *mlir.Context) *const mlir.Attribute { - return (IteratorTypeAttr.get(ctx, .{ .value = self }) catch unreachable).attribute(); - } -}; - -/// `#vector.kind<...>`. -pub const CombiningKindAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAVectorCombiningKind; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "kind"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: CombiningKind, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirVectorCombiningKindAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) CombiningKind { - return @fromBackingInt(@intCast(c.mlirVectorCombiningKindAttrGetValue(self.ptr()))); - } -}; - -/// `#vector.iterator_type<...>`. -pub const IteratorTypeAttr = opaque { - const Self = @This(); - const M = mlir.Methods(Self, c.MlirAttribute); - pub const isAFn = c.mlirAttributeIsAVectorIteratorType; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - pub const isA = M.isA; - pub const mnemonic = "iterator_type"; - pub fn attribute(self: *const Self) *const mlir.Attribute { - return @ptrCast(self); - } - pub const InitArgs = struct { - value: IteratorType, - }; - pub fn get(ctx: *mlir.Context, args: InitArgs) mlir.Error!*const Self { - const result = c.mlirVectorIteratorTypeAttrGet(ctx.ptr(), @backingInt(args.value)); - return @ptrCast(result.ptr orelse return error.InvalidMlir); - } - pub fn value(self: *const Self) IteratorType { - return @fromBackingInt(@intCast(c.mlirVectorIteratorTypeAttrGetValue(self.ptr()))); - } -}; - -// ============================================================================= -// Element / lane access — extract / insert / from_elements / to_elements -// ============================================================================= - -/// vector.extract — pull a single element or sub-vector out of `src`. Static -/// position entries equal to `kPoison` (== `INT64_MIN` per VectorOps.td) come -/// from `dynamic_position`. The result type is provided by the caller. -pub const kPoison: i64 = std.math.minInt(i64); - -pub fn extract( - ctx: *mlir.Context, - src: *const mlir.Value, - static_position: []const i64, - dynamic_position: []const *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - operands_buf.appendAssumeCapacity(src); - operands_buf.appendSliceAssumeCapacity(dynamic_position); - - return mlir.Operation.make(ctx, "vector.extract", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "static_position", .denseArray(ctx, .i64, static_position)), - }, - .location = location, - }); -} - -/// vector.insert — write `value` into `dest` at the given position. The result -/// has the same type as `dest`. -pub fn insert( - ctx: *mlir.Context, - value: *const mlir.Value, - dest: *const mlir.Value, - static_position: []const i64, - dynamic_position: []const *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ value, dest }); - operands_buf.appendSliceAssumeCapacity(dynamic_position); - - return mlir.Operation.make(ctx, "vector.insert", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{dest.type_()} }, - .attributes = &.{ - .named(ctx, "static_position", .denseArray(ctx, .i64, static_position)), - }, - .location = location, - }); -} - -/// vector.from_elements — build a vector from an exact-size list of scalars. -pub fn from_elements( - ctx: *mlir.Context, - elements: []const *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.from_elements", .{ - .operands = .{ .flat = elements }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// vector.to_elements — destructure a vector into its scalar lanes. -pub fn to_elements( - ctx: *mlir.Context, - src: *const mlir.Value, - result_types: []const *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.to_elements", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = result_types }, - .location = location, - }); -} - -// ============================================================================= -// Strided slice — extract / insert with offsets/sizes/strides -// ============================================================================= - -pub const StridedSliceArgs = struct { - offsets: []const i64, - sizes: []const i64, - strides: []const i64, -}; - -pub fn extract_strided_slice( - ctx: *mlir.Context, - src: *const mlir.Value, - args: StridedSliceArgs, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.extract_strided_slice", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "offsets", .intArray(ctx, i64, args.offsets)), - .named(ctx, "sizes", .intArray(ctx, i64, args.sizes)), - .named(ctx, "strides", .intArray(ctx, i64, args.strides)), - }, - .location = location, - }); -} - -pub fn insert_strided_slice( - ctx: *mlir.Context, - src: *const mlir.Value, - dest: *const mlir.Value, - offsets: []const i64, - strides: []const i64, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.insert_strided_slice", .{ - .operands = .{ .flat = &.{ src, dest } }, - .results = .{ .flat = &.{dest.type_()} }, - .attributes = &.{ - .named(ctx, "offsets", .intArray(ctx, i64, offsets)), - .named(ctx, "strides", .intArray(ctx, i64, strides)), - }, - .location = location, - }); -} - -// ============================================================================= -// Shape manipulation -// ============================================================================= - -/// vector.broadcast — broadcast a scalar / lower-rank vector to a wider one. -pub fn broadcast( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.broadcast", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// vector.shape_cast — reshape a vector preserving total number of lanes. -pub fn shape_cast( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.shape_cast", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// vector.bitcast — same total bits, possibly different element type / shape. -pub fn bitcast( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.bitcast", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// vector.type_cast — cast a `memref>` to `memref<...>` (and vice- -/// versa). Used in vector lowering / unrolling. -pub fn type_cast( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.type_cast", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// vector.transpose — permute dims of a vector. -pub fn transpose( - ctx: *mlir.Context, - src: *const mlir.Value, - permutation: []const i64, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.transpose", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "permutation", .denseArray(ctx, .i64, permutation)), - }, - .location = location, - }); -} - -// ============================================================================= -// Lane shuffle — shuffle / interleave / deinterleave -// ============================================================================= - -/// vector.shuffle — concat lhs and rhs into a logical vector (rank 1) and -/// pick `mask`-listed lanes to form the result. -pub fn shuffle( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - mask: []const i64, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.shuffle", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "mask", .denseArray(ctx, .i64, mask)), - }, - .location = location, - }); -} - -/// vector.interleave — interleave even/odd lanes from lhs/rhs. -pub fn interleave( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.interleave", .{ - .operands = .{ .flat = &.{ lhs, rhs } }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// vector.deinterleave — split a vector into (even-lanes, odd-lanes). -pub fn deinterleave( - ctx: *mlir.Context, - src: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.deinterleave", .{ - .operands = .{ .flat = &.{src} }, - .results = .{ .flat = &.{ result_type, result_type } }, - .location = location, - }); -} - -// ============================================================================= -// Compute — fma / step / vscale -// ============================================================================= - -/// vector.fma — element-wise fused multiply-add `lhs * rhs + acc`. -pub fn fma( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - acc: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.fma", .{ - .operands = .{ .flat = &.{ lhs, rhs, acc } }, - .result_type_inference = true, - .location = location, - }); -} - -/// vector.step — produce `<0, 1, ..., N-1>` of the result type. -pub fn step( - ctx: *mlir.Context, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.step", .{ - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -/// vector.vscale — runtime scalable-vector size factor (index-typed). -pub fn vscale(ctx: *mlir.Context, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.vscale", .{ - .results = .{ .flat = &.{.index(ctx)} }, - .location = location, - }); -} - -/// vector.yield — terminator for vector.mask / vector.warp regions. -pub fn yield(ctx: *mlir.Context, values: []const *const mlir.Value, location: *const mlir.Location) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.yield", .{ - .operands = .{ .flat = values }, - .verify = false, - .location = location, - }); -} - -// ============================================================================= -// Reductions — reduction / multi_reduction / contract -// ============================================================================= - -pub const ReductionOpts = struct { - /// Optional accumulator (scalar of result type). When provided the op is - /// `result = combine(reduce(src), acc)`. - acc: ?*const mlir.Value = null, -}; - -/// vector.reduction — reduce a 1-D vector to a scalar with `kind`. -pub fn reduction( - ctx: *mlir.Context, - kind: CombiningKind, - src: *const mlir.Value, - result_type: *const mlir.Type, - opts: ReductionOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 2) = .empty; - operands_buf.appendAssumeCapacity(src); - if (opts.acc) |a| operands_buf.appendAssumeCapacity(a); - - return mlir.Operation.make(ctx, "vector.reduction", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "kind", kind.attribute(ctx)), - }, - .location = location, - }); -} - -/// vector.multi_reduction — reduce a multi-D vector along `reduction_dims`. -pub fn multi_reduction( - ctx: *mlir.Context, - kind: CombiningKind, - src: *const mlir.Value, - acc: *const mlir.Value, - reduction_dims: []const i64, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.multi_reduction", .{ - .operands = .{ .flat = &.{ src, acc } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "kind", kind.attribute(ctx)), - .named(ctx, "reduction_dims", .denseArray(ctx, .i64, reduction_dims)), - }, - .location = location, - }); -} - -pub const ContractOpts = struct { - kind: CombiningKind = .add, -}; - -/// vector.contract — generalized contraction (matmul, dot, batched matmul, -/// reductions). `indexing_maps` is one AffineMapAttr per (lhs, rhs, acc). -/// `iterator_types` is one IteratorType per dim of the iteration space. -pub fn contract( - ctx: *mlir.Context, - lhs: *const mlir.Value, - rhs: *const mlir.Value, - acc: *const mlir.Value, - indexing_maps: []const *const mlir.Attribute, - iterator_types: []const IteratorType, - result_type: *const mlir.Type, - opts: ContractOpts, - location: *const mlir.Location, -) *mlir.Operation { - var iter_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; - for (iterator_types) |it| iter_attrs.appendAssumeCapacity(it.attribute(ctx)); - - return mlir.Operation.make(ctx, "vector.contract", .{ - .operands = .{ .flat = &.{ lhs, rhs, acc } }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "indexing_maps", .array(ctx, indexing_maps)), - .named(ctx, "iterator_types", .array(ctx, iter_attrs.constSlice())), - .named(ctx, "kind", opts.kind.attribute(ctx)), - }, - .location = location, - }); -} - -// ============================================================================= -// Memory transfers — load / store / masked / gather / scatter / transfer_* -// ============================================================================= - -pub const LoadStoreOpts = struct { - nontemporal: bool = false, - alignment: ?i64 = null, -}; - -fn loadStoreAttrs(ctx: *mlir.Context, opts: LoadStoreOpts, dst: *stdx.BoundedArray(mlir.NamedAttribute, 2)) void { - if (opts.nontemporal) { - dst.appendAssumeCapacity(.named(ctx, "nontemporal", .boolean(ctx, true))); - } - if (opts.alignment) |a| { - dst.appendAssumeCapacity(.named(ctx, "alignment", .int(ctx, .i64, a))); - } -} - -/// vector.load — strided contiguous load from `base[indices...]` into the -/// result vector. -pub fn load( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const *const mlir.Value, - result_type: *const mlir.Type, - opts: LoadStoreOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 16) = .empty; - operands_buf.appendAssumeCapacity(base); - operands_buf.appendSliceAssumeCapacity(indices); - - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - loadStoreAttrs(ctx, opts, &attrs); - - return mlir.Operation.make(ctx, "vector.load", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// vector.store — strided contiguous store of `value` to `base[indices...]`. -pub fn store( - ctx: *mlir.Context, - value: *const mlir.Value, - base: *const mlir.Value, - indices: []const *const mlir.Value, - opts: LoadStoreOpts, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 17) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ value, base }); - operands_buf.appendSliceAssumeCapacity(indices); - - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 2) = .empty; - loadStoreAttrs(ctx, opts, &attrs); - - return mlir.Operation.make(ctx, "vector.store", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .attributes = attrs.constSlice(), - .location = location, - }); -} - -/// vector.maskedload — like load but lanes whose `mask` bit is 0 are read -/// from `passthru` instead. -pub fn masked_load( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const *const mlir.Value, - mask: *const mlir.Value, - passthru: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; - operands_buf.appendAssumeCapacity(base); - operands_buf.appendSliceAssumeCapacity(indices); - operands_buf.appendSliceAssumeCapacity(&.{ mask, passthru }); - - return mlir.Operation.make(ctx, "vector.maskedload", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn masked_store( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const *const mlir.Value, - mask: *const mlir.Value, - value: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; - operands_buf.appendAssumeCapacity(base); - operands_buf.appendSliceAssumeCapacity(indices); - operands_buf.appendSliceAssumeCapacity(&.{ mask, value }); - - return mlir.Operation.make(ctx, "vector.maskedstore", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .location = location, - }); -} - -/// vector.gather — gather scalar elements from `base[indices...]` plus per-lane -/// `index_vec` offsets. -pub fn gather( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const *const mlir.Value, - index_vec: *const mlir.Value, - mask: *const mlir.Value, - passthru: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; - operands_buf.appendAssumeCapacity(base); - operands_buf.appendSliceAssumeCapacity(indices); - operands_buf.appendSliceAssumeCapacity(&.{ index_vec, mask, passthru }); - - return mlir.Operation.make(ctx, "vector.gather", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn scatter( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const *const mlir.Value, - index_vec: *const mlir.Value, - mask: *const mlir.Value, - value: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; - operands_buf.appendAssumeCapacity(base); - operands_buf.appendSliceAssumeCapacity(indices); - operands_buf.appendSliceAssumeCapacity(&.{ index_vec, mask, value }); - - return mlir.Operation.make(ctx, "vector.scatter", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .location = location, - }); -} - -pub fn expand_load( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const *const mlir.Value, - mask: *const mlir.Value, - passthru: *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; - operands_buf.appendAssumeCapacity(base); - operands_buf.appendSliceAssumeCapacity(indices); - operands_buf.appendSliceAssumeCapacity(&.{ mask, passthru }); - - return mlir.Operation.make(ctx, "vector.expandload", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} - -pub fn compress_store( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const *const mlir.Value, - mask: *const mlir.Value, - value: *const mlir.Value, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 18) = .empty; - operands_buf.appendAssumeCapacity(base); - operands_buf.appendSliceAssumeCapacity(indices); - operands_buf.appendSliceAssumeCapacity(&.{ mask, value }); - - return mlir.Operation.make(ctx, "vector.compressstore", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .location = location, - }); -} - -// ============================================================================= -// Transfer ops — vector.transfer_read / vector.transfer_write -// ============================================================================= - -pub const TransferReadArgs = struct { - /// `affine_map<(d0,...) -> (...)>` attribute. Build with - /// `mlir.parseAffineMapAttribute` or `mlir.Attribute.affineMap(...)`. - permutation_map: *const mlir.Attribute, - /// One bool per result-vector dim. Length must equal vector rank. - in_bounds: []const bool, - /// Optional i1 vector mask. - mask: ?*const mlir.Value = null, -}; - -/// vector.transfer_read — supervector load with permutation map and optional -/// mask. AttrSizedOperandSegments: (base, indices, padding, [mask]). -pub fn transfer_read( - ctx: *mlir.Context, - base: *const mlir.Value, - indices: []const *const mlir.Value, - padding: *const mlir.Value, - args: TransferReadArgs, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; - operands_buf.appendAssumeCapacity(base); - operands_buf.appendSliceAssumeCapacity(indices); - operands_buf.appendAssumeCapacity(padding); - const mask_len: i32 = if (args.mask) |m| blk: { - operands_buf.appendAssumeCapacity(m); - break :blk 1; - } else 0; - - const seg_sizes = [4]i32{ 1, @intCast(indices.len), 1, mask_len }; - - var in_bounds_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; - for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(.boolean(ctx, b)); - - return mlir.Operation.make(ctx, "vector.transfer_read", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "permutation_map", args.permutation_map), - .named(ctx, "in_bounds", .array(ctx, in_bounds_attrs.constSlice())), - }, - .location = location, - }); -} - -pub const TransferWriteArgs = struct { - permutation_map: *const mlir.Attribute, - in_bounds: []const bool, - mask: ?*const mlir.Value = null, - /// When `dest` is a tensor, the op produces a new tensor — pass its type - /// here. For memref destinations leave null. - result_type: ?*const mlir.Type = null, -}; - -/// vector.transfer_write — supervector store with permutation map and optional -/// mask. AttrSizedOperandSegments: (vector, base, indices, [mask]). -pub fn transfer_write( - ctx: *mlir.Context, - value: *const mlir.Value, - base: *const mlir.Value, - indices: []const *const mlir.Value, - args: TransferWriteArgs, - location: *const mlir.Location, -) *mlir.Operation { - var operands_buf: stdx.BoundedArray(*const mlir.Value, 20) = .empty; - operands_buf.appendSliceAssumeCapacity(&.{ value, base }); - operands_buf.appendSliceAssumeCapacity(indices); - const mask_len: i32 = if (args.mask) |m| blk: { - operands_buf.appendAssumeCapacity(m); - break :blk 1; - } else 0; - - const seg_sizes = [4]i32{ 1, 1, @intCast(indices.len), mask_len }; - - var in_bounds_attrs: stdx.BoundedArray(*const mlir.Attribute, 16) = .empty; - for (args.in_bounds) |b| in_bounds_attrs.appendAssumeCapacity(.boolean(ctx, b)); - - var result_types: stdx.BoundedArray(*const mlir.Type, 1) = .empty; - if (args.result_type) |rt| result_types.appendAssumeCapacity(rt); - - return mlir.Operation.make(ctx, "vector.transfer_write", .{ - .operands = .{ .flat = operands_buf.constSlice() }, - .results = .{ .flat = result_types.constSlice() }, - .attributes = &.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, &seg_sizes)), - .named(ctx, "permutation_map", args.permutation_map), - .named(ctx, "in_bounds", .array(ctx, in_bounds_attrs.constSlice())), - }, - .location = location, - }); -} - -// ============================================================================= -// Masks — constant_mask / create_mask -// ============================================================================= - -/// vector.constant_mask — constant boolean mask. Each entry of -/// `mask_dim_sizes` is the number of leading true lanes along that dim. -pub fn constant_mask( - ctx: *mlir.Context, - mask_dim_sizes: []const i64, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.constant_mask", .{ - .results = .{ .flat = &.{result_type} }, - .attributes = &.{ - .named(ctx, "mask_dim_sizes", .denseArray(ctx, .i64, mask_dim_sizes)), - }, - .location = location, - }); -} - -/// vector.create_mask — runtime mask from per-dim count operands. `operands` -/// length must equal the rank of `result_type`. -pub fn create_mask( - ctx: *mlir.Context, - operands: []const *const mlir.Value, - result_type: *const mlir.Type, - location: *const mlir.Location, -) *mlir.Operation { - return mlir.Operation.make(ctx, "vector.create_mask", .{ - .operands = .{ .flat = operands }, - .results = .{ .flat = &.{result_type} }, - .location = location, - }); -} diff --git a/mlir/mlir.h b/mlir/mlir.h index b897c82b..4ad1523e 100644 --- a/mlir/mlir.h +++ b/mlir/mlir.h @@ -1,15 +1,9 @@ -//===- mlir.h - MLIR C API headers bound by the mlir module -------*- C -*-===// - -#ifndef MLIR_MLIR_H -#define MLIR_MLIR_H - -#include "mlir-c/AffineExpr.h" -#include "mlir-c/AffineMap.h" -#include "mlir-c/BuiltinAttributes.h" -#include "mlir-c/BuiltinTypes.h" -#include "mlir-c/IR.h" -#include "mlir-c/IntegerSet.h" -#include "mlir-c/Pass.h" -#include "mlir-c/Transforms.h" - -#endif // MLIR_MLIR_H +#include +#include +#include +#include +#include +#include +#include +#include +#include diff --git a/mlir/mlir.zig b/mlir/mlir.zig index 57e57bd2..6f4bef3b 100644 --- a/mlir/mlir.zig +++ b/mlir/mlir.zig @@ -9,57 +9,6 @@ test { std.testing.refAllDecls(@This()); } -test "variadic operation groups preserve order and dense array segment sizes" { - const ctx = try Context.init(.{}); - defer ctx.deinit(); - ctx.setAllowUnregisteredDialects(true); - const loc = Location.unknown(ctx); - const i32_type = Type.int(ctx, .i32); - const i64_type = Type.int(ctx, .i64); - const block = Block.init(&.{ i32_type, i64_type, i32_type }, &.{ loc, loc, loc }); - defer block.deinit(); - const op = Operation.make(ctx, "test.segmented", .{ - .operands = .{ .variadic = &.{ - &.{block.argument(0)}, - &.{}, - &.{ block.argument(1), block.argument(2) }, - } }, - .results = .{ .variadic = &.{ - &.{}, - &.{ i64_type, i32_type }, - &.{i64_type}, - } }, - .location = loc, - }); - defer op.deinit(); - try std.testing.expectEqual(@as(usize, 3), op.numOperands()); - for (0..3) |i| try std.testing.expect(op.operand(i).eql(block.argument(i))); - try std.testing.expectEqual(@as(usize, 3), op.numResults()); - try std.testing.expect(op.result(0).type_().eql(i64_type)); - try std.testing.expect(op.result(1).type_().eql(i32_type)); - try std.testing.expect(op.result(2).type_().eql(i64_type)); - try std.testing.expect(op.attributeByName("operandSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 1, 0, 2 }))); - try std.testing.expect(op.attributeByName("resultSegmentSizes").?.eql(.denseArray(ctx, .i32, &.{ 0, 2, 1 }))); -} - -test "unrealized_conversion_cast builds a verified operation" { - const ctx = try Context.init(.{}); - defer ctx.deinit(); - const loc = Location.unknown(ctx); - const block = Block.init(&.{Type.int(ctx, .i32)}, &.{loc}); - defer block.deinit(); - const cast = unrealized_conversion_cast(ctx, &.{block.argument(0)}, &.{Type.index(ctx)}, loc).appendTo(block); - try std.testing.expect(cast.verify()); - try std.testing.expect(cast.result(0).type_().eql(Type.index(ctx))); -} - -test "nested symbol references print their path" { - const ctx = try Context.init(.{}); - defer ctx.deinit(); - const symbol = Attribute.symbolRef(ctx, "kernels", &.{FlatSymbolRefAttribute.init(ctx, "add")}); - try std.testing.expect(symbol.eql(try Attribute.parse(ctx, "@kernels::@add"))); -} - pub const Error = error{ /// Invalid Mlir was created. InvalidMlir, @@ -102,23 +51,23 @@ pub fn registerPasses(comptime passes: []const u8) void { } fn isPtrConst(comptime T: type) bool { - return @typeInfo(T).pointer.attrs.@"const"; + return @typeInfo(T).pointer.is_const; } fn CastedPtr(comptime T1: type, comptime T2: type) type { - return if (@typeInfo(T1).pointer.attrs.@"const") *const T2 else *T2; + return if (@typeInfo(T1).pointer.is_const) *const T2 else *T2; } pub fn Methods(comptime T: type, comptime NativeT: type) type { return struct { - pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.attrs.@"const"; + pub const IsPtrConst = @typeInfo(@typeInfo(@FieldType(NativeT, "ptr")).optional.child).pointer.is_const; pub const Ptr = if (IsPtrConst) *const T else *T; pub fn ptr(self: anytype) NativeT { return .{ .ptr = @ptrCast(@constCast(self)) }; } - pub fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { + fn format(f: anytype) fn (self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { return struct { fn format(self: *const T, writer: *std.Io.Writer) std.Io.Writer.Error!void { var ctx: StringCallbackCtx = .{ .writer = writer }; @@ -300,14 +249,6 @@ pub const Context = opaque { return c.mlirContextIsRegisteredOperation(self.ptr(), stringRef(op)); } - pub fn setAllowUnregisteredDialects(self: *Context, allow: bool) void { - c.mlirContextSetAllowUnregisteredDialects(self.ptr(), allow); - } - - pub fn allowUnregisteredDialects(self: *const Context) bool { - return c.mlirContextGetAllowUnregisteredDialects(self.ptr()); - } - pub fn format(self: *const Context, writer: *std.Io.Writer) !void { try writer.print("{s}{{ .numThreads = {d}, .numRegisteredDialects = {d}, .numLoadedDialects = {d} }}", .{ @typeName(Context), @@ -362,7 +303,7 @@ pub const Location = opaque { pub fn namedFmt(loc: *const Location, ctx: *Context, comptime fmt: []const u8, args: anytype) *const Location { var buf: [256]u8 = undefined; - return loc.named(ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { + return loc.named(loc, ctx, std.fmt.bufPrint(&buf, fmt, args) catch blk: { buf[buf.len - 3 ..].* = "...".*; break :blk &buf; }); @@ -375,77 +316,6 @@ pub const Location = opaque { pub fn unknown(ctx: *Context) *const Location { return @ptrCast(c.mlirLocationUnknownGet(ctx.ptr()).ptr); } - - pub const Tagged = union(enum) { - unknown, - named: *const Named, - callsite: *const Callsite, - file_line_col: *const FileLineCol, - fused: *const Fused, - }; - - pub fn inspect(location: *const Location) Tagged { - const loc_ptr = location.ptr(); - return if (c.mlirLocationIsAUnknown(loc_ptr)) - .unknown - else if (c.mlirLocationIsAName(loc_ptr)) - .{ .named = @ptrCast(location) } - else if (c.mlirLocationIsACallSite(loc_ptr)) - .{ .callsite = @ptrCast(location) } - else if (c.mlirLocationIsAFileLineColRange(loc_ptr)) - .{ .file_line_col = @ptrCast(location) } - else if (c.mlirLocationIsAFused(loc_ptr)) - .{ .fused = @ptrCast(location) } - else - // Theoritically the C++ API has "OpaqueLoc" but it's not part of C bindings, - // I think in this case it's fine to fall back to .unknown - // https://mlir.llvm.org/docs/Dialects/Builtin/#opaqueloc - .unknown; - } - - pub const Named = opaque { - pub fn name(location: *const Named) []const u8 { - const identifier: *const Identifier = @ptrCast(c.mlirLocationNameGetName(.{ .ptr = location }).ptr); - return identifier.str(); - } - - pub fn childLoc(location: *const Named) *const Location { - return @ptrCast(c.mlirLocationNameGetChildLoc(.{ .ptr = location }).ptr); - } - }; - - pub const Callsite = opaque { - pub fn callee(location: *const Callsite) *const Location { - return @ptrCast(c.mlirLocationCallSiteGetCallee(.{ .ptr = location }).ptr); - } - - pub fn caller(location: *const Callsite) *const Location { - return @ptrCast(c.mlirLocationCallSiteGetCaller(.{ .ptr = location }).ptr); - } - }; - - pub const FileLineCol = opaque { - pub fn src(location: *const FileLineCol) *const std.builtin.SourceLocation { - const file_id: *const Identifier = @ptrCast(c.mlirLocationFileLineColRangeGetFilename(.{ .ptr = location }).ptr); - return .{ - .file = file_id.str(), - .line = @truncate(c.mlirLocationFileLineColRangeGetStartLine(.{ .ptr = location })), - .column = @truncate(c.mlirLocationFileLineColRangeGetStartColumn(.{ .ptr = location })), - }; - } - }; - - pub const Fused = opaque { - pub fn len(location: *const Fused) u32 { - return @intCast(c.mlirLocationFusedGetNumLocations(.{ .ptr = location })); - } - - pub fn locations(location: *const Fused, allocator: std.mem.Allocator) error{OutOfMemory}![]*const Location { - const result = try allocator.alloc(*const Location, location.len()); - c.mlirLocationFusedGetLocations(.{ .ptr = location }, @ptrCast(result.ptr)); - return result; - } - }; }; pub const Type = opaque { @@ -459,56 +329,6 @@ pub const Type = opaque { pub fn parse(ctx: *Context, str: []const u8) Error!*const Type { return @ptrCast(c.mlirTypeParseGet(ctx.ptr(), stringRef(str)).ptr orelse return Error.InvalidMlir); } - - pub fn isFloat(self: *const Type) bool { - return c.mlirTypeIsAFloat(self.ptr()); - } - - pub fn index(ctx: *Context) *const Type { - return @ptrCast(IndexType.get(ctx)); - } - - pub fn int(ctx: *Context, it: IntegerTypes) *const Type { - return @ptrCast(IntegerType.get(ctx, it)); - } - - pub fn float(ctx: *Context, ft: FloatTypes) *const Type { - return switch (ft) { - inline else => |v| @ptrCast(FloatType(v).get(ctx)), - }; - } - - pub fn complex(ctx: *Context, complex_type: ComplexTypes) *const Type { - return @ptrCast(ComplexType.get(ctx, complex_type)); - } - - pub fn rankedTensor(dimensions: []const i64, elem_type: *const Type) *const Type { - return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); - } - - pub fn rankedTensorWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const Type { - return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); - } - - pub fn function(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { - return @ptrCast(FunctionType.init(ctx, args, results)); - } - - pub fn memRef(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { - return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); - } - - pub fn element(self: *const VectorType) *const Type { - return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); - } - - pub fn vector(shape: []const i64, element_type: *const Type) *const Type { - return @ptrCast(VectorType.init(shape, element_type)); - } - - pub fn unrankedMemRef(element_type: *const Type, memory_space: ?*const Attribute) *const Type { - return @ptrCast(UnrankedMemRefType.init(element_type, memory_space)); - } }; pub const IndexType = opaque { @@ -524,6 +344,10 @@ pub const IndexType = opaque { } }; +pub fn indexType(ctx: *Context) *const Type { + return @ptrCast(IndexType.get(ctx)); +} + pub const IntegerTypes = enum { i1, i2, @@ -585,10 +409,6 @@ pub const IntegerType = opaque { pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); - pub fn width(self: *const IntegerType) u32 { - return c.mlirIntegerTypeGetWidth(self.ptr()); - } - fn get(ctx: *Context, it: IntegerTypes) *const IntegerType { return exact(ctx, it.bitwidth(), it.signedness()); } @@ -602,6 +422,10 @@ pub const IntegerType = opaque { } }; +pub fn integerType(ctx: *Context, it: IntegerTypes) *const Type { + return @ptrCast(IntegerType.get(ctx, it)); +} + pub const FloatTypes = enum { f4e2m1fn, f6e2m3fn, @@ -658,6 +482,12 @@ pub fn FloatType(comptime ft: FloatTypes) type { }; } +pub fn floatType(ctx: *Context, ft: FloatTypes) *const Type { + return switch (ft) { + inline else => |v| @ptrCast(FloatType(v).get(ctx)), + }; +} + pub const ComplexTypes = enum { c64, c128, @@ -673,12 +503,16 @@ pub const ComplexType = opaque { pub fn get(ctx: *Context, complex_type: ComplexTypes) *const ComplexType { return @ptrCast(c.mlirComplexTypeGet(switch (complex_type) { - .c64 => Type.float(ctx, .f32).ptr(), - .c128 => Type.float(ctx, .f64).ptr(), + .c64 => floatType(ctx, .f32).ptr(), + .c128 => floatType(ctx, .f64).ptr(), }).ptr); } }; +pub fn complexType(ctx: *Context, complex_type: ComplexTypes) *const Type { + return @ptrCast(ComplexType.get(ctx, complex_type)); +} + pub const Attribute = opaque { const M = Methods(Attribute, c.MlirAttribute); @@ -698,71 +532,6 @@ pub const Attribute = opaque { pub fn dialect(self: *const Context) *const Dialect { return @ptrCast(c.mlirAttributeGetDialect(self.ptr()).ptr); } - - pub fn string(ctx: *Context, str: []const u8) *const Attribute { - return @ptrCast(StringAttribute.init(ctx, str)); - } - - pub fn int(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { - return @ptrCast(IntegerAttribute.init(ctx, it, value_)); - } - - pub fn intFromType(it: *const IntegerType, value_: i64) *const Attribute { - return @ptrCast(IntegerAttribute.fromType(it, value_)); - } - - pub fn float(ctx: *Context, comptime ft: FloatTypes, value_: anytype) *const Attribute { - return @ptrCast(FloatAttribute.init(ctx, ft, value_)); - } - pub fn boolean(ctx: *Context, value: bool) *const Attribute { - return @ptrCast(BoolAttribute.init(ctx, value)); - } - - pub fn typeAttr(type_: *const Type) *const Attribute { - return @ptrCast(TypeAttribute.init(type_)); - } - - pub fn dict(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { - return @ptrCast(DictionaryAttribute.init(ctx, attributes)); - } - - pub fn denseElements(shaped_type: *const ShapedType, values: anytype) *const Attribute { - return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); - } - - pub fn denseArray(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { - return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); - } - pub fn array(ctx: *Context, attrs: []const *const Attribute) *const Attribute { - return @ptrCast(ArrayAttribute.init(ctx, attrs)); - } - - pub fn intArray(ctx: *Context, comptime T: type, values: []const T) *const Attribute { - const it: IntegerTypes = comptime @field(IntegerTypes, @typeName(T)); - var buf: stdx.BoundedArray(*const Attribute, ShapedType.MAX_RANK) = .empty; - for (values) |v| buf.appendAssumeCapacity(.int(ctx, it, v)); - return .array(ctx, buf.constSlice()); - } - - pub fn flatSymbolRef(ctx: *Context, symbol: []const u8) *const Attribute { - return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); - } - - pub fn symbolRef(ctx: *Context, root: []const u8, nested: []const *const FlatSymbolRefAttribute) *const Attribute { - return @ptrCast(SymbolRefAttribute.init(ctx, root, nested)); - } - - pub fn unit(ctx: *Context) *const Attribute { - return @ptrCast(UnitAttribute.get(ctx)); - } - - pub fn affineMap(map: *const AffineMap) *const Attribute { - return @ptrCast(AffineMapAttribute.init(map)); - } - - pub fn integerSet(set: *const IntegerSet) *const Attribute { - return @ptrCast(IntegerSetAttribute.init(set)); - } }; pub const StringAttribute = opaque { @@ -782,6 +551,10 @@ pub const StringAttribute = opaque { } }; +pub fn stringAttribute(ctx: *Context, str: []const u8) *const Attribute { + return @ptrCast(StringAttribute.init(ctx, str)); +} + pub const IntegerAttribute = opaque { const M = Methods(IntegerAttribute, c.MlirAttribute); @@ -791,11 +564,10 @@ pub const IntegerAttribute = opaque { pub const format = M.format(c.mlirAttributePrint); pub fn init(ctx: *Context, it: IntegerTypes, value_: anytype) *const IntegerAttribute { - return .fromType(.get(ctx, it), @intCast(value_)); - } - - pub fn fromType(it: *const IntegerType, value_: i64) *const IntegerAttribute { - return @ptrCast(c.mlirIntegerAttrGet(it.ptr(), value_).ptr); + return @ptrCast(c.mlirIntegerAttrGet( + IntegerType.get(ctx, it).ptr(), + @intCast(value_), + ).ptr); } pub fn value(self: *const IntegerAttribute, comptime T: type) T { @@ -807,6 +579,10 @@ pub const IntegerAttribute = opaque { } }; +pub fn integerAttribute(ctx: *Context, it: IntegerTypes, value_: anytype) *const Attribute { + return @ptrCast(IntegerAttribute.init(ctx, it, value_)); +} + pub const FloatAttribute = opaque { const M = Methods(FloatAttribute, c.MlirAttribute); @@ -824,6 +600,10 @@ pub const FloatAttribute = opaque { } }; +pub fn floatAttribute(ctx: *Context, comptime ft: FloatTypes, value_: anytype) *const Attribute { + return @ptrCast(FloatAttribute.init(ctx, ft, value_)); +} + pub const BoolAttribute = opaque { const M = Methods(BoolAttribute, c.MlirAttribute); @@ -841,6 +621,10 @@ pub const BoolAttribute = opaque { } }; +pub fn boolAttribute(ctx: *Context, value: bool) *const Attribute { + return @ptrCast(BoolAttribute.init(ctx, value)); +} + pub const TypeAttribute = opaque { const M = Methods(TypeAttribute, c.MlirAttribute); @@ -858,6 +642,10 @@ pub const TypeAttribute = opaque { } }; +pub fn typeAttribute(type_: *const Type) *const Attribute { + return @ptrCast(TypeAttribute.init(type_)); +} + pub const Dialect = struct { const M = Methods(Dialect, c.MlirDialect); @@ -1009,24 +797,10 @@ pub const Block = opaque { } } - pub fn firstOperation(self: *const Block) ?*Operation { - const op = c.mlirBlockGetFirstOperation(self.ptr()); - if (op.ptr == null) return null; - return @ptrCast(op.ptr); - } - - pub fn insertOwnedOperationBefore(self: *Block, reference: *Operation, op: *Operation) void { - c.mlirBlockInsertOwnedOperationBefore(self.ptr(), reference.ptr(), op.ptr()); - } - pub fn parentOperation(self: *const Block) ?*Operation { return @ptrCast(c.mlirBlockGetParentOperation(self.ptr()).ptr); } - pub fn parentRegion(self: *const Block) *Region { - return @ptrCast(c.mlirBlockGetParentRegion(self.ptr()).ptr); - } - pub fn detach(self: *Block) void { c.mlirBlockDetach(self.ptr()); } @@ -1107,6 +881,10 @@ pub const DictionaryAttribute = opaque { } }; +pub fn dictionaryAttribute(ctx: *Context, attributes: []const NamedAttribute) *const Attribute { + return @ptrCast(DictionaryAttribute.init(ctx, attributes)); +} + pub const Value = opaque { const M = Methods(Value, c.MlirValue); @@ -1248,7 +1026,7 @@ pub const Operation = opaque { } pub fn name(self: *const Operation) []const u8 { - return string(c.mlirIdentifierStr(c.mlirOperationGetName(self.ptr()))); + return string(c.mlirOperationGetName(self.ptr())); } pub fn clone(self: *const Operation) *Operation { @@ -1264,14 +1042,6 @@ pub const Operation = opaque { return self; } - pub fn numRegions(self: *const Operation) usize { - return @intCast(c.mlirOperationGetNumRegions(self.ptr())); - } - - pub fn region(self: *const Operation, index: usize) *Region { - return @ptrCast(c.mlirOperationGetRegion(self.ptr(), @intCast(index)).ptr); - } - pub fn parent(self: *const Operation) ?*Operation { return @ptrCast(c.mlirOperationGetParentOperation(self.ptr()).ptr); } @@ -1312,16 +1082,6 @@ pub const Operation = opaque { c.mlirOperationSetAttributeByName(self.ptr(), stringRef(name_), attribute.ptr()); } - /// Sets `attribute` under its name, as a dialect's attribute helper names it. - pub fn setAttribute(self: *Operation, attribute: NamedAttribute) void { - self.setAttributeByName(attribute.name().str(), attribute.attribute()); - } - - pub fn attributeByName(self: *const Operation, name_: []const u8) ?*const Attribute { - const attr = c.mlirOperationGetAttributeByName(self.ptr(), stringRef(name_)); - return if (attr.ptr == null) null else @ptrCast(attr.ptr); - } - pub const PrintFlags = struct { elide_large_elements_attrs: ?usize = null, debug_info: bool = false, @@ -1329,7 +1089,6 @@ pub const Operation = opaque { print_generic_op_form: bool = false, use_local_scope: bool = false, assume_verified: bool = false, - print_name_loc_as_prefix: bool = false, const Ctx = struct { self: *const Operation, @@ -1355,9 +1114,6 @@ pub const Operation = opaque { if (self.assume_verified) { c.mlirOpPrintingFlagsAssumeVerified(flags); } - if (self.print_name_loc_as_prefix) { - c.mlirOpPrintingFlagsPrintNameLocAsPrefix(flags); - } return flags; } }; @@ -1398,7 +1154,7 @@ pub const Operation = opaque { attributes: ?[]const NamedAttribute = null, blocks: ?[]const *Block = null, verify: bool = true, - location: ?*const Location, + location: ?*const Location = null, }; pub fn try_make(ctx: *Context, name_: []const u8, args: MakeArgs) !*Operation { @@ -1409,26 +1165,26 @@ pub const Operation = opaque { if (args.operands) |operands| switch (operands) { .flat => |v| state.addOperands(v), .variadic => |segments| { - var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .empty; + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; for (segments) |segment_operands| { state.addOperands(segment_operands); sizes.appendAssumeCapacity(@intCast(segment_operands.len)); } state.addAttributes(&.{ - .named(ctx, "operandSegmentSizes", .denseArray(ctx, .i32, sizes.constSlice())), + .named(ctx, "operandSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), }); }, }; if (args.results) |results| switch (results) { .flat => |v| state.addResults(v), .variadic => |segments| { - var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .empty; + var sizes: stdx.BoundedArray(i32, MAX_SEGMENTS) = .{}; for (segments) |segment_results| { state.addResults(segment_results); sizes.appendAssumeCapacity(@intCast(segment_results.len)); } state.addAttributes(&.{ - .named(ctx, "resultSegmentSizes", .denseArray(ctx, .i32, sizes.constSlice())), + .named(ctx, "resultSegmentSizes", denseElementsAttribute(RankedTensorType.get(&.{@intCast(sizes.len)}, integerType(ctx, .i32), null).shaped(), sizes.constSlice())), }); }, }; @@ -1436,11 +1192,11 @@ pub const Operation = opaque { state.addAttributes(attrs); } if (args.blocks) |blocks| { + const region = Region.init(); for (blocks) |block_| { - const body_region = Region.init(); - body_region.appendOwnedBlock(block_); - state.addOwnedRegions(&.{body_region}); + region.appendOwnedBlock(block_); } + state.addOwnedRegions(&.{region}); } const new_op = try Operation.init(&state); errdefer new_op.deinit(); @@ -1490,11 +1246,11 @@ pub const Operation = opaque { (struct { pub fn callback(op: c.MlirOperation, ctx_: ?*anyopaque) callconv(.c) c.MlirWalkResult { const inner_ctx_: *ContextType = @ptrCast(@alignCast(ctx_)); - return @backingInt(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); + return @intFromEnum(walkfn(inner_ctx_.ctx, @ptrCast(op.ptr))); } }).callback, &inner_ctx, - @backingInt(order), + @intFromEnum(order), ); } @@ -1548,16 +1304,6 @@ pub const Operation = opaque { } }; -/// builtin.unrealized_conversion_cast — `inputs` as values of `outputs_types`, -/// a cast between type systems that a later conversion resolves. -pub fn unrealized_conversion_cast(ctx: *Context, inputs: []const *const Value, outputs_types: []const *const Type, location: *const Location) *Operation { - return Operation.make(ctx, "builtin.unrealized_conversion_cast", .{ - .operands = .{ .flat = inputs }, - .results = .{ .flat = outputs_types }, - .location = location, - }); -} - pub const Region = opaque { const M = Methods(Region, c.MlirRegion); @@ -1623,14 +1369,6 @@ pub const ShapedType = opaque { pub fn dimension(self: *const ShapedType, dim: usize) i64 { return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); } - - pub fn rankedTensor(dimensions: []const i64, elem_type: *const Type) *const ShapedType { - return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); - } - - pub fn rankedTensorWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const ShapedType { - return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); - } }; pub const RankedTensorType = opaque { @@ -1641,24 +1379,23 @@ pub const RankedTensorType = opaque { pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); - pub fn get(dimensions: []const i64, elem_type: *const Type, encoding: ?*Attribute) *const RankedTensorType { + pub fn get(dimensions: []const i64, elemType: *const Type, encoding: ?*Attribute) *const RankedTensorType { return @ptrCast(c.mlirRankedTensorTypeGet( @intCast(dimensions.len), @ptrCast(dimensions), - elem_type.ptr(), + elemType.ptr(), if (encoding) |e| e.ptr() else c.mlirAttributeGetNull(), ).ptr); } pub fn fromShaped(other: *const ShapedType) *const RankedTensorType { - var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .empty; + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; for (0..other.rank()) |i| { dims.appendAssumeCapacity(other.dimension(i)); } return get( dims.constSlice(), other.elementType(), - null, ); } @@ -1679,6 +1416,14 @@ pub const RankedTensorType = opaque { } }; +pub fn rankedTensorType(dimensions: []const i64, elem_type: *const Type) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, null)); +} + +pub fn rankedTensorTypeWithEncoding(dimensions: []const i64, elem_type: *const Type, encoding: *Attribute) *const Type { + return @ptrCast(RankedTensorType.get(dimensions, elem_type, encoding)); +} + pub const DenseElementsAttribute = opaque { const M = Methods(DenseElementsAttribute, c.MlirAttribute); @@ -1697,6 +1442,10 @@ pub const DenseElementsAttribute = opaque { } }; +pub fn denseElementsAttribute(shaped_type: *const ShapedType, values: anytype) *const Attribute { + return @ptrCast(DenseElementsAttribute.init(shaped_type, values)); +} + pub const DenseArrayTypes = enum { bool, i8, @@ -1756,6 +1505,10 @@ pub fn DenseArrayAttribute(comptime ElementType: DenseArrayTypes) type { }; } +pub fn denseArrayAttribute(ctx: *Context, comptime ElementType: DenseArrayTypes, values: []const ElementType.ZigType()) *const Attribute { + return @ptrCast(DenseArrayAttribute(ElementType).init(ctx, @ptrCast(values))); +} + pub const ArrayAttribute = opaque { const M = Methods(ArrayAttribute, c.MlirAttribute); @@ -1777,6 +1530,10 @@ pub const ArrayAttribute = opaque { } }; +pub fn arrayAttribute(ctx: *Context, attrs: []const *const Attribute) *const Attribute { + return @ptrCast(ArrayAttribute.init(ctx, attrs)); +} + pub const FunctionType = opaque { const M = Methods(FunctionType, c.MlirType); pub const isAFn = c.mlirTypeIsAFunction; @@ -1796,6 +1553,10 @@ pub const FunctionType = opaque { } }; +pub fn functionType(ctx: *Context, args: []const *const Type, results: []const *const Type) *const Type { + return @ptrCast(FunctionType.init(ctx, args, results)); +} + pub const FlatSymbolRefAttribute = opaque { const M = Methods(FlatSymbolRefAttribute, c.MlirAttribute); @@ -1813,19 +1574,9 @@ pub const FlatSymbolRefAttribute = opaque { } }; -/// `@root::@nested...`. -pub const SymbolRefAttribute = opaque { - const M = Methods(SymbolRefAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsASymbolRef; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(ctx: *Context, root: []const u8, nested: []const *const FlatSymbolRefAttribute) *const SymbolRefAttribute { - return @ptrCast(c.mlirSymbolRefAttrGet(ctx.ptr(), stringRef(root), @intCast(nested.len), @ptrCast(nested)).ptr); - } -}; +pub fn flatSymbolRefAttribute(ctx: *Context, symbol: []const u8) *const Attribute { + return @ptrCast(FlatSymbolRefAttribute.init(ctx, symbol)); +} pub const UnitAttribute = opaque { const M = Methods(UnitAttribute, c.MlirAttribute); @@ -1840,27 +1591,30 @@ pub const UnitAttribute = opaque { } }; +pub fn unitAttribute(ctx: *Context) *const Attribute { + return @ptrCast(UnitAttribute.get(ctx)); +} + pub const MemRefType = opaque { const M = Methods(MemRefType, c.MlirType); - pub const isAFn = c.mlirTypeIsAMemRef; + pub const isAFn = c.mlirTypeIsAInteger; pub const ptr = M.ptr; pub const eql = M.eql(c.mlirTypeEqual); pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - pub fn init(element_type: *const Type, shape: []const i64, layout_attr: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { + pub fn init(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const MemRefType { return @ptrCast(c.mlirMemRefTypeGet( element_type.ptr(), @intCast(shape.len), @ptrCast(shape), - if (layout_attr) |l| l.ptr() else .{ .ptr = null }, - if (memory_space) |m| m.ptr() else .{ .ptr = null }, + if (layout) |l| l.ptr() else .{}, + if (memory_space) |m| m.ptr() else .{}, ).ptr); } pub fn fromShaped(other: *const ShapedType) *const MemRefType { - var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .empty; + var dims: stdx.BoundedArray(i64, ShapedType.MAX_RANK) = .{}; for (0..other.rank()) |i| { dims.appendAssumeCapacity(other.dimension(i)); } @@ -1872,244 +1626,11 @@ pub const MemRefType = opaque { ); } - pub fn elementType(self: *const MemRefType) *const Type { - return @ptrCast(c.mlirShapedTypeGetElementType(self.ptr()).ptr); - } - - pub fn rank(self: *const MemRefType) usize { - return @intCast(c.mlirShapedTypeGetRank(self.ptr())); - } - - pub fn dimension(self: *const MemRefType, dim: usize) i64 { - return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); - } - - pub fn layout(self: *const MemRefType) *const Attribute { - return @ptrCast(c.mlirMemRefTypeGetLayout(self.ptr()).ptr); - } - - pub fn memorySpace(self: *const MemRefType) ?*const Attribute { - const a = c.mlirMemRefTypeGetMemorySpace(self.ptr()); - return if (a.ptr == null) null else @ptrCast(a.ptr); - } - pub fn shaped(self: *const MemRefType) *const ShapedType { return @ptrCast(self); } }; -pub const VectorType = opaque { - const M = Methods(VectorType, c.MlirType); - - pub const isAFn = c.mlirTypeIsAVector; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - /// Plain (non-scalable) vector type. - pub fn init(shape: []const i64, element_type: *const Type) *const VectorType { - return @ptrCast(c.mlirVectorTypeGet( - @intCast(shape.len), - @ptrCast(shape), - element_type.ptr(), - ).ptr); - } - - /// Vector type with explicit per-dim scalability flags. `scalable.len` - /// must equal `shape.len`. - pub fn initScalable( - shape: []const i64, - scalable: []const bool, - element_type: *const Type, - ) *const VectorType { - std.debug.assert(shape.len == scalable.len); - return @ptrCast(c.mlirVectorTypeGetScalable( - @intCast(shape.len), - @ptrCast(shape), - @ptrCast(scalable), - element_type.ptr(), - ).ptr); - } - - pub fn rank(self: *const VectorType) usize { - return @intCast(c.mlirShapedTypeGetRank(self.ptr())); - } - - pub fn dimension(self: *const VectorType, dim: usize) i64 { - return c.mlirShapedTypeGetDimSize(self.ptr(), @intCast(dim)); - } - - pub fn shaped(self: *const VectorType) *const ShapedType { - return @ptrCast(self); - } -}; - -pub const UnrankedMemRefType = opaque { - const M = Methods(UnrankedMemRefType, c.MlirType); - - pub const isAFn = c.mlirTypeIsAUnrankedMemRef; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirTypeEqual); - pub const format = M.format(c.mlirTypePrint); - pub const isA = M.isA; - - pub fn init(element_type: *const Type, memory_space: ?*const Attribute) *const UnrankedMemRefType { - return @ptrCast(c.mlirUnrankedMemRefTypeGet( - element_type.ptr(), - if (memory_space) |m| m.ptr() else .{ .ptr = null }, - ).ptr); - } -}; - -pub const AffineExpr = opaque { - const M = Methods(AffineExpr, c.MlirAffineExpr); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAffineExprEqual); - pub const format = M.format(c.mlirAffineExprPrint); - - pub fn dim(ctx: *Context, position: usize) *const AffineExpr { - return @ptrCast(c.mlirAffineDimExprGet(ctx.ptr(), @intCast(position)).ptr); - } - - pub fn symbol(ctx: *Context, position: usize) *const AffineExpr { - return @ptrCast(c.mlirAffineSymbolExprGet(ctx.ptr(), @intCast(position)).ptr); - } - - pub fn constant(ctx: *Context, value: i64) *const AffineExpr { - return @ptrCast(c.mlirAffineConstantExprGet(ctx.ptr(), value).ptr); - } - - pub fn add(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { - return @ptrCast(c.mlirAffineAddExprGet(self.ptr(), other.ptr()).ptr); - } - - pub fn mul(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { - return @ptrCast(c.mlirAffineMulExprGet(self.ptr(), other.ptr()).ptr); - } - - pub fn floorDiv(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { - return @ptrCast(c.mlirAffineFloorDivExprGet(self.ptr(), other.ptr()).ptr); - } - - pub fn ceilDiv(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { - return @ptrCast(c.mlirAffineCeilDivExprGet(self.ptr(), other.ptr()).ptr); - } - - pub fn mod(self: *const AffineExpr, other: *const AffineExpr) *const AffineExpr { - return @ptrCast(c.mlirAffineModExprGet(self.ptr(), other.ptr()).ptr); - } -}; - -pub const AffineMap = opaque { - const M = Methods(AffineMap, c.MlirAffineMap); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAffineMapEqual); - pub const format = M.format(c.mlirAffineMapPrint); - - /// Empty `() -> ()` map. - pub fn empty(ctx: *Context) *const AffineMap { - return @ptrCast(c.mlirAffineMapEmptyGet(ctx.ptr()).ptr); - } - - /// Zero-result map: `(d0, d1, ...) -> ()`. - pub fn zeroResult(ctx: *Context, dim_count: usize, symbol_count: usize) *const AffineMap { - return @ptrCast(c.mlirAffineMapZeroResultGet( - ctx.ptr(), - @intCast(dim_count), - @intCast(symbol_count), - ).ptr); - } - - /// Identity map: `(d0, d1, ..., dN-1) -> (d0, d1, ..., dN-1)`. - pub fn multiDimIdentity(ctx: *Context, num_dims: usize) *const AffineMap { - return @ptrCast(c.mlirAffineMapMultiDimIdentityGet(ctx.ptr(), @intCast(num_dims)).ptr); - } - - /// Build a map from explicit results. - pub fn get( - ctx: *Context, - dim_count: usize, - symbol_count: usize, - results: []const *const AffineExpr, - ) *const AffineMap { - return @ptrCast(c.mlirAffineMapGet( - ctx.ptr(), - @intCast(dim_count), - @intCast(symbol_count), - @intCast(results.len), - @ptrCast(@constCast(results.ptr)), - ).ptr); - } - - pub fn numDims(self: *const AffineMap) usize { - return @intCast(c.mlirAffineMapGetNumDims(self.ptr())); - } - - pub fn numSymbols(self: *const AffineMap) usize { - return @intCast(c.mlirAffineMapGetNumSymbols(self.ptr())); - } - - pub fn numResults(self: *const AffineMap) usize { - return @intCast(c.mlirAffineMapGetNumResults(self.ptr())); - } -}; - -/// AffineMapAttr — `affine_map<(d0,...) -> (...)>` wrapped as an attribute. -pub const AffineMapAttribute = opaque { - const M = Methods(AffineMapAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsAAffineMap; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(map: *const AffineMap) *const AffineMapAttribute { - return @ptrCast(c.mlirAffineMapAttrGet(map.ptr()).ptr); - } - - pub fn value(self: *const AffineMapAttribute) *const AffineMap { - return @ptrCast(c.mlirAffineMapAttrGetValue(self.ptr()).ptr); - } -}; - -pub const IntegerSet = opaque { - const M = Methods(IntegerSet, c.MlirIntegerSet); - - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirIntegerSetEqual); - pub const format = M.format(c.mlirIntegerSetPrint); - - pub fn get( - ctx: *Context, - num_dims: usize, - num_symbols: usize, - constraints: []const *const AffineExpr, - eqs: []const bool, - ) *const IntegerSet { - std.debug.assert(constraints.len == eqs.len); - return @ptrCast(c.mlirIntegerSetGet( - ctx.ptr(), - @intCast(num_dims), - @intCast(num_symbols), - @intCast(constraints.len), - @ptrCast(constraints), - @ptrCast(eqs), - ).ptr); - } -}; - -pub const IntegerSetAttribute = opaque { - const M = Methods(IntegerSetAttribute, c.MlirAttribute); - - pub const isAFn = c.mlirAttributeIsAIntegerSet; - pub const ptr = M.ptr; - pub const eql = M.eql(c.mlirAttributeEqual); - pub const format = M.format(c.mlirAttributePrint); - - pub fn init(set: *const IntegerSet) *const IntegerSetAttribute { - return @ptrCast(c.mlirIntegerSetAttrGet(set.ptr()).ptr); - } -}; +pub fn memRefType(element_type: *const Type, shape: []const i64, layout: ?*const Attribute, memory_space: ?*const Attribute) *const Type { + return @ptrCast(MemRefType.init(element_type, shape, layout, memory_space)); +} diff --git a/pjrt/BUILD.bazel b/pjrt/BUILD.bazel index 5370afdc..e9f365d1 100644 --- a/pjrt/BUILD.bazel +++ b/pjrt/BUILD.bazel @@ -4,7 +4,6 @@ zig_library( name = "pjrt", srcs = ["ffi.zig"], main = "pjrt.zig", - tags = ["manual"], visibility = ["//visibility:public"], deps = [ "@xla//xla/ffi/api:c_api", @@ -20,5 +19,6 @@ zig_library( "@xla//xla/pjrt/c:pjrt_c_api_profiler_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_raw_buffer_extension_hdrs", "@xla//xla/pjrt/c:pjrt_c_api_stream_extension_hdrs", + "@xla//xla/pjrt/c:pjrt_c_api_triton_extension_hdrs", ], ) diff --git a/pjrt/ffi.zig b/pjrt/ffi.zig index cebec841..83fad4d2 100644 --- a/pjrt/ffi.zig +++ b/pjrt/ffi.zig @@ -11,7 +11,7 @@ const Stream = @import("pjrt.zig").Stream; const log = std.log.scoped(.pjrt); comptime { - if (@typeInfo(TypeId).@"struct".field_names.len != 1) @compileError("TypeId has changed"); + if (@typeInfo(TypeId).@"struct".fields.len != 1) @compileError("TypeId has changed"); } /// The signature of a generic custom call. @@ -27,7 +27,7 @@ pub const ApiVersion = extern struct { minor_version: i32, }; -pub const ExtensionType = enum(c.XLA_FFI_InternalExtension_Type) { +pub const ExtensionType = enum(c.XLA_FFI_Extension_Type) { metadata = c.XLA_FFI_Extension_Metadata, }; @@ -152,17 +152,17 @@ pub const ExecutionStage = enum(c.XLA_FFI_ExecutionStage) { }; pub const ExecutionContext = opaque { - pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_InvokeContext).to; + pub const inner = TransmuteMixin(ExecutionContext, c.XLA_FFI_ExecutionContext).to; pub fn getContext(self: *const ExecutionContext, type_id: TypeId, api: *const Api) pjrt.ApiError!*anyopaque { - var ret: c.XLA_FFI_InvokeContext_Get_Args = .{ - .struct_size = pjrt.meta.structSize(c.XLA_FFI_InvokeContext_Get_Args), + var ret: c.XLA_FFI_ExecutionContext_Get_Args = .{ + .struct_size = pjrt.pjrtStructSize(c.XLA_FFI_ExecutionContext_Get_Args), .extension_start = api.inner().extension_start, .ctx = @ptrCast(@constCast(self)), .type_id = @constCast(&type_id), - .data = undefined, // set by XLA_FFI_InvokeContext_Get. + .data = undefined, // set by XLA_FFI_ExecutionContext_Get. }; - const maybe_err = api.inner().XLA_FFI_InvokeContext_Get.?(&ret); + const maybe_err = api.inner().XLA_FFI_ExecutionContext_Get.?(&ret); if (maybe_err) |ffi_error| { const err = Error.fromInner(ffi_error); @@ -252,12 +252,11 @@ pub const DataType = enum(c.XLA_FFI_DataType) { f8e4m3b11fnuz = c.XLA_FFI_DataType_F8E4M3B11FNUZ, f8e5m2fnuz = c.XLA_FFI_DataType_F8E5M2FNUZ, f8e4m3fnuz = c.XLA_FFI_DataType_F8E4M3FNUZ, - f4e2m1 = c.XLA_FFI_DataType_F4E2M1FN, }; pub const Buffer = extern struct { struct_size: usize, - extension_start: ?*c.XLA_FFI_InternalExtension, + extension_start: ?*c.XLA_FFI_Extension_Base, dtype: DataType, data: [*]u8, rank: u64, @@ -274,7 +273,7 @@ pub const Buffer = extern struct { pub const Args = extern struct { struct_size: usize, - extension_start: ?*const c.XLA_FFI_InternalExtension, + extension_start: ?*const c.XLA_FFI_Extension_Base, len: u64, types: [*]const Type, ptr: [*]*const Buffer, @@ -290,7 +289,7 @@ pub const Args = extern struct { pub const Rets = extern struct { struct_size: usize, - extension_start: ?*const c.XLA_FFI_InternalExtension, + extension_start: ?*const c.XLA_FFI_Extension_Base, len: u64, types: [*]const Type, ptr: [*]*const Buffer, @@ -450,7 +449,7 @@ pub const Error = opaque { pub fn create(api: *const Api, error_code: ErrorCode, message: []const u8) *Error { var ret: pjrt.meta.Struct(c.XLA_FFI_Error_Create_Args) = .{ .message = message.ptr, - .errc = @backingInt(error_code), + .errc = @intFromEnum(error_code), }; return fromInner(api.inner().XLA_FFI_Error_Create.?(@ptrCast(&ret)).?); } @@ -461,10 +460,10 @@ pub const Error = opaque { } pub fn getMessage(err: *Error, api: *const Api) [:0]const u8 { - var ret: pjrt.meta.Struct(c.XLA_FFI_Error_GetDetails_Args) = .{ + var ret: pjrt.meta.Struct(c.XLA_FFI_Error_GetMessage_Args) = .{ .@"error" = err.inner(), }; - api.inner().XLA_FFI_Error_GetDetails.?(@ptrCast(&ret)); + api.inner().XLA_FFI_Error_GetMessage.?(@ptrCast(&ret)); return std.mem.span(ret.message); } }; diff --git a/pjrt/pjrt.zig b/pjrt/pjrt.zig index 3715361f..073691e5 100644 --- a/pjrt/pjrt.zig +++ b/pjrt/pjrt.zig @@ -5,8 +5,7 @@ const c = @import("c"); pub const ffi = @import("ffi.zig"); -const log_scope = .pjrt; -const log = std.log.scoped(log_scope); +const log = std.log.scoped(.pjrt); test { std.testing.refAllDecls(@This()); @@ -32,18 +31,18 @@ pub const meta = struct { } pub fn Struct(comptime T: type) type { - const info = @typeInfo(T).@"struct"; - var names: [info.field_names.len][]const u8 = undefined; - var types: [info.field_names.len]type = undefined; - var attributes: [info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; - for (info.field_names, info.field_types, &names, &types, &attributes) |field_name, Field, *name, *type_, *attr| { - name.* = field_name; - type_.* = Field; + const fields = std.meta.fields(T); + var names: [fields.len][]const u8 = undefined; + var types: [fields.len]type = undefined; + var attributes: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (fields, &names, &types, &attributes) |field, *name, *type_, *attr| { + name.* = field.name; + type_.* = field.type; attr.* = .{ - .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field_name, "struct_size")) + .default_value_ptr = @ptrCast(if (std.mem.eql(u8, field.name, "struct_size")) &structSize(T) else - &std.mem.zeroes(Field)), + &std.mem.zeroes(field.type)), }; } return @Struct( @@ -75,15 +74,6 @@ pub const ApiError = error{ Unauthenticated, }; -inline fn interpretPjrtError(api: *const Api, pjrt_error: *Error, context: []const u8, comptime log_level: std.log.Level) ApiError { - defer pjrt_error.deinit(api); - const err_code = pjrt_error.getCode(api).toApiError(); - if (comptime std.log.logEnabled(log_level, log_scope)) { - std.options.logFn(log_level, log_scope, "[{s}] {t}: {s}", .{ context, err_code, pjrt_error.getMessage(api) }); - } - return err_code; -} - fn InnerMixin(comptime innerT: type) type { return struct { fn inner(self: anytype) *innerT { @@ -109,6 +99,7 @@ pub const Api = struct { pub fn loadFrom(library: [:0]const u8) !*const Api { const basename = std.Io.Dir.path.basename(library); log.info("Loading: {s}...", .{basename}); + defer log.info("Loaded: {s}", .{basename}); var lib: std.DynLib = switch (builtin.os.tag) { .linux, .macos => blk: { @@ -121,33 +112,28 @@ pub const Api = struct { break :blk .{ .inner = .{ .handle = std.c.dlopen(library, rtld) orelse { - log.err("Unable to dlopen plugin {s}\n{s}", .{ library, std.c.dlerror() orelse "" }); + log.err("Unable to dlopen plugin: {s}", .{library}); return error.FileNotFound; }, }, }; }, else => std.DynLib.open(library) catch |err| { - log.err("Unable to dlopen plugin {s}: {}", .{ library, err }); + log.err("Unable to dlopen plugin: {s}", .{library}); return err; }, }; - const api = fromDynLib(&lib) catch |err| { - log.err("Unable to load PJRT API from plugin {s}: {}", .{ library, err }); - return err; - }; - log.info("Loaded: {s}", .{basename}); - return api; + return fromDynLib(&lib); } pub fn fromDynLib(lib: *std.DynLib) !*const Api { const DynGetPjrtApi = lib.lookup(*const fn () callconv(.c) *const Api, "GetPjrtApi") orelse { - return error.MissingGetPjrtApi; + std.debug.panic("Unable to find GetPjrtApi symbol in library", .{}); }; const api = DynGetPjrtApi(); - _ = try api.call(.PJRT_Plugin_Initialize, .{}); + _ = api.call(.PJRT_Plugin_Initialize, .{}) catch unreachable; return api; } @@ -156,7 +142,7 @@ pub const Api = struct { const fti = @typeInfo(@FieldType(c.PJRT_Api, @tagName(func))); const fn_ptr = @typeInfo(fti.optional.child); const fn_type_info = @typeInfo(fn_ptr.pointer.child); - const arg_array_type_info = @typeInfo(fn_type_info.@"fn".param_types[0].?); + const arg_array_type_info = @typeInfo(fn_type_info.@"fn".params[0].type.?); return arg_array_type_info.pointer.child; } @@ -168,33 +154,25 @@ pub const Api = struct { }; } - inline fn innerCall(self: *const Api, comptime method: Funcs, arg: *PJRTFnArg(method)) ?*Error { + inline fn innerCall(self: *const Api, comptime method: Funcs, arg: *PJRTFnArg(method)) ApiError!void { if (@offsetOf(c.PJRT_Api, @tagName(method)) > self.inner.struct_size) { std.debug.panic("PJRT Api method {s} not available in this plugin", .{@tagName(method)}); } const fn_ptr = @field(&self.inner, @tagName(method)).?; const result = fn_ptr(arg); if (@TypeOf(result) == void) { - return null; + return; } if (result) |pjrt_c_error| { const pjrt_error: *Error = @ptrCast(pjrt_c_error); - return pjrt_error; + log.err("[{s}] {s}", .{ @tagName(method), pjrt_error.getMessage(self) }); + return pjrt_error.getCode(self).toApiError(); } - return null; } inline fn call(self: *const Api, comptime method: Funcs, arg: PJRTFnArgWithDefault(method)) ApiError!PJRTFnArgWithDefault(method) { var ret = arg; - const pjrt_error = innerCall(self, method, @ptrCast(&ret)); - if (pjrt_error) |err| return interpretPjrtError(self, err, @tagName(method), .warn); - return ret; - } - - inline fn callSilent(self: *const Api, comptime method: Funcs, arg: PJRTFnArgWithDefault(method)) ApiError!PJRTFnArgWithDefault(method) { - var ret = arg; - const pjrt_error = innerCall(self, method, @ptrCast(&ret)); - if (pjrt_error) |err| return interpretPjrtError(self, err, @tagName(method), .debug); + try innerCall(self, method, @ptrCast(&ret)); return ret; } @@ -202,10 +180,12 @@ pub const Api = struct { pub const Type = enum(c.PJRT_Extension_Type) { custom_call = c.PJRT_Extension_Type_Gpu_Custom_Call, profiler = c.PJRT_Extension_Type_Profiler, + custom_partitioner = c.PJRT_Extension_Type_Custom_Partitioner, stream = c.PJRT_Extension_Type_Stream, layouts = c.PJRT_Extension_Type_Layouts, ffi = c.PJRT_Extension_Type_FFI, memory_descriptions = c.PJRT_Extension_Type_MemoryDescriptions, + triton = c.PJRT_Extension_Type_Triton, raw_buffer = c.PJRT_Extension_Type_RawBuffer, phase_compile = c.PJRT_Extension_Type_PhaseCompile, unknown = c.PJRT_Extension_Type_Unknown, @@ -214,10 +194,12 @@ pub const Api = struct { pub const Extension = union(Type) { custom_call: *const c.PJRT_Gpu_Custom_Call, profiler: *const c.PJRT_Profiler_Extension, + custom_partitioner: *const c.PJRT_Custom_Partitioner_Extension, stream: *const c.PJRT_Stream_Extension, layouts: *const c.PJRT_Layouts_Extension, ffi: *const c.PJRT_FFI_Extension, memory_descriptions: *const c.PJRT_MemoryDescriptions_Extension, + triton: *const c.PJRT_Triton_Extension, raw_buffer: *const c.PJRT_RawBuffer_Extension, phase_compile: *const c.PJRT_PhaseCompile_Extension, unknown: *const c.PJRT_Extension_Base, @@ -267,15 +249,15 @@ pub const Api = struct { pub fn stablehloCurrentVersion(self: *const Api) ?[]const u8 { const state = struct { var buf: [32]u8 = undefined; - var str: ?[]const u8 = null; + var str: ?[:0]const u8 = null; }; if (state.str) |str| { return str; } - if (self.pluginAttribute("stablehlo_current_version")) |nv| { + if (self.getPluginAttribute("stablehlo_current_version")) |nv| { switch (nv.value()) { .int64list => |v| { - state.str = std.mem.print(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; + state.str = std.fmt.bufPrintZ(&state.buf, "{d}.{d}.{d}", .{ v[0], v[1], v[2] }) catch unreachable; }, else => unreachable, } @@ -295,16 +277,8 @@ pub const Api = struct { return null; } - pub fn profiler(self: *const Api, options_pb: []const u8) ApiError!?Profiler { - if (self.extension(.profiler)) |ext| { - return try Profiler.init(self, ext.profiler, options_pb); - } - - return null; - } - - pub fn pluginAttribute(api: *const Api, key: []const u8) ?NamedValue { - const attributes = api.pluginAttributes(); + fn getPluginAttribute(api: *const Api, key: []const u8) ?NamedValue { + const attributes = api.getPluginAttributes(); for (attributes) |attr| { if (std.mem.eql(u8, attr.name(), key)) { return attr; @@ -313,7 +287,7 @@ pub const Api = struct { return null; } - pub fn pluginAttributes(api: *const Api) []const NamedValue { + fn getPluginAttributes(api: *const Api) []const NamedValue { const ret = api.call(.PJRT_Plugin_Attributes, .{ .extension_start = null, }) catch unreachable; @@ -365,24 +339,22 @@ pub const ErrorCode = enum(c.PJRT_Error_Code) { }; pub const Error = opaque { - const inner = InnerMixin(c.PJRT_Error).inner; - pub fn deinit(self: *Error, api: *const Api) void { _ = api.call(.PJRT_Error_Destroy, .{ - .@"error" = self.inner(), + .@"error" = @ptrCast(self), }) catch unreachable; } pub fn getCode(self: *Error, api: *const Api) ErrorCode { const ret = api.call(.PJRT_Error_GetCode, .{ - .@"error" = self.inner(), + .@"error" = @ptrCast(self), }) catch unreachable; - return @fromBackingInt(@intCast(ret.code)); + return @enumFromInt(ret.code); } pub fn getMessage(self: *Error, api: *const Api) []const u8 { const ret = api.call(.PJRT_Error_Message, .{ - .@"error" = self.inner(), + .@"error" = @ptrCast(self), }) catch unreachable; return ret.message[0..ret.message_size]; } @@ -402,7 +374,7 @@ pub const ShapeSpec = extern struct { .inner = .{ .dims = @ptrCast(@constCast(dims_)), .num_dims = dims_.len, - .element_type = @backingInt(bt), + .element_type = @intFromEnum(bt), }, }; } @@ -412,7 +384,7 @@ pub const ShapeSpec = extern struct { } pub fn bufferType(self: ShapeSpec) BufferType { - return @fromBackingInt(@intCast(self.inner.element_type)); + return @enumFromInt(self.inner.element_type); } }; @@ -494,7 +466,6 @@ pub const Client = opaque { buffer_type: BufferType, dims: []const i64, byte_strides: ?[]const i64, - layout: MemoryLayout, host_buffer_semantics: HostBufferSemantics, dst: union(enum) { device: *const Device, @@ -506,15 +477,14 @@ pub const Client = opaque { const ret = try api.call(.PJRT_Client_BufferFromHostBuffer, .{ .client = self.inner(), .data = @constCast(args.data), - .type = @backingInt(args.buffer_type), + .type = @intFromEnum(args.buffer_type), .dims = @ptrCast(@constCast(args.dims)), .num_dims = args.dims.len, .byte_strides = if (args.byte_strides) |bs| @ptrCast(@constCast(bs)) else null, .num_byte_strides = if (args.byte_strides) |bs| bs.len else 0, - .device_layout = @ptrCast(@constCast(&args.layout.toCStruct())), - .host_buffer_semantics = @backingInt(args.host_buffer_semantics), + .host_buffer_semantics = @intFromEnum(args.host_buffer_semantics), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, - .memory = if (args.dst == .memory) args.dst.memory.inner() else null, + .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, }); return .{ @@ -533,14 +503,11 @@ pub const Client = opaque { } pub const CreateViewOfDeviceBufferArgs = struct { - device_buffer_ptr: *anyopaque, + data: *anyopaque, dims: []const i64, element_type: BufferType, layout: MemoryLayout, device: ?*const Device = null, - /// The memory holding `device_buffer_ptr`. Takes precedence over `device`, - /// which designates its default memory. - memory: ?*const Memory = null, on_delete_callback: *const fn (device_buffer_ptr: ?*anyopaque, ctx: ?*anyopaque) callconv(.c) void = &struct { fn call(_: ?*anyopaque, _: ?*anyopaque) callconv(.c) void {} }.call, @@ -552,16 +519,15 @@ pub const Client = opaque { const layout = args.layout.toCStruct(); const ret = try api.call(.PJRT_Client_CreateViewOfDeviceBuffer, .{ .client = self.inner(), - .device_buffer_ptr = @constCast(args.device_buffer_ptr), + .device_buffer_ptr = @constCast(args.data), .dims = args.dims.ptr, .num_dims = args.dims.len, - .element_type = @backingInt(args.element_type), + .element_type = @intFromEnum(args.element_type), .layout = @ptrCast(@constCast(&layout)), .device = @ptrCast(@constCast(args.device)), .on_delete_callback = args.on_delete_callback, .on_delete_callback_arg = args.on_delete_callback_arg, .stream = @bitCast(@intFromPtr(args.stream)), - .memory = if (args.memory) |memory| memory.inner() else null, }); return @ptrCast(ret.buffer.?); } @@ -604,68 +570,11 @@ pub const Client = opaque { .num_shape_specs = args.shape_specs.len, .device_layouts = if (args.device_layouts) |layouts| @ptrCast(@constCast(layouts)) else null, .num_device_layouts = if (args.device_layouts) |layouts| layouts.len else 0, - .memory = args.memory.inner(), + .memory = @ptrCast(@constCast(args.memory)), }); return @ptrCast(ret.transfer_manager.?); } - pub fn defaultMemoryLayout( - self: *const Client, - api: *const Api, - element_type: BufferType, - dims: []const i64, - ) !DefaultMemoryLayout { - const ext = if (api.extension(.layouts)) |e| e.layouts else return error.LayoutsExtensionUnavailable; - - var get_args: c.PJRT_Layouts_PJRT_Client_GetDefaultLayout_Args = .{ - .struct_size = c.PJRT_Layouts_PJRT_Client_GetDefaultLayout_Args_STRUCT_SIZE, - .extension_start = null, - .client = self.inner(), - .type = @backingInt(element_type), - .dims = dims.ptr, - .num_dims = dims.len, - .layout = null, - }; - if (ext.PJRT_Layouts_PJRT_Client_GetDefaultLayout.?(&get_args)) |pjrt_c_error| { - const pjrt_error: *Error = @ptrCast(pjrt_c_error); - defer pjrt_error.deinit(api); - return pjrt_error.getCode(api).toApiError(); - } - defer { - var destroy_args: c.PJRT_Layouts_MemoryLayout_Destroy_Args = .{ - .struct_size = c.PJRT_Layouts_MemoryLayout_Destroy_Args_STRUCT_SIZE, - .extension_start = null, - .layout = get_args.layout, - }; - if (ext.PJRT_Layouts_MemoryLayout_Destroy.?(&destroy_args)) |pjrt_c_error| { - const pjrt_error: *Error = @ptrCast(pjrt_c_error); - defer pjrt_error.deinit(api); - log.err("[PJRT_Layouts_MemoryLayout_Destroy] {s}", .{pjrt_error.getMessage(api)}); - } - } - - var serialize_args: c.PJRT_Layouts_MemoryLayout_Serialize_Args = .{ - .struct_size = c.PJRT_Layouts_MemoryLayout_Serialize_Args_STRUCT_SIZE, - .extension_start = null, - .layout = get_args.layout, - .serialized_bytes = null, - .serialized_bytes_size = 0, - .serialized_layout = null, - .serialized_layout_deleter = null, - }; - if (ext.PJRT_Layouts_MemoryLayout_Serialize.?(&serialize_args)) |pjrt_c_error| { - const pjrt_error: *Error = @ptrCast(pjrt_c_error); - defer pjrt_error.deinit(api); - return pjrt_error.getCode(api).toApiError(); - } - defer if (serialize_args.serialized_layout_deleter) |deleter| { - deleter(serialize_args.serialized_layout); - }; - - const serialized = serialize_args.serialized_bytes[0..serialize_args.serialized_bytes_size]; - return try DefaultMemoryLayout.parseSerialized(serialized); - } - pub const CreateUninitializedBufferArgs = struct { dims: []const i64, element_type: BufferType, @@ -682,10 +591,10 @@ pub const Client = opaque { .client = self.inner(), .shape_dims = args.dims.ptr, .shape_num_dims = args.dims.len, - .shape_element_type = @backingInt(args.element_type), + .shape_element_type = @intFromEnum(args.element_type), .shape_layout = @ptrCast(&layout), .device = if (args.dst == .device) @ptrCast(@constCast(args.dst.device)) else null, - .memory = if (args.dst == .memory) args.dst.memory.inner() else null, + .memory = if (args.dst == .memory) @ptrCast(@constCast(args.dst.memory)) else null, }); return @ptrCast(ret.buffer.?); } @@ -694,40 +603,6 @@ pub const Client = opaque { pub const Device = opaque { const inner = InnerMixin(c.PJRT_Device).inner; - pub const Attributes = struct { - values: []const NamedValue, - state: *c.PJRT_Device_Attributes, - deleter: *const fn (?*c.PJRT_Device_Attributes) callconv(.c) void, - - pub fn deinit(self: Attributes) void { - self.deleter(self.state); - } - - /// String and list values remain valid until this owner is deinitialized. - pub fn get(self: Attributes, name: []const u8) ?NamedValue.Value { - for (self.values) |attribute| { - if (std.mem.eql(u8, attribute.name(), name)) return attribute.value(); - } - return null; - } - }; - - pub fn attributes(self: *const Device, api: *const Api) ApiError!Attributes { - const ret = try api.call(.PJRT_Device_GetAttributes, .{ .device = self.inner() }); - - const deleter = ret.attributes_deleter.?; - errdefer deleter(ret.device_attributes); - - const state = ret.device_attributes.?; - std.debug.assert((ret.attributes == null) == (ret.num_attributes == 0)); - - return .{ - .values = if (ret.attributes == null) &.{} else @ptrCast(ret.attributes[0..ret.num_attributes]), - .state = state, - .deleter = deleter, - }; - } - pub fn getDescription(self: *const Device, api: *const Api) *const DeviceDescription { const ret = api.call(.PJRT_Device_GetDescription, .{ .device = self.inner(), @@ -817,11 +692,10 @@ pub const Device = opaque { }; pub fn memoryStats(self: *const Device, api: *const Api) ApiError!MemoryStats { - // callSilent: If this API is not available no point in printing it at every call - const ret = try api.callSilent(.PJRT_Device_MemoryStats, .{ + const ret = try api.call(.PJRT_Device_MemoryStats, .{ .device = self.inner(), }); - return .fromCStruct(ret); + return .fromCStruct(@bitCast(ret)); } }; @@ -867,9 +741,6 @@ pub const DeviceDescription = opaque { const ret = api.call(.PJRT_DeviceDescription_Attributes, .{ .device_description = self.inner(), }) catch unreachable; - - if (ret.attributes == null) return &.{}; - return @ptrCast(ret.attributes[0..ret.num_attributes]); } @@ -1120,128 +991,6 @@ pub const MemoryLayout = union(MemoryLayoutType) { }, }; } - - fn fromCStruct(self: c.PJRT_Buffer_MemoryLayout) MemoryLayout { - return switch (self.type) { - c.PJRT_Buffer_MemoryLayout_Type_Tiled => b: { - const tiled = self.unnamed_0.tiled; - - const minor_to_major: []const i64 = if (tiled.minor_to_major == null) &.{} else tiled.minor_to_major[0..tiled.minor_to_major_size]; - const tile_dims_sizes: []const usize = if (tiled.tile_dim_sizes == null) &.{} else tiled.tile_dim_sizes[0..tiled.num_tiles]; - - const tile_dims: []const i64 = if (tiled.tile_dims == null) &.{} else td: { - var tile_dim_len: usize = 0; - for (tile_dims_sizes) |size| tile_dim_len += size; - break :td tiled.tile_dims[0..tile_dim_len]; - }; - - break :b .{ - .tiled = .{ - .minor_to_major = minor_to_major, - .tile_dims = tile_dims, - .tile_dims_sizes = tile_dims_sizes, - }, - }; - }, - c.PJRT_Buffer_MemoryLayout_Type_Strides => .{ - .strides = .{ .byte_strides = self.unnamed_0.strides.byte_strides[0..self.unnamed_0.strides.num_byte_strides] }, - }, - else => unreachable, - }; - } -}; - -pub const DefaultMemoryLayout = struct { - pub const MAX_MINOR_TO_MAJOR: usize = 16; - pub const MAX_TILE_DIMS: usize = 128; - pub const MAX_NUM_TILES: usize = 16; - - minor_to_major_storage: [MAX_MINOR_TO_MAJOR]i64 = undefined, - minor_to_major_len: usize = 0, - - tile_dims_storage: [MAX_TILE_DIMS]i64 = undefined, - tile_dims_len: usize = 0, - - tile_dims_sizes_storage: [MAX_NUM_TILES]usize = undefined, - tile_dims_sizes_len: usize = 0, - - pub fn toMemoryLayout(self: *const DefaultMemoryLayout) MemoryLayout { - return .{ - .tiled = .{ - .minor_to_major = self.minor_to_major_storage[0..self.minor_to_major_len], - .tile_dims = self.tile_dims_storage[0..self.tile_dims_len], - .tile_dims_sizes = self.tile_dims_sizes_storage[0..self.tile_dims_sizes_len], - }, - }; - } - - pub fn parseSerialized(serialized: []const u8) !DefaultMemoryLayout { - var parsed: DefaultMemoryLayout = .{}; - try parseSerializedInto(serialized, &parsed); - return parsed; - } - - fn parseSerializedInto(serialized: []const u8, out: *DefaultMemoryLayout) !void { - out.* = .{}; - const raw = std.mem.trim(u8, serialized, " \t\n\r\x00"); - - if (raw.len < 2 or raw[0] != '{' or raw[raw.len - 1] != '}') { - return error.InvalidLayoutString; - } - - const body = raw[1 .. raw.len - 1]; - const first_colon = std.mem.indexOfScalar(u8, body, ':'); - - const minor_to_major_str = std.mem.trim(u8, if (first_colon) |index| body[0..index] else body, " \t\n\r"); - - if (minor_to_major_str.len > 0) { - var mtm_iter = std.mem.splitScalar(u8, minor_to_major_str, ','); - while (mtm_iter.next()) |part_raw| { - if (out.minor_to_major_len >= MAX_MINOR_TO_MAJOR) return error.LayoutTooLarge; - out.minor_to_major_storage[out.minor_to_major_len] = try parseI64Token(part_raw); - out.minor_to_major_len += 1; - } - } - - if (first_colon) |index| { - const attrs = body[index + 1 ..]; - var cursor: usize = 0; - while (cursor < attrs.len) { - const t_pos = std.mem.indexOfScalarPos(u8, attrs, cursor, 'T') orelse break; - cursor = t_pos + 1; - while (cursor < attrs.len and std.ascii.isWhitespace(attrs[cursor])) : (cursor += 1) {} - if (cursor >= attrs.len or attrs[cursor] != '(') continue; - - while (cursor < attrs.len and attrs[cursor] == '(') { - const tuple_start = cursor + 1; - const tuple_end = std.mem.indexOfScalarPos(u8, attrs, tuple_start, ')') orelse return error.InvalidLayoutString; - const tuple = attrs[tuple_start..tuple_end]; - - var dims_in_tile: usize = 0; - var tuple_iter = std.mem.splitScalar(u8, tuple, ','); - while (tuple_iter.next()) |dim_raw| { - if (out.tile_dims_len >= MAX_TILE_DIMS) return error.LayoutTooLarge; - out.tile_dims_storage[out.tile_dims_len] = try parseI64Token(dim_raw); - out.tile_dims_len += 1; - dims_in_tile += 1; - } - if (dims_in_tile == 0) return error.InvalidLayoutString; - - if (out.tile_dims_sizes_len >= MAX_NUM_TILES) return error.LayoutTooLarge; - out.tile_dims_sizes_storage[out.tile_dims_sizes_len] = dims_in_tile; - out.tile_dims_sizes_len += 1; - - cursor = tuple_end + 1; - } - } - } - } - - fn parseI64Token(token_raw: []const u8) !i64 { - const token = std.mem.trim(u8, token_raw, " \t\n\r"); - if (token.len == 0) return error.InvalidLayoutString; - return std.fmt.parseInt(i64, token, 10); - } }; pub const HostBufferSemantics = enum(c.PJRT_HostBufferSemantics) { @@ -1300,7 +1049,7 @@ pub const Buffer = opaque { const ret = api.call(.PJRT_Buffer_ElementType, .{ .buffer = self.inner(), }) catch unreachable; - return @fromBackingInt(@intCast(ret.type)); + return @enumFromInt(ret.type); } pub fn dimensions(self: *const Buffer, api: *const Api) []const i64 { @@ -1346,10 +1095,7 @@ pub const Buffer = opaque { const ret = try api.call(.PJRT_Buffer_OpaqueDeviceMemoryDataPointer, .{ .buffer = self.inner(), }); - return ret.device_memory_ptr orelse { - log.err("[PJRT_Buffer_OpaqueDeviceMemoryDataPointer] plugin returned success with null device_memory_ptr", .{}); - return error.Internal; - }; + return ret.device_memory_ptr.?; } pub fn copyRawToHost(self: *const Buffer, api: *const Api, dst: []u8, offset: i64) ApiError!?*Event { @@ -1365,7 +1111,7 @@ pub const Buffer = opaque { pub fn copyToMemory(self: *const Buffer, api: *const Api, dst_memory: *const Memory) ApiError!*Buffer { const ret = try api.call(.PJRT_Buffer_CopyToMemory, .{ .buffer = self.inner(), - .dst_memory = dst_memory.inner(), + .dst_memory = @ptrCast(@constCast(dst_memory)), }); return @ptrCast(ret.dst_buffer); } @@ -1388,11 +1134,6 @@ pub const Buffer = opaque { .buffer = self.inner(), }); } - - pub fn memoryLayout(self: *const Buffer, api: *const Api) ApiError!MemoryLayout { - const ret = try api.call(.PJRT_Buffer_GetMemoryLayout, .{ .buffer = self.inner() }); - return .fromCStruct(ret.layout); - } }; pub const Event = opaque { @@ -1421,8 +1162,7 @@ pub const Event = opaque { pub fn await(self: *Event, api: *const Api, io: std.Io) ApiError!void { if (self.isReady(api)) { - const err = self.getEventError(api) orelse return; - return interpretPjrtError(api, err, "PJRT_Event_Await", .warn); + return; } const Ctx = struct { @@ -1439,7 +1179,12 @@ pub const Event = opaque { }.call, &ctx); ctx.event.waitUncancelable(io); - if (ctx.err) |err| return interpretPjrtError(api, err, "PJRT_Event_OnReady", .warn); + if (ctx.err) |e| { + defer e.deinit(api); + const err_code = e.getCode(api).toApiError(); + log.err("{t} {s}", .{ err_code, e.getMessage(api) }); + return err_code; + } } pub fn awaitRaw(self: *const Event, api: *const Api) ApiError!void { @@ -1630,7 +1375,7 @@ pub const NamedValue = extern struct { }; pub fn kind(self: NamedValue) Kind { - return @fromBackingInt(@intCast(self.inner.type)); + return @enumFromInt(self.inner.type); } pub fn name(self: NamedValue) []const u8 { @@ -1647,14 +1392,14 @@ pub const NamedValue = extern struct { }; } - pub fn init(comptime kind_: Kind, name_: []const u8, value_: @FieldType(Value, @tagName(kind_))) NamedValue { + pub fn init(comptime kind_: Kind, name_: []const u8, value_: std.meta.fieldInfo(Value, kind_).type) NamedValue { return .{ .inner = .{ .struct_size = c.PJRT_NamedValue_STRUCT_SIZE, .extension_start = null, .name = @ptrCast(name_), .name_size = name_.len, - .type = @backingInt(kind_), + .type = @intFromEnum(kind_), .unnamed_0 = switch (kind_) { .string => .{ .string_value = @ptrCast(@constCast(value_)) }, .int64 => .{ .int64_value = value_ }, @@ -1767,98 +1512,3 @@ pub const Ffi = extern struct { } } }; - -pub const Profiler = struct { - pjrt_api: *const Api, - api: *const c.PLUGIN_Profiler_Api, - inner: *c.PLUGIN_Profiler, - - pub fn init(api_: *const Api, prof_ext: *const c.PJRT_Profiler_Extension, options_pb: []const u8) ApiError!Profiler { - var args: c.PLUGIN_Profiler_Create_Args = .{ - .struct_size = meta.structSize(c.PLUGIN_Profiler_Create_Args), - .options = options_pb.ptr, - .options_size = options_pb.len, - .profiler = null, - }; - - const profiler_api: *const c.PLUGIN_Profiler_Api = @ptrCast(prof_ext.profiler_api); - if (profiler_api.create.?(&args)) |err| { - const pjrt_err: *Error = @ptrCast(err); - return pjrt_err.getCode(api_).toApiError(); - } - - return .{ - .pjrt_api = api_, - .api = profiler_api, - .inner = args.profiler.?, - }; - } - - pub fn start(self: *Profiler) ApiError!void { - var args: c.PLUGIN_Profiler_Start_Args = .{ - .struct_size = meta.structSize(c.PLUGIN_Profiler_Start_Args), - .profiler = self.inner, - }; - - if (self.api.start.?(&args)) |err| { - const pjrt_err: *Error = @ptrCast(err); - return pjrt_err.getCode(self.pjrt_api).toApiError(); - } - } - - pub fn stop(self: *Profiler) ApiError!void { - var args: c.PLUGIN_Profiler_Stop_Args = .{ - .struct_size = meta.structSize(c.PLUGIN_Profiler_Stop_Args), - .profiler = self.inner, - }; - - if (self.api.stop.?(&args)) |err| { - const pjrt_err: *Error = @ptrCast(err); - return pjrt_err.getCode(self.pjrt_api).toApiError(); - } - } - - pub fn collectData(self: *Profiler, allocator: std.mem.Allocator) ![]u8 { - var args: c.PLUGIN_Profiler_CollectData_Args = .{ - .struct_size = meta.structSize(c.PLUGIN_Profiler_CollectData_Args), - .profiler = self.inner, - .buffer = null, - .buffer_size_in_bytes = 0, - }; - - if (self.api.collect_data.?(&args)) |err| { - const pjrt_err: *Error = @ptrCast(err); - return pjrt_err.getCode(self.pjrt_api).toApiError(); - } - - if (args.buffer != null) { - var data = args.buffer[0..args.buffer_size_in_bytes]; - if (data.len > 0 and data[data.len - 1] == 0) { - data = data[0 .. data.len - 1]; - } - return allocator.dupe(u8, data); - } - - if (args.buffer_size_in_bytes == 0) return &[_]u8{}; - - const buffer = try allocator.alloc(u8, args.buffer_size_in_bytes); - errdefer allocator.free(buffer); - - args.buffer = buffer.ptr; - if (self.api.collect_data.?(&args)) |err| { - const pjrt_err: *Error = @ptrCast(err); - return pjrt_err.getCode(self.pjrt_api).toApiError(); - } - - // Trim trailing null byte logic - var data = buffer[0..args.buffer_size_in_bytes]; - if (data.len > 0 and data[data.len - 1] == 0) { - data = data[0 .. data.len - 1]; - } - - const pb = try allocator.dupe(u8, data); - allocator.free(buffer); - - return pb; - } -}; diff --git a/platform_mappings b/platform_mappings new file mode 100644 index 00000000..ed02329d --- /dev/null +++ b/platform_mappings @@ -0,0 +1,29 @@ +platforms: + @zml//platforms:linux_amd64 + --cpu=k8 + + @zml//platforms:linux_arm64 + --cpu=aarch64 + + @zml//platforms:macos_arm64 + --cpu=darwin_arm64 + --apple_platform_type=macos + + @zml//platforms:macos_amd64 + --cpu=darwin_amd64 + --apple_platform_type=macos + +flags: + --cpu=darwin_arm64 + --apple_platform_type=macos + @zml//platforms:macos_arm64 + + --cpu=darwin_amd64 + --apple_platform_type=macos + @zml//platforms:macos_amd64 + + --cpu=k8 + @zml//platforms:linux_amd64 + + --cpu=aarch64 + @zml//platforms:linux_arm64 diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 5ee2f176..a2a023b1 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -1,6 +1,5 @@ load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") load("@rules_zig//zig:defs.bzl", "zig_library") -load("//bazel:zig_options.bzl", "zig_option", "zig_options") PLATFORMS = { "cpu": True, @@ -8,9 +7,6 @@ PLATFORMS = { "rocm": False, "tpu": False, "neuron": False, - "oneapi": False, - "metal": False, - "furiosa": False, } [ @@ -21,75 +17,20 @@ PLATFORMS = { for runtime, default in PLATFORMS.items() ] -[ - zig_option( - name = "{}_enabled".format(runtime), - bool = select({ - "//platforms:{}.enabled".format(runtime): True, - "//conditions:default": False, - }), - visibility = ["//platforms:__pkg__"], - ) - for runtime in PLATFORMS -] - -zig_options( - name = "options", - import_name = "platforms/options", - settings = [":{}_enabled".format(runtime) for runtime in PLATFORMS], - visibility = ["//platforms:__subpackages__"], -) - -# x86_64 and aarch64 [ config_setting( name = "{}.enabled".format(runtime), flag_values = {":{}".format(runtime): "True"}, visibility = ["//visibility:public"], ) - for runtime in [ - "cpu", - "cuda", - ] + for runtime in PLATFORMS.keys() ] -# x86_64 only -[ - config_setting( - name = "{}.enabled".format(runtime), - constraint_values = [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - flag_values = {":{}".format(runtime): "True"}, - visibility = ["//visibility:public"], - ) - for runtime in [ - "rocm", - "tpu", - "neuron", - "oneapi", - "furiosa", - ] -] - -config_setting( - name = "metal.enabled", - constraint_values = [ - "@platforms//cpu:aarch64", - "@platforms//os:macos", - ], - flag_values = {":metal": "True"}, - visibility = ["//visibility:public"], -) - zig_library( name = "platforms", main = "platforms.zig", - tags = ["manual"], visibility = ["//visibility:public"], deps = [ - ":options", "//pjrt", ] + [ "//platforms/{}".format(runtime) @@ -102,11 +43,9 @@ platform( constraint_values = [ "@platforms//cpu:x86_64", "@platforms//os:linux", - "@llvm//constraints/libc:gnu.2.28", ], flags = [ "--force_pic=true", # required for protobuf upb on Linux - "--cpu=x86_64", ], visibility = ["//visibility:public"], ) @@ -116,11 +55,9 @@ platform( constraint_values = [ "@platforms//cpu:aarch64", "@platforms//os:linux", - "@llvm//constraints/libc:gnu.2.28", ], flags = [ "--force_pic=true", # required for protobuf upb on Linux - "--cpu=aarch64", ], visibility = ["//visibility:public"], ) @@ -142,26 +79,3 @@ platform( ], visibility = ["//visibility:public"], ) - -constraint_setting(name = "execution_location") - -constraint_value( - name = "local_execution", - constraint_setting = ":execution_location", - visibility = ["//visibility:public"], -) - -# Special constrained host platform that toolchain can use as exec_compatible_with -# if they want to only be used when executing on the local machine. -# -# This is currently used by Zig toolchains so that they are only resolved for -# the host platform and thus can be used for no-remote-exec actions. -platform( - name = "local_execution_platform", - constraint_values = [":local_execution"], - exec_properties = { - "no-remote-exec": "1", - }, - parents = ["@platforms//host"], - visibility = ["//visibility:public"], -) diff --git a/platforms/cpu/BUILD.bazel b/platforms/cpu/BUILD.bazel index 418c7837..16e059d5 100644 --- a/platforms/cpu/BUILD.bazel +++ b/platforms/cpu/BUILD.bazel @@ -1,4 +1,4 @@ -load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") @@ -18,43 +18,42 @@ config_setting( ], ) +cc_library( + name = "empty", + defines = ["ZML_RUNTIME_CPU_DISABLED"], +) + copy_to_directory( name = "sandbox", srcs = select({ - ":darwin_arm64": ["@libzml_cpu_darwin_arm64//:libzml_cpu"], - ":darwin_amd64": ["@libzml_cpu_darwin_amd64//:libzml_cpu"], - "@platforms//os:linux": ["@libzml_cpu_linux_amd64//:libzml_cpu"], + ":darwin_arm64": ["@libpjrt_cpu_darwin_arm64//:libpjrt_cpu"], + ":darwin_amd64": ["@libpjrt_cpu_darwin_amd64//:libpjrt_cpu"], + "@platforms//os:linux": ["@libpjrt_cpu_linux_amd64//:libpjrt_cpu"], }), - out = "sandbox", + out = "sandbox/lib", include_external_repositories = ["**"], - replace_prefixes = { - "libzml_cpu_so": "lib", - "usr/lib/x86_64-linux-gnu": "lib", - }, ) cc_library( - name = "libzml_cpu", + name = "libpjrt_cpu", data = [":sandbox"], - tags = ["manual"], + defines = ["ZML_RUNTIME_CPU"], ) zig_library( name = "cpu", import_name = "platforms/cpu", main = "cpu.zig", - tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", - "//platforms:options", ] + select({ "//platforms:cpu.enabled": [ - ":libzml_cpu", + ":libpjrt_cpu", "//bazel", "//stdx", "@rules_zig//zig/runfiles", ], - "//conditions:default": [], + "//conditions:default": [":empty"], }), ) diff --git a/platforms/cpu/cpu.bzl b/platforms/cpu/cpu.bzl index 08817afe..35d0746e 100644 --- a/platforms/cpu/cpu.bzl +++ b/platforms/cpu/cpu.bzl @@ -1,5 +1,4 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//platforms:packages.bzl", "packages") _BUILD_FILE_DEFAULT_VISIBILITY = """\ @@ -7,69 +6,47 @@ package(default_visibility = ["//visibility:public"]) """ _BUILD_LINUX = "\n".join([ - packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), - packages.patchelf( - name = "libzml_cpu_so", - src = "lib/libzml_cpu.so", - set_rpath = "$ORIGIN", - ), packages.filegroup( - name = "libzml_cpu", - srcs = [":libzml_cpu_so", "@llvm-libunwind1//:libunwind"], + name = "libpjrt_cpu", + srcs = ["libpjrt_cpu.so"], visibility = ["@zml//platforms/cpu:__subpackages__"], ), ]) _BUILD_DARWIN = packages.filegroup( - name = "libzml_cpu", - srcs = ["lib/libzml_cpu.dylib"], + name = "libpjrt_cpu", + srcs = ["libpjrt_cpu.dylib"], visibility = ["@zml//platforms/cpu:__subpackages__"], ) -def _cpu_plugin_impl(mctx): - loaded_packages = packages.read(mctx, ["@zml//platforms/cpu:packages.lock.json"]) - pkg = loaded_packages["llvm-libunwind1"]["amd64"] - http_deb_archive( - name = "llvm-libunwind1", - urls = pkg["urls"], - sha256 = pkg["sha256"], - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + packages.filegroup( - name = "libunwind", - srcs = ["usr/lib/x86_64-linux-gnu/libunwind.so.1"], - ), - ) - +def _cpu_pjrt_plugin_impl(mctx): http_archive( - name = "libzml_cpu_linux_amd64", + name = "libpjrt_cpu_linux_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_LINUX, - sha256 = "3d6a6d1b047a0943b50fd9e63e5f6dff6b7cf184b186c0cc1052de40f0dbec92", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-linux-amd64.tar.zst", + sha256 = "880a9575be4c950cd49b6ca20c2782869cea7ec445bac843ac55b21053b648ef", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_linux-amd64.tar.gz", ) http_archive( - name = "libzml_cpu_darwin_amd64", + name = "libpjrt_cpu_darwin_amd64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "d6b4b15b13a2d1a3e0b84f143f568cd572bef5db91784f9f7a826173ed626c7e", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-amd64.tar.zst", + sha256 = "e1bd183a862fa1c504ade5aca1e000b273628ab2bd2a9eb2971067584934d882", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_darwin-amd64.tar.gz", ) http_archive( - name = "libzml_cpu_darwin_arm64", + name = "libpjrt_cpu_darwin_arm64", build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + _BUILD_DARWIN, - sha256 = "c06babc001edeab313bc63daab3aa3d40e087266a3a29634d182628fb65fad14", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cpu-darwin-arm64.tar.zst", + sha256 = "fb733aeec1284bfe4573acbe78ca38292ab521e8f207a3547d56d1a6d7173d47", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cpu_darwin-arm64.tar.gz", ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = [ - "libzml_cpu_linux_amd64", - "libzml_cpu_darwin_amd64", - "libzml_cpu_darwin_arm64", - ], + root_module_direct_deps = "all", root_module_direct_dev_deps = [], ) -cpu_plugin = module_extension( - implementation = _cpu_plugin_impl, +cpu_pjrt_plugin = module_extension( + implementation = _cpu_pjrt_plugin_impl, ) diff --git a/platforms/cpu/cpu.zig b/platforms/cpu/cpu.zig index 523cd119..37131347 100644 --- a/platforms/cpu/cpu.zig +++ b/platforms/cpu/cpu.zig @@ -3,20 +3,20 @@ const builtin = @import("builtin"); const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); const pjrt = @import("pjrt"); -const platforms_options = @import("platforms/options"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/cpu"); pub fn isEnabled() bool { - return platforms_options.cpu_enabled; + return @hasDecl(c, "ZML_RUNTIME_CPU"); } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = io; - _ = allocator; + _ = io; // autofix + _ = allocator; // autofix if (comptime !isEnabled()) { return error.Unavailable; } @@ -24,7 +24,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("zml/platforms/cpu/sandbox", &path_buf) orelse { + const sandbox_path = try r.rlocation("zml/platforms/cpu/sandbox/lib", &path_buf) orelse { log.err("Failed to find sandbox path for CPU runtime", .{}); return error.FileNotFound; }; @@ -37,7 +37,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { }; var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_cpu" ++ ext }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "libpjrt_cpu" ++ ext }); break :blk .loadFrom(path); }; } diff --git a/platforms/cpu/packages.lock.json b/platforms/cpu/packages.lock.json deleted file mode 100644 index 7e9963e3..00000000 --- a/platforms/cpu/packages.lock.json +++ /dev/null @@ -1,65 +0,0 @@ -{ - "packages": [ - { - "arch": "amd64", - "dependencies": [ - { - "key": "libc6_2.36-9-p-deb12u10_amd64", - "name": "libc6", - "version": "2.36-9+deb12u10" - }, - { - "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", - "name": "libgcc-s1", - "version": "12.2.0-14+deb12u1" - }, - { - "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", - "name": "gcc-12-base", - "version": "12.2.0-14+deb12u1" - } - ], - "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_amd64", - "name": "llvm-libunwind1", - "sha256": "04cb76d70da569e5bcf2395f28dd7e9b9e9f0bf4521cf3b008b0f62ae9d2b712", - "urls": [ - "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_amd64.deb" - ], - "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libc6_2.36-9-p-deb12u10_amd64", - "name": "libc6", - "sha256": "5dc83256f10ca4d0f2a53dd6583ffd0d0e319af30074ea6c82fb0ae77bd16365", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/glibc/libc6_2.36-9+deb12u10_amd64.deb" - ], - "version": "2.36-9+deb12u10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", - "name": "libgcc-s1", - "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" - ], - "version": "12.2.0-14+deb12u1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", - "name": "gcc-12-base", - "sha256": "1896a2aacf4ad681ff5eacc24a5b0ca4d5d9c9b9c9e4b6de5197bc1e116ea619", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_amd64.deb" - ], - "version": "12.2.0-14+deb12u1" - } - ], - "version": 1 -} \ No newline at end of file diff --git a/platforms/cpu/packages.yaml b/platforms/cpu/packages.yaml deleted file mode 100644 index d6f97d05..00000000 --- a/platforms/cpu/packages.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# -# bazel run @apt_cpu//:lock -# -version: 1 - -sources: - - channel: bookworm main - url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z - - channel: llvm-toolchain-bookworm-22 main - url: https://apt.llvm.org/bookworm - -archs: - - "amd64" - -packages: - - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-bookworm-22. diff --git a/platforms/cuda/BUILD.bazel b/platforms/cuda/BUILD.bazel index 7f7aed55..3bb7cfc0 100644 --- a/platforms/cuda/BUILD.bazel +++ b/platforms/cuda/BUILD.bazel @@ -1,17 +1,11 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") - -zig_binary( - name = "compat_probe", - main = "compat_probe.zig", - visibility = ["//visibility:public"], -) +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") zig_shared_library( name = "zmlxcuda", main = "zmlxcuda.zig", shared_lib_name = "libzmlxcuda.so.0", - visibility = ["//visibility:public"], + visibility = ["@libpjrt_cuda//:__subpackages__"], # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. zigopts = ["-fno-stack-check"], @@ -19,48 +13,37 @@ zig_shared_library( ) cc_library( - name = "libzml_cuda_repo", - target_compatible_with = ["@platforms//os:linux"], - deps = select({ - "@llvm//platforms/config:linux_x86_64": ["@libzml_cuda_linux_amd64//:libzml_cuda"], - "@llvm//platforms/config:linux_aarch64": ["@libzml_cuda_linux_arm64//:libzml_cuda"], - }), + name = "empty", + defines = ["ZML_RUNTIME_CUDA_DISABLED"], ) cc_library( - name = "cuda_nvtx_headers", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["//visibility:public"], - deps = select({ - "@llvm//platforms/config:linux_x86_64": ["@cuda_nvtx_linux_x86_64//:headers"], - "@llvm//platforms/config:linux_aarch64": ["@cuda_nvtx_linux_sbsa//:headers"], - }), -) - -cc_library( - name = "libzml_cuda", - hdrs = ["libzml_cuda.h"], - tags = ["manual"], - deps = [":libzml_cuda_repo"], + name = "libpjrt_cuda", + hdrs = ["libpjrt_cuda.h"], + defines = ["ZML_RUNTIME_CUDA"], + deps = ["@libpjrt_cuda"], ) zig_library( name = "cuda", - srcs = ["compat_probe.zig"], + data = select({ + "//platforms:cuda.enabled": [ + "@flashattn//:libflashattn", + ], + "//conditions:default": [":empty"], + }), import_name = "platforms/cuda", main = "cuda.zig", - tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", - "//platforms:options", ] + select({ "//platforms:cuda.enabled": [ - ":libzml_cuda", + ":libpjrt_cuda", "//bazel", "//stdx", "@rules_zig//zig/runfiles", ], - "//conditions:default": [], + "//conditions:default": [":empty"], }), ) diff --git a/platforms/cuda/compat_probe.zig b/platforms/cuda/compat_probe.zig deleted file mode 100644 index 3f890c6a..00000000 --- a/platforms/cuda/compat_probe.zig +++ /dev/null @@ -1,39 +0,0 @@ -const std = @import("std"); - -pub const ExitCode = enum(u8) { - Success = 0, - CompatNotSupportedOnDevice = 1, - SystemDriverMismatch = 2, - UnexpectedError = 3, -}; - -const CUDA_SUCCESS = 0; -const CUDA_ERROR_SYSTEM_DRIVER_MISMATCH = 803; -const CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE = 804; - -pub fn main(init: std.process.Init) !u8 { - const arena = init.arena; - const io = init.io; - - const self_exe_dir = try std.process.executableDirPathAlloc(io, arena.allocator()); - const libcuda_path = try std.Io.Dir.path.joinZ(arena.allocator(), &.{ self_exe_dir, "..", "lib", "compat", "libcuda.so.1" }); - - var libcuda = try std.DynLib.open(libcuda_path); - const cuInit = libcuda.lookup(*const fn (c_uint) callconv(.c) c_int, "cuInit").?; - const cuGetErrorString = libcuda.lookup(*const fn (c_int, *[*c]const u8) callconv(.c) c_int, "cuGetErrorString").?; - - return @backingInt(switch (cuInit(0)) { - CUDA_SUCCESS => ExitCode.Success, - CUDA_ERROR_SYSTEM_DRIVER_MISMATCH => ExitCode.SystemDriverMismatch, - CUDA_ERROR_COMPAT_NOT_SUPPORTED_ON_DEVICE => ExitCode.CompatNotSupportedOnDevice, - else => |v| blk: { - var err_str_ptr: [*c]const u8 = undefined; - if (cuGetErrorString(v, &err_str_ptr) == CUDA_SUCCESS) { - std.log.err("cuInit returned unexpected error code: {d}: {s}", .{ v, err_str_ptr }); - } else { - std.log.err("cuInit returned unexpected error code: {d}", .{v}); - } - break :blk ExitCode.UnexpectedError; - }, - }); -} diff --git a/platforms/cuda/cuda.bzl b/platforms/cuda/cuda.bzl index 8482e52a..14e4867e 100644 --- a/platforms/cuda/cuda.bzl +++ b/platforms/cuda/cuda.bzl @@ -1,49 +1,35 @@ load("@bazel_skylib//lib:paths.bzl", "paths") -load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//platforms:packages.bzl", "packages") -ARCHS = ["linux-x86_64", "linux-sbsa"] - -CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" -CUDA_VERSION = "13.3.0" -CUDA_VARIANT = "cuda13.3" -CUDA_REDIST_JSON_SHA256 = "507eddaab1360336bc0fe17b77552e0b7dfe1e74da888671c3a2f5fad7775db1" - -CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" -CUDNN_VERSION = "9.24.0" -CUDNN_REDIST_JSON_SHA256 = "89a0626f1925d9f4ceb8d6bc0b2de160b30e92afaae5471e6cfbb50234ae5488" - -NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" -NVSHMEM_VERSION = "3.6.5" -NVSHMEM_REDIST_JSON_SHA256 = "afbf1ad5c4174c25a66cadd5b925d6814e679e8f836ef1fee1442afb6cb7fdd3" - _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_cc//cc:cc_import.bzl", "cc_import") -load("@zml//bazel:patchelf.bzl", "patchelf") """ -CUDA_COMPAT_FILES = [ - "libcuda.so.1", - "libcudadebugger.so.1", - "libnvidia-nvvm.so.4", - "libnvidia-nvvm70.so.4", - "libnvidia-ptxjitcompiler.so.1", -] +ARCH = "linux-x86_64" + +CUDA_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cuda/redist/" +CUDA_VERSION = "13.1.1" +CUDA_REDIST_JSON_SHA256 = "97cf605ccc4751825b1865f4af571c9b50dd29ffd13e9a38b296a9ecb1f0d422" + +CUDNN_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/cudnn/redist/" +CUDNN_VERSION = "9.19.0" +CUDNN_REDIST_JSON_SHA256 = "6ad4f2c047ee03131d1efb95db56f78c24953cb30880a2edcf99407ff9362a04" + +NVSHMEM_REDIST_PREFIX = "https://developer.download.nvidia.com/compute/nvshmem/redist/" +NVSHMEM_VERSION = "3.5.19" +NVSHMEM_REDIST_JSON_SHA256 = "6dced4193eb728542504b346cfb768da6e3de2abca0cded95fda3a69729994d2" + +_UBUNTU_PACKAGES = { + "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), +} CUDA_PACKAGES = { - "cuda_nvml_dev": [ - packages.cc_library( - name = "nvml", - hdrs = ["include/nvml.h"], - includes = ["include"], - visibility = ["//visibility:public"], - ), - ], - "cuda_cudart": [ + "cuda_cudart": "\n".join([ # Driver API only packages.cc_library( name = "cuda", @@ -55,74 +41,34 @@ CUDA_PACKAGES = { name = "cuda_cudart", srcs = ["lib/libcudart.so.13"], ), - ], - "cuda_cupti": [ - packages.filegroup( - name = "cuda_cupti", - srcs = ["lib/libcupti.so.13"], - ), - ], - "cuda_nvtx": [ - packages.cc_library_hdrs_glob( - name = "headers", - hdrs_glob = ["include/nvtx3/**"], - includes = ["include"], - ), + ]), + "cuda_cupti": packages.filegroup( + name = "cuda_cupti", + srcs = ["lib/libcupti.so.13"], + ), + "cuda_nvtx": "\n".join([ packages.filegroup( name = "cuda_nvtx", srcs = ["lib/libnvtx3interop.so"], ), - ], - "cuda_compat": [ - packages.patchelf( - name = "{}.patchelf".format(file), - set_rpath = "$ORIGIN", - src = "compat/{}".format(file), - soname = file, - ) - for file in CUDA_COMPAT_FILES - ] + [ - """filegroup( - name = "cuda_compat", - srcs = [ - "compat/libnvidia-gpucomp.so.610.43.02", - "compat/libnvidia-tileiras.so.610.43.02", - ] + {} + select({{ - "@llvm//platforms/config:linux_x86_64": [ - "compat/libnvidia-pkcs11-openssl3.so.610.43.02", - ], - "@llvm//platforms/config:linux_aarch64": [], - }}), - )""".format(repr([ - ":{}.patchelf".format(file) - for file in CUDA_COMPAT_FILES - ])), - ], - "libcufft": [ - packages.filegroup( - name = "libcufft", - srcs = ["lib/libcufft.so.12"], - ), - ], - "libcusolver": [ - packages.filegroup( - name = "libcusolver", - srcs = ["lib/libcusolver.so.12"], - ), - ], - "libcusparse": [ - packages.filegroup( - name = "libcusparse", - srcs = ["lib/libcusparse.so.12"], - ), - ], - "libnvjitlink": [ - packages.filegroup( - name = "libnvjitlink", - srcs = ["lib/libnvJitLink.so.13"], - ), - ], - "cuda_nvcc": [ + ]), + "libcufft": packages.filegroup( + name = "libcufft", + srcs = ["lib/libcufft.so.12"], + ), + "libcusolver": packages.filegroup( + name = "libcusolver", + srcs = ["lib/libcusolver.so.12"], + ), + "libcusparse": packages.filegroup( + name = "libcusparse", + srcs = ["lib/libcusparse.so.12"], + ), + "libnvjitlink": packages.filegroup( + name = "libnvjitlink", + srcs = ["lib/libnvJitLink.so.13"], + ), + "cuda_nvcc": "\n".join([ packages.filegroup( name = "cuda_nvcc", srcs = [ @@ -130,35 +76,34 @@ CUDA_PACKAGES = { "bin/nvlink", ], ), - ], - "cuda_tileiras": [ - packages.filegroup( - name = "cuda_tileiras", - srcs = ["bin/tileiras"], + packages.cc_import( + name = "nvptxcompiler", + static_library = "lib/libnvptxcompiler_static.a", ), - ], - "libnvvm": [ + ]), + "libnvvm": "\n".join([ packages.filegroup( name = "libnvvm", srcs = [ "nvvm/bin/cicc", "nvvm/libdevice/libdevice.10.bc", - "nvvm/lib64/libnvvm.so", - "nvvm/lib64/libnvvm.so.4", - "nvvm/lib64/libnvvm.so.4.0.0", ], ), - ], - "cuda_nvrtc": [ + packages.cc_import( + name = "nvptxcompiler", + static_library = "lib/libnvptxcompiler_static.a", + ), + ]), + "cuda_nvrtc": "\n".join([ packages.filegroup( name = "cuda_nvrtc", srcs = [ "lib/libnvrtc.so.13", - "lib/libnvrtc-builtins.so.13.3", + "lib/libnvrtc-builtins.so.13.1", ], ), - ], - "libcublas": [ + ]), + "libcublas": "\n".join([ packages.filegroup( name = "libcublas", srcs = [ @@ -166,11 +111,11 @@ CUDA_PACKAGES = { "lib/libcublas.so.13", ], ), - ], + ]), } CUDNN_PACKAGES = { - "cudnn": [ + "cudnn": "\n".join([ packages.filegroup( name = "cudnn", srcs = [ @@ -181,75 +126,23 @@ CUDNN_PACKAGES = { "lib/libcudnn_graph.so.9", "lib/libcudnn_engines_precompiled.so.9", "lib/libcudnn_engines_runtime_compiled.so.9", - "lib/libcudnn_engines_tensor_ir.so.9", "lib/libcudnn_heuristic.so.9", ], ), - ], + ]), } NVSHMEM_PACKAGES = { - "libnvshmem": [ - packages.filegroup( - name = "libnvshmem", - srcs = [ - "lib/libnvshmem_host.so.3", - "lib/nvshmem_bootstrap_uid.so.3", - "lib/nvshmem_transport_ibrc.so.5", - ], - ), - ], + "libnvshmem": packages.filegroup( + name = "libnvshmem", + srcs = [ + "lib/libnvshmem_host.so.3", + "lib/nvshmem_bootstrap_uid.so.3", + "lib/nvshmem_transport_ibrc.so.4", + ], + ), } -_UBUNTU_PACKAGES = { - "llvm-libunwind1": [ - """filegroup( - name = "libunwind", - srcs = select({ - "@llvm//platforms/config:linux_x86_64": ["usr/lib/x86_64-linux-gnu/libunwind.so.1"], - "@llvm//platforms/config:linux_aarch64": ["usr/lib/aarch64-linux-gnu/libunwind.so.1"], - }), - )""", - ], - "zlib1g": [ - """filegroup( - name = "zlib1g", - srcs = select({ - "@llvm//platforms/config:linux_x86_64": ["lib/x86_64-linux-gnu/libz.so.1"], - "@llvm//platforms/config:linux_aarch64": ["lib/aarch64-linux-gnu/libz.so.1"], - }), - )""", - ], -} - -_PLUGINS = { - "amd64": { - "sha256": "72b1359384ac50e727b800e81c56d7134f9d6a061a982c4e36b7fef62cf7c331", - "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-amd64.tar.zst", - }, - "arm64": { - "sha256": "caeefd717960fa2acec6304cd8296299e262abdba669ecf4d487e3af76d73a7b", - "url": "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-cuda-linux-arm64.tar.zst", - }, -} - -_NCCL_ASSETS = { - "amd64": { - "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_x86_64.whl", - "sha256": "534dbf3058cadb625f08ab0d17f1dffad3b961a2bfa360d66633fcf21be53f57", - }, - "arm64": { - "url": "https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.30.4-py3-none-manylinux_2_18_aarch64.whl", - "sha256": "e99308a3a89fba78918d50886e81072a6c8b0b4199feb02c3903e63713a6525a", - }, -} - -def _repo_suffix(arch): - return "linux_{}".format(arch) - -def _repo_name(name, arch): - return "{}_{}".format(name, _repo_suffix(arch)) - def _read_redist_json(mctx, url, sha256): fname = ".{}.json".format(sha256) mctx.download( @@ -263,7 +156,6 @@ def _cuda_impl(mctx): loaded_packages = packages.read(mctx, [ "@zml//platforms/cuda:packages.lock.json", ]) - CUDA_REDIST = _read_redist_json( mctx, url = CUDA_REDIST_PREFIX + "redistrib_{}.json".format(CUDA_VERSION), @@ -282,100 +174,77 @@ def _cuda_impl(mctx): sha256 = CUDNN_REDIST_JSON_SHA256, ) - #TODO(cerisier): for each architecture for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): - pkgs = loaded_packages[pkg_name] - for arch, pkg in pkgs.items(): - http_deb_archive( - name = _repo_name(pkg_name, arch), - urls = pkg["urls"], - sha256 = pkg["sha256"], - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), - ) + pkg = loaded_packages[pkg_name] + http_deb_archive( + name = pkg_name, + urls = pkg["urls"], + sha256 = pkg["sha256"], + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + ) for pkg, build_file_content in CUDA_PACKAGES.items(): pkg_data = CUDA_REDIST[pkg] - for arch in ARCHS: - pkg_repo_name = pkg + "_" + arch.replace("-", "_") - arch_data = pkg_data.get(arch) - if not arch_data: - fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) - arch_data = arch_data.get(CUDA_VARIANT, None) or arch_data - http_archive( - name = pkg_repo_name, - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), - url = CUDA_REDIST_PREFIX + arch_data["relative_path"], - sha256 = arch_data["sha256"], - strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), - ) + arch_data = pkg_data.get(ARCH) + if not arch_data: + continue + http_archive( + name = pkg, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + url = CUDA_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) for pkg, build_file_content in CUDNN_PACKAGES.items(): - for arch in ARCHS: - pkg_repo_name = pkg + "_" + arch.replace("-", "_") - pkg_data = CUDNN_REDIST[pkg] - arch_data = pkg_data.get(arch) - if not arch_data: - fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) - arch_data = arch_data.get("cuda13", arch_data) - http_archive( - name = pkg_repo_name, - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), - url = CUDNN_REDIST_PREFIX + arch_data["relative_path"], - sha256 = arch_data["sha256"], - strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), - ) + pkg_data = CUDNN_REDIST[pkg] + arch_data = pkg_data.get(ARCH) + if not arch_data: + continue + arch_data = arch_data.get("cuda13", arch_data) + http_archive( + name = pkg, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + url = CUDNN_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), + ) for pkg, build_file_content in NVSHMEM_PACKAGES.items(): - for arch in ARCHS: - pkg_repo_name = pkg + "_" + arch.replace("-", "_") - pkg_data = NVSHMEM_REDIST[pkg] - arch_data = pkg_data.get(arch) - if not arch_data: - fail("CUDA redist package {} does not have data for architecture {}".format(pkg, arch)) - arch_data = arch_data.get("cuda13", arch_data) - http_archive( - name = pkg_repo_name, - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + "\n".join(build_file_content), - url = NVSHMEM_REDIST_PREFIX + arch_data["relative_path"], - sha256 = arch_data["sha256"], - strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), - ) - - for arch, arch_config in _NCCL_ASSETS.items(): + pkg_data = NVSHMEM_REDIST[pkg] + arch_data = pkg_data.get(ARCH) + if not arch_data: + continue + arch_data = arch_data.get("cuda13", arch_data) http_archive( - name = _repo_name("nccl", arch), - urls = [arch_config["url"]], - type = "zip", - sha256 = arch_config["sha256"], - build_file_content = "\n".join([ - _BUILD_FILE_DEFAULT_VISIBILITY, - packages.filegroup( - name = "nccl", - srcs = ["nvidia/nccl/lib/libnccl.so.2"], - ), - ]), + name = pkg, + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, + url = NVSHMEM_REDIST_PREFIX + arch_data["relative_path"], + sha256 = arch_data["sha256"], + strip_prefix = paths.basename(arch_data["relative_path"]).replace(".tar.xz", ""), ) - for arch, arch_config in _PLUGINS.items(): - http_archive( - name = _repo_name("libzml_cuda", arch), - build_file = "libzml_cuda.BUILD.bazel", - url = arch_config["url"], - sha256 = arch_config["sha256"], - ) + http_archive( + name = "nccl", + urls = ["https://pypi.nvidia.com/nvidia-nccl-cu13/nvidia_nccl_cu13-2.29.3-py3-none-manylinux_2_18_x86_64.whl"], + type = "zip", + sha256 = "2a321629f49490e4e0122ecb578a4b4a6f89e72740dd988e04dfa4758fab7fc3", + build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + packages.filegroup( + name = "nccl", + srcs = ["nvidia/nccl/lib/libnccl.so.2"], + ), + ) + + http_archive( + name = "libpjrt_cuda", + build_file = "libpjrt_cuda.BUILD.bazel", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v15.0.2/pjrt-cuda_linux-amd64.tar.gz", + sha256 = "ebc5f0fa54d38ec85346f0f02b29a6497ee394b9e5fa4832da5db16d3296ae84", + ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = [ - "libzml_cuda_linux_amd64", - "libzml_cuda_linux_arm64", - "cuda_nvml_dev_linux_x86_64", - "cuda_nvml_dev_linux_sbsa", - "zlib1g_linux_arm64", - "zlib1g_linux_amd64", - "cuda_nvtx_linux_x86_64", - "cuda_nvtx_linux_sbsa", - ], + root_module_direct_deps = ["libpjrt_cuda"], root_module_direct_dev_deps = [], ) diff --git a/platforms/cuda/cuda.zig b/platforms/cuda/cuda.zig index 692ea226..b0ff4425 100644 --- a/platforms/cuda/cuda.zig +++ b/platforms/cuda/cuda.zig @@ -5,71 +5,24 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); -const platforms_options = @import("platforms/options"); const stdx = @import("stdx"); -const compat_probe = @import("compat_probe.zig"); - const nvidiaLibsPath = "/cuda/"; const log = std.log.scoped(.@"zml/platforms/cuda"); -fn findCudaSandbox( - r: anytype, - buffer: *[std.Io.Dir.max_path_bytes]u8, -) !?[]const u8 { - const candidate = switch (builtin.cpu.arch) { - .aarch64 => "libzml_cuda_linux_arm64/sandbox", - .x86_64 => "libzml_cuda_linux_amd64/sandbox", - else => return null, - }; - return try r.rlocation(candidate, buffer); -} - pub fn isEnabled() bool { - return platforms_options.cuda_enabled; -} - -pub fn needsCudaCompat(io: std.Io, sandbox_path: []const u8) !bool { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const nvidia_compat_path = try stdx.Io.Dir.path.bufJoinZ(&buf, &.{ sandbox_path, "bin", "compat_probe" }); - - var child = try std.process.spawn(io, .{ - .argv = &[_][]const u8{nvidia_compat_path}, - .cwd = .{ .path = sandbox_path }, - .stdin = .ignore, - .stdout = .inherit, - .stderr = .inherit, - }); - defer child.kill(io); - - const res = child.wait(io) catch |err| { - log.err("Failed to run CUDA compatibility probe: {any}", .{err}); - return err; - }; - const result: compat_probe.ExitCode = @fromBackingInt(@intCast(res.exited)); - - return switch (result) { - .Success => true, - .SystemDriverMismatch, .CompatNotSupportedOnDevice => false, - .UnexpectedError => blk: { - log.err("CUDA compatibility probe returned unexpected error code", .{}); - break :blk false; - }, - }; + return @hasDecl(c, "ZML_RUNTIME_CUDA"); } fn hasNvidiaDevice(io: std.Io) bool { - for (&[_][]const u8{ "/dev/nvidiactl", "/dev/dxg" }) |dev| { - std.Io.Dir.accessAbsolute(io, dev, .{ .read = true }) catch continue; - return true; - } - return false; + std.Io.Dir.accessAbsolute(io, "/dev/nvidiactl", .{ .read = true }) catch return false; + return true; } fn hasCudaPathInLDPath() bool { const ldLibraryPath = std.c.getenv("LD_LIBRARY_PATH") orelse return false; - return std.ascii.findIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; + return std.ascii.indexOfIgnoreCase(std.mem.span(ldLibraryPath), nvidiaLibsPath) != null; } fn setupXlaGpuCudaDirFlag(allocator: std.mem.Allocator, sandbox: []const u8) !void { @@ -98,7 +51,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try findCudaSandbox(r, &path_buf) orelse { + const sandbox_path = try r.rlocation("libpjrt_cuda/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for CUDA runtime", .{}); return error.FileNotFound; }; @@ -108,25 +61,17 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { try setupXlaGpuCudaDirFlag(arena.allocator(), sandbox_path); { - const cudaCompat = needsCudaCompat(io, sandbox_path) catch |err| blk: { - log.err("Unable to determine wether or not to use CUDA Compat, disabling: {any}", .{err}); - break :blk false; + var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libnvtx3interop.so" }); + _ = std.c.dlopen(path, .{ .NOW = true, .GLOBAL = true }) orelse { + log.err("Unable to dlopen libnvtx3interop.so: {s}", .{std.c.dlerror().?}); + return error.DlError; }; - - if (cudaCompat) { - log.info("Detected NVIDIA GPU that requires CUDA compatibility libraries.", .{}); - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "compat", "libcuda.so.1" }); - _ = std.c.dlopen(path, .{ .NOW = true }) orelse { - log.warn("Failed to load CUDA compatibility library from {s}: {any}", .{ path, std.mem.span(std.c.dlerror()) }); - }; - log.info("Loaded CUDA compatibility libraries.", .{}); - } } return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_cuda.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_cuda.so" }); break :blk .loadFrom(path); }; } diff --git a/platforms/cuda/flashattn/BUILD.bazel b/platforms/cuda/flashattn/BUILD.bazel new file mode 100644 index 00000000..9ef2bbf8 --- /dev/null +++ b/platforms/cuda/flashattn/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "flashattn", + import_name = "platforms/cuda/flashattn", + main = "flashattn.zig", + visibility = ["//visibility:public"], + deps = [ + "//bazel", + "@flashattn//:c", + "@rules_zig//zig/runfiles", + ], +) diff --git a/platforms/cuda/flashattn/flashattn.zig b/platforms/cuda/flashattn/flashattn.zig new file mode 100644 index 00000000..2a743d33 --- /dev/null +++ b/platforms/cuda/flashattn/flashattn.zig @@ -0,0 +1,112 @@ +const std = @import("std"); + +const bazel = @import("bazel"); +const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); +pub const FA2MhaFwdParams = c.FA2MhaFwdParams; +pub const FA2MhaVarlenFwdParams = c.FA2MhaVarlenFwdParams; +pub const FA3MhaFwdParams = c.FA3MhaFwdParams; +const runfiles = @import("runfiles"); + +pub const Tensor = struct { + inner: c.FlashattnTensor, + + pub fn init(ptr: *anyopaque, dims: []const i64, strides: []const i64, dtype: DataType) Tensor { + const rank = dims.len; + var ret: Tensor = .{ + .inner = .{ + .ptr = ptr, + .rank = @intCast(rank), + .dims = undefined, + .strides = undefined, + .dtype = @intCast(@intFromEnum(dtype)), + }, + }; + @memcpy(ret.inner.dims[0..rank], dims); + @memcpy(ret.inner.strides[0..rank], strides); + return ret; + } +}; + +pub const DataType = enum(c_int) { + f32 = c.CAPI_FLOAT, + f16 = c.CAPI_FLOAT16, + bf16 = c.CAPI_BFLOAT16, + i32 = c.CAPI_INT32, + i8 = c.CAPI_INT8, +}; + +const Fa2MhaFwdFunc = *const fn ( + q: *const Tensor, + k: *const Tensor, + v: *const Tensor, + out: *const Tensor, + softmax_lse: ?*const Tensor, + alibi_slopes_: ?*const Tensor, + softmax_lse_accum: ?*const Tensor, + out_accum: ?*const Tensor, + params: ?*const FA2MhaFwdParams, + stream: ?*anyopaque, +) callconv(.c) void; + +const Fa2MhaVarlenFwdFunc = *const fn ( + q: *const Tensor, + k: *const Tensor, + v: *const Tensor, + out: *const Tensor, + cu_seqlens_q: ?*const Tensor, + cu_seqlens_k: ?*const Tensor, + seqused_k: ?*const Tensor, + block_table: ?*const Tensor, + softmax_lse: ?*const Tensor, + alibi_slopes_: ?*const Tensor, + softmax_lse_accum: ?*const Tensor, + out_accum: ?*const Tensor, + params: ?*const FA2MhaVarlenFwdParams, + stream: ?*anyopaque, +) callconv(.c) void; + +const Fa3MhaFwdFunc = *const fn ( + q: ?*const Tensor, + k: ?*const Tensor, + v: ?*const Tensor, + out: ?*const Tensor, + cu_seqlens_q: ?*const Tensor, + cu_seqlens_k: ?*const Tensor, + seqused_q: ?*const Tensor, + seqused_k: ?*const Tensor, + page_table: ?*const Tensor, + q_descale: ?*const Tensor, + k_descale: ?*const Tensor, + v_descale: ?*const Tensor, + softmax_lse: ?*const Tensor, + softmax_lse_accum: ?*const Tensor, + out_accum: ?*const Tensor, + scheduler_metadata: ?*const Tensor, + s_aux: ?*const Tensor, + cp_tot_seqused_k: ?*const Tensor, + params: ?*const FA3MhaFwdParams, + stream: ?*anyopaque, +) callconv(.c) void; + +pub var fa2_mha_fwd: Fa2MhaFwdFunc = undefined; +pub var fa2_mha_varlen_fwd: Fa2MhaVarlenFwdFunc = undefined; +pub var fa3_mha_fwd: Fa3MhaFwdFunc = undefined; + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { + _ = io; // autofix + _ = allocator; // autofix + + const r = try bazel.runfiles(bazel_builtin.current_repository); + + var buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; + const library = (try r.rlocation("flashattn/flashattn/lib/libflashattn.so", &buffer)) orelse return error.NotFound; + var lib = std.DynLib.open(library) catch |err| { + std.log.err("Failed to open libflashattn.so: {any}", .{err}); + return err; + }; + + fa2_mha_varlen_fwd = lib.lookup(Fa2MhaVarlenFwdFunc, "fa2_mha_varlen_fwd") orelse return error.NotFound; + fa2_mha_fwd = lib.lookup(Fa2MhaFwdFunc, "fa2_mha_fwd") orelse return error.NotFound; + fa3_mha_fwd = lib.lookup(Fa3MhaFwdFunc, "fa3_mha_fwd") orelse return error.NotFound; +} diff --git a/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel b/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel deleted file mode 100644 index 8a61e394..00000000 --- a/platforms/cuda/flashinfer_cutlass_moe/BUILD.bazel +++ /dev/null @@ -1,48 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -_DEPS = [ - "//bazel", - "@rules_zig//zig/runfiles", -] + select({ - "@llvm//platforms/config:linux_x86_64": [ - "@flashinfer_cutlass_moe_linux_amd64//:c", - ], - "@llvm//platforms/config:linux_aarch64": [ - "@flashinfer_cutlass_moe_linux_arm64//:c", - ], - "//conditions:default": [], -}) - -_DATA = select({ - "@llvm//platforms/config:linux_x86_64": [ - "@flashinfer_cutlass_moe_linux_amd64//:libraries", - ], - "@llvm//platforms/config:linux_aarch64": [ - "@flashinfer_cutlass_moe_linux_arm64//:libraries", - ], - "//conditions:default": [], -}) - -zig_library( - name = "flashinfer_cutlass_moe", - data = _DATA, - import_name = "platforms/cuda/flashinfer_cutlass_moe", - main = "flashinfer_cutlass_moe.zig", - tags = ["manual"], - target_compatible_with = [ - "@platforms//os:linux", - ], - visibility = ["//visibility:public"], - deps = _DEPS, -) - -zig_test( - name = "test", - data = _DATA, - main = "flashinfer_cutlass_moe.zig", - tags = ["manual"], - target_compatible_with = [ - "@platforms//os:linux", - ], - deps = _DEPS, -) diff --git a/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig b/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig deleted file mode 100644 index 0b4360cf..00000000 --- a/platforms/cuda/flashinfer_cutlass_moe/flashinfer_cutlass_moe.zig +++ /dev/null @@ -1,165 +0,0 @@ -const std = @import("std"); - -const bazel = @import("bazel"); -const bazel_builtin = @import("bazel_builtin"); -const builtin = @import("builtin"); -pub const c = @import("c"); - -pub const Status = c.zml_fi_cutlass_moe_status; -pub const DType = c.zml_fi_cutlass_moe_dtype; -pub const Activation = c.zml_fi_cutlass_moe_activation; -pub const RunnerOptions = c.zml_fi_cutlass_moe_runner_options; -pub const Context = c.zml_fi_cutlass_moe_context; -pub const Io = c.zml_fi_cutlass_moe_io; -pub const Workspace = c.zml_fi_cutlass_moe_workspace; -pub const WorkspaceRequirements = c.zml_fi_cutlass_moe_workspace_requirements; -pub const Runner = c.zml_fi_cutlass_moe_runner; - -pub const abiVersionExpected = c.ZML_FI_CUTLASS_MOE_ABI_VERSION; - -const AbiVersionFn = @TypeOf(&c.zml_fi_cutlass_moe_abi_version); -const CompiledSmFn = @TypeOf(&c.zml_fi_cutlass_moe_compiled_sm); -const LastErrorFn = @TypeOf(&c.zml_fi_cutlass_moe_last_error); -const DeviceIsSupportedFn = @TypeOf(&c.zml_fi_cutlass_moe_device_is_supported); -const RunnerCreateFn = @TypeOf(&c.zml_fi_cutlass_moe_runner_create); -const RunnerDestroyFn = @TypeOf(&c.zml_fi_cutlass_moe_runner_destroy); -const GetTacticCountsFn = @TypeOf(&c.zml_fi_cutlass_moe_get_tactic_counts); -const GetTacticOccupancyFn = @TypeOf(&c.zml_fi_cutlass_moe_get_tactic_occupancy); -const GetWorkspaceRequirementsFn = @TypeOf(&c.zml_fi_cutlass_moe_get_workspace_requirements); -const RunFn = @TypeOf(&c.zml_fi_cutlass_moe_run); - -pub const Api = struct { - library: std.DynLib, - abiVersion: AbiVersionFn, - compiledSm: CompiledSmFn, - lastError: LastErrorFn, - deviceIsSupported: DeviceIsSupportedFn, - runnerCreate: RunnerCreateFn, - runnerDestroy: RunnerDestroyFn, - getTacticCounts: GetTacticCountsFn, - getTacticOccupancy: GetTacticOccupancyFn, - getWorkspaceRequirements: GetWorkspaceRequirementsFn, - run: RunFn, -}; - -var loaded_api: ?Api = null; - -fn libraryRunfile(sm: u16, buffer: []u8) ![]const u8 { - if (comptime builtin.os.tag != .linux) return error.UnsupportedPlatform; - const cpu_arch = comptime switch (builtin.cpu.arch) { - .aarch64 => "arm64", - .x86_64 => "amd64", - else => return error.UnsupportedPlatform, - }; - return try std.fmt.bufPrint( - buffer, - "flashinfer_cutlass_moe_linux_{s}/lib/libflashinfer_cutlass_moe_sm{d}.so", - .{ cpu_arch, sm }, - ); -} - -fn loadApi(path: []const u8) !Api { - var library = std.DynLib.open(path) catch |err| { - std.log.err("Failed to open FlashInfer CUTLASS MoE library {s}: {any}", .{ path, err }); - return err; - }; - errdefer library.close(); - - const api: Api = .{ - .library = library, - .abiVersion = library.lookup(AbiVersionFn, "zml_fi_cutlass_moe_abi_version") orelse - return error.SymbolNotFound, - .compiledSm = library.lookup(CompiledSmFn, "zml_fi_cutlass_moe_compiled_sm") orelse - return error.SymbolNotFound, - .lastError = library.lookup(LastErrorFn, "zml_fi_cutlass_moe_last_error") orelse - return error.SymbolNotFound, - .deviceIsSupported = library.lookup(DeviceIsSupportedFn, "zml_fi_cutlass_moe_device_is_supported") orelse - return error.SymbolNotFound, - .runnerCreate = library.lookup(RunnerCreateFn, "zml_fi_cutlass_moe_runner_create") orelse - return error.SymbolNotFound, - .runnerDestroy = library.lookup(RunnerDestroyFn, "zml_fi_cutlass_moe_runner_destroy") orelse - return error.SymbolNotFound, - .getTacticCounts = library.lookup(GetTacticCountsFn, "zml_fi_cutlass_moe_get_tactic_counts") orelse - return error.SymbolNotFound, - .getTacticOccupancy = library.lookup(GetTacticOccupancyFn, "zml_fi_cutlass_moe_get_tactic_occupancy") orelse - return error.SymbolNotFound, - .getWorkspaceRequirements = library.lookup( - GetWorkspaceRequirementsFn, - "zml_fi_cutlass_moe_get_workspace_requirements", - ) orelse return error.SymbolNotFound, - .run = library.lookup(RunFn, "zml_fi_cutlass_moe_run") orelse - return error.SymbolNotFound, - }; - if (api.abiVersion() != abiVersionExpected) return error.AbiVersionMismatch; - return api; -} - -pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { - _ = allocator; - if (loaded_api) |*api| { - if (api.compiledSm() != sm) return error.ArchitectureAlreadyLoaded; - return; - } - - const runfiles = try bazel.runfiles(bazel_builtin.current_repository); - var runfile_name_buffer: [160]u8 = undefined; - const runfile_name = try libraryRunfile(sm, &runfile_name_buffer); - var runfile_path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const runfile_path = (try runfiles.rlocation(runfile_name, &runfile_path_buffer)) orelse - return error.NotFound; - - // Preserve the producer library's $ORIGIN-relative dependency lookup when - // the development repository exposes it through Bazel symlinks. - var canonical_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const canonical_length = if (std.fs.path.isAbsolute(runfile_path)) - try std.Io.Dir.realPathFileAbsolute(io, runfile_path, &canonical_buffer) - else - try std.Io.Dir.cwd().realPathFile(io, runfile_path, &canonical_buffer); - var api = try loadApi(canonical_buffer[0..canonical_length]); - errdefer api.library.close(); - if (api.compiledSm() != sm) return error.WrongArchitectureLibrary; - loaded_api = api; -} - -pub fn apiForDevice(device: i32) !*Api { - const api = if (loaded_api) |*value| value else return error.BackendNotLoaded; - var supported: u8 = 0; - const status = api.deviceIsSupported(device, &supported); - if (status != c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) { - if (api.lastError()) |message| { - std.log.err("FlashInfer CUTLASS MoE device probe failed: {s}", .{std.mem.span(message)}); - } - return error.DeviceProbeFailed; - } - if (supported != 0) return api; - return error.UnsupportedArchitecture; -} - -test "select host and GPU architecture-specific FlashInfer CUTLASS MoE library" { - const test_case = &.{ - .{ - .compat_version = 120, - .expected = switch (builtin.cpu.arch) { - .aarch64 => "flashinfer_cutlass_moe_linux_arm64/lib/libflashinfer_cutlass_moe_sm120.so", - .x86_64 => "flashinfer_cutlass_moe_linux_amd64/lib/libflashinfer_cutlass_moe_sm120.so", - }, - }, - .{ - .compat_version = 103, - .expected = switch (builtin.cpu.arch) { - .aarch64 => "flashinfer_cutlass_moe_linux_arm64/lib/libflashinfer_cutlass_moe_sm103.so", - .x86_64 => "flashinfer_cutlass_moe_linux_amd64/lib/libflashinfer_cutlass_moe_sm103.so", - }, - }, - }; - - inline for (test_case) |t| { - var buffer: [160]u8 = undefined; - const path = try libraryRunfile(t.compat_version, &buffer); - try std.testing.expectEqualStrings(t.expected, path); - - try load(std.testing.allocator, std.testing.io, 120); - const api = if (loaded_api) |*value| value else return error.NotLoaded; - try std.testing.expectEqual(@as(i32, 120), api.compiledSm()); - } -} diff --git a/platforms/cuda/libpjrt_cuda.BUILD.bazel b/platforms/cuda/libpjrt_cuda.BUILD.bazel new file mode 100644 index 00000000..372874a9 --- /dev/null +++ b/platforms/cuda/libpjrt_cuda.BUILD.bazel @@ -0,0 +1,70 @@ +load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@zml//bazel:patchelf.bzl", "patchelf") +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +patchelf( + name = "libpjrt_cuda_so", + src = "libpjrt_cuda.so", + add_needed = [ + "libzmlxcuda.so.0", + ], + rename_dynamic_symbols = { + "dlopen": "zmlxcuda_dlopen", + }, + replace_needed = { + "nvshmem_transport_ibrc.so.3": "nvshmem_transport_ibrc.so.4", + "libnvrtc-builtins.so.13.0": "libnvrtc-builtins.so.13.1", + }, + set_rpath = "$ORIGIN", +) + +copy_to_directory( + name = "sandbox", + srcs = [ + ":libpjrt_cuda_so", + "@cuda_cudart", + "@cuda_cupti", + "@cuda_nvcc", + "@libnvvm", + "@cuda_nvrtc", + "@cuda_nvtx", + "@cudnn", + "@libnvshmem", + "@libcublas", + "@libcufft", + "@libcusolver", + "@libcusparse", + "@libnvjitlink", + "@nccl", + "@zlib1g", + "@zml//platforms/cuda:zmlxcuda", + ], + replace_prefixes = { + "nvidia/nccl/lib": "lib", + "nvvm/lib64": "lib", + "libpjrt_cuda_so": "lib", + "lib/x86_64-linux-gnu": "lib", + "platforms/cuda": "lib", + }, + add_directory_to_runfiles = False, + include_external_repositories = ["**"], +) + +cc_library( + name = "libpjrt_cuda", + data = [":sandbox"], + deps = [ + "@cuda_cudart//:cuda", + ], + linkopts = [ + # Defer function call resolution until the function is called + # (lazy loading) rather than at load time. + # + # This is required because we want to let downstream use weak CUDA symbols. + # + # We force it here because -z,now (which resolve all symbols at load time), + # is the default in most bazel CC toolchains as well as in certain linkers. + "-Wl,-z,lazy", + ], + visibility = ["@zml//platforms/cuda:__subpackages__"], +) diff --git a/platforms/cuda/libzml_cuda.h b/platforms/cuda/libpjrt_cuda.h similarity index 100% rename from platforms/cuda/libzml_cuda.h rename to platforms/cuda/libpjrt_cuda.h diff --git a/platforms/cuda/libzml_cuda.BUILD.bazel b/platforms/cuda/libzml_cuda.BUILD.bazel deleted file mode 100644 index e79ab900..00000000 --- a/platforms/cuda/libzml_cuda.BUILD.bazel +++ /dev/null @@ -1,108 +0,0 @@ -load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@zml//bazel:patchelf.bzl", "patchelf") -load("@zml//platforms/cuda:cuda.bzl", "CUDA_COMPAT_FILES") - -patchelf( - name = "libzml_cuda_so", - src = "lib/libzml_cuda.so", - add_needed = [ - "libzmlxcuda.so.0", - ], - rename_dynamic_symbols = { - "dlopen": "zmlxcuda_dlopen", - }, - set_rpath = "$ORIGIN", -) - -copy_to_directory( - name = "sandbox", - srcs = [ - "lib/cutlass_ir.so", - ":libzml_cuda_so", - "@zml//platforms/cuda:compat_probe", - "@zml//platforms/cuda:zmlxcuda", - ] + select({ - "@llvm//platforms/config:linux_x86_64": [ - "@cuda_compat_linux_x86_64//:cuda_compat", - "@cuda_cudart_linux_x86_64//:cuda_cudart", - "@cuda_cupti_linux_x86_64//:cuda_cupti", - "@cuda_nvcc_linux_x86_64//:cuda_nvcc", - "@cuda_nvrtc_linux_x86_64//:cuda_nvrtc", - "@cuda_nvtx_linux_x86_64//:cuda_nvtx", - "@cuda_tileiras_linux_x86_64//:cuda_tileiras", - "@cudnn_linux_x86_64//:cudnn", - "@libcublas_linux_x86_64//:libcublas", - "@libcufft_linux_x86_64//:libcufft", - "@libcusolver_linux_x86_64//:libcusolver", - "@libcusparse_linux_x86_64//:libcusparse", - "@libnvjitlink_linux_x86_64//:libnvjitlink", - "@libnvshmem_linux_x86_64//:libnvshmem", - "@libnvvm_linux_x86_64//:libnvvm", - "@llvm-libunwind1_linux_amd64//:libunwind", - "@nccl_linux_amd64//:nccl", - "@zlib1g_linux_amd64//:zlib1g", - ], - "@llvm//platforms/config:linux_aarch64": [ - "@cuda_compat_linux_sbsa//:cuda_compat", - "@cuda_cudart_linux_sbsa//:cuda_cudart", - "@cuda_cupti_linux_sbsa//:cuda_cupti", - "@cuda_nvcc_linux_sbsa//:cuda_nvcc", - "@cuda_nvrtc_linux_sbsa//:cuda_nvrtc", - "@cuda_nvtx_linux_sbsa//:cuda_nvtx", - "@cuda_tileiras_linux_sbsa//:cuda_tileiras", - "@cudnn_linux_sbsa//:cudnn", - "@libcublas_linux_sbsa//:libcublas", - "@libcufft_linux_sbsa//:libcufft", - "@libcusolver_linux_sbsa//:libcusolver", - "@libcusparse_linux_sbsa//:libcusparse", - "@libnvjitlink_linux_sbsa//:libnvjitlink", - "@libnvshmem_linux_sbsa//:libnvshmem", - "@libnvvm_linux_sbsa//:libnvvm", - "@llvm-libunwind1_linux_arm64//:libunwind", - "@nccl_linux_arm64//:nccl", - "@zlib1g_linux_arm64//:zlib1g", - ], - }), - add_directory_to_runfiles = False, - include_external_repositories = ["**"], - replace_prefixes = { - "compat": "lib/compat", - "nvidia/nccl/lib": "lib", - "nvvm/lib64": "lib", - "libzml_cuda_so": "lib", - "lib/x86_64-linux-gnu": "lib", - "lib/aarch64-linux-gnu": "lib", - "usr/lib/x86_64-linux-gnu": "lib", - "usr/lib/aarch64-linux-gnu": "lib", - "platforms/cuda/compat_probe": "bin/compat_probe", - "platforms/cuda/libzmlxcuda": "lib/libzmlxcuda", - } | { - "{}.patchelf".format(file): "lib/compat" - for file in CUDA_COMPAT_FILES - }, -) - -cc_library( - name = "libzml_cuda", - data = [":sandbox"], - linkopts = [ - # Defer function call resolution until the function is called - # (lazy loading) rather than at load time. - # - # This is required because we want to let downstream use weak CUDA symbols. - # - # We force it here because -z,now (which resolve all symbols at load time), - # is the default in most bazel CC toolchains as well as in certain linkers. - "-Wl,-z,lazy", - ], - visibility = ["@zml//platforms/cuda:__subpackages__"], - deps = select({ - "@llvm//platforms/config:linux_x86_64": [ - "@cuda_cudart_linux_x86_64//:cuda", - ], - "@llvm//platforms/config:linux_aarch64": [ - "@cuda_cudart_linux_sbsa//:cuda", - ], - }), -) diff --git a/platforms/cuda/packages.lock.json b/platforms/cuda/packages.lock.json index 3563bc61..b80a2c36 100755 --- a/platforms/cuda/packages.lock.json +++ b/platforms/cuda/packages.lock.json @@ -19,13 +19,13 @@ "version": "12.2.0-14+deb12u1" } ], - "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_amd64", - "name": "llvm-libunwind1", - "sha256": "04cb76d70da569e5bcf2395f28dd7e9b9e9f0bf4521cf3b008b0f62ae9d2b712", + "key": "zlib1g_1-1.2.13.dfsg-1_amd64", + "name": "zlib1g", + "sha256": "d7dd1d1411fedf27f5e27650a6eff20ef294077b568f4c8c5e51466dc7c08ce4", "urls": [ - "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_amd64.deb" + "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb" ], - "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" + "version": "1:1.2.13.dfsg-1" }, { "arch": "amd64", @@ -59,228 +59,6 @@ "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_amd64.deb" ], "version": "12.2.0-14+deb12u1" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libc6_2.36-9-p-deb12u10_amd64", - "name": "libc6", - "version": "2.36-9+deb12u10" - }, - { - "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", - "name": "libgcc-s1", - "version": "12.2.0-14+deb12u1" - }, - { - "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", - "name": "gcc-12-base", - "version": "12.2.0-14+deb12u1" - } - ], - "key": "zlib1g_1-1.2.13.dfsg-1_amd64", - "name": "zlib1g", - "sha256": "d7dd1d1411fedf27f5e27650a6eff20ef294077b568f4c8c5e51466dc7c08ce4", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_amd64.deb" - ], - "version": "1:1.2.13.dfsg-1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "cuda-compat-13-4_615.71.09-2ubuntu1_amd64", - "name": "cuda-compat-13-4", - "sha256": "1b814aa705b4dfe955420c967bd545f39ef7b67f32bd0f285a005ea296b02831", - "urls": [ - "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/./cuda-compat-13-4_615.71.09-2ubuntu1_amd64.deb" - ], - "version": "615.71.09-2ubuntu1" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", - "name": "libstdc++6", - "version": "12.2.0-14+deb12u1" - }, - { - "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", - "name": "libgcc-s1", - "version": "12.2.0-14+deb12u1" - }, - { - "key": "libc6_2.36-9-p-deb12u10_amd64", - "name": "libc6", - "version": "2.36-9+deb12u10" - }, - { - "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", - "name": "gcc-12-base", - "version": "12.2.0-14+deb12u1" - } - ], - "key": "libnccl2_2.32.3-1-p-cuda13.4_amd64", - "name": "libnccl2", - "sha256": "55a3e32a3f560f20e2f423cd38f1f284951e92093d1e8fbaabb4c42ac76f3195", - "urls": [ - "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/./libnccl2_2.32.3-1+cuda13.4_amd64.deb" - ], - "version": "2.32.3-1+cuda13.4" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", - "name": "libstdc++6", - "sha256": "5cd3171216d4ab0fc911cfe9c35509bf2dd8f47761c43b7f6a4296701551a24d", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_amd64.deb" - ], - "version": "12.2.0-14+deb12u1" - }, - { - "arch": "arm64", - "dependencies": [ - { - "key": "libc6_2.36-9-p-deb12u10_arm64", - "name": "libc6", - "version": "2.36-9+deb12u10" - }, - { - "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", - "name": "libgcc-s1", - "version": "12.2.0-14+deb12u1" - }, - { - "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", - "name": "gcc-12-base", - "version": "12.2.0-14+deb12u1" - } - ], - "key": "llvm-libunwind1_1-22.1.8_-p--p-20260714014842-p-ca7933e47d3a-1_exp1_20260714134950.86_arm64", - "name": "llvm-libunwind1", - "sha256": "05890076ab166d0fe3eab3ab08f9076bca1edc7322d123df8970760ae26c25e8", - "urls": [ - "https://apt.llvm.org/bookworm/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86_arm64.deb" - ], - "version": "1:22.1.8~++20260714014842+ca7933e47d3a-1~exp1~20260714134950.86" - }, - { - "arch": "arm64", - "dependencies": [], - "key": "libc6_2.36-9-p-deb12u10_arm64", - "name": "libc6", - "sha256": "9b25c7c2aaa9328b2cfd92b47cc48b24631bfdff0c61afe05cba937911fcf239", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/glibc/libc6_2.36-9+deb12u10_arm64.deb" - ], - "version": "2.36-9+deb12u10" - }, - { - "arch": "arm64", - "dependencies": [], - "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", - "name": "libgcc-s1", - "sha256": "576926b283613db80168ddf76380a3bd877602778cf0d226caa7bfbfa71eacf3", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_arm64.deb" - ], - "version": "12.2.0-14+deb12u1" - }, - { - "arch": "arm64", - "dependencies": [], - "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", - "name": "gcc-12-base", - "sha256": "674cf6cba6d432bd200c45fe866c1652c7a53523cc2e7a613e05bc4abf7b5440", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.2.0-14+deb12u1_arm64.deb" - ], - "version": "12.2.0-14+deb12u1" - }, - { - "arch": "arm64", - "dependencies": [ - { - "key": "libc6_2.36-9-p-deb12u10_arm64", - "name": "libc6", - "version": "2.36-9+deb12u10" - }, - { - "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", - "name": "libgcc-s1", - "version": "12.2.0-14+deb12u1" - }, - { - "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", - "name": "gcc-12-base", - "version": "12.2.0-14+deb12u1" - } - ], - "key": "zlib1g_1-1.2.13.dfsg-1_arm64", - "name": "zlib1g", - "sha256": "52b8b8a145bbe1956bba82034f77022cbef0c3d0885c9e32d9817a7932fe1913", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/z/zlib/zlib1g_1.2.13.dfsg-1_arm64.deb" - ], - "version": "1:1.2.13.dfsg-1" - }, - { - "arch": "arm64", - "dependencies": [], - "key": "cuda-compat-13-4_615.71.09-2ubuntu1_arm64", - "name": "cuda-compat-13-4", - "sha256": "83cdca90db73a36784a077693e3b48565d852158c63f604194c1f0d42c2e5458", - "urls": [ - "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/./cuda-compat-13-4_615.71.09-2ubuntu1_arm64.deb" - ], - "version": "615.71.09-2ubuntu1" - }, - { - "arch": "arm64", - "dependencies": [ - { - "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_arm64", - "name": "libstdc++6", - "version": "12.2.0-14+deb12u1" - }, - { - "key": "libgcc-s1_12.2.0-14-p-deb12u1_arm64", - "name": "libgcc-s1", - "version": "12.2.0-14+deb12u1" - }, - { - "key": "libc6_2.36-9-p-deb12u10_arm64", - "name": "libc6", - "version": "2.36-9+deb12u10" - }, - { - "key": "gcc-12-base_12.2.0-14-p-deb12u1_arm64", - "name": "gcc-12-base", - "version": "12.2.0-14+deb12u1" - } - ], - "key": "libnccl2_2.32.3-1-p-cuda13.4_arm64", - "name": "libnccl2", - "sha256": "809a6eea2f8a570050229bc218e30d4280829cbe01e824e052c96ee9203a7fe5", - "urls": [ - "https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/./libnccl2_2.32.3-1+cuda13.4_arm64.deb" - ], - "version": "2.32.3-1+cuda13.4" - }, - { - "arch": "arm64", - "dependencies": [], - "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_arm64", - "name": "libstdc++6", - "sha256": "26e138c677a985775331373828a6c286c551ff397cb735d00e2383cb273d1cb2", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_arm64.deb" - ], - "version": "12.2.0-14+deb12u1" } ], "version": 1 diff --git a/platforms/cuda/packages.yaml b/platforms/cuda/packages.yaml index 010ca572..4b3b04c0 100644 --- a/platforms/cuda/packages.yaml +++ b/platforms/cuda/packages.yaml @@ -6,21 +6,9 @@ version: 1 sources: - channel: bookworm main url: https://snapshot-cloudflare.debian.org/archive/debian/20250711T030400Z - - channel: llvm-toolchain-bookworm-22 main - url: https://apt.llvm.org/bookworm - - channel: / - url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64 - archs: [amd64] - - channel: / - url: https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa - archs: [arm64] archs: - - "amd64" - - "arm64" + - "amd64" packages: - - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-bookworm-22. - - "zlib1g" - - "cuda-compat-13-4" - - "libnccl2" + - "zlib1g" diff --git a/platforms/furiosa/BUILD.bazel b/platforms/furiosa/BUILD.bazel deleted file mode 100644 index 3118e6d0..00000000 --- a/platforms/furiosa/BUILD.bazel +++ /dev/null @@ -1,21 +0,0 @@ -load("@rules_shell//shell:sh_test.bzl", "sh_test") -load("@rules_zig//zig:defs.bzl", "zig_library") - -exports_files(["libzml_furiosa.BUILD.bazel"]) - -zig_library( - name = "furiosa", - import_name = "platforms/furiosa", - main = "furiosa.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//bazel", - "//pjrt", - "//platforms:options", - "//stdx", - ] + select({ - "//platforms:furiosa.enabled": ["@libzml_furiosa"], - "//conditions:default": [], - }), -) diff --git a/platforms/furiosa/furiosa.bzl b/platforms/furiosa/furiosa.bzl deleted file mode 100644 index afb26864..00000000 --- a/platforms/furiosa/furiosa.bzl +++ /dev/null @@ -1,21 +0,0 @@ -"""Consume the complete Furiosa sandbox produced by XLA.""" - -load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") - -def _furiosa_impl(mctx): - http_archive( - name = "libzml_furiosa", - build_file = "libzml_furiosa.BUILD.bazel", - url = "https://mirror.zml.ai/plugins/202611131503.373.1.be246d178d7d/zml-furiosa-linux-amd64.tar.zst", - sha256 = "c7cdf0120818589aee3b53d8a6813e2b5e244447701acc34eb697b30f46e22f9", - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["libzml_furiosa"], - root_module_direct_dev_deps = [], - ) - -furiosa_packages = module_extension( - implementation = _furiosa_impl, -) diff --git a/platforms/furiosa/furiosa.zig b/platforms/furiosa/furiosa.zig deleted file mode 100644 index ee33fd58..00000000 --- a/platforms/furiosa/furiosa.zig +++ /dev/null @@ -1,57 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const bazel = @import("bazel"); -const bazel_builtin = @import("bazel_builtin"); -const pjrt = @import("pjrt"); -const platforms_options = @import("platforms/options"); -const stdx = @import("stdx"); - -const log = std.log.scoped(.@"zml/platforms/furiosa"); - -extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; - -fn setEnv(name: [*:0]const u8, value: [:0]const u8, overwrite: c_int) !void { - if (setenv(name, value, overwrite) != 0) return error.SetEnvFailed; -} - -pub fn isEnabled() bool { - return platforms_options.furiosa_enabled; -} - -fn hasFuriosaDevice(io: std.Io) bool { - std.Io.Dir.accessAbsolute(io, "/dev/rngd", .{ .read = true }) catch return false; - return true; -} - -fn hasDmaHeapAccess(io: std.Io) bool { - std.Io.Dir.accessAbsolute(io, "/dev/dma_heap/system", .{ .read = true }) catch return false; - return true; -} - -pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - if (comptime !isEnabled() or builtin.os.tag != .linux or builtin.cpu.arch != .x86_64) return error.Unavailable; - if (!hasFuriosaDevice(io)) { - return error.Unavailable; - } - if (!hasDmaHeapAccess(io)) { - std.debug.panic("Furiosa needs read access to /dev/dma_heap/system for pinned host memory", .{}); - } - - const r = try bazel.runfiles(bazel_builtin.current_repository); - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox = try r.rlocation("libzml_furiosa/sandbox", &path_buf) orelse { - log.err("Missing Furiosa sandbox runfile", .{}); - return error.FileNotFound; - }; - - const root = try std.Io.Dir.cwd().realPathFileAlloc(io, sandbox, allocator); - defer allocator.free(root); - - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - try setEnv("FURIOSA_SDK_ROOT", try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{root}), 1); - - const library = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ root, "lib", "libzml_furiosa.so" }); - log.info("Loading Furiosa plugin: {s}", .{library}); - return .loadFrom(library); -} diff --git a/platforms/furiosa/libzml_furiosa.BUILD.bazel b/platforms/furiosa/libzml_furiosa.BUILD.bazel deleted file mode 100644 index 3a2da19a..00000000 --- a/platforms/furiosa/libzml_furiosa.BUILD.bazel +++ /dev/null @@ -1,23 +0,0 @@ -load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -# XLA owns the full payload. ZML only places it in the platform's runfiles. -copy_to_directory( - name = "sandbox", - srcs = glob( - [ - "bin/**", - "include/**", - "lib/**", - ], - allow_empty = False, - ), - add_directory_to_runfiles = True, - visibility = ["//visibility:public"], -) - -cc_library( - name = "libzml_furiosa", - data = [":sandbox"], - visibility = ["//visibility:public"], -) diff --git a/platforms/metal/BUILD.bazel b/platforms/metal/BUILD.bazel deleted file mode 100644 index 48bca771..00000000 --- a/platforms/metal/BUILD.bazel +++ /dev/null @@ -1,45 +0,0 @@ -load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library") - -copy_to_directory( - name = "metal_toolchain", - srcs = [ - "@metal_toolchain//:metal_compile_runtime", - "@metal_toolchain//:metal_compile_tools", - ], - out = "metal_toolchain", - include_external_repositories = ["**"], - replace_prefixes = { - "Metal.xctoolchain/usr/metal/32023": ".", - }, - target_compatible_with = ["@platforms//os:macos"], -) - -cc_library( - name = "libzml_metal", - hdrs = ["metal.h"], - data = [":metal_toolchain"], - tags = ["manual"], - deps = ["@libzml_metal"], -) - -zig_library( - name = "metal", - import_name = "platforms/metal", - main = "metal.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//pjrt", - "//platforms:options", - ] + select({ - "//platforms:metal.enabled": [ - ":libzml_metal", - "//bazel", - "//stdx", - "@rules_zig//zig/runfiles", - ], - "//conditions:default": [], - }), -) diff --git a/platforms/metal/libzml_metal.BUILD.bazel b/platforms/metal/libzml_metal.BUILD.bazel deleted file mode 100644 index 7bac767d..00000000 --- a/platforms/metal/libzml_metal.BUILD.bazel +++ /dev/null @@ -1,17 +0,0 @@ -load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -copy_to_directory( - name = "sandbox", - srcs = [ - "lib/libzml_metal.dylib", - ], - add_directory_to_runfiles = False, - include_external_repositories = ["**"], -) - -cc_library( - name = "libzml_metal", - data = [":sandbox"], - visibility = ["@zml//platforms/metal:__subpackages__"], -) diff --git a/platforms/metal/metal.bzl b/platforms/metal/metal.bzl deleted file mode 100644 index 71e1858a..00000000 --- a/platforms/metal/metal.bzl +++ /dev/null @@ -1,19 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -def _metal_impl(mctx): - http_archive( - name = "libzml_metal", - build_file = "libzml_metal.BUILD.bazel", - sha256 = "89881d1a1db336e3337a9373db2c4e94f71ddfb77619dee75c0c12e2903913ae", - url = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-metal-darwin-arm64.tar.zst", - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["libzml_metal"], - root_module_direct_dev_deps = [], - ) - -metal_packages = module_extension( - implementation = _metal_impl, -) diff --git a/platforms/metal/metal.h b/platforms/metal/metal.h deleted file mode 100644 index 933e4f31..00000000 --- a/platforms/metal/metal.h +++ /dev/null @@ -1 +0,0 @@ - #include diff --git a/platforms/metal/metal.zig b/platforms/metal/metal.zig deleted file mode 100644 index ddde2a30..00000000 --- a/platforms/metal/metal.zig +++ /dev/null @@ -1,53 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const bazel = @import("bazel"); -const bazel_builtin = @import("bazel_builtin"); -const c = @import("c"); -const pjrt = @import("pjrt"); -const platforms_options = @import("platforms/options"); -const stdx = @import("stdx"); - -const log = std.log.scoped(.@"zml/platforms/metal"); - -pub fn isEnabled() bool { - return platforms_options.metal_enabled; -} - -fn setMetalToolchainEnv(r: anytype) !void { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const root = try r.rlocation("zml/platforms/metal/metal_toolchain", &buf) orelse - return error.ToolchainNotInRunfiles; - - var bindir_z: [std.Io.Dir.max_path_bytes]u8 = undefined; - _ = c.setenv("METAL_TOOLCHAIN", try stdx.Io.Dir.path.bufJoinZ(&bindir_z, &.{ root, "bin" }), 0); -} - -pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = io; - if (comptime !isEnabled()) { - return error.Unavailable; - } - - if (comptime builtin.os.tag != .macos) { - return error.Unavailable; - } - - const r = try bazel.runfiles(bazel_builtin.current_repository); - - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libzml_metal/sandbox", &path_buf) orelse { - log.err("Failed to find sandbox path for Metal runtime", .{}); - return error.FileNotFound; - }; - - setMetalToolchainEnv(r) catch |err| { - log.err("Bundled Metal toolchain not found in runfiles ({any})", .{err}); - }; - - return blk: { - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_metal.dylib" }); - break :blk pjrt.Api.loadFrom(path); - }; -} diff --git a/platforms/metal/metal_toolchain.BUILD.bazel b/platforms/metal/metal_toolchain.BUILD.bazel deleted file mode 100644 index ca70fff0..00000000 --- a/platforms/metal/metal_toolchain.BUILD.bazel +++ /dev/null @@ -1,382 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -# Static BUILD file for the 17B54 MetalToolchain asset. - -filegroup( - name = "toolchain_runtime", - srcs = glob(["Metal.xctoolchain/**"]), -) - -exports_files([ - "Metal.xctoolchain/ToolchainInfo.plist", - "Metal.xctoolchain/usr/bin/air-ar", - "Metal.xctoolchain/usr/bin/air-arch", - "Metal.xctoolchain/usr/bin/air-as", - "Metal.xctoolchain/usr/bin/air-binary-perf", - "Metal.xctoolchain/usr/bin/air-config", - "Metal.xctoolchain/usr/bin/air-dsymutil", - "Metal.xctoolchain/usr/bin/air-libtool", - "Metal.xctoolchain/usr/bin/air-link", - "Metal.xctoolchain/usr/bin/air-lipo", - "Metal.xctoolchain/usr/bin/air-lld", - "Metal.xctoolchain/usr/bin/air-nm", - "Metal.xctoolchain/usr/bin/air-nt", - "Metal.xctoolchain/usr/bin/air-objcopy", - "Metal.xctoolchain/usr/bin/air-objdump", - "Metal.xctoolchain/usr/bin/air-opt", - "Metal.xctoolchain/usr/bin/air-pack", - "Metal.xctoolchain/usr/bin/air-package", - "Metal.xctoolchain/usr/bin/air-ranlib", - "Metal.xctoolchain/usr/bin/air-readobj", - "Metal.xctoolchain/usr/bin/air-size", - "Metal.xctoolchain/usr/bin/air-strip", - "Metal.xctoolchain/usr/bin/air-tt", - "Metal.xctoolchain/usr/bin/air-vtool", - "Metal.xctoolchain/usr/bin/amdgpu-nt", - "Metal.xctoolchain/usr/bin/applegpu-nt", - "Metal.xctoolchain/usr/bin/intelgpu-nt", - "Metal.xctoolchain/usr/bin/metal", - "Metal.xctoolchain/usr/bin/metal-ar", - "Metal.xctoolchain/usr/bin/metal-arch", - "Metal.xctoolchain/usr/bin/metal-as", - "Metal.xctoolchain/usr/bin/metal-binary-perf", - "Metal.xctoolchain/usr/bin/metal-config", - "Metal.xctoolchain/usr/bin/metal-dsymutil", - "Metal.xctoolchain/usr/bin/metal-libtool", - "Metal.xctoolchain/usr/bin/metal-link", - "Metal.xctoolchain/usr/bin/metal-lipo", - "Metal.xctoolchain/usr/bin/metal-lld", - "Metal.xctoolchain/usr/bin/metal-nm", - "Metal.xctoolchain/usr/bin/metal-nt", - "Metal.xctoolchain/usr/bin/metal-objcopy", - "Metal.xctoolchain/usr/bin/metal-objdump", - "Metal.xctoolchain/usr/bin/metal-opt", - "Metal.xctoolchain/usr/bin/metal-pack", - "Metal.xctoolchain/usr/bin/metal-package", - "Metal.xctoolchain/usr/bin/metal-ranlib", - "Metal.xctoolchain/usr/bin/metal-readobj", - "Metal.xctoolchain/usr/bin/metal-size", - "Metal.xctoolchain/usr/bin/metal-source", - "Metal.xctoolchain/usr/bin/metal-strip", - "Metal.xctoolchain/usr/bin/metal-tt", - "Metal.xctoolchain/usr/bin/metal-vtool", - "Metal.xctoolchain/usr/bin/metallib", - "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libAMDIL902.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libGCNASM.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/AMDShared.bundle/Contents/PlugIns/libSC.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libAMDIL902.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libGCNASM.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/amd_13/AMDShared.bundle/Contents/PlugIns/libSC.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/amd_13/libAMDNTPlugin.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_ios.metallib", - "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_iosmac.metallib", - "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_iossim.metallib", - "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_osx.metallib", - "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_tvos.metallib", - "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_tvossim.metallib", - "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_watchos.metallib", - "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_watchossim.metallib", - "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_xros.metallib", - "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/lib/darwin/libtracepoint_rt_xrossim.metallib", - "Metal.xctoolchain/usr/metal/32023/lib/libAMDNTPlugin.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/libMTLIGCCompilerPlugin.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/libMTLPasses.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu-nt.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu23-nt.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/libapplegpu24-nt.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/libapplegpuG9G12-nt.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/libigc.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/metalfe/libmetal.dylib", - "Metal.xctoolchain/usr/metal/32023/lib/runtime.gpuencode.split", - "Metal.xctoolchain/usr/metal/32023/lib/runtime.igc64.split", - "Metal.xctoolchain/usr/metal/32023/lib/wrapper-nt.dylib", -]) - -alias(name = "toolchain_info", actual = "Metal.xctoolchain/ToolchainInfo.plist") - -alias( - name = "air-ar", - actual = "Metal.xctoolchain/usr/bin/air-ar", -) - -alias( - name = "air-arch", - actual = "Metal.xctoolchain/usr/bin/air-arch", -) - -alias( - name = "air-as", - actual = "Metal.xctoolchain/usr/bin/air-as", -) - -alias( - name = "air-binary-perf", - actual = "Metal.xctoolchain/usr/bin/air-binary-perf", -) - -alias( - name = "air-config", - actual = "Metal.xctoolchain/usr/bin/air-config", -) - -alias( - name = "air-dsymutil", - actual = "Metal.xctoolchain/usr/bin/air-dsymutil", -) - -alias( - name = "air-libtool", - actual = "Metal.xctoolchain/usr/bin/air-libtool", -) - -alias( - name = "air-link", - actual = "Metal.xctoolchain/usr/bin/air-link", -) - -alias( - name = "air-lipo", - actual = "Metal.xctoolchain/usr/bin/air-lipo", -) - -alias( - name = "air-lld", - actual = "Metal.xctoolchain/usr/bin/air-lld", -) - -alias( - name = "air-nm", - actual = "Metal.xctoolchain/usr/bin/air-nm", -) - -alias( - name = "air-nt", - actual = "Metal.xctoolchain/usr/bin/air-nt", -) - -alias( - name = "air-objcopy", - actual = "Metal.xctoolchain/usr/bin/air-objcopy", -) - -alias( - name = "air-objdump", - actual = "Metal.xctoolchain/usr/bin/air-objdump", -) - -alias( - name = "air-opt", - actual = "Metal.xctoolchain/usr/bin/air-opt", -) - -alias( - name = "air-pack", - actual = "Metal.xctoolchain/usr/bin/air-pack", -) - -alias( - name = "air-package", - actual = "Metal.xctoolchain/usr/bin/air-package", -) - -alias( - name = "air-ranlib", - actual = "Metal.xctoolchain/usr/bin/air-ranlib", -) - -alias( - name = "air-readobj", - actual = "Metal.xctoolchain/usr/bin/air-readobj", -) - -alias( - name = "air-size", - actual = "Metal.xctoolchain/usr/bin/air-size", -) - -alias( - name = "air-strip", - actual = "Metal.xctoolchain/usr/bin/air-strip", -) - -alias( - name = "air-tt", - actual = "Metal.xctoolchain/usr/bin/air-tt", -) - -alias( - name = "air-vtool", - actual = "Metal.xctoolchain/usr/bin/air-vtool", -) - -alias( - name = "amdgpu-nt", - actual = "Metal.xctoolchain/usr/bin/amdgpu-nt", -) - -alias( - name = "applegpu-nt", - actual = "Metal.xctoolchain/usr/bin/applegpu-nt", -) - -alias( - name = "intelgpu-nt", - actual = "Metal.xctoolchain/usr/bin/intelgpu-nt", -) - -alias( - name = "metal", - actual = "Metal.xctoolchain/usr/bin/metal", -) - -alias( - name = "metal-ar", - actual = "Metal.xctoolchain/usr/bin/metal-ar", -) - -alias( - name = "metal-arch", - actual = "Metal.xctoolchain/usr/bin/metal-arch", -) - -alias( - name = "metal-as", - actual = "Metal.xctoolchain/usr/bin/metal-as", -) - -alias( - name = "metal-binary-perf", - actual = "Metal.xctoolchain/usr/bin/metal-binary-perf", -) - -alias( - name = "metal-config", - actual = "Metal.xctoolchain/usr/bin/metal-config", -) - -alias( - name = "metal-dsymutil", - actual = "Metal.xctoolchain/usr/bin/metal-dsymutil", -) - -alias( - name = "metal-libtool", - actual = "Metal.xctoolchain/usr/bin/metal-libtool", -) - -alias( - name = "metal-link", - actual = "Metal.xctoolchain/usr/bin/metal-link", -) - -alias( - name = "metal-lipo", - actual = "Metal.xctoolchain/usr/bin/metal-lipo", -) - -alias( - name = "metal-lld", - actual = "Metal.xctoolchain/usr/bin/metal-lld", -) - -alias( - name = "metal-nm", - actual = "Metal.xctoolchain/usr/bin/metal-nm", -) - -alias( - name = "metal-nt", - actual = "Metal.xctoolchain/usr/bin/metal-nt", -) - -alias( - name = "metal-objcopy", - actual = "Metal.xctoolchain/usr/bin/metal-objcopy", -) - -alias( - name = "metal-objdump", - actual = "Metal.xctoolchain/usr/bin/metal-objdump", -) - -alias( - name = "metal-opt", - actual = "Metal.xctoolchain/usr/bin/metal-opt", -) - -alias( - name = "metal-pack", - actual = "Metal.xctoolchain/usr/bin/metal-pack", -) - -alias( - name = "metal-package", - actual = "Metal.xctoolchain/usr/bin/metal-package", -) - -alias( - name = "metal-ranlib", - actual = "Metal.xctoolchain/usr/bin/metal-ranlib", -) - -alias( - name = "metal-readobj", - actual = "Metal.xctoolchain/usr/bin/metal-readobj", -) - -alias( - name = "metal-size", - actual = "Metal.xctoolchain/usr/bin/metal-size", -) - -alias( - name = "metal-source", - actual = "Metal.xctoolchain/usr/bin/metal-source", -) - -alias( - name = "metal-strip", - actual = "Metal.xctoolchain/usr/bin/metal-strip", -) - -alias( - name = "metal-tt", - actual = "Metal.xctoolchain/usr/bin/metal-tt", -) - -alias( - name = "metal-vtool", - actual = "Metal.xctoolchain/usr/bin/metal-vtool", -) - -alias( - name = "metallib", - actual = "Metal.xctoolchain/usr/bin/metallib", -) - -filegroup( - name = "metal_headers", - srcs = glob(["Metal.xctoolchain/usr/metal/*/lib/clang/*/include/**/*.h"]), -) - -filegroup( - name = "tracepoint_runtime_metallibs", - srcs = glob(["Metal.xctoolchain/usr/metal/*/lib/clang/*/lib/darwin/*.metallib"]), -) - -filegroup( - name = "metal_compile_tools", - srcs = [ - "Metal.xctoolchain/usr/metal/32023/bin/metal", - "Metal.xctoolchain/usr/metal/32023/bin/air-opt", - "Metal.xctoolchain/usr/metal/32023/bin/air-as", - "Metal.xctoolchain/usr/metal/32023/bin/metallib", - ], -) - -filegroup( - name = "metal_compile_runtime", - srcs = glob([ - "Metal.xctoolchain/usr/metal/32023/lib/clang/32023.830/**", - ]), -) diff --git a/platforms/neuron/BUILD.bazel b/platforms/neuron/BUILD.bazel index ad537191..e095d025 100644 --- a/platforms/neuron/BUILD.bazel +++ b/platforms/neuron/BUILD.bazel @@ -1,32 +1,21 @@ +load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library", "zig_shared_library") -load("@zml//bazel:cc_import.bzl", "cc_import") load("@zml//bazel:runfiles.bzl", "runfiles_to_default") -exports_files([ - "neuronx_cc_shim.py", - "nki_kernel_compiler.py", -]) - # A proxy PJRT Plugin that loads the Neuron PJRT Plugin # and returns the instance from nested GetPjrtApi. # -# The proxy also initializes the Neuron runtime environment and exposes the -# sandboxed compiler binaries to libneuronxla 3.0. +# Additionally, it provides a way to load implicit transitive dependencies +# of neuronx-cc (see add_needed of the patchelf target below). zig_shared_library( - name = "libzml_neuron", - srcs = ["neuron.zig"], - main = "libzml_neuron.zig", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["@libzml_neuron//:__subpackages__"], + name = "libpjrt_neuron", + main = "libpjrt_neuron.zig", + visibility = ["@libpjrt_neuron//:__subpackages__"], zigopts = ["-fno-stack-check"], deps = [ - ":libnrt_headers", - ":neuron_nrt", - "//bazel", - "//pjrt", - "//platforms:options", + ":libpython", "//stdx", "@rules_zig//zig/runfiles", "@xla//xla/pjrt/c:pjrt_c_api_hdrs", @@ -37,109 +26,67 @@ zig_binary( name = "neuronx-cc", data = ["@neuron_py_deps//neuronx_cc"], linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], - main = "neuronxcc_compiler_launcher.zig", + main = "neuronx-cc.zig", tags = ["manual"], - target_compatible_with = ["@platforms//os:linux"], - deps = [ - ":libpython", - ":python_launcher", - ], + deps = [":libpython"], ) runfiles_to_default( name = "neuronx-cc_files", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["@libzml_neuron//:__subpackages__"], + visibility = ["@libpjrt_neuron//:__subpackages__"], deps = [":neuronx-cc"], ) -runfiles_to_default( - name = "nki-cc_files", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["@libzml_neuron//:__subpackages__"], - deps = [":nki-cc"], -) - cc_library( name = "libpython", hdrs = ["libpython.h"], - tags = ["manual"], - visibility = ["//platforms/neuron:__subpackages__"], deps = [ "@rules_python//python/cc:current_py_cc_headers", "@rules_python//python/cc:current_py_cc_libs", ], ) +cc_library( + name = "empty", + defines = ["ZML_RUNTIME_NEURON_DISABLED"], +) + +cc_library( + name = "zmlxneuron", + defines = ["ZML_RUNTIME_NEURON"], +) + cc_library( name = "libnrt_headers", hdrs = ["nrt.h"], - tags = ["manual"], - visibility = [ - "//bin/zml-smi:__subpackages__", - ], - deps = ["@libzml_neuron//:libnrt_headers"], + deps = ["@libpjrt_neuron//:libnrt_headers"], ) -cc_import( - name = "neuron_ncfw", - shared_library = "@aws-neuronx-runtime-lib//:libncfw.patchelf", - tags = ["manual"], - target_compatible_with = [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], +upb_c_proto_library( + name = "xla_data_upb", + deps = ["@xla//xla:xla_data_proto"], ) -cc_import( - name = "neuron_nrt", - shared_library = "@aws-neuronx-runtime-lib//:libnrt.patchelf", - tags = ["manual"], - target_compatible_with = [ - "@platforms//cpu:x86_64", - "@platforms//os:linux", - ], - deps = [":neuron_ncfw"], +upb_c_proto_library( + name = "hlo_proto_upb", + deps = ["@xla//xla/service:hlo_proto"], ) -zig_library( - name = "python_launcher", - import_name = "platforms/neuron/python_launcher", - main = "python_launcher.zig", - tags = ["manual"], - visibility = ["//platforms/neuron:__subpackages__"], - deps = [":libpython"], -) - -zig_binary( - name = "nki-cc", - data = [ - "nki_kernel_compiler.py", - "@neuron_py_deps//neuronx_cc", - "@neuron_py_deps//nki", +zig_shared_library( + name = "libneuronxla", + main = "libneuronxla.zig", + shared_lib_name = "libneuronxla.so", + visibility = ["@libpjrt_neuron//:__subpackages__"], + zigopts = [ + "-fno-stack-check", + "-fPIC", ], - linkopts = ["-Wl,-rpath,$$ORIGIN/../lib"], - main = "nki_compiler_launcher.zig", - tags = ["manual"], - target_compatible_with = ["@platforms//os:linux"], - visibility = ["//platforms/neuron:__pkg__"], deps = [ + ":hlo_proto_upb", ":libpython", - ":python_launcher", - ], -) - -zig_library( - name = "nki_kernel", - import_name = "platforms/neuron/nki_kernel", - main = "nki_kernel.zig", - tags = ["manual"], - visibility = ["//zml:__pkg__"], - deps = [ - ":neuron", - "//bazel", + ":xla_data_upb", "//stdx", - "@rules_zig//zig/runfiles", + "//upb", ], ) @@ -154,6 +101,7 @@ uv_lock( "--upgrade", "--python-platform=x86_64-unknown-linux-gnu", ], + python_version = "3.12", tags = ["manual"], ) @@ -161,32 +109,19 @@ zig_library( name = "neuron", import_name = "platforms/neuron", main = "neuron.zig", - tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", - "//platforms:options", ] + select({ "//platforms:neuron.enabled": [ ":libnrt_headers", - ":neuron_nrt", + ":libpython", + ":zmlxneuron", "//bazel", "//stdx", - "@libzml_neuron", + "@libpjrt_neuron", "@rules_zig//zig/runfiles", ], - "//conditions:default": [], + "//conditions:default": [":empty"], }), ) - -zig_library( - name = "topology", - data = [":libzml_neuron"], - import_name = "platforms/neuron/topology", - main = "topology.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - ":neuron", - ], -) diff --git a/platforms/neuron/libneuronxla.zig b/platforms/neuron/libneuronxla.zig new file mode 100644 index 00000000..8185cf56 --- /dev/null +++ b/platforms/neuron/libneuronxla.zig @@ -0,0 +1,256 @@ +const std = @import("std"); + +const c = @import("c"); +const stdx = @import("stdx"); +const upb = @import("upb"); + +const log = std.log.scoped(.@"zml/platforms/neuron/libneuronxla"); + +pub fn makeTempDir(io: std.Io, buf: []u8, prefix: []const u8) ![]const u8 { + const tmp_dir = std.posix.getenv("TMPDIR") orelse "/tmp"; + const ret = try std.fmt.bufPrint(buf, "{s}{s}{s}{d}", .{ + tmp_dir, + std.Io.Dir.path.sep_str_posix, + prefix, + (try std.Io.Clock.now(.real, io)).toNanoseconds(), + }); + try std.Io.Dir.createDir(.cwd(), io, ret, .fromMode(0o700)); + return ret; +} + +const ModuleState = struct { + threaded: std.Io.Threaded, +}; + +var module_def: c.PyModuleDef = .{ + .m_base = std.mem.zeroes(c.PyModuleDef_Base), + .m_doc = "Zig bindings for libneuronxla", + .m_name = "libneuronxla", + .m_size = @sizeOf(ModuleState), + .m_methods = @constCast(&[_]c.PyMethodDef{ + .{ + .ml_name = "hook", + .ml_meth = @ptrCast(&hook), + .ml_flags = c.METH_NOARGS, + .ml_doc = "Return a greeting from Zig.", + }, + .{ + .ml_name = "neuronx_cc", + .ml_meth = @ptrCast(&neuronx_cc), + .ml_flags = c.METH_FASTCALL, + .ml_doc = "Return a greeting from Zig.", + }, + std.mem.zeroes(c.PyMethodDef), + }), + .m_slots = @constCast(&[_]c.PyModuleDef_Slot{ + .{ .slot = c.Py_mod_exec, .value = @ptrCast(@constCast(&module_exec)) }, + std.mem.zeroes(c.PyModuleDef_Slot), + }), + .m_traverse = null, + .m_clear = null, + .m_free = null, +}; + +fn module_exec(module: ?*c.PyObject) callconv(.c) c_int { + const state: *ModuleState = @ptrCast(@alignCast(c.PyModule_GetState(module))); + state.* = .{ + .threaded = .init(std.heap.c_allocator, .{}), + }; + return 0; +} + +fn hook(self: ?*c.PyObject, args: ?*c.PyObject) callconv(.c) ?*c.PyObject { + _ = self; + _ = args; + const none = c.Py_None(); + defer c.Py_IncRef(none); + return none; +} + +pub fn PyBytes_AsStringAndSize(object: *c.PyObject) []u8 { + var buf: [*c]u8 = undefined; + var len: c.Py_ssize_t = undefined; + _ = c.PyBytes_AsStringAndSize(object, &buf, &len); + return buf[0..@intCast(len)]; +} + +fn wrapNeffAsCustomCall(allocator: std.mem.Allocator, io: std.Io, hlo_code: []const u8, neff_file_path: []const u8) ![]const u8 { + var upb_alloc: upb.Allocator = .init(allocator); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + + const hlo_module = try upb.parse(c.xla_HloModuleProto, upb_arena, hlo_code); + + const entry = blk: { + var size: usize = undefined; + const computations = c.xla_HloModuleProto_mutable_computations(hlo_module, &size)[0..size]; + for (computations) |comp| { + if (c.xla_HloComputationProto_id(comp) == c.xla_HloModuleProto_entry_computation_id(hlo_module)) { + break :blk comp; + } + } else return error.ComputationNotFound; + }; + + const entry_instructions = blk: { + var size: usize = undefined; + break :blk c.xla_HloComputationProto_instructions(entry, &size)[0..size]; + }; + c.xla_HloComputationProto_clear_instructions(entry); + + const fused_root = blk: { + for (entry_instructions) |instruction| { + if (c.xla_HloInstructionProto_id(instruction) == c.xla_HloComputationProto_root_id(entry)) { + break :blk try upb.shallowClone(c.xla_HloInstructionProto, upb_arena, instruction); + } + } else return error.ComputationNotFound; + }; + + c.xla_HloInstructionProto_set_opcode(fused_root, upb.stringView("custom-call")); + c.xla_HloInstructionProto_set_custom_call_target(fused_root, upb.stringView("AwsNeuronNeff")); + c.xla_HloInstructionProto_set_backend_config( + fused_root, + upb.stringView( + try stdx.Io.Dir.readFileAlloc(.cwd(), io, neff_file_path, allocator, .unlimited), + ), + ); + + const parameters_len = blk: { + var size: usize = undefined; + _ = c.xla_ProgramShapeProto_parameters( + c.xla_HloComputationProto_program_shape(entry), + &size, + ); + break :blk size; + }; + + { + var operand_ids: std.ArrayList(i64) = .initBuffer(c.xla_HloInstructionProto_resize_operand_ids(fused_root, parameters_len + 1, upb_arena)[0 .. parameters_len + 1]); + var new_instructions: std.ArrayList(*const c.xla_HloInstructionProto) = .initBuffer(@ptrCast(c.xla_HloComputationProto_resize_instructions(entry, parameters_len + 1, upb_arena)[0 .. parameters_len + 1])); + for (entry_instructions) |instruction| { + if (std.mem.eql(u8, upb.slice(c.xla_HloInstructionProto_opcode(instruction)) orelse continue, "parameter")) { + const id = c.xla_HloInstructionProto_id(instruction); + operand_ids.appendAssumeCapacity(id); + new_instructions.appendAssumeCapacity(instruction); + } + } + new_instructions.appendAssumeCapacity(fused_root); + } + + { + const fa = c.xla_HloInstructionProto_mutable_frontend_attributes(fused_root, upb_arena); + const map = c._xla_FrontendAttributes_map_mutable_upb_map(fa, upb_arena); + _ = c.upb_Map_Set( + map, + .{ .str_val = upb.stringView("valid_inputs") }, + .{ .str_val = blk: { + const valid_inputs_value = try allocator.alloc(u8, parameters_len * 2 - 1); + for (valid_inputs_value, 0..) |*char, i| { + char.* = if (i % 2 == 0) '1' else ','; + } + break :blk upb.stringView(valid_inputs_value); + } }, + upb_arena, + ); + } + + return try upb.serialize(hlo_module, upb_arena); +} + +fn neuronx_cc_(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) !?*c.PyObject { + const state: *ModuleState = @ptrCast(@alignCast(c.PyModule_GetState(self))); + const io = state.threaded.io(); + + var arena = std.heap.ArenaAllocator.init(std.heap.c_allocator); + defer arena.deinit(); + + const args = args_[0..@intCast(nargs_)]; + + const code = PyBytes_AsStringAndSize(args[0]); + const platform_version = PyBytes_AsStringAndSize(args[2]); + + const target = std.StaticStringMap([]const u8).initComptime(.{ + .{ "1.0", "inf1" }, + .{ "2.0", "trn1" }, + .{ "3.0", "trn2" }, + }).get(platform_version) orelse { + log.err("Unknown platform version: {s}\n", .{platform_version}); + return error.UnknownPlatformVersion; + }; + + var tmp_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const tmp_dir = try makeTempDir(io, &tmp_dir_buf, "zml-neuronxcc-"); + defer std.Io.Dir.deleteTree(.cwd(), io, tmp_dir) catch |err| { + log.err("Error deleting temporary directory {s}: {}\n", .{ tmp_dir, err }); + }; + + const code_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.code" }); + try std.Io.Dir.writeFile(.cwd(), io, .{ + .data = code, + .sub_path = code_file, + .flags = .{ .truncate = true }, + }); + + const neff_file = try std.Io.Dir.path.join(arena.allocator(), &.{ tmp_dir, "file.neff" }); + + var neuronx_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + var child: std.process.Child = .init(&.{ + try stdx.Io.Dir.path.bufJoin(&neuronx_cc_buf, &.{ + stdx.process.selfSharedObjectDirPath(), + "..", + "bin", + "neuronx-cc", + }), + "compile", + "--framework=XLA", + "--target", + target, + "--verbose=user", + "--enable-internal-neff-wrapper", + "--output", + neff_file, + "--optlevel=1", + // generic is the default, but it fails on transformers, force it + "--model-type=transformer", + // disable it, we do our own + "--auto-cast=none", + "--enable-fast-loading-neuron-binaries", + code_file, + }, arena.allocator()); + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; + child.cwd = tmp_dir; + _ = try child.spawnAndWait(io); + + std.debug.print(">>>> {s}\n", .{tmp_dir}); + + const neff_hlo_bytes = wrapNeffAsCustomCall(arena.allocator(), io, code, neff_file) catch |err| { + log.err("Error wrapping NEFF as custom call: {}\n", .{err}); + return err; + }; + + return c.PyTuple_Pack( + 2, + c.PyLong_FromLongLong(0), + c.PyBytes_FromStringAndSize(@ptrCast(neff_hlo_bytes), @intCast(neff_hlo_bytes.len)), + ); +} + +fn neuronx_cc(self: ?*c.PyObject, args_: [*c]*c.PyObject, nargs_: c.Py_ssize_t) callconv(.c) ?*c.PyObject { + return neuronx_cc_(self, args_, nargs_) catch |err| { + log.err("Error in neuronx_cc: {}\n", .{err}); + + const none = c.Py_None(); + c.Py_IncRef(none); + const tuple = c.PyTuple_New(2) orelse { + c.Py_DecRef(none); + return null; + }; + _ = c.PyTuple_SetItem(tuple, 0, c.PyLong_FromLongLong(400).?); + _ = c.PyTuple_SetItem(tuple, 1, none); + return tuple; + }; +} + +pub export fn PyInit_libneuronxla() callconv(.c) ?*c.PyObject { + return c.PyModuleDef_Init(&module_def); +} diff --git a/platforms/neuron/libzml_neuron.BUILD.bazel b/platforms/neuron/libpjrt_neuron.BUILD.bazel similarity index 56% rename from platforms/neuron/libzml_neuron.BUILD.bazel rename to platforms/neuron/libpjrt_neuron.BUILD.bazel index 8d8a7119..df831ad4 100644 --- a/platforms/neuron/libzml_neuron.BUILD.bazel +++ b/platforms/neuron/libpjrt_neuron.BUILD.bazel @@ -1,6 +1,5 @@ -load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") load("@bazel_skylib//rules:select_file.bzl", "select_file") -load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@zml//bazel:patchelf.bzl", "patchelf") load("@zml//bazel:runfiles.bzl", "runfiles_to_default") @@ -17,86 +16,38 @@ select_file( patchelf( name = "libneuronpjrt_so", - src = ":libneuronpjrt_so_orig", set_rpath = "$ORIGIN", + src = ":libneuronpjrt_so_orig", soname = "libneuronpjrt.so", ) patchelf( - name = "libzml_neuron_so", - src = "@zml//platforms/neuron:libzml_neuron", + name = "libpjrt_neuron_so", set_rpath = "$ORIGIN", - soname = "libzml_neuron.so", + src = "@zml//platforms/neuron:libpjrt_neuron", + soname = "libpjrt_neuron.so", ) -patchelf( - name = "neuron_explorer", - src = "@aws-neuronx-tools//:neuron-explorer", - add_needed = ["libzml_neuron.so"], - set_rpath = "$ORIGIN/../lib", -) - -# This is a compiler-enabled sandbox. Python stays packaged because AWS exposes -# neuronx-cc and NKI compilation through Python entrypoints; ZML wraps those as -# native ELF launchers in bin/ so the image does not need shell scripts or host -# Python. Keep AWS compiler/runtime payloads intact and only trim Python -# files that are not part of Neuron compilation. copy_to_directory( name = "sandbox", srcs = [ ":libneuronpjrt_so", - ":libneuronxla_files", - ":libzml_neuron_so", - ":neuron_explorer", - "@aws-neuronx-collectives//:libnccom", - "@aws-neuronx-runtime-lib//:libncfw.patchelf", - "@aws-neuronx-runtime-lib//:libnrt.patchelf", - "@aws-neuronx-runtime-lib//:libnrtucode_extisa", - "@libgcc_s1", - "@libgomp1", - "@libstdcpp6", - "@rules_python//python:current_py_toolchain", - "@zlib1g", + ":libpjrt_neuron_so", "@zml//platforms/neuron:neuronx-cc_files", - "@zml//platforms/neuron:neuronx_cc_shim.py", - "@zml//platforms/neuron:nki-cc_files", - "@zml//platforms/neuron:nki_kernel_compiler.py", - ], - add_directory_to_runfiles = False, - allow_overwrites = True, - exclude_srcs_patterns = [ - "**/__pycache__/**", - "**/*.pyc", - "**/tests/**", - "lib/libtcl*.so*", - "lib/libtk*.so*", - "lib/python*/idlelib/**", - "lib/python*/tkinter/**", - "lib/python*/turtledemo/**", - "lib/python*/ensurepip/**", - # "**/include/**", - "**/_solib_*/**", - ], - hardlink = "on", - include_external_repositories = ["**"], - include_srcs_patterns = [ - "platforms/neuron/neuronx-cc", - "platforms/neuron/neuronx_cc_shim.py", - "platforms/neuron/nki-cc", - "platforms/neuron/nki_kernel_compiler.py", - "neuron_explorer/**", - "**/lib**", - "lib/python*/**", - "site-packages/**", + "@zml//platforms/neuron:libneuronxla", + "@rules_python//python:current_py_toolchain", + "@aws-neuronx-runtime-lib//:libnrt.patchelf", + "@aws-neuronx-runtime-lib//:libncfw.patchelf", + "@aws-neuronx-runtime-lib//:libnrtucode_extisa", + "@aws-neuronx-collectives//:libnccom", + "@zlib1g", + "@libgomp1", ], replace_prefixes = { "platforms/neuron/neuronx-cc": "bin/neuronx-cc", - "platforms/neuron/neuronx_cc_shim.py": "bin/python-shims/neuronx-cc", - "platforms/neuron/nki-cc": "bin/nki-cc", - "platforms/neuron/nki_kernel_compiler.py": "site-packages/nki_kernel_compiler.py", - "neuron_explorer": "bin", + "platforms/neuron/libneuronxla.so": "site-packages/libneuronxla.so", "libneuronpjrt_so": "lib", - "libzml_neuron_so": "lib", + "libpjrt_neuron_so": "lib", "libnrt.patchelf": "lib", "libncfw.patchelf": "lib", "libnrtucode_extisa": "lib", @@ -104,15 +55,28 @@ copy_to_directory( "usr/lib/x86_64-linux-gnu": "lib", "opt/neuron": "lib", }, - visibility = [ - "@zml//tools/neuron:__pkg__", - "@zml//zml:__pkg__", - "@zml//zml/kernels/flash_attention/nki:__pkg__", + exclude_srcs_patterns = [ + "**/tests/**", + # "**/include/**", + "**/_solib_*/**", ], + include_srcs_patterns = [ + "platforms/neuron/neuronx-cc", + "**/lib**", + "lib/python*/**", + "site-packages/**", + ], + allow_overwrites = True, + add_directory_to_runfiles = False, + include_external_repositories = ["**"], + hardlink = "on", ) cc_library( name = "libnrt_headers", + deps = [ + "@aws-neuronx-runtime-lib//:libnrt_headers", + ], linkopts = [ # Defer function call resolution until the function is called # (lazy loading) rather than at load time. @@ -124,13 +88,10 @@ cc_library( "-Wl,-z,lazy", ], visibility = ["@zml//platforms/neuron:__subpackages__"], - deps = [ - "@aws-neuronx-runtime-lib//:libnrt_headers", - ], ) cc_library( - name = "libzml_neuron", + name = "libpjrt_neuron", data = [":sandbox"], visibility = ["@zml//platforms/neuron:__subpackages__"], ) diff --git a/platforms/neuron/libzml_neuron.zig b/platforms/neuron/libpjrt_neuron.zig similarity index 61% rename from platforms/neuron/libzml_neuron.zig rename to platforms/neuron/libpjrt_neuron.zig index 404d8cc0..1d25cde6 100644 --- a/platforms/neuron/libzml_neuron.zig +++ b/platforms/neuron/libpjrt_neuron.zig @@ -2,16 +2,14 @@ const std = @import("std"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); -const neuron = @import("neuron.zig"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/neuron"); fn findFreeTcpPort(io: std.Io) !u16 { - const addr: std.Io.net.IpAddress = .{ .ip4 = .{ .bytes = .{ 127, 0, 0, 1 }, .port = 0 } }; var server = try std.Io.net.IpAddress.listen( - &addr, + .{ .ip4 = .{ .bytes = .{ 127, 0, 0, 1 }, .port = 0 } }, io, .{}, ); @@ -39,14 +37,11 @@ pub export fn zmlxneuron_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaqu } extern fn setenv(name: [*:0]const u8, value: [*:0]const u8, overwrite: c_int) c_int; - -fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { - const scratchpad_page_size = "1024"; - - var root_comm_buf: [256]u8 = undefined; +fn setupNeuronEnv(io: std.Io) !void { + var buf: [256]u8 = undefined; _ = setenv( "NEURON_RT_ROOT_COMM_ID", - try std.mem.printSentinel(&root_comm_buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}, 0), + try std.fmt.bufPrintZ(&buf, "127.0.0.1:{d}", .{try findFreeTcpPort(io)}), 1, ); _ = setenv( @@ -57,51 +52,80 @@ fn setupNeuronEnv(io: std.Io, sandbox_path: []const u8) !void { }), 1, ); - _ = setenv("NEURON_RT_LOG_LEVEL", "error", 0); - _ = setenv("NEURON_RT_DISABLE_EXECUTION_BARRIER", "1", 0); - _ = setenv("NEURON_RT_STOCHASTIC_ROUNDING_EN", "1", 1); - _ = setenv("NEURON_RT_ASYNC_EXEC_MAX_INFLIGHT_REQUESTS", "3", 0); - _ = setenv("NEURON_SCRATCHPAD_PAGE_SIZE", scratchpad_page_size, 0); - - var bin_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const bin_path = try stdx.Io.Dir.path.bufJoin(&bin_path_buf, &.{ sandbox_path, "bin" }); - - const old_path = if (std.c.getenv("PATH")) |path| std.mem.span(path) else ""; - var new_path_buf: [std.posix.PATH_MAX:0]u8 = undefined; - const new_path = if (old_path.len == 0) - try std.mem.printSentinel(&new_path_buf, "{s}", .{bin_path}, 0) - else - try std.mem.printSentinel(&new_path_buf, "{s}:{s}", .{ bin_path, old_path }, 0); - if (setenv("PATH", new_path.ptr, 1) != 0) return error.SetEnvFailed; - - if (std.c.getenv("NEURON_CC_FLAGS") != null) return; - - const log_level = if (std.c.getenv("NEURON_RT_LOG_LEVEL")) |level| std.mem.span(level) else "error"; - - var flags_buf: [512:0]u8 = undefined; - const instance = try neuron.instance(); - const target = @tagName(instance.compilerTarget()); - const flags = try std.mem.printSentinel( - &flags_buf, - "--target={s} " ++ - "--optlevel=1 " ++ - "--model-type=transformer " ++ - "--enable-fast-loading-neuron-binaries " ++ - "--enable-fast-context-switch " ++ - "--hbm-scratchpad-page-size={s} " ++ - "--verbose {s} " ++ - "--logfile-verbose {s} " ++ - "--logfile=./log-neuron-cc.txt", - .{ - target, - scratchpad_page_size, - log_level, - log_level, - }, - 0, + _ = setenv( + "NEURON_RT_STOCHASTIC_ROUNDING_EN", + "1", + 1, ); +} - _ = setenv("NEURON_CC_FLAGS", flags.ptr, 0); +fn pyStatusCheck(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { + if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); + path_with_null[len] = 0; + return path_with_null; +} + +// redefine it because for some reason PyThreadState can't be sized +extern fn PyEval_SaveThread() *anyopaque; + +fn setupPythonEnv(sandbox_path: []const u8) !void { + const Static = struct { + var py_config: c.PyConfig = undefined; + }; + + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyStatusCheck(c.Py_PreInitialize(&preconfig)); + } + + c.PyConfig_InitIsolatedConfig(&Static.py_config); + + Static.py_config.module_search_paths_set = 1; + Static.py_config.optimization_level = 2; + Static.py_config.write_bytecode = 0; + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ + std.Io.Dir.path.fmtJoin(&.{ + sandbox_path, + "lib", + "python", + }), + c.PY_MAJOR_VERSION, + c.PY_MINOR_VERSION, + }); + pyStatusCheck(c.PyConfig_SetBytesString(&Static.py_config, &Static.py_config.home, home)); + pyStatusCheck(c.PyWideStringList_Append(&Static.py_config.module_search_paths, &try toPosixPathW(home))); + } + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const site_packages = try stdx.Io.Dir.path.bufJoin(&buf, &.{ + sandbox_path, + "site-packages", + }); + pyStatusCheck(c.PyWideStringList_Append(&Static.py_config.module_search_paths, &try toPosixPathW(site_packages))); + } + + pyStatusCheck(c.Py_InitializeFromConfig(&Static.py_config)); + + // release the GIL + _ = PyEval_SaveThread(); } // Duplicates a PJRT Api object while being careful about struct size differences @@ -131,7 +155,8 @@ fn getPjrtApi() !*c.PJRT_Api { "..", }); - try setupNeuronEnv(io, sandbox_path); + try setupNeuronEnv(io); + try setupPythonEnv(sandbox_path); Static.inner = blk: { const GetPjrtApi_inner = GetPjrtApi_blk: { diff --git a/platforms/neuron/neuron.bzl b/platforms/neuron/neuron.bzl index 8b947c04..409904ca 100644 --- a/platforms/neuron/neuron.bzl +++ b/platforms/neuron/neuron.bzl @@ -1,4 +1,4 @@ -load("@python_versions//3.12:defs.bzl", _py_binary = "py_binary") +load("@python_versions//3.11:defs.bzl", _py_binary = "py_binary") load("@rules_python//python:defs.bzl", "PyInfo") load("@with_cfg.bzl", "with_cfg") load("//bazel:http_deb_archive.bzl", "http_deb_archive") @@ -6,31 +6,15 @@ load("//bazel:simple_repository.bzl", "simple_repository") load("//platforms:packages.bzl", "packages") BASE_URL = "https://apt.repos.neuron.amazonaws.com" -STRIP_PREFIX = "./opt/aws/neuron" +STRIP_PREFIX = "opt/aws/neuron" _BUILD_FILE_PRELUDE = """\ package(default_visibility = ["//visibility:public"]) - -load("@rules_cc//cc:cc_library.bzl", "cc_library") """ _UBUNTU_PACKAGES = { - "libgcc_s1": { - "package": "libgcc-s1", - "build": packages.filegroup(name = "libgcc_s1", srcs = ["lib/x86_64-linux-gnu/libgcc_s.so.1"]), - }, - "libgomp1": { - "package": "libgomp1", - "build": packages.filegroup(name = "libgomp1", srcs = ["usr/lib/x86_64-linux-gnu/libgomp.so.1"]), - }, - "libstdcpp6": { - "package": "libstdc++6", - "build": packages.filegroup(name = "libstdcpp6", srcs = ["usr/lib/x86_64-linux-gnu/libstdc++.so.6"]), - }, - "zlib1g": { - "package": "zlib1g", - "build": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), - }, + "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), + "libgomp1": packages.filegroup(name = "libgomp1", srcs = ["usr/lib/x86_64-linux-gnu/libgomp.so.1"]), } _NEURON_PACKAGES = { @@ -73,12 +57,6 @@ _NEURON_PACKAGES = { srcs = ["lib/libnccom.so.2"], ), ]), - "aws-neuronx-tools": "\n".join([ - packages.filegroup( - name = "neuron-explorer", - srcs = ["bin/neuron-explorer"], - ), - ]), } def _neuron_impl(mctx): @@ -87,21 +65,21 @@ def _neuron_impl(mctx): ]) simple_repository( - name = "libzml_neuron", - build_file = ":libzml_neuron.BUILD.bazel", + name = "libpjrt_neuron", + build_file = ":libpjrt_neuron.BUILD.bazel", ) - for repo_name, package in _UBUNTU_PACKAGES.items(): - pkg = loaded_packages[package["package"]]["amd64"] + for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): + pkg = loaded_packages[pkg_name] http_deb_archive( - name = repo_name, + name = pkg_name, urls = pkg["urls"], sha256 = pkg["sha256"], - build_file_content = _BUILD_FILE_PRELUDE + package["build"], + build_file_content = _BUILD_FILE_PRELUDE + build_file_content, ) for pkg_name, build_file_content in _NEURON_PACKAGES.items(): - pkg = loaded_packages[pkg_name]["amd64"] + pkg = loaded_packages[pkg_name] http_deb_archive( name = pkg_name, urls = pkg["urls"], @@ -112,15 +90,7 @@ def _neuron_impl(mctx): return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = [ - "aws-neuronx-collectives", - "aws-neuronx-runtime-lib", - "aws-neuronx-tools", - "libgcc_s1", - "libgomp1", - "libstdcpp6", - "libzml_neuron", - ], + root_module_direct_deps = "all", root_module_direct_dev_deps = [], ) @@ -131,4 +101,4 @@ neuron_packages = module_extension( py_binary_with_script, _py_binary_internal = with_cfg( kind = _py_binary, extra_providers = [PyInfo], -).build() +).set(Label("@rules_python//python/config_settings:bootstrap_impl"), "script").build() diff --git a/platforms/neuron/neuron.zig b/platforms/neuron/neuron.zig index e0365fe9..8751f5ac 100644 --- a/platforms/neuron/neuron.zig +++ b/platforms/neuron/neuron.zig @@ -5,98 +5,12 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); -const platforms_options = @import("platforms/options"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/neuron"); -pub const CompilerTarget = enum { - trn1, - inf2, - trn1n, - trn2, - trn2n, - trn3, -}; - -pub const Family = enum(u32) { - unknown = 0, - inf1 = 1, - trn1 = 2, - trn1n = 3, - inf2 = 4, - trn2 = 5, - trn2n = 6, - inf2e = 7, - trn2p = 8, - trn2u = 9, - trn2e = 10, - trn2eu = 11, - trn2ac = 12, - trn2uac = 13, - trn3 = 14, - trn3pds98 = 15, - - pub fn compilerTarget(self: Family) CompilerTarget { - return switch (self) { - .unknown, .inf1 => unreachable, - .trn1 => .trn1, - .trn1n => .trn1n, - .inf2, .inf2e => .inf2, - .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac => .trn2, - .trn3, .trn3pds98 => .trn3, - }; - } -}; - -pub const Size = enum(u32) { - xl1 = 0, - xl2 = 1, - xl4 = 2, - xl6 = 3, - xl8 = 4, - xl24 = 5, - xl32 = 6, - xl48 = 7, - xl3 = 8, - unknown = 9, -}; - -pub const Instance = struct { - family: Family, - size: Size, - - pub fn compilerTarget(self: Instance) CompilerTarget { - return self.family.compilerTarget(); - } - - pub fn coresPerChip(self: Instance) !usize { - return switch (self.family) { - .inf1 => 4, - .trn1, .trn1n, .inf2, .inf2e => 2, - .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac, .trn3, .trn3pds98 => 8, - .unknown => unreachable, - }; - } -}; - -pub fn instance() !Instance { - var info = std.mem.zeroes(c.struct_nrt_instance_info); - info.family = @backingInt(Family.unknown); - info.size = @backingInt(Size.unknown); - - if (c.nrt_get_instance_info(&info, @sizeOf(c.struct_nrt_instance_info)) != c.NRT_SUCCESS) { - return error.Unavailable; - } - - return .{ - .family = @fromBackingInt(@intCast(info.family)), - .size = @fromBackingInt(@intCast(info.size)), - }; -} - pub fn isEnabled() bool { - return platforms_options.neuron_enabled; + return @hasDecl(c, "ZML_RUNTIME_NEURON"); } fn hasNeuronDevice(io: std.Io) bool { @@ -120,7 +34,7 @@ fn isRunningOnEC2(io: std.Io) !bool { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = allocator; + _ = allocator; // autofix if (comptime !isEnabled()) { return error.Unavailable; } @@ -134,17 +48,17 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { return error.Unavailable; } - const r = try bazel.runfiles(bazel_builtin.current_repository); + const r = try bazel.runfiles(io, bazel_builtin.current_repository); var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libzml_neuron/sandbox", &sandbox_path_buf) orelse { + const sandbox_path = try r.rlocation("libpjrt_neuron/sandbox", &sandbox_path_buf) orelse { log.err("Failed to find sandbox path for NEURON runtime", .{}); return error.FileNotFound; }; return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_neuron.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_neuron.so" }); break :blk .loadFrom(path); }; } diff --git a/platforms/neuron/neuronx-cc.zig b/platforms/neuron/neuronx-cc.zig new file mode 100644 index 00000000..e77c4858 --- /dev/null +++ b/platforms/neuron/neuronx-cc.zig @@ -0,0 +1,99 @@ +const std = @import("std"); + +const c = @import("c"); +const runfiles = @import("runfiles"); + +fn pyStatusCheck(status: c.PyStatus) void { + if (c.PyStatus_Exception(status) != 0) { + if (c.PyStatus_IsExit(status) != 0) { + std.process.exit(@intCast(status.exitcode)); + } + c.Py_ExitStatusException(status); + } +} + +pub fn toPosixPathW(file_path: []const u8) error{NameTooLong}![std.posix.PATH_MAX - 1:0]c.wchar_t { + if (file_path.len >= std.posix.PATH_MAX) return error.NameTooLong; + + var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; + const len = c.mbstowcs(&path_with_null, file_path.ptr, file_path.len); + path_with_null[len] = 0; + return path_with_null; +} + +pub fn main() !void { + const allocator = std.heap.c_allocator; + + var threaded: std.Io.Threaded = .init(allocator, .{}); + defer threaded.deinit(); + + const io = threaded.io(); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + { + var preconfig: c.PyPreConfig = undefined; + c.PyPreConfig_InitIsolatedConfig(&preconfig); + preconfig.utf8_mode = 1; + pyStatusCheck(c.Py_PreInitialize(&preconfig)); + } + + var config: c.PyConfig = undefined; + c.PyConfig_InitIsolatedConfig(&config); + defer c.PyConfig_Clear(&config); + + config.module_search_paths_set = 1; + config.optimization_level = 2; + config.write_bytecode = 0; + + _ = c.PyConfig_SetBytesArgv(&config, @intCast(std.os.argv.len), @ptrCast(std.os.argv)); + + var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const self_exe_dir = blk: { + const n = try std.process.executableDirPath(io, &self_exe_dir_buf); + break :blk self_exe_dir_buf[0..n]; + }; + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const home = try std.fmt.bufPrintZ(&buf, "{f}{d}.{d}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "lib", + "python", + }), + c.PY_MAJOR_VERSION, + c.PY_MINOR_VERSION, + }); + pyStatusCheck(c.PyConfig_SetBytesString(&config, &config.home, home)); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); + } + + { + var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ + std.Io.Dir.path.fmtJoin(&.{ + self_exe_dir, + "..", + "site-packages", + }), + }); + pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); + } + + pyStatusCheck(c.Py_InitializeFromConfig(&config)); + defer c.Py_Finalize(); + + const neuronxcc_main = blk: { + const module = c.PyImport_ImportModule("neuronxcc.driver.CommandDriver"); + std.debug.print(">>> MODULE: {any}\n", .{module}); + defer c.Py_DecRef(module); + break :blk c.PyObject_GetAttrString(module, "main"); + }; + defer c.Py_DecRef(neuronxcc_main); + + const result = c.PyObject_CallNoArgs(neuronxcc_main); + defer c.Py_DecRef(result); +} diff --git a/platforms/neuron/neuronx_cc_shim.py b/platforms/neuron/neuronx_cc_shim.py deleted file mode 100755 index deff0b5d..00000000 --- a/platforms/neuron/neuronx_cc_shim.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 -"""Python-compatible shim for harness NKI standalone compilation.""" - -import os -import sys - - -def main() -> None: - native = os.path.join( - os.path.dirname(os.path.realpath(__file__)), "..", "neuronx-cc" - ) - os.execv(native, [native, *sys.argv[1:]]) - - -if __name__ == "__main__": - main() diff --git a/platforms/neuron/neuronxcc_compiler_launcher.zig b/platforms/neuron/neuronxcc_compiler_launcher.zig deleted file mode 100644 index 066923e2..00000000 --- a/platforms/neuron/neuronxcc_compiler_launcher.zig +++ /dev/null @@ -1,9 +0,0 @@ -const std = @import("std"); - -const python_launcher = @import("platforms/neuron/python_launcher"); - -pub fn main(init: std.process.Init) !void { - try python_launcher.runModuleEntrypoint(init, .{ - .module_name = "neuronxcc.driver.CommandDriver", - }); -} diff --git a/platforms/neuron/nki_compiler_launcher.zig b/platforms/neuron/nki_compiler_launcher.zig deleted file mode 100644 index f8a6b884..00000000 --- a/platforms/neuron/nki_compiler_launcher.zig +++ /dev/null @@ -1,12 +0,0 @@ -const std = @import("std"); - -const python_launcher = @import("platforms/neuron/python_launcher"); - -pub fn main(init: std.process.Init) !void { - _ = std; - try python_launcher.runScriptMain(init, &.{ - "..", - "site-packages", - "nki_kernel_compiler.py", - }); -} diff --git a/platforms/neuron/nki_kernel.zig b/platforms/neuron/nki_kernel.zig deleted file mode 100644 index 4d897be8..00000000 --- a/platforms/neuron/nki_kernel.zig +++ /dev/null @@ -1,160 +0,0 @@ -const std = @import("std"); - -const bazel = @import("bazel"); -const bazel_builtin = @import("bazel_builtin"); -const stdx = @import("stdx"); - -const neuron = @import("platforms/neuron"); - -const log = std.log.scoped(.@"zml/platforms/neuron/nki_kernel"); - -pub const TensorSignature = struct { - dtype: []const u8, - dims: []const i64, -}; - -pub const Kernel = struct { - name: []const u8, - entrypoint: []const u8, - source_path: []const u8, - compiler_target: []const u8, - inputs: []const TensorSignature, - outputs: []const TensorSignature, -}; - -const CompilerRequest = struct { - entrypoint: []const u8, - source: []const u8, - target: []const u8, - tool_bin_dir: []const u8, - neuronx_cc_args: []const []const u8, - inputs: []const TensorSignature, - outputs: []const TensorSignature, -}; - -const CompilerResponse = struct { - backend_config_b64: []const u8, -}; - -pub fn compilerTargetFromInstance() neuron.CompilerTarget { - const instance = neuron.instance() catch |err| { - std.debug.panic("failed to query Neuron instance for NKI compiler target: {}", .{err}); - }; - return instance.compilerTarget(); -} - -pub fn compileNkiKernel(allocator: std.mem.Allocator, io: std.Io, kernel: Kernel) ![]const u8 { - const compiler_flags = std.mem.span(std.c.getenv("NEURON_CC_FLAGS") orelse return error.MissingNeuronCompilerFlags); - var compiler_args: std.ArrayList([]const u8) = .empty; - defer compiler_args.deinit(allocator); - var compiler_flags_iter = std.mem.tokenizeScalar(u8, compiler_flags, ' '); - while (compiler_flags_iter.next()) |arg| { - try compiler_args.append(allocator, arg); - } - - const tmp_root_path = if (std.c.getenv("TMPDIR")) |tmpdir| std.mem.span(tmpdir) else "/tmp"; - var tmp_root = try std.Io.Dir.openDir(.cwd(), io, tmp_root_path, .{}); - defer tmp_root.close(io); - - var dir_name_buf: [128]u8 = undefined; - const dir_name = try std.fmt.bufPrint(&dir_name_buf, "zml-nki-{d}", .{std.Io.Timestamp.now(io, .real).nanoseconds}); - - var tmp_dir = try tmp_root.createDirPathOpen(io, dir_name, .{ .permissions = .fromMode(0o700) }); - defer tmp_dir.close(io); - - var cwd_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const cwd = try tmp_dir.realPath(io, &cwd_buf); - - const request_file = try tmp_dir.createFile(io, "kernel.request.json", .{ .read = true }); - defer request_file.close(io); - - const result_file = try tmp_dir.createFile(io, "kernel.result.json", .{ .read = true }); - defer result_file.close(io); - - var request_file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const request_file_path = try request_file.realPath(io, &request_file_buf); - - var result_file_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const result_file_path = try result_file.realPath(io, &result_file_buf); - - const r = try bazel.runfiles(bazel_builtin.current_repository); - - var sandbox_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libzml_neuron/sandbox", &sandbox_path_buf) orelse return error.FileNotFound; - - var source_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const source_path = try r.rlocation(kernel.source_path, &source_path_buf) orelse return error.FileNotFound; - const source = try std.Io.Dir.cwd().readFileAlloc(io, source_path, allocator, .unlimited); - defer allocator.free(source); - - var nki_cc_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const nki_cc_path = try stdx.Io.Dir.path.bufJoin(&nki_cc_buf, &.{ sandbox_path, "bin", "nki-cc" }); - - const bin_dirpath = std.fs.path.dirname(nki_cc_path) orelse return error.InvalidSandboxPath; - - const request: CompilerRequest = .{ - .entrypoint = kernel.entrypoint, - .source = source, - .target = kernel.compiler_target, - .tool_bin_dir = bin_dirpath, - .neuronx_cc_args = compiler_args.items, - .inputs = kernel.inputs, - .outputs = kernel.outputs, - }; - - const request_json = try std.fmt.allocPrint(allocator, "{f}", .{std.json.fmt(request, .{ .emit_null_optional_fields = false })}); - defer allocator.free(request_json); - - try request_file.writePositionalAll(io, request_json, 0); - - var spawn_threaded: std.Io.Threaded = .init(std.heap.c_allocator, .{}); - defer spawn_threaded.deinit(); - - const spawn_io = spawn_threaded.io(); - - var child = try std.process.spawn(spawn_io, .{ - .argv = &.{ - nki_cc_path, - "--request", - request_file_buf[0..request_file_path], - "--result", - result_file_buf[0..result_file_path], - }, - .stdin = .ignore, - .stdout = .inherit, - .stderr = .inherit, - .cwd = .{ .path = cwd_buf[0..cwd] }, - }); - - const term = try child.wait(spawn_io); - switch (term) { - .exited => |code| { - if (code != 0) { - log.err("nki-cc exited with code {}", .{code}); - return error.NkiCcFailed; - } - }, - .signal => |sig| { - log.err("nki-cc terminated by signal {}", .{sig}); - return error.NkiCcFailed; - }, - else => |status| { - log.err("nki-cc terminated unexpectedly: {}", .{status}); - return error.NkiCcFailed; - }, - } - - var result_reader = result_file.reader(io, &.{}); - const data = try result_reader.interface.readAlloc(allocator, try result_file.length(io)); - defer allocator.free(data); - - const parsed_response = try std.json.parseFromSlice(CompilerResponse, allocator, data, .{ - .ignore_unknown_fields = true, - }); - defer parsed_response.deinit(); - - const backend_config = try allocator.dupe(u8, parsed_response.value.backend_config_b64); - errdefer allocator.free(backend_config); - - return backend_config; -} diff --git a/platforms/neuron/nki_kernel_compiler.py b/platforms/neuron/nki_kernel_compiler.py deleted file mode 100755 index 207d0c26..00000000 --- a/platforms/neuron/nki_kernel_compiler.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Compile inline ZML NKI kernels into Neuron custom-native backend configs. - -The `nki_kernel.zig` helper writes one JSON request per inline kernel. -This script asks NKI to produce BIR and returns the backend_config expected by -`AwsNeuronCustomNativeKernel`. -""" - -import argparse -import inspect -import json -import re -import sys -import tempfile -import types -from dataclasses import dataclass -from pathlib import Path - -import ml_dtypes -import numpy as np - - -@dataclass(frozen=True) -class TensorSpec: - dtype: str - shape: tuple[int, ...] - - -def dtype_from_name(name: str): - mapping = { - "bool": np.bool_, - "i8": np.int8, - "i16": np.int16, - "i32": np.int32, - "i64": np.int64, - "u8": np.uint8, - "u16": np.uint16, - "u32": np.uint32, - "u64": np.uint64, - "f16": np.float16, - "f32": np.float32, - "f64": np.float64, - } - - if name == "bf16": - return ml_dtypes.bfloat16 - - try: - return mapping[name] - except KeyError as exc: - raise ValueError(f"unsupported dtype: {name}") from exc - - -def parse_tensor_specs(raw_specs: list[dict]) -> tuple[TensorSpec, ...]: - return tuple( - TensorSpec( - dtype=spec["dtype"], - shape=tuple(spec.get("shape", spec["dims"])), - ) - for spec in raw_specs - ) - - -def load_kernel(source_path: Path, source: str, entrypoint: str): - module_name = re.sub(r"\W", "_", source_path.stem) or "kernel" - if module_name[0].isdigit(): - module_name = f"kernel_{module_name}" - - module = types.ModuleType(module_name) - module.__file__ = str(source_path) - module.__dict__["__builtins__"] = __builtins__ - sys.modules[module_name] = module - exec(compile(source, str(source_path), "exec"), module.__dict__) - - try: - return module.__dict__[entrypoint] - except KeyError as exc: - raise KeyError(f"entrypoint {entrypoint!r} not found in {source_path}") from exc - - -def make_inputs(inputs: tuple[TensorSpec, ...]) -> list[np.ndarray]: - """Create zero-filled specialization inputs matching the declared ZML signature.""" - arrays: list[np.ndarray] = [] - for input_spec in inputs: - arrays.append( - np.zeros( - shape=input_spec.shape, - dtype=dtype_from_name(input_spec.dtype), - ) - ) - return arrays - - -def compile_kernel( - source: str, - entrypoint: str, - target: str, - tool_bin_dir: str, - neuronx_cc_args: tuple[str, ...], - inputs: tuple[TensorSpec, ...], - outputs: tuple[TensorSpec, ...], -) -> str: - """Run the full embedded-kernel pipeline for one custom-call.""" - tmp_dir = Path(tempfile.mkdtemp(prefix="zml-nki-", dir=".")) - - source_path = tmp_dir / "kernel.py" - artifacts_dir = tmp_dir / "artifacts" - artifacts_dir.mkdir(parents=True, exist_ok=False) - - source_path.write_text(source, encoding="utf-8") - - kernel = load_kernel(source_path, source, entrypoint) - - input_names = [ - parameter.name - for parameter in inspect.signature(kernel).parameters.values() - if parameter.kind - in (inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD) - ] - - if len(input_names) != len(inputs): - raise ValueError( - f"{entrypoint} expects {len(input_names)} inputs, ZML declared {len(inputs)}" - ) - - from nki.compiler.driver import compile_to_bir - from nki.compiler.frontend import ParserFrontend - from nki.compiler.ncc_driver import CompilationMode, CompileOptions - - compile_opts = CompileOptions( - target=target, - artifacts_dir=str(artifacts_dir), - output_path=str(artifacts_dir / "unused.neff"), - neuronx_cc_path=str(Path(tool_bin_dir) / "neuronx-cc"), - neuronx_cc_args=neuronx_cc_args, - mode=CompilationMode.INTEGRATION, - ).disable_backend_optimizations() - - nir = compile_to_bir( - kernel, - frontend=ParserFrontend(), - inputs=dict(zip(input_names, make_inputs(inputs), strict=True)), - compile_opts=compile_opts, - output_names=[f"output{i}" for i in range(len(outputs))], - ) - - return nir.build_config().backend_config_b64.decode("utf-8") - - -def main() -> None: - parser = argparse.ArgumentParser( - description="Compile a ZML NKI kernel to a custom-native backend config" - ) - parser.add_argument("--request", required=True) - parser.add_argument("--result", required=True) - args = parser.parse_args() - - request_path = Path(args.request) - result_path = Path(args.result) - - request = json.loads(request_path.read_text(encoding="utf-8")) - - backend_config_b64 = compile_kernel( - source=request["source"], - entrypoint=request["entrypoint"], - target=request["target"], - tool_bin_dir=request["tool_bin_dir"], - neuronx_cc_args=tuple(request["neuronx_cc_args"]), - inputs=parse_tensor_specs(request["inputs"]), - outputs=parse_tensor_specs(request["outputs"]), - ) - - result_path.write_text( - json.dumps({"backend_config_b64": backend_config_b64}), - encoding="utf-8", - ) - - -if __name__ == "__main__": - main() diff --git a/platforms/neuron/nrt.h b/platforms/neuron/nrt.h index 9970d0bd..d391b8eb 100644 --- a/platforms/neuron/nrt.h +++ b/platforms/neuron/nrt.h @@ -1,5 +1,2 @@ #include #include -#include -#include -#include diff --git a/platforms/neuron/packages.lock.json b/platforms/neuron/packages.lock.json index 2f275bba..134d55e4 100755 --- a/platforms/neuron/packages.lock.json +++ b/platforms/neuron/packages.lock.json @@ -3,24 +3,60 @@ { "arch": "amd64", "dependencies": [ + { + "key": "debianutils_5.7-0.5_deb12u1_amd64", + "name": "debianutils", + "version": "5.7-0.5~deb12u1" + }, { "key": "libc6_2.36-9-p-deb12u10_amd64", "name": "libc6", "version": "2.36-9+deb12u10" }, + { + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "version": "12.2.0-14+deb12u1" + }, { "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", "name": "gcc-12-base", "version": "12.2.0-14+deb12u1" + }, + { + "key": "base-files_12.4-p-deb12u11_amd64", + "name": "base-files", + "version": "12.4+deb12u11" + }, + { + "key": "mawk_1.3.4.20200120-3.1_amd64", + "name": "mawk", + "version": "1.3.4.20200120-3.1" + }, + { + "key": "libtinfo6_6.4-4_amd64", + "name": "libtinfo6", + "version": "6.4-4" } ], - "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", - "name": "libgcc-s1", - "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", + "key": "bash_5.2.15-2-p-b8_amd64", + "name": "bash", + "sha256": "23910515e6b8b4fb4a3b50549464ef2efdc149b568150e2aa45115b37b6e6578", "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/b/bash/bash_5.2.15-2+b8_amd64.deb" ], - "version": "12.2.0-14+deb12u1" + "version": "5.2.15-2+b8" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "debianutils_5.7-0.5_deb12u1_amd64", + "name": "debianutils", + "sha256": "55f951359670eb3236c9e2ccd5fac9ccb3db734f5a22aff21589e7a30aee48c9", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/d/debianutils/debianutils_5.7-0.5~deb12u1_amd64.deb" + ], + "version": "5.7-0.5~deb12u1" }, { "arch": "amd64", @@ -33,6 +69,17 @@ ], "version": "2.36-9+deb12u10" }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", + "name": "libgcc-s1", + "sha256": "3016e62cb4b7cd8038822870601f5ed131befe942774d0f745622cc77d8a88f7", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libgcc-s1_12.2.0-14+deb12u1_amd64.deb" + ], + "version": "12.2.0-14+deb12u1" + }, { "arch": "amd64", "dependencies": [], @@ -44,6 +91,39 @@ ], "version": "12.2.0-14+deb12u1" }, + { + "arch": "amd64", + "dependencies": [], + "key": "base-files_12.4-p-deb12u11_amd64", + "name": "base-files", + "sha256": "c0b28aa96c88ca2e2f18c44911b6cd07de9335e0de7922fba778886a50061e64", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/b/base-files/base-files_12.4+deb12u11_amd64.deb" + ], + "version": "12.4+deb12u11" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "mawk_1.3.4.20200120-3.1_amd64", + "name": "mawk", + "sha256": "bcbc83f391854ea9d50ce2a4101aacf330de3b8b71d81a798faadba14a157f78", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/m/mawk/mawk_1.3.4.20200120-3.1_amd64.deb" + ], + "version": "1.3.4.20200120-3.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtinfo6_6.4-4_amd64", + "name": "libtinfo6", + "sha256": "072d908f38f51090ca28ca5afa3b46b2957dc61fe35094c0b851426859a49a51", + "urls": [ + "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/n/ncurses/libtinfo6_6.4-4_amd64.deb" + ], + "version": "6.4-4" + }, { "arch": "amd64", "dependencies": [ @@ -71,33 +151,6 @@ ], "version": "12.2.0-14+deb12u1" }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "libgcc-s1_12.2.0-14-p-deb12u1_amd64", - "name": "libgcc-s1", - "version": "12.2.0-14+deb12u1" - }, - { - "key": "libc6_2.36-9-p-deb12u10_amd64", - "name": "libc6", - "version": "2.36-9+deb12u10" - }, - { - "key": "gcc-12-base_12.2.0-14-p-deb12u1_amd64", - "name": "gcc-12-base", - "version": "12.2.0-14+deb12u1" - } - ], - "key": "libstdc-p--p-6_12.2.0-14-p-deb12u1_amd64", - "name": "libstdc++6", - "sha256": "5cd3171216d4ab0fc911cfe9c35509bf2dd8f47761c43b7f6a4296701551a24d", - "urls": [ - "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/g/gcc-12/libstdc++6_12.2.0-14+deb12u1_amd64.deb" - ], - "version": "12.2.0-14+deb12u1" - }, { "arch": "amd64", "dependencies": [ @@ -128,13 +181,13 @@ { "arch": "amd64", "dependencies": [], - "key": "aws-neuronx-runtime-lib_2.32.31.0-0234f5ed2_amd64", + "key": "aws-neuronx-runtime-lib_2.29.40.0-f954cd7a5_amd64", "name": "aws-neuronx-runtime-lib", - "sha256": "ed69354674b926490d917a6aeb7dea8b6300278a37958ee9a412e2cd89c41792", + "sha256": "60f5dc0c8731fdd48a9726493375dbb0f0d178d05d84f70716bcc36789fd9be9", "urls": [ - "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-runtime-lib/aws-neuronx-runtime-lib_2.32.31.0-0234f5ed2_amd64.deb" + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-runtime-lib/aws-neuronx-runtime-lib_2.29.40.0-f954cd7a5_amd64.deb" ], - "version": "2.32.31.0-0234f5ed2" + "version": "2.29.40.0-f954cd7a5" }, { "arch": "amd64", @@ -280,13 +333,13 @@ "version": "252.36-1~deb12u1" } ], - "key": "aws-neuronx-collectives_2.32.28.0-452cba8de_amd64", + "key": "aws-neuronx-collectives_2.29.41.0-681fef5f5_amd64", "name": "aws-neuronx-collectives", - "sha256": "ec737c6647b6fbd38550e11f25e27d0f1a96dbb2cd28ae5bbf3f827faba5bdc7", + "sha256": "c4bd37dfd6a18883ea87b9e7e665ee266f3e344536d0f4e9c9d6da4aa38e7417", "urls": [ - "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-collectives/aws-neuronx-collectives_2.32.28.0-452cba8de_amd64.deb" + "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-collectives/aws-neuronx-collectives_2.29.41.0-681fef5f5_amd64.deb" ], - "version": "2.32.28.0-452cba8de" + "version": "2.29.41.0-681fef5f5" }, { "arch": "amd64", @@ -562,17 +615,6 @@ "https://snapshot-cloudflare.debian.org/archive/debian/20250529T205323Z/pool/main/s/systemd/libudev1_252.36-1~deb12u1_amd64.deb" ], "version": "252.36-1~deb12u1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "aws-neuronx-tools_2.30.10.0-39feaa591_amd64", - "name": "aws-neuronx-tools", - "sha256": "b972e4db5e84a429aa237d1eb9347fcaa78c19e848049a3495efef734a0b240f", - "urls": [ - "https://apt.repos.neuron.amazonaws.com/pool/main/a/aws-neuronx-tools/aws-neuronx-tools_2.30.10.0-39feaa591_amd64.deb" - ], - "version": "2.30.10.0-39feaa591" } ], "version": 1 diff --git a/platforms/neuron/packages.yaml b/platforms/neuron/packages.yaml index eb399f49..6c3de28a 100644 --- a/platforms/neuron/packages.yaml +++ b/platforms/neuron/packages.yaml @@ -18,10 +18,8 @@ archs: - "amd64" packages: - - "libgcc-s1" # implicit dep of libneuronpjrt/libstdc++ + - "bash" # needed to launch neuronx-cc - "libgomp1" # implicit dep of neuronx-cc - - "libstdc++6" # implicit dep of libneuronpjrt/libnccom - - "zlib1g" # implicit dep of the sandboxed Python runtime + - "zlib1g" # implicit dep of libpython - "aws-neuronx-runtime-lib" - "aws-neuronx-collectives" - - "aws-neuronx-tools" diff --git a/platforms/neuron/python_launcher.zig b/platforms/neuron/python_launcher.zig deleted file mode 100644 index 075034ab..00000000 --- a/platforms/neuron/python_launcher.zig +++ /dev/null @@ -1,155 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); - -// Shared isolated-Python launcher used by Neuron helper binaries. The sandbox -// contains the Python stdlib, site-packages, and these tiny Zig executables; the -// helpers configure Python from their own location so they are hermetic. - -pub const ModuleEntrypoint = struct { - module_name: [:0]const u8, - attr_name: [:0]const u8 = "main", -}; - -// Run `module.attr()` inside the sandboxed Python interpreter. -pub fn runModuleEntrypoint(init: std.process.Init, entrypoint: ModuleEntrypoint) !void { - var config: c.PyConfig = undefined; - _ = try configureInterpreter(init, &config); - defer c.PyConfig_Clear(&config); - - pyStatusCheck(c.Py_InitializeFromConfig(&config)); - defer c.Py_Finalize(); - - const module = c.PyImport_ImportModule(entrypoint.module_name.ptr) orelse { - std.log.err("Failed to import {s}", .{entrypoint.module_name}); - c.PyErr_Print(); - return error.FailedToImportModule; - }; - defer c.Py_DecRef(module); - - const main_fn = c.PyObject_GetAttrString(module, entrypoint.attr_name.ptr) orelse { - std.log.err("Failed to resolve {s}.{s}", .{ entrypoint.module_name, entrypoint.attr_name }); - c.PyErr_Print(); - return error.FailedToResolveEntrypoint; - }; - defer c.Py_DecRef(main_fn); - - const result = c.PyObject_CallNoArgs(main_fn); - if (result == null) { - c.PyErr_Print(); - std.process.exit(1); - } - defer c.Py_DecRef(result); -} - -// Run a Python file relative to the helper binary and call its `main()`. -pub fn runScriptMain(init: std.process.Init, relative_script_path: []const []const u8) !void { - const arena = init.arena; - - var config: c.PyConfig = undefined; - const self_exe_dir = try configureInterpreter(init, &config); - defer c.PyConfig_Clear(&config); - - pyStatusCheck(c.Py_InitializeFromConfig(&config)); - defer c.Py_Finalize(); - - const script_path = try std.Io.Dir.path.join(arena.allocator(), relative_script_path); - const full_script_path = try std.Io.Dir.path.join(arena.allocator(), &.{ self_exe_dir, script_path }); - const bootstrap = try std.fmt.allocPrint(arena.allocator(), - \\import importlib.util - \\spec = importlib.util.spec_from_file_location("_zml_neuron_python_launcher", "{s}") - \\module = importlib.util.module_from_spec(spec) - \\assert spec.loader is not None - \\spec.loader.exec_module(module) - \\module.main() - , .{full_script_path}); - - if (c.PyRun_SimpleStringFlags(bootstrap.ptr, null) != 0) { - c.PyErr_Print(); - return error.CompilerExecutionFailed; - } -} - -fn pyStatusCheck(status: c.PyStatus) void { - if (c.PyStatus_Exception(status) != 0) { - if (c.PyStatus_IsExit(status) != 0) { - std.process.exit(@intCast(status.exitcode)); - } - c.Py_ExitStatusException(status); - } -} - -fn toPosixPathW(file_path: []const u8) error{ InvalidUtf8, NameTooLong, CodepointTooLarge }![std.posix.PATH_MAX - 1:0]c.wchar_t { - var view = std.unicode.Utf8View.init(file_path) catch return error.InvalidUtf8; - var it = view.iterator(); - - var path_with_null: [std.posix.PATH_MAX - 1:0]c.wchar_t = undefined; - var len: usize = 0; - while (it.nextCodepoint()) |cp| { - if (len + 1 >= path_with_null.len) return error.NameTooLong; - path_with_null[len] = std.math.cast(c.wchar_t, cp) orelse return error.CodepointTooLarge; - len += 1; - } - path_with_null[len] = 0; - return path_with_null; -} - -fn configureInterpreter(init: std.process.Init, config: *c.PyConfig) ![]const u8 { - const arena = init.arena; - const io = init.io; - - { - var preconfig: c.PyPreConfig = undefined; - c.PyPreConfig_InitIsolatedConfig(&preconfig); - preconfig.utf8_mode = 1; - pyStatusCheck(c.Py_PreInitialize(&preconfig)); - } - - c.PyConfig_InitIsolatedConfig(config); - config.module_search_paths_set = 1; - config.optimization_level = 2; - config.write_bytecode = 0; - - const args = try init.minimal.args.toSlice(arena.allocator()); - const c_args = try arena.allocator().alloc([*:0]const u8, args.len); - for (args, 0..) |arg, i| { - c_args[i] = (try arena.allocator().dupeSentinel(u8, arg, 0)).ptr; - } - pyStatusCheck(c.PyConfig_SetBytesArgv(config, @intCast(c_args.len), @ptrCast(c_args.ptr))); - - var self_exe_dir_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const self_exe_dir = blk: { - const n = try std.process.executableDirPath(io, &self_exe_dir_buf); - break :blk self_exe_dir_buf[0..n]; - }; - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const home = try std.mem.printSentinel(&buf, "{f}{d}.{d}", .{ - std.Io.Dir.path.fmtJoin(&.{ - self_exe_dir, - "..", - "lib", - "python", - }), - c.PY_MAJOR_VERSION, - c.PY_MINOR_VERSION, - }, 0); - pyStatusCheck(c.PyConfig_SetBytesString(config, &config.home, home)); - pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(home))); - } - - { - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const site_packages = try std.fmt.bufPrint(&buf, "{f}", .{ - std.Io.Dir.path.fmtJoin(&.{ - self_exe_dir, - "..", - "site-packages", - }), - }); - pyStatusCheck(c.PyWideStringList_Append(&config.module_search_paths, &try toPosixPathW(site_packages))); - } - - return self_exe_dir; -} diff --git a/platforms/neuron/requirements.in b/platforms/neuron/requirements.in index 51174303..f574686b 100644 --- a/platforms/neuron/requirements.in +++ b/platforms/neuron/requirements.in @@ -1,5 +1,4 @@ --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html --extra-index-url=https://pip.repos.neuron.amazonaws.com -libneuronxla>=3.0 -neuronx-cc>=2.25 -nki>=0.4 +libneuronxla>=2.2 +neuronx-cc>=2.22 diff --git a/platforms/neuron/requirements.lock.txt b/platforms/neuron/requirements.lock.txt index 0945aa7d..19c7bb90 100644 --- a/platforms/neuron/requirements.lock.txt +++ b/platforms/neuron/requirements.lock.txt @@ -4,159 +4,143 @@ --extra-index-url https://pip.repos.neuron.amazonaws.com --find-links https://mirror.zml.ai/pypi/aws-neuronx-runtime-discovery/index.html -boto3==1.43.14 \ - --hash=sha256:574335744656cfed0b362a0a0467aaf2eb2bf15526edcd02d31d3c661f4b09e4 \ - --hash=sha256:5c0a994b3182061ee101812e721100717a4d664f9f4ceaf4a86b6d032ce9fc2d +boto3==1.42.24 \ + --hash=sha256:8ed6ad670a5a2d7f66c1b0d3362791b48392c7a08f78479f5d8ab319a4d9118f \ + --hash=sha256:c47a2f40df933e3861fc66fd8d6b87ee36d4361663a7e7ba39a87f5a78b2eae1 # via libneuronxla -botocore==1.43.14 \ - --hash=sha256:1f4a2a95ea78c10398e78431e98c1fe47adb54a7b10a32975144c1f541186658 \ - --hash=sha256:b9e500737e43d2f147c9d4e23b54360335e77d4c0ba90a318f51b65e06cb8516 +botocore==1.42.24 \ + --hash=sha256:8fca9781d7c84f7ad070fceffaff7179c4aa7a5ffb27b43df9d1d957801e0a8d \ + --hash=sha256:be8d1bea64fb91eea08254a1e5fea057e4428d08e61f4e11083a02cafc1f8cc6 # via # boto3 # libneuronxla # s3transfer -certifi==2026.5.20 \ - --hash=sha256:3c52e209ba0a4ad7aebe60436a4ab349c39e1e602e8c134221e546902ad25897 \ - --hash=sha256:69dea482ab64caa7b9f6aba1c6bf48bb6a5448d1c0f1b17ab42ad8c763a5344d +certifi==2026.1.4 \ + --hash=sha256:9943707519e4add1115f44c2bc244f782c0249876bf51b6599fee1ffbedd685c \ + --hash=sha256:ac726dd470482006e014ad384921ed6438c457018f4b3d204aea4281258b2120 # via requests -charset-normalizer==3.4.7 \ - --hash=sha256:007d05ec7321d12a40227aae9e2bc6dca73f3cb21058999a1df9e193555a9dcc \ - --hash=sha256:03853ed82eeebbce3c2abfdbc98c96dc205f32a79627688ac9a27370ea61a49c \ - --hash=sha256:07d9e39b01743c3717745f4c530a6349eadbfa043c7577eef86c502c15df2c67 \ - --hash=sha256:08e721811161356f97b4059a9ba7bafb23ea5ee2255402c42881c214e173c6b4 \ - --hash=sha256:0c96c3b819b5c3e9e165495db84d41914d6894d55181d2d108cc1a69bfc9cce0 \ - --hash=sha256:0ea948db76d31190bf08bd371623927ee1339d5f2a0b4b1b4a4439a65298703c \ - --hash=sha256:0f7eb884681e3938906ed0434f20c63046eacd0111c4ba96f27b76084cd679f5 \ - --hash=sha256:12a6fff75f6bc66711b73a2f0addfc4c8c15a20e805146a02d147a318962c444 \ - --hash=sha256:12d8baf840cc7889b37c7c770f478adea7adce3dcb3944d02ec87508e2dcf153 \ - --hash=sha256:14265bfe1f09498b9d8ec91e9ec9fa52775edf90fcbde092b25f4a33d444fea9 \ - --hash=sha256:16d971e29578a5e97d7117866d15889a4a07befe0e87e703ed63cd90cb348c01 \ - --hash=sha256:177a0ba5f0211d488e295aaf82707237e331c24788d8d76c96c5a41594723217 \ - --hash=sha256:1a87ca9d5df6fe460483d9a5bbf2b18f620cbed41b432e2bddb686228282d10b \ - --hash=sha256:1c2a768fdd44ee4a9339a9b0b130049139b8ce3c01d2ce09f67f5a68048d477c \ - --hash=sha256:1c2aed2e5e41f24ea8ef1590b8e848a79b56f3a5564a65ceec43c9d692dc7d8a \ - --hash=sha256:1dc8b0ea451d6e69735094606991f32867807881400f808a106ee1d963c46a83 \ - --hash=sha256:1efde3cae86c8c273f1eb3b287be7d8499420cf2fe7585c41d370d3e790054a5 \ - --hash=sha256:202389074300232baeb53ae2569a60901f7efadd4245cf3a3bf0617d60b439d7 \ - --hash=sha256:203104ed3e428044fd943bc4bf45fa73c0730391f9621e37fe39ecf477b128cb \ - --hash=sha256:2257141f39fe65a3fdf38aeccae4b953e5f3b3324f4ff0daf9f15b8518666a2c \ - --hash=sha256:298930cec56029e05497a76988377cbd7457ba864beeea92ad7e844fe74cd1f1 \ - --hash=sha256:2cd4a60d0e2fb04537162c62bbbb4182f53541fe0ede35cdf270a1c1e723cc42 \ - --hash=sha256:2d6eb928e13016cea4f1f21d1e10c1cebd5a421bc57ddf5b1142ae3f86824fab \ - --hash=sha256:2fe249cb4651fd12605b7288b24751d8bfd46d35f12a20b1ba33dea122e690df \ - --hash=sha256:30b8d1d8c52a48c2c5690e152c169b673487a2a58de1ec7393196753063fcd5e \ - --hash=sha256:320ade88cfb846b8cd6b4ddf5ee9e80ee0c1f52401f2456b84ae1ae6a1a5f207 \ - --hash=sha256:3534e7dcbdcf757da6b85a0bbf5b6868786d5982dd959b065e65481644817a18 \ - --hash=sha256:36836d6ff945a00b88ba1e4572d721e60b5b8c98c155d465f56ad19d68f23734 \ - --hash=sha256:38c0109396c4cfc574d502df99742a45c72c08eff0a36158b6f04000043dbf38 \ - --hash=sha256:3946fa46a0cf3e4c8cb1cc52f56bb536310d34f25f01ca9b6c16afa767dab110 \ - --hash=sha256:3bec022aec2c514d9cf199522a802bd007cd588ab17ab2525f20f9c34d067c18 \ - --hash=sha256:3c9a494bc5ec77d43cea229c4f6db1e4d8fe7e1bbffa8b6f0f0032430ff8ab44 \ - --hash=sha256:3dce51d0f5e7951f8bb4900c257dad282f49190fdbebecd4ba99bcc41fef404d \ - --hash=sha256:3dedcc22d73ec993f42055eff4fcfed9318d1eeb9a6606c55892a26964964e48 \ - --hash=sha256:4042d5c8f957e15221d423ba781e85d553722fc4113f523f2feb7b188cc34c5e \ - --hash=sha256:481551899c856c704d58119b5025793fa6730adda3571971af568f66d2424bb5 \ - --hash=sha256:4dc1e73c36828f982bfe79fadf5919923f8a6f4df2860804db9a98c48824ce8d \ - --hash=sha256:4e5163c14bffd570ef2affbfdd77bba66383890797df43dc8b4cc7d6f500bf53 \ - --hash=sha256:511ef87c8aec0783e08ac18565a16d435372bc1ac25a91e6ac7f5ef2b0bff790 \ - --hash=sha256:532bc9bf33a68613fd7d65e4b1c71a6a38d7d42604ecf239c77392e9b4e8998c \ - --hash=sha256:54523e136b8948060c0fa0bc7b1b50c32c186f2fceee897a495406bb6e311d2b \ - --hash=sha256:5649fd1c7bade02f320a462fdefd0b4bd3ce036065836d4f42e0de958038e116 \ - --hash=sha256:56be790f86bfb2c98fb742ce566dfb4816e5a83384616ab59c49e0604d49c51d \ - --hash=sha256:5b77459df20e08151cd6f8b9ef8ef1f961ef73d85c21a555c7eed5b79410ec10 \ - --hash=sha256:5ed6ab538499c8644b8a3e18debabcd7ce684f3fa91cf867521a7a0279cab2d6 \ - --hash=sha256:6178f72c5508bfc5fd446a5905e698c6212932f25bcdd4b47a757a50605a90e2 \ - --hash=sha256:6370e8686f662e6a3941ee48ed4742317cafbe5707e36406e9df792cdb535776 \ - --hash=sha256:64f02c6841d7d83f832cd97ccf8eb8a906d06eb95d5276069175c696b024b60a \ - --hash=sha256:65bcd23054beab4d166035cabbc868a09c1a49d1efe458fe8e4361215df40265 \ - --hash=sha256:66671f93accb62ed07da56613636f3641f1a12c13046ce91ffc923721f23c008 \ - --hash=sha256:6696b7688f54f5af4462118f0bfa7c1621eeb87154f77fa04b9295ce7a8f2943 \ - --hash=sha256:6785f414ae0f3c733c437e0f3929197934f526d19dfaa75e18fdb4f94c6fb374 \ - --hash=sha256:67f6279d125ca0046a7fd386d01b311c6363844deac3e5b069b514ba3e63c246 \ - --hash=sha256:6c114670c45346afedc0d947faf3c7f701051d2518b943679c8ff88befe14f8e \ - --hash=sha256:6e0d51f618228538a3e8f46bd246f87a6cd030565e015803691603f55e12afb5 \ - --hash=sha256:6ed74185b2db44f41ef35fd1617c5888e59792da9bbc9190d6c7300617182616 \ - --hash=sha256:708838739abf24b2ceb208d0e22403dd018faeef86ddac04319a62ae884c4f15 \ - --hash=sha256:715479b9a2802ecac752a3b0efa2b0b60285cf962ee38414211abdfccc233b41 \ - --hash=sha256:733784b6d6def852c814bce5f318d25da2ee65dd4839a0718641c696e09a2960 \ - --hash=sha256:750e02e074872a3fad7f233b47734166440af3cdea0add3e95163110816d6752 \ - --hash=sha256:752a45dc4a6934060b3b0dab47e04edc3326575f82be64bc4fc293914566503e \ - --hash=sha256:7579e913a5339fb8fa133f6bbcfd8e6749696206cf05acdbdca71a1b436d8e72 \ - --hash=sha256:7641bb8895e77f921102f72833904dcd9901df5d6d72a2ab8f31d04b7e51e4e7 \ - --hash=sha256:7804338df6fcc08105c7745f1502ba68d900f45fd770d5bdd5288ddccb8a42d8 \ - --hash=sha256:80d04837f55fc81da168b98de4f4b797ef007fc8a79ab71c6ec9bc4dd662b15b \ - --hash=sha256:813c0e0132266c08eb87469a642cb30aaff57c5f426255419572aaeceeaa7bf4 \ - --hash=sha256:82b271f5137d07749f7bf32f70b17ab6eaabedd297e75dce75081a24f76eb545 \ - --hash=sha256:84c018e49c3bf790f9c2771c45e9313a08c2c2a6342b162cd650258b57817706 \ - --hash=sha256:8751d2787c9131302398b11e6c8068053dcb55d5a8964e114b6e196cf16cb366 \ - --hash=sha256:8778f0c7a52e56f75d12dae53ae320fae900a8b9b4164b981b9c5ce059cd1fcb \ - --hash=sha256:87fad7d9ba98c86bcb41b2dc8dbb326619be2562af1f8ff50776a39e55721c5a \ - --hash=sha256:8d828b6667a32a728a1ad1d93957cdf37489c57b97ae6c4de2860fa749b8fc1e \ - --hash=sha256:8e385e4267ab76874ae30db04c627faaaf0b509e1ccc11a95b3fc3e83f855c00 \ - --hash=sha256:92a0a01ead5e668468e952e4238cccd7c537364eb7d851ab144ab6627dbbe12f \ - --hash=sha256:94e1885b270625a9a828c9793b4d52a64445299baa1fea5a173bf1d3dd9a1a5a \ - --hash=sha256:a180c5e59792af262bf263b21a3c49353f25945d8d9f70628e73de370d55e1e1 \ - --hash=sha256:a277ab8928b9f299723bc1a2dabb1265911b1a76341f90a510368ca44ad9ab66 \ - --hash=sha256:a5fe03b42827c13cdccd08e6c0247b6a6d4b5e3cdc53fd1749f5896adcdc2356 \ - --hash=sha256:a6c5863edfbe888d9eff9c8b8087354e27618d9da76425c119293f11712a6319 \ - --hash=sha256:a89c23ef8d2c6b27fd200a42aa4ac72786e7c60d40efdc76e6011260b6e949c4 \ - --hash=sha256:adb2597b428735679446b46c8badf467b4ca5f5056aae4d51a19f9570301b1ad \ - --hash=sha256:ae196f021b5e7c78e918242d217db021ed2a6ace2bc6ae94c0fc596221c7f58d \ - --hash=sha256:ae89db9e5f98a11a4bf50407d4363e7b09b31e55bc117b4f7d80aab97ba009e5 \ - --hash=sha256:aed52fea0513bac0ccde438c188c8a471c4e0f457c2dd20cdbf6ea7a450046c7 \ - --hash=sha256:aef65cd602a6d0e0ff6f9930fcb1c8fec60dd2cfcb6facaf4bdb0e5873042db0 \ - --hash=sha256:af21eb4409a119e365397b2adbaca4c9ccab56543a65d5dbd9f920d6ac29f686 \ - --hash=sha256:b14b2d9dac08e28bb8046a1a0434b1750eb221c8f5b87a68f4fa11a6f97b5e34 \ - --hash=sha256:bb6d88045545b26da47aa879dd4a89a71d1dce0f0e549b1abcb31dfe4a8eac49 \ - --hash=sha256:bb8cc7534f51d9a017b93e3e85b260924f909601c3df002bcdb58ddb4dc41a5c \ - --hash=sha256:bc17a677b21b3502a21f66a8cc64f5bfad4df8a0b8434d661666f8ce90ac3af1 \ - --hash=sha256:bd6c2a1c7573c64738d716488d2cdd3c00e340e4835707d8fdb8dc1a66ef164e \ - --hash=sha256:bd9b23791fe793e4968dba0c447e12f78e425c59fc0e3b97f6450f4781f3ee60 \ - --hash=sha256:c03a41a8784091e67a39648f70c5f97b5b6a37f216896d44d2cdcb82615339a0 \ - --hash=sha256:c0f081d69a6e58272819b70288d3221a6ee64b98df852631c80f293514d3b274 \ - --hash=sha256:c35abb8bfff0185efac5878da64c45dafd2b37fb0383add1be155a763c1f083d \ - --hash=sha256:c36c333c39be2dbca264d7803333c896ab8fa7d4d6f0ab7edb7dfd7aea6e98c0 \ - --hash=sha256:c45e9440fb78f8ddabcf714b68f936737a121355bf59f3907f4e17721b9d1aae \ - --hash=sha256:c593052c465475e64bbfe5dbd81680f64a67fdc752c56d7a0ae205dc8aeefe0f \ - --hash=sha256:cdd68a1fb318e290a2077696b7eb7a21a49163c455979c639bf5a5dcdc46617d \ - --hash=sha256:ce3412fbe1e31eb81ea42f4169ed94861c56e643189e1e75f0041f3fe7020abe \ - --hash=sha256:cf1493cd8607bec4d8a7b9b004e699fcf8f9103a9284cc94962cb73d20f9d4a3 \ - --hash=sha256:cf29836da5119f3c8a8a70667b0ef5fdca3bb12f80fd06487cfa575b3909b393 \ - --hash=sha256:d4a48e5b3c2a489fae013b7589308a40146ee081f6f509e047e0e096084ceca1 \ - --hash=sha256:d560742f3c0d62afaccf9f41fe485ed69bd7661a241f86a3ef0f0fb8b1a397af \ - --hash=sha256:d6038d37043bced98a66e68d3aa2b6a35505dc01328cd65217cefe82f25def44 \ - --hash=sha256:d61f00a0869d77422d9b2aba989e2d24afa6ffd552af442e0e58de4f35ea6d00 \ - --hash=sha256:d635aab80466bc95771bb78d5370e74d36d1fe31467b6b29b8b57b2a3cd7d22c \ - --hash=sha256:dca4bbc466a95ba9c0234ef56d7dd9509f63da22274589ebd4ed7f1f4d4c54e3 \ - --hash=sha256:dd915403e231e6b1809fe9b6d9fc55cf8fb5e02765ac625d9cd623342a7905d7 \ - --hash=sha256:e044c39e41b92c845bc815e5ae4230804e8e7bc29e399b0437d64222d92809dd \ - --hash=sha256:e060d01aec0a910bdccb8be71faf34e7799ce36950f8294c8bf612cba65a2c9e \ - --hash=sha256:e1421b502d83040e6d7fb2fb18dff63957f720da3d77b2fbd3187ceb63755d7b \ - --hash=sha256:e17b8d5d6a8c47c85e68ca8379def1303fd360c3e22093a807cd34a71cd082b8 \ - --hash=sha256:e5f4d355f0a2b1a31bc3edec6795b46324349c9cb25eed068049e4f472fb4259 \ - --hash=sha256:e712b419df8ba5e42b226c510472b37bd57b38e897d3eca5e8cfd410a29fa859 \ - --hash=sha256:e74327fb75de8986940def6e8dee4f127cc9752bee7355bb323cc5b2659b6d46 \ - --hash=sha256:e80c8378d8f3d83cd3164da1ad2df9e37a666cdde7b1cb2298ed0b558064be30 \ - --hash=sha256:e8ac484bf18ce6975760921bb6148041faa8fef0547200386ea0b52b5d27bf7b \ - --hash=sha256:eca9705049ad3c7345d574e3510665cb2cf844c2f2dcfe675332677f081cbd46 \ - --hash=sha256:ed065083d0898c9d5b4bbec7b026fd755ff7454e6e8b73a67f8c744b13986e24 \ - --hash=sha256:edac0f1ab77644605be2cbba52e6b7f630731fc42b34cb0f634be1a6eface56a \ - --hash=sha256:effc3f449787117233702311a1b7d8f59cba9ced946ba727bdc329ec69028e24 \ - --hash=sha256:f22dec1690b584cea26fade98b2435c132c1b5f68e39f5a0b7627cd7ae31f1dc \ - --hash=sha256:f495a1652cf3fbab2eb0639776dad966c2fb874d79d87ca07f9d5f059b8bd215 \ - --hash=sha256:f496c9c3cc02230093d8330875c4c3cdfc3b73612a5fd921c65d39cbcef08063 \ - --hash=sha256:f59099f9b66f0d7145115e6f80dd8b1d847176df89b234a5a6b3f00437aa0832 \ - --hash=sha256:f59ad4c0e8f6bba240a9bb85504faa1ab438237199d4cce5f622761507b8f6a6 \ - --hash=sha256:fbccdc05410c9ee21bbf16a35f4c1d16123dcdeb8a1d38f33654fa21d0234f79 \ - --hash=sha256:fea24543955a6a729c45a73fe90e08c743f0b3334bbf3201e6c4bc1b0c7fa464 +charset-normalizer==3.4.4 \ + --hash=sha256:027f6de494925c0ab2a55eab46ae5129951638a49a34d87f4c3eda90f696b4ad \ + --hash=sha256:077fbb858e903c73f6c9db43374fd213b0b6a778106bc7032446a8e8b5b38b93 \ + --hash=sha256:0a98e6759f854bd25a58a73fa88833fba3b7c491169f86ce1180c948ab3fd394 \ + --hash=sha256:0d3d8f15c07f86e9ff82319b3d9ef6f4bf907608f53fe9d92b28ea9ae3d1fd89 \ + --hash=sha256:0f04b14ffe5fdc8c4933862d8306109a2c51e0704acfa35d51598eb45a1e89fc \ + --hash=sha256:11d694519d7f29d6cd09f6ac70028dba10f92f6cdd059096db198c283794ac86 \ + --hash=sha256:194f08cbb32dc406d6e1aea671a68be0823673db2832b38405deba2fb0d88f63 \ + --hash=sha256:1bee1e43c28aa63cb16e5c14e582580546b08e535299b8b6158a7c9c768a1f3d \ + --hash=sha256:21d142cc6c0ec30d2efee5068ca36c128a30b0f2c53c1c07bd78cb6bc1d3be5f \ + --hash=sha256:2437418e20515acec67d86e12bf70056a33abdacb5cb1655042f6538d6b085a8 \ + --hash=sha256:244bfb999c71b35de57821b8ea746b24e863398194a4014e4c76adc2bbdfeff0 \ + --hash=sha256:2677acec1a2f8ef614c6888b5b4ae4060cc184174a938ed4e8ef690e15d3e505 \ + --hash=sha256:277e970e750505ed74c832b4bf75dac7476262ee2a013f5574dd49075879e161 \ + --hash=sha256:2aaba3b0819274cc41757a1da876f810a3e4d7b6eb25699253a4effef9e8e4af \ + --hash=sha256:2b7d8f6c26245217bd2ad053761201e9f9680f8ce52f0fcd8d0755aeae5b2152 \ + --hash=sha256:2c9d3c380143a1fedbff95a312aa798578371eb29da42106a29019368a475318 \ + --hash=sha256:3162d5d8ce1bb98dd51af660f2121c55d0fa541b46dff7bb9b9f86ea1d87de72 \ + --hash=sha256:31fd66405eaf47bb62e8cd575dc621c56c668f27d46a61d975a249930dd5e2a4 \ + --hash=sha256:362d61fd13843997c1c446760ef36f240cf81d3ebf74ac62652aebaf7838561e \ + --hash=sha256:376bec83a63b8021bb5c8ea75e21c4ccb86e7e45ca4eb81146091b56599b80c3 \ + --hash=sha256:44c2a8734b333e0578090c4cd6b16f275e07aa6614ca8715e6c038e865e70576 \ + --hash=sha256:47cc91b2f4dd2833fddaedd2893006b0106129d4b94fdb6af1f4ce5a9965577c \ + --hash=sha256:4902828217069c3c5c71094537a8e623f5d097858ac6ca8252f7b4d10b7560f1 \ + --hash=sha256:4bd5d4137d500351a30687c2d3971758aac9a19208fc110ccb9d7188fbe709e8 \ + --hash=sha256:4fe7859a4e3e8457458e2ff592f15ccb02f3da787fcd31e0183879c3ad4692a1 \ + --hash=sha256:542d2cee80be6f80247095cc36c418f7bddd14f4a6de45af91dfad36d817bba2 \ + --hash=sha256:554af85e960429cf30784dd47447d5125aaa3b99a6f0683589dbd27e2f45da44 \ + --hash=sha256:5833d2c39d8896e4e19b689ffc198f08ea58116bee26dea51e362ecc7cd3ed26 \ + --hash=sha256:5947809c8a2417be3267efc979c47d76a079758166f7d43ef5ae8e9f92751f88 \ + --hash=sha256:5ae497466c7901d54b639cf42d5b8c1b6a4fead55215500d2f486d34db48d016 \ + --hash=sha256:5bd2293095d766545ec1a8f612559f6b40abc0eb18bb2f5d1171872d34036ede \ + --hash=sha256:5bfbb1b9acf3334612667b61bd3002196fe2a1eb4dd74d247e0f2a4d50ec9bbf \ + --hash=sha256:5cb4d72eea50c8868f5288b7f7f33ed276118325c1dfd3957089f6b519e1382a \ + --hash=sha256:5dbe56a36425d26d6cfb40ce79c314a2e4dd6211d51d6d2191c00bed34f354cc \ + --hash=sha256:5f819d5fe9234f9f82d75bdfa9aef3a3d72c4d24a6e57aeaebba32a704553aa0 \ + --hash=sha256:64b55f9dce520635f018f907ff1b0df1fdc31f2795a922fb49dd14fbcdf48c84 \ + --hash=sha256:6515f3182dbe4ea06ced2d9e8666d97b46ef4c75e326b79bb624110f122551db \ + --hash=sha256:65e2befcd84bc6f37095f5961e68a6f077bf44946771354a28ad434c2cce0ae1 \ + --hash=sha256:6aee717dcfead04c6eb1ce3bd29ac1e22663cdea57f943c87d1eab9a025438d7 \ + --hash=sha256:6b39f987ae8ccdf0d2642338faf2abb1862340facc796048b604ef14919e55ed \ + --hash=sha256:6e1fcf0720908f200cd21aa4e6750a48ff6ce4afe7ff5a79a90d5ed8a08296f8 \ + --hash=sha256:74018750915ee7ad843a774364e13a3db91682f26142baddf775342c3f5b1133 \ + --hash=sha256:74664978bb272435107de04e36db5a9735e78232b85b77d45cfb38f758efd33e \ + --hash=sha256:74bb723680f9f7a6234dcf67aea57e708ec1fbdf5699fb91dfd6f511b0a320ef \ + --hash=sha256:752944c7ffbfdd10c074dc58ec2d5a8a4cd9493b314d367c14d24c17684ddd14 \ + --hash=sha256:778d2e08eda00f4256d7f672ca9fef386071c9202f5e4607920b86d7803387f2 \ + --hash=sha256:780236ac706e66881f3b7f2f32dfe90507a09e67d1d454c762cf642e6e1586e0 \ + --hash=sha256:798d75d81754988d2565bff1b97ba5a44411867c0cf32b77a7e8f8d84796b10d \ + --hash=sha256:799a7a5e4fb2d5898c60b640fd4981d6a25f1c11790935a44ce38c54e985f828 \ + --hash=sha256:7a32c560861a02ff789ad905a2fe94e3f840803362c84fecf1851cb4cf3dc37f \ + --hash=sha256:7c308f7e26e4363d79df40ca5b2be1c6ba9f02bdbccfed5abddb7859a6ce72cf \ + --hash=sha256:7fa17817dc5625de8a027cb8b26d9fefa3ea28c8253929b8d6649e705d2835b6 \ + --hash=sha256:81d5eb2a312700f4ecaa977a8235b634ce853200e828fbadf3a9c50bab278328 \ + --hash=sha256:82004af6c302b5d3ab2cfc4cc5f29db16123b1a8417f2e25f9066f91d4411090 \ + --hash=sha256:837c2ce8c5a65a2035be9b3569c684358dfbf109fd3b6969630a87535495ceaa \ + --hash=sha256:840c25fb618a231545cbab0564a799f101b63b9901f2569faecd6b222ac72381 \ + --hash=sha256:8a6562c3700cce886c5be75ade4a5db4214fda19fede41d9792d100288d8f94c \ + --hash=sha256:8af65f14dc14a79b924524b1e7fffe304517b2bff5a58bf64f30b98bbc5079eb \ + --hash=sha256:8ef3c867360f88ac904fd3f5e1f902f13307af9052646963ee08ff4f131adafc \ + --hash=sha256:94537985111c35f28720e43603b8e7b43a6ecfb2ce1d3058bbe955b73404e21a \ + --hash=sha256:99ae2cffebb06e6c22bdc25801d7b30f503cc87dbd283479e7b606f70aff57ec \ + --hash=sha256:9a26f18905b8dd5d685d6d07b0cdf98a79f3c7a918906af7cc143ea2e164c8bc \ + --hash=sha256:9b35f4c90079ff2e2edc5b26c0c77925e5d2d255c42c74fdb70fb49b172726ac \ + --hash=sha256:9cd98cdc06614a2f768d2b7286d66805f94c48cde050acdbbb7db2600ab3197e \ + --hash=sha256:9d1bb833febdff5c8927f922386db610b49db6e0d4f4ee29601d71e7c2694313 \ + --hash=sha256:9f7fcd74d410a36883701fafa2482a6af2ff5ba96b9a620e9e0721e28ead5569 \ + --hash=sha256:a59cb51917aa591b1c4e6a43c132f0cdc3c76dbad6155df4e28ee626cc77a0a3 \ + --hash=sha256:a61900df84c667873b292c3de315a786dd8dac506704dea57bc957bd31e22c7d \ + --hash=sha256:a79cfe37875f822425b89a82333404539ae63dbdddf97f84dcbc3d339aae9525 \ + --hash=sha256:a8a8b89589086a25749f471e6a900d3f662d1d3b6e2e59dcecf787b1cc3a1894 \ + --hash=sha256:a8bf8d0f749c5757af2142fe7903a9df1d2e8aa3841559b2bad34b08d0e2bcf3 \ + --hash=sha256:a9768c477b9d7bd54bc0c86dbaebdec6f03306675526c9927c0e8a04e8f94af9 \ + --hash=sha256:ac1c4a689edcc530fc9d9aa11f5774b9e2f33f9a0c6a57864e90908f5208d30a \ + --hash=sha256:af2d8c67d8e573d6de5bc30cdb27e9b95e49115cd9baad5ddbd1a6207aaa82a9 \ + --hash=sha256:b435cba5f4f750aa6c0a0d92c541fb79f69a387c91e61f1795227e4ed9cece14 \ + --hash=sha256:b5b290ccc2a263e8d185130284f8501e3e36c5e02750fc6b6bdeb2e9e96f1e25 \ + --hash=sha256:b5d84d37db046c5ca74ee7bb47dd6cbc13f80665fdde3e8040bdd3fb015ecb50 \ + --hash=sha256:b7cf1017d601aa35e6bb650b6ad28652c9cd78ee6caff19f3c28d03e1c80acbf \ + --hash=sha256:bc7637e2f80d8530ee4a78e878bce464f70087ce73cf7c1caf142416923b98f1 \ + --hash=sha256:c0463276121fdee9c49b98908b3a89c39be45d86d1dbaa22957e38f6321d4ce3 \ + --hash=sha256:c4ef880e27901b6cc782f1b95f82da9313c0eb95c3af699103088fa0ac3ce9ac \ + --hash=sha256:c8ae8a0f02f57a6e61203a31428fa1d677cbe50c93622b4149d5c0f319c1d19e \ + --hash=sha256:ca5862d5b3928c4940729dacc329aa9102900382fea192fc5e52eb69d6093815 \ + --hash=sha256:cb01158d8b88ee68f15949894ccc6712278243d95f344770fa7593fa2d94410c \ + --hash=sha256:cb6254dc36b47a990e59e1068afacdcd02958bdcce30bb50cc1700a8b9d624a6 \ + --hash=sha256:cc00f04ed596e9dc0da42ed17ac5e596c6ccba999ba6bd92b0e0aef2f170f2d6 \ + --hash=sha256:cd09d08005f958f370f539f186d10aec3377d55b9eeb0d796025d4886119d76e \ + --hash=sha256:cd4b7ca9984e5e7985c12bc60a6f173f3c958eae74f3ef6624bb6b26e2abbae4 \ + --hash=sha256:ce8a0633f41a967713a59c4139d29110c07e826d131a316b50ce11b1d79b4f84 \ + --hash=sha256:cead0978fc57397645f12578bfd2d5ea9138ea0fac82b2f63f7f7c6877986a69 \ + --hash=sha256:d055ec1e26e441f6187acf818b73564e6e6282709e9bcb5b63f5b23068356a15 \ + --hash=sha256:d1f13550535ad8cff21b8d757a3257963e951d96e20ec82ab44bc64aeb62a191 \ + --hash=sha256:d9c7f57c3d666a53421049053eaacdd14bbd0a528e2186fcb2e672effd053bb0 \ + --hash=sha256:d9e45d7faa48ee908174d8fe84854479ef838fc6a705c9315372eacbc2f02897 \ + --hash=sha256:da3326d9e65ef63a817ecbcc0df6e94463713b754fe293eaa03da99befb9a5bd \ + --hash=sha256:de00632ca48df9daf77a2c65a484531649261ec9f25489917f09e455cb09ddb2 \ + --hash=sha256:e1f185f86a6f3403aa2420e815904c67b2f9ebc443f045edd0de921108345794 \ + --hash=sha256:e824f1492727fa856dd6eda4f7cee25f8518a12f3c4a56a74e8095695089cf6d \ + --hash=sha256:e912091979546adf63357d7e2ccff9b44f026c075aeaf25a52d0e95ad2281074 \ + --hash=sha256:eaabd426fe94daf8fd157c32e571c85cb12e66692f15516a83a03264b08d06c3 \ + --hash=sha256:ebf3e58c7ec8a8bed6d66a75d7fb37b55e5015b03ceae72a8e7c74495551e224 \ + --hash=sha256:ecaae4149d99b1c9e7b88bb03e3221956f68fd6d50be2ef061b2381b61d20838 \ + --hash=sha256:eecbc200c7fd5ddb9a7f16c7decb07b566c29fa2161a16cf67b8d068bd21690a \ + --hash=sha256:f155a433c2ec037d4e8df17d18922c3a0d9b3232a396690f17175d2946f0218d \ + --hash=sha256:f1e34719c6ed0b92f418c7c780480b26b5d9c50349e9a9af7d76bf757530350d \ + --hash=sha256:f34be2938726fc13801220747472850852fe6b1ea75869a048d6f896838c896f \ + --hash=sha256:f820802628d2694cb7e56db99213f930856014862f3fd943d290ea8438d07ca8 \ + --hash=sha256:f8bf04158c6b607d747e93949aa60618b61312fe647a6369f88ce2ff16043490 \ + --hash=sha256:f8e160feb2aed042cd657a72acc0b481212ed28b1b9a95c0cee1621b524e1966 \ + --hash=sha256:f9d332f8c2a2fcbffe1378594431458ddbef721c1769d78e2cbc06280d8155f9 \ + --hash=sha256:fa09f53c465e532f4d3db095e0c55b615f010ad81803d383195b6b5ca6cbf5f3 \ + --hash=sha256:faa3a41b2b66b6e50f84ae4a68c64fcd0c44355741c6374813a800cd6695db9e \ + --hash=sha256:fd44c878ea55ba351104cb93cc85e74916eb8fa440ca7903e57575e97394f608 # via requests ec2-metadata==2.17.0 \ --hash=sha256:a696a899cb8ee1f0101eb9ec7bc2050e1ffc8574ee4cd748a52d9c748fb19d9e \ --hash=sha256:ad98a5afd8f4f49f688e4899dc505257da3237348761eceeb0fc7fc7d00cc90d # via neuronx-cc -idna==3.16 \ - --hash=sha256:cc246e3a3f89580c3a951b5ad298ca4638078b2cdd4f115654332b5c26daded5 \ - --hash=sha256:d7a6da03db833450fca25d2358ac9ff06cd624577a4aea3a596d5c0f77b8e03d +idna==3.11 \ + --hash=sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea \ + --hash=sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902 # via requests islpy==2023.2.5 \ --hash=sha256:06446395028b88701fd84b685b1c82b9de493728906f96f576a4dca69d12dd1f \ @@ -200,15 +184,15 @@ islpy==2023.2.5 \ --hash=sha256:ed0c62e64d160735ed1a7fc314d4c0f501711c282bb84a00ceaf590b4c4bd61f \ --hash=sha256:eeec97588bd5282c983bd73f9010156e5e251e9e5307af547e2cb53b02d4639a # via neuronx-cc -jmespath==1.1.0 \ - --hash=sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d \ - --hash=sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64 +jmespath==1.0.1 \ + --hash=sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 \ + --hash=sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe # via # boto3 # botocore -libneuronxla==3.0.2891.0+e2a4b1f5 \ - --hash=sha256:b453709b37565902acc85bcfb4011d55e99381b81d831a5559847bba71f9cec3 - # via -r platforms/neuron/requirements.in +libneuronxla==2.2.14584.0+06ac23d1 \ + --hash=sha256:d1e594b27716bc59b937ccd8f40e7f2b74f6c309643e83dcf511b7ea392924f2 + # via -r runtimes/neuron/requirements.in lockfile==0.12.2 \ --hash=sha256:6aed02de03cba24efabcd600b30540140634fc06cfa603822d508d5361e9f799 \ --hash=sha256:6c3cb24f344923d30b2785d5ad75182c8ea7ac1b6171b08657258ec7429d50fa @@ -253,137 +237,131 @@ ml-dtypes==0.5.4 \ --hash=sha256:ce756d3a10d0c4067172804c9cc276ba9cc0ff47af9078ad439b075d1abdc29b \ --hash=sha256:d81fdb088defa30eb37bf390bb7dde35d3a83ec112ac8e33d75ab28cc29dd8b0 \ --hash=sha256:f21c9219ef48ca5ee78402d5cc831bd58ea27ce89beda894428bc67a52da5328 - # via - # neuronx-cc - # nki + # via neuronx-cc networkx==2.8.8 \ --hash=sha256:230d388117af870fce5647a3c52401fcf753e94720e6ea6b4197a5355648885e \ --hash=sha256:e435dfa75b1d7195c7b8378c3859f0445cd88c6b0375c181ed66823a9ceb7524 # via neuronx-cc -neuronx-cc==2.25.3371.0+f524f7f8 \ - --hash=sha256:c71cb7333d5320b6ea91e13e0168eec01be5e7ddfd5515bf62aa47088ee274c2 +neuronx-cc==2.22.12471.0+b4a00d10 \ + --hash=sha256:19fcf633f6f453b73e791b6f23d0e3f741fd967a61b7b82208bd74d80908c81a # via - # -r platforms/neuron/requirements.in + # -r runtimes/neuron/requirements.in # libneuronxla -nki==0.4.0+25940409122.gd30719f9 \ - --hash=sha256:09b520140e55790ba00ebe3e7d53da1e5f221ff3d6c1823414acfebe4b794879 - # via - # -r platforms/neuron/requirements.in - # neuronx-cc -numpy==2.4.6 \ - --hash=sha256:001fbb8e08d942dd57599e781f2472269ee7f2755fae407b4f67b2f0b17da3f1 \ - --hash=sha256:0280e0356c0829a18d9de1cb7eee50ec22ca639878d7240307ca0943d73cd2c4 \ - --hash=sha256:043191bfa8eab18c776647b62723ac9dddece59743b13f49b2016094129c2b3f \ - --hash=sha256:06ca2f61ec4385a07a6977c55ba998a4466c123642b4a32694d3128fce18c079 \ - --hash=sha256:0a041d3d761dc3c35cc56ce0351506a02bcbc25f7b169f652435141a17db9096 \ - --hash=sha256:0ab0a9c4ffb1a6d95ef519fe4247dba8eb6b18ad93999f76b7f657039acabd47 \ - --hash=sha256:0c9136e14ed34a9e343a31c533d78a9813a69a3148332bce5e9821cb2f996e66 \ - --hash=sha256:110f8b71aacb688ec69062bb7f6938a0f8acb01b7c1c4beb453c65b6d234584d \ - --hash=sha256:112b06a867b235ef466ed3508ddf0238050df9c727cafb5301ac385b899189a1 \ - --hash=sha256:17f9ade344e7d9b464a084d69bcf18fc691cb1db67c62ed80820bf4926d78f0e \ - --hash=sha256:1e254a00cdf42b1e4d5b3d68d33af63268d41340d8885df2ab6470f2e1500147 \ - --hash=sha256:1e978ec1e8bd0e0e4de6bb75de9d30cbb74db6b6a2bb727618613703ca0167dd \ - --hash=sha256:25c692919ac5a01f170a3bfcd62d745b24fd095c353d50812637d6fcab442e75 \ - --hash=sha256:260a5d70215b61ab4fadf5c7baacd64821842975eea312125ed3c39a6391b063 \ - --hash=sha256:2803abfebfc990042cd494d8ce2d5f82e9d847af6d35ec486923aa19dbad5e73 \ - --hash=sha256:29a287e0cf63ff528da061de6b9f64a4618da591ca1046aafc54062e40ca7eab \ - --hash=sha256:29cb7f67d10b479ff07c17d33e39f78c07f71c40ef30d63c153d340e96cd3fb4 \ - --hash=sha256:3213d622a0283a39a93d188f3cf72b26862df52fbb4ca3697f51705016523d41 \ - --hash=sha256:33111801a01c12a8a1e3721f0a9232f8cfc8ae2c6b7098167e6f623c6073f402 \ - --hash=sha256:357cc07a6d7b0b182ff02249616a03742827ebb1277546b5c7cd7f7620a45698 \ - --hash=sha256:38efbc8de75c7a0fc1ac190162d892787f3f47b57cc291231aafee36b80982b7 \ - --hash=sha256:4081eb135ac24158bd51cdfbef16f1c64df7063b1143f24731387137c092bec8 \ - --hash=sha256:40fdc1ae7125e518ea98e53e69a4ebc27e1fd50510c47b7ea130cf21e5e1d42b \ - --hash=sha256:4cfe66903cc32a9921a6733d96b19bb6abf310397581bbad89c228f5abaf0ee8 \ - --hash=sha256:511dbaf848decaaaf4b4ca48032619fb3138710c4bf7da7617765edad1ef96b0 \ - --hash=sha256:55cced7c52e981362f708ad635198e97a752dfba412cc03c23bbf3bd8d5cd662 \ - --hash=sha256:56b39e5e0622a09a25bf5baf62f4bcf0cb8a41ae6e2819cf49bbc5a74c083f91 \ - --hash=sha256:5dbbdb29840ca3d91ee0fece42fc29278886d908280bfec0a5846c6f901a3eb0 \ - --hash=sha256:5f9fb9157b4ce2971008323afe46053787b526ef624fea915b261468a8421a0f \ - --hash=sha256:6180d8b35af935aed8ece3a85e0a43f87393ae0ac87c8d2c8bd2c993f7270ef3 \ - --hash=sha256:68a5124b13fa6cc2086764a20005d30bc0548146f7f5322f02fce212ca14317f \ - --hash=sha256:68bb27509ac1b9a3443094260f6326150663b06abe40b73a2f81160623da5b67 \ - --hash=sha256:6f41ae150c4e32db4f3310cdaf64b1593a03dbabe29eec77fc9b50fe64061df6 \ - --hash=sha256:7265a2f3d436e54ef9f2b52b5c937e6be778781bd97a590319d7348f1c1ca997 \ - --hash=sha256:72fbe16c6fac95aedf5937fa873445cec2110be35d8a4e9433d7501fd98dae6b \ - --hash=sha256:7d92c3819208a60205a12a245c91ad70cb0a85336659b19b834205573ac8456e \ - --hash=sha256:8155154c7c691289fe18f510b5d4657c68c67989f293f0535a91360392ff6538 \ - --hash=sha256:81a1cca95ed5bb92aa8b10dd2cdc9a0d3853a50fad926c28b5d7e8ea54389627 \ - --hash=sha256:89cd468399cfd2504718f0ba50e410dca55a170b61a02ad92bb18c8a65186e93 \ - --hash=sha256:8ad03c0965fb3c692200e74d458ca28c1dbb4ce96f9a479a8aa041ad5fabca02 \ - --hash=sha256:90f9849678c75fe7afa2d348ac842c168b0a4d3d61919687216dfc547976d853 \ - --hash=sha256:948424b06129ce883307e8cff868c31396d8dc7630a59c61d70d98dbe70f222c \ - --hash=sha256:9cd5ffd25db4e7ba6a375693b3fc0fc1791ec636c17db3720da19bde7180ec43 \ - --hash=sha256:a0df0043bdb289bde1f62da130d20df23d58b45429f752bc7a8fc5325a225ecd \ - --hash=sha256:a2c306dea656c12c68f51f4cea133cbe78ca7435eb28c735eac1d3ebe73be6e8 \ - --hash=sha256:a7830bab239b79cda9c08c2da014761cafb48da6150e1da17ac06283f43b6089 \ - --hash=sha256:a7c711e21628b52034bb5ab8d1bce291f752fcc5e92accc615778acee1ff4778 \ - --hash=sha256:aaf159caa35993cb1f56fb9b8e4610d35758e7ca005412eb1daa856a78c9c4b1 \ - --hash=sha256:ae506e6902902557576a26ff33eda8695e7ecb3cb36c3b573a0765dee114ebdb \ - --hash=sha256:b507f5c4c1d508876d1819b6bf9a49d365b96320b5d4993426b33a23ca4b8261 \ - --hash=sha256:bf162abab1c1a736333192707cef898e735a5ca00f38f27eeedf44b39d9e85eb \ - --hash=sha256:c1a2af6c6ef86344a6b0db6b97834208bf598db514f2b155042439b62605601a \ - --hash=sha256:c2d37ab77531417474168eb79d6d80b14f821a966818505d03013d0833edb7a8 \ - --hash=sha256:c4fc99836233ea196540b17ab0983aff60ed07941751930f5f4d05bc3b3b7359 \ - --hash=sha256:d581b735e177fdcdce6fed8e7e8880a3fb6ee4e3653a3ac6af01c6f4c03effc5 \ - --hash=sha256:d6da64deb6b8ed903e7560180a92f2d804ee1ba5eeb849ac2748b8c1aba1f6d7 \ - --hash=sha256:d8e8286dd7cea7895157318d1b91cdacac64c479f3cbc8dce548331728484751 \ - --hash=sha256:ddea102b48f9e339f3948bf22040944184627a30fdf7f858667673b9c5f033c8 \ - --hash=sha256:dfa20cc6ca228e6b155b11da03825975ce66aea520985dbbddf0f2a5a495c605 \ - --hash=sha256:e3e5193ef5a3dc73bceee50f7fdc2c90dbb76c42df8d8fae3d1067a583df579e \ - --hash=sha256:e3eeb0aabd6bd5ce64faae67e9935203a6991b4bc2a485a767fbafb2c5125f45 \ - --hash=sha256:e5805d5a22fd19c8ccff10a9561f9df94436b0545619ea579db2d3c35294bce2 \ - --hash=sha256:e85b752a1e912b70eaad4fafbd4d1238007ab221de2009b9a2f5ae7461239895 \ - --hash=sha256:eaf7fa2de5c0be8ae6ff8e9bea2ccd725e980541244521d8d4b5f3354a27babe \ - --hash=sha256:ebfb099f8dcf083deef3ac1ca4c1503f387cf76296fcb3816b66f5ecb5f54fdb \ - --hash=sha256:ece3d2cfe132e7d51f44a832b303895e6f2d499c5e74dfbdb06ee246147a304a \ - --hash=sha256:ed9749eef4cbd126da3dc1d6bcb3a57f5eb7ac6a6484146bdbf743f552dfc577 \ - --hash=sha256:ede83e07a75dd06bc501566c1eca2afc0d61677c1472ac9ad93fdee6e638a48d \ - --hash=sha256:ef4aea96ce4d3b074422cb4f2f64e216bf9e213004bb58ecfdf50ea02ea8eb9a \ - --hash=sha256:f3a3570c4a2a16746ac2c31a7c7c7b0c186b95ce902e33db6f28094ed7387dda \ - --hash=sha256:f407cb6b8e9d6d8c626bc73c945db1706035af8fd632295547bf1c9e46d092d6 \ - --hash=sha256:f74a575920ab21fe304421a3fc28793d82e299cae9eccb37084e9fc7f3617c20 +numpy==2.4.0 \ + --hash=sha256:009bd0ea12d3c784b6639a8457537016ce5172109e585338e11334f6a7bb88ee \ + --hash=sha256:02a2038eb27f9443a8b266a66911e926566b5a6ffd1a689b588f7f35b81e7dc3 \ + --hash=sha256:0804a8e4ab070d1d35496e65ffd3cf8114c136a2b81f61dfab0de4b218aacfd5 \ + --hash=sha256:0feafc9e03128074689183031181fac0897ff169692d8492066e949041096548 \ + --hash=sha256:1889b3a3f47a7b5bee16bc25a2145bd7cb91897f815ce3499db64c7458b6d91d \ + --hash=sha256:1d8f9fde5f6dc1b6fc34df8162f3b3079365468703fee7f31d4e0cc8c63baed9 \ + --hash=sha256:1ee32359fb7543b7b7bd0b2f46294db27e29e7bbdf70541e81b190836cd83ded \ + --hash=sha256:20c115517513831860c573996e395707aa9fb691eb179200125c250e895fcd93 \ + --hash=sha256:21555da4ec4a0c942520ead42c3b0dc9477441e085c42b0fbdd6a084869a6f6b \ + --hash=sha256:23a3e9d1a6f360267e8fbb38ba5db355a6a7e9be71d7fce7ab3125e88bb646c8 \ + --hash=sha256:2a8b6bb8369abefb8bd1801b054ad50e02b3275c8614dc6e5b0373c305291037 \ + --hash=sha256:2e284ca13d5a8367e43734148622caf0b261b275673823593e3e3634a6490f83 \ + --hash=sha256:2f585f52b2baf07ff3356158d9268ea095e221371f1074fadea2f42544d58b4d \ + --hash=sha256:316b2f2584682318539f0bcaca5a496ce9ca78c88066579ebd11fd06f8e4741e \ + --hash=sha256:32ed06d0fe9cae27d8fb5f400c63ccee72370599c75e683a6358dd3a4fb50aaf \ + --hash=sha256:355354388cba60f2132df297e2d53053d4063f79077b67b481d21276d61fc4df \ + --hash=sha256:35ddc8f4914466e6fc954c76527aa91aa763682a4f6d73249ef20b418fe6effb \ + --hash=sha256:36667db4d6c1cea79c8930ab72fadfb4060feb4bfe724141cd4bd064d2e5f8ce \ + --hash=sha256:36cbfb13c152b1c7c184ddac43765db8ad672567e7bafff2cc755a09917ed2e6 \ + --hash=sha256:39699233bc72dd482da1415dcb06076e32f60eddc796a796c5fb6c5efce94667 \ + --hash=sha256:39b19251dec4de8ff8496cd0806cbe27bf0684f765abb1f4809554de93785f2d \ + --hash=sha256:3d857f55e7fdf7c38ab96c4558c95b97d1c685be6b05c249f5fdafcbd6f9899e \ + --hash=sha256:3de19cfecd1465d0dcf8a5b5ea8b3155b42ed0b639dba4b71e323d74f2a3be5e \ + --hash=sha256:3f3096405acc48887458bbf9f6814d43785ac7ba2a57ea6442b581dedbc60ce6 \ + --hash=sha256:40483b2f2d3ba7aad426443767ff5632ec3156ef09742b96913787d13c336471 \ + --hash=sha256:413aa561266a4be2d06cd2b9665e89d9f54c543f418773076a76adcf2af08bc7 \ + --hash=sha256:44aba2f0cafd287871a495fb3163408b0bd25bbce135c6f621534a07f4f7875c \ + --hash=sha256:49ff32b09f5aa0cd30a20c2b39db3e669c845589f2b7fc910365210887e39344 \ + --hash=sha256:4d1cfce39e511069b11e67cd0bd78ceff31443b7c9e5c04db73c7a19f572967c \ + --hash=sha256:57c540ed8fb1f05cb997c6761cd56db72395b0d6985e90571ff660452ade4f98 \ + --hash=sha256:5f48cb3e88fbc294dc90e215d86fbaf1c852c63dbdb6c3a3e63f45c4b57f7344 \ + --hash=sha256:5fe44e277225fd3dff6882d86d3d447205d43532c3627313d17e754fb3905a0e \ + --hash=sha256:60e8c196cd82cbbd4f130b5290007e13e6de3eca79f0d4d38014769d96a7c475 \ + --hash=sha256:65167da969cd1ec3a1df31cb221ca3a19a8aaa25370ecb17d428415e93c1935e \ + --hash=sha256:680060061adb2d74ce352628cb798cfdec399068aa7f07ba9fb818b2b3305f98 \ + --hash=sha256:6bbaebf0d11567fa8926215ae731e1d58e6ec28a8a25235b8a47405d301332db \ + --hash=sha256:6c05483c3136ac4c91b4e81903cb53a8707d316f488124d0398499a4f8e8ef51 \ + --hash=sha256:6dc8b7e2f4eb184b37655195f421836cfae6f58197b67e3ffc501f1333d993fa \ + --hash=sha256:6e504f7b16118198f138ef31ba24d985b124c2c469fe8467007cf30fd992f934 \ + --hash=sha256:74623f2ab5cc3f7c886add4f735d1031a1d2be4a4ae63c0546cfd74e7a31ddf6 \ + --hash=sha256:77f0d13fa87036d7553bf81f0e1fe3ce68d14c9976c9851744e4d3e91127e95f \ + --hash=sha256:85eef4cb5625c47ee6425c58a3502555e10f45ee973da878ac8248ad58c136f3 \ + --hash=sha256:882567b7ae57c1b1a0250208cc21a7976d8cbcc49d5a322e607e6f09c9e0bd53 \ + --hash=sha256:8b986403023c8f3bf8f487c2e6186afda156174d31c175f747d8934dfddf3479 \ + --hash=sha256:8cfa5f29a695cb7438965e6c3e8d06e0416060cf0d709c1b1c1653a939bf5c2a \ + --hash=sha256:8eafe7c36c8430b7794edeab3087dec7bf31d634d92f2af9949434b9d1964cba \ + --hash=sha256:9198f447e1dc5647d07c9a6bbe2063cc0132728cc7175b39dbc796da5b54920d \ + --hash=sha256:92190db305a6f48734d3982f2c60fa30d6b5ee9bff10f2887b930d7b40119f4c \ + --hash=sha256:927ccf5cd17c48f801f4ed43a7e5673a2724bd2171460be3e3894e6e332ef83a \ + --hash=sha256:98e81648e0b36e325ab67e46b5400a7a6d4a22b8a7c8e8bbfe20e7db7906bf95 \ + --hash=sha256:9a818668b674047fd88c4cddada7ab8f1c298812783e8328e956b78dc4807f9f \ + --hash=sha256:a152d86a3ae00ba5f47b3acf3b827509fd0b6cb7d3259665e63dafbad22a75ea \ + --hash=sha256:a2718c1de8504121714234b6f8241d0019450353276c88b9453c9c3d92e101db \ + --hash=sha256:a39fb973a726e63223287adc6dafe444ce75af952d711e400f3bf2b36ef55a7b \ + --hash=sha256:a899699294f28f7be8992853c0c60741f16ff199205e2e6cdca155762cbaa59d \ + --hash=sha256:a8fdfed3deaf1928fb7667d96e0567cdf58c2b370ea2ee7e586aa383ec2cb346 \ + --hash=sha256:aabb081ca0ec5d39591fc33018cd4b3f96e1a2dd6756282029986d00a785fba4 \ + --hash=sha256:b1f5b45829ac1848893f0ddf5cb326110604d6df96cdc255b0bf9edd154104d4 \ + --hash=sha256:b48e35f4ab6f6a7597c46e301126ceba4c44cd3280e3750f85db48b082624fa4 \ + --hash=sha256:b54c83f1c0c0f1d748dca0af516062b8829d53d1f0c402be24b4257a9c48ada6 \ + --hash=sha256:ba0cb30acd3ef11c94dc27fbfba68940652492bc107075e7ffe23057f9425681 \ + --hash=sha256:bb50ce5fb202a26fd5404620e7ef820ad1ab3558b444cb0b55beb7ef66cd2d63 \ + --hash=sha256:c95eb6db2884917d86cde0b4d4cf31adf485c8ec36bf8696dd66fa70de96f36b \ + --hash=sha256:d0ffd9e2e4441c96a9c91ec1783285d80bf835b677853fc2770a89d50c1e48ac \ + --hash=sha256:d57b5046c120561ba8fa8e4030fbb8b822f3063910fa901ffadf16e2b7128ad6 \ + --hash=sha256:d9e6a7664ddd9746e20b7325351fe1a8408d0a2bf9c63b5e898290ddc8f09544 \ + --hash=sha256:dc578891de1db95b2a35001b695451767b580bb45753717498213c5ff3c41d63 \ + --hash=sha256:e0434aa22c821f44eeb4c650b81c7fbdd8c0122c6c4b5a576a76d5a35625ecd9 \ + --hash=sha256:e06a922a469cae9a57100864caf4f8a97a1026513793969f8ba5b63137a35d25 \ + --hash=sha256:e493962256a38f58283de033d8af176c5c91c084ea30f15834f7545451c42059 \ + --hash=sha256:ecb0019d44f4cdb50b676c5d0cb4b1eae8e15d1ed3d3e6639f986fc92b2ec52c \ + --hash=sha256:f935c4493eda9069851058fa0d9e39dbf6286be690066509305e52912714dbb2 # via # ml-dtypes # neuronx-cc - # nki # scipy pgzip==0.4.0 \ --hash=sha256:41e44c03a9f1bbbd390dd6b6e37a11484113f8c159a049fa81cd8a9ddde82a21 \ --hash=sha256:91f6526f0c6e1f6c2d3522707777e5843fad7c791401745593fd01bd869b7a05 # via neuronx-cc -protobuf==7.35.0 \ - --hash=sha256:4c4617b83ade0e279d1d2bfe04025a1adb87f9ed657de038620dc0ff959357f6 \ - --hash=sha256:4cbf5cc286130e06a6c9bbefac442431173906dfcc979712183d4adcc01b37ee \ - --hash=sha256:66be6c513931c794fa92c080ffee41671390da3d79da219cf9c0c0907f035dda \ - --hash=sha256:6c0f98f10c8a05ea30f8993dfef2de093d27b490fdae78bb60c8343795d55011 \ - --hash=sha256:a2efd84605f41e559f1881b0912b44099d0a2ac9bf46b3474823f10fb393b0e6 \ - --hash=sha256:c13f325cf242bad135c350629eeb5d54b24228eb472fb3e2e9ebbd4c5dc20ca0 \ - --hash=sha256:f05bcadf9a2a6b8dda047007075135fb7d08c73d9177aabc067e1be46881a201 \ - --hash=sha256:fcbe42a4ac09d3ec9c987ddfcd956afd0b15f1ff613bd8371bde9405ffd5c8e5 +protobuf==6.33.2 \ + --hash=sha256:1f8017c48c07ec5859106533b682260ba3d7c5567b1ca1f24297ce03384d1b4f \ + --hash=sha256:2981c58f582f44b6b13173e12bb8656711189c2a70250845f264b877f00b1913 \ + --hash=sha256:56dc370c91fbb8ac85bc13582c9e373569668a290aa2e66a590c2a0d35ddb9e4 \ + --hash=sha256:7109dcc38a680d033ffb8bf896727423528db9163be1b6a02d6a49606dcadbfe \ + --hash=sha256:7636aad9bb01768870266de5dc009de2d1b936771b38a793f73cbbf279c91c5c \ + --hash=sha256:87eb388bd2d0f78febd8f4c8779c79247b26a5befad525008e49a6955787ff3d \ + --hash=sha256:8cd7640aee0b7828b6d03ae518b5b4806fdfc1afe8de82f79c3454f8aef29872 \ + --hash=sha256:b5d3b5625192214066d99b2b605f5783483575656784de223f00a8d00754fc0e \ + --hash=sha256:d9b19771ca75935b3a4422957bc518b0cecb978b31d1dd12037b088f6bcc0e43 \ + --hash=sha256:fc2a0e8b05b180e5fc0dd1559fe8ebdae21a27e81ac77728fb6c42b12c7419b4 # via neuronx-cc -psutil==7.2.2 \ - --hash=sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372 \ - --hash=sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9 \ - --hash=sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841 \ - --hash=sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63 \ - --hash=sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979 \ - --hash=sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a \ - --hash=sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b \ - --hash=sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9 \ - --hash=sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee \ - --hash=sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312 \ - --hash=sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b \ - --hash=sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9 \ - --hash=sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e \ - --hash=sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc \ - --hash=sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1 \ - --hash=sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf \ - --hash=sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea \ - --hash=sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988 \ - --hash=sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486 \ - --hash=sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00 \ - --hash=sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8 +psutil==7.2.1 \ + --hash=sha256:05cc68dbb8c174828624062e73078e7e35406f4ca2d0866c272c2410d8ef06d1 \ + --hash=sha256:08a2f175e48a898c8eb8eace45ce01777f4785bc744c90aa2cc7f2fa5462a266 \ + --hash=sha256:0d67c1822c355aa6f7314d92018fb4268a76668a536f133599b91edd48759442 \ + --hash=sha256:2ceae842a78d1603753561132d5ad1b2f8a7979cb0c283f5b52fb4e6e14b1a79 \ + --hash=sha256:35630d5af80d5d0d49cfc4d64c1c13838baf6717a13effb35869a5919b854cdf \ + --hash=sha256:3fce5f92c22b00cdefd1645aa58ab4877a01679e901555067b1bd77039aa589f \ + --hash=sha256:494c513ccc53225ae23eec7fe6e1482f1b8a44674241b54561f755a898650679 \ + --hash=sha256:5e38404ca2bb30ed7267a46c02f06ff842e92da3bb8c5bfdadbd35a5722314d8 \ + --hash=sha256:81442dac7abfc2f4f4385ea9e12ddf5a796721c0f6133260687fec5c3780fa49 \ + --hash=sha256:923f8653416604e356073e6e0bccbe7c09990acef442def2f5640dd0faa9689f \ + --hash=sha256:93f3f7b0bb07711b49626e7940d6fe52aa9940ad86e8f7e74842e73189712129 \ + --hash=sha256:99a4cd17a5fdd1f3d014396502daa70b5ec21bf4ffe38393e152f8e449757d67 \ + --hash=sha256:ab2b98c9fc19f13f59628d94df5cc4cc4844bc572467d113a8b517d634e362c6 \ + --hash=sha256:b1b0671619343aa71c20ff9767eced0483e4fc9e1f489d50923738caf6a03c17 \ + --hash=sha256:b2e953fcfaedcfbc952b44744f22d16575d3aa78eb4f51ae74165b4e96e55f42 \ + --hash=sha256:ba9f33bb525b14c3ea563b2fd521a84d2fa214ec59e3e6a2858f78d0844dd60d \ + --hash=sha256:cfbe6b40ca48019a51827f20d830887b3107a74a79b01ceb8cc8de4ccb17b672 \ + --hash=sha256:d34d2ca888208eea2b5c68186841336a7f5e0b990edec929be909353a202768a \ + --hash=sha256:ea46c0d060491051d39f0d2cff4f98d5c72b288289f57a21556cc7d504db37fc \ + --hash=sha256:f7583aec590485b43ca601dd9cea0dcd65bd7bb21d30ef4ddbf4ea6b5ed1bdd3 \ + --hash=sha256:f78baafb38436d5a128f837fab2d92c276dfb48af01a240b861ae02b2413ada8 # via neuronx-cc python-daemon==3.1.2 \ --hash=sha256:b906833cef63502994ad48e2eab213259ed9bb18d54fa8774dcba2ff7864cec6 \ @@ -393,9 +371,9 @@ python-dateutil==2.9.0.post0 \ --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 # via botocore -requests==2.34.2 \ - --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ - --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed +requests==2.32.5 \ + --hash=sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6 \ + --hash=sha256:dbba0bac56e100853db0ea71b82b4dfd5fe2bf6d3754a8893c3af500cec7d7cf # via # ec2-metadata # requests-unixsocket @@ -403,88 +381,88 @@ requests-unixsocket==0.4.1 \ --hash=sha256:60c4942e9dbecc2f64d611039fb1dfc25da382083c6434ac0316dca3ff908f4d \ --hash=sha256:b2596158c356ecee68d27ba469a52211230ac6fb0cde8b66afb19f0ed47a1995 # via neuronx-cc -s3transfer==0.17.0 \ - --hash=sha256:9edeb6d1c3c2f89d6050348548834ad8289610d886e5bf7b7207728bd43ce33a \ - --hash=sha256:ce3801712acf4ad3e89fb9990df97b4972e93f4b3b0004d214be5bce12814c20 +s3transfer==0.16.0 \ + --hash=sha256:18e25d66fed509e3868dc1572b3f427ff947dd2c56f844a5bf09481ad3f3b2fe \ + --hash=sha256:8e990f13268025792229cd52fa10cb7163744bf56e719e0b9cb925ab79abf920 # via boto3 -scipy==1.17.1 \ - --hash=sha256:010f4333c96c9bb1a4516269e33cb5917b08ef2166d5556ca2fd9f082a9e6ea0 \ - --hash=sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458 \ - --hash=sha256:08b900519463543aa604a06bec02461558a6e1cef8fdbb8098f77a48a83c8118 \ - --hash=sha256:131f5aaea57602008f9822e2115029b55d4b5f7c070287699fe45c661d051e39 \ - --hash=sha256:158dd96d2207e21c966063e1635b1063cd7787b627b6f07305315dd73d9c679e \ - --hash=sha256:1cc682cea2ae55524432f3cdff9e9a3be743d52a7443d0cba9017c23c87ae2f6 \ - --hash=sha256:1f95b894f13729334fb990162e911c9e5dc1ab390c58aa6cbecb389c5b5e28ec \ - --hash=sha256:200e1050faffacc162be6a486a984a0497866ec54149a01270adc8a59b7c7d21 \ - --hash=sha256:2040ad4d1795a0ae89bfc7e8429677f365d45aa9fd5e4587cf1ea737f927b4a1 \ - --hash=sha256:2b64ca7d4aee0102a97f3ba22124052b4bd2152522355073580bf4845e2550b6 \ - --hash=sha256:2ceb2d3e01c5f1d83c4189737a42d9cb2fc38a6eeed225e7515eef71ad301dce \ - --hash=sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8 \ - --hash=sha256:37425bc9175607b0268f493d79a292c39f9d001a357bebb6b88fdfaff13f6448 \ - --hash=sha256:3877ac408e14da24a6196de0ddcace62092bfc12a83823e92e49e40747e52c19 \ - --hash=sha256:3fd1fcdab3ea951b610dc4cef356d416d5802991e7e32b5254828d342f7b7e0b \ - --hash=sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87 \ - --hash=sha256:43af8d1f3bea642559019edfe64e9b11192a8978efbd1539d7bc2aaa23d92de4 \ - --hash=sha256:45abad819184f07240d8a696117a7aacd39787af9e0b719d00285549ed19a1e9 \ - --hash=sha256:4b400bdc6f79fa02a4d86640310dde87a21fba0c979efff5248908c6f15fad1b \ - --hash=sha256:4eb6c25dd62ee8d5edf68a8e1c171dd71c292fdae95d8aeb3dd7d7de4c364082 \ - --hash=sha256:581b2264fc0aa555f3f435a5944da7504ea3a065d7029ad60e7c3d1ae09c5464 \ - --hash=sha256:5cf36e801231b6a2059bf354720274b7558746f3b1a4efb43fcf557ccd484a87 \ - --hash=sha256:5e3c5c011904115f88a39308379c17f91546f77c1667cea98739fe0fccea804c \ - --hash=sha256:6609bc224e9568f65064cfa72edc0f24ee6655b47575954ec6339534b2798369 \ - --hash=sha256:6e3dcd57ab780c741fde8dc68619de988b966db759a3c3152e8e9142c26295ad \ - --hash=sha256:6fac755ca3d2c3edcb22f479fceaa241704111414831ddd3bc6056e18516892f \ - --hash=sha256:744b2bf3640d907b79f3fd7874efe432d1cf171ee721243e350f55234b4cec4c \ - --hash=sha256:74cbb80d93260fe2ffa334efa24cb8f2f0f622a9b9febf8b483c0b865bfb3475 \ - --hash=sha256:766e0dc5a616d026a3a1cffa379af959671729083882f50307e18175797b3dfd \ - --hash=sha256:7bdf2da170b67fdf10bca777614b1c7d96ae3ca5794fd9587dce41eb2966e866 \ - --hash=sha256:7ff200bf9d24f2e4d5dc6ee8c3ac64d739d3a89e2326ba68aaf6c4a2b838fd7d \ - --hash=sha256:844e165636711ef41f80b4103ed234181646b98a53c8f05da12ca5ca289134f6 \ - --hash=sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb \ - --hash=sha256:94055a11dfebe37c656e70317e1996dc197e1a15bbcc351bcdd4610e128fe1ca \ - --hash=sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0 \ - --hash=sha256:9cdc1a2fcfd5c52cfb3045feb399f7b3ce822abdde3a193a6b9a60b3cb5854ca \ - --hash=sha256:9ecb4efb1cd6e8c4afea0daa91a87fbddbce1b99d2895d151596716c0b2e859d \ - --hash=sha256:a3472cfbca0a54177d0faa68f697d8ba4c80bbdc19908c3465556d9f7efce9ee \ - --hash=sha256:a4328d245944d09fd639771de275701ccadf5f781ba0ff092ad141e017eccda4 \ - --hash=sha256:a48a72c77a310327f6a3a920092fa2b8fd03d7deaa60f093038f22d98e096717 \ - --hash=sha256:a720477885a9d2411f94a93d16f9d89bad0f28ca23c3f8daa521e2dcc3f44d49 \ - --hash=sha256:a77cbd07b940d326d39a1d1b37817e2ee4d79cb30e7338f3d0cddffae70fcaa2 \ - --hash=sha256:a9956e4d4f4a301ebf6cde39850333a6b6110799d470dbbb1e25326ac447f52a \ - --hash=sha256:adb2642e060a6549c343603a3851ba76ef0b74cc8c079a9a58121c7ec9fe2350 \ - --hash=sha256:beeda3d4ae615106d7094f7e7cef6218392e4465cc95d25f900bebabfded0950 \ - --hash=sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b \ - --hash=sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086 \ - --hash=sha256:cd96a1898c0a47be4520327e01f874acfd61fb48a9420f8aa9f6483412ffa444 \ - --hash=sha256:d2650c1fb97e184d12d8ba010493ee7b322864f7d3d00d3f9bb97d9c21de4068 \ - --hash=sha256:d30e57c72013c2a4fe441c2fcb8e77b14e152ad48b5464858e07e2ad9fbfceff \ - --hash=sha256:d59c30000a16d8edc7e64152e30220bfbd724c9bbb08368c054e24c651314f0a \ - --hash=sha256:dbc12c9f3d185f5c737d801da555fb74b3dcfa1a50b66a1a93e09190f41fab50 \ - --hash=sha256:e18f12c6b0bc5a592ed23d3f7b891f68fd7f8241d69b7883769eb5d5dfb52696 \ - --hash=sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21 \ - --hash=sha256:e30bdeaa5deed6bc27b4cc490823cd0347d7dae09119b8803ae576ea0ce52e4c \ - --hash=sha256:eb092099205ef62cd1782b006658db09e2fed75bffcae7cc0d44052d8aa0f484 \ - --hash=sha256:eee2cfda04c00a857206a4330f0c5e3e56535494e30ca445eb19ec624ae75118 \ - --hash=sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3 \ - --hash=sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea \ - --hash=sha256:f8885db0bc2bffa59d5c1b72fad7a6a92d3e80e7257f967dd81abb553a90d293 \ - --hash=sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76 +scipy==1.16.3 \ + --hash=sha256:0151a0749efeaaab78711c78422d413c583b8cdd2011a3c1d6c794938ee9fdb2 \ + --hash=sha256:01e87659402762f43bd2fee13370553a17ada367d42e7487800bf2916535aecb \ + --hash=sha256:03192a35e661470197556de24e7cb1330d84b35b94ead65c46ad6f16f6b28f2a \ + --hash=sha256:0553371015692a898e1aa858fed67a3576c34edefa6b7ebdb4e9dde49ce5c203 \ + --hash=sha256:062246acacbe9f8210de8e751b16fc37458213f124bef161a5a02c7a39284304 \ + --hash=sha256:0c3b4dd3d9b08dbce0f3440032c52e9e2ab9f96ade2d3943313dfe51a7056959 \ + --hash=sha256:0c623a54f7b79dd88ef56da19bc2873afec9673a48f3b85b18e4d402bdd29a5a \ + --hash=sha256:16b8bc35a4cc24db80a0ec836a9286d0e31b2503cb2fd7ff7fb0e0374a97081d \ + --hash=sha256:1fb2472e72e24d1530debe6ae078db70fb1605350c88a3d14bc401d6306dbffe \ + --hash=sha256:21d9d6b197227a12dcbf9633320a4e34c6b0e51c57268df255a0942983bac562 \ + --hash=sha256:2a207a6ce9c24f1951241f4693ede2d393f59c07abc159b2cb2be980820e01fb \ + --hash=sha256:2b71d93c8a9936046866acebc915e2af2e292b883ed6e2cbe5c34beb094b82d9 \ + --hash=sha256:2d1ae2cf0c350e7705168ff2429962a89ad90c2d49d1dd300686d8b2a5af22fc \ + --hash=sha256:3a4c460301fb2cffb7f88528f30b3127742cff583603aa7dc964a52c463b385d \ + --hash=sha256:3d4a07a8e785d80289dfe66b7c27d8634a773020742ec7187b85ccc4b0e7b686 \ + --hash=sha256:40be6cf99e68b6c4321e9f8782e7d5ff8265af28ef2cd56e9c9b2638fa08ad97 \ + --hash=sha256:4aff59800a3b7f786b70bfd6ab551001cb553244988d7d6b8299cb1ea653b353 \ + --hash=sha256:50a3dbf286dbc7d84f176f9a1574c705f277cb6565069f88f60db9eafdbe3ee2 \ + --hash=sha256:532fb5ad6a87e9e9cd9c959b106b73145a03f04c7d57ea3e6f6bb60b86ab0876 \ + --hash=sha256:53c3844d527213631e886621df5695d35e4f6a75f620dca412bcd292f6b87d78 \ + --hash=sha256:56edc65510d1331dae01ef9b658d428e33ed48b4f77b1d51caf479a0253f96dc \ + --hash=sha256:57d01cb6f85e34f0946b33caa66e892aae072b64b034183f3d87c4025802a119 \ + --hash=sha256:5803c5fadd29de0cf27fa08ccbfe7a9e5d741bf63e4ab1085437266f12460ff9 \ + --hash=sha256:6020470b9d00245926f2d5bb93b119ca0340f0d564eb6fbaad843eaebf9d690f \ + --hash=sha256:63d3cdacb8a824a295191a723ee5e4ea7768ca5ca5f2838532d9f2e2b3ce2135 \ + --hash=sha256:663b8d66a8748051c3ee9c96465fb417509315b99c71550fda2591d7dd634234 \ + --hash=sha256:72d1717fd3b5e6ec747327ce9bda32d5463f472c9dce9f54499e81fbd50245a1 \ + --hash=sha256:7dc1360c06535ea6116a2220f760ae572db9f661aba2d88074fe30ec2aa1ff88 \ + --hash=sha256:7f68154688c515cdb541a31ef8eb66d8cd1050605be9dcd74199cbd22ac739bc \ + --hash=sha256:81fc5827606858cf71446a5e98715ba0e11f0dbc83d71c7409d05486592a45d6 \ + --hash=sha256:875555ce62743e1d54f06cdf22c1e0bc47b91130ac40fe5d783b6dfa114beeb6 \ + --hash=sha256:8b3c820ddb80029fe9f43d61b81d8b488d3ef8ca010d15122b152db77dc94c22 \ + --hash=sha256:8be1ca9170fcb6223cc7c27f4305d680ded114a1567c0bd2bfcbf947d1b17511 \ + --hash=sha256:8d09d72dc92742988b0e7750bddb8060b0c7079606c0d24a8cc8e9c9c11f9079 \ + --hash=sha256:9452781bd879b14b6f055b26643703551320aa8d79ae064a71df55c00286a184 \ + --hash=sha256:96491a6a54e995f00a28a3c3badfff58fd093bf26cd5fb34a2188c8c756a3a2c \ + --hash=sha256:9b9c9c07b6d56a35777a1b4cc8966118fb16cfd8daf6743867d17d36cfad2d40 \ + --hash=sha256:a8a26c78ef223d3e30920ef759e25625a0ecdd0d60e5a8818b7513c3e5384cf2 \ + --hash=sha256:aadd23f98f9cb069b3bd64ddc900c4d277778242e961751f77a8cb5c4b946fb0 \ + --hash=sha256:b7180967113560cca57418a7bc719e30366b47959dd845a93206fbed693c867e \ + --hash=sha256:b7c5f1bda1354d6a19bc6af73a649f8285ca63ac6b52e64e658a5a11d4d69800 \ + --hash=sha256:b81c27fc41954319a943d43b20e07c40bdcd3ff7cf013f4fb86286faefe546c4 \ + --hash=sha256:bb61878c18a470021fb515a843dc7a76961a8daceaaaa8bad1332f1bf4b54657 \ + --hash=sha256:bea0a62734d20d67608660f69dcda23e7f90fb4ca20974ab80b6ed40df87a005 \ + --hash=sha256:c5192722cffe15f9329a3948c4b1db789fbb1f05c97899187dcf009b283aea70 \ + --hash=sha256:c97176013d404c7346bf57874eaac5187d969293bf40497140b0a2b2b7482e07 \ + --hash=sha256:cd13e354df9938598af2be05822c323e97132d5e6306b83a3b4ee6724c6e522e \ + --hash=sha256:d2ec56337675e61b312179a1ad124f5f570c00f920cc75e1000025451b88241c \ + --hash=sha256:d3837938ae715fc0fe3c39c0202de3a8853aff22ca66781ddc2ade7554b7e2cc \ + --hash=sha256:d9f48cafc7ce94cf9b15c6bffdc443a81a27bf7075cf2dcd5c8b40f85d10c4e7 \ + --hash=sha256:da7763f55885045036fabcebd80144b757d3db06ab0861415d1c3b7c69042146 \ + --hash=sha256:deb3841c925eeddb6afc1e4e4a45e418d19ec7b87c5df177695224078e8ec733 \ + --hash=sha256:e1d27cbcb4602680a49d787d90664fa4974063ac9d4134813332a8c53dbe667c \ + --hash=sha256:e5d42a9472e7579e473879a1990327830493a7047506d58d73fc429b84c1d49d \ + --hash=sha256:e7efa2681ea410b10dde31a52b18b0154d66f2485328830e45fdf183af5aefc6 \ + --hash=sha256:eab43fae33a0c39006a88096cd7b4f4ef545ea0447d250d5ac18202d40b6611d \ + --hash=sha256:f2622206f5559784fa5c4b53a950c3c7c1cf3e84ca1b9c4b6c03f062f289ca26 \ + --hash=sha256:f379b54b77a597aa7ee5e697df0d66903e41b9c85a6dd7946159e356319158e8 \ + --hash=sha256:f667a4542cc8917af1db06366d3f78a5c8e83badd56409f94d1eac8d8d9133fa \ + --hash=sha256:fb4b29f4cf8cc5a8d628bc8d8e26d12d7278cd1f219f22698a378c3d67db5e4b \ + --hash=sha256:ffa6eea95283b2b8079b821dc11f50a17d0571c92b43e2b5b12764dc5f9b285d # via neuronx-cc six==1.17.0 \ --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 # via python-dateutil -tqdm==4.67.3 \ - --hash=sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb \ - --hash=sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf +tqdm==4.67.1 \ + --hash=sha256:26445eca388f82e72884e0d580d5464cd801a3ea01e63e5601bdff9ba6a48de2 \ + --hash=sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2 # via neuronx-cc typing-extensions==4.15.0 \ --hash=sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466 \ --hash=sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 # via neuronx-cc -urllib3==2.7.0 \ - --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ - --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 +urllib3==2.6.3 \ + --hash=sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed \ + --hash=sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4 # via # botocore # requests diff --git a/platforms/neuron/topology.zig b/platforms/neuron/topology.zig deleted file mode 100644 index 201db15f..00000000 --- a/platforms/neuron/topology.zig +++ /dev/null @@ -1,222 +0,0 @@ -const std = @import("std"); - -const neuron = @import("platforms/neuron"); - -// References: -// - `neuron.Instance` exposes the NRT instance family and size used below. -// - Neuron instance families and sizes are documented here: -// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/index.html -// - Inf1: -// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/inf1-arch.html -// - Inf2: -// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/inf2-arch.html -// - Trn1 / Trn1n: -// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn1-arch.html -// - Trn2 / Trn2 UltraServer: -// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn2-arch.html -// - Trn3: -// https://awsdocs-neuron.readthedocs-hosted.com/en/latest/about-neuron/arch/neuron-hardware/trn3-arch.html -// -// The same instance-architecture pages are also the basis for the inter-chip -// topology inference below, including 2D torus and ring-over-torus fabrics. -pub const InterChipTopology = union(enum) { - none: void, - linear: usize, - ring: usize, - torus2d: struct { - x: usize, - y: usize, - }, - torus2d_ring: struct { - x: usize, - y: usize, - z: usize, - }, - point_to_point: usize, -}; - -pub const Axis = struct { - fabric: Fabric, - size: usize, - geometry: Geometry, -}; - -pub const Fabric = enum { - core, - chip_x, - chip_y, - chip_z, -}; - -pub const Geometry = enum { - linear_ring, - closed_ring, - torus, - point_to_point, -}; - -pub const Placement = struct { - input_index: usize, - nc_id: usize, - chip: usize, - core: usize, -}; - -pub const VisibleMesh = struct { - instance: neuron.Instance, - chips: usize, - cores_per_chip: usize, - inter_chip_topology: InterChipTopology, - placements: []Placement, - axes_buffer: [4]Axis, - axes_len: usize, - - pub fn deinit(self: *VisibleMesh, allocator: std.mem.Allocator) void { - allocator.free(self.placements); - self.* = undefined; - } - - pub fn axes(self: *const VisibleMesh) []const Axis { - return self.axes_buffer[0..self.axes_len]; - } - - pub fn format(self: VisibleMesh, writer: *std.Io.Writer) !void { - try writer.print( - "VisibleMesh(family={s} size={s} chips={d} cores_per_chip={d} inter_chip_topology=", - .{ - @tagName(self.instance.family), - @tagName(self.instance.size), - self.chips, - self.cores_per_chip, - }, - ); - switch (self.inter_chip_topology) { - .none => try writer.writeAll("none"), - .linear => |size| try writer.print("linear({d})", .{size}), - .ring => |size| try writer.print("ring({d})", .{size}), - .torus2d => |dims| try writer.print("torus2d({d}x{d})", .{ dims.x, dims.y }), - .torus2d_ring => |dims| try writer.print("torus2d_ring({d}x{d}x{d})", .{ dims.x, dims.y, dims.z }), - .point_to_point => |size| try writer.print("point_to_point({d})", .{size}), - } - try writer.writeAll(")"); - } -}; - -pub fn visibleMeshFromNcIds(allocator: std.mem.Allocator, nc_ids: []const usize) !VisibleMesh { - if (nc_ids.len == 0) return error.InvalidDeviceTopology; - - const instance = try neuron.instance(); - const physical_cores_per_chip = try instance.coresPerChip(); - const placements = try allocator.alloc(Placement, nc_ids.len); - errdefer allocator.free(placements); - - for (nc_ids, placements, 0..) |nc_id, *placement, input_index| { - placement.* = .{ - .input_index = input_index, - .nc_id = nc_id, - .chip = chipOf(nc_id, physical_cores_per_chip), - .core = coreOf(nc_id, physical_cores_per_chip), - }; - } - - const Sort = struct { - fn lessThan(_: void, a: Placement, b: Placement) bool { - return if (a.chip == b.chip) a.core < b.core else a.chip < b.chip; - } - }; - std.mem.sort(Placement, placements, {}, Sort.lessThan); - - const first_chip = placements[0].chip; - const last_chip = placements[placements.len - 1].chip; - const selected_chip_count = last_chip - first_chip + 1; - const selected_cores_per_chip = try std.math.divExact(usize, placements.len, selected_chip_count); - const inter_chip_topology = interChipTopology(instance, selected_chip_count); - - var res: VisibleMesh = .{ - .instance = instance, - .chips = selected_chip_count, - .cores_per_chip = selected_cores_per_chip, - .inter_chip_topology = inter_chip_topology, - .placements = placements, - .axes_buffer = undefined, - .axes_len = 0, - }; - - switch (inter_chip_topology) { - .none => {}, - .linear => |size| appendAxis(&res, .chip_x, size, .linear_ring), - .ring => |size| appendAxis(&res, .chip_x, size, .closed_ring), - .torus2d => |dims| { - appendAxis(&res, .chip_x, dims.x, .torus); - appendAxis(&res, .chip_y, dims.y, .torus); - }, - .torus2d_ring => |dims| { - appendAxis(&res, .chip_x, dims.x, .torus); - appendAxis(&res, .chip_y, dims.y, .torus); - appendAxis(&res, .chip_z, dims.z, .closed_ring); - }, - .point_to_point => |size| appendAxis(&res, .chip_x, size, .point_to_point), - } - - // Inter-chip topology only describes NeuronLink fabric between chips. Add a - // local core axis for multi-core chips, and keep single-chip meshes non-empty. - if (res.cores_per_chip > 1 or res.axes_len == 0) { - appendAxis(&res, .core, res.cores_per_chip, .point_to_point); - } - - return res; -} - -pub fn interChipTopology(instance: neuron.Instance, chip_count: usize) InterChipTopology { - if (chip_count == 1) return .none; - - // References for chip-count and fabric-shape inference: - // - Inf1 instance sizes and 1/4/16-chip layouts - // - Inf2 instance sizes and 1/6/12-chip layouts - // - Trn1 / Trn1n 16-chip 2D torus - // - Trn2 16-chip torus and Trn2 UltraServer 64-chip ring-over-torus - // - Trn3 all-to-all / point-to-point scale-up - return switch (instance.family) { - .inf1 => switch (chip_count) { - 4 => .{ .linear = 4 }, - else => .{ .ring = chip_count }, - }, - .inf2, .inf2e => switch (chip_count) { - 6 => .{ .torus2d = .{ .x = 3, .y = 2 } }, - 12 => .{ .torus2d = .{ .x = 4, .y = 3 } }, - else => switch (instance.size) { - .xl24 => .{ .torus2d = .{ .x = 3, .y = 2 } }, - .xl48 => .{ .torus2d = .{ .x = 4, .y = 3 } }, - else => .{ .ring = chip_count }, - }, - }, - .trn1, .trn1n => switch (chip_count) { - 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, - else => .{ .ring = chip_count }, - }, - .trn2, .trn2n, .trn2p, .trn2u, .trn2e, .trn2eu, .trn2ac, .trn2uac => switch (chip_count) { - 16 => .{ .torus2d = .{ .x = 4, .y = 4 } }, - 64 => .{ .torus2d_ring = .{ .x = 4, .y = 4, .z = 4 } }, - else => .{ .ring = chip_count }, - }, - .trn3, .trn3pds98 => .{ .point_to_point = chip_count }, - .unknown => .{ .ring = chip_count }, - }; -} - -pub fn chipOf(nc_id: usize, physical_cores_per_chip: usize) usize { - return @divFloor(nc_id, physical_cores_per_chip); -} - -pub fn coreOf(nc_id: usize, physical_cores_per_chip: usize) usize { - return @mod(nc_id, physical_cores_per_chip); -} - -fn appendAxis(self: *VisibleMesh, fabric: Fabric, size: usize, geometry: Geometry) void { - self.axes_buffer[self.axes_len] = .{ - .fabric = fabric, - .size = size, - .geometry = geometry, - }; - self.axes_len += 1; -} diff --git a/platforms/oneapi/BUILD.bazel b/platforms/oneapi/BUILD.bazel deleted file mode 100644 index aaa06989..00000000 --- a/platforms/oneapi/BUILD.bazel +++ /dev/null @@ -1,39 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") - -zig_shared_library( - name = "zmlxoneapi", - main = "zmlxoneapi.zig", - shared_lib_name = "libzmlxoneapi.so.0", - target_compatible_with = ["@platforms//os:linux"], - visibility = ["@libzml_oneapi//:__subpackages__"], - zigopts = ["-fno-stack-check"], - deps = ["//stdx"], -) - -cc_library( - name = "libzml_oneapi", - hdrs = ["oneapi.h"], - tags = ["manual"], - deps = ["@libzml_oneapi"], -) - -zig_library( - name = "oneapi", - import_name = "platforms/oneapi", - main = "oneapi.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//bazel", - "//pjrt", - "//platforms:options", - "//stdx", - "@rules_zig//zig/runfiles", - ] + select({ - "//platforms:oneapi.enabled": [ - ":libzml_oneapi", - ], - "//conditions:default": [], - }), -) diff --git a/platforms/oneapi/README.md b/platforms/oneapi/README.md deleted file mode 100644 index d450bba6..00000000 --- a/platforms/oneapi/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# EXPERIMENTAL: oneAPI - - -# About - -This directory contains the oneAPI backend for ZML. It is currently in an early stage of development, and only supports a subset of ZML's features. The oneAPI backend is only available on Linux, and requires an Intel GPU. - -The sharding is not yet supported, so only models that fit in the GPU memory can be run. - -# Getting Started - -## Prerequisites - -- An Intel GPU -- bazel see [Getting Started](../README.md#getting-started) for installation instructions. - -### Linux Only - -One GPU is supported on Linux. -ONEAPI_DEVICE_SELECTOR environment variable must be set to select the GPU device. - -For example, to select the first GPU: -```bash -export ONEAPI_DEVICE_SELECTOR=level_zero:0 -``` - -#### Run - - -To run the LLM example with oneAPI and the Llama 3.2 1B model: -```bash -ONEAPI_DEVICE_SELECTOR=level_zero:0 \ -bazel run //examples/llm \ - --config=release \ - --@zml//platforms:cpu=false \ - --@zml//platforms:oneapi=true \ - -- \ - --model=hf://meta-llama/Llama-3.2-1B-Instruct \ - --topk=1 \ - --prompt="Tell me a story about a cat in 2 lines" -``` - -To run the LLM example with oneAPI and a local model: -```bash -ONEAPI_DEVICE_SELECTOR=level_zero:0 \ -bazel run //examples/llm \ - --config=release \ - --@zml//platforms:cpu=false \ - --@zml//platforms:oneapi=true \ - -- \ - --model=path/to/your/model/Llama-3.1-8B-Instruct \ - --topk=1 \ - --prompt="Tell me a story about a cat in 2 lines" -``` diff --git a/platforms/oneapi/libzml_oneapi.BUILD.bazel b/platforms/oneapi/libzml_oneapi.BUILD.bazel deleted file mode 100644 index eb08e652..00000000 --- a/platforms/oneapi/libzml_oneapi.BUILD.bazel +++ /dev/null @@ -1,154 +0,0 @@ -load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@zml//bazel:patchelf.bzl", "patchelf") - -patchelf( - name = "libzml_oneapi_so", - src = "lib/libzml_oneapi.so", - add_needed = [ - "libzmlxoneapi.so.0", - ], - rename_dynamic_symbols = { - "dlopen": "zmlxoneapi_dlopen", - }, - set_rpath = "$ORIGIN", - soname = "libzml_oneapi.so", -) - -patchelf( - name = "libdnnl_so_3", - src = "@intel-oneapi-dnnl-2026.0//:libdnnl_so_3", - set_rpath = "$ORIGIN", - soname = "libdnnl.so.3", -) - -patchelf( - name = "libtbb_so_12", - src = "@intel-oneapi-tbb-2023.1//:libtbb_so_12", - set_rpath = "$ORIGIN", - soname = "libtbb.so.12", -) - -patchelf( - name = "libsycl_so", - src = "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1//:libsycl_so", - set_rpath = "$ORIGIN", -) - -patchelf( - name = "libccl_legacy_so", - src = "@intel-oneapi-ccl-2022.1//:libccl_legacy_so", - set_rpath = "$ORIGIN", -) - -patchelf( - name = "libze_intel_gpu_so", - src = "@libze-intel-gpu1//:libze_intel_gpu_so", - set_rpath = "$ORIGIN", - soname = "libze_intel_gpu.so.1", -) - -patchelf( - name = "libigc_so_2", - src = "@libigc2//:libigc_so_2", - set_rpath = "$ORIGIN", - soname = "libigc.so.2", -) - -patchelf( - name = "libiga64_so_2", - src = "@libigc2//:libiga64_so_2", - set_rpath = "$ORIGIN", - soname = "libiga64.so.2", -) - -patchelf( - name = "libigdfcl_so_2", - src = "@libigdfcl2//:libigdfcl_so_2", - set_rpath = "$ORIGIN", - soname = "libigdfcl.so.2", -) - -ONEAPI_VERSION = "2026.1" - -ONEAPI_CCL_LIB = "ccl/2022.1/lib" - -ONEAPI_MPI_LIB = "mpi/2021.18/lib" - -ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" - -ONEAPI_TCM_VERSION = "1.5" - -ONEAPI_UMF_VERSION = "1.1" - -ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) - -ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) - -ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) - -ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) - -copy_to_directory( - name = "sandbox", - srcs = [ - ":libdnnl_so_3", - ":libiga64_so_2", - ":libigc_so_2", - ":libigdfcl_so_2", - ":libsycl_so", - ":libtbb_so_12", - ":libze_intel_gpu_so", - ":libzml_oneapi_so", - "@intel-oneapi-ccl-2022.1//:ccl_runtime", - "@intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1//:sycl_runtime", - "@intel-oneapi-compiler-shared-runtime-2026.1//:compiler_runtime", - "@intel-oneapi-mkl-core-2026.1//:mkl_core_runtime", - "@intel-oneapi-mkl-sycl-blas-2026.1//:mkl_sycl_blas", - "@intel-oneapi-mkl-sycl-dft-2026.1//:mkl_sycl_dft", - "@intel-oneapi-mkl-sycl-lapack-2026.1//:mkl_sycl_lapack", - "@intel-oneapi-mkl-sycl-rng-2026.1//:mkl_sycl_rng", - "@intel-oneapi-mkl-sycl-sparse-2026.1//:mkl_sycl_sparse", - "@intel-oneapi-mpi-2021.18//:mpi_runtime", - "@intel-oneapi-tcm-1.5//:hwloc", - "@intel-oneapi-umf-1.1//:umf", - "@libigdfcl2//:libopencl_clang", - "@libigdgmm12", - "@libnl-3-200//:libnl_3", - "@libnl-genl-3-200//:libnl_genl_3", - "@libnl-route-3-200//:libnl_route_3", - "@libnuma1", - "@libze1", - "@zlib1g", - "@zml//platforms/oneapi:zmlxoneapi", - ], - add_directory_to_runfiles = False, - allow_overwrites = True, - hardlink = "auto", - include_external_repositories = ["**"], - replace_prefixes = { - "libiga64_so_2": "lib", - "libigdfcl_so_2": "lib", - "libigc_so_2": "lib", - "libdnnl_so_3": "lib", - "libzml_oneapi_so": "lib", - "libsycl_so": "lib", - "libtbb_so_12": "lib", - ONEAPI_CCL_LIB: "lib", - ONEAPI_COMPILER_LIB: "lib", - ONEAPI_MPI_LIB: "lib", - ONEAPI_MPI_LIBFABRIC_LIB: "lib", - ONEAPI_MKL_LIB: "lib", - ONEAPI_TCM_LIB: "lib", - ONEAPI_UMF_LIB: "lib", - "lib/x86_64-linux-gnu": "lib", - "libze_intel_gpu_so": "lib", - "platforms/oneapi/libzmlxoneapi": "lib/libzmlxoneapi", - }, - root_paths = ["."], -) - -cc_library( - name = "libzml_oneapi", - data = [":sandbox"], - visibility = ["@zml//platforms/oneapi:__subpackages__"], -) diff --git a/platforms/oneapi/oneapi.bzl b/platforms/oneapi/oneapi.bzl deleted file mode 100644 index a3c968d3..00000000 --- a/platforms/oneapi/oneapi.bzl +++ /dev/null @@ -1,325 +0,0 @@ -load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") -load("//bazel:http_deb_archive.bzl", "http_deb_archive") - -_PLUGIN_ONEAPI_URL = "https://mirror.zml.ai/plugins/202611132003.375.1.7d765f57a407/zml-oneapi-linux-amd64.tar.zst" -_PLUGIN_ONEAPI_SHA256 = "294722670832c543c8ac4c074c30f636f2a3b0854820f7c5a070fa82c0d6f3d5" - -ONEAPI_VERSION = "2026.1" - -# www.intel.com/content/www/us/en/developer/tools/oneapi/onednn-download.html -ONEAPI_DNNL_VERSION = "2026.0" - -# https://github.com/uxlfoundation/onetbb -ONEAPI_TBB_VERSION = "2023.1" -ONEAPI_TCM_VERSION = "1.5" -ONEAPI_UMF_VERSION = "1.1" - -ONEAPI_CCL_LIB = "ccl/2022.1/lib" -ONEAPI_COMPILER_LIB = "compiler/{}/lib".format(ONEAPI_VERSION) -ONEAPI_DNNL_LIB = "dnnl/{}/lib".format(ONEAPI_DNNL_VERSION) -ONEAPI_MPI_LIB = "mpi/2021.18/lib" -ONEAPI_MPI_LIBFABRIC_LIB = "mpi/2021.18/opt/mpi/libfabric/lib" -ONEAPI_MKL_LIB = "mkl/{}/lib".format(ONEAPI_VERSION) -ONEAPI_TBB_LIB = "tbb/{}/lib".format(ONEAPI_TBB_VERSION) -ONEAPI_TCM_LIB = "tcm/{}/lib".format(ONEAPI_TCM_VERSION) -ONEAPI_UMF_LIB = "umf/{}/lib".format(ONEAPI_UMF_VERSION) - -_BUILD_FILE_DEFAULT_VISIBILITY = """\ -package(default_visibility = ["//visibility:public"]) -""" - -_UBUNTU_STRIP_PREFIX = "./usr" -_ONEAPI_STRIP_PREFIX = "./opt/intel/oneapi" - -def _read_packages(mctx, labels): - ret = {} - for label in labels: - data = json.decode(mctx.read(Label(label))) - for pkg in data["packages"]: - ret.setdefault(pkg["name"], {})[pkg["arch"]] = pkg - return ret - -_UBUNTU_PACKAGES = { - "libnl-3-200": """ -filegroup( - name = "libnl_3", - srcs = ["lib/x86_64-linux-gnu/libnl-3.so.200"], -)""", - "libnl-genl-3-200": """ -filegroup( - name = "libnl_genl_3", - srcs = ["lib/x86_64-linux-gnu/libnl-genl-3.so.200"], -)""", - "libnl-route-3-200": """ -filegroup( - name = "libnl_route_3", - srcs = ["lib/x86_64-linux-gnu/libnl-route-3.so.200"], -)""", - "libnuma1": """ -filegroup( - name = "libnuma1", - srcs = ["lib/x86_64-linux-gnu/libnuma.so.1"], -)""", - "libigdgmm12": """ -filegroup( - name = "libigdgmm12", - srcs = ["lib/x86_64-linux-gnu/libigdgmm.so.12"], -)""", - "libigc2": """ -filegroup( - name = "libigc_so_2", - srcs = ["lib/x86_64-linux-gnu/libigc.so.2.41.15+0"], -) -filegroup( - name = "libiga64_so_2", - srcs = ["lib/x86_64-linux-gnu/libiga64.so.2.41.15+0"], -) -filegroup( - name = "libigc2", - srcs = [ - ":libiga64_so_2", - ":libigc_so_2", - ], -)""", - "libigdfcl2": """ -filegroup( - name = "libigdfcl_so_2", - srcs = ["lib/x86_64-linux-gnu/libigdfcl.so.2.41.15+0"], -) -filegroup( - name = "libopencl_clang", - srcs = ["lib/x86_64-linux-gnu/libopencl-clang.so.22.1"], -) -filegroup( - name = "libigdfcl2", - srcs = [ - ":libigdfcl_so_2", - ":libopencl_clang", - ], -)""", - "libze-intel-gpu1": """ -filegroup( - name = "libze_intel_gpu_so", - srcs = ["lib/x86_64-linux-gnu/libze_intel_gpu.so.1"], -) -filegroup( - name = "libze_intel_gpu", - srcs = ["lib/x86_64-linux-gnu/libze_intel_gpu.so.1"], -)""", - "libze1": """ -filegroup( - name = "libze1", - srcs = [ - "lib/x86_64-linux-gnu/libze_loader.so.1", - "lib/x86_64-linux-gnu/libze_tracing_layer.so.1", - "lib/x86_64-linux-gnu/libze_validation_layer.so.1", - ], -)""", - "zlib1g": """ -filegroup( - name = "zlib1g", - srcs = ["lib/x86_64-linux-gnu/libz.so.1"], -)""", -} - -_ONEAPI_PACKAGES = { - "intel-oneapi-ccl-2022.1": """ -filegroup( - name = "libccl_legacy_so", - srcs = ["{ONEAPI_CCL_LIB}/libccl_legacy.so"], -) -filegroup( - name = "ccl_runtime", - srcs = [ - "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so", - "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so.1", - "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl.so.1.0", - "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so", - "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so.0", - "{ONEAPI_CCL_LIB}/ccl/cpu/lib/libccl_openmp.so.0.1", - "{ONEAPI_CCL_LIB}/ccl/kernels/kernels.spv", - "{ONEAPI_CCL_LIB}/libccl.so", - "{ONEAPI_CCL_LIB}/libccl.so.1", - "{ONEAPI_CCL_LIB}/libccl.so.1.0", - "{ONEAPI_CCL_LIB}/libccl.so.2", - "{ONEAPI_CCL_LIB}/libccl.so.2.0", - "{ONEAPI_CCL_LIB}/libccl_legacy_cpu.so", - "{ONEAPI_CCL_LIB}/libccl_openmp.so", - "{ONEAPI_CCL_LIB}/libccl_openmp.so.0", - "{ONEAPI_CCL_LIB}/libccl_openmp.so.0.1", - ], -)""".format(ONEAPI_CCL_LIB = ONEAPI_CCL_LIB), - "intel-oneapi-mpi-2021.18": """ -filegroup( - name = "mpi_runtime", - srcs = [ - "{ONEAPI_MPI_LIB}/libmpi.so", - "{ONEAPI_MPI_LIB}/libmpi.so.12", - "{ONEAPI_MPI_LIB}/libmpi.so.12.0", - "{ONEAPI_MPI_LIB}/libmpi.so.12.0.0", - "{ONEAPI_MPI_LIB}/libmpi_abi.so", - "{ONEAPI_MPI_LIB}/libmpi_abi.so.1", - "{ONEAPI_MPI_LIB}/libmpi_abi.so.1.0", - "{ONEAPI_MPI_LIB}/libmpi_ilp64.so", - "{ONEAPI_MPI_LIB}/libmpi_ilp64.so.4", - "{ONEAPI_MPI_LIB}/libmpi_ilp64.so.4.1", - "{ONEAPI_MPI_LIB}/libmpi_shm_heap_proxy.so", - "{ONEAPI_MPI_LIB}/libmpicxx.so", - "{ONEAPI_MPI_LIB}/libmpicxx.so.12", - "{ONEAPI_MPI_LIB}/libmpicxx.so.12.0", - "{ONEAPI_MPI_LIB}/libmpicxx.so.12.0.0", - "{ONEAPI_MPI_LIB}/libmpifort.so", - "{ONEAPI_MPI_LIB}/libmpifort.so.12", - "{ONEAPI_MPI_LIB}/libmpifort.so.12.0", - "{ONEAPI_MPI_LIB}/libmpifort.so.12.0.0", - "{ONEAPI_MPI_LIB}/libmpifort_abi.so", - "{ONEAPI_MPI_LIB}/libmpifort_abi.so.1", - "{ONEAPI_MPI_LIB}/libmpifort_abi.so.1.0", - "{ONEAPI_MPI_LIB}/libmpijava.so", - "{ONEAPI_MPI_LIB}/libmpijava.so.1", - "{ONEAPI_MPI_LIB}/libmpijava.so.1.0", - "{ONEAPI_MPI_LIB}/mpi/libmpi_ze_hooks.so", - "{ONEAPI_MPI_LIBFABRIC_LIB}/libfabric.so", - "{ONEAPI_MPI_LIBFABRIC_LIB}/libfabric.so.1", - "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libefa-fi.so", - "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libmlx-fi.so", - "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libpsm3-fi.so", - "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libpsmx2-fi.so", - "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/librxm-fi.so", - "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libshm-fi.so", - "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libtcp-fi.so", - "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libverbs-1.1-fi.so", - "{ONEAPI_MPI_LIBFABRIC_LIB}/prov/libverbs-1.12-fi.so", - ], -)""".format( - ONEAPI_MPI_LIB = ONEAPI_MPI_LIB, - ONEAPI_MPI_LIBFABRIC_LIB = ONEAPI_MPI_LIBFABRIC_LIB, - ), - "intel-oneapi-dnnl-2026.0": """ -filegroup( - name = "libdnnl_so_3", - srcs = ["{ONEAPI_DNNL_LIB}/libdnnl.so.3.11"], -)""".format(ONEAPI_DNNL_LIB = ONEAPI_DNNL_LIB), - "intel-oneapi-tbb-2023.1": """ -filegroup( - name = "libtbb_so_12", - srcs = ["{ONEAPI_TBB_LIB}/libtbb.so.12.19"], -)""".format(ONEAPI_TBB_LIB = ONEAPI_TBB_LIB), - "intel-oneapi-tcm-1.5": """ -filegroup( - name = "hwloc", - srcs = ["{ONEAPI_TCM_LIB}/libhwloc.so.15"], -)""".format(ONEAPI_TCM_LIB = ONEAPI_TCM_LIB), - "intel-oneapi-umf-1.1": """ -filegroup( - name = "umf", - srcs = ["{ONEAPI_UMF_LIB}/libumf.so.1"], -)""".format(ONEAPI_UMF_LIB = ONEAPI_UMF_LIB), - "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1": """ -filegroup( - name = "libsycl_so", - srcs = ["{ONEAPI_COMPILER_LIB}/libsycl.so.9"], -) -filegroup( - name = "sycl_runtime", - srcs = [ - "{ONEAPI_COMPILER_LIB}/libur_adapter_level_zero.so.0", - "{ONEAPI_COMPILER_LIB}/libur_adapter_level_zero_v2.so.0", - "{ONEAPI_COMPILER_LIB}/libur_adapter_opencl.so.0", - "{ONEAPI_COMPILER_LIB}/libur_loader.so.0", - ], -)""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), - "intel-oneapi-compiler-shared-runtime-2026.1": """ -filegroup( - name = "compiler_runtime", - srcs = [ - "{ONEAPI_COMPILER_LIB}/libOpenCL.so", - "{ONEAPI_COMPILER_LIB}/libOpenCL.so.1", - "{ONEAPI_COMPILER_LIB}/libOpenCL.so.1.0.0", - "{ONEAPI_COMPILER_LIB}/libimf.so", - "{ONEAPI_COMPILER_LIB}/libintlc.so.5", - "{ONEAPI_COMPILER_LIB}/libirc.so", - "{ONEAPI_COMPILER_LIB}/libirng.so", - "{ONEAPI_COMPILER_LIB}/libsvml.so", - ], -)""".format(ONEAPI_COMPILER_LIB = ONEAPI_COMPILER_LIB), - "intel-oneapi-mkl-core-2026.1": """ -filegroup( - name = "mkl_core_runtime", - srcs = [ - "{ONEAPI_MKL_LIB}/libmkl_core.so.3", - "{ONEAPI_MKL_LIB}/libmkl_def.so.3", - "{ONEAPI_MKL_LIB}/libmkl_intel_ilp64.so.3", - "{ONEAPI_MKL_LIB}/libmkl_sequential.so.3", - ], -)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-blas-2026.1": """ -filegroup( - name = "mkl_sycl_blas", - srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_blas.so.6"], -)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-dft-2026.1": """ -filegroup( - name = "mkl_sycl_dft", - srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_dft.so.6"], -)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-lapack-2026.1": """ -filegroup( - name = "mkl_sycl_lapack", - srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_lapack.so.6"], -)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-rng-2026.1": """ -filegroup( - name = "mkl_sycl_rng", - srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_rng.so.6"], -)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), - "intel-oneapi-mkl-sycl-sparse-2026.1": """ -filegroup( - name = "mkl_sycl_sparse", - srcs = ["{ONEAPI_MKL_LIB}/libmkl_sycl_sparse.so.6"], -)""".format(ONEAPI_MKL_LIB = ONEAPI_MKL_LIB), -} - -_ROOT_MODULE_DIRECT_DEPS = ["libzml_oneapi"] - -def _oneapi_impl(mctx): - loaded_packages = _read_packages(mctx, [ - "@zml//platforms/oneapi:packages.lock.json", - ]) - - http_archive( - name = "libzml_oneapi", - build_file = "libzml_oneapi.BUILD.bazel", - sha256 = _PLUGIN_ONEAPI_SHA256, - url = _PLUGIN_ONEAPI_URL, - ) - - for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): - pkg = loaded_packages[pkg_name]["amd64"] - http_deb_archive( - name = pkg_name, - urls = pkg["urls"], - sha256 = pkg["sha256"], - strip_prefix = _UBUNTU_STRIP_PREFIX, - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, - ) - - for pkg_name, build_file_content in _ONEAPI_PACKAGES.items(): - pkg = loaded_packages[pkg_name]["amd64"] - http_deb_archive( - name = pkg_name, - urls = pkg["urls"], - sha256 = pkg["sha256"], - strip_prefix = _ONEAPI_STRIP_PREFIX, - build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, - ) - - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = _ROOT_MODULE_DIRECT_DEPS, - root_module_direct_dev_deps = [], - ) - -oneapi_packages = module_extension( - implementation = _oneapi_impl, -) diff --git a/platforms/oneapi/oneapi.h b/platforms/oneapi/oneapi.h deleted file mode 100644 index 933e4f31..00000000 --- a/platforms/oneapi/oneapi.h +++ /dev/null @@ -1 +0,0 @@ - #include diff --git a/platforms/oneapi/oneapi.zig b/platforms/oneapi/oneapi.zig deleted file mode 100644 index c5654741..00000000 --- a/platforms/oneapi/oneapi.zig +++ /dev/null @@ -1,73 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const bazel = @import("bazel"); -const bazel_builtin = @import("bazel_builtin"); -const c = @import("c"); -const pjrt = @import("pjrt"); -const platforms_options = @import("platforms/options"); -const stdx = @import("stdx"); - -const log = std.log.scoped(.@"zml/platforms/oneapi"); - -pub fn isEnabled() bool { - return platforms_options.oneapi_enabled; -} - -fn hasOneApiDevice(io: std.Io) bool { - var dir = std.Io.Dir.openDirAbsolute(io, "/dev/dri", .{ .iterate = true }) catch return false; - defer dir.close(io); - - var it = dir.iterate(); - while (it.next(io) catch null) |entry| { - if (std.mem.startsWith(u8, entry.name, "renderD")) { - return true; - } - } - - return false; -} - -fn setupOneAPIEnv(driver_path: [:0]const u8) void { - _ = c.setenv("CCL_LOG_LEVEL", c.getenv("CCL_LOG_LEVEL") orelse "error", 1); - _ = c.setenv("CCL_ATL_TRANSPORT", "ofi", 1); - _ = c.setenv("FI_PROVIDER", "shm", 1); - // collective-permute through oneccl recv/send often relies on sycl d2d memcpy underneath, - // that doesn't work because of peer residency likely this: - // https://github.com/intel/compute-runtime/issues/953 - _ = c.setenv("XLA_ONECCL_COLLECTIVE_PERMUTE_BYPASS_SYCL_P2P", "1", 1); - _ = c.setenv("ONEAPI_DEVICE_SELECTOR", "level_zero:*", 0); - _ = c.setenv("ZE_ENABLE_ALT_DRIVERS", driver_path, 0); -} - -pub fn load(_: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - if (comptime !isEnabled()) { - return error.Unavailable; - } - - if (comptime builtin.os.tag != .linux) { - return error.Unavailable; - } - - if (!hasOneApiDevice(io)) { - return error.Unavailable; - } - - const r = try bazel.runfiles(bazel_builtin.current_repository); - - var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libzml_oneapi/sandbox", &path_buf) orelse { - log.err("Failed to find sandbox path for oneAPI runtime", .{}); - return error.FileNotFound; - }; - - var driver_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const driver_path = try stdx.Io.Dir.path.bufJoinZ(&driver_path_buf, &.{ sandbox_path, "lib", "libze_intel_gpu.so.1" }); - setupOneAPIEnv(driver_path); - - return blk: { - var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_oneapi.so" }); - break :blk pjrt.Api.loadFrom(path); - }; -} diff --git a/platforms/oneapi/packages.lock.json b/platforms/oneapi/packages.lock.json deleted file mode 100644 index 140ce102..00000000 --- a/platforms/oneapi/packages.lock.json +++ /dev/null @@ -1,269 +0,0 @@ -{ - "packages": [ - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-ccl-2022.1_2022.1.2-9_amd64", - "name": "intel-oneapi-ccl-2022.1", - "sha256": "432e62469978f60a9b35b55216cfd495b0adc76c334707d3bbc81f69b7cc2948", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-ccl-2022.1-2022.1.2-9_amd64.deb" - ], - "version": "2022.1.2-9" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-mpi-2021.18_2021.18.1-9_amd64", - "name": "intel-oneapi-mpi-2021.18", - "sha256": "f6f76f39755329bbac890deeb72fb105df064dc5dcb67cde1396e4d6ef0f6706", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mpi-2021.18-2021.18.1-9_amd64.deb" - ], - "version": "2021.18.1-9" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1_2026.1.1-325_amd64", - "name": "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1", - "sha256": "df0413d57cff5a9c67bd02c1e958d7e0512f77878bbcf976c9e08af7c0b1069c", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1-2026.1.1-325_amd64.deb" - ], - "version": "2026.1.1-325" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-compiler-shared-runtime-2026.1_2026.1.1-325_amd64", - "name": "intel-oneapi-compiler-shared-runtime-2026.1", - "sha256": "c0c77700243af0413f7fbefd26649943c4959171ead91b6ace36ef0c8dfb2630", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-compiler-shared-runtime-2026.1-2026.1.1-325_amd64.deb" - ], - "version": "2026.1.1-325" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-dnnl-2026.0_2026.0.2-40_amd64", - "name": "intel-oneapi-dnnl-2026.0", - "sha256": "4b0f7e23415a695f204f8080328977225759534158d813c2225403f8851433e8", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-dnnl-2026.0-2026.0.2-40_amd64.deb" - ], - "version": "2026.0.2-40" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-mkl-core-2026.1_2026.1.0-236_amd64", - "name": "intel-oneapi-mkl-core-2026.1", - "sha256": "3b230c02fc0ac44f8f71062c20cd883b37c6c1e00940dc0a7f13b02d9c47bd93", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-core-2026.1-2026.1.0-236_amd64.deb" - ], - "version": "2026.1.0-236" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-mkl-sycl-blas-2026.1_2026.1.0-236_amd64", - "name": "intel-oneapi-mkl-sycl-blas-2026.1", - "sha256": "9668d51a7cf68ae4b8944a8cdaa24f6d4ef72604c048f68fd8603b2b792bd783", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-blas-2026.1-2026.1.0-236_amd64.deb" - ], - "version": "2026.1.0-236" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-mkl-sycl-dft-2026.1_2026.1.0-236_amd64", - "name": "intel-oneapi-mkl-sycl-dft-2026.1", - "sha256": "31862dd325a31ccacc1e5e9c93140f4c8bd8a0353e0b3658e2b87e329e58d373", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-dft-2026.1-2026.1.0-236_amd64.deb" - ], - "version": "2026.1.0-236" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-mkl-sycl-lapack-2026.1_2026.1.0-236_amd64", - "name": "intel-oneapi-mkl-sycl-lapack-2026.1", - "sha256": "f701832f5e49b29731c15a27a094122bebe6a1fbd90ddf14c431b7923b8fd921", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-lapack-2026.1-2026.1.0-236_amd64.deb" - ], - "version": "2026.1.0-236" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-mkl-sycl-rng-2026.1_2026.1.0-236_amd64", - "name": "intel-oneapi-mkl-sycl-rng-2026.1", - "sha256": "b0748e152f2042dedd1d21c784fb2a8b551c573b7e7ed0361ada2423deb275bf", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-rng-2026.1-2026.1.0-236_amd64.deb" - ], - "version": "2026.1.0-236" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-mkl-sycl-sparse-2026.1_2026.1.0-236_amd64", - "name": "intel-oneapi-mkl-sycl-sparse-2026.1", - "sha256": "6a499dc1c5f049ad522a5288151ff0c93f535b8e8ab34edcd773785338e07d45", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-mkl-sycl-sparse-2026.1-2026.1.0-236_amd64.deb" - ], - "version": "2026.1.0-236" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-tbb-2023.1_2023.1.0-151_amd64", - "name": "intel-oneapi-tbb-2023.1", - "sha256": "f6e4ae469436a8c5ee2250203268344164c9ff455a2207219c75c463aab99a8b", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-tbb-2023.1-2023.1.0-151_amd64.deb" - ], - "version": "2023.1.0-151" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-tcm-1.5_1.5.0-489_amd64", - "name": "intel-oneapi-tcm-1.5", - "sha256": "7122ed7748d54342b24f0a2838a64f24b54e430ac4d06b2961a8eefee4ffb089", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-tcm-1.5-1.5.0-489_amd64.deb" - ], - "version": "1.5.0-489" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "intel-oneapi-umf-1.1_1.1.0-340_amd64", - "name": "intel-oneapi-umf-1.1", - "sha256": "8705a997042c76f67228b4135184754e1cd3f8f68b6629bcb261c1d5b958ce78", - "urls": [ - "https://apt.repos.intel.com/oneapi/pool/main/intel-oneapi-umf-1.1-1.1.0-340_amd64.deb" - ], - "version": "1.1.0-340" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libnl-3-200_3.7.0-0.3build1.1_amd64", - "name": "libnl-3-200", - "sha256": "15b3f61db753dc3db6f62f4259f8b822299343e500886776486612a13a2ed247", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-3-200_3.7.0-0.3build1.1_amd64.deb" - ], - "version": "3.7.0-0.3build1.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libnl-genl-3-200_3.7.0-0.3build1.1_amd64", - "name": "libnl-genl-3-200", - "sha256": "9ca0cb63f295d00e6346b9924278c07cfc61311cbd288b805b06e25aa08c9fd8", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-genl-3-200_3.7.0-0.3build1.1_amd64.deb" - ], - "version": "3.7.0-0.3build1.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libnl-route-3-200_3.7.0-0.3build1.1_amd64", - "name": "libnl-route-3-200", - "sha256": "c9dbd29cb7fdcf1a0587726cbc5432a7d3e81242108dfcfa2dd6716c44c74d4d", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/libn/libnl3/libnl-route-3-200_3.7.0-0.3build1.1_amd64.deb" - ], - "version": "3.7.0-0.3build1.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libnuma1_2.0.18-1ubuntu0.24.04.1_amd64", - "name": "libnuma1", - "sha256": "f333b8edf6f0b705c19f2a67008194df66f7aab0fa2350dc98510569a033ff29", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/n/numactl/libnuma1_2.0.18-1ubuntu0.24.04.1_amd64.deb" - ], - "version": "2.0.18-1ubuntu0.24.04.1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libigdgmm12_22.10.1-1_24.04_ppa1_amd64", - "name": "libigdgmm12", - "sha256": "2c68fcb150cba1b679063266cdf0054f26f18af90de71d6d3b614eaf95c98f80", - "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-gmmlib/libigdgmm12_22.10.1-1~24.04~ppa1_amd64.deb" - ], - "version": "22.10.1-1~24.04~ppa1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libigc2_2.41.15-1_24.04_amd64", - "name": "libigc2", - "sha256": "c3e5f181eecd40dc8b656c3ff5b7e3d4f690e93181255726516416e61ade1d1f", - "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigc2_2.41.15-1~24.04_amd64.deb" - ], - "version": "2.41.15-1~24.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libigdfcl2_2.41.15-1_24.04_amd64", - "name": "libigdfcl2", - "sha256": "8a2058fd9f5e7d0a4b05001ff861780d2f72529135fa70fb7fbba0f0ff8cd74f", - "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/i/intel-graphics-compiler/libigdfcl2_2.41.15-1~24.04_amd64.deb" - ], - "version": "2.41.15-1~24.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libze-intel-gpu1_26.31.39395.14-1_24.04_ppa1_amd64", - "name": "libze-intel-gpu1", - "sha256": "39f10740e40cf0de1015997d0de094a3c3dc6371c3b9ea1dcd15771d64559c14", - "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z/pool/main/i/intel-compute-runtime/libze-intel-gpu1_26.31.39395.14-1~24.04~ppa1_amd64.deb" - ], - "version": "26.31.39395.14-1~24.04~ppa1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libze1_1.32.0-1_24.04_ppa1_amd64", - "name": "libze1", - "sha256": "d4001dc474d3cb74ded50a449b9f646cfdfbb31e69e70b8af38f2d18273dfce1", - "urls": [ - "https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z/pool/main/l/level-zero-loader/libze1_1.32.0-1~24.04~ppa1_amd64.deb" - ], - "version": "1.32.0-1~24.04~ppa1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "zlib1g_1-1.3.dfsg-3.1ubuntu2.1_amd64", - "name": "zlib1g", - "sha256": "7074b6a2f6367a10d280c00a1cb02e74277709180bab4f2491a2f355ab2d6c20", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20260715T000000Z/pool/main/z/zlib/zlib1g_1.3.dfsg-3.1ubuntu2.1_amd64.deb" - ], - "version": "1:1.3.dfsg-3.1ubuntu2.1" - } - ], - "version": 1 -} diff --git a/platforms/oneapi/packages.yaml b/platforms/oneapi/packages.yaml deleted file mode 100644 index 5a829ac5..00000000 --- a/platforms/oneapi/packages.yaml +++ /dev/null @@ -1,53 +0,0 @@ -# -# bazel run @apt_oneapi//:lock -# -version: 1 - -sources: - - channel: all main - url: https://apt.repos.intel.com/oneapi - - channel: noble main - url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z - - channel: noble-security main - url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z - - channel: noble-updates main - url: https://snapshot.ubuntu.com/ubuntu/20260715T000000Z - - channel: noble main - url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20260930T000000Z - - # IGC 2.41.15 is newer than the graphics snapshot above; the other graphics - # packages are pinned below to keep their versions from it. Use snapshots, - # not the live PPA: the PPA deletes superseded packages from its pool. - - channel: noble main - url: https://snapshot.ppa.launchpadcontent.net/kobuk-team/intel-graphics/ubuntu/20261029T000000Z - -archs: - - "amd64" - -packages: - # Do not use "intel-oneapi-toolkit" package - # It will install an opaque deb file that will at preinst download the sdk toolkit - - "intel-oneapi-ccl-2022.1" - - "intel-oneapi-mpi-2021.18" - - "intel-oneapi-compiler-dpcpp-cpp-runtime-2026.1" - - "intel-oneapi-compiler-shared-runtime-2026.1" - - "intel-oneapi-dnnl-2026.0" - - "intel-oneapi-mkl-core-2026.1" - - "intel-oneapi-mkl-sycl-blas-2026.1" - - "intel-oneapi-mkl-sycl-dft-2026.1" - - "intel-oneapi-mkl-sycl-lapack-2026.1" - - "intel-oneapi-mkl-sycl-rng-2026.1" - - "intel-oneapi-mkl-sycl-sparse-2026.1" - - "intel-oneapi-tbb-2023.1" - - "intel-oneapi-tcm-1.5" - - "intel-oneapi-umf-1.1" - - "libnl-3-200" - - "libnl-genl-3-200" - - "libnl-route-3-200" - - "libnuma1" - - "libigdgmm12 (= 22.10.1-1~24.04~ppa1)" - - "libigc2 (= 2.41.15-1~24.04)" - - "libigdfcl2 (= 2.41.15-1~24.04)" - - "libze-intel-gpu1 (= 26.31.39395.14-1~24.04~ppa1)" - - "libze1" - - "zlib1g" diff --git a/platforms/oneapi/zmlxoneapi.zig b/platforms/oneapi/zmlxoneapi.zig deleted file mode 100644 index 6be2505a..00000000 --- a/platforms/oneapi/zmlxoneapi.zig +++ /dev/null @@ -1,31 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -pub export fn zmlxoneapi_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { - const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ - .{ "libOpenCL.so", "libOpenCL.so.1" }, - .{ "libze_loader.so", "libze_loader.so.1" }, - .{ "libumf.so", "libumf.so.1" }, - .{ "libur_loader.so", "libur_loader.so.0" }, - .{ "libur_adapter_level_zero.so", "libur_adapter_level_zero.so.0" }, - .{ "libur_adapter_level_zero_v2.so", "libur_adapter_level_zero_v2.so.0" }, - .{ "libur_adapter_opencl.so", "libur_adapter_opencl.so.0" }, - .{ "libsycl.so", "libsycl.so.9" }, - .{ "libmkl_core.so", "libmkl_core.so.3" }, - .{ "libmkl_intel_ilp64.so", "libmkl_intel_ilp64.so.3" }, - .{ "libmkl_sequential.so", "libmkl_sequential.so.3" }, - .{ "libmkl_sycl_blas.so", "libmkl_sycl_blas.so.6" }, - }); - - var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const new_filename: [*c]const u8 = if (filename) |f| blk: { - const replacement = replacements.get(std.Io.Dir.path.basename(std.mem.span(f))) orelse break :blk f; - break :blk stdx.Io.Dir.path.bufJoinZ(&buf, &.{ - stdx.process.selfSharedObjectDirPath(), - replacement, - }) catch unreachable; - } else null; - - return std.c.dlopen(new_filename, @bitCast(flags)); -} diff --git a/platforms/packages.bzl b/platforms/packages.bzl index 6d3117cd..633f98b1 100644 --- a/platforms/packages.bzl +++ b/platforms/packages.bzl @@ -45,8 +45,10 @@ def _read(mctx, labels): ret = {} for label in labels: data = json.decode(mctx.read(Label(label))) - for pkg in data["packages"]: - ret.setdefault(pkg["name"], {})[pkg["arch"]] = pkg + ret.update({ + pkg["name"]: pkg + for pkg in data["packages"] + }) return ret packages = struct( diff --git a/platforms/platforms.zig b/platforms/platforms.zig index 789c9a10..4033415c 100644 --- a/platforms/platforms.zig +++ b/platforms/platforms.zig @@ -2,49 +2,35 @@ const std = @import("std"); const cpu = @import("platforms/cpu"); const cuda = @import("platforms/cuda"); -const furiosa = @import("platforms/furiosa"); -const metal = @import("platforms/metal"); const neuron = @import("platforms/neuron"); -const oneapi = @import("platforms/oneapi"); const pjrt = @import("pjrt"); -const platforms_options = @import("platforms/options"); const rocm = @import("platforms/rocm"); const tpu = @import("platforms/tpu"); -const platforms = @This(); - pub const Platform = enum { cpu, cuda, rocm, tpu, neuron, - oneapi, - metal, - furiosa, - - pub fn selectFirstAcceleratorEnabled() error{Unavailable}!Platform { - inline for (@typeInfo(Platform).@"enum".field_names) |field| { - const target: Platform = @field(Platform, field); - if (comptime (target != .cpu and target.isEnabled())) return target; - } - - if (!comptime Platform.cpu.isEnabled()) @compileError("No platform was enabled, use --@zml//platforms:cuda=true to eg enable Cuda"); - return .cpu; - } - - pub fn isEnabled(target: Platform) bool { - return switch (target) { - inline else => |tag| @field(platforms_options, @tagName(tag) ++ "_enabled"), - }; - } - - pub fn load(target: Platform, allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - return switch (target) { - inline else => |tag| @field(platforms, @tagName(tag)).load(allocator, io), - }; - } }; -pub const load = Platform.load; -pub const isEnabled = Platform.isEnabled; +pub fn load(allocator: std.mem.Allocator, io: std.Io, tag: Platform) !*const pjrt.Api { + return switch (tag) { + .cpu => try cpu.load(allocator, io), + .cuda => try cuda.load(allocator, io), + .rocm => try rocm.load(allocator, io), + .tpu => try tpu.load(allocator, io), + .neuron => try neuron.load(allocator, io), + }; +} + +pub fn isEnabled(tag: Platform) bool { + return switch (tag) { + .cpu => cpu.isEnabled(), + .cuda => cuda.isEnabled(), + .rocm => rocm.isEnabled(), + .tpu => tpu.isEnabled(), + .neuron => neuron.isEnabled(), + }; +} diff --git a/platforms/rocm/BUILD.bazel b/platforms/rocm/BUILD.bazel index 91651372..76ee4e9f 100644 --- a/platforms/rocm/BUILD.bazel +++ b/platforms/rocm/BUILD.bazel @@ -1,49 +1,47 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_shared_library") -exports_files( - ["merge_amdgpu_ids.zig"], - visibility = ["@libzml_rocm//:__subpackages__"], -) - zig_shared_library( name = "zmlxrocm", main = "zmlxrocm.zig", shared_lib_name = "libzmlxrocm.so.0", - target_compatible_with = ["@platforms//os:linux"], - visibility = [ - "//bin/zml-smi:__pkg__", - "@libzml_rocm//:__subpackages__", - ], + visibility = ["@libpjrt_rocm//:__subpackages__"], # Use Clang's compiler-rt, but disable stack checking # to avoid requiring on the _zig_probe_stack symbol. zigopts = ["-fno-stack-check"], deps = ["//stdx"], ) +alias( + name = "hipblaslt", + actual = "@libpjrt_rocm//:hipblaslt", +) + cc_library( - name = "libzml_rocm", - hdrs = ["rocm.h"], - tags = ["manual"], - deps = ["@libzml_rocm"], + name = "empty", + defines = ["ZML_RUNTIME_ROCM_DISABLED"], +) + +cc_library( + name = "libpjrt_rocm", + defines = ["ZML_RUNTIME_ROCM"], + deps = ["@libpjrt_rocm"], ) zig_library( name = "rocm", import_name = "platforms/rocm", main = "rocm.zig", - tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//pjrt", - "//platforms:options", ] + select({ "//platforms:rocm.enabled": [ - ":libzml_rocm", + ":libpjrt_rocm", "//bazel", "//stdx", "@rules_zig//zig/runfiles", ], - "//conditions:default": [], + "//conditions:default": [":empty"], }), ) diff --git a/platforms/rocm/libpjrt_rocm.BUILD.bazel b/platforms/rocm/libpjrt_rocm.BUILD.bazel new file mode 100644 index 00000000..4aaabacf --- /dev/null +++ b/platforms/rocm/libpjrt_rocm.BUILD.bazel @@ -0,0 +1,98 @@ +load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@aspect_bazel_lib//lib:tar.bzl", "tar") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_list_flag") +load("@zml//bazel:patchelf.bzl", "patchelf") + +bool_flag( + name = "hipblaslt", + build_setting_default = False, + visibility = ["//visibility:public"], +) + +config_setting( + name = "_hipblaslt", + flag_values = {":hipblaslt": "True"}, +) + +patchelf( + name = "libpjrt_rocm_so", + src = "libpjrt_rocm.so", + add_needed = [ + "libzmlxrocm.so.0", + ], + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + set_rpath = "$ORIGIN", +) + +copy_to_directory( + name = "sandbox", + srcs = [ + ":libpjrt_rocm_so", + "@comgr//:amd_comgr", + "@hip-runtime-amd//:amdhip", + "@hip-runtime-amd//:hiprtc", + "@hipblaslt//:hipblaslt", + "@hipblaslt//:rocroller", + "@hipfft", + "@hipsolver", + "@hsa-amd-aqlprofile//:hsa-amd-aqlprofile", + "@hsa-rocr//:hsa-runtime", + "@libdrm-amdgpu-amdgpu1", + "@libdrm-amdgpu-common//:amdgpu_ids", + "@libdrm2-amdgpu", + "@libelf1", + "@libnuma1", + "@libtinfo6", + "@libzstd1", + "@miopen-hip//:MIOpen", + "@rccl", + "@rocblas//:rocblas", + "@rocblas//:runfiles", + "@rocfft", + "@rocm-core", + "@rocm-device-libs//:runfiles", + "@rocm-smi-lib//:rocm_smi", + "@rocprofiler-register", + "@rocprofiler-sdk//:rocprofiler-sdk", + "@rocprofiler-sdk-rocpd//:rocprofiler-sdk-rocpd", + "@rocprofiler-sdk-roctx//:rocprofiler-sdk-roctx", + "@rocsolver", + "@roctracer", + "@roctracer//:roctx", + "@zlib1g", + "@zml//platforms/rocm:zmlxrocm", + ] + select({ + ":_hipblaslt": ["@hipblaslt//:runfiles"], + "//conditions:default": [], + }), + replace_prefixes = { + "amdhip": "lib", + "hipblaslt": "lib", + "lib/x86_64-linux-gnu": "lib", + "libdrm-amdgpu-amdgpu1": "lib", + "libelf1": "lib", + "libpjrt_rocm_so": "lib", + "opt/amdgpu/lib/x86_64-linux-gnu": "lib", + "opt/amdgpu/share": "share", + "rocblas": "lib", + "rocroller": "lib", + "platforms/rocm": "lib", + "usr/lib/x86_64-linux-gnu": "lib", + }, + add_directory_to_runfiles = True, + include_external_repositories = ["**"], +) + +tar( + name = "amdgpu_ids_layer", + srcs = ["@libdrm-amdgpu-common//:amdgpu_ids"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "libpjrt_rocm", + data = [":sandbox"], + visibility = ["@zml//platforms/rocm:__subpackages__"], +) diff --git a/platforms/rocm/libzml_rocm.BUILD.bazel b/platforms/rocm/libzml_rocm.BUILD.bazel deleted file mode 100644 index 7996b001..00000000 --- a/platforms/rocm/libzml_rocm.BUILD.bazel +++ /dev/null @@ -1,122 +0,0 @@ -load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@bazel_skylib//rules:run_binary.bzl", "run_binary") -load("@rules_cc//cc:cc_library.bzl", "cc_library") -load("@rules_zig//zig:defs.bzl", "zig_binary") -load("@zml//bazel:patchelf.bzl", "patchelf") - -patchelf( - name = "libzml_rocm_so", - src = "lib/libzml_rocm.so", - add_needed = [ - "libzmlxrocm.so.0", - ], - rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - }, - set_rpath = "$ORIGIN:$ORIGIN/rocm_sysdeps/lib", -) - -zig_binary( - name = "merge_amdgpu_ids", - main = "@zml//platforms/rocm:merge_amdgpu_ids.zig", -) - -run_binary( - name = "amdgpu_ids", - srcs = [ - "@libdrm-common//:amdgpu_ids", - "@libdrm_mesa_amdgpu_ids//file:amdgpu.ids", - ], - outs = ["amdgpu.ids"], - args = [ - "$(location @libdrm-common//:amdgpu_ids)", - "$(location @libdrm_mesa_amdgpu_ids//file:amdgpu.ids)", - "$(location amdgpu.ids)", - ], - tool = ":merge_amdgpu_ids", - visibility = ["@zml//bin/zml-smi:__pkg__"], -) - -copy_to_directory( - name = "sandbox", - srcs = [ - ":amdgpu_ids", - ":libzml_rocm_so", - "@amdrocm-amdsmi//:libamd_smi", - "@amdrocm-base//:rocm-core", - "@amdrocm-base//:rocprofiler-register", - "@amdrocm-blas//:hipblaslt", - "@amdrocm-blas//:hipblaslt_runfiles", - "@amdrocm-blas//:rocblas", - "@amdrocm-blas//:rocblas_runfiles", - "@amdrocm-dnn//:MIOpen", - "@amdrocm-fft//:hipfft", - "@amdrocm-fft//:rocfft", - "@amdrocm-llvm//:llvm_libs", - "@amdrocm-llvm//:rocm_device_libs_runfiles", - "@amdrocm-math-common//:host_math_libs", - "@amdrocm-profiler-base//:hsa-amd-aqlprofile", - "@amdrocm-profiler-base//:rocprofiler-sdk", - "@amdrocm-profiler-base//:rocprofv3_tool", - "@amdrocm-profiler-base//:roctracer", - "@amdrocm-profiler-base//:roctx", - "@amdrocm-rccl//:rccl", - "@amdrocm-runtime//:amd_comgr", - "@amdrocm-runtime//:amdhip", - "@amdrocm-runtime//:hiprtc", - "@amdrocm-runtime//:hsa-runtime", - "@amdrocm-solver//:hipsolver", - "@amdrocm-solver//:rocsolver", - "@amdrocm-sparse//:rocsparse", - "@amdrocm-sysdeps//:system_libs", - "@libatomic1", - "@llvm-libunwind1//:libunwind", - "@zml//platforms/rocm:zmlxrocm", - ], - add_directory_to_runfiles = True, - include_external_repositories = ["**"], - replace_prefixes = { - "MIOpen": "lib", - "amdhip_so": "lib", - "amdhip": "lib", - "amd_comgr": "lib", - "hipdnn_backend": "lib", - "hipblas": "lib", - "hiprtc_so": "lib", - "hipfft": "lib", - "hiprtc": "lib", - "hipsolver": "lib", - "hipsparse": "lib", - "hipsparselt": "lib", - "hsa-amd-aqlprofile": "lib", - "hsa-runtime": "lib", - "hipblaslt": "lib", - "lib/host-math/lib": "lib", - "lib/llvm/lib": "lib", - "lib/llvm/amdgcn": "amdgcn", - "libzml_rocm_so": "lib", - "amdgpu.ids": "share/libdrm/amdgpu.ids", - "rccl": "lib", - "rocblas": "lib", - "rocfft": "lib", - "rocm-core": "lib", - "rocprofiler-register": "lib", - "rocprofiler-sdk_so": "lib", - "rocprofiler-sdk": "lib", - "rocm_smi": "lib", - "rocsolver": "lib", - "rocroller": "lib", - "rocsparse": "lib", - "roctracer": "lib", - "roctx": "lib", - "platforms/rocm": "lib", - "usr/lib/x86_64-linux-gnu": "lib", - }, - visibility = ["//visibility:public"], -) - -cc_library( - name = "libzml_rocm", - data = [":sandbox"], - visibility = ["@zml//platforms/rocm:__subpackages__"], -) diff --git a/platforms/rocm/merge_amdgpu_ids.zig b/platforms/rocm/merge_amdgpu_ids.zig deleted file mode 100644 index b939df82..00000000 --- a/platforms/rocm/merge_amdgpu_ids.zig +++ /dev/null @@ -1,73 +0,0 @@ -const std = @import("std"); - -pub fn main(init: std.process.Init) !void { - const io = init.io; - const arena = init.arena.allocator(); - - const args = try init.minimal.args.toSlice(arena); - - if (args.len != 4) { - try std.Io.File.stderr().writeStreamingAll(io, "Usage: merge_amdgpu_ids \n"); - std.process.exit(1); - } - - var entries: std.StringHashMapUnmanaged([]const u8) = .empty; - defer entries.deinit(arena); - - const cwd = std.Io.Dir.cwd(); - - try parseFile(io, cwd, arena, args[2], &entries); - try parseFile(io, cwd, arena, args[1], &entries); - - const out = try cwd.createFile(io, args[3], .{}); - defer out.close(io); - - var buf: [4096]u8 = undefined; - var writer = out.writer(io, &buf); - - try writer.interface.print("# List of AMDGPU IDs\n" ++ - "#\n" ++ - "# Syntax:\n" ++ - "# device_id,\trevision_id,\tproduct_name <-- single tab after comma\n" ++ - "\n" ++ - "1.0.0\n", .{}); - - var it = entries.valueIterator(); - while (it.next()) |v| { - try writer.interface.print("{s}\n", .{v.*}); - } - - try writer.flush(); -} - -fn parseFile( - io: std.Io, - cwd: std.Io.Dir, - arena: std.mem.Allocator, - path: []const u8, - entries: *std.StringHashMapUnmanaged([]const u8), -) !void { - const file = try cwd.openFile(io, path, .{ .mode = .read_only }); - defer file.close(io); - - var buf: [4096]u8 = undefined; - var reader = file.reader(io, &buf); - - while (true) { - const raw = try reader.interface.takeDelimiter('\n') orelse break; - - const line = std.mem.trimEnd(u8, raw, " \t\r"); - if (line.len == 0 or line[0] == '#') continue; - - var iter = std.mem.splitSequence(u8, line, ","); - - const device = std.mem.trim(u8, iter.next() orelse continue, " \t"); - const revision = std.mem.trim(u8, iter.next() orelse continue, " \t"); - const product = std.mem.trim(u8, iter.rest(), " \t"); - - const key = try std.fmt.allocPrint(arena, "{s},{s}", .{ device, revision }); - const value = try std.fmt.allocPrint(arena, "{s},\t{s},\t{s}", .{ device, revision, product }); - - try entries.put(arena, key, value); - } -} diff --git a/platforms/rocm/packages.lock.json b/platforms/rocm/packages.lock.json old mode 100644 new mode 100755 index 2b2a4836..67139f24 --- a/platforms/rocm/packages.lock.json +++ b/platforms/rocm/packages.lock.json @@ -4,129 +4,48 @@ "arch": "amd64", "dependencies": [ { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" + "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70200-2278374.22.04" } ], - "key": "llvm-libunwind1_1-22.1.8_-p--p-20260613092327-p-e80beda6e255-1_exp1_20260613092437.81_amd64", - "name": "llvm-libunwind1", - "sha256": "0da2b5596faee4e53456b7bdf51458e7bd5ec06852af93f2538e74c122032df2", + "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-common", + "sha256": "baa11bc7e3f8e6931a69dea7f1485c2f982da184811ac7545342953ffb64abc4", "urls": [ - "https://apt.llvm.org/jammy/pool/main/l/llvm-toolchain-22/llvm-libunwind1_22.1.8~++20260613092327+e80beda6e255-1~exp1~20260613092437.81_amd64.deb" + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu-common/libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_all.deb" ], - "version": "1:22.1.8~++20260613092327+e80beda6e255-1~exp1~20260613092437.81" + "version": "1.0.0.70200-2278374.22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "sha256": "d8e36ba6cc55719d728c38166bf753650710878e02e30af2aa79aefa244eb275", + "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", + "name": "amdgpu-core", + "sha256": "916fdee8e388c36a67abda89e0868e5789f4b77a3c351f945db4218fbcbe226c", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6_2.35-0ubuntu3.10_amd64.deb" + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/a/amdgpu-core/amdgpu-core_7.2.70200-2278374.22.04_all.deb" ], - "version": "2.35-0ubuntu3.10" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "sha256": "3fa566e9f861a08736cbc5a97562d9d6e4f0c00450fbeafcb6d7583423b04a98", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt1_4.4.27-1_amd64.deb" - ], - "version": "1:4.4.27-1" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "sha256": "8ca7b573446d22ef46a97b4ec537c38acc8653f76391127253040acc7b74af5f", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "sha256": "80a1f5df097af3978f7df2808006ca77197865e5eb7a923712aee6876e7a14cb", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" + "version": "1:7.2.70200-2278374.22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70200-43~22.04" }, { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" }, { "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", "name": "libgcc-s1", "version": "12.3.0-1ubuntu1~22.04" }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - } - ], - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "sha256": "56ec991983c7c89a2c15bd5134534531d722ec5b51a60679f7ceaa1a09b37a09", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libatomic1_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libdrm-common_2.4.113-2_ubuntu0.22.04.1_amd64", - "name": "libdrm-common", - "sha256": "35a306712d8b15b30c42ecd73ec087813eb01c0b3125dc8f7ca2b5134e133522", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libd/libdrm/libdrm-common_2.4.113-2~ubuntu0.22.04.1_all.deb" - ], - "version": "2.4.113-2~ubuntu0.22.04.1" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" - }, { "key": "libc6_2.35-0ubuntu3.10_amd64", "name": "libc6", @@ -137,16 +56,16 @@ "name": "libcrypt1", "version": "1:4.4.27-1" }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, { "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", "name": "gcc-12-base", "version": "12.3.0-1ubuntu1~22.04" }, + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, { "key": "python3_3.10.6-1_22.04.1_amd64", "name": "python3", @@ -302,11 +221,273 @@ "name": "libmpdec3", "version": "2.5.1-2build2" }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70200-2278374.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70200-2278374.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70200-2278374.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70200-2278374.22.04" + } + ], + "key": "hsa-rocr_1.18.0.70200-43_22.04_amd64", + "name": "hsa-rocr", + "sha256": "c2ee2205b61a0c6b00478e70fa0cde4441fc9658cddcedb8bff07ab07569719c", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-rocr/hsa-rocr_1.18.0.70200-43~22.04_amd64.deb" + ], + "version": "1.18.0.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [ { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", "name": "libstdc++6", "version": "12.3.0-1ubuntu1~22.04" }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, { "key": "libffi8_3.4.2-4_amd64", "name": "libffi8", @@ -348,24 +529,79 @@ "version": "3.10.6-1~22.04.1" } ], - "key": "amdrocm-amdsmi10.1_10.1.0-3_amd64", - "name": "amdrocm-amdsmi10.1", - "sha256": "994e715d38a16d9c408eebd508c66123ff2f64cf4d4f4ca15780c4500e85103a", + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "sha256": "144d516c55f368e82c231df469c1d3097ee4f44302ef3b71fe3eeded4199f5d3", "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-amdsmi10.1_10.1.0-3_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-register/rocprofiler-register_0.6.0.70200-43~22.04_amd64.deb" ], - "version": "10.1.0-3" + "version": "0.6.0.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "sha256": "a2493d98144a586df49489c4b4b7a7449c9eb123597515929699b1b5fccfd916", + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "sha256": "83873058e692936a09649ede2f8e70b87dde1f3f5488db53da8081b81c79a5d9", "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sysdeps10.1_10.1.0-3_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.3.0-1ubuntu1~22.04_amd64.deb" ], - "version": "10.1.0-3" + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "sha256": "8ca7b573446d22ef46a97b4ec537c38acc8653f76391127253040acc7b74af5f", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libgcc-s1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "sha256": "d8e36ba6cc55719d728c38166bf753650710878e02e30af2aa79aefa244eb275", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6_2.35-0ubuntu3.10_amd64.deb" + ], + "version": "2.35-0ubuntu3.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "sha256": "3fa566e9f861a08736cbc5a97562d9d6e4f0c00450fbeafcb6d7583423b04a98", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt1_4.4.27-1_amd64.deb" + ], + "version": "1:4.4.27-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "sha256": "80a1f5df097af3978f7df2808006ca77197865e5eb7a923712aee6876e7a14cb", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/gcc-12-base_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "sha256": "9e8c61b3b260547f88b795f521d82249dfae50f9f82522c6ba9a4fad8fe10f0a", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-core/rocm-core_7.2.0.70200-43~22.04_amd64.deb" + ], + "version": "7.2.0.70200-43~22.04" }, { "arch": "amd64", @@ -708,17 +944,6 @@ ], "version": "2.5.1-2build2" }, - { - "arch": "amd64", - "dependencies": [], - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "sha256": "83873058e692936a09649ede2f8e70b87dde1f3f5488db53da8081b81c79a5d9", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libstdc++6_12.3.0-1ubuntu1~22.04_amd64.deb" - ], - "version": "12.3.0-1ubuntu1~22.04" - }, { "arch": "amd64", "dependencies": [], @@ -809,364 +1034,55 @@ }, { "arch": "amd64", - "dependencies": [ - { - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-llvm10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - } - ], - "key": "amdrocm-base10.1_10.1.0-3_amd64", - "name": "amdrocm-base10.1", - "sha256": "a3d48032edee53c51016b28884a1fe81e2d2f84e087c2546dd20753402ad53c9", + "dependencies": [], + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "sha256": "2cc781614b045b450a1e073ee3b0902f393ee58ede1d2ea4526ad805fcccf538", "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-base10.1_10.1.0-3_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/e/elfutils/libelf1_0.186-1ubuntu0.1_amd64.deb" ], - "version": "10.1.0-3" + "version": "0.186-1ubuntu0.1" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "sha256": "2cf4fa3b82f9b37be9fc18ca6886e1791585c1b8041758ccba9e3c5dadb7c6b1", + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "sha256": "0721c89001fbbd1ada23e89da5d60e762763c1a7b3dc814a2e9a518480a8043d", "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-runtime10.1_10.1.0-3_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/numactl/libnuma1_2.0.14-3ubuntu2_amd64.deb" ], - "version": "10.1.0-3" + "version": "2.0.14-3ubuntu2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "sha256": "460f6dd499dbff0d4bc368d40b0c3f943dbf981df77f362ac05cce7befcf7779", + "urls": [ + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-amdgpu1_2.4.125.70200-2278374.22.04_amd64.deb" + ], + "version": "1:2.4.125.70200-2278374.22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm2-amdgpu", + "sha256": "a5a620541bcd852d6feb8299dda804c41cb979a3870cf82a783112b58ad9d7e4", + "urls": [ + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm2-amdgpu_2.4.125.70200-2278374.22.04_amd64.deb" + ], + "version": "1:2.4.125.70200-2278374.22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -1189,79 +1105,24 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" }, { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" }, { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" }, { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -1363,35 +1224,158 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, { "key": "libnsl2_1.3.0-2build2_amd64", "name": "libnsl2", "version": "1.3.0-2build2" }, { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" }, { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" }, { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + } + ], + "key": "hsa-amd-aqlprofile_1.0.0.70200-43_22.04_amd64", + "name": "hsa-amd-aqlprofile", + "sha256": "c4eabbbac5adb81d30442fe4172660591b4b11f70596035c0cd6b5dc80bd27be", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-amd-aqlprofile/hsa-amd-aqlprofile_1.0.0.70200-43~22.04_amd64.deb" + ], + "version": "1.0.0.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "sha256": "a62cc2bbf9cfc46b263b17ff90bea66ae01267c03014b55a9764eab6895db4a7", + "urls": [ + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/h/hsa-runtime-rocr4wsl-amdgpu/hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64.deb" + ], + "version": "25.30.13-2281980.22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", "version": "2.35-0ubuntu3.10" }, { - "key": "amdrocm-base10.1_10.1.0-3_amd64", - "name": "amdrocm-base10.1", - "version": "10.1.0-3" + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" }, { "key": "python3_3.10.6-1_22.04.1_amd64", @@ -1413,6 +1397,96 @@ "name": "libuuid1", "version": "2.37.2-4ubuntu3.4" }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, { "key": "libtinfo6_6.3-2ubuntu0.1_amd64", "name": "libtinfo6", @@ -1433,6 +1507,11 @@ "name": "readline-common", "version": "8.1.2-1" }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, { "key": "libncursesw6_6.3-2ubuntu0.1_amd64", "name": "libncursesw6", @@ -1482,15 +1561,3196 @@ "key": "python3-minimal_3.10.6-1_22.04.1_amd64", "name": "python3-minimal", "version": "3.10.6-1~22.04.1" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" } ], - "key": "amdrocm-llvm10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm10.1", - "sha256": "6a4727102d830f6c97b911351192b15623189278afad9b4558514a6fe51ca58a", + "key": "comgr_3.0.0.70200-43_22.04_amd64", + "name": "comgr", + "sha256": "8f58b1958811430ba19426489608b3cbf685e5a78c1d6d9414c7f23cc8311a13", "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-llvm10.1_10.1.0-3_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/c/comgr/comgr_3.0.0.70200-43~22.04_amd64.deb" ], - "version": "10.1.0-3" + "version": "3.0.0.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "sha256": "145736b5bec52cd4fc1b2c09517759383b95255f9fd304ace1de7d830295bb33", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libtinfo-dev_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "sha256": "1de196ffdd885b5953a44cde13e68b77dc7ceb9c9921482dc84e358bd1ad8b02", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses-dev_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "sha256": "6382f5c1d7d0bf39cbb94e208ece2bdf556cb3a8dfe278748542c7b8b57a2e6d", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/ncurses-bin_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "sha256": "c2e3b50af761886617462ca47ebc9303a83968129acca2add42b8eba0a7abca3", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6-dev_2.35-0ubuntu3.10_amd64.deb" + ], + "version": "2.35-0ubuntu3.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "sha256": "e93aa448fe708a4bbfcc40b2a8cab6f5f88896174a301caab2719f734e08a621", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl-dev_1.3.0-2build2_amd64.deb" + ], + "version": "1.3.0-2build2" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "sha256": "f4254aff5a32ef3c9fca06d3f4f2eff71c93b9393cc1ecc718f7478a5d6a83da", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc-dev_1.3.2-2ubuntu0.1_amd64.deb" + ], + "version": "1.3.2-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "sha256": "447c6d8f7624a3d728a12232440d3f10e6e53bbf60cc202b7beb8a3bc9f7acc9", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/rpcsvc-proto/rpcsvc-proto_1.4.2-0ubuntu6_amd64.deb" + ], + "version": "1.4.2-0ubuntu6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "sha256": "a79be2f6e45823dcc09e04d5e98c88ec88d07d5b8895d05b875a8ade8b345efa", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt-dev_4.4.27-1_amd64.deb" + ], + "version": "1:4.4.27-1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "sha256": "f534bb052e96efb9f9e08279b18950d1b1f2a7958f3a06c426fed7d2c1500840", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/l/linux/linux-libc-dev_5.15.0-143.153_amd64.deb" + ], + "version": "5.15.0-143.153" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "sha256": "d93f29bc5ad18f69a4fe292fd392dc17e0a9d9bf0a22ee11fcf43e6395b6ecf8", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc-dev-bin_2.35-0ubuntu3.10_amd64.deb" + ], + "version": "2.35-0ubuntu3.10" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "sha256": "4a828e6cb2a721be6f4ed7b34256d3fc9aab1495b2b4923fd579d4dc589496fd", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/n/ncurses/libncurses6_6.3-2ubuntu0.1_amd64.deb" + ], + "version": "6.3-2ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocprofiler-sdk-rocpd_1.1.0-43_22.04_amd64", + "name": "rocprofiler-sdk-rocpd", + "version": "1.1.0-43~22.04" + }, + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocprofiler-sdk-roctx_1.1.0-43_22.04_amd64", + "name": "rocprofiler-sdk-roctx", + "version": "1.1.0-43~22.04" + }, + { + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70200-43~22.04" + } + ], + "key": "rocprofiler-sdk_1.1.0-43_22.04_amd64", + "name": "rocprofiler-sdk", + "sha256": "adaebb45a2b713fb7677cca10d50afbce33b025f6bbf5a84cf37707fc413127d", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk/rocprofiler-sdk_1.1.0-43~22.04_amd64.deb" + ], + "version": "1.1.0-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocprofiler-sdk-rocpd_1.1.0-43_22.04_amd64", + "name": "rocprofiler-sdk-rocpd", + "sha256": "2dbfad4c5c1e55bb65771bb3aa34460c52e6c7952767f26756d8aa7d98c47414", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk-rocpd/rocprofiler-sdk-rocpd_1.1.0-43~22.04_amd64.deb" + ], + "version": "1.1.0-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocprofiler-sdk-roctx_1.1.0-43_22.04_amd64", + "name": "rocprofiler-sdk-roctx", + "sha256": "4c74438154cd91d81768c5ecbbed24859f787db638dcacce6b70445abf8577c7", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocprofiler-sdk-roctx/rocprofiler-sdk-roctx_1.1.0-43~22.04_amd64.deb" + ], + "version": "1.1.0-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocrand_4.2.0.70200-43_22.04_amd64", + "name": "rocrand", + "version": "4.2.0.70200-43~22.04" + }, + { + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", + "name": "hipblaslt", + "version": "1.2.1.70200-43~22.04" + }, + { + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "name": "roctracer", + "version": "4.1.70200.70200-43~22.04" + }, + { + "key": "rocblas_5.2.0.70200-43_22.04_amd64", + "name": "rocblas", + "version": "5.2.0.70200-43~22.04" + }, + { + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.2.26015.70200-43~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70200-43~22.04" + }, + { + "key": "comgr_3.0.0.70200-43_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70200-43~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70200-43~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + } + ], + "key": "miopen-hip_3.5.1.70200-43_22.04_amd64", + "name": "miopen-hip", + "sha256": "c9b6ea5b4f07aa6ee21f95f69ac3dd88ef87449e52c41d751b01b071ab121656", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/m/miopen-hip/miopen-hip_3.5.1.70200-43~22.04_amd64.deb" + ], + "version": "3.5.1.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocrand_4.2.0.70200-43_22.04_amd64", + "name": "rocrand", + "sha256": "2b22a1cc2861eff622c6756b7cba160fb0b9211cbd9452b89b7d3ab3ce19e6c8", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocrand/rocrand_4.2.0.70200-43~22.04_amd64.deb" + ], + "version": "4.2.0.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "name": "roctracer", + "version": "4.1.70200.70200-43~22.04" + } + ], + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", + "name": "hipblaslt", + "sha256": "a7baff52908f4a25b134d7e2613c1eb2b51b369509465d2a502b457f3ce3c1b2", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblaslt/hipblaslt_1.2.1.70200-43~22.04_amd64.deb" + ], + "version": "1.2.1.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "name": "roctracer", + "sha256": "4b40ae9c1981979f3fb69e9713c5b9fc1a17f868a83732f38f8fb5e53be3a612", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/roctracer/roctracer_4.1.70200.70200-43~22.04_amd64.deb" + ], + "version": "4.1.70200.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.2.26015.70200-43~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70200-43~22.04" + }, + { + "key": "comgr_3.0.0.70200-43_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70200-43~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70200-43~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", + "name": "hipblaslt", + "version": "1.2.1.70200-43~22.04" + }, + { + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "name": "roctracer", + "version": "4.1.70200.70200-43~22.04" + } + ], + "key": "rocblas_5.2.0.70200-43_22.04_amd64", + "name": "rocblas", + "sha256": "0581ada80350b0d959e1863af4ff4be3ef31fe9838e941d09b8f0693dda7d23d", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocblas/rocblas_5.2.0.70200-43~22.04_amd64.deb" + ], + "version": "5.2.0.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70200-43~22.04" + }, + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "comgr_3.0.0.70200-43_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70200-43~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70200-43~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + } + ], + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "name": "hip-runtime-amd", + "sha256": "97f477244025fb3bd8762d8d730d18545cafe080cb938f3f6520aeaff6e05c42", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hip-runtime-amd/hip-runtime-amd_7.2.26015.70200-43~22.04_amd64.deb" + ], + "version": "7.2.26015.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "name": "rocminfo", + "sha256": "6b8bcd3f5d0589a3b6447353c140b4ef676c0fbe319050a0f584fde4aa83f16f", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocminfo/rocminfo_1.0.0.70200-43~22.04_amd64.deb" + ], + "version": "1.0.0.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "sha256": "830f2957662723860d3ebac949b22867df00d9a4c5ab1c2bf98e10674dc1c756", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/pciutils_3.7.0-6_amd64.deb" + ], + "version": "1:3.7.0-6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "sha256": "624bea7b5a9073a5bc2dfa5d956571d6127c20400fcfcfca1e47b7448ba5d351", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pciutils/libpci3_3.7.0-6_amd64.deb" + ], + "version": "1:3.7.0-6" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "sha256": "026609ed71363cd6ed13ace2dac0c9813d94be3874c778539a8a64d6d2cc5566", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/p/pci.ids/pci.ids_0.0~2022.01.22-1ubuntu0.1_all.deb" + ], + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "sha256": "451d4eaa84dbeabc045611937a78489710ad2c2baf277a63aacda7ee5226e3ea", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/s/systemd/libudev1_249.11-0ubuntu3.16_amd64.deb" + ], + "version": "249.11-0ubuntu3.16" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "sha256": "f355cebb66405a84cc88bb313440abd39f97cf45d3f468ea9ec7ba34e11b6c8a", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/k/kmod/libkmod2_29-1ubuntu1_amd64.deb" + ], + "version": "29-1ubuntu1" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "rocm-smi-lib_7.8.0.70200-43_22.04_amd64", + "name": "rocm-smi-lib", + "version": "7.8.0.70200-43~22.04" + }, + { + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.2.26015.70200-43~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70200-43~22.04" + }, + { + "key": "comgr_3.0.0.70200-43_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70200-43~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70200-43~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + } + ], + "key": "rccl_2.27.7.70200-43_22.04_amd64", + "name": "rccl", + "sha256": "bb450a46ee36578a2b171b9af891f5e94ef04e9f743f75ec16e99246ddaff878", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rccl/rccl_2.27.7.70200-43~22.04_amd64.deb" + ], + "version": "2.27.7.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocm-smi-lib_7.8.0.70200-43_22.04_amd64", + "name": "rocm-smi-lib", + "sha256": "bfd4ef1c3382eab7685c92c218ef14593ea1bd06836bb8cc903e4100e9125d79", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-smi-lib/rocm-smi-lib_7.8.0.70200-43~22.04_amd64.deb" + ], + "version": "7.8.0.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + } + ], + "key": "rocm-device-libs_1.0.0.70200-43_22.04_amd64", + "name": "rocm-device-libs", + "sha256": "f1fe8f8fea614c3f734d002f6ae688929156bcfde0f27457412a7c3074d18d51", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-device-libs/rocm-device-libs_1.0.0.70200-43~22.04_amd64.deb" + ], + "version": "1.0.0.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [ + { + "key": "hipcc_1.1.1.70200-43_22.04_amd64", + "name": "hipcc", + "version": "1.1.1.70200-43~22.04" + }, + { + "key": "rocm-llvm_22.0.0.26014.70200-43_22.04_amd64", + "name": "rocm-llvm", + "version": "22.0.0.26014.70200-43~22.04" + }, + { + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" + }, + { + "key": "libc6_2.35-0ubuntu3.10_amd64", + "name": "libc6", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libcrypt1_1-4.4.27-1_amd64", + "name": "libcrypt1", + "version": "1:4.4.27-1" + }, + { + "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgcc-s1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", + "name": "gcc-12-base", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" + }, + { + "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc3", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-common", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", + "name": "libgssapi-krb5-2", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5support0", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", + "name": "libkrb5-3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libssl3_3.0.2-0ubuntu1.19_amd64", + "name": "libssl3", + "version": "3.0.2-0ubuntu1.19" + }, + { + "key": "debconf_1.5.79ubuntu1_amd64", + "name": "debconf", + "version": "1.5.79ubuntu1" + }, + { + "key": "perl-base_5.34.0-3ubuntu1.4_amd64", + "name": "perl-base", + "version": "5.34.0-3ubuntu1.4" + }, + { + "key": "dpkg_1.21.1ubuntu2.3_amd64", + "name": "dpkg", + "version": "1.21.1ubuntu2.3" + }, + { + "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", + "name": "tar", + "version": "1.34+dfsg-1ubuntu0.1.22.04.2" + }, + { + "key": "libselinux1_3.3-1build2_amd64", + "name": "libselinux1", + "version": "3.3-1build2" + }, + { + "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", + "name": "libpcre2-8-0", + "version": "10.39-3ubuntu0.1" + }, + { + "key": "libacl1_2.3.1-1_amd64", + "name": "libacl1", + "version": "2.3.1-1" + }, + { + "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", + "name": "zlib1g", + "version": "1:1.2.11.dfsg-2ubuntu9.2" + }, + { + "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", + "name": "libzstd1", + "version": "1.4.8+dfsg-3build1" + }, + { + "key": "liblzma5_5.2.5-2ubuntu1_amd64", + "name": "liblzma5", + "version": "5.2.5-2ubuntu1" + }, + { + "key": "libbz2-1.0_1.0.8-5build1_amd64", + "name": "libbz2-1.0", + "version": "1.0.8-5build1" + }, + { + "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", + "name": "libkeyutils1", + "version": "1.6.1-2ubuntu3" + }, + { + "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", + "name": "libk5crypto3", + "version": "1.19.2-2ubuntu0.7" + }, + { + "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", + "name": "libcom-err2", + "version": "1.46.5-2ubuntu1.2" + }, + { + "key": "libtinfo6_6.3-2ubuntu0.1_amd64", + "name": "libtinfo6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", + "name": "libsqlite3-0", + "version": "3.37.2-2ubuntu0.4" + }, + { + "key": "libreadline8_8.1.2-1_amd64", + "name": "libreadline8", + "version": "8.1.2-1" + }, + { + "key": "readline-common_8.1.2-1_amd64", + "name": "readline-common", + "version": "8.1.2-1" + }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, + { + "key": "libncursesw6_6.3-2ubuntu0.1_amd64", + "name": "libncursesw6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libmpdec3_2.5.1-2build2_amd64", + "name": "libmpdec3", + "version": "2.5.1-2build2" + }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libffi8_3.4.2-4_amd64", + "name": "libffi8", + "version": "3.4.2-4" + }, + { + "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", + "name": "libdb5.3", + "version": "5.3.28+dfsg1-0.8ubuntu3" + }, + { + "key": "media-types_7.0.0_amd64", + "name": "media-types", + "version": "7.0.0" + }, + { + "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10_3.10.12-1_22.04.10_amd64", + "name": "python3.10", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", + "name": "python3.10-minimal", + "version": "3.10.12-1~22.04.10" + }, + { + "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", + "name": "libexpat1", + "version": "2.4.7-1ubuntu0.6" + }, + { + "key": "python3-minimal_3.10.6-1_22.04.1_amd64", + "name": "python3-minimal", + "version": "3.10.6-1~22.04.1" + }, + { + "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libgcc-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", + "name": "libquadmath0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libubsan1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", + "name": "libtsan0", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", + "name": "gcc-11-base", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", + "name": "liblsan0", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", + "name": "libasan6", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libitm1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libgomp1", + "version": "12.3.0-1ubuntu1~22.04" + }, + { + "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", + "name": "libstdc++-11-dev", + "version": "11.4.0-1ubuntu1~22.04" + }, + { + "key": "libc6-dev_2.35-0ubuntu3.10_amd64", + "name": "libc6-dev", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "libnsl-dev_1.3.0-2build2_amd64", + "name": "libnsl-dev", + "version": "1.3.0-2build2" + }, + { + "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", + "name": "libtirpc-dev", + "version": "1.3.2-2ubuntu0.1" + }, + { + "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", + "name": "rpcsvc-proto", + "version": "1.4.2-0ubuntu6" + }, + { + "key": "libcrypt-dev_1-4.4.27-1_amd64", + "name": "libcrypt-dev", + "version": "1:4.4.27-1" + }, + { + "key": "linux-libc-dev_5.15.0-143.153_amd64", + "name": "linux-libc-dev", + "version": "5.15.0-143.153" + }, + { + "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", + "name": "libc-dev-bin", + "version": "2.35-0ubuntu3.10" + }, + { + "key": "hsa-rocr-dev_1.18.0.70200-43_22.04_amd64", + "name": "hsa-rocr-dev", + "version": "1.18.0.70200-43~22.04" + }, + { + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "libdrm-amdgpu-dev_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-dev", + "version": "1:2.4.125.70200-2278374.22.04" + }, + { + "key": "libdrm-amdgpu-radeon1_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-radeon1", + "version": "1:2.4.125.70200-2278374.22.04" + }, + { + "key": "libdrm2-amdgpu_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm2-amdgpu", + "version": "1:2.4.125.70200-2278374.22.04" + }, + { + "key": "amdgpu-core_1-7.2.70200-2278374.22.04_amd64", + "name": "amdgpu-core", + "version": "1:7.2.70200-2278374.22.04" + }, + { + "key": "libdrm-amdgpu-amdgpu1_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-amdgpu1", + "version": "1:2.4.125.70200-2278374.22.04" + }, + { + "key": "libdrm-amdgpu-common_1.0.0.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-common", + "version": "1.0.0.70200-2278374.22.04" + }, + { + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.2.26015.70200-43~22.04" + }, + { + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" + }, + { + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70200-43~22.04" + }, + { + "key": "comgr_3.0.0.70200-43_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70200-43~22.04" + }, + { + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70200-43~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "file_1-5.41-3ubuntu0.1_amd64", + "name": "file", + "version": "1:5.41-3ubuntu0.1" + }, + { + "key": "libmagic1_1-5.41-3ubuntu0.1_amd64", + "name": "libmagic1", + "version": "1:5.41-3ubuntu0.1" + }, + { + "key": "libmagic-mgc_1-5.41-3ubuntu0.1_amd64", + "name": "libmagic-mgc", + "version": "1:5.41-3ubuntu0.1" + } + ], + "key": "hip-dev_7.2.26015.70200-43_22.04_amd64", + "name": "hip-dev", + "sha256": "19d62d6f520b4e521fe90246d9ae872b30528e490f80c6e265f83a583302f0ad", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hip-dev/hip-dev_7.2.26015.70200-43~22.04_amd64.deb" + ], + "version": "7.2.26015.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "hipcc_1.1.1.70200-43_22.04_amd64", + "name": "hipcc", + "sha256": "3c037af2114bd9058fb1420c00586c5d1c01a63d1d34500d8e61b39802ac570b", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipcc/hipcc_1.1.1.70200-43~22.04_amd64.deb" + ], + "version": "1.1.1.70200-43~22.04" + }, + { + "arch": "amd64", + "dependencies": [], + "key": "rocm-llvm_22.0.0.26014.70200-43_22.04_amd64", + "name": "rocm-llvm", + "sha256": "88d604fe0ab4a2502f0c7e0f8d3b7b49e829ae3057971de681a68b2fdb2505d2", + "urls": [ + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocm-llvm/rocm-llvm_22.0.0.26014.70200-43~22.04_amd64.deb" + ], + "version": "22.0.0.26014.70200-43~22.04" }, { "arch": "amd64", @@ -1569,6 +4829,17 @@ ], "version": "11.4.0-1ubuntu1~22.04" }, + { + "arch": "amd64", + "dependencies": [], + "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", + "name": "libatomic1", + "sha256": "56ec991983c7c89a2c15bd5134534531d722ec5b51a60679f7ceaa1a09b37a09", + "urls": [ + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/gcc-12/libatomic1_12.3.0-1ubuntu1~22.04_amd64.deb" + ], + "version": "12.3.0-1ubuntu1~22.04" + }, { "arch": "amd64", "dependencies": [], @@ -1605,2065 +4876,76 @@ { "arch": "amd64", "dependencies": [], - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "sha256": "c2e3b50af761886617462ca47ebc9303a83968129acca2add42b8eba0a7abca3", + "key": "hsa-rocr-dev_1.18.0.70200-43_22.04_amd64", + "name": "hsa-rocr-dev", + "sha256": "71233ffaac8f28550c35bcf3e287a377a3c5c938a1e815e65d4155139cf4bdff", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc6-dev_2.35-0ubuntu3.10_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hsa-rocr-dev/hsa-rocr-dev_1.18.0.70200-43~22.04_amd64.deb" ], - "version": "2.35-0ubuntu3.10" + "version": "1.18.0.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "sha256": "e93aa448fe708a4bbfcc40b2a8cab6f5f88896174a301caab2719f734e08a621", + "key": "libdrm-amdgpu-dev_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-dev", + "sha256": "fec583602169ae32d3c9313dcb24c00732911234712495cbb6bb10f2194d2aee", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libn/libnsl/libnsl-dev_1.3.0-2build2_amd64.deb" + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-dev_2.4.125.70200-2278374.22.04_amd64.deb" ], - "version": "1.3.0-2build2" + "version": "1:2.4.125.70200-2278374.22.04" }, { "arch": "amd64", "dependencies": [], - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "sha256": "f4254aff5a32ef3c9fca06d3f4f2eff71c93b9393cc1ecc718f7478a5d6a83da", + "key": "libdrm-amdgpu-radeon1_1-2.4.125.70200-2278374.22.04_amd64", + "name": "libdrm-amdgpu-radeon1", + "sha256": "70049c919cff1b728107a03cd893b69106de8c00ac344e2d4019922d6bdbcd8b", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libt/libtirpc/libtirpc-dev_1.3.2-2ubuntu0.1_amd64.deb" + "https://repo.radeon.com/graphics/7.2/ubuntu/pool/main/libd/libdrm-amdgpu/libdrm-amdgpu-radeon1_2.4.125.70200-2278374.22.04_amd64.deb" ], - "version": "1.3.2-2ubuntu0.1" + "version": "1:2.4.125.70200-2278374.22.04" }, { "arch": "amd64", "dependencies": [], - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "sha256": "447c6d8f7624a3d728a12232440d3f10e6e53bbf60cc202b7beb8a3bc9f7acc9", + "key": "file_1-5.41-3ubuntu0.1_amd64", + "name": "file", + "sha256": "b63b48e8b32c6f0fce5ad37d5ec3269cbbdc8feed13170aae1f416a43ebe898b", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/r/rpcsvc-proto/rpcsvc-proto_1.4.2-0ubuntu6_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/f/file/file_5.41-3ubuntu0.1_amd64.deb" ], - "version": "1.4.2-0ubuntu6" + "version": "1:5.41-3ubuntu0.1" }, { "arch": "amd64", "dependencies": [], - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "sha256": "a79be2f6e45823dcc09e04d5e98c88ec88d07d5b8895d05b875a8ade8b345efa", + "key": "libmagic1_1-5.41-3ubuntu0.1_amd64", + "name": "libmagic1", + "sha256": "344d0cc65fea4175ca1ab222c20b5b95fb1357acafea6b016add9988760829d0", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/libx/libxcrypt/libcrypt-dev_4.4.27-1_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/f/file/libmagic1_5.41-3ubuntu0.1_amd64.deb" ], - "version": "1:4.4.27-1" + "version": "1:5.41-3ubuntu0.1" }, { "arch": "amd64", "dependencies": [], - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "sha256": "f534bb052e96efb9f9e08279b18950d1b1f2a7958f3a06c426fed7d2c1500840", + "key": "libmagic-mgc_1-5.41-3ubuntu0.1_amd64", + "name": "libmagic-mgc", + "sha256": "23fcefb8b9ee73eb74e127fc230330918a8f81e127fa54015f191cfd34726b3c", "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/l/linux/linux-libc-dev_5.15.0-143.153_amd64.deb" + "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/f/file/libmagic-mgc_5.41-3ubuntu0.1_amd64.deb" ], - "version": "5.15.0-143.153" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "sha256": "d93f29bc5ad18f69a4fe292fd392dc17e0a9d9bf0a22ee11fcf43e6395b6ecf8", - "urls": [ - "https://snapshot.ubuntu.com/ubuntu/20250711T030400Z/pool/main/g/glibc/libc-dev-bin_2.35-0ubuntu3.10_amd64.deb" - ], - "version": "2.35-0ubuntu3.10" + "version": "1:5.41-3ubuntu0.1" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", - "name": "amdrocm-sparse-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-llvm10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "amdrocm-base10.1_10.1.0-3_amd64", - "name": "amdrocm-base10.1", - "version": "10.1.0-3" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", - "name": "amdrocm-profiler-base10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", - "name": "amdrocm-solver-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-math-common10.1_10.1.0-3_amd64", - "name": "amdrocm-math-common10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", - "name": "amdrocm-blas-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1010", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1010", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1010", - "version": "10.1.0-3" - } - ], - "key": "amdrocm-blas10.1_10.1.0-3_amd64", - "name": "amdrocm-blas10.1", - "sha256": "2a8a63f7c3e5b5e11974433c9cb6abdd297c9939c07cd061f83e0a8d3867e785", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx950", - "sha256": "ba944cc7b634d098344315357b126a6ab7fff39ab34970f798d20a3e8b1177d2", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx950_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx950", - "sha256": "089bf10ade33b38698d8eb02af7cd8ec9768ad1946afd2e76b18175d7dbbaa26", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx950_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx950", - "sha256": "b0714db7f27322097d7d1940e9c95a431d9ec11ff7ea93081ca549f8b72a379c", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx950_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", - "name": "amdrocm-sparse-host10.1", - "sha256": "44faf2440577e65c51b9b6474c1b70a96503a44c7939e6b8ec75512ab928f08e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse-host10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-llvm10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm10.1", - "version": "10.1.0-3" - }, - { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "amdrocm-base10.1_10.1.0-3_amd64", - "name": "amdrocm-base10.1", - "version": "10.1.0-3" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - } - ], - "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", - "name": "amdrocm-profiler-base10.1", - "sha256": "5f027faf16f293f66fe42cd794ec6b984377183417f49c49c8c2daf3a5675d23", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-profiler-base10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", - "name": "amdrocm-solver-host10.1", - "sha256": "1ec2e8677941eadcb4b87f48c5cfae18511b1fc2ac08794d61740843eabba022", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver-host10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-math-common10.1_10.1.0-3_amd64", - "name": "amdrocm-math-common10.1", - "sha256": "1e4b3e1fc77bc8431e3f5bf81b9193376c9025dd22b6c8a28f87d448cd3d7eaf", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-math-common10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", - "name": "amdrocm-blas-host10.1", - "sha256": "9c35dc27bc27b797f80d42d01cf3d8e042c5f6cc1db66f1a54180d5f21ccf3a3", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas-host10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx942", - "sha256": "f988f42ba63f83ca7e65086a5439ac693a0c76c5a963e871b1d8440c7e892c34", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx942_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx942", - "sha256": "c7964290972e15f3db7d73a91c460332ecbb19fa1c187cbdf5836c73804c2748", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx942_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx942", - "sha256": "0dadc1402ca074a627918c52680d77dfe99b22abdad643ad36fdfb964ef3fa16", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx942_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx90a", - "sha256": "a0799f0aad9e9378f245288ce5ae653ff1b4f716962fe0f8d6eb160820fb9870", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx90a_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx90a", - "sha256": "f329191044bc2aee16a5eb33ab989f393dfbef42c4705542e4d5a0112e406b8d", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx90a_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx90a", - "sha256": "200cae46a18b621385e50b191f9acbacceea04b8c7757148442661c65ea58c40", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx908", - "sha256": "da0bf3ce9c571e9c2a60cc1964e287ff0c2496ebb156003962212650c8a3a42c", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx908_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx908", - "sha256": "0c7e23e418f4892d6d341d9c89f774477d52e105595204801272f2834be1f40b", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx908_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx908", - "sha256": "54d6ddf65e98722e9e5c0b39909d7ccd689ec273c6d7020f6095ba7274404152", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx908_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1250", - "sha256": "44db7d9e96ecddd26bd8b069a0a5a89610276c8db859ca7cad9f619f0bf0dd60", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1250_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1250", - "sha256": "7e76b6e1b314d87b55c268f5b6f0b9e300f571eb34de551524bc6302b9ab276b", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1250_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1250", - "sha256": "d555ef3296a63874af1fa99fc7f8f48e57d932abce3c57ec16b2d4cb24ee027e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1201", - "sha256": "1906724e7d338bcbf8013e6f02a96618120d39d5d7c8f3b5a40da185be236bc2", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1201_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1201", - "sha256": "43214b4df226aac08647646f71c13dec6cd3be561f9278f0a9fa36b3a5827337", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1201_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1201", - "sha256": "4612811ba76d77f1faa19b1f4da9db829a473253058aa2b170a2ee555de26a15", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1200", - "sha256": "b4df7e69d7c5c0e6e0ebe262411574153cca723c1ef4e9d445dbb539e1724a16", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1200_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1200", - "sha256": "757a0c725ebcf7701ced94331d1cb4d615aef9e34849c96a84c3aa9dc0e4a754", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1200_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1200", - "sha256": "740dd6b691abe24651412c7d5eac34f10a3878eec939759b5dc39f720d530084", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1153", - "sha256": "dfaec5607aad9e14282c27c1d1f7440cedbde1d147d2e42c11aada69cfb5c30d", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1153_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1153", - "sha256": "b9cfe52bea768867d858a4b0db6de109b95455c1470528cfb51ba5ef4a62f58d", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1153_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1153", - "sha256": "fac6655f10a9e857c844e0f0c9941213ad543c375335272e97ca9a05c3adf64a", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1152", - "sha256": "3cb059af8f1ce850cf6fabe3182a1904ada1f2460f638476ad2fd57c43423988", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1152_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1152", - "sha256": "93cccffec257c00cbc45e0970f980fcefe06b2a55893dda128bd97b1b6be29cb", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1152_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1152", - "sha256": "2b348c1379a9ffbfafea1d606d47574d6b009fb7e729eeaa586a4bcac19830b5", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1151", - "sha256": "9de48ee15f185b471fc6b0e836abea882153e7d3cced6e3157a5f091f21ea4cd", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1151_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1151", - "sha256": "d22416ee83fc82dddf4d094eebe28a1a699b09153ed4d996b7763d8686365ab4", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1151_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1151", - "sha256": "8ab4582f863ceeb2bd0f9bc607786f57d03d34515bed3fb38e4bcaa89b05e8f2", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1150", - "sha256": "7f2fa46ad609d5cf9b8ef181cf0f76e288880d8916530e33b74df33efe508d37", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1150_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1150", - "sha256": "ffd9c28d8313d64a83401614d6164e4308ade4f7eca52209206c7974f605e541", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1150_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1150", - "sha256": "81626b7367523f6bb0ba918aa284c9ddcbe59093fad6110462d76cf51dfe06a5", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1103", - "sha256": "2bda07dcfbf333c35dd9b063e8eb4436bb653afea0aa0ef23fac58ebc54ae021", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1103_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1103", - "sha256": "b20f76bd6424de48b4a3d7881bc7ee4dc7c0e060970b237a8db53297b46c2acc", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1103_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1103", - "sha256": "1089660ba5ab633ac0c6f7acc0da10e39926713e71c581f3e27bb7add04cfa59", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1102", - "sha256": "626a668f339768a2842a9679a417009ad5c885e9a727e0e34ff7386ab9262f71", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1102_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1102", - "sha256": "33d78823a628a6b0bf77f4a36cf977c1d1c816f1aa7e1b55bd2af86370bd3f6c", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1102_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1102", - "sha256": "fcabe97aadc69b9a6f4d006191d0e4fb77df22b734b688763b2c437071c9e11a", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1101", - "sha256": "736385257ca8d3680f49500667fd7bc3fe483e249e80b550da6386e17ac92cb8", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1101_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1101", - "sha256": "49610fda5782cbf81a0ade0ecea1e4cda4a23ea5f619bbeb19954fd3159bc511", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1101_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1101", - "sha256": "28470e954ec30f8a6cbfff23b85fcdbc0c486b839f904d61a2c922e3157349d0", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1100", - "sha256": "7998acee523f9ac9e2ce6c36514c8343fb9a651629275f0354404e47e080f076", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1100_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1100", - "sha256": "71d7fdbe3619d77d01db2f7895f73330365ca3e1fb792ccc80f8accfb067db7b", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1100_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1100", - "sha256": "15f0a8421d49acdc19365063b7ce4d3b85102bbfbb517e83bc888e4107061694", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1036", - "sha256": "dcf5d14b3781669526b279a40866068d60e4d119aa5b628f38fd69e065fba450", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1036_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1036", - "sha256": "7e65239be07b34e784e22f1fd98aa1b02a89a713aad05c3640ee88b08cf38a7d", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1036_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1036", - "sha256": "028a6598a0f90384b08bdfb0ee42ca2031206e8bf5b65730eeef2a77c201cf87", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1035", - "sha256": "5e0397e891522100a5432d433039e8877479b5111f7226401ed185700a56d2ef", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1035_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1035", - "sha256": "2763f459121dcca48edc4997dc861763cb3c237d102a0cf771209f5d4e0865db", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1035_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1035", - "sha256": "2aed6082214e2ef53cc472d169128238a64ec7ec9300291759a5b6416e173d20", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1034", - "sha256": "370b7644ad5031bde2dcbb0fdc32441a741869d44d953ff315abe7ba263dbb72", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1034_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1034", - "sha256": "aa24693000c9b848f66135dde49722d19673ef1d3385736b91b5ebe298509b28", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1034_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1034", - "sha256": "0ab0d5238ba6a8d2c0b46e57ace593316c10d93759aa1328937e170c348d1a1f", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1033", - "sha256": "cf3d658fb0a76fbead063bad0c9c892d7624f9b3c40e41de67cbe49169dfae58", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1033_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1033", - "sha256": "208976c7e784551983664f24330c8ba9734b4bb505e25ae6dac233c98a711052", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1033_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1033", - "sha256": "ced0d2f76cf3113b6bf7246ce1ddd90c638293a61f0c78df1b8099f3cc89a5d7", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1032", - "sha256": "03aa4d150527f6fb4ab0c84ed650e8ce63a132935ec50641e595693d378529da", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1032_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1032", - "sha256": "9a79590868ea114964ce9ca634b73213600f7800c0cd2d5b26f3e964bf496d28", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1032_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1032", - "sha256": "4182454edef4240bbc911ec01f00deae83e71a8ff878fbcceff70db3d52eaefe", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1031", - "sha256": "2553208d7552421a7e0100334a0135320dfd07d70746a04d6a75ac49c7c5514d", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1031_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1031", - "sha256": "dc2453d80d4e7690f918ff3dbe9eff6663b1f837aaae3975144282126d7b1527", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1031_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1031", - "sha256": "3f0bb24ff77b733eee320eba671a0b169b1527fc8bf2dfae6c0b4cc5c363ec3f", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1030", - "sha256": "4b56e03309b8339d2584a7af18609b0f5f8d041dbf08653c1a32f4a36fa9e73b", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1030_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1030", - "sha256": "8c3802bb51b012a36a9012d86c020694f26ebea07492f9bdb3189248b87ec2d1", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1030_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1030", - "sha256": "e8c255c31835bcf6a5193f9ff8ed91b6a94369097ed8c364168dd71d2c163e6e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1012", - "sha256": "95c5aa73aaa87b4fb944d5cd05088b1ce500d71f012d3ee1eae76d7db105b906", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1012_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1012", - "sha256": "11afcd2eebbb54385d0334e126a2ff63b90ccf820b04ed0f81ae4042273bfd66", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1012_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1012", - "sha256": "ade96cb7aa4f0a16f23920eafb683686efabee5e5aaaac023d5b11434037d181", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1011", - "sha256": "7142eefd24ec5363036bb03267e49caf7af820a8f275df4ae63afcb53471cee6", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1011_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1011", - "sha256": "b689b157fd60e30ba111dbea21077086020e0e1eda146995e83b5b44cd117b43", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1011_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1011", - "sha256": "4a4e94348c97148c7332ceb32073cbf330623f507a6022ae398ccfeb6ddc2878", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1010", - "sha256": "052bdaa575e9a47e59ec21d492b5288ee289999d0121c7d0b82484f4a4f3c19f", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas10.1-gfx1010_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1010", - "sha256": "969e5b48158bb841dbdc3db2388e7b13d58016649104b138e2ce307a087f4afa", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1-gfx1010_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1010", - "sha256": "905dca5b90e3e301512e422eb0d5ae5631e341cb4b8dab4f5f2324ec2947a33f", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64", - "name": "amdrocm-hipblas-common-dev10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-runtime-dev10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime-dev10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm-dev10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-llvm10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-base10.1_10.1.0-3_amd64", - "name": "amdrocm-base10.1", - "version": "10.1.0-3" + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -3886,64 +5168,44 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" + "key": "rocblas_5.2.0.70200-43_22.04_amd64", + "name": "rocblas", + "version": "5.2.0.70200-43~22.04" }, { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.2.26015.70200-43~22.04" }, { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" }, { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70200-43~22.04" }, { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" + "key": "comgr_3.0.0.70200-43_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70200-43~22.04" }, { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" }, { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" }, { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" }, { "key": "libc6-dev_2.35-0ubuntu3.10_amd64", @@ -3979,49 +5241,78 @@ "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", "name": "libc-dev-bin", "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70200-43~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", + "name": "hipblaslt", + "version": "1.2.1.70200-43~22.04" + }, + { + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "name": "roctracer", + "version": "4.1.70200.70200-43~22.04" } ], - "key": "amdrocm-blas-dev10.1_10.1.0-3_amd64", - "name": "amdrocm-blas-dev10.1", - "sha256": "c7b6b25d7f1143742a7b7a2bc6819cab56a166995b60ebdc5b282d5ddc67770f", + "key": "rocsolver_3.32.0.70200-43_22.04_amd64", + "name": "rocsolver", + "sha256": "f18b57d3ebeba075b2b91f9e71d92da3efba73f34c5cff697cea1259b829547a", "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-blas-dev10.1_10.1.0-3_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocsolver/rocsolver_3.32.0.70200-43~22.04_amd64.deb" ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64", - "name": "amdrocm-hipblas-common-dev10.1", - "sha256": "e8456d1a340533ae73e5a7cbe9aa66feea1f8cedef72971f00930167b80037a3", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-hipblas-common-dev10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" + "version": "3.32.0.70200-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm-dev10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-llvm10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-base10.1_10.1.0-3_amd64", - "name": "amdrocm-base10.1", - "version": "10.1.0-3" + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4244,64 +5535,49 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" + "key": "rocsolver_3.32.0.70200-43_22.04_amd64", + "name": "rocsolver", + "version": "3.32.0.70200-43~22.04" }, { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" + "key": "rocblas_5.2.0.70200-43_22.04_amd64", + "name": "rocblas", + "version": "5.2.0.70200-43~22.04" }, { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" + "key": "hip-runtime-amd_7.2.26015.70200-43_22.04_amd64", + "name": "hip-runtime-amd", + "version": "7.2.26015.70200-43~22.04" }, { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" + "key": "libnuma1_2.0.14-3ubuntu2_amd64", + "name": "libnuma1", + "version": "2.0.14-3ubuntu2" }, { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" + "key": "rocprofiler-register_0.6.0.70200-43_22.04_amd64", + "name": "rocprofiler-register", + "version": "0.6.0.70200-43~22.04" }, { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" + "key": "comgr_3.0.0.70200-43_22.04_amd64", + "name": "comgr", + "version": "3.0.0.70200-43~22.04" }, { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" + "key": "libtinfo-dev_6.3-2ubuntu0.1_amd64", + "name": "libtinfo-dev", + "version": "6.3-2ubuntu0.1" }, { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" + "key": "libncurses-dev_6.3-2ubuntu0.1_amd64", + "name": "libncurses-dev", + "version": "6.3-2ubuntu0.1" }, { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" + "key": "ncurses-bin_6.3-2ubuntu0.1_amd64", + "name": "ncurses-bin", + "version": "6.3-2ubuntu0.1" }, { "key": "libc6-dev_2.35-0ubuntu3.10_amd64", @@ -4337,69 +5613,78 @@ "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", "name": "libc-dev-bin", "version": "2.35-0ubuntu3.10" + }, + { + "key": "libncurses6_6.3-2ubuntu0.1_amd64", + "name": "libncurses6", + "version": "6.3-2ubuntu0.1" + }, + { + "key": "rocminfo_1.0.0.70200-43_22.04_amd64", + "name": "rocminfo", + "version": "1.0.0.70200-43~22.04" + }, + { + "key": "pciutils_1-3.7.0-6_amd64", + "name": "pciutils", + "version": "1:3.7.0-6" + }, + { + "key": "libpci3_1-3.7.0-6_amd64", + "name": "libpci3", + "version": "1:3.7.0-6" + }, + { + "key": "pci.ids_0.0_2022.01.22-1ubuntu0.1_amd64", + "name": "pci.ids", + "version": "0.0~2022.01.22-1ubuntu0.1" + }, + { + "key": "libudev1_249.11-0ubuntu3.16_amd64", + "name": "libudev1", + "version": "249.11-0ubuntu3.16" + }, + { + "key": "libkmod2_29-1ubuntu1_amd64", + "name": "libkmod2", + "version": "29-1ubuntu1" + }, + { + "key": "hsa-runtime-rocr4wsl-amdgpu_25.30.13-2281980.22.04_amd64", + "name": "hsa-runtime-rocr4wsl-amdgpu", + "version": "25.30.13-2281980.22.04" + }, + { + "key": "libelf1_0.186-1ubuntu0.1_amd64", + "name": "libelf1", + "version": "0.186-1ubuntu0.1" + }, + { + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", + "name": "hipblaslt", + "version": "1.2.1.70200-43~22.04" + }, + { + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "name": "roctracer", + "version": "4.1.70200.70200-43~22.04" } ], - "key": "amdrocm-runtime-dev10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime-dev10.1", - "sha256": "452b3a3b31a4fa4da8d4c08976a5698aa05732b52ab88daa85cb4b4d2c84754c", + "key": "hipsolver_3.2.0.70200-43_22.04_amd64", + "name": "hipsolver", + "sha256": "0fbff8007eada3d952db641c0653df26854854ee9e7058b4bd63202db66f2af8", "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-runtime-dev10.1_10.1.0-3_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipsolver/hipsolver_3.2.0.70200-43~22.04_amd64.deb" ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-llvm-dev10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm-dev10.1", - "sha256": "351db702fd37be1fda7e5221aedf19484e99d27f2a3e7d2df5f0f531bcebcfd5", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-llvm-dev10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" + "version": "3.2.0.70200-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-dnn10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", - "name": "amdrocm-sparse-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-llvm10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -4422,79 +5707,24 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" }, { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" }, { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" }, { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -4596,56 +5826,6 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "amdrocm-base10.1_10.1.0-3_amd64", - "name": "amdrocm-base10.1", - "version": "10.1.0-3" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, { "key": "libtinfo6_6.3-2ubuntu0.1_amd64", "name": "libtinfo6", @@ -4666,6 +5846,11 @@ "name": "readline-common", "version": "8.1.2-1" }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, { "key": "libncursesw6_6.3-2ubuntu0.1_amd64", "name": "libncursesw6", @@ -4676,6 +5861,11 @@ "name": "libmpdec3", "version": "2.5.1-2build2" }, + { + "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", + "name": "libstdc++6", + "version": "12.3.0-1ubuntu1~22.04" + }, { "key": "libffi8_3.4.2-4_amd64", "name": "libffi8", @@ -4717,822 +5907,42 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", - "name": "amdrocm-profiler-base10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", - "name": "amdrocm-solver-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-math-common10.1_10.1.0-3_amd64", - "name": "amdrocm-math-common10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", - "name": "amdrocm-blas-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn-host10.1_10.1.0-3_amd64", - "name": "amdrocm-dnn-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1010", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1010", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1010", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1010", - "version": "10.1.0-3" + "key": "rocfft_1.0.36.70200-43_22.04_amd64", + "name": "rocfft", + "version": "1.0.36.70200-43~22.04" } ], - "key": "amdrocm-dnn10.1_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1", - "sha256": "2a65817ed4bd283ff908a6789c1beaa95e1232043b9c0261edb4f7b031e1880e", + "key": "hipfft_1.0.22.70200-43_22.04_amd64", + "name": "hipfft", + "sha256": "55a791b56dbeadcc3a1f1d0d770afc71a7fbb06aed4cd190b75ee27c19c18c61", "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1_10.1.0-3_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipfft/hipfft_1.0.22.70200-43~22.04_amd64.deb" ], - "version": "10.1.0-3" + "version": "1.0.22.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-dnn10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx950", - "sha256": "b945f4d2f35c53226a43ef798c036e0ea23eb4844c12a9f917fb450a41e756d4", + "key": "rocfft_1.0.36.70200-43_22.04_amd64", + "name": "rocfft", + "sha256": "8dbfef90c0055ef0d572451cb4a7a3380c3394485ed33cab4c116283d2fea7fe", "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx950_10.1.0-3_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/r/rocfft/rocfft_1.0.36.70200-43~22.04_amd64.deb" ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn-host10.1_10.1.0-3_amd64", - "name": "amdrocm-dnn-host10.1", - "sha256": "199c66da1e43484d21dfe1581bcf84deec3ee82d106605058b932d8c3dc86359", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn-host10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx942", - "sha256": "78e43e7ecf4a52b895fc9500e778abc0d4882a7ad5f60caff8830bb2afd64119", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx942_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx90a", - "sha256": "6fa5ee1edecb61cc9434659cfece977c636da00b63d9886646848b876a60cba6", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx90a_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx908", - "sha256": "349a3f6c0c9cf7981d5cc2476be98e0893a5b54033a541660e084a3011efba17", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx908_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1250", - "sha256": "bffbfd01dd43330cdc2a6957b4100c64983d9e94e0fecda43dd799ce6b545534", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1250_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1201", - "sha256": "d53c6f605df4faa6cfbc4ea9a3f3e75549dbec3400ab899ab502bb8d68681d3f", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1201_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1200", - "sha256": "e8e27e5fe95a8092af0c7b287e1c8b2ebb4a4099677cb274a56d09d04f6effd6", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1200_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1153", - "sha256": "39609097b14134dfce5937940c0d6118f089f8963f41b88aaf82c6dbe36d8d88", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1153_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1152", - "sha256": "f9383ce30973bca37538eab93173f7a6f277189dc1f2c32285902284524b9f6c", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1152_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1151", - "sha256": "daf6239c37012e979b4b31612cc27c1f6b5df16c4d0a3eb9ffdb7149f6c0c88e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1151_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1150", - "sha256": "57370823faa258ee6c28650dbaa1afe4a983f0d431c0a4f59b45c52bfd5aa776", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1150_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1103", - "sha256": "31103c9e996fb78bb9fbee801b67401fb757bf0f909d35d22ee3c555f6f2419c", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1103_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1102", - "sha256": "ba17cd8597c96637216596edc2146297bf3606e32e9bac7dce06147a8d171a06", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1102_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1101", - "sha256": "25844db830121a1bb8cb3e64e1d04eb8fbda654a14c7e08b96eafce54891c789", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1101_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1100", - "sha256": "7b742fc6696895809f20edaf373a22cd50fd19f17086c56d6cb144d025b32712", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1100_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1036", - "sha256": "46658d6cf72bfc08aae94e13c7d7c34209714e700dd033b2c109203addc158b1", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1036_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1035", - "sha256": "9d2165f2a2771936204ab97105a427c3b489ddc6984cebf8d3786822fbd566e9", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1035_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1034", - "sha256": "d3c75ffa1b9f79165bfcad631d009149742780114e132b3cd3e4912eb0f23632", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1034_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1033", - "sha256": "ebb48929bd2faf317aaa457096d4cefd6a9a2f8b9cbe2492bffb2e52135bdea3", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1033_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1032", - "sha256": "ee57ec63c768f031899bb630b42ca780bd753ce5c2543d04aea25bd86a04bf33", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1032_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1031", - "sha256": "6ce6789fcbaaac92404402adeec89e1170ccd04cccc7340240338f506f3e0e27", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1031_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1030", - "sha256": "e33aeb4b4acd112e37a278e006fe214a19927451e63d8bfd8db7cf4fb04a03a6", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1030_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1012", - "sha256": "21723b94ee22e0cc5caf839dab16c431e0b6b7f91d9a06cb22e58cc486ae7a5f", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1012_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1011", - "sha256": "4b1722d77fc4e0509802a1c56466cc72a8e208ade66f83b92c32cb54ec9c357e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1011_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-dnn10.1-gfx1010", - "sha256": "c9e57b5b1c75dcbf41dc69310adb3cdac7ef807e3119dd51a0357f2a9429f97a", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-dnn10.1-gfx1010_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" + "version": "1.0.36.70200-43~22.04" }, { "arch": "amd64", "dependencies": [ { - "key": "amdrocm-fft10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx950", - "version": "10.1.0-3" + "key": "hipblaslt_1.2.1.70200-43_22.04_amd64", + "name": "hipblaslt", + "version": "1.2.1.70200-43~22.04" }, { - "key": "amdrocm-fft-host10.1_10.1.0-3_amd64", - "name": "amdrocm-fft-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" + "key": "rocm-core_7.2.0.70200-43_22.04_amd64", + "name": "rocm-core", + "version": "7.2.0.70200-43~22.04" }, { "key": "libc6_2.35-0ubuntu3.10_amd64", @@ -5555,89 +5965,24 @@ "version": "12.3.0-1ubuntu1~22.04" }, { - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "version": "10.1.0-3" + "key": "python3_3.10.6-1_22.04.1_amd64", + "name": "python3", + "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-llvm10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm10.1", - "version": "10.1.0-3" + "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", + "name": "libpython3-stdlib", + "version": "3.10.6-1~22.04.1" }, { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" + "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", + "name": "libpython3.10-stdlib", + "version": "3.10.12-1~22.04.10" }, { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" + "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", + "name": "libuuid1", + "version": "2.37.2-4ubuntu3.4" }, { "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", @@ -5739,56 +6084,6 @@ "name": "libcom-err2", "version": "1.46.5-2ubuntu1.2" }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "amdrocm-base10.1_10.1.0-3_amd64", - "name": "amdrocm-base10.1", - "version": "10.1.0-3" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, { "key": "libtinfo6_6.3-2ubuntu0.1_amd64", "name": "libtinfo6", @@ -5809,6 +6104,11 @@ "name": "readline-common", "version": "8.1.2-1" }, + { + "key": "libnsl2_1.3.0-2build2_amd64", + "name": "libnsl2", + "version": "1.3.0-2build2" + }, { "key": "libncursesw6_6.3-2ubuntu0.1_amd64", "name": "libncursesw6", @@ -5819,769 +6119,11 @@ "name": "libmpdec3", "version": "2.5.1-2build2" }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "amdrocm-fft10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-fft10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1010", - "version": "10.1.0-3" - } - ], - "key": "amdrocm-fft10.1_10.1.0-3_amd64", - "name": "amdrocm-fft10.1", - "sha256": "0246b0530ab01388018cafba38ee64736933c3b5eeea0e21dcd9dafb44a5950b", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx950", - "sha256": "3be1ec8ef93a0d7c4d3c463fa92a0ab0f7422f09e5c278d486e42424897704c8", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx950_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft-host10.1_10.1.0-3_amd64", - "name": "amdrocm-fft-host10.1", - "sha256": "5eb075b4dd89ea9d500921959be55a469c60675f53c89e846e186da1d353c27e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft-host10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx942", - "sha256": "415771e4caefb3af0e91264a6ac232ab0a1401aa68d0e9c9b836cc177c609ac2", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx942_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx90a", - "sha256": "c626c38512d954de19bcc298cfde38e419d4fc21d57665c10d92523c18419265", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx90a_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx908", - "sha256": "3310f7fe026228c270df2b9b8100c2a64ef7aaae783a54e55ca5099fe57a718b", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx908_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1250", - "sha256": "7b8cb472a25096913a4329f5dfc2e124e98621764aff2e04b79db5d7f6ce3ea8", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1250_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1201", - "sha256": "eb10f46c03ffdbc1be901e2fefc16fcd3dcd36079acc17bf2a991dda010ca22e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1201_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1200", - "sha256": "04f88bce83b753730025bc352c61b16027c2a6d565f85763ce8990faf155231f", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1200_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1153", - "sha256": "cd7270fd487b7abea0498a832122c11186ac234981822274af84520b8980c0c3", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1153_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1152", - "sha256": "67250ae45a6190b8e0f0bea35a8019a8fd5a5da070432206a32c4ae3388f2e0e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1152_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1151", - "sha256": "f70434a361d0207188a0feff30d677164c1fdcdb0af970acbc55561b3bfc42f5", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1151_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1150", - "sha256": "26b23859562047c84e8eb4ff7387bcab69b475eeb8655e577b0c95e436e6c2f4", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1150_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1103", - "sha256": "0cd8464f08f8b6b08ed10773352a34d1d1efea4de3c039a36e6f3bb4fdfb0f9d", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1103_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1102", - "sha256": "ec4b9439df683ee782241ec4551daa4e0ec4e4ae4a8c2b92f1bd41285cad596e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1102_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1101", - "sha256": "2ed4b2fbabb310fdb7fdf8ed01c9ca1f0f37fe97b5e3027451737be4815492a8", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1101_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1100", - "sha256": "df323baa9cfb3a2c2dd81bd81510021db4780eb074717b22abb9dbd80d992dc0", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1100_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1036", - "sha256": "6eb45f4a815e10d2119370e0ed68fe44bc34c2999c3b0a799d15d23d96f17ed8", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1036_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1035", - "sha256": "1da409970e160027c8e5fbf9d446394fa9108b92163ba69796b7d7f598f65d18", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1035_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1034", - "sha256": "58351fe6a54cc374a3d94a61416c7c4079303e2b38e974d54381bd4f5b53f83e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1034_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1033", - "sha256": "98bb8653582f9840ba723b0344204be8f6c55c758e756acec6ee372e9233119d", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1033_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1032", - "sha256": "e4b0fc5dab2eaac470a54811d19bef3b9c1f903e94d5dfa56a4f3f05b8d5977d", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1032_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1031", - "sha256": "998eae13396245c496da21504fff6832654b50323cc40cb694060a79f943a51c", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1031_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1030", - "sha256": "e321ced70752e27d4c8b70eaf3b8590b9ce6019d64896cb5cfca75d4ad26bf89", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1030_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1012", - "sha256": "b8910ca5eea38a9b9e68d3752637e214ded07e84b4ac83da39f15157cc56bf64", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1012_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1011", - "sha256": "f6cc9a8657250ba79d739ca65bc8cb13491629695b9c486060652ea9fb508656", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1011_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-fft10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-fft10.1-gfx1010", - "sha256": "378078f09766726732a4f192397b6f1d9fee346b68efd519b58238f9c41f895a", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-fft10.1-gfx1010_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "amdrocm-rccl10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl-host10.1_10.1.0-3_amd64", - "name": "amdrocm-rccl-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", - "name": "amdrocm-profiler-base10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-llvm10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm10.1", - "version": "10.1.0-3" - }, - { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" - }, { "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", "name": "libstdc++6", "version": "12.3.0-1ubuntu1~22.04" }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "amdrocm-base10.1_10.1.0-3_amd64", - "name": "amdrocm-base10.1", - "version": "10.1.0-3" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, { "key": "libffi8_3.4.2-4_amd64", "name": "libffi8", @@ -6623,1844 +6165,34 @@ "version": "3.10.6-1~22.04.1" }, { - "key": "amdrocm-amdsmi10.1_10.1.0-3_amd64", - "name": "amdrocm-amdsmi10.1", - "version": "10.1.0-3" + "key": "roctracer_4.1.70200.70200-43_22.04_amd64", + "name": "roctracer", + "version": "4.1.70200.70200-43~22.04" }, { - "key": "amdrocm-rccl10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1010", - "version": "10.1.0-3" + "key": "hipblas-common-dev_1.4.0.70200-43_22.04_amd64", + "name": "hipblas-common-dev", + "version": "1.4.0.70200-43~22.04" } ], - "key": "amdrocm-rccl10.1_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1", - "sha256": "18b5e340191a729524ede9ec9d1810f3673d035b4940743307189e1b4a96d293", + "key": "hipblaslt-dev_1.2.1.70200-43_22.04_amd64", + "name": "hipblaslt-dev", + "sha256": "7bb01f62e51be15495ea4a124db7b6c4e600d27d401ce636fb9b3daaf6b4b958", "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1_10.1.0-3_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblaslt-dev/hipblaslt-dev_1.2.1.70200-43~22.04_amd64.deb" ], - "version": "10.1.0-3" + "version": "1.2.1.70200-43~22.04" }, { "arch": "amd64", "dependencies": [], - "key": "amdrocm-rccl10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx950", - "sha256": "d9a77415ca9392af357d188df8e805f86c8b0f7574705133ee919b88fec66f4d", + "key": "hipblas-common-dev_1.4.0.70200-43_22.04_amd64", + "name": "hipblas-common-dev", + "sha256": "1eed050f8367b58d697db452c3928780ae408ee055694f1676326d54f4caa739", "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx950_10.1.0-3_amd64.deb" + "https://repo.radeon.com/rocm/apt/7.2/pool/main/h/hipblas-common-dev/hipblas-common-dev_1.4.0.70200-43~22.04_amd64.deb" ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl-host10.1_10.1.0-3_amd64", - "name": "amdrocm-rccl-host10.1", - "sha256": "b38172c360fc8fdf89d038edfa3cb5c3cb874804f089a4c9ba91c384be7c9707", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl-host10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx942", - "sha256": "c0c4be063b49364858d76af509cbf88e4642146489a1fcde2088f66a2e1f31a9", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx942_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx90a", - "sha256": "ffcc6bce6b2271b3c6cd1c5d7b568fe2773a0022a2ba4d24c177f7df5d6c5225", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx90a_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx908", - "sha256": "2eebb6a869970c182dd64105da18f956801ed7a048693f47f99a05a087b01f59", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx908_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1250", - "sha256": "941da91fc0f827732d0c4e0010403723643a27dd3ca2494270aa97d0a8a38f35", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1250_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1201", - "sha256": "4c589250c3c587d4865551b15f32adb46cfbb5aaa07e147eb82b3d6634cff213", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1201_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1200", - "sha256": "33086fcdf5aa8e353c90ac38effddca41cb3d8ae88272333df2c580b08826abe", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1200_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1151", - "sha256": "4d7b24fc17351d69c63028867b5bdf4b9015e563175c868a814c8ca34b7bbcce", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1151_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1102", - "sha256": "935b928d8f2901861021d3f9cb2c43953d309975c4e5803b4ac496d6bd4ee007", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1102_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1101", - "sha256": "4d418a9fb372ec15bd9e42faa02f9bf945adc5de2853cb77db54cb658286c67c", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1101_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1100", - "sha256": "ceddf293510c927d2a5a8fee30cc76fce6cd0d3cc6977422396514bedbcc597e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1100_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1036", - "sha256": "8fb0c4dfa528a82c872df49b0feeb7205e9664ce06d9760d0fa85a1892d97f79", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1036_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1035", - "sha256": "174385d65d68e5c5aa7c51f5cfcf90fae9005c37944f4515883f370760830b74", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1035_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1034", - "sha256": "c3e4727c061c15aa67b89b61539d58be182dca850df13defe1790b7b5b7ffd4d", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1034_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1033", - "sha256": "521d5e2bc7c7d8f4d941044472fd9f800d5f172d4c9a3c015b035e2454c2dba1", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1033_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1032", - "sha256": "198345dc05c10db202080744337fee857c367786a315004e662dc24340cff787", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1032_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1031", - "sha256": "91761516a93977fcc70cb771e5ad2e7f958f8147905ca86adf592e5f7d79dab5", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1031_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1030", - "sha256": "076013938bd6ef83e31d5c4f444d7fb136d0733a4edfdfccf05d8108776c3388", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1030_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1012", - "sha256": "473c3be9bf38e2f30dac5f6019a521c3aad2b96e10e28389f542c799141b8f09", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1012_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1011", - "sha256": "842068f0334f6c3d4977394c9ee5098f8a8254012206b5f66f5b2aaa0e510ce1", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1011_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [], - "key": "amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-rccl10.1-gfx1010", - "sha256": "09c782685d0d6a93e767a580dc5e35183cf9f742859953271d8abaa977a74c6e", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-rccl10.1-gfx1010_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", - "name": "amdrocm-blas-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", - "name": "amdrocm-profiler-base10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-llvm10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm10.1", - "version": "10.1.0-3" - }, - { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "amdrocm-base10.1_10.1.0-3_amd64", - "name": "amdrocm-base10.1", - "version": "10.1.0-3" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", - "name": "amdrocm-sparse-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", - "name": "amdrocm-solver-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-math-common10.1_10.1.0-3_amd64", - "name": "amdrocm-math-common10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1010", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1010", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1010", - "version": "10.1.0-3" - } - ], - "key": "amdrocm-solver10.1_10.1.0-3_amd64", - "name": "amdrocm-solver10.1", - "sha256": "66f914aa374d13a700da9006af9dcd4aeae20c02a50b4807c45d3ed990da0711", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-solver10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" - }, - { - "arch": "amd64", - "dependencies": [ - { - "key": "amdrocm-sparse10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx950_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx950", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver-host10.1_10.1.0-3_amd64", - "name": "amdrocm-solver-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-math-common10.1_10.1.0-3_amd64", - "name": "amdrocm-math-common10.1", - "version": "10.1.0-3" - }, - { - "key": "libc6_2.35-0ubuntu3.10_amd64", - "name": "libc6", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libcrypt1_1-4.4.27-1_amd64", - "name": "libcrypt1", - "version": "1:4.4.27-1" - }, - { - "key": "libgcc-s1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgcc-s1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "gcc-12-base_12.3.0-1ubuntu1_22.04_amd64", - "name": "gcc-12-base", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "amdrocm-blas-host10.1_10.1.0-3_amd64", - "name": "amdrocm-blas-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-profiler-base10.1_10.1.0-3_amd64", - "name": "amdrocm-profiler-base10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sysdeps10.1_10.1.0-3_amd64", - "name": "amdrocm-sysdeps10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-runtime10.1_10.1.0-3_amd64", - "name": "amdrocm-runtime10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-llvm10.1_10.1.0-3_amd64", - "name": "amdrocm-llvm10.1", - "version": "10.1.0-3" - }, - { - "key": "libgcc-11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libgcc-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libquadmath0_12.3.0-1ubuntu1_22.04_amd64", - "name": "libquadmath0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libubsan1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libubsan1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p-6_12.3.0-1ubuntu1_22.04_amd64", - "name": "libstdc++6", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libtsan0_11.4.0-1ubuntu1_22.04_amd64", - "name": "libtsan0", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "gcc-11-base_11.4.0-1ubuntu1_22.04_amd64", - "name": "gcc-11-base", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "liblsan0_12.3.0-1ubuntu1_22.04_amd64", - "name": "liblsan0", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libasan6_11.4.0-1ubuntu1_22.04_amd64", - "name": "libasan6", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libatomic1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libatomic1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libitm1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libitm1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libgomp1_12.3.0-1ubuntu1_22.04_amd64", - "name": "libgomp1", - "version": "12.3.0-1ubuntu1~22.04" - }, - { - "key": "libstdc-p--p--11-dev_11.4.0-1ubuntu1_22.04_amd64", - "name": "libstdc++-11-dev", - "version": "11.4.0-1ubuntu1~22.04" - }, - { - "key": "libc6-dev_2.35-0ubuntu3.10_amd64", - "name": "libc6-dev", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "libnsl-dev_1.3.0-2build2_amd64", - "name": "libnsl-dev", - "version": "1.3.0-2build2" - }, - { - "key": "libtirpc-dev_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-dev", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc3_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc3", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libtirpc-common_1.3.2-2ubuntu0.1_amd64", - "name": "libtirpc-common", - "version": "1.3.2-2ubuntu0.1" - }, - { - "key": "libgssapi-krb5-2_1.19.2-2ubuntu0.7_amd64", - "name": "libgssapi-krb5-2", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5support0_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5support0", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libkrb5-3_1.19.2-2ubuntu0.7_amd64", - "name": "libkrb5-3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libssl3_3.0.2-0ubuntu1.19_amd64", - "name": "libssl3", - "version": "3.0.2-0ubuntu1.19" - }, - { - "key": "debconf_1.5.79ubuntu1_amd64", - "name": "debconf", - "version": "1.5.79ubuntu1" - }, - { - "key": "perl-base_5.34.0-3ubuntu1.4_amd64", - "name": "perl-base", - "version": "5.34.0-3ubuntu1.4" - }, - { - "key": "dpkg_1.21.1ubuntu2.3_amd64", - "name": "dpkg", - "version": "1.21.1ubuntu2.3" - }, - { - "key": "tar_1.34-p-dfsg-1ubuntu0.1.22.04.2_amd64", - "name": "tar", - "version": "1.34+dfsg-1ubuntu0.1.22.04.2" - }, - { - "key": "libselinux1_3.3-1build2_amd64", - "name": "libselinux1", - "version": "3.3-1build2" - }, - { - "key": "libpcre2-8-0_10.39-3ubuntu0.1_amd64", - "name": "libpcre2-8-0", - "version": "10.39-3ubuntu0.1" - }, - { - "key": "libacl1_2.3.1-1_amd64", - "name": "libacl1", - "version": "2.3.1-1" - }, - { - "key": "zlib1g_1-1.2.11.dfsg-2ubuntu9.2_amd64", - "name": "zlib1g", - "version": "1:1.2.11.dfsg-2ubuntu9.2" - }, - { - "key": "libzstd1_1.4.8-p-dfsg-3build1_amd64", - "name": "libzstd1", - "version": "1.4.8+dfsg-3build1" - }, - { - "key": "liblzma5_5.2.5-2ubuntu1_amd64", - "name": "liblzma5", - "version": "5.2.5-2ubuntu1" - }, - { - "key": "libbz2-1.0_1.0.8-5build1_amd64", - "name": "libbz2-1.0", - "version": "1.0.8-5build1" - }, - { - "key": "libkeyutils1_1.6.1-2ubuntu3_amd64", - "name": "libkeyutils1", - "version": "1.6.1-2ubuntu3" - }, - { - "key": "libk5crypto3_1.19.2-2ubuntu0.7_amd64", - "name": "libk5crypto3", - "version": "1.19.2-2ubuntu0.7" - }, - { - "key": "libcom-err2_1.46.5-2ubuntu1.2_amd64", - "name": "libcom-err2", - "version": "1.46.5-2ubuntu1.2" - }, - { - "key": "libnsl2_1.3.0-2build2_amd64", - "name": "libnsl2", - "version": "1.3.0-2build2" - }, - { - "key": "rpcsvc-proto_1.4.2-0ubuntu6_amd64", - "name": "rpcsvc-proto", - "version": "1.4.2-0ubuntu6" - }, - { - "key": "libcrypt-dev_1-4.4.27-1_amd64", - "name": "libcrypt-dev", - "version": "1:4.4.27-1" - }, - { - "key": "linux-libc-dev_5.15.0-143.153_amd64", - "name": "linux-libc-dev", - "version": "5.15.0-143.153" - }, - { - "key": "libc-dev-bin_2.35-0ubuntu3.10_amd64", - "name": "libc-dev-bin", - "version": "2.35-0ubuntu3.10" - }, - { - "key": "amdrocm-base10.1_10.1.0-3_amd64", - "name": "amdrocm-base10.1", - "version": "10.1.0-3" - }, - { - "key": "python3_3.10.6-1_22.04.1_amd64", - "name": "python3", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3-stdlib_3.10.6-1_22.04.1_amd64", - "name": "libpython3-stdlib", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "libpython3.10-stdlib_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-stdlib", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libuuid1_2.37.2-4ubuntu3.4_amd64", - "name": "libuuid1", - "version": "2.37.2-4ubuntu3.4" - }, - { - "key": "libtinfo6_6.3-2ubuntu0.1_amd64", - "name": "libtinfo6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libsqlite3-0_3.37.2-2ubuntu0.4_amd64", - "name": "libsqlite3-0", - "version": "3.37.2-2ubuntu0.4" - }, - { - "key": "libreadline8_8.1.2-1_amd64", - "name": "libreadline8", - "version": "8.1.2-1" - }, - { - "key": "readline-common_8.1.2-1_amd64", - "name": "readline-common", - "version": "8.1.2-1" - }, - { - "key": "libncursesw6_6.3-2ubuntu0.1_amd64", - "name": "libncursesw6", - "version": "6.3-2ubuntu0.1" - }, - { - "key": "libmpdec3_2.5.1-2build2_amd64", - "name": "libmpdec3", - "version": "2.5.1-2build2" - }, - { - "key": "libffi8_3.4.2-4_amd64", - "name": "libffi8", - "version": "3.4.2-4" - }, - { - "key": "libdb5.3_5.3.28-p-dfsg1-0.8ubuntu3_amd64", - "name": "libdb5.3", - "version": "5.3.28+dfsg1-0.8ubuntu3" - }, - { - "key": "media-types_7.0.0_amd64", - "name": "media-types", - "version": "7.0.0" - }, - { - "key": "libpython3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "libpython3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10_3.10.12-1_22.04.10_amd64", - "name": "python3.10", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "python3.10-minimal_3.10.12-1_22.04.10_amd64", - "name": "python3.10-minimal", - "version": "3.10.12-1~22.04.10" - }, - { - "key": "libexpat1_2.4.7-1ubuntu0.6_amd64", - "name": "libexpat1", - "version": "2.4.7-1ubuntu0.6" - }, - { - "key": "python3-minimal_3.10.6-1_22.04.1_amd64", - "name": "python3-minimal", - "version": "3.10.6-1~22.04.1" - }, - { - "key": "amdrocm-sparse-host10.1_10.1.0-3_amd64", - "name": "amdrocm-sparse-host10.1", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx942_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx942", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx90a_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx90a", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx908_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx908", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1250_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1250", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1201_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1201", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1200_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1200", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1153_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1153", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1152_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1152", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1151_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1151", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1150_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1150", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1103_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1103", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1102_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1102", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1101_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1101", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1100_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1100", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1036_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1036", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1035_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1035", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1034_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1034", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1033_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1033", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1032_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1032", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1031_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1031", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1030_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1030", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1012_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1012", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1011_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1011", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-sparse10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1-gfx1010", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-blas10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-blas10.1-gfx1010", - "version": "10.1.0-3" - }, - { - "key": "amdrocm-solver10.1-gfx1010_10.1.0-3_amd64", - "name": "amdrocm-solver10.1-gfx1010", - "version": "10.1.0-3" - } - ], - "key": "amdrocm-sparse10.1_10.1.0-3_amd64", - "name": "amdrocm-sparse10.1", - "sha256": "71a68bcbb89fdac799535916b602a7f456269a24e2e40d0151abd003c7ad1b58", - "urls": [ - "https://stable.repo.amd.com/rocm/core/packages/ubuntu2204/pool/main/amdrocm-sparse10.1_10.1.0-3_amd64.deb" - ], - "version": "10.1.0-3" + "version": "1.4.0.70200-43~22.04" } ], "version": 1 diff --git a/platforms/rocm/packages.yaml b/platforms/rocm/packages.yaml index c9116561..b9b96902 100644 --- a/platforms/rocm/packages.yaml +++ b/platforms/rocm/packages.yaml @@ -4,10 +4,10 @@ version: 1 sources: - - channel: llvm-toolchain-jammy-22 main - url: https://apt.llvm.org/jammy - - channel: stable main - url: https://stable.repo.amd.com/rocm/core/packages/ubuntu2204 + - channel: jammy main + url: https://repo.radeon.com/rocm/apt/7.2 + - channel: jammy main + url: https://repo.radeon.com/graphics/7.2/ubuntu - channel: jammy main url: https://snapshot.ubuntu.com/ubuntu/20250711T030400Z - channel: jammy-security main @@ -18,19 +18,25 @@ sources: archs: - "amd64" +# readelf -d libpjrt_rocm.so | grep NEEDED packages: - - "llvm-libunwind1" # LLVM 22 runtime from llvm-toolchain-jammy-22. - - "libatomic1" - - "libdrm-common" - - "amdrocm-amdsmi10.1" - - "amdrocm-base10.1" - - "amdrocm-blas10.1" - - "amdrocm-blas-dev10.1" - - "amdrocm-dnn10.1" - - "amdrocm-fft10.1" - - "amdrocm-llvm10.1" - - "amdrocm-profiler-base10.1" - - "amdrocm-rccl10.1" - - "amdrocm-runtime-dev10.1" - - "amdrocm-solver10.1" - - "amdrocm-sparse10.1" + # - "rocm-smi-lib" + - "libdrm-amdgpu-common" + # Do not use hsa-rocr from amdgpu repo because it is overriden by hsa-runtime-rocr4wsl-amdgpu + - "hsa-rocr (<< 20)" + - "hsa-amd-aqlprofile" + - "comgr" + - "rocprofiler-register" + - "rocprofiler-sdk" + - "miopen-hip" + - "rccl" + - "rocm-device-libs" + - "hip-dev" + - "rocblas" + - "rocsolver" + - "hipsolver" + - "hipfft" + - "roctracer" + - "hipblaslt" + - "hipblaslt-dev" + - "hip-runtime-amd" diff --git a/platforms/rocm/rocm.bzl b/platforms/rocm/rocm.bzl index 41b01b60..78171a3b 100644 --- a/platforms/rocm/rocm.bzl +++ b/platforms/rocm/rocm.bzl @@ -1,236 +1,126 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") -load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("//bazel:http_deb_archive.bzl", "http_deb_archive") load("//platforms:packages.bzl", "packages") _BUILD_FILE_DEFAULT_VISIBILITY = """\ package(default_visibility = ["//visibility:public"]) - -load("@rules_cc//cc:cc_library.bzl", "cc_library") """ -_ROCM_VERSION = "10.1" -_ROCM_STRIP_PREFIX = "./opt/rocm/core-" + _ROCM_VERSION -_PLUGIN_ROCM_URL = "https://mirror.zml.ai/plugins/202611101651.363.1.d46c7471c419/zml-rocm-linux-amd64.tar.zst" -_PLUGIN_ROCM_SHA256 = "cdc4434f2023842f0e1a34c08aaed87f082df19f92c4ae21c9d4452a9739b604" - -def _rocm_package_name(name): - return name + _ROCM_VERSION - -def _rocm_repo_name(package_name): - return package_name.replace(_ROCM_VERSION, "") +_ROCM_STRIP_PREFIX = "opt/rocm-7.2.0" _UBUNTU_PACKAGES = { - "llvm-libunwind1": packages.filegroup(name = "libunwind", srcs = ["usr/lib/x86_64-linux-gnu/libunwind.so.1"]), - "libatomic1": packages.filegroup(name = "libatomic1", srcs = ["usr/lib/x86_64-linux-gnu/libatomic.so.1"]), - "libdrm-common": packages.filegroup(name = "amdgpu_ids", srcs = ["usr/share/libdrm/amdgpu.ids"]), -} - -def _rocm_dlopen_patchelf(name, src): - return "\n".join([ + "libdrm2-amdgpu": packages.filegroup(name = "libdrm2-amdgpu", srcs = ["opt/amdgpu/lib/x86_64-linux-gnu/libdrm.so.2"]), + "libelf1": "\n".join([ packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), packages.patchelf( - name = name, - src = src, + name = "libelf1", + src = "usr/lib/x86_64-linux-gnu/libelf.so.1", + set_rpath = "$ORIGIN", + ), + ]), + "libdrm-amdgpu-common": packages.filegroup(name = "amdgpu_ids", srcs = ["opt/amdgpu/share/libdrm/amdgpu.ids"]), + "libnuma1": packages.filegroup(name = "libnuma1", srcs = ["usr/lib/x86_64-linux-gnu/libnuma.so.1"]), + "libzstd1": packages.filegroup(name = "libzstd1", srcs = ["usr/lib/x86_64-linux-gnu/libzstd.so.1"]), + "libdrm-amdgpu-amdgpu1": "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = "libdrm-amdgpu-amdgpu1", + src = "opt/amdgpu/lib/x86_64-linux-gnu/libdrm_amdgpu.so.1", add_needed = ["libzmlxrocm.so.0"], - set_rpath = "$ORIGIN:$ORIGIN/rocm_sysdeps/lib", + set_rpath = "$ORIGIN", rename_dynamic_symbols = { - "dlopen": "zmlxrocm_dlopen", - "pthread_attr_setstacksize": "zmlxrocm_pthread_attr_setstacksize", + "fopen64": "zmlxrocm_fopen64", }, ), - ]) + ]), + "libtinfo6": packages.filegroup(name = "libtinfo6", srcs = ["lib/x86_64-linux-gnu/libtinfo.so.6"]), + "zlib1g": packages.filegroup(name = "zlib1g", srcs = ["lib/x86_64-linux-gnu/libz.so.1"]), +} -def _glob_filegroup(name, srcs): - return """filegroup( - name = "{name}", - srcs = glob({srcs}), -)""".format(name = name, srcs = repr(srcs)) - -def _family_runfile_package_names(loaded_packages, family): - prefix = "amdrocm-{}{}-gfx".format(family, _ROCM_VERSION) - return sorted([ - pkg_name - for pkg_name in loaded_packages.keys() - if pkg_name.startswith(prefix) - ]) - -def _family_runfile_labels(loaded_packages, family): - return [ - "@{}//:runfiles".format(_rocm_repo_name(pkg_name)) - for pkg_name in _family_runfile_package_names(loaded_packages, family) - ] - -def _rocm_runfile_build_files(loaded_packages): - rocm_packages = {} - - for pkg_name in _family_runfile_package_names(loaded_packages, "blas"): - rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ - ".kpack/**", - "lib/hipblaslt/library/**", - "lib/rocblas/library/**", - ]) - - for pkg_name in _family_runfile_package_names(loaded_packages, "dnn"): - rocm_packages[pkg_name] = _glob_filegroup("runfiles", [ - ".kpack/**", - "lib/libMIOpenCKGroupedConv_*.so", - "share/miopen/db/**", - ]) - - for pkg_name in _family_runfile_package_names(loaded_packages, "fft"): - rocm_packages[pkg_name] = _glob_filegroup("runfiles", [".kpack/**"]) - - for pkg_name in _family_runfile_package_names(loaded_packages, "rccl"): - rocm_packages[pkg_name] = _glob_filegroup("runfiles", [".kpack/**"]) - - return rocm_packages - -def _rocm_base_build_files(loaded_packages): - return { - _rocm_package_name("amdrocm-amdsmi"): "\n\n".join([ - packages.cc_library(name = "amdsmi", hdrs = ["include/amd_smi/amdsmi.h"], includes = ["include/amd_smi"]), - packages.filegroup(name = "libamd_smi", srcs = ["lib/libamd_smi.so.27"]), +_ROCM_PACKAGES = { + "rocm-core": packages.filegroup(name = "rocm-core", srcs = ["lib/librocm-core.so.1"]), + "rocm-smi-lib": packages.filegroup(name = "rocm_smi", srcs = ["lib/librocm_smi64.so.1"]), + "rocprofiler-sdk": "\n".join([ + packages.filegroup(name = "rocprofiler-sdk", srcs = [ + "lib/librocprofiler-sdk.so.1", + "lib/librocprofiler-sdk-attach.so.1", ]), - _rocm_package_name("amdrocm-base"): "\n\n".join([ - _rocm_dlopen_patchelf(name = "rocm-core", src = "lib/librocm-core.so.1"), - _rocm_dlopen_patchelf(name = "rocprofiler-register", src = "lib/librocprofiler-register.so.0"), - ]), - _rocm_package_name("amdrocm-blas-dev"): _glob_filegroup("headers", ["include/**"]), - _rocm_package_name("amdrocm-blas-host"): "\n\n".join([ - _rocm_dlopen_patchelf(name = "hipblas", src = "lib/libhipblas.so.3"), - packages.filegroup(name = "hipblaslt", srcs = [ - "lib/libhipblaslt.so.1", - "lib/liborigami.so.1", - ]), - packages.filegroup(name = "rocblas", srcs = ["lib/librocblas.so.5"]), - _rocm_dlopen_patchelf(name = "rocroller", src = "lib/librocroller.so.1"), - _glob_filegroup("hipblaslt_support", [ - "lib/hipdnn_plugins/engines/libhipblaslt_plugin.so", - "share/hipblaslt/**", - ]), - ]), - _rocm_package_name("amdrocm-blas"): "\n\n".join([ - packages.filegroup(name = "hipblaslt", srcs = [ - "@amdrocm-blas-host//:hipblaslt", - "@amdrocm-blas-host//:rocroller", - ]), - packages.filegroup(name = "hipblaslt_runfiles", srcs = _family_runfile_labels(loaded_packages, "blas") + [ - "@amdrocm-blas-host//:hipblaslt_support", - ]), - packages.filegroup(name = "rocblas", srcs = [ - "@amdrocm-blas-host//:hipblas", - "@amdrocm-blas-host//:rocblas", - ]), - packages.filegroup(name = "rocblas_runfiles", srcs = _family_runfile_labels(loaded_packages, "blas")), - ]), - _rocm_package_name("amdrocm-dnn-host"): "\n\n".join([ - _rocm_dlopen_patchelf(name = "MIOpen", src = "lib/libMIOpen.so.1"), - _rocm_dlopen_patchelf(name = "hipdnn_backend", src = "lib/libhipdnn_backend.so"), - _glob_filegroup("runfiles", [ - "lib/hipdnn_plugins/**", - "share/miopen/**", - ]), - ]), - _rocm_package_name("amdrocm-dnn"): packages.filegroup(name = "MIOpen", srcs = [ - "@amdrocm-dnn-host//:MIOpen", - "@amdrocm-dnn-host//:hipdnn_backend", - "@amdrocm-dnn-host//:runfiles", - ] + _family_runfile_labels(loaded_packages, "dnn")), - _rocm_package_name("amdrocm-fft-host"): "\n\n".join([ - _rocm_dlopen_patchelf(name = "hipfft", src = "lib/libhipfft.so.0"), - _rocm_dlopen_patchelf(name = "hipfftw", src = "lib/libhipfftw.so.0"), - packages.filegroup(name = "rocfft", srcs = ["lib/librocfft.so.0"]), - ]), - _rocm_package_name("amdrocm-fft"): "\n\n".join([ - packages.filegroup(name = "hipfft", srcs = [ - "@amdrocm-fft-host//:hipfft", - "@amdrocm-fft-host//:hipfftw", - ]), - packages.filegroup(name = "rocfft", srcs = [ - "@amdrocm-fft-host//:rocfft", - ] + _family_runfile_labels(loaded_packages, "fft")), - ]), - _rocm_package_name("amdrocm-llvm"): "\n\n".join([ - packages.filegroup(name = "llvm_libs", srcs = [ - "lib/llvm/lib/libLLVM.so.24.0git", - "lib/llvm/lib/libclang-cpp.so.24.0git", - ]), - _glob_filegroup("rocm_device_libs_runfiles", ["lib/llvm/amdgcn/**"]), - ]), - _rocm_package_name("amdrocm-math-common"): _glob_filegroup("host_math_libs", ["lib/host-math/lib/*.so*"]), - _rocm_package_name("amdrocm-profiler-base"): "\n\n".join([ - _rocm_dlopen_patchelf(name = "hsa-amd-aqlprofile", src = "lib/libhsa-amd-aqlprofile64.so.1"), - _rocm_dlopen_patchelf(name = "rocprofiler-sdk_so", src = "lib/librocprofiler-sdk.so.1"), - packages.filegroup(name = "rocprofiler-sdk-attach", srcs = ["lib/librocprofiler-sdk-attach.so.1"]), - packages.filegroup(name = "rocprofiler-sdk-rocattach", srcs = ["lib/librocprofiler-sdk-rocattach.so.1"]), - _rocm_dlopen_patchelf(name = "roctracer", src = "lib/libroctracer64.so.4"), - _rocm_dlopen_patchelf(name = "roctx", src = "lib/libroctx64.so.4"), - packages.cc_library_hdrs_glob( - name = "roctx_headers", - hdrs_glob = ["include/rocprofiler-sdk-roctx/**/*.h"], - includes = ["include"], - ), - packages.filegroup(name = "rocprofiler-sdk", srcs = [ - ":rocprofiler-sdk_so", - ":rocprofiler-sdk-attach", - ":rocprofiler-sdk-rocattach", - ]), - _glob_filegroup("rocprofv3_tool", [ - "bin/rocprof*", - "bin/rocpd*", - "libexec/rocprofiler-sdk/**", - "share/rocprofiler-sdk/**", - ]), - ]), - _rocm_package_name("amdrocm-rccl-host"): packages.filegroup(name = "rccl", srcs = ["lib/librccl.so.1"]), - _rocm_package_name("amdrocm-rccl"): packages.filegroup(name = "rccl", srcs = [ - "@amdrocm-rccl-host//:rccl", - ] + _family_runfile_labels(loaded_packages, "rccl")), - _rocm_package_name("amdrocm-runtime"): "\n\n".join([ - _rocm_dlopen_patchelf(name = "amd_comgr", src = "lib/libamd_comgr.so.3"), - _rocm_dlopen_patchelf(name = "amdhip_so", src = "lib/libamdhip64.so.7"), - _rocm_dlopen_patchelf(name = "hiprtc_so", src = "lib/libhiprtc.so.7"), - _rocm_dlopen_patchelf(name = "hsa-runtime", src = "lib/libhsa-runtime64.so.1"), - packages.filegroup(name = "amdhip", srcs = [ - ":amdhip_so", - "lib/librocm_kpack.so.0", - ]), - packages.filegroup(name = "hiprtc", srcs = [ - ":hiprtc_so", - "lib/libhiprtc-builtins.so.7", - ]), - ]), - _rocm_package_name("amdrocm-runtime-dev"): _glob_filegroup("headers", ["include/**"]), - _rocm_package_name("amdrocm-solver-host"): "\n\n".join([ - _rocm_dlopen_patchelf(name = "hipsolver", src = "lib/libhipsolver.so.1"), - packages.filegroup(name = "rocsolver", srcs = ["lib/librocsolver.so.0"]), - ]), - _rocm_package_name("amdrocm-solver"): "\n\n".join([ - packages.filegroup(name = "hipsolver", srcs = ["@amdrocm-solver-host//:hipsolver"]), - packages.filegroup(name = "rocsolver", srcs = [ - "@amdrocm-solver-host//:rocsolver", - "@amdrocm-math-common//:host_math_libs", - ]), - ]), - _rocm_package_name("amdrocm-sparse-host"): "\n\n".join([ - _rocm_dlopen_patchelf(name = "hipsparse", src = "lib/libhipsparse.so.4"), - packages.filegroup(name = "rocsparse", srcs = ["lib/librocsparse.so.1"]), - ]), - _rocm_package_name("amdrocm-sparse"): packages.filegroup(name = "rocsparse", srcs = [ - "@amdrocm-sparse-host//:hipsparse", - "@amdrocm-sparse-host//:rocsparse", - ]), - _rocm_package_name("amdrocm-sysdeps"): _glob_filegroup("system_libs", [ - "lib/rocm_sysdeps/lib/*.so*", - "lib/rocm_sysdeps/share/**", - ]), - } - -def _rocm_build_files(loaded_packages): - rocm_packages = _rocm_base_build_files(loaded_packages) - rocm_packages.update(_rocm_runfile_build_files(loaded_packages)) - return rocm_packages + ]), + "rocprofiler-sdk-rocpd": packages.filegroup(name = "rocprofiler-sdk-rocpd", srcs = ["lib/librocprofiler-sdk-rocpd.so.1"]), + "rocprofiler-sdk-roctx": packages.filegroup(name = "rocprofiler-sdk-roctx", srcs = ["lib/librocprofiler-sdk-roctx.so.1"]), + "hsa-rocr": packages.filegroup(name = "hsa-runtime", srcs = ["lib/libhsa-runtime64.so.1"]), + "hsa-amd-aqlprofile": packages.filegroup(name = "hsa-amd-aqlprofile", srcs = ["lib/libhsa-amd-aqlprofile64.so.1"]), + "comgr": packages.filegroup(name = "amd_comgr", srcs = ["lib/libamd_comgr.so.3"]), + "rocprofiler-register": packages.filegroup(name = "rocprofiler-register", srcs = ["lib/librocprofiler-register.so.0"]), + "miopen-hip": "\n".join([ + packages.filegroup(name = "MIOpen", srcs = ["lib/libMIOpen.so.1"]), + """filegroup(name = "runfiles", srcs = glob(["share/miopen/**"]))""", + ]), + "rccl": packages.filegroup(name = "rccl", srcs = ["lib/librccl.so.1"]), + "rocm-device-libs": """filegroup(name = "runfiles", srcs = glob(["amdgcn/**"]))""", + "hip-dev": """filegroup(name = "runfiles", srcs = glob(["share/**"]))""", + "rocblas": "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = "rocblas", + src = "lib/librocblas.so.5", + add_needed = ["libzmlxrocm.so.0"], + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + ), + """filegroup( + name = "runfiles", + srcs = glob(["lib/rocblas/library/**"]), + ) + """ + ]), + "rocfft": packages.filegroup(name = "rocfft", srcs = ["lib/librocfft.so.0"]), + "rocsolver": packages.filegroup(name = "rocsolver", srcs = ["lib/librocsolver.so.0"]), + "roctracer": "\n".join([ + packages.filegroup(name = "roctracer", srcs = ["lib/libroctracer64.so.4"]), + packages.filegroup(name = "roctx", srcs = ["lib/libroctx64.so.4"]), + ]), + "hipblaslt": "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = "hipblaslt", + src = "lib/libhipblaslt.so.1", + add_needed = ["libzmlxrocm.so.0"], + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + ), + packages.patchelf( + name = "rocroller", + src = "lib/librocroller.so.1", + set_rpath = "$ORIGIN", + add_needed = ["libzmlxrocm.so.0"], + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + ), + """filegroup( + name = "runfiles", + srcs = glob(["lib/hipblaslt/library/**"]), + ) + """ + ]), + "hipfft": packages.filegroup(name = "hipfft", srcs = ["lib/libhipfft.so.0"]), + "hip-runtime-amd": "\n".join([ + packages.load_("@zml//bazel:patchelf.bzl", "patchelf"), + packages.patchelf( + name = "amdhip", + src = "lib/libamdhip64.so.7", + add_needed = ["libzmlxrocm.so.0"], + rename_dynamic_symbols = { + "dlopen": "zmlxrocm_dlopen", + }, + ), + packages.filegroup(name = "hiprtc", srcs = ["lib/libhiprtc.so.7", "lib/libhiprtc-builtins.so.7"]), + ]), + "hipsolver": packages.filegroup(name = "hipsolver", srcs = ["lib/libhipsolver.so.1"]), +} def _rocm_impl(mctx): loaded_packages = packages.read(mctx, [ @@ -238,7 +128,7 @@ def _rocm_impl(mctx): ]) for pkg_name, build_file_content in _UBUNTU_PACKAGES.items(): - pkg = loaded_packages[pkg_name]["amd64"] + pkg = loaded_packages[pkg_name] http_deb_archive( name = pkg_name, urls = pkg["urls"], @@ -246,50 +136,26 @@ def _rocm_impl(mctx): build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, ) - for pkg_name, build_file_content in _rocm_build_files(loaded_packages).items(): - pkg = loaded_packages[pkg_name]["amd64"] + for pkg_name, build_file_content in _ROCM_PACKAGES.items(): + pkg = loaded_packages[pkg_name] http_deb_archive( - name = _rocm_repo_name(pkg_name), + name = pkg_name, urls = pkg["urls"], sha256 = pkg["sha256"], strip_prefix = _ROCM_STRIP_PREFIX, build_file_content = _BUILD_FILE_DEFAULT_VISIBILITY + build_file_content, ) - http_file( - name = "libdrm_mesa_amdgpu_ids", - url = "https://gitlab.freedesktop.org/mesa/libdrm/-/raw/979f607906ad64f629967ac1f3ba3590e756442c/data/amdgpu.ids?inline=false", - sha256 = "ffd2a8f1bfa755f4d90f537b4969fc4676f116e5af051ce2f18ef93a96d8beb6", - downloaded_file_path = "amdgpu.ids", - ) - http_archive( - name = "libzml_rocm", - build_file = "libzml_rocm.BUILD.bazel", - url = _PLUGIN_ROCM_URL, - sha256 = _PLUGIN_ROCM_SHA256, + name = "libpjrt_rocm", + build_file = "libpjrt_rocm.BUILD.bazel", + url = "https://github.com/zml/pjrt-artifacts/releases/download/v16.0.0/pjrt-rocm_linux-amd64.tar.gz", + sha256 = "6a70cf6a5a8330809f173072dd5fae5332c3ec8fba80acb9b77ddd97bda6572b", ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = [ - "amdrocm-amdsmi", - "amdrocm-base", - "amdrocm-blas", - "amdrocm-dnn", - "amdrocm-fft", - "amdrocm-llvm", - "amdrocm-math-common", - "amdrocm-profiler-base", - "amdrocm-rccl", - "amdrocm-runtime", - "amdrocm-solver", - "amdrocm-sparse", - "amdrocm-sysdeps", - "libatomic1", - "libdrm-common", - "libzml_rocm", - ], + root_module_direct_deps = ["libpjrt_rocm", "hipblaslt", "rocblas"], root_module_direct_dev_deps = [], ) diff --git a/platforms/rocm/rocm.h b/platforms/rocm/rocm.h deleted file mode 100644 index 933e4f31..00000000 --- a/platforms/rocm/rocm.h +++ /dev/null @@ -1 +0,0 @@ - #include diff --git a/platforms/rocm/rocm.zig b/platforms/rocm/rocm.zig index ee31bd83..79053208 100644 --- a/platforms/rocm/rocm.zig +++ b/platforms/rocm/rocm.zig @@ -5,14 +5,13 @@ const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); const c = @import("c"); const pjrt = @import("pjrt"); -const platforms_options = @import("platforms/options"); const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/rocm"); pub fn isEnabled() bool { - return platforms_options.rocm_enabled; + return @hasDecl(c, "ZML_RUNTIME_ROCM"); } fn hasRocmDevices(io: std.Io) bool { @@ -25,23 +24,10 @@ fn hasRocmDevices(io: std.Io) bool { fn setupRocmEnv(rocm_data_dir: []const u8) !void { var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; _ = c.setenv("ROCM_PATH", try stdx.Io.Dir.path.bufJoinZ(&buf, &.{rocm_data_dir}), 1); // must be zero terminated - - // Use one HIP graph stream to reduce cross-stream synchronization overhead. - _ = c.setenv("DEBUG_HIP_FORCE_GRAPH_QUEUES", "1", 0); - // Share one hardware queue per device so auxiliary barriers stay on the kernel queue. - _ = c.setenv("GPU_MAX_HW_QUEUES", "1", 0); - // Disable rocprofiler queue interposition to avoid its dispatch overhead outside profiling. - _ = c.setenv("ROCPROFILER_QUEUE_INTERPOSITION", "0", 0); - - if (c.getenv("ROCPROFILER_LOG_LEVEL") == null) { - // rocprofiler defaults to the `warning` log level, which can produce unnecessary log output. - // Set the log level to `fatal` to suppress all but fatal error messages. - _ = c.setenv("ROCPROFILER_LOG_LEVEL", "error", 0); - } } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = allocator; + _ = allocator; // autofix if (comptime !isEnabled()) { return error.Unavailable; } @@ -55,7 +41,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libzml_rocm/sandbox", &path_buf) orelse { + const sandbox_path = try r.rlocation("libpjrt_rocm/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for ROCm runtime", .{}); return error.FileNotFound; }; @@ -73,7 +59,7 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { // executing the destructor. Accessing this variable results in a segmentation fault... return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_rocm.so" }); + const lib_path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_rocm.so" }); break :blk .loadFrom(lib_path); }; } diff --git a/platforms/rocm/zmlxrocm.zig b/platforms/rocm/zmlxrocm.zig index 069c08bb..e3024de6 100644 --- a/platforms/rocm/zmlxrocm.zig +++ b/platforms/rocm/zmlxrocm.zig @@ -4,11 +4,22 @@ const stdx = @import("stdx"); pub export fn zmlxrocm_dlopen(filename: [*c]const u8, flags: c_int) ?*anyopaque { const replacements: std.StaticStringMap([:0]const u8) = .initComptime(.{ - .{ "libamdhip64.so", "libamdhip64.so.7" }, + .{ "librocm-core.so", "librocm-core.so.1" }, + .{ "librocm_smi64.so", "librocm_smi64.so.1" }, + .{ "libhsa-runtime64.so", "libhsa-runtime64.so.1" }, .{ "libhsa-amd-aqlprofile64.so", "libhsa-amd-aqlprofile64.so.1" }, - .{ "librocprofiler-sdk-attach.so", "librocprofiler-sdk-attach.so.1" }, - .{ "librocprofiler-sdk.so", "librocprofiler-sdk.so.1" }, + .{ "libamd_comgr.so", "libamd_comgr.so.3" }, + .{ "librocprofiler-register.so", "librocprofiler-register.so.0" }, + .{ "libMIOpen.so", "libMIOpen.so.1" }, + .{ "libMIOpen.so.1", "libMIOpen.so.1" }, + .{ "librccl.so", "librccl.so.1" }, + .{ "librocblas.so.5", "librocblas.so.5" }, + .{ "librocblas.so", "librocblas.so.5" }, + .{ "libroctracer64.so", "libroctracer64.so.4" }, .{ "libroctx64.so", "libroctx64.so.4" }, + .{ "libhipblaslt.so", "libhipblaslt.so.1" }, + .{ "libamdhip64.so", "libamdhip64.so.7" }, + .{ "libhiprtc.so", "libhiprtc.so.7" }, }); var buf: [std.Io.Dir.max_path_bytes]u8 = undefined; @@ -39,29 +50,3 @@ pub export fn zmlxrocm_fopen64(pathname: [*c]const u8, mode: [*c]const u8) ?*std return std.c.fopen64(new_pathname, mode); } - -extern fn pthread_attr_setstacksize(attr: ?*anyopaque, stacksize: usize) c_int; - -// ROCclr asks for small helper-thread stacks: CQ_THREAD_STACK_SIZE is 256 KiB, -// then ROCclr adds the pthread guard size before calling pthread_attr_setstacksize. -// https://github.com/ROCm/clr/blob/1736f3eba9134a5ee4b56475acf15665f065ab05/rocclr/utils/flags.hpp#L25 -// https://github.com/ROCm/clr/blob/1736f3eba9134a5ee4b56475acf15665f065ab05/rocclr/os/os_posix.cpp#L378-L385 -// glibc 2.28 accepts stack sizes >= PTHREAD_STACK_MIN at attribute-set time, -// but pthread_create can still reject stacks that do not leave room for the -// guard, static TLS, and MINIMAL_REST_STACK: -// https://github.com/bminor/glibc/blob/glibc-2.28/nptl/pthread_attr_setstacksize.c#L35-L38 -// https://github.com/bminor/glibc/blob/glibc-2.28/nptl/allocatestack.c#L532-L544 -// The 1 MiB floor below is a local compatibility margin for that glibc path; -// it is intentionally larger than ROCclr's request and not a ROCm ABI value. -pub export fn zmlxrocm_pthread_attr_setstacksize(attr: ?*anyopaque, stacksize: usize) c_int { - const min_stack_size = 1024 * 1024; - const adjusted_stacksize = blk: { - if (stacksize < min_stack_size) { - break :blk min_stack_size; - } else { - break :blk stacksize; - } - }; - - return pthread_attr_setstacksize(attr, adjusted_stacksize); -} diff --git a/platforms/tpu/BUILD.bazel b/platforms/tpu/BUILD.bazel index 1aab4031..af25cd2f 100644 --- a/platforms/tpu/BUILD.bazel +++ b/platforms/tpu/BUILD.bazel @@ -2,38 +2,30 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( - name = "libzml_tpu", - tags = ["manual"], - deps = ["@libzml_tpu"], + name = "empty", + defines = ["ZML_RUNTIME_TPU_DISABLED"], +) + +cc_library( + name = "libpjrt_tpu", + defines = ["ZML_RUNTIME_TPU"], + deps = ["@libpjrt_tpu"], ) zig_library( name = "tpu", import_name = "platforms/tpu", main = "tpu.zig", - tags = ["manual"], visibility = ["//visibility:public"], deps = [ - "//bazel", "//pjrt", - "//platforms:options", - "//stdx", ] + select({ "//platforms:tpu.enabled": [ - ":libzml_tpu", + ":libpjrt_tpu", + "//bazel", + "//stdx", + "@rules_zig//zig/runfiles", ], - "//conditions:default": [], + "//conditions:default": [":empty"], }), ) - -zig_library( - name = "ragged_paged", - import_name = "platforms/tpu/ragged_paged", - main = "ragged_paged.zig", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//dsl/mosaic_tpu:builder", - "//mlir", - ], -) diff --git a/platforms/tpu/libpjrt_tpu.BUILD.bazel b/platforms/tpu/libpjrt_tpu.BUILD.bazel new file mode 100644 index 00000000..767cd004 --- /dev/null +++ b/platforms/tpu/libpjrt_tpu.BUILD.bazel @@ -0,0 +1,21 @@ +load("@aspect_bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") + +copy_file( + name = "libpjrt_tpu.so", + src = "libtpu/libtpu.so", + out = "lib/libpjrt_tpu.so", +) + +copy_to_directory( + name = "sandbox", + srcs = [ + "lib/libpjrt_tpu.so", + ], +) + +cc_library( + name = "libpjrt_tpu", + data = [":sandbox"], + visibility = ["@zml//platforms/tpu:__subpackages__"], +) diff --git a/platforms/tpu/libzml_tpu.BUILD.bazel b/platforms/tpu/libzml_tpu.BUILD.bazel deleted file mode 100644 index 1186510c..00000000 --- a/platforms/tpu/libzml_tpu.BUILD.bazel +++ /dev/null @@ -1,35 +0,0 @@ -load("@bazel_lib//lib:copy_to_directory.bzl", "copy_to_directory") -load("@bazel_skylib//rules:copy_file.bzl", "copy_file") - -copy_file( - name = "libzml_tpu.so", - src = "libtpu/libtpu.so", - out = "lib/libzml_tpu.so", -) - -copy_to_directory( - name = "sandbox", - srcs = [ - "lib/libzml_tpu.so", - ], - add_directory_to_runfiles = False, - allow_overwrites = True, - exclude_srcs_patterns = [ - "**/include/**", - "**/_solib_*/**", - ], - hardlink = "auto", - include_external_repositories = ["**"], - include_srcs_patterns = [ - "**/lib**", - ], - replace_prefixes = { - "lib/x86_64-linux-gnu": "lib", - }, -) - -cc_library( - name = "libzml_tpu", - data = [":sandbox"], - visibility = ["@zml//platforms/tpu:__subpackages__"], -) diff --git a/platforms/tpu/ragged_paged.zig b/platforms/tpu/ragged_paged.zig deleted file mode 100644 index 6a0fdfae..00000000 --- a/platforms/tpu/ragged_paged.zig +++ /dev/null @@ -1,715 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const mtt = @import("dsl/mosaic_tpu/builder"); - -pub const Cfg = struct { - // Shapes that vary at compile-of-kernel time. - num_q_tokens: i64, - num_q_heads: i64, - num_kv_heads: i64, - head_dim: i64, - total_num_pages: i64, - page_size: i64, - max_num_seqs: i64, - pages_per_seq: i64, - - // Block-level tuning knobs. - num_kv_pages_per_block: i64, - num_queries_per_block: i64, - - // Element types. - q_dtype: mtt.DType = .bf16, - kv_dtype: mtt.DType = .bf16, - - // Numeric knobs. - sm_scale: f32, - mask_value: f32 = -2.38197633e+38, - /// Pre-mask softcap: `qk = soft_cap * tanh(qk / soft_cap)`. - soft_cap: ?f32 = null, - /// K dequant scale applied right after `stridedLoadKv` (mirrors Pallas's - /// `k.astype(f32) * k_scale → k.astype(q_dtype)`). - k_scale: ?f32 = null, - v_scale: ?f32 = null, - - /// Whether to mask keys beyond the current query row. - is_causal: bool = true, - - /// Sliding-window mask added on top of the validity/causal mask: any - /// `(row, col)` with `row - sliding_window >= col` is masked out. Mirrors - /// `kernel.py:391-393` in causal mode. - sliding_window: ?i64 = null, - vmem_limit_bytes: ?i64 = null, - - // Derived shape helpers — keeps the body readable. - pub fn numCombinedKvHeads(self: Cfg) i64 { - return 2 * self.num_kv_heads; - } - pub fn numQHeadsPerKvHead(self: Cfg) i64 { - return @divExact(self.num_q_heads, self.num_kv_heads); - } - pub fn numQHeadsPerBlk(self: Cfg) i64 { - return self.num_q_heads; - } - pub fn numCombinedKvHeadsPerBlk(self: Cfg) i64 { - return self.numCombinedKvHeads(); - } - pub fn numKvHeadsPerBlk(self: Cfg) i64 { - return @divExact(self.numCombinedKvHeadsPerBlk(), 2); - } - pub fn numQPerBlk(self: Cfg) i64 { - return self.num_queries_per_block; - } - pub fn numKvPerBlk(self: Cfg) i64 { - return self.num_kv_pages_per_block * self.page_size; - } - pub fn numHeadsBlks(self: Cfg) i64 { - return @divExact(self.num_q_heads, self.numQHeadsPerBlk()); - } - pub fn numQBlks(self: Cfg) i64 { - return @divExact(self.num_q_tokens, self.numQPerBlk()); - } - pub fn flashQRows(self: Cfg) i64 { - return self.numQPerBlk() * self.numQHeadsPerKvHead(); - } - pub fn flashKRows(self: Cfg) i64 { - return self.numKvPerBlk(); - } - - /// Pallas's `get_dtype_packing(dtype) = 32 // bitwidth(dtype)`. Mirrors - /// `kernel.py:get_dtype_packing` exactly. - pub fn kvPackingFactor(self: Cfg) i64 { - return switch (self.kv_dtype) { - .i32, .f32 => 1, - .i16, .f16, .bf16 => 2, - .i8, .f8e4m3fn, .f8e5m2 => 4, - else => std.debug.panic("ragged_paged: unsupported KV dtype {s}", .{@tagName(self.kv_dtype)}), - }; - } - /// Same packing rule applied to `q_dtype` — used by `fold_on_2nd_minor` - /// to decide whether to cast q to f32 before the trailing-dim reshape. - pub fn qPackingFactor(self: Cfg) i64 { - return switch (self.q_dtype) { - .f32 => 1, - .f16, .bf16 => 2, - else => std.debug.panic("ragged_paged: unsupported Q dtype {s}", .{@tagName(self.q_dtype)}), - }; - } - /// `kv_load_step = max(1, kv_packing // 2)` — number of (k, v) pairs the - /// caller unpacks from a single `b` strided-load (Pallas's outer-loop step). - pub fn kvLoadStep(self: Cfg) i64 { - const p = self.kvPackingFactor(); - return @max(@as(i64, 1), @divFloor(p, 2)); - } - pub fn kvFlatRows(self: Cfg) i64 { - return self.num_kv_pages_per_block * self.page_size * self.numCombinedKvHeadsPerBlk(); - } - pub fn kvFlatI32Rows(self: Cfg) i64 { - return @divExact(self.kvFlatRows(), self.kvPackingFactor()); - } -}; - -fn maskedStoreLM( - k: *mtt.Builder, - ref: mtt.Value, - val: mtt.Value, - kv_head_idx_idx: mtt.Value, - start: mtt.Value, - end: mtt.Value, - group: i64, -) void { - const shape = val.shape().constSlice(); - const iota = k.iota(shape, .i32, &.{0}); - const iota_div = k.divFloor(iota, @as(i32, @intCast(group))); - const ge = k.cmpi(.sge, iota_div, k.broadcastTo(start, shape)); - const lt = k.cmpi(.slt, iota_div, k.broadcastTo(end, shape)); - const mask = k.andi(ge, lt); - const slot_shape = [_]i64{ 1, shape[0], shape[1] }; - const val_3d = k.shapeCast(val, &slot_shape); - const mask_3d = k.shapeCast(mask, &slot_shape); - k.refStoreOpts(ref, val_3d, &.{ kv_head_idx_idx, k.constIndex(0), k.constIndex(0) }, .{ .mask = mask_3d, .strides = &.{} }); -} - -/// `pltpu.store(acc_ref, val, mask=…)` — group=1 so no floor-div. -fn maskedStoreAcc( - k: *mtt.Builder, - ref: mtt.Value, - val: mtt.Value, - q_head_offset_idx: mtt.Value, - start: mtt.Value, - end: mtt.Value, -) void { - const shape = val.shape().constSlice(); - const iota = k.iota(shape, .i32, &.{0}); - const ge = k.cmpi(.sge, iota, k.broadcastTo(start, shape)); - const lt = k.cmpi(.slt, iota, k.broadcastTo(end, shape)); - const mask = k.andi(ge, lt); - k.refStoreOpts(ref, val, &.{ k.constIndex(0), q_head_offset_idx, k.constIndex(0) }, .{ .mask = mask, .strides = &.{} }); -} - -/// `MultiPageAsyncCopyDescriptor.__init__` — `tpu.memref_slice → squeeze -/// → enqueue_dma` chain that Pallas emits for one async page-copy. -fn createKvAsyncCopyDescriptors( - k: *mtt.Builder, - cfg: Cfg, - a: anytype, - heads_blk_idx: mtt.Value, // i32 - seq_idx: mtt.Value, // i32 - kv_blk_idx: mtt.Value, // i32 — the block-relative page index - buf_idx: mtt.Value, // i32 -) void { - const start_kv_page_idx = k.muli(kv_blk_idx, k.constant(.i32, cfg.num_kv_pages_per_block)); - const kv_at_seq = k.scalarLoad(a.kv_lens, &.{k.toIndex(seq_idx)}); - const end_kv_page_idx = k.minsi( - k.divsi(k.addi(kv_at_seq, k.constant(.i32, cfg.page_size - 1)), k.constant(.i32, cfg.page_size)), - k.constant(.i32, cfg.pages_per_seq), - ); - const heads_start = k.muli(heads_blk_idx, k.constant(.i32, cfg.numCombinedKvHeadsPerBlk())); - - const within = k.cmpi(.slt, start_kv_page_idx, end_kv_page_idx); - const page_idx_safe = k.select(within, start_kv_page_idx, k.constant(.i32, 0)); - const page_table = k.scalarLoad(a.page_indices, &.{ k.toIndex(seq_idx), k.toIndex(page_idx_safe) }); - - const src0 = k.memRefSlice(a.kv_pages, &.{ k.constant(.i32, 0), k.constant(.i32, 0), heads_start, k.constant(.i32, 0) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); - const src1 = k.memRefSlice(src0, &.{ page_table, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); - const src = k.memRefSqueeze(src1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); - - const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); - const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); - const dst = k.memRefSqueeze(dst1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); - - const sem_slice = k.memRefSlice(a.sems, &.{buf_idx}, &.{1}, &.{}); - const sem = k.memRefSqueeze(sem_slice, &.{}); - - k.enqueueDma(src, dst, sem, .{}); -} - -fn waitKvDma( - k: *mtt.Builder, - cfg: Cfg, - a: anytype, - heads_blk_idx: mtt.Value, - seq_idx: mtt.Value, - kv_blk_idx: mtt.Value, - buf_idx: mtt.Value, -) void { - const start_kv_page_idx = k.muli(kv_blk_idx, k.constant(.i32, cfg.num_kv_pages_per_block)); - const kv_at_seq = k.scalarLoad(a.kv_lens, &.{k.toIndex(seq_idx)}); - const end_kv_page_idx = k.minsi( - k.divsi(k.addi(kv_at_seq, k.constant(.i32, cfg.page_size - 1)), k.constant(.i32, cfg.page_size)), - k.constant(.i32, cfg.pages_per_seq), - ); - const heads_start = k.muli(heads_blk_idx, k.constant(.i32, cfg.numCombinedKvHeadsPerBlk())); - const within = k.cmpi(.slt, start_kv_page_idx, end_kv_page_idx); - const page_idx_safe = k.select(within, start_kv_page_idx, k.constant(.i32, 0)); - const page_table = k.scalarLoad(a.page_indices, &.{ k.toIndex(seq_idx), k.toIndex(page_idx_safe) }); - - const src0 = k.memRefSlice(a.kv_pages, &.{ k.constant(.i32, 0), k.constant(.i32, 0), heads_start, k.constant(.i32, 0) }, &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); - const src1 = k.memRefSlice(src0, &.{ page_table, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); - const src = k.memRefSqueeze(src1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); - - const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); - const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); - const dst = k.memRefSqueeze(dst1, &.{ cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); - - const sem_slice = k.memRefSlice(a.sems, &.{buf_idx}, &.{1}, &.{}); - const sem = k.memRefSqueeze(sem_slice, &.{}); - - k.waitDma2(sem, src, dst, .{}); -} - -const KvPair = struct { k: mtt.Value, v: mtt.Value }; - -/// Mirrors Pallas's `b_ref[b_start::b_step, :]` lowering: -/// - `b_step == 1` → `vector.load` (the slice is contiguous). -/// - `last_dim ≤ 128` and `b_step > 1` → `tpu.strided_load`. -/// - `last_dim > 128` → per-row `vector.load` + `tpu.concatenate` -/// (`jnp.stack([ref[start + i*step, :] for i in range(num_rows)])`). -fn loadStridedOrStack( - k: *mtt.Builder, - ref: mtt.Value, - start_row: i64, - stride: i32, - out_rows: i64, - out_last_dim: i64, -) mtt.Value { - if (out_last_dim <= 128) { - if (stride == 1) { - return k.vectorLoadShape(ref, &.{ k.constIndex(start_row), k.constIndex(0) }, &.{ out_rows, out_last_dim }); - } - return k.stridedLoadShape(ref, &.{ k.constIndex(start_row), k.constIndex(0) }, &.{ stride, 1 }, &.{ out_rows, out_last_dim }); - } - const rows_count: usize = @intCast(out_rows); - const rows = k.arena.allocator().alloc(mtt.Value, rows_count) catch @panic("loadStridedOrStack OOM"); - for (0..rows_count) |i| { - const row_idx: i64 = start_row + @as(i64, @intCast(i)) * @as(i64, stride); - rows[i] = k.vectorLoadShape(ref, &.{ k.constIndex(row_idx), k.constIndex(0) }, &.{ 1, out_last_dim }); - } - return k.concatenate(rows, 0); -} - -/// Pallas's `broadcast_to_shape(arr, shape)` — emit `tpu.concatenate` of -/// `shape[1] / arr.shape[1]` copies when expanding axis 1 (e.g., -/// `` → `` for `head_dim > 128`). -fn broadcastToShape(k: *mtt.Builder, arr: mtt.Value, shape: []const i64) mtt.Value { - const arr_shape = arr.shape(); - std.debug.assert(arr_shape.len == 2 and shape.len == 2); - if (arr_shape.get(1) == shape[1]) return arr; - std.debug.assert(arr_shape.get(0) == shape[0]); - std.debug.assert(@mod(shape[1], arr_shape.get(1)) == 0); - const repeats: usize = @intCast(@divExact(shape[1], arr_shape.get(1))); - const sources = k.arena.allocator().alloc(mtt.Value, repeats) catch @panic("broadcastToShape OOM"); - for (sources) |*s| s.* = arr; - return k.concatenate(sources, 1); -} - -/// `strided_load_kv(ref, kv_head_chunk_idx * 2, num_combined_kv_heads_per_blk)` -/// — full Pallas parity. -/// -/// - `packing == 1` (i32/f32): two strided loads in original dtype. -/// - `packing == 2` bf16: special-case shli + and + tpu.bitcast(f32) + truncf. -/// - `packing == 2` f16/i16, `packing == 4` (i8/u8/f8): generic shrui + trunci -/// + tpu.bitcast unpack producing `kv_load_step` (k, v) pairs. -fn stridedLoadKv( - k: *mtt.Builder, - cfg: Cfg, - a: anytype, - buf_idx: mtt.Value, - kv_head_chunk_idx: i64, -) []const KvPair { - const packing = cfg.kvPackingFactor(); - const start: i64 = kv_head_chunk_idx * 2; - const step: i32 = @intCast(cfg.numCombinedKvHeadsPerBlk()); - - const dst0 = k.memRefSlice(a.kv_bufs, &.{ buf_idx, k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0), k.constant(.i32, 0) }, &.{ 1, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, &.{}); - const dst1 = k.memRefSqueeze(dst0, &.{ cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }); - const flat = k.memRefReshape(dst1, &.{ cfg.kvFlatRows(), cfg.head_dim }); - - if (packing == 1) { - const k_v = loadStridedOrStack(k, flat, start, step, cfg.flashKRows(), cfg.head_dim); - const v_v = loadStridedOrStack(k, flat, start + 1, step, cfg.flashKRows(), cfg.head_dim); - const result = k.arena.allocator().alloc(KvPair, 1) catch @panic("stridedLoadKv OOM"); - result[0] = .{ .k = k_v, .v = v_v }; - return result; - } - - std.debug.assert(packing == 2 or packing == 4 or packing == 8); - std.debug.assert(@mod(step, packing) == 0); - - const ref_i32 = k.memRefBitcastShape(flat, &.{ cfg.kvFlatI32Rows(), cfg.head_dim }, .i32); - const b_stride: i32 = @intCast(@divExact(cfg.numCombinedKvHeadsPerBlk(), packing)); - const b_start: i64 = @divExact(start, packing); - const b = loadStridedOrStack(k, ref_i32, b_start, b_stride, cfg.flashKRows(), cfg.head_dim); - - const splat_shape = &[_]i64{ cfg.flashKRows(), cfg.head_dim }; - - if (cfg.kv_dtype == .bf16) { - const bk = k.shli(b, k.splat(@as(i32, 16), splat_shape, .i32)); - const bv = k.andi(b, k.splat(@as(i32, @bitCast(@as(u32, 0xFFFF0000))), splat_shape, .i32)); - const k_v = k.arithTruncf(bk.bitCast(.f32), cfg.kv_dtype); - const v_v = k.arithTruncf(bv.bitCast(.f32), cfg.kv_dtype); - const result = k.arena.allocator().alloc(KvPair, 1) catch @panic("stridedLoadKv OOM"); - result[0] = .{ .k = k_v, .v = v_v }; - return result; - } - - const bitwidth: i64 = @divExact(@as(i64, 32), packing); - const trunc_dtype: mtt.DType = switch (bitwidth) { - 16 => .i16, - 8 => .i8, - else => std.debug.panic("ragged_paged: unsupported packing bitwidth {d}", .{bitwidth}), - }; - const kv_load_step: usize = @intCast(cfg.kvLoadStep()); - const result = k.arena.allocator().alloc(KvPair, kv_load_step) catch @panic("stridedLoadKv OOM"); - var i: usize = 0; - while (i < kv_load_step) : (i += 1) { - const shift_k: i32 = @intCast(@as(i64, @intCast(i)) * 2 * bitwidth); - const shift_v: i32 = @intCast((@as(i64, @intCast(i)) * 2 + 1) * bitwidth); - const bk = k.shrui(b, k.splat(shift_k, splat_shape, .i32)); - const bv = k.shrui(b, k.splat(shift_v, splat_shape, .i32)); - const bk_trunc = k.trunci(bk, trunc_dtype); - const bv_trunc = k.trunci(bv, trunc_dtype); - result[i] = .{ - .k = bk_trunc.bitCast(cfg.kv_dtype), - .v = bv_trunc.bitCast(cfg.kv_dtype), - }; - } - return result; -} - -/// `flash_attention(q, k, v, ...)` — one online-softmax step over `num_kv_per_blk` -/// keys. -fn flashAttention( - k: *mtt.Builder, - cfg: Cfg, - a: anytype, - q: mtt.Value, - kv_k: mtt.Value, - v: mtt.Value, - /// Element type of `kv_k` as currently held — equal to `cfg.kv_dtype` - /// unless `cfg.k_scale` was applied upstream (then `cfg.q_dtype`). - /// Used for the masked-zero round-trip Pallas's `.astype(k.dtype)` does. - k_input_dtype: mtt.DType, - /// Same role as `k_input_dtype`, for `v`. - v_input_dtype: mtt.DType, - kv_head_idx: i64, - q_head_idx: i64, - kv_blk_idx: mtt.Value, - kv_len: mtt.Value, - q_start: mtt.Value, - q_end: mtt.Value, - q_len: mtt.Value, - q_len_start: mtt.Value, -) void { - const flash_q: i64 = cfg.flashQRows(); - const flash_k: i64 = cfg.flashKRows(); - const head_dim: i64 = cfg.head_dim; - const num_q_per_blk: i64 = cfg.numQPerBlk(); - const num_q_heads_per_kv_head: i64 = cfg.numQHeadsPerKvHead(); - - const kv_len_start = k.muli(kv_blk_idx, k.constant(.i32, cfg.numKvPerBlk())); - - const kv_mask = k.cmpi( - .slt, - k.iota(&.{ flash_k, head_dim }, .i32, &.{0}), - k.broadcastTo(k.subi(kv_len, kv_len_start), &.{ flash_k, head_dim }), - ); - const k_zeroed = k.select(kv_mask, kv_k.to(.f32), k.zeros(&.{ flash_k, head_dim }, .f32)).to(k_input_dtype); - const v_zeroed = k.select(kv_mask, v.to(.f32), k.zeros(&.{ flash_k, head_dim }, .f32)).to(v_input_dtype); - - k.traceStart(10, "nd,md->nm"); - const qk_raw = k.matmulOpts( - q, - k_zeroed, - k.zeros(&.{ flash_q, flash_k }, .f32), - .{ .dimension_numbers = k.dotDimensionNumbers(&.{1}, &.{1}, &.{0}, &.{0}, &.{ 0, 0, 1, 0 }, &.{}, &.{}) }, - ); - k.traceStop(); - var qk = k.mulf(qk_raw, k.splat(cfg.sm_scale, &.{ flash_q, flash_k }, .f32)); - - const store_start = k.maxsi(k.subi(q_start, q_len_start), k.constant(.i32, 0)); - const store_end = k.minsi(k.subi(q_end, q_len_start), k.constant(.i32, num_q_per_blk)); - - const row_offset = k.subi(k.addi(k.subi(kv_len, q_len), q_len_start), q_start); - const row_iota = k.iota(&.{ flash_q, flash_k }, .i32, &.{0}); - const row_q = k.divFloor(row_iota, @as(i32, @intCast(num_q_heads_per_kv_head))); - const row_ids = k.addi(k.broadcastTo(row_offset, &.{ flash_q, flash_k }), row_q); - const col_ids = k.addi( - k.broadcastTo(kv_len_start, &.{ flash_q, flash_k }), - k.iota(&.{ flash_q, flash_k }, .i32, &.{1}), - ); - var score_mask = if (cfg.is_causal) - k.cmpi(.slt, row_ids, col_ids) - else - // `row_ids < row_ids` is an all-false mask with the same bool shape - // as the causal mask; sliding-window masking can still be OR-ed below. - k.cmpi(.slt, row_ids, row_ids); - if (cfg.sliding_window) |sw| { - const sw_splat = k.splat(@as(i32, @intCast(sw)), &.{ flash_q, flash_k }, .i32); - const window_mask = k.cmpi(.sge, k.subi(row_ids, sw_splat), col_ids); - score_mask = k.ori(score_mask, window_mask); - } - if (cfg.soft_cap) |sc| { - const sc_splat = k.splat(sc, &.{ flash_q, flash_k }, .f32); - qk = k.mulf(sc_splat, k.tanh(k.divf(qk, sc_splat))); - } - const qk_masked = k.addf( - qk, - k.select(score_mask, k.splat(cfg.mask_value, &.{ flash_q, flash_k }, .f32), k.zeros(&.{ flash_q, flash_k }, .f32)), - ); - - // jnp.max → maximumf (NaN-propagating). - const m_curr_v = k.multiReduction(.maximumf, qk_masked, k.splat(@as(f32, -std.math.inf(f32)), &.{flash_q}, .f32), &.{1}); - const m_curr_2d = k.shapeCast(m_curr_v, &.{ flash_q, 1 }); - const s_curr = k.exp(k.subf(qk_masked, k.broadcastTo(m_curr_2d, &.{ flash_q, flash_k }))); - - const qkv = k.matmulOpts( - s_curr, - v_zeroed, - k.zeros(&.{ flash_q, head_dim }, .f32), - .{ .dimension_numbers = k.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}) }, - ); - - // Pallas hardcodes the m/l-ref trailing dim to 128 (the TPU lane width) - // regardless of `head_dim`. Broadcast m/l values to `head_dim` only - // immediately before mixing with `qkv` / `o_curr`. - const lm_lane: i64 = 128; - - const m_curr_bc = k.broadcastTo(m_curr_2d, &.{ flash_q, lm_lane }); - const l_curr_v = k.multiReduction(.add, s_curr, k.zeros(&.{flash_q}, .f32), &.{1}); - const l_curr_bc = k.broadcastTo(k.shapeCast(l_curr_v, &.{ flash_q, 1 }), &.{ flash_q, lm_lane }); - - const head_l_ref_idx = k.constIndex(kv_head_idx); - const m_prev_load = k.shapeCast( - k.vectorLoadShape(a.m_ref, &.{ head_l_ref_idx, k.constIndex(0), k.constIndex(0) }, &.{ 1, flash_q, lm_lane }), - &.{ flash_q, lm_lane }, - ); - const m_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), k.splat(@as(f32, -std.math.inf(f32)), &.{ flash_q, lm_lane }, .f32), m_prev_load); - const l_prev_load = k.shapeCast( - k.vectorLoadShape(a.l_ref, &.{ head_l_ref_idx, k.constIndex(0), k.constIndex(0) }, &.{ 1, flash_q, lm_lane }), - &.{ flash_q, lm_lane }, - ); - const l_prev = k.select(k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), k.zeros(&.{ flash_q, lm_lane }, .f32), l_prev_load); - - const m_next = k.maximumf(m_prev, m_curr_bc); - maskedStoreLM(k, a.m_ref, m_next, head_l_ref_idx, store_start, store_end, num_q_heads_per_kv_head); - - const alpha = k.exp(k.subf(m_prev, m_next)); - const beta = k.exp(k.subf(m_curr_bc, m_next)); - const l_alpha = k.mulf(alpha, l_prev); - const l_next = k.addf(l_alpha, k.mulf(beta, l_curr_bc)); - const l_next_safe = k.select( - k.cmpf(.oeq, l_next, k.zeros(&.{ flash_q, lm_lane }, .f32)), - k.splat(@as(f32, 1.0), &.{ flash_q, lm_lane }, .f32), - l_next, - ); - maskedStoreLM(k, a.l_ref, l_next_safe, head_l_ref_idx, store_start, store_end, num_q_heads_per_kv_head); - - const q_head_idx_idx = k.constIndex(q_head_idx); - const o_curr_3d = k.vectorLoadShape(a.acc_ref, &.{ k.constIndex(0), q_head_idx_idx, k.constIndex(0) }, &.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }); - const o_curr_pre = k.select( - k.cmpi(.eq, kv_blk_idx, k.constant(.i32, 0)), - k.zeros(&.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }, .f32), - o_curr_3d, - ); - const o_curr = k.shapeCast(o_curr_pre, &.{ flash_q, head_dim }); - const l_alpha_bc = broadcastToShape(k, l_alpha, &.{ flash_q, head_dim }); - const beta_bc = broadcastToShape(k, beta, &.{ flash_q, head_dim }); - const l_next_safe_bc = broadcastToShape(k, l_next_safe, &.{ flash_q, head_dim }); - const out = k.divf(k.addf(k.mulf(l_alpha_bc, o_curr), k.mulf(beta_bc, qkv)), l_next_safe_bc); - const out_3d = k.shapeCast(out, &.{ num_q_per_blk, num_q_heads_per_kv_head, head_dim }); - maskedStoreAcc(k, a.acc_ref, out_3d, q_head_idx_idx, store_start, store_end); -} - -/// Emit the kernel body into a pre-opened `mtt.Builder`. The factory in -/// `zml/kernel/mosaic_tpu/mosaic_tpu.zig` opens the builder, calls this `run`, and finalizes the -/// IR with `b.finishOpts(&.{}, .{ .canonicalize = true })`. -pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { - const q_transform: []const mtt.ArgSpec.TransformReturn = &.{ - .{ .program_id = 1 }, - .{ .program_id = 0 }, - .zero, - }; - - const a = try b.declareArgsOpts(.{ - .heads_blk_idx = .{ .scalar = .i32 }, - .q_blk_idx = .{ .scalar = .i32 }, - - .kv_lens = .{ .ref = .{ .shape = &.{cfg.max_num_seqs}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, - .page_indices = .{ .ref = .{ .shape = &.{ cfg.max_num_seqs, cfg.pages_per_seq }, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, - .cu_q_lens = .{ .ref = .{ .shape = &.{cfg.max_num_seqs + 1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, - .seq_buf_idx = .{ .ref = .{ .shape = &.{2}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, - .num_seqs = .{ .ref = .{ .shape = &.{1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, - - .q = .{ .ref = .{ - .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, - .dtype = cfg.q_dtype, - .window = mtt.ArgSpec.WindowSpec{ .transform_returns = q_transform }, - } }, - .kv_pages = .{ .ref = .{ - .shape = &.{ cfg.total_num_pages, cfg.page_size, cfg.numCombinedKvHeads(), cfg.head_dim }, - .dtype = cfg.kv_dtype, - .memory_space = .any, - } }, - .o = .{ .ref = .{ - .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, - .dtype = cfg.q_dtype, - .role = .output, - .window = mtt.ArgSpec.WindowSpec{ .transform_returns = q_transform }, - } }, - - .kv_bufs = .{ .ref = .{ - .shape = &.{ 2, cfg.num_kv_pages_per_block, cfg.page_size, cfg.numCombinedKvHeadsPerBlk(), cfg.head_dim }, - .dtype = cfg.kv_dtype, - .role = .scratch, - } }, - .sems = .{ .sem_array = .{ .shape = &.{2}, .kind = .dma } }, - // Pallas hardcodes the trailing dim to 128 (TPU lane width); see - // `kernel.py:lm_scratch`. - .l_ref = .{ .ref = .{ .shape = &.{ cfg.numKvHeadsPerBlk(), cfg.flashQRows(), 128 }, .dtype = .f32, .role = .scratch } }, - .m_ref = .{ .ref = .{ .shape = &.{ cfg.numKvHeadsPerBlk(), cfg.flashQRows(), 128 }, .dtype = .f32, .role = .scratch } }, - .acc_ref = .{ .ref = .{ .shape = &.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, .dtype = .f32, .role = .scratch } }, - }, &.{}, .{ - .dimension_semantics = &.{ .arbitrary, .arbitrary }, - .iteration_bounds = &.{ cfg.numHeadsBlks(), cfg.numQBlks() }, - .scalar_prefetch = 5, - .scratch_operands = 5, - .pallas_window_params = true, - }); - - const k = b; - - const num_seqs = k.scalarLoad(a.num_seqs, &.{k.constIndex(0)}); - const init_seq_idx = k.scalarLoad(a.seq_buf_idx, &.{k.constIndex(0)}); - const init_buf_idx = k.scalarLoad(a.seq_buf_idx, &.{k.constIndex(1)}); - const q_len_start = k.muli(a.q_blk_idx, k.constant(.i32, cfg.numQPerBlk())); - const q_len_end = k.addi(q_len_start, k.constant(.i32, cfg.numQPerBlk())); - - var i = k.openIf(k.cmpi(.eq, k.addi(a.heads_blk_idx, a.q_blk_idx), k.constant(.i32, 0))); - { - const first_seq_idx = nextNonemptyKvSequence(k, cfg, a, init_seq_idx, num_seqs); - var nonempty = k.openIf(k.cmpi(.slt, first_seq_idx, num_seqs)); - { - createKvAsyncCopyDescriptors(k, cfg, a, a.heads_blk_idx, first_seq_idx, k.constant(.i32, 0), init_buf_idx); - nonempty.close(); - } - i.close(); - } - - // Empty rows skip flashAttention, so their accumulator must already be zero. - k.vectorStoreAt(a.acc_ref, k.zeros(&.{ cfg.numQPerBlk(), cfg.numQHeadsPerBlk(), cfg.head_dim }, .f32), &.{ k.constIndex(0), k.constIndex(0), k.constIndex(0) }); - - var q_loop = k.openWhile(.{ k.constant(.i32, 0), init_seq_idx, init_buf_idx }, .{ .i32, .i32, .i32 }); - { - const done = q_loop.before_carried[0]; - const cur_seq_idx = q_loop.before_carried[1]; - const cur_buf_idx = q_loop.before_carried[2]; - const cu_at_num_seqs = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(num_seqs)}); - const should_run = k.andi( - k.cmpi(.slt, q_len_start, cu_at_num_seqs), - k.cmpi(.slt, cur_seq_idx, num_seqs), - ); - const cond = k.andi(k.cmpi(.eq, done, k.constant(.i32, 0)), should_run); - q_loop.yieldBefore(cond, .{ done, cur_seq_idx, cur_buf_idx }); - } - { - const done = q_loop.after_carried[0]; - const cur_seq_idx = q_loop.after_carried[1]; - const cur_buf_idx = q_loop.after_carried[2]; - - const q_start = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(cur_seq_idx)}); - const q_end = k.scalarLoad(a.cu_q_lens, &.{k.toIndex(k.addi(cur_seq_idx, k.constant(.i32, 1)))}); - const q_len = k.subi(q_end, q_start); - const kv_len = k.scalarLoad(a.kv_lens, &.{k.toIndex(cur_seq_idx)}); - - var kv_loop = k.openWhile(.{ k.constant(.i32, 0), cur_buf_idx }, .{ .i32, .i32 }); - { - const kv_blk = kv_loop.before_carried[0]; - const buf = kv_loop.before_carried[1]; - const cond = k.cmpi(.slt, k.muli(kv_blk, k.constant(.i32, cfg.numKvPerBlk())), kv_len); - kv_loop.yieldBefore(cond, .{ kv_blk, buf }); - } - { - const kv_blk = kv_loop.after_carried[0]; - const cur_buf = kv_loop.after_carried[1]; - - const next_kv_blk_raw = k.addi(kv_blk, k.constant(.i32, 1)); - const is_last_kv = k.cmpi(.sge, k.muli(next_kv_blk_raw, k.constant(.i32, cfg.numKvPerBlk())), kv_len); - const next_kv_blk = k.select(is_last_kv, k.constant(.i32, 0), next_kv_blk_raw); - const cur_seq_end = k.cmpi(.sle, q_end, q_len_end); - const next_nonempty_seq = blk: { - // Only sequence transitions need to scan for empty KV rows. - var next_seq = k.openIfElse(k.andi(is_last_kv, cur_seq_end), .{.i32}); - { - const start_seq_idx = k.addi(cur_seq_idx, k.constant(.i32, 1)); - next_seq.yieldThen(.{nextNonemptyKvSequence(k, cfg, a, start_seq_idx, num_seqs)}); - } - { - next_seq.yieldElse(.{cur_seq_idx}); - } - break :blk next_seq.results[0]; - }; - const is_last_seq = k.cmpi(.eq, next_nonempty_seq, num_seqs); - const next_seq_idx = k.select(is_last_seq, k.constant(.i32, 0), next_nonempty_seq); - const next_heads_blk_idx = k.select(is_last_seq, k.addi(a.heads_blk_idx, k.constant(.i32, 1)), a.heads_blk_idx); - const next_buf = k.extui(k.cmpi(.eq, cur_buf, k.constant(.i32, 0)), .i32); - - var ii = k.openIf(k.cmpi(.slt, next_heads_blk_idx, k.constant(.i32, cfg.numHeadsBlks()))); - { - createKvAsyncCopyDescriptors(k, cfg, a, next_heads_blk_idx, next_seq_idx, next_kv_blk, next_buf); - ii.close(); - } - - waitKvDma(k, cfg, a, a.heads_blk_idx, cur_seq_idx, kv_blk, cur_buf); - - // for kv_head_chunk_idx in range(0, num_kv_heads_per_blk, kv_load_step) — unrolled at IR-emit time. - const num_kv_heads_per_blk: usize = @intCast(cfg.numKvHeadsPerBlk()); - const kv_load_step: usize = @intCast(cfg.kvLoadStep()); - var kv_head_chunk_idx: usize = 0; - while (kv_head_chunk_idx < num_kv_heads_per_blk) : (kv_head_chunk_idx += kv_load_step) { - const kv_pairs = stridedLoadKv(k, cfg, a, cur_buf, @intCast(kv_head_chunk_idx)); - var step_idx: usize = 0; - while (step_idx < kv_load_step) : (step_idx += 1) { - var kv_k_val = kv_pairs[step_idx].k; - var kv_v_val = kv_pairs[step_idx].v; - var k_input_dtype = cfg.kv_dtype; - var v_input_dtype = cfg.kv_dtype; - if (cfg.k_scale) |ks| { - const ks_splat = k.splat(ks, &.{ cfg.flashKRows(), cfg.head_dim }, .f32); - kv_k_val = k.mulf(kv_k_val.to(.f32), ks_splat).to(cfg.q_dtype); - k_input_dtype = cfg.q_dtype; - } - if (cfg.v_scale) |vs| { - const vs_splat = k.splat(vs, &.{ cfg.flashKRows(), cfg.head_dim }, .f32); - kv_v_val = k.mulf(kv_v_val.to(.f32), vs_splat).to(cfg.q_dtype); - v_input_dtype = cfg.q_dtype; - } - const kv_head_idx: i64 = @intCast(kv_head_chunk_idx + step_idx); - const q_head_idx = kv_head_idx * cfg.numQHeadsPerKvHead(); - - // `fold_on_2nd_minor` (kernel.py:255-262): when the - // 2nd-to-last dim is not a multiple of q's packing - // factor, cast q to f32 before the trailing-dim reshape - // so the layout pass can fold it cleanly. - var q_block = k.vectorLoadShape(a.q, &.{ k.constIndex(0), k.constIndex(q_head_idx), k.constIndex(0) }, &.{ cfg.numQPerBlk(), cfg.numQHeadsPerKvHead(), cfg.head_dim }); - if (@mod(cfg.numQHeadsPerKvHead(), cfg.qPackingFactor()) != 0) { - q_block = q_block.to(.f32); - } - const q_2d = k.shapeCast(q_block, &.{ cfg.flashQRows(), cfg.head_dim }); - flashAttention(k, cfg, a, q_2d, kv_k_val, kv_v_val, k_input_dtype, v_input_dtype, kv_head_idx, q_head_idx, kv_blk, kv_len, q_start, q_end, q_len, q_len_start); - } - } - - kv_loop.yieldAfter(.{ k.addi(kv_blk, k.constant(.i32, 1)), next_buf }); - } - const next_buf_idx = kv_loop.results[1]; - - const next_seq_idx_outer = k.select(k.cmpi(.sle, q_end, q_len_end), k.addi(cur_seq_idx, k.constant(.i32, 1)), cur_seq_idx); - const next_done = k.select(k.cmpi(.slt, q_end, q_len_end), done, k.constant(.i32, 1)); - q_loop.yieldAfter(.{ next_done, next_seq_idx_outer, next_buf_idx }); - } - const seq_idx_out = q_loop.results[1]; - const buf_idx_out = q_loop.results[2]; - - k.scalarStore(a.seq_buf_idx, k.select(k.cmpi(.slt, seq_idx_out, num_seqs), seq_idx_out, k.constant(.i32, 0)), &.{k.constIndex(0)}); - k.scalarStore(a.seq_buf_idx, buf_idx_out, &.{k.constIndex(1)}); - - k.vectorStoreAt(a.o, k.refLoad(a.acc_ref).to(cfg.q_dtype), &.{ k.constIndex(0), k.constIndex(0), k.constIndex(0) }); -} - -// Fixed drafter blocks can have query tokens without visible KV tokens. -// Keep their query positions, but never schedule a DMA for an empty KV row. -fn nextNonemptyKvSequence( - k: *mtt.Builder, - cfg: Cfg, - a: anytype, - start_seq_idx: mtt.Value, - num_seqs: mtt.Value, -) mtt.Value { - var loop = k.openWhile(.{start_seq_idx}, .{.i32}); - { - const seq_idx = loop.before_carried[0]; - // The terminal index can equal the metadata array length. - const safe_seq_idx = k.minsi(seq_idx, k.constant(.i32, cfg.max_num_seqs - 1)); - const kv_len = k.scalarLoad(a.kv_lens, &.{k.toIndex(safe_seq_idx)}); - const skip = k.andi( - k.cmpi(.slt, seq_idx, num_seqs), - k.cmpi(.sle, kv_len, k.constant(.i32, 0)), - ); - loop.yieldBefore(skip, .{seq_idx}); - } - { - loop.yieldAfter(.{k.addi(loop.after_carried[0], k.constant(.i32, 1))}); - } - return loop.results[0]; -} - -/// Open a builder, run the kernel body, and return the canonicalized MLIR IR. -/// Used by offline tooling (`examples/mosaic_ragged_paged`); production model -/// code goes through `zml.kernel.mosaic_tpu.Kernel(...).emit(...)` instead. -pub fn buildIr(allocator: std.mem.Allocator, ctx: *mlir.Context, cfg: Cfg) ![:0]const u8 { - var b = try mtt.Builder.open(allocator, ctx, "ragged_paged_attention_kernel"); - defer b.deinit(); - try run(&b, cfg); - return b.finishOpts(&.{}, .{ .canonicalize = true }); -} diff --git a/platforms/tpu/tpu.bzl b/platforms/tpu/tpu.bzl index 2fbc2a9b..fe111015 100644 --- a/platforms/tpu/tpu.bzl +++ b/platforms/tpu/tpu.bzl @@ -1,17 +1,17 @@ -load("@llvm//:http_bsdtar_archive.bzl", http_archive = "http_bsdtar_archive") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def _tpu_impl(mctx): # https://storage.googleapis.com/jax-releases/libtpu_releases.html http_archive( - name = "libzml_tpu", - url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu/libtpu-0.0.50.dev20261005+nightly-cp314-cp314t-manylinux_2_31_x86_64.whl", + name = "libpjrt_tpu", + url = "https://storage.googleapis.com/libtpu-nightly-releases/wheels/libtpu/libtpu-0.0.33.dev20251218+nightly-cp314-cp314t-manylinux_2_31_x86_64.whl", type = "zip", - sha256 = "a1e8615a33f60e1bfce15ec9d570642f882a8bec1d46763531e3b831edb2bb42", - build_file = "libzml_tpu.BUILD.bazel", + sha256 = "b7d9855c74a7fc56d11c4c7a2c55d5395c80594d43caec1549c69d5f908583cf", + build_file = "libpjrt_tpu.BUILD.bazel", ) return mctx.extension_metadata( reproducible = True, - root_module_direct_deps = ["libzml_tpu"], + root_module_direct_deps = ["libpjrt_tpu"], root_module_direct_dev_deps = [], ) diff --git a/platforms/tpu/tpu.zig b/platforms/tpu/tpu.zig index 3c8143bb..a9316699 100644 --- a/platforms/tpu/tpu.zig +++ b/platforms/tpu/tpu.zig @@ -3,14 +3,15 @@ const builtin = @import("builtin"); const bazel = @import("bazel"); const bazel_builtin = @import("bazel_builtin"); +const c = @import("c"); const pjrt = @import("pjrt"); -const platforms_options = @import("platforms/options"); +const runfiles = @import("runfiles"); const stdx = @import("stdx"); const log = std.log.scoped(.@"zml/platforms/tpu"); pub fn isEnabled() bool { - return platforms_options.tpu_enabled; + return @hasDecl(c, "ZML_RUNTIME_TPU"); } /// Check if running on Google Compute Engine, because TPUs will poll the @@ -33,7 +34,7 @@ fn isOnGCP(io: std.Io) !bool { } pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { - _ = allocator; + _ = allocator; // autofix if (comptime !isEnabled()) { return error.Unavailable; } @@ -47,14 +48,14 @@ pub fn load(allocator: std.mem.Allocator, io: std.Io) !*const pjrt.Api { const r = try bazel.runfiles(bazel_builtin.current_repository); var path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const sandbox_path = try r.rlocation("libzml_tpu/sandbox", &path_buf) orelse { + const sandbox_path = try r.rlocation("libpjrt_tpu/sandbox", &path_buf) orelse { log.err("Failed to find sandbox path for TPU runtime", .{}); return error.FileNotFound; }; return blk: { var lib_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libzml_tpu.so" }); + const path = try stdx.Io.Dir.path.bufJoinZ(&lib_path_buf, &.{ sandbox_path, "lib", "libpjrt_tpu.so" }); break :blk .loadFrom(path); }; } diff --git a/stdx/BUILD.bazel b/stdx/BUILD.bazel index d7c94d82..1dcfc56d 100644 --- a/stdx/BUILD.bazel +++ b/stdx/BUILD.bazel @@ -5,22 +5,20 @@ zig_library( srcs = [ "Io.zig", "bounded_array.zig", - "crypto.zig", "debug.zig", "flags.zig", "fmt.zig", - "cpu.zig", "json.zig", "math.zig", "meta.zig", "process.zig", "queue.zig", "segmented_list.zig", + "signature.zig", "time.zig", "unicode.zig", ], main = "stdx.zig", - tags = ["manual"], visibility = ["//visibility:public"], zigopts = ["-lc"], ) @@ -30,9 +28,3 @@ zig_test( testonly = False, deps = [":stdx"], ) - -zig_test( - name = "cpu_test", - main = "cpu.zig", - zigopts = ["-lc"], -) diff --git a/stdx/bounded_array.zig b/stdx/bounded_array.zig index 1f77a14d..bdee5bee 100644 --- a/stdx/bounded_array.zig +++ b/stdx/bounded_array.zig @@ -27,16 +27,12 @@ pub fn BoundedArray(comptime T: type, comptime buffer_capacity: usize) type { /// only known at runtime, but whose maximum size is known at comptime, without /// requiring an `Allocator`. /// ```zig -/// var a: BoundedArrayAligned(u8, 16, 2) = .empty; -/// try a.append(255); -/// try a.append(255); -/// const b: *const [1]u16 = @ptrCast(a.constSlice().ptr); -/// try testing.expectEqual(@as(u16, 65535), b[0]); +// var a = try BoundedArrayAligned(u8, 16, 2).init(0); +// try a.append(255); +// try a.append(255); +// const b = @ptrCast(*const [1]u16, a.constSlice().ptr); +// try testing.expectEqual(@as(u16, 65535), b[0]); /// ``` -/// -/// Note: there is no init anymore. It's generally unsafe because it makes `slice()` return undefined.; -/// If you used `try init(0)` use `.empty` -/// If you used `try init(n)` use `.{ .buffer = undefined, .len = n }` and fill `buffer` yourself. pub fn BoundedArrayAligned( comptime T: type, comptime alignment: Alignment, @@ -44,24 +40,14 @@ pub fn BoundedArrayAligned( ) type { return struct { const Self = @This(); - buffer: [buffer_capacity]T align(alignment.toByteUnits()), - len: usize, + buffer: [buffer_capacity]T align(alignment.toByteUnits()) = undefined, + len: usize = 0, - pub const max_len: usize = buffer_capacity; - - pub const empty: Self = .{ .buffer = undefined, .len = 0 }; - - pub fn init(m: []const T) Self { - var buf: Self = .{ .buffer = undefined, .len = m.len }; - @memcpy(buf.buffer[0..m.len], m); - return buf; - } - - pub fn repeat(value: T, len: usize) Self { - std.debug.assert(len <= buffer_capacity); - var buf: Self = .{ .buffer = undefined, .len = len }; - @memset(buf.buffer[0..len], value); - return buf; + /// Set the actual length of the slice. + /// Returns error.Overflow if it exceeds the length of the backing array. + pub fn init(len: usize) error{Overflow}!Self { + if (len > buffer_capacity) return error.Overflow; + return Self{ .len = len }; } /// View the internal array as a slice whose size was previously set. @@ -92,20 +78,19 @@ pub fn BoundedArrayAligned( /// Copy the content of an existing slice. pub fn fromSlice(m: []const T) error{Overflow}!Self { - if (m.len > buffer_capacity) return error.Overflow; - return .init(m); + var list = try init(m.len); + @memcpy(list.slice(), m); + return list; } /// Return the element at index `i` of the slice. pub fn get(self: Self, i: usize) T { - std.debug.assert(i < self.len); - return self.buffer[i]; + return self.constSlice()[i]; } /// Set the value of the element at index `i` of the slice. pub fn set(self: *Self, i: usize, item: T) void { - std.debug.assert(i < self.len); - self.buffer[i] = item; + self.slice()[i] = item; } /// Return the maximum length of a slice. @@ -288,6 +273,17 @@ pub fn BoundedArrayAligned( @memset(self.slice()[old_len..self.len], value); } + //pub const Writer = if (T != u8) + // @compileError("The Writer interface is only defined for BoundedArray(u8, ...) " ++ + // "but the given type is BoundedArray(" ++ @typeName(T) ++ ", ...)") + //else + // std.io.GenericWriter(*Self, error{Overflow}, appendWrite); + + ///// Initializes a writer which will write into the array. + //pub fn writer(self: *Self) Writer { + // return .{ .context = self }; + //} + /// Same as `appendSlice` except it returns the number of bytes written, which is always the same /// as `m.len`. The purpose of this function existing is to match `std.io.GenericWriter` API. fn appendWrite(self: *Self, m: []const u8) error{Overflow}!usize { @@ -298,7 +294,7 @@ pub fn BoundedArrayAligned( } test BoundedArray { - var a: BoundedArray(u8, 64) = .{ .buffer = undefined, .len = 32 }; + var a = try BoundedArray(u8, 64).init(32); try testing.expectEqual(a.capacity(), 64); try testing.expectEqual(a.slice().len, 32); @@ -307,8 +303,8 @@ test BoundedArray { try a.resize(48); try testing.expectEqual(a.len, 48); - const x: [10]u8 = @splat(1); - a = try .fromSlice(&x); + const x = [_]u8{1} ** 10; + a = try BoundedArray(u8, 64).fromSlice(&x); try testing.expectEqualSlices(u8, &x, a.constSlice()); var a2 = a; @@ -397,13 +393,14 @@ test BoundedArray { try testing.expectEqual(a.len, 36); while (a.pop()) |_| {} + const w = a.writer(); const s = "hello, this is a test string"; - _ = try a.appendWrite(s); + try w.writeAll(s); try testing.expectEqualStrings(s, a.constSlice()); } test "BoundedArrayAligned" { - var a: BoundedArrayAligned(u8, .@"16", 4) = .empty; + var a = try BoundedArrayAligned(u8, .@"16", 4).init(0); try a.append(0); try a.append(0); try a.append(255); diff --git a/stdx/cpu.zig b/stdx/cpu.zig deleted file mode 100644 index c51b32d4..00000000 --- a/stdx/cpu.zig +++ /dev/null @@ -1,411 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -/// Restricts the calling thread and existing threads to the CPUs of a NUMA -/// node within the caller's current affinity, preserving each thread's narrower -/// affinity. Threads with no overlap are left alone. Returns the number of -/// threads whose affinity changed. -/// Call during startup, before threads can concurrently create other threads. -/// On failure, previously changed affinities remain in effect. -pub fn pinProcessToNumaNode(io: std.Io, node: u32) !usize { - if (builtin.os.tag != .linux) return error.Unsupported; - - var path_buf: [64]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "/sys/devices/system/node/node{d}/cpulist", .{node}); - - var cpulist_buf: [4096]u8 = undefined; - const cpulist = try std.Io.Dir.cwd().readFile(io, path, &cpulist_buf); - - var set: std.os.linux.cpu_set_t = @splat(0); - try parseCpuList(cpulist, &set); - - return pinProcessToCpuSet(io, set); -} - -/// CPUs the calling thread may run on, ordered for worker pools. -pub const AllowedCores = struct { - /// Allowed CPU ids: one per physical core first, then the remaining SMT - /// siblings, round by round. - ids: []const usize, - /// `ids[0..physical_count]` holds exactly one CPU per physical core. Equals - /// `ids.len` when the topology is unknown. - physical_count: usize, - - /// One CPU per physical core, skipping SMT siblings. - pub fn physical(self: AllowedCores) []const usize { - return self.ids[0..self.physical_count]; - } - - pub fn deinit(self: AllowedCores, allocator: std.mem.Allocator) void { - allocator.free(self.ids); - } -}; - -/// Returns the CPUs the calling thread may run on. Linux uses sysfs topology so -/// that a second logical CPU of a physical core comes only after every allowed -/// physical core got one. -pub fn allowedCores(allocator: std.mem.Allocator, io: std.Io) !AllowedCores { - if (builtin.os.tag != .linux) return naturalOrder(allocator, try std.Thread.getCpuCount()); - - const cpus = linuxAllowedCoreIds(allocator) catch |err| switch (err) { - error.OutOfMemory => return err, - else => return naturalOrder(allocator, try std.Thread.getCpuCount()), - }; - errdefer allocator.free(cpus); - - const physical_count = linuxOrderBySiblings(allocator, io, cpus) catch |err| switch (err) { - error.OutOfMemory => return err, - else => cpus.len, - }; - return .{ .ids = cpus, .physical_count = physical_count }; -} - -fn pinProcessToCpuSet(io: std.Io, node_set: std.os.linux.cpu_set_t) !usize { - const linux = std.os.linux; - // Keep the caller's taskset/numactl choice as the process-wide limit, and - // intersect each thread's own affinity below so private bindings stay narrow. - const set = intersectCpuSets(node_set, try threadAffinity(0)); - if (std.mem.allEqual(usize, &set, 0)) return 0; - - var tasks = try std.Io.Dir.cwd().openDir(io, "/proc/self/task", .{ .iterate = true }); - defer tasks.close(io); - - // Pin the caller first so threads it creates inherit the target mask. - var pinned: usize = @intFromBool(try pinThreadToCpuSet(0, set)); - var it = tasks.iterate(); - while (try it.next(io)) |entry| { - const tid = std.fmt.parseInt(linux.pid_t, entry.name, 10) catch continue; - // Threads may exit while the list is being walked. - const changed = pinThreadToCpuSet(tid, set) catch |err| switch (err) { - error.ThreadGone => continue, - else => return err, - }; - pinned += @intFromBool(changed); - } - - return pinned; -} - -fn intersectCpuSets(a: std.os.linux.cpu_set_t, b: std.os.linux.cpu_set_t) std.os.linux.cpu_set_t { - var set = a; - for (&set, b) |*word, other| word.* &= other; - return set; -} - -fn threadAffinity(tid: std.os.linux.pid_t) error{ ThreadGone, GetAffinityFailed }!std.os.linux.cpu_set_t { - const linux = std.os.linux; - var set: linux.cpu_set_t = @splat(0); - switch (linux.errno(linux.sched_getaffinity(tid, @sizeOf(linux.cpu_set_t), &set))) { - .SUCCESS => return set, - .SRCH => return error.ThreadGone, - else => return error.GetAffinityFailed, - } -} - -fn setThreadAffinity(tid: std.os.linux.pid_t, set: *const std.os.linux.cpu_set_t) error{ ThreadGone, SetAffinityFailed }!void { - const linux = std.os.linux; - const rc = linux.syscall3(.sched_setaffinity, @as(usize, @bitCast(@as(isize, tid))), @sizeOf(linux.cpu_set_t), @intFromPtr(set)); - switch (linux.errno(rc)) { - .SUCCESS => {}, - .SRCH => return error.ThreadGone, - else => return error.SetAffinityFailed, - } -} - -fn pinThreadToCpuSet(tid: std.os.linux.pid_t, target: std.os.linux.cpu_set_t) !bool { - const current = try threadAffinity(tid); - const set = intersectCpuSets(target, current); - if (std.mem.allEqual(usize, &set, 0) or std.mem.eql(usize, &set, ¤t)) return false; - try setThreadAffinity(tid, &set); - return true; -} - -/// Parses Linux CPU lists, including singleton CPUs and inclusive ranges. -fn parseCpuList(cpulist: []const u8, set: *std.os.linux.cpu_set_t) !void { - const trimmed = std.mem.trim(u8, cpulist, " \t\r\n"); - if (trimmed.len == 0) return; - - const bits_per_word = @bitSizeOf(usize); - - var ranges = std.mem.splitScalar(u8, trimmed, ','); - while (ranges.next()) |raw_range| { - const range = std.mem.trim(u8, raw_range, " \t\r\n"); - var ends = std.mem.splitScalar(u8, range, '-'); - const first = std.fmt.parseInt(usize, ends.first(), 10) catch return error.InvalidCpuList; - - const last = if (ends.next()) |end| - std.fmt.parseInt(usize, end, 10) catch return error.InvalidCpuList - else - first; - - if (last < first or ends.next() != null or last >= set.len * bits_per_word) return error.InvalidCpuList; - for (first..last + 1) |cpu| { - set[cpu / bits_per_word] |= @as(usize, 1) << @intCast(cpu % bits_per_word); - } - } -} - -fn linuxAllowedCoreIds(allocator: std.mem.Allocator) ![]usize { - const set = try threadAffinity(0); - - var core_ids: std.ArrayList(usize) = .empty; - errdefer core_ids.deinit(allocator); - - var core_id: usize = 0; - while (core_id < @bitSizeOf(std.os.linux.cpu_set_t)) : (core_id += 1) { - if (cpuSetContains(set, core_id)) { - try core_ids.append(allocator, core_id); - } - } - - if (core_ids.items.len == 0) return error.MissingCpuTopology; - - return core_ids.toOwnedSlice(allocator); -} - -fn cpuSetContains(set: std.os.linux.cpu_set_t, core_id: usize) bool { - const word_bit_count = @bitSizeOf(usize); - const word_index = core_id / word_bit_count; - const bit_index: std.math.Log2Int(usize) = @intCast(core_id % word_bit_count); - return word_index < set.len and (set[word_index] & (@as(usize, 1) << bit_index)) != 0; -} - -fn linuxOrderBySiblings(allocator: std.mem.Allocator, io: std.Io, cpus: []usize) !usize { - const leaders = try allocator.alloc(usize, cpus.len); - defer allocator.free(leaders); - for (cpus, leaders) |cpu, *leader| leader.* = try linuxCoreLeader(io, cpu); - return orderBySiblings(allocator, cpus, leaders); -} - -/// Returns the lowest CPU id sharing a physical core with `cpu`. -fn linuxCoreLeader(io: std.Io, cpu: usize) !usize { - var cpulist_buf: [4096]u8 = undefined; - // `core_cpus_list` replaced `thread_siblings_list` in Linux 5.7. - const cpulist = linuxReadTopology(io, cpu, "core_cpus_list", &cpulist_buf) catch |err| switch (err) { - error.FileNotFound => try linuxReadTopology(io, cpu, "thread_siblings_list", &cpulist_buf), - else => return err, - }; - - var set: std.os.linux.cpu_set_t = @splat(0); - try parseCpuList(cpulist, &set); - for (set, 0..) |word, word_index| { - if (word != 0) return word_index * @bitSizeOf(usize) + @ctz(word); - } - return error.MissingCpuTopology; -} - -fn linuxReadTopology(io: std.Io, cpu: usize, name: []const u8, buf: []u8) ![]const u8 { - var path_buf: [96]u8 = undefined; - const path = try std.fmt.bufPrint(&path_buf, "/sys/devices/system/cpu/cpu{d}/topology/{s}", .{ cpu, name }); - return std.Io.Dir.cwd().readFile(io, path, buf); -} - -/// Reorders `cpus` so that each round holds at most one CPU per physical core, -/// in ascending order, before the next round of SMT siblings. `leaders[i]` -/// identifies the physical core of `cpus[i]`. Returns the number of physical -/// cores, which is also the size of the first round. -fn orderBySiblings(allocator: std.mem.Allocator, cpus: []usize, leaders: []const usize) !usize { - const Entry = struct { - round: usize, - cpu: usize, - - fn lessThan(_: void, lhs: @This(), rhs: @This()) bool { - return if (lhs.round != rhs.round) lhs.round < rhs.round else lhs.cpu < rhs.cpu; - } - }; - - const entries = try allocator.alloc(Entry, cpus.len); - defer allocator.free(entries); - - var physical_count: usize = 0; - for (entries, cpus, leaders, 0..) |*entry, cpu, leader, index| { - var round: usize = 0; - for (leaders[0..index]) |previous| round += @intFromBool(previous == leader); - entry.* = .{ .round = round, .cpu = cpu }; - physical_count += @intFromBool(round == 0); - } - - std.mem.sort(Entry, entries, {}, Entry.lessThan); - for (cpus, entries) |*cpu, entry| cpu.* = entry.cpu; - return physical_count; -} - -fn naturalOrder(allocator: std.mem.Allocator, cpu_count: usize) !AllowedCores { - const ids = try allocator.alloc(usize, cpu_count); - for (ids, 0..) |*id, index| id.* = index; - return .{ .ids = ids, .physical_count = ids.len }; -} - -test "CPU lists respect mask boundaries" { - var set: std.os.linux.cpu_set_t = @splat(0); - try parseCpuList("0-2,64,66-67\n", &set); - try std.testing.expectEqual(@as(usize, 0b111), set[0]); - try std.testing.expectEqual(@as(usize, 0b1101), set[1]); - const capacity = @bitSizeOf(std.os.linux.cpu_set_t); - var text: [32]u8 = undefined; - try parseCpuList(try std.fmt.bufPrint(&text, "{d}", .{capacity - 1}), &set); - try std.testing.expect((set[set.len - 1] & (@as(usize, 1) << (@bitSizeOf(usize) - 1))) != 0); - try std.testing.expectError(error.InvalidCpuList, parseCpuList(try std.fmt.bufPrint(&text, "{d}", .{capacity}), &set)); - try std.testing.expectError(error.InvalidCpuList, parseCpuList("3-1", &set)); - try std.testing.expectError(error.InvalidCpuList, parseCpuList("1-2-3", &set)); -} - -const AffinityTest = struct { - const CpuSet = std.os.linux.cpu_set_t; - - fn available(count: usize) !CpuSet { - if (builtin.os.tag != .linux) return error.SkipZigTest; - const current = try threadAffinity(0); - var selected: CpuSet = @splat(0); - var remaining = count; - for (0..@bitSizeOf(CpuSet)) |core_id| { - if (!cpuSetContains(current, core_id)) continue; - selected[core_id / @bitSizeOf(usize)] |= @as(usize, 1) << @intCast(core_id % @bitSizeOf(usize)); - remaining -= 1; - if (remaining == 0) break; - } - if (remaining != 0) return error.SkipZigTest; - return selected; - } - - fn check(caller: CpuSet, worker: CpuSet, node: CpuSet, expected: CpuSet) !void { - const io = std.testing.io; - const Worker = struct { - io: std.Io, - set: CpuSet, - tid: std.os.linux.pid_t = undefined, - ready: std.Io.Event = .unset, - stop: std.Io.Event = .unset, - failure: ?anyerror = null, - - fn run(self: *@This()) void { - self.tid = @intCast(std.Thread.getCurrentId()); - setThreadAffinity(0, &self.set) catch |err| { - self.failure = err; - }; - self.ready.set(self.io); - self.stop.waitUncancelable(self.io); - } - }; - var state: Worker = .{ .io = io, .set = worker }; - const thread = try std.Thread.spawn(.{}, Worker.run, .{&state}); - defer { - state.stop.set(io); - thread.join(); - } - try state.ready.wait(io); - if (state.failure) |err| return err; - - // Restore every thread touched by the process-wide helper, including - // any test runner IO workers, before the next test executes. - var saved: std.ArrayList(struct { tid: std.os.linux.pid_t, set: CpuSet }) = .empty; - defer { - for (saved.items) |entry| setThreadAffinity(entry.tid, &entry.set) catch |err| switch (err) { - error.ThreadGone => {}, - error.SetAffinityFailed => @panic("failed to restore test thread affinity"), - }; - saved.deinit(std.testing.allocator); - } - var tasks = try std.Io.Dir.cwd().openDir(io, "/proc/self/task", .{ .iterate = true }); - defer tasks.close(io); - var it = tasks.iterate(); - while (try it.next(io)) |entry| { - const tid = std.fmt.parseInt(std.os.linux.pid_t, entry.name, 10) catch continue; - const set = threadAffinity(tid) catch |err| switch (err) { - error.ThreadGone => continue, - else => return err, - }; - try saved.append(std.testing.allocator, .{ .tid = tid, .set = set }); - } - - try setThreadAffinity(0, &caller); - _ = try pinProcessToCpuSet(io, node); - try std.testing.expectEqualSlices(usize, &expected, &(try threadAffinity(state.tid))); - const expected_caller = intersectCpuSets(caller, node); - try std.testing.expectEqualSlices(usize, &expected_caller, &(try threadAffinity(0))); - } -}; - -test "process NUMA affinity preserves a narrower worker binding" { - const caller = try AffinityTest.available(3); - const node = try AffinityTest.available(2); - const worker = try AffinityTest.available(1); - try AffinityTest.check(caller, worker, node, worker); -} - -test "process NUMA affinity visits workers when the caller is already pinned" { - const caller = try AffinityTest.available(1); - const worker = try AffinityTest.available(2); - try AffinityTest.check(caller, worker, worker, caller); -} - -test "process NUMA affinity leaves a worker with no overlapping CPUs alone" { - const caller = try AffinityTest.available(2); - const node = try AffinityTest.available(1); - var worker = caller; - for (&worker, node) |*word, excluded| word.* &= ~excluded; - try AffinityTest.check(caller, worker, node, worker); -} - -test "CPU lists accept whitespace and empty node CPU sets" { - var set: std.os.linux.cpu_set_t = @splat(0); - try parseCpuList(" \t\r\n", &set); - try std.testing.expect(std.mem.allEqual(usize, &set, 0)); - try parseCpuList(" \t0-2, 64,66-67\r\n", &set); - try std.testing.expectEqual(@as(usize, 0b111), set[0]); - try std.testing.expectEqual(@as(usize, 0b1101), set[1]); -} - -test "CPU lists reject malformed ranges" { - for ([_][]const u8{ "3-1", "x", "-1", "1-", "1-2-3", "1,,2", "1,", ",1", "999999999999999999999999999999" }) |text| { - var set: std.os.linux.cpu_set_t = @splat(0); - try std.testing.expectError(error.InvalidCpuList, parseCpuList(text, &set)); - } -} - -test "natural order treats every CPU as physical" { - const cores = try naturalOrder(std.testing.allocator, 4); - defer cores.deinit(std.testing.allocator); - - try std.testing.expectEqualSlices(usize, &.{ 0, 1, 2, 3 }, cores.ids); - try std.testing.expectEqualSlices(usize, &.{ 0, 1, 2, 3 }, cores.physical()); -} - -test "sibling order puts one CPU per physical core first" { - const Case = struct { cpus: []const usize, leaders: []const usize, expected: []const usize, physical_count: usize }; - const cases = [_]Case{ - // Siblings numbered after all cores (typical Intel). - .{ .cpus = &.{ 0, 1, 2, 3 }, .leaders = &.{ 0, 1, 0, 1 }, .expected = &.{ 0, 1, 2, 3 }, .physical_count = 2 }, - // Adjacent siblings (typical AMD, some ARM). - .{ .cpus = &.{ 0, 1, 2, 3 }, .leaders = &.{ 0, 0, 2, 2 }, .expected = &.{ 0, 2, 1, 3 }, .physical_count = 2 }, - // Affinity allows only some siblings of cores {0,4}, {1,5}, {2,6}. - .{ .cpus = &.{ 0, 1, 2, 5 }, .leaders = &.{ 0, 1, 2, 1 }, .expected = &.{ 0, 1, 2, 5 }, .physical_count = 3 }, - // Affinity allows only the second sibling of some cores. - .{ .cpus = &.{ 1, 4, 5 }, .leaders = &.{ 1, 0, 1 }, .expected = &.{ 1, 4, 5 }, .physical_count = 2 }, - // No SMT. - .{ .cpus = &.{ 0, 1, 2 }, .leaders = &.{ 0, 1, 2 }, .expected = &.{ 0, 1, 2 }, .physical_count = 3 }, - }; - - for (cases) |case| { - const cpus = try std.testing.allocator.dupe(usize, case.cpus); - defer std.testing.allocator.free(cpus); - - const physical_count = try orderBySiblings(std.testing.allocator, cpus, case.leaders); - try std.testing.expectEqualSlices(usize, case.expected, cpus); - try std.testing.expectEqual(case.physical_count, physical_count); - } -} - -test "allowed cores cover the thread affinity" { - const cores = try allowedCores(std.testing.allocator, std.testing.io); - defer cores.deinit(std.testing.allocator); - - try std.testing.expect(cores.physical_count > 0 and cores.physical_count <= cores.ids.len); - if (builtin.os.tag != .linux) return; - - const affinity = try threadAffinity(0); - var expected_count: usize = 0; - for (affinity) |word| expected_count += @popCount(word); - try std.testing.expectEqual(expected_count, cores.ids.len); - for (cores.ids) |cpu| try std.testing.expect(cpuSetContains(affinity, cpu)); -} diff --git a/stdx/crypto.zig b/stdx/crypto.zig deleted file mode 100644 index eb5729b6..00000000 --- a/stdx/crypto.zig +++ /dev/null @@ -1,41 +0,0 @@ -const std = @import("std"); - -pub fn MacWriter(comptime T: type) type { - return struct { - const Self = @This(); - - mac: *T, - interface: std.Io.Writer, - - pub fn init(mac_: *T) MacWriter(T) { - return .{ - .mac = mac_, - .interface = .{ - .buffer = &.{}, - .vtable = &.{ - .drain = drain, - }, - }, - }; - } - - pub fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - const self: *Self = @alignCast(@fieldParentPtr("interface", w)); - var total: usize = 0; - for (data) |chunk| { - self.mac.update(chunk); - total += chunk.len; - } - const last = data[data.len - 1]; - for (0..splat - 1) |_| { - self.mac.update(last); - total += last.len; - } - return total; - } - }; -} - -pub fn hmacWriter(hmac: anytype) MacWriter(std.meta.Child(@TypeOf(hmac))) { - return .init(hmac); -} diff --git a/stdx/debug.zig b/stdx/debug.zig index a217eca1..4686b3b5 100644 --- a/stdx/debug.zig +++ b/stdx/debug.zig @@ -10,12 +10,11 @@ pub inline fn internalAssert(check: bool, comptime msg: []const u8, args: anytyp pub inline fn assert(check: bool, comptime msg: []const u8, args: anytype) void { if (!check) { - @branchHint(.cold); panic(msg, args); } } -pub fn panic(comptime format: []const u8, args: anytype) noreturn { +pub inline fn panic(comptime format: []const u8, args: anytype) noreturn { std.debug.panic(format, args); } diff --git a/stdx/flags.zig b/stdx/flags.zig index d0c4352d..9a0f32ae 100644 --- a/stdx/flags.zig +++ b/stdx/flags.zig @@ -93,9 +93,8 @@ pub fn parseProcessArgs(init_: std.process.Init.Minimal, comptime CliArgs: type) } fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Commands { - comptime assert(@typeInfo(Commands) == .@"union"); - const field_names = std.meta.fieldNames(Commands); - comptime assert(field_names.len >= 2); + comptime assert(@typeInfo(Commands) == .Union); + comptime assert(std.meta.fields(Commands).len >= 2); const first_arg = args.next() orelse fatal( "subcommand required, expected {s}", @@ -105,15 +104,15 @@ fn parse_commands(args: *std.process.Args.Iterator, comptime Commands: type) Com // NB: help must be declared as *pub* const to be visible here. if (@hasDecl(Commands, "help")) { if (std.mem.eql(u8, first_arg, "-h") or std.mem.eql(u8, first_arg, "--help")) { - std.debug.print(Commands.help, .{}); + std.io.getStdOut().writeAll(Commands.help) catch std.process.exit(1); std.process.exit(0); } } - inline for (field_names, std.meta.fieldTypes(Commands)) |field_name, FieldType| { - comptime assert(std.mem.indexOf(u8, field_name, "_") == null); - if (std.mem.eql(u8, first_arg, field_name)) { - return @unionInit(Commands, field_name, parse_flags(args, FieldType)); + inline for (comptime std.meta.fields(Commands)) |field| { + comptime assert(std.mem.indexOf(u8, field.name, "_") == null); + if (std.mem.eql(u8, first_arg, field.name)) { + return @unionInit(Commands, field.name, parse_flags(args, field.type)); } } fatal("unknown subcommand: '{s}'", .{first_arg}); @@ -129,61 +128,52 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { return {}; } - const flags_info = @typeInfo(Flags).@"struct"; + assert(@typeInfo(Flags) == .@"struct"); - const total_num_fields = flags_info.field_names.len; - comptime var field_names: [total_num_fields][]const u8 = undefined; - comptime var field_types: [total_num_fields]type = undefined; - comptime var field_defaults: [total_num_fields]?*const anyopaque = undefined; + comptime var fields: [std.meta.fields(Flags).len]std.builtin.Type.StructField = undefined; comptime var field_count = 0; - const positional_infos: ?std.lang.Type.Struct = if (@hasField(Flags, "positional")) - @typeInfo(@FieldType(Flags, "positional")).@"struct" - else - null; + comptime var positional_fields: []const std.builtin.Type.StructField = &.{}; - comptime for (flags_info.field_names, flags_info.field_types, flags_info.field_attrs) |field_name, FieldType, field_attr| { - if (std.mem.eql(u8, field_name, "positional")) { + comptime for (std.meta.fields(Flags)) |field| { + if (std.mem.eql(u8, field.name, "positional")) { + assert(@typeInfo(field.type) == .@"struct"); + positional_fields = std.meta.fields(field.type); var optional_tail = false; - for (positional_infos.?.field_types, positional_infos.?.field_attrs) |PosType, pos_attr| { - const default: ?PosType = pos_attr.defaultValue(PosType); - if (default == null) { + for (positional_fields) |positional_field| { + if (default_value(positional_field) == null) { if (optional_tail) @panic("optional positional arguments must be last"); } else { optional_tail = true; } - switch (@typeInfo(PosType)) { + switch (@typeInfo(positional_field.type)) { .optional => |optional| { // optional flags should have a default - assert(default != null); - assert(default.? == null); + assert(default_value(positional_field) != null); + assert(default_value(positional_field).? == null); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(PosType); + assert_valid_value_type(positional_field.type); }, } } } else { - field_names[field_count] = field_name; - field_types[field_count] = FieldType; - field_defaults[field_count] = field_attr.default_value_ptr; + fields[field_count] = field; field_count += 1; - const default: ?FieldType = field_attr.defaultValue(FieldType); - - switch (@typeInfo(FieldType)) { + switch (@typeInfo(field.type)) { .bool => { // boolean flags should have a default - debug.assertComptime(default != null and default.? == false, "boolean flag --{s} should default to false", .{field_name}); + debug.assertComptime(default_value(field) != null and default_value(field).? == false, "boolean flag --{s} should default to false", .{field.name}); }, .optional => |optional| { // optional flags should have a default - debug.assertComptime(default != null and default.? == null, "optional flag --{s} should have a null default value", .{field_name}); + debug.assertComptime(default_value(field) != null and default_value(field).? == null, "optional flag --{s} should have a null default value", .{field.name}); assert_valid_value_type(optional.child); }, else => { - assert_valid_value_type(FieldType); + assert_valid_value_type(field.type); }, } } @@ -192,33 +182,29 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { var result: Flags = undefined; // Would use std.enums.EnumFieldStruct(Flags, u32, 0) here but Flags is a struct not an Enum. var counts = comptime blk: { - const count_field_types: [total_num_fields]type = @splat(u32); - const count_field_attrs: [total_num_fields]std.builtin.Type.Struct.FieldAttributes = @splat(.{ - .@"align" = @alignOf(u32), - .default_value_ptr = @ptrCast(&@as(u32, 0)), - }); - break :blk @Struct(.auto, null, flags_info.field_names, &count_field_types, &count_field_attrs){}; + const f = std.meta.fields(Flags); + var count_field_names: [fields.len][]const u8 = undefined; + var count_field_types: [fields.len]type = undefined; + var count_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (f, &count_field_names, &count_field_types, &count_field_attrs) |field, *field_name, *field_type, *field_attr| { + field_name.* = field.name; + field_type.* = u32; + field_attr.* = .{ + .@"align" = @alignOf(u32), + .default_value_ptr = @ptrCast(&@as(u32, 0)), + }; + } + break :blk @Struct(.auto, null, &count_field_names, &count_field_types, &count_field_attrs){}; }; // When parsing arguments, we must consider longer arguments first, such that `--foo-bar=92` is // not confused for a misspelled `--foo=92`. Using `std.sort` for comptime-only values does not // work, so open-code insertion sort, and comptime assert order during the actual parsing. comptime { - for ( - 0.., - field_names[0..field_count], - field_types[0..field_count], - field_defaults[0..field_count], - ) |i, *name_right, *type_right, *default_right| { - for ( - field_names[0..i], - field_types[0..i], - field_defaults[0..i], - ) |*name_left, *type_left, *default_left| { - if (name_left.len < name_right.len) { - std.mem.swap([]const u8, name_left, name_right); - std.mem.swap(type, type_left, type_right); - std.mem.swap(?*const anyopaque, default_left, default_right); + for (fields[0..field_count], 0..) |*field_right, i| { + for (fields[0..i]) |*field_left| { + if (field_left.name.len < field_right.name.len) { + std.mem.swap(std.builtin.Type.StructField, field_left, field_right); } } } @@ -226,38 +212,30 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { var parsed_positional = false; next_arg: while (args.next()) |arg| { - // NB: help must be declared as *pub* const to be visible here. - if (@hasDecl(Flags, "help")) { - if (std.mem.eql(u8, arg, "-h") or std.mem.eql(u8, arg, "--help")) { - std.debug.print(Flags.help, .{}); - std.process.exit(0); - } - } - comptime var field_len_prev = std.math.maxInt(usize); - inline for (field_names[0..field_count], field_types[0..field_count]) |field_name, FieldType| { - const flag = comptime flag_name(field_name); + inline for (fields[0..field_count]) |field| { + const flag = comptime flag_name(field); - comptime assert(field_len_prev >= field_name.len); - field_len_prev = field_name.len; + comptime assert(field_len_prev >= field.name.len); + field_len_prev = field.name.len; if (std.mem.startsWith(u8, arg, flag)) { if (parsed_positional) { fatal("unexpected trailing option: '{s}'", .{arg}); } - @field(counts, field_name) += 1; - const flag_value = parse_flag(FieldType, flag, arg); - @field(result, field_name) = flag_value; + @field(counts, field.name) += 1; + const flag_value = parse_flag(field.type, flag, arg); + @field(result, field.name) = flag_value; continue :next_arg; } } - if (positional_infos) |pos_infos| { + if (@hasField(Flags, "positional")) { counts.positional += 1; switch (counts.positional - 1) { - inline 0...pos_infos.field_names.len - 1 => |positional_index| { - const positional_field_name = pos_infos.field_names[positional_index]; - const flag = comptime flag_name_positional(positional_field_name); + inline 0...positional_fields.len - 1 => |positional_index| { + const positional_field = positional_fields[positional_index]; + const flag = comptime flag_name_positional(positional_field); if (arg.len == 0) fatal("{s}: empty argument", .{flag}); // Prevent ambiguity between a flag and positional argument value. We could add @@ -266,8 +244,8 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { if (arg[0] == '-') fatal("unexpected argument: '{s}'", .{arg}); parsed_positional = true; - @field(result.positional, positional_field_name) = - parse_value(pos_infos.field_types[positional_index], flag, arg); + @field(result.positional, positional_field.name) = + parse_value(positional_field.type, flag, arg); continue :next_arg; }, else => {}, // Fall-through to the unexpected argument error. @@ -277,15 +255,11 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { fatal("unexpected argument: '{s}'", .{arg}); } - inline for ( - field_names[0..field_count], - field_types[0..field_count], - field_defaults[0..field_count], - ) |field_name, FieldType, default_ptr| { - const flag = flag_name(field_name); - switch (@field(counts, field_name)) { - 0 => if (defaultValue(FieldType, default_ptr)) |default| { - @field(result, field_name) = default; + inline for (fields[0..field_count]) |field| { + const flag = flag_name(field); + switch (@field(counts, field.name)) { + 0 => if (default_value(field)) |default| { + @field(result, field.name) = default; } else { fatal("{s}: argument is required", .{flag}); }, @@ -294,19 +268,15 @@ fn parse_flags(args: *std.process.Args.Iterator, comptime Flags: type) Flags { } } - if (positional_infos) |pos_infos| { - assert(counts.positional <= pos_infos.field_names.len); - inline for ( - 0.., - pos_infos.field_names, - pos_infos.field_types, - pos_infos.field_attrs, - ) |positional_index, name, PosFieldType, attr| { + if (@hasField(Flags, "positional")) { + assert(counts.positional <= positional_fields.len); + inline for (positional_fields, 0..) |positional_field, positional_index| { if (positional_index >= counts.positional) { - if (attr.defaultValue(PosFieldType)) |default| { - @field(result.positional, name) = default; + const flag = comptime flag_name_positional(positional_field); + if (default_value(positional_field)) |default| { + @field(result.positional, positional_field.name) = default; } else { - fatal("<{s}>: argument is required", .{name}); + fatal("{s}: argument is required", .{flag}); } } } @@ -321,8 +291,8 @@ fn assert_valid_value_type(comptime T: type) void { if (@typeInfo(T) == .@"enum") { const info = @typeInfo(T).@"enum"; - assert(info.mode == .exhaustive); - assert(info.field_names.len >= 2); + assert(info.is_exhaustive); + assert(info.fields.len >= 2); return; } @@ -471,7 +441,7 @@ pub const ByteSize = struct { else ByteUnit.bytes; - _ = std.math.mul(u64, amount, @backingInt(unit)) catch { + _ = std.math.mul(u64, amount, @intFromEnum(unit)) catch { return ByteSizeParseError.BytesOverflow; }; @@ -482,7 +452,7 @@ pub const ByteSize = struct { return std.math.mul( u64, size.value, - @backingInt(size.unit), + @intFromEnum(size.unit), ) catch unreachable; } @@ -564,7 +534,7 @@ fn parse_value_float(comptime T: type, flag: []const u8, value: [:0]const u8) T fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { assert((flag[0] == '-' and flag[1] == '-') or flag[0] == '<'); - comptime assert(@typeInfo(E).@"enum".mode == .exhaustive); + comptime assert(@typeInfo(E).@"enum".is_exhaustive); return std.meta.stringToEnum(E, value) orelse fatal( "{s}: expected one of {s}, but found '{s}'", @@ -574,44 +544,53 @@ fn parse_value_enum(comptime E: type, flag: []const u8, value: [:0]const u8) E { fn fields_to_comma_list(comptime E: type) []const u8 { comptime { - const field_names = std.meta.fieldNames(E); - const field_count = field_names.len; + const field_count = std.meta.fields(E).len; assert(field_count >= 2); var result: []const u8 = ""; - for (field_names, 0..) |field_name, field_index| { + for (std.meta.fields(E), 0..) |field, field_index| { const separator = switch (field_index) { 0 => "", else => ", ", field_count - 1 => if (field_count == 2) " or " else ", or ", }; - result = result ++ separator ++ "'" ++ field_name ++ "'"; + result = result ++ separator ++ "'" ++ field.name ++ "'"; } return result; } } -/// Maps a field name to its CLI flag, e.g. `model_path` to `--model-path`. -pub fn flag_name(comptime field_name: []const u8) []const u8 { - comptime assert(!std.mem.eql(u8, field_name, "positional")); - comptime var name = field_name[0..field_name.len].*; - comptime std.mem.replaceScalar(u8, &name, '_', '-'); - const dashed = name; - return "--" ++ dashed; +pub fn flag_name(comptime field: std.builtin.Type.StructField) []const u8 { + // TODO(Zig): Cleanup when this is fixed after Zig 0.11. + // Without comptime blk, the compiler thinks the result is a runtime slice returning a UAF. + return comptime blk: { + assert(!std.mem.eql(u8, field.name, "positional")); + + var result: []const u8 = "--"; + var index = 0; + while (std.mem.indexOf(u8, field.name[index..], "_")) |i| { + result = result ++ field.name[index..][0..i] ++ "-"; + index = index + i + 1; + } + result = result ++ field.name[index..]; + break :blk result; + }; } test flag_name { - try std.testing.expectEqualStrings("--statsd", flag_name("statsd")); - try std.testing.expectEqualStrings("--model-path", flag_name("model_path")); - try std.testing.expectEqualStrings("--a-b-c", flag_name("a_b_c")); + const field = @typeInfo(struct { statsd: bool }).@"struct".fields[0]; + try std.testing.expectEqualStrings(flag_name(field), "--statsd"); } -fn flag_name_positional(comptime field_name: []const u8) []const u8 { - comptime assert(std.mem.indexOf(u8, field_name, "_") == null); - return "<" ++ field_name ++ ">"; +fn flag_name_positional(comptime field: std.builtin.Type.StructField) []const u8 { + comptime assert(std.mem.indexOf(u8, field.name, "_") == null); + return "<" ++ field.name ++ ">"; } -pub fn defaultValue(comptime FieldType: type, default_value_ptr: ?*const anyopaque) ?FieldType { - const dp: *const FieldType = @ptrCast(@alignCast(default_value_ptr orelse return null)); - return dp.*; +/// This is essentially `field.default_value`, but with a useful type instead of `?*anyopaque`. +pub fn default_value(comptime field: std.builtin.Type.StructField) ?field.type { + return if (field.default_value_ptr) |default_opaque| + @as(*const field.type, @ptrCast(@alignCast(default_opaque))).* + else + null; } diff --git a/stdx/fmt.zig b/stdx/fmt.zig index a62d4f7d..ec3a0343 100644 --- a/stdx/fmt.zig +++ b/stdx/fmt.zig @@ -5,16 +5,6 @@ pub fn slice(any_slice: anytype) FmtSlice(std.meta.Elem(@TypeOf(any_slice))) { return .{ .slice = any_slice }; } -/// Properly format a slice of []const u8. -pub fn strings(strs: []const []const u8) FmtStrings { - return .{ .strings = strs }; -} - -/// Properly format a slice of [*:0]const u8. -pub fn stringsZ(strs: []const [*:0]const u8) FmtZeroStrings { - return .{ .strings = strs }; -} - fn FmtSlice(T: type) type { return struct { slice: []const T, @@ -41,58 +31,21 @@ fn FmtSlice(T: type) type { }; } -pub const FmtStrings = struct { - strings: []const []const u8, - - pub fn format(f: FmtStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return try formatSliceCustom(formatStr, f.strings, .{}, 1, writer); - } - - pub fn formatNumber(f: FmtStrings, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - return try formatIntSlice(f.strings, n, 1, writer); - } -}; - -pub const FmtZeroStrings = struct { - strings: []const [*:0]const u8, - - pub fn format(f: FmtZeroStrings, writer: *std.Io.Writer) std.Io.Writer.Error!void { - return try formatSliceCustom(formatStrZ, f.strings, .{}, 1, writer); - } -}; - pub fn formatFloat(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { const x = switch (@typeInfo(@TypeOf(value))) { .@"struct" => value.toF32(), .float => value, else => @compileError("formatFloat expects a float, got: " ++ @typeName(@TypeOf(value))), }; - switch (@typeInfo(@TypeOf(x))) { - .float => try writer.printFloat(x, spec), - // Handle F4.toF32() being a [2]f32 - .array => inline for (0.., x) |i, xi| { - try writer.printFloat(xi, spec); - if (i < x.len - 1) try writer.writeByte(','); - }, - else => @compileError("formatFloat expects a float, got: " ++ @typeName(@TypeOf(value))), - } + return writer.printFloat(x, spec); } pub fn formatInt(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { - return switch (@typeInfo(@TypeOf(value))) { - .int => try writer.printInt(value, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }), - .vector => { - const vector = @typeInfo(@TypeOf(value)).vector; - const array: [vector.len]vector.child = value; - for (&array, 0..) |elem, i| { - try writer.printInt(elem, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }); - if (i < array.len - 1) { - try writer.writeAll(", "); - } - } - }, + switch (@typeInfo(@TypeOf(value))) { + .int => {}, else => @compileError("formatInt expects an int, got: " ++ @typeName(@TypeOf(value))), - }; + } + return writer.printInt(value, spec.mode.base().?, spec.case, .{ .alignment = spec.alignment, .fill = spec.fill }); } pub fn formatComplex(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { @@ -107,24 +60,10 @@ pub fn formatBool(value: bool, spec: std.fmt.Number, writer: *std.Io.Writer) !vo try writer.alignBufferOptions(if (value) "1" else "0", .{ .alignment = spec.alignment, .fill = spec.fill }); } -pub fn formatStr(value: []const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { - try writer.writeAll(value); -} - -pub fn formatStrZ(value: [*:0]const u8, _: std.fmt.Number, writer: *std.Io.Writer) !void { - try writer.writeAll(std.mem.span(value)); -} - pub fn formatAny(value: anytype, spec: std.fmt.Number, writer: *std.Io.Writer) !void { var buf: [48]u8 = undefined; const T = @TypeOf(value); - const fmt = switch (@typeInfo(T)) { - .@"struct", .@"union", .@"enum", .@"opaque" => if (@hasDecl(T, "formatNumber")) - "{d}" - else - "{f}", - else => "{f}", - }; + const fmt = if (@hasDecl(T, "formatNumber")) "{d}" else "{f}"; const s = std.fmt.bufPrint(&buf, fmt, .{value}) catch blk: { buf[45..].* = "...".*; @@ -222,18 +161,30 @@ fn printValue( switch (@typeInfo(T)) { .@"struct" => |info| { try w.writeAll(".{ "); - inline for (info.field_names, 0..) |field_name, i| { + inline for (info.fields, 0..) |f, i| { if (i > 0) try w.writeAll(", "); if (!info.is_tuple) { try w.writeByte('.'); - try w.writeAll(field_name); + try w.writeAll(f.name); try w.writeAll(" = "); } - try printValue(w, options, @field(value, field_name), max_depth - 1); + try printValue(w, options, @field(value, f.name), max_depth - 1); } try w.writeAll(" }"); }, inline else => try w.printValue("any", options, value, max_depth - 1), } } + +pub const FormatDuration = struct { + inner: std.Io.Duration, + + pub fn formatDuration(self: FormatDuration, w: *std.Io.Writer) std.Io.Writer.Error!void { + try w.printDurationSigned(@intCast(self.inner.toNanoseconds())); + } +}; + +pub fn fmtDuration(ts: std.Io.Duration) FormatDuration { + return .{ .inner = ts }; +} diff --git a/stdx/json.zig b/stdx/json.zig index c1f71a4a..c99cfb00 100644 --- a/stdx/json.zig +++ b/stdx/json.zig @@ -17,23 +17,22 @@ const ParseFromValueError = std.json.ParseFromValueError; /// /// ```zig /// const Answer = union { -/// number: i32, -/// numbers: []const i32, -/// -/// const Helpers = stdx.json.UnionHelpers(@This()); -/// pub const jsonParse = Helpers.jsonParse; -/// pub const jsonParseFromValue = Helpers.jsonParseFromValue; -/// pub const jsonStringify = Helpers.jsonStringify; +/// number: i32, +/// numbers: []const i32, /// }; /// /// const Message = struct { /// question: []const u8; -/// answer: Answer; +/// answer: stdx.json.Union(Answer); /// } /// ``` -pub fn UnionHelpers(comptime T: type) type { +pub fn Union(comptime T: type) type { return struct { - pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!T { + const Self = @This(); + + value: T, + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { return jsonParseFromValue( allocator, try std.json.innerParse(std.json.Value, allocator, source, options), @@ -41,23 +40,21 @@ pub fn UnionHelpers(comptime T: type) type { ); } - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!T { - inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { - switch (Field) { - bool => if (source == .bool) return @unionInit(T, field_name, source.bool), + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { + inline for (std.meta.fields(T)) |field| { + switch (field.type) { + bool => if (source == .bool) return .{ .value = @unionInit(T, field.name, source.bool) }, []const u8 => switch (source) { - .string, .number_string => |v| { - const dupe = try allocator.dupe(u8, v); - return @unionInit(T, field_name, dupe); - }, + .string => |v| return .{ .value = @unionInit(T, field.name, v) }, + .number_string => |v| return .{ .value = @unionInit(T, field.name, v) }, else => {}, }, - else => switch (@typeInfo(Field)) { - .int => if (source == .integer) return @unionInit(T, field_name, @intCast(source.integer)), - .float => if (source == .float) return @unionInit(T, field_name, @floatCast(source.float)), - .@"struct" => if (source == .object) return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), + else => switch (@typeInfo(field.type)) { + .int => if (source == .integer) return .{ .value = @unionInit(T, field.name, @intCast(source.integer)) }, + .float => if (source == .float) return .{ .value = @unionInit(T, field.name, @floatCast(source.float)) }, + .@"struct" => if (source == .object) return .{ .value = @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source.object, options)) }, inline else => switch (source) { - .number_string, .array => return @unionInit(T, field_name, try std.json.innerParseFromValue(Field, allocator, source, options)), + .number_string, .array => return .{ .value = @unionInit(T, field.name, try std.json.innerParseFromValue(field.type, allocator, source, options)) }, else => {}, }, }, @@ -65,36 +62,6 @@ pub fn UnionHelpers(comptime T: type) type { } return error.UnknownField; } - - pub fn jsonStringify(self: T, jw: anytype) !void { - return switch (self) { - inline else => |v| jw.write(v), - }; - } - }; -} - -/// Deprecated: prefer importing `stdx.json.UnionHelpers` method into your union. -/// This will remove the nesting required by `stdx.json.Union`. -pub fn Union(comptime T: type) type { - return struct { - const Self = @This(); - - value: T, - - const Helpers = UnionHelpers(T); - - pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { - return .{ .value = try Helpers.jsonParse(allocator, source, options) }; - } - - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { - return .{ .value = try Helpers.jsonParseFromValue(allocator, source, options) }; - } - - pub fn jsonStringify(self: Self, jw: anytype) !void { - return try Helpers.jsonStringify(self.value, jw); - } }; } @@ -114,31 +81,19 @@ pub fn Union(comptime T: type) type { /// /// ```zig /// const Entry = union { -/// faq: struct { question: []const u8, answer: u32 }, -/// address: struct { city: []const u8, zipcode: []const u8 }, -/// -/// const Helpers = stdx.json.TaggedUnionHelpers(@This(), "type"); -/// pub const jsonParse = Helpers.jsonParse; -/// pub const jsonParseFromValue = Helpers.jsonParseFromValue; -/// pub const jsonStringify = Helpers.jsonStringify; +/// faq: struct { question: []const u8, answer: u32 }, +/// address: struct { city: []const u8, zipcode: []const u8 }, /// }; /// -/// const Message = []const Entry; +/// const Message = []const stdx.json.TaggedUnion(Entry, "type"); /// ``` -pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) type { - comptime { - if (@typeInfo(T) != .@"union") { - @compileError("TaggedUnion expects a union type, found " ++ @typeName(T)); - } - for (std.meta.fieldNames(T), std.meta.fieldTypes(T)) |field_name, Field| { - if (@typeInfo(Field) != .@"struct") { - @compileError("TaggedUnion member '" ++ field_name ++ "' must be a struct, found " ++ @typeName(Field)); - } - } - } - +pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { return struct { - pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!T { + const Self = @This(); + + value: T, + + pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { return jsonParseFromValue( allocator, try std.json.innerParse(std.json.Value, allocator, source, options), @@ -146,66 +101,27 @@ pub fn TaggedUnionHelpers(comptime T: type, comptime tag_name: [:0]const u8) typ ); } - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!T { + pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { errdefer std.log.warn("failed to parse: {} as {s}", .{ source, @typeName(T) }); if (source != .object) return error.UnexpectedToken; - var o = source.object; - const tag = (o.fetchSwapRemove(tag_name) orelse return error.MissingField).value; + const o = source.object; + const tag = o.get(tag_name) orelse return error.MissingField; + for (o.keys(), o.values()) |k, v| { + std.log.warn("object['{s}'] = {}", .{ k, v }); + } if (tag != .string) return error.LengthMismatch; - inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { - if (std.mem.eql(u8, field_name, tag.string)) { - const inner: Field = std.json.parseFromValueLeaky(Field, allocator, .{ .object = o }, options) catch |err| { - std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(Field), err }); + inline for (std.meta.fields(T)) |field| { + if (std.mem.eql(u8, field.name, tag.string)) { + const inner_source = o.get(field.name) orelse return error.MissingField; + const inner: field.type = std.json.innerParseFromValue(field.type, allocator, inner_source, options) catch |err| { + std.log.warn("failed to interpret {s} as a {s}: {}", .{ tag.string, @typeName(field.type), err }); return err; }; - return @unionInit(T, field_name, inner); + return .{ .value = @unionInit(T, field.name, inner) }; } } return error.InvalidEnumTag; } - - pub fn jsonStringify(self: T, jw: anytype) !void { - try jw.beginObject(); - switch (self) { - inline else => |v, tag| { - // emit a field containing the enum tag. - try jw.objectField(tag_name); - try jw.write(@tagName(tag)); - switch (@typeInfo(@TypeOf(v))) { - .@"struct" => inline for (comptime std.meta.fieldNames(@TypeOf(v))) |name| { - try jw.objectField(name); - try jw.write(@field(v, name)); - }, - else => unreachable, // Would have failed with comptime check. - } - }, - } - try jw.endObject(); - } - }; -} - -/// Deprecated: prefer importing `stdx.json.TaggedUnionHelpers` method into your union. -/// This will remove the nesting required by `stdx.json.TaggedUnion`. -pub fn TaggedUnion(comptime T: type, comptime tag_name: [:0]const u8) type { - return struct { - const Self = @This(); - - value: T, - - const Helpers = TaggedUnionHelpers(T, tag_name); - - pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Self { - return .{ .value = try Helpers.jsonParse(allocator, source, options) }; - } - - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) ParseFromValueError!Self { - return .{ .value = try Helpers.jsonParseFromValue(allocator, source, options) }; - } - - pub fn jsonStringify(self: Self, jw: anytype) !void { - return try Helpers.jsonStringify(self.value, jw); - } }; } @@ -222,155 +138,16 @@ pub fn NeverNull(comptime T: type, comptime default_value: T) type { pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) !Self { return .{ .value = (try std.json.innerParseFromValue(?T, allocator, source, options)) orelse default_value }; } - - pub fn jsonStringify(self: Self, jw: anytype) !void { - try jw.write(self.value); - } }; } pub fn fillDefaultStructValues(comptime T: type, r: *T) !void { - const info = @typeInfo(T).@"struct"; - inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, attrs| { - if (attrs.defaultValue(Field)) |default| { - if (@field(r, field_name) == null) { - @field(r, field_name) = default; + inline for (@typeInfo(T).Struct.fields) |field| { + if (field.default_value) |default_ptr| { + if (@field(r, field.name) == null) { + const default = @as(*align(1) const field.type, @ptrCast(default_ptr)).*; + @field(r, field.name) = default; } } } } - -test "union" { - const Answer = union(enum) { - number: i32, - numbers: []const i32, - flag: bool, - ratio: f64, - text: []const u8, - - const Helpers = UnionHelpers(@This()); - pub const jsonParse = Helpers.jsonParse; - pub const jsonParseFromValue = Helpers.jsonParseFromValue; - pub const jsonStringify = Helpers.jsonStringify; - }; - - const Message = struct { - question: []const u8, - answer: Answer, - }; - - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - - const number_case = "{\"question\":\"How old are you?\",\"answer\":5}"; - const parsed_number = try std.json.parseFromSliceLeaky(Message, allocator, number_case, .{}); - try std.testing.expectEqualStrings("How old are you?", parsed_number.question); - try std.testing.expectEqual(@as(i32, 5), parsed_number.answer.number); - try std.testing.expectFmt(number_case, "{f}", .{std.json.fmt(parsed_number, .{})}); - - const numbers_case = "{\"question\":\"Count to three.\",\"answer\":[1,2,3]}"; - const parsed_numbers = try std.json.parseFromSliceLeaky(Message, allocator, numbers_case, .{}); - try std.testing.expectEqual(@as(usize, 3), parsed_numbers.answer.numbers.len); - try std.testing.expectEqual(@as(i32, 1), parsed_numbers.answer.numbers[0]); - try std.testing.expectEqual(@as(i32, 2), parsed_numbers.answer.numbers[1]); - try std.testing.expectEqual(@as(i32, 3), parsed_numbers.answer.numbers[2]); - try std.testing.expectFmt(numbers_case, "{f}", .{std.json.fmt(parsed_numbers, .{})}); - - const flag_case = "{\"question\":\"Are you ready?\",\"answer\":true}"; - const parsed_flag = try std.json.parseFromSliceLeaky(Message, allocator, flag_case, .{}); - try std.testing.expectEqual(true, parsed_flag.answer.flag); - try std.testing.expectFmt(flag_case, "{f}", .{std.json.fmt(parsed_flag, .{})}); - - const ratio_case = "{\"question\":\"What is pi?\",\"answer\":3.5}"; - const parsed_ratio = try std.json.parseFromSliceLeaky(Message, allocator, ratio_case, .{}); - try std.testing.expectEqual(@as(f64, 3.5), parsed_ratio.answer.ratio); - try std.testing.expectFmt(ratio_case, "{f}", .{std.json.fmt(parsed_ratio, .{})}); - - const text_case = "{\"question\":\"Your name?\",\"answer\":\"zml\"}"; - const parsed_text = try std.json.parseFromSliceLeaky(Message, allocator, text_case, .{}); - try std.testing.expectEqualStrings("zml", parsed_text.answer.text); - try std.testing.expectFmt(text_case, "{f}", .{std.json.fmt(parsed_text, .{})}); - - const null_value = try std.json.parseFromSliceLeaky(std.json.Value, allocator, "null", .{}); - try std.testing.expectError(error.UnknownField, Answer.jsonParseFromValue(allocator, null_value, .{})); -} - -test "strings in union can be freed" { - const Answer = union(enum) { - string: []const u8, - number: u32, - - const Helpers = UnionHelpers(@This()); - pub const jsonParse = Helpers.jsonParse; - pub const jsonParseFromValue = Helpers.jsonParseFromValue; - pub const jsonStringify = Helpers.jsonStringify; - }; - const Struct = struct { - answer1: Answer, - answer2: Answer, - }; - const struct_json = - \\{"answer1":"I'm 29","answer2": 1e29} - ; - var parsed_value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, struct_json, .{ .parse_numbers = false }); - defer parsed_value.deinit(); - - const parsed = try std.json.parseFromValueLeaky(Struct, std.testing.allocator, parsed_value.value, .{}); - - try std.testing.expect(parsed.answer1 == .string); - try std.testing.expectEqualStrings("I'm 29", parsed.answer1.string); - std.testing.allocator.free(parsed.answer1.string); - - try std.testing.expect(parsed.answer2 == .string); - try std.testing.expectEqualStrings("1e29", parsed.answer2.string); - std.testing.allocator.free(parsed.answer2.string); -} - -test "tagged union" { - const Message = union(enum) { - faq: struct { - question: []const u8, - answer: u32, - }, - address: struct { - city: []const u8, - zipcode: []const u8, - }, - - const Helpers = TaggedUnionHelpers(@This(), "type"); - pub const jsonParse = Helpers.jsonParse; - pub const jsonParseFromValue = Helpers.jsonParseFromValue; - pub const jsonStringify = Helpers.jsonStringify; - }; - - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - - const faq_case = "{\"type\":\"faq\",\"question\":\"How old are you?\",\"answer\":5}"; - const parsed_faq = try std.json.parseFromSliceLeaky(Message, allocator, faq_case, .{}); - try std.testing.expectEqualStrings("How old are you?", parsed_faq.faq.question); - try std.testing.expectEqual(@as(u32, 5), parsed_faq.faq.answer); - try std.testing.expectFmt(faq_case, "{f}", .{std.json.fmt(parsed_faq, .{})}); - - const address_case = "{\"type\":\"address\",\"city\":\"NYC\",\"zipcode\":\"49130\"}"; - const parsed_address = try std.json.parseFromSliceLeaky(Message, allocator, address_case, .{}); - try std.testing.expectEqualStrings("NYC", parsed_address.address.city); - try std.testing.expectEqualStrings("49130", parsed_address.address.zipcode); - try std.testing.expectFmt(address_case, "{f}", .{std.json.fmt(parsed_address, .{})}); -} - -test "never null" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - - const non_null = try std.json.parseFromSliceLeaky(NeverNull(i32, 10), allocator, "3", .{}); - try std.testing.expectEqual(@as(i32, 3), non_null.value); - try std.testing.expectFmt("3", "{f}", .{std.json.fmt(non_null, .{})}); - - const was_null = try std.json.parseFromSliceLeaky(NeverNull(i32, 10), allocator, "null", .{}); - try std.testing.expectEqual(@as(i32, 10), was_null.value); - try std.testing.expectFmt("10", "{f}", .{std.json.fmt(was_null, .{})}); -} diff --git a/stdx/meta.zig b/stdx/meta.zig index 9b1c05af..9a096122 100644 --- a/stdx/meta.zig +++ b/stdx/meta.zig @@ -2,6 +2,8 @@ const std = @import("std"); const debug = @import("debug.zig"); const compileError = debug.compileError; +pub const FnSignature = @import("signature.zig").FnSignature; +pub const Signature = @import("signature.zig").Signature; pub fn isStruct(comptime T: type) bool { return switch (@typeInfo(T)) { @@ -20,8 +22,8 @@ pub fn isTuple(comptime T: type) bool { pub fn isStructOf(comptime T: type, comptime Elem: type) bool { return switch (@typeInfo(T)) { .@"struct" => |info| blk: { - inline for (info.field_types) |FieldType| { - if (FieldType != Elem) { + inline for (info.fields) |field| { + if (field.type != Elem) { break :blk false; } } @@ -34,8 +36,8 @@ pub fn isStructOf(comptime T: type, comptime Elem: type) bool { pub fn isStructOfAny(comptime T: type, comptime f: fn (comptime type) bool) bool { return switch (@typeInfo(T)) { .@"struct" => |info| blk: { - inline for (info.field_types) |FieldType| { - if (f(FieldType) == false) { + inline for (info.fields) |field| { + if (f(field.type) == false) { break :blk false; } } @@ -121,9 +123,9 @@ pub fn asSlice(comptime T: type) type { } pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize) type { - const all_field_types = std.meta.fieldTypes(T); + const fields = std.meta.fields(T); const start_ = start orelse 0; - const end_ = end orelse all_field_types.len; + const end_ = end orelse fields.len; if (start_ == end_) { return @Tuple(&.{}); @@ -131,80 +133,49 @@ pub fn TupleRange(comptime T: type, comptime start: ?usize, comptime end: ?usize var field_types: [end_ - start_]type = undefined; inline for (start_..end_, 0..) |i, j| { - field_types[j] = all_field_types[i]; + field_types[j] = fields[i].type; } return @Tuple(&field_types); } pub fn FnParam(comptime func: anytype, comptime n: comptime_int) type { - const params = @typeInfo(@TypeOf(func)).@"fn".param_types; - if (n >= params.len) { - @compileError("param doesn't exist in func"); - } - return params[n] orelse @compileError("anytype is not supported"); + return @typeInfo(@TypeOf(func)).@"fn".params[n].type orelse @compileError("anytype is not supported"); } -pub fn FnReturn(comptime func: anytype) type { +pub fn FnArgs(comptime func: anytype) type { + debug.assertComptime(!@typeInfo(@TypeOf(func)).@"fn".is_generic, "FnArgs expects non generic function, got: {}", .{@TypeOf(func)}); + return FnSignature(func, null).ArgsT; +} + +pub fn FnArgsWithHint(comptime func: anytype, ArgsT: type) type { + debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".is_generic, "FnArgsWithHint expects a generic function, got: {}", .{@TypeOf(func)}); + return FnSignature(func, ArgsT).ArgsT; +} + +pub fn FnResult(comptime func: anytype) type { return @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); } -pub fn FnReturnPayload(comptime func: anytype) type { - const rt = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); - return switch (@typeInfo(rt)) { - .error_union => |u| u.payload, - else => rt, - }; -} - -pub fn FnReturnErrorSet(comptime func: anytype) ?type { - const rt = @typeInfo(@TypeOf(func)).@"fn".return_type orelse @compileError("anytype is not supported"); - return switch (@typeInfo(rt)) { - .error_union => |u| u.error_set, - else => null, - }; -} - -pub fn Head(comptime Tuple: type) type { +pub fn Head(Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { - if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); - return struct_info.field_types[0]; + if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); + return struct_info.fields[0].type; }, else => @compileError("Head works on tuple type"), }; } -pub fn Tail(comptime Tuple: type) type { +pub fn Tail(Tuple: type) type { return switch (@typeInfo(Tuple)) { .@"struct" => |struct_info| { - if (struct_info.field_types.len == 0) @compileError("Can't tail empty tuple"); - var types: [struct_info.field_types.len - 1]type = undefined; - for (struct_info.field_types[1..], &types) |FieldType, *type_| { - type_.* = FieldType; + if (struct_info.fields.len == 0) @compileError("Can't tail empty tuple"); + var types: [struct_info.fields.len - 1]type = undefined; + for (struct_info.fields[1..], &types) |field, *type_| { + type_.* = field.type; } return @Tuple(&types); }, else => @compileError("Tail works on tuple type"), }; } - -pub fn fnName(comptime func: anytype) []const u8 { - const T = @TypeOf(func); - std.debug.assert(@typeInfo(T) == .@"fn"); - const wrapper_name = @typeName(Wrapper(func)); - - var it = std.mem.splitSequence(u8, wrapper_name, ".Wrapper((function '"); - _ = it.first(); - const rest = it.next().?; - return rest[0 .. rest.len - 3]; -} - -fn Wrapper(comptime arg: anytype) type { - return struct { - const stuff = arg; - }; -} - -test fnName { - try std.testing.expectEqualStrings("Tail", fnName(Tail)); -} diff --git a/stdx/process.zig b/stdx/process.zig index 91cf5586..434a5bf7 100644 --- a/stdx/process.zig +++ b/stdx/process.zig @@ -15,10 +15,10 @@ fn selfSharedObjectPathImpl(addr: usize) []const u8 { return std.mem.span(info.dli_fname); } -pub noinline fn selfSharedObjectPath() []const u8 { +pub fn selfSharedObjectPath() []const u8 { return selfSharedObjectPathImpl(@returnAddress()); } -pub noinline fn selfSharedObjectDirPath() []const u8 { +pub fn selfSharedObjectDirPath() []const u8 { return std.fs.path.dirname(selfSharedObjectPathImpl(@returnAddress())).?; } diff --git a/stdx/segmented_list.zig b/stdx/segmented_list.zig index 567534be..ee115ce2 100644 --- a/stdx/segmented_list.zig +++ b/stdx/segmented_list.zig @@ -107,7 +107,7 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type pub const prealloc_count = prealloc_item_count; fn AtType(comptime SelfType: type) type { - if (@typeInfo(SelfType).pointer.attrs.@"const") { + if (@typeInfo(SelfType).pointer.is_const) { return *const T; } else { return *T; diff --git a/stdx/signature.zig b/stdx/signature.zig new file mode 100644 index 00000000..955a1887 --- /dev/null +++ b/stdx/signature.zig @@ -0,0 +1,59 @@ +const std = @import("std"); + +const compileError = @import("debug.zig").compileError; + +pub fn ArgsTuple(comptime funcT: anytype, comptime ArgsT: ?type) type { + const params = @typeInfo(funcT).@"fn".params; + if (params.len == 0) { + return @TypeOf(.{}); + } + + if (@typeInfo(funcT).@"fn".is_generic == false) { + return std.meta.ArgsTuple(funcT); + } + + const args = std.meta.fields(ArgsT orelse @compileError("generic function requires an explicit ArgsTuple")); + var tuple_types: [params.len]type = undefined; + if (params.len != args.len) { + compileError("function {} expected {} args, got {}", .{ funcT, params.len, args.len }); + } + inline for (params, args, 0..) |param, arg, i| { + if (param.type == null) { + tuple_types[i] = arg.type; + continue; + } + tuple_types[i] = param.type.?; + } + + return @Tuple(tuple_types); +} + +pub const Signature = struct { + Func: type, + FuncT: type, + ArgsT: type, + ReturnT: type, + ReturnPayloadT: type, + ReturnErrorSet: ?type, +}; + +pub fn FnSignature(comptime func: anytype, comptime argsT_: ?type) Signature { + const argsT = ArgsTuple(@TypeOf(func), argsT_); + const return_type = @TypeOf(@call(.auto, func, @as(argsT, undefined))); + return Signature{ + .Func = struct { + pub const Value = func; + }, + .FuncT = @TypeOf(func), + .ArgsT = argsT, + .ReturnT = return_type, + .ReturnPayloadT = switch (@typeInfo(return_type)) { + .error_union => |u| u.payload, + else => return_type, + }, + .ReturnErrorSet = switch (@typeInfo(return_type)) { + .error_union => |u| u.error_set, + else => null, + }, + }; +} diff --git a/stdx/stdx.zig b/stdx/stdx.zig index 1b19e638..83e14277 100644 --- a/stdx/stdx.zig +++ b/stdx/stdx.zig @@ -3,8 +3,6 @@ const builtin = @import("builtin"); pub const BoundedArray = @import("bounded_array.zig").BoundedArray; pub const BoundedArrayAligned = @import("bounded_array.zig").BoundedArrayAligned; -pub const cpu = @import("cpu.zig"); -pub const crypto = @import("crypto.zig"); pub const debug = @import("debug.zig"); pub const flags = @import("flags.zig"); pub const fmt = @import("fmt.zig"); @@ -28,22 +26,13 @@ pub inline fn stackSlice(comptime max_len: usize, T: type, len: usize) []T { return storage[0..len]; } -pub const noalloc: std.mem.Allocator = if (builtin.mode == .fast) undefined else std.testing.failing_allocator; - -pub fn arenaWithCapacity(parent: std.mem.Allocator, initial_capacity: usize) std.mem.Allocator.Error!std.heap.ArenaAllocator { - var a: std.heap.ArenaAllocator = .init(parent); - - _ = try a.allocator().alloc(u8, initial_capacity); - std.debug.assert(a.state.used_list.?.end_index == initial_capacity); - a.state.used_list.?.end_index = 0; - return a; -} +pub const noalloc: std.mem.Allocator = if (builtin.mode == .ReleaseFast) undefined else std.testing.failing_allocator; pub fn pinToCore(core_id: usize) void { if (builtin.os.tag == .linux) { const CPUSet = std.bit_set.ArrayBitSet(usize, std.os.linux.CPU_SETSIZE * @sizeOf(usize)); - var set: CPUSet = .empty; + var set: CPUSet = .initEmpty(); set.set(core_id); std.os.linux.sched_setaffinity(0, @ptrCast(&set.masks)) catch {}; } @@ -86,60 +75,3 @@ pub fn Once(comptime f: anytype) type { } }; } - -/// Packed storage of sub-byte values -/// -/// Note: before Zig 0.17, @Vector(u2, 4) was always one byte and could be used as a storage type, -/// But that's not true anymore. -pub fn Packed(T: type) type { - return switch (@bitSizeOf(T)) { - 2 => packed struct(u8) { - x: T, - y: T, - z: T, - w: T, - - const P = @This(); - pub fn pack(x: T, y: T, z: T, w: T) P { - return .{ .x = x, .y = y, .z = z, .w = w }; - } - - pub fn unpack(p: P) [4]T { - return .{ p.x, p.y, p.z, p.w }; - } - - pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - try formatPackedNumber(T, &p.unpack(), w, n); - } - }, - 4 => packed struct(u8) { - x: T, - y: T, - - const P = @This(); - pub fn pack(x: T, y: T) P { - return .{ .x = x, .y = y }; - } - - pub fn unpack(p: P) [2]T { - return .{ p.x, p.y }; - } - - pub fn formatNumber(p: P, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - try formatPackedNumber(T, &p.unpack(), w, n); - } - }, - else => debug.compileError("stdx.Packed expects a bitsize of 2 or 4, got type {} with bitsize {d}", .{ T, @bitSizeOf(T) }), - }; -} - -fn formatPackedNumber(T: type, values: []const T, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - for (0.., values) |i, val| { - if (i > 0) try w.writeByte(','); - switch (@typeInfo(T)) { - .@"struct" => try val.formatNumber(w, n), - .int => try fmt.formatInt(val, n, w), - else => try w.print("{}", .{val}), - } - } -} diff --git a/third_party/abseil-cpp/bmi2intrin.patch b/third_party/abseil-cpp/bmi2intrin.patch deleted file mode 100644 index 8273731b..00000000 --- a/third_party/abseil-cpp/bmi2intrin.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h -index 0d05ad7..85ed32d 100644 ---- a/absl/container/internal/raw_hash_set.h -+++ b/absl/container/internal/raw_hash_set.h -@@ -226,7 +226,7 @@ static_assert(std::numeric_limits::is_iec559, - #endif - - #ifdef __BMI2__ --#include -+#include - #endif // __BMI2__ - - #if defined(__SSE2__) || defined(__aarch64__) || \ \ No newline at end of file diff --git a/third_party/arocc/arocc.bazel b/third_party/arocc/arocc.bazel index a5de1d98..3e3c62bd 100644 --- a/third_party/arocc/arocc.bazel +++ b/third_party/arocc/arocc.bazel @@ -1,89 +1,54 @@ -load("@bazel_skylib//rules:run_binary.bzl", "run_binary") +load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") -load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") -zig_option( - name = "enable_linker_build_id", - bool = False, +write_file( + name = "system_defaults_zig", + out = "system_defaults.zig", + content = ["""\ +pub const enable_linker_build_id: bool = false; +pub const linker: []const u8 = ""; +pub const sysroot: []const u8 = ""; +pub const gcc_install_prefix: []const u8 = ""; +pub const rtlib: []const u8 = ""; +pub const unwindlib: []const u8 = ""; +"""], ) -zig_option( - name = "linker", - string = "", -) - -zig_option( - name = "sysroot", - string = "", -) - -zig_option( - name = "gcc_install_prefix", - string = "", -) - -zig_option( - name = "rtlib", - string = "", -) - -zig_option( - name = "unwindlib", - string = "", -) - -zig_options( +zig_library( name = "system_defaults", - settings = [ - ":enable_linker_build_id", - ":linker", - ":sysroot", - ":gcc_install_prefix", - ":rtlib", - ":unwindlib", - ], + main = ":system_defaults_zig", ) -zig_option( - name = "enable_tracy", - bool = False, +write_file( + name = "aro_options_zig", + out = "aro_options.zig", + content = ["""\ +pub const enable_tracy: bool = false; +pub const enable_tracy_callstack: bool = false; +pub const enable_tracy_allocation: bool = false; +pub const debug_allocations: bool = false; +pub const version_str: []const u8 = ""; +"""], ) -zig_option( - name = "enable_tracy_callstack", - bool = False, -) - -zig_option( - name = "enable_tracy_allocation", - bool = False, -) - -zig_option( - name = "debug_allocations", - bool = False, -) - -zig_option( - name = "version_str", - string = "", -) - -zig_options( +zig_library( name = "build_options", - settings = [ - ":enable_tracy", - ":enable_tracy_callstack", - ":enable_tracy_allocation", - ":debug_allocations", - ":version_str", - ], + main = ":aro_options_zig", +) + +zig_library( + name = "zig", + srcs = glob(["deps/zig/**/*.zig"]), + main = "deps/zig/lib.zig", ) zig_library( name = "backend", main = "src/backend.zig", - deps = [":build_options"], + deps = [ + ":build_options", + ":zig", + ], ) zig_library( @@ -118,6 +83,7 @@ zig_binary( ":assembly_backend", ":build_options", ], + zigopts = ["-fm"], ) zig_binary( @@ -126,15 +92,12 @@ zig_binary( ) [ - run_binary( + genrule( name = "def_{}".format(name), srcs = [name], outs = ["{}.zig".format(name)], - args = [ - "$(location {})".format(name), - "$(location {}.zig)".format(name), - ], - tool = ":generate-def", + cmd = """$(location :generate-def) $< $@""", + tools = [":generate-def"], ) for name in glob(["src/aro/**/*.def"]) ] diff --git a/third_party/arocc/repo.bzl b/third_party/arocc/repo.bzl index 15ff6d7f..510445d1 100644 --- a/third_party/arocc/repo.bzl +++ b/third_party/arocc/repo.bzl @@ -1,10 +1,9 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") def repo(): - # https://github.com/zml/arocc/releases/tag/0.17.0 - git_repository( + new_git_repository( name = "arocc", - remote = "https://github.com/zml/arocc", - commit = "d0c8c4d9c55daa7ef6e40cf0f630a5b5e900989b", - build_file = Label("//third_party/arocc:arocc.bazel"), + remote = "https://github.com/zml/arocc.git", + commit = "73acf3aa8164cc8fee16b0f79036b92b07dab053", + build_file = "//third_party/arocc:arocc.bazel", ) diff --git a/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel b/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel deleted file mode 100644 index 563999d2..00000000 --- a/third_party/cloud_accelerator_diagnostics/cloud_accelerator_diagnostics.BUILD.bazel +++ /dev/null @@ -1,34 +0,0 @@ -load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") -load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") -load("@grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") - -proto_library( - name = "tpu_telemetry_proto", - srcs = ["tpu_telemetry.proto"], - import_prefix = "tpu_info/proto", - visibility = ["//visibility:public"], -) - -proto_library( - name = "tpu_metric_service_proto", - srcs = ["tpu_metric_service.proto"], - deps = [ - ":tpu_telemetry_proto", - "@com_google_protobuf//:timestamp_proto", - ], - visibility = ["//visibility:public"], -) - -cc_proto_library( - name = "tpu_metric_service_cc_proto", - deps = [":tpu_metric_service_proto"], - visibility = ["//visibility:public"], -) - -cc_grpc_library( - name = "tpu_metric_service_cc_grpc", - srcs = [":tpu_metric_service_proto"], - grpc_only = True, - deps = [":tpu_metric_service_cc_proto"], - visibility = ["//visibility:public"], -) diff --git a/third_party/cloud_accelerator_diagnostics/repo.bzl b/third_party/cloud_accelerator_diagnostics/repo.bzl deleted file mode 100644 index 661e45ec..00000000 --- a/third_party/cloud_accelerator_diagnostics/repo.bzl +++ /dev/null @@ -1,29 +0,0 @@ -_COMMIT = "6eccaf47364904d1c55209dd8795e3bf9cac21e1" -_URL_PREFIX = "https://raw.githubusercontent.com/AI-Hypercomputer/cloud-accelerator-diagnostics/" + _COMMIT + "/tpu_info/tpu_info/proto/" - -_PROTOS = { - "tpu_metric_service.proto": "a89d80938c75177ff45fb4c6130f5d71bc5164df473a77948e31b7dedd96a1bb", - "tpu_telemetry.proto": "4ad6096e40de5ba00b6c7badea1218c801069b82be829ec56dc0ce693b5a2505", -} - -def _impl(ctx): - for output, sha256 in _PROTOS.items(): - ctx.download( - url = _URL_PREFIX + output, - sha256 = sha256, - output = output, - ) - ctx.template("BUILD.bazel", ctx.attr.build_file) - -_cloud_accelerator_diagnostics = repository_rule( - implementation = _impl, - attrs = { - "build_file": attr.label(mandatory = True), - }, -) - -def repo(): - _cloud_accelerator_diagnostics( - name = "cloud_accelerator_diagnostics", - build_file = "//third_party/cloud_accelerator_diagnostics:cloud_accelerator_diagnostics.BUILD.bazel", - ) diff --git a/third_party/com_google_sentencepiece/BUILD.bazel b/third_party/com_google_sentencepiece/BUILD.bazel index 598dce0b..53e12d3a 100644 --- a/third_party/com_google_sentencepiece/BUILD.bazel +++ b/third_party/com_google_sentencepiece/BUILD.bazel @@ -28,12 +28,12 @@ cc_library( includes = ["."], visibility = ["//visibility:public"], deps = [ - "@com_google_absl//absl/container:flat_hash_map", - "@com_google_absl//absl/container:flat_hash_set", - "@com_google_absl//absl/flags:flag", - "@com_google_absl//absl/flags:parse", - "@com_google_absl//absl/flags:usage", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:string_view", + "@abseil-cpp//absl/container:flat_hash_map", + "@abseil-cpp//absl/container:flat_hash_set", + "@abseil-cpp//absl/flags:flag", + "@abseil-cpp//absl/flags:parse", + "@abseil-cpp//absl/flags:usage", + "@abseil-cpp//absl/strings", + "@abseil-cpp//absl/strings:string_view", ], ) diff --git a/third_party/cuda_tile/cuda_tile.bazel b/third_party/cuda_tile/cuda_tile.bazel deleted file mode 100644 index bfa768e9..00000000 --- a/third_party/cuda_tile/cuda_tile.bazel +++ /dev/null @@ -1,450 +0,0 @@ -# XLA's third_party/cuda_tile/cuda_tile.BUILD, plus the C API library it does -# not build (cuda_tile_capi at the bottom). -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") -load("@rules_cc//cc:cc_binary.bzl", "cc_binary") -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -cc_binary( - name = "cuda-tile-tblgen", - srcs = glob([ - "tools/cuda-tile-tblgen/*.cpp", - "tools/cuda-tile-tblgen/*.h", - ]), - deps = [ - "@llvm-project//llvm:Support", - "@llvm-project//llvm:TableGen", - "@llvm-project//mlir:MlirTableGenMain", - "@llvm-project//mlir:Support", - "@llvm-project//mlir:TableGen", - ], -) - -exports_files(["LICENSE"]) - -td_library( - name = "CudaTileTdFiles", - srcs = glob(["include/cuda_tile/Dialect/CudaTile/IR/*.td"]), - includes = ["include"], - deps = [ - "@llvm-project//mlir:BuiltinDialectTdFiles", - "@llvm-project//mlir:ControlFlowInterfacesTdFiles", - "@llvm-project//mlir:FunctionInterfacesTdFiles", - "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - "@llvm-project//mlir:ViewLikeInterfaceTdFiles", - ], -) - -gentbl_cc_library( - name = "CudaTileDialectIncGen", - tbl_outs = [ - ( - [ - "-gen-dialect-decls", - "-dialect=cuda_tile", - ], - "include/cuda_tile/Dialect/CudaTile/IR/Dialect.h.inc", - ), - ( - [ - "-gen-dialect-defs", - "-dialect=cuda_tile", - ], - "include/cuda_tile/Dialect/CudaTile/IR/Dialect.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/IR/Dialect.td", - deps = [":CudaTileTdFiles"], -) - -gentbl_cc_library( - name = "CudaTileInterfacesIncGen", - tbl_outs = [ - ( - ["-gen-attr-interface-decls"], - "include/cuda_tile/Dialect/CudaTile/IR/AttrInterfaces.h.inc", - ), - ( - ["-gen-attr-interface-defs"], - "include/cuda_tile/Dialect/CudaTile/IR/AttrInterfaces.cpp.inc", - ), - ( - ["-gen-type-interface-decls"], - "include/cuda_tile/Dialect/CudaTile/IR/TypeInterfaces.h.inc", - ), - ( - ["-gen-type-interface-defs"], - "include/cuda_tile/Dialect/CudaTile/IR/TypeInterfaces.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/IR/Interfaces.td", - deps = [":CudaTileTdFiles"], -) - -gentbl_cc_library( - name = "CudaTileTypesIncGen", - tbl_outs = [ - ( - [ - "-gen-typedef-decls", - "-typedefs-dialect=cuda_tile", - ], - "include/cuda_tile/Dialect/CudaTile/IR/Types.h.inc", - ), - ( - [ - "-gen-typedef-defs", - "-typedefs-dialect=cuda_tile", - ], - "include/cuda_tile/Dialect/CudaTile/IR/Types.cpp.inc", - ), - ( - [ - "-gen-type-constraint-decls", - "-dialect=cuda_tile", - ], - "include/cuda_tile/Dialect/CudaTile/IR/TypeConstraints.h.inc", - ), - ( - [ - "-gen-type-constraint-defs", - "-dialect=cuda_tile", - ], - "include/cuda_tile/Dialect/CudaTile/IR/TypeConstraints.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/IR/Types.td", - deps = [":CudaTileTdFiles"], -) - -gentbl_cc_library( - name = "CudaTileAttrDefsIncGen", - tbl_outs = [ - ( - [ - "-gen-attrdef-decls", - "-attrdefs-dialect=cuda_tile", - ], - "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.h.inc", - ), - ( - [ - "-gen-attrdef-defs", - "-attrdefs-dialect=cuda_tile", - ], - "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.cpp.inc", - ), - ( - [ - "-gen-enum-decls", - "-dialect=cuda_tile", - ], - "include/cuda_tile/Dialect/CudaTile/IR/Enums.h.inc", - ), - ( - [ - "-gen-enum-defs", - "-dialect=cuda_tile", - ], - "include/cuda_tile/Dialect/CudaTile/IR/Enums.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.td", - deps = [":CudaTileTdFiles"], -) - -gentbl_cc_library( - name = "CudaTileAttrCodeGenIncGen", - tbl_outs = [ - ( - ["-gen-enum-attr-list"], - "include/cuda_tile/Dialect/CudaTile/IR/CudaTileEnumAttrs.inc", - ), - ( - ["-gen-gpu-arch-def"], - "include/cuda_tile/Dialect/CudaTile/IR/GpuArchitectures.inc", - ), - ( - ["-gen-gpu-arch-constants"], - "include/cuda_tile/Dialect/CudaTile/IR/GpuArchConstants.inc", - ), - ( - ["-gen-hint-key-accessors"], - "include/cuda_tile/Dialect/CudaTile/IR/HintKeyAccessors.inc", - ), - ( - ["-gen-hint-key-impl"], - "include/cuda_tile/Dialect/CudaTile/IR/HintKeyImpl.inc", - ), - ], - tblgen = ":cuda-tile-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/IR/AttrDefs.td", - deps = [":CudaTileTdFiles"], -) - -gentbl_cc_library( - name = "CudaTileRemarksIncGen", - tbl_outs = [ - ( - ["-gen-enum-decls"], - "include/cuda_tile/Dialect/CudaTile/IR/TileIRRemarks.h.inc", - ), - ( - ["-gen-enum-defs"], - "include/cuda_tile/Dialect/CudaTile/IR/TileIRRemarks.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/IR/Remarks.td", - deps = [":CudaTileTdFiles"], -) - -gentbl_cc_library( - name = "CudaTileOpsIncGen", - tbl_outs = [ - ( - ["-gen-op-decls"], - "include/cuda_tile/Dialect/CudaTile/IR/Ops.h.inc", - ), - ( - ["-gen-op-defs"], - "include/cuda_tile/Dialect/CudaTile/IR/Ops.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/IR/Ops.td", - deps = [":CudaTileTdFiles"], -) - -gentbl_cc_library( - name = "CudaTileOpsCanonicalizationIncGen", - tbl_outs = [ - ( - ["-gen-rewriters"], - "lib/Dialect/CudaTile/IR/OpsCanonicalization.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "lib/Dialect/CudaTile/IR/OpsCanonicalization.td", - deps = [ - ":CudaTileTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - ], -) - -cc_library( - name = "CudaTileDialect", - srcs = glob( - ["lib/Dialect/CudaTile/IR/*.cpp"], - exclude = ["lib/Dialect/CudaTile/IR/CudaTileTesting.cpp"], - ), - hdrs = glob(["include/cuda_tile/Dialect/CudaTile/IR/*.h"]), - includes = [ - "include", - "lib/Dialect/CudaTile/IR", - ], - visibility = ["//visibility:public"], - deps = [ - ":CudaTileAttrCodeGenIncGen", - ":CudaTileAttrDefsIncGen", - ":CudaTileDialectIncGen", - ":CudaTileInterfacesIncGen", - ":CudaTileOpsCanonicalizationIncGen", - ":CudaTileOpsIncGen", - ":CudaTileRemarksIncGen", - ":CudaTileTypesIncGen", - "@llvm-project//llvm:Core", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:BytecodeOpInterface", - "@llvm-project//mlir:ControlFlowInterfaces", - "@llvm-project//mlir:FunctionInterfaces", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:InliningUtils", - "@llvm-project//mlir:SideEffectInterfaces", - "@llvm-project//mlir:Support", - "@llvm-project//mlir:ViewLikeInterface", - ], -) - -td_library( - name = "CudaTileTransformsTdFiles", - srcs = ["include/cuda_tile/Dialect/CudaTile/Transforms/Passes.td"], - includes = ["include"], - deps = [ - "@llvm-project//mlir:PassBaseTdFiles", - ], -) - -gentbl_cc_library( - name = "CudaTileTransformsIncGen", - tbl_outs = [ - ( - [ - "-gen-pass-decls", - "-name=CudaTile", - ], - "include/cuda_tile/Dialect/CudaTile/Transforms/Passes.h.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/Transforms/Passes.td", - deps = [":CudaTileTransformsTdFiles"], -) - -cc_library( - name = "CudaTileTransforms", - srcs = glob(["lib/Dialect/CudaTile/Transforms/*.cpp"]), - hdrs = glob(["include/cuda_tile/Dialect/CudaTile/Transforms/*.h"]), - includes = ["include"], - visibility = ["//visibility:public"], - deps = [ - ":CudaTileDialect", - ":CudaTileTransformsIncGen", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:Pass", - "@llvm-project//mlir:SideEffectInterfaces", - "@llvm-project//mlir:TransformUtils", - ], -) - -gentbl_cc_library( - name = "CudaTileBytecodeOpsIncGen", - tbl_outs = [ - ( - ["-gen-cuda-tile-bytecode"], - "include/cuda_tile/Bytecode/Writer/Bytecode.inc", - ), - ( - ["-gen-cuda-tile-bytecode-reader"], - "include/cuda_tile/Bytecode/Reader/BytecodeReader.inc", - ), - ], - tblgen = ":cuda-tile-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/IR/Ops.td", - deps = [":CudaTileTdFiles"], -) - -gentbl_cc_library( - name = "CudaTileBytecodeOpcodesIncGen", - tbl_outs = [ - ( - ["-gen-cuda-tile-opcodes"], - "include/cuda_tile/Bytecode/Common/StaticOpcodes.inc", - ), - ], - tblgen = ":cuda-tile-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeOpcodes.td", - deps = [":CudaTileTdFiles"], -) - -gentbl_cc_library( - name = "CudaTileBytecodeTypeIncGen", - tbl_outs = [ - ( - ["-gen-cuda-tile-type-bytecode"], - "include/cuda_tile/Bytecode/Writer/TypeBytecode.inc", - ), - ( - ["-gen-cuda-tile-type-bytecode-reader"], - "include/cuda_tile/Bytecode/Reader/TypeBytecodeReader.inc", - ), - ], - tblgen = ":cuda-tile-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeTypeOpcodes.td", - deps = [":CudaTileTdFiles"], -) - -gentbl_cc_library( - name = "CudaTileBytecodeAttrIncGen", - tbl_outs = [ - ( - ["-gen-cuda-tile-attr-bytecode"], - "include/cuda_tile/Bytecode/Writer/AttrBytecode.inc", - ), - ], - tblgen = ":cuda-tile-tblgen", - td_file = "include/cuda_tile/Dialect/CudaTile/IR/BytecodeAttrOpcodes.td", - deps = [":CudaTileTdFiles"], -) - -cc_library( - name = "CudaTileBytecode", - srcs = glob([ - "lib/Bytecode/**/*.cpp", - "lib/Bytecode/**/*.h", - ]), - hdrs = glob([ - "include/cuda_tile/Bytecode/**/*.h", - ]), - includes = [ - "include", - "include/cuda_tile/Bytecode/Common", - "include/cuda_tile/Bytecode/Reader", - "include/cuda_tile/Bytecode/Writer", - ], - visibility = ["//visibility:public"], - deps = [ - ":CudaTileBytecodeAttrIncGen", - ":CudaTileBytecodeOpcodesIncGen", - ":CudaTileBytecodeOpsIncGen", - ":CudaTileBytecodeTypeIncGen", - ":CudaTileDialect", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:FunctionInterfaces", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:Support", - "@llvm-project//mlir:TranslateLib", - ], -) - -cc_library( - name = "CudaTileOptimizer", - srcs = ["lib/Dialect/CudaTile/Optimizer/CudaTileOptimizer.cpp"], - hdrs = ["include/cuda_tile/Dialect/CudaTile/Optimizer/CudaTileOptimizer.h"], - includes = ["include"], - visibility = ["//visibility:public"], - deps = [ - ":CudaTileBytecode", - ":CudaTileDialect", - ":CudaTileTransforms", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:Parser", - "@llvm-project//mlir:Pass", - "@llvm-project//mlir:Support", - "@llvm-project//mlir:Transforms", - ], -) - -# The MLIR C API for the dialect: types, enum attributes, bytecode, passes, and -# the dialect handle `mlirGetDialectHandle__cuda_tile__` that ZML resolves at -# comptime. Mirrors lib/CAPI/CMakeLists.txt + lib/CAPI/Dialect/CMakeLists.txt. -cc_library( - name = "cuda_tile_capi", - srcs = [ - "lib/CAPI/Dialect/CudaTileDialect.cpp", - "lib/CAPI/Dialect/CudaTileOptimizer.cpp", - "lib/CAPI/Registration.cpp", - ], - hdrs = glob(["include/cuda_tile-c/**/*.h"]), - includes = ["include"], - visibility = ["//visibility:public"], - deps = [ - ":CudaTileBytecode", - ":CudaTileDialect", - ":CudaTileOptimizer", - ":CudaTileTransforms", - "@llvm-project//mlir:CAPIIR", - ], - # MLIR_DEFINE_CAPI_DIALECT_REGISTRATION expands to a global static - # initializer that the linker would otherwise dead-strip. - alwayslink = True, -) diff --git a/third_party/cuda_tile/repo.bzl b/third_party/cuda_tile/repo.bzl deleted file mode 100644 index 79076c51..00000000 --- a/third_party/cuda_tile/repo.bzl +++ /dev/null @@ -1,10 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -# v13.4.0 with upstream LLVM compatibility fixes -def repo(): - git_repository( - name = "cuda_tile", - remote = "https://github.com/NVIDIA/cuda-tile.git", - commit = "5b3fa4c0dd04d90edeb935a1887c20bf14a86ebe", - build_file = "//third_party/cuda_tile:cuda_tile.bazel", - ) diff --git a/third_party/cute_ir/cute_ir.bazel b/third_party/cute_ir/cute_ir.bazel deleted file mode 100644 index 0c0aed19..00000000 --- a/third_party/cute_ir/cute_ir.bazel +++ /dev/null @@ -1,104 +0,0 @@ -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") -load("@rules_cc//cc:defs.bzl", "cc_library") - -package(default_visibility = ["//visibility:public"]) - -licenses(["notice"]) # BSD-3-Clause - -# The CuTe dialect of the cutlass_compiler/ tree, as released. The tree that -# builds it (//mlir/dialects/cute_ir) adds the DSL compiler's own attributes, -# types and operations and compiles these sources with them. -CUTE_IR_INCLUDE = "cute_ir/include" - -exports_files([ - "LICENSE.txt", - CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h", - "cute_ir/lib/Dialect/IR/CuteAttributes.cpp", - "cute_ir/lib/Dialect/IR/CuteDialect.cpp", -]) - - -# cutegen, the CuTe layout algebra, with the `?{div=N}` leaf added by -# cutegen_dynamic_divisibility.patch: a dynamic extent the DSL knows to be a -# multiple of N, which the release's cutegen cannot read. -cc_library( - name = "cutegen", - hdrs = glob(["cutegen/include/cutegen/*.hpp"]), - includes = ["cutegen/include"], - deps = [ - "@llvm-project//llvm:Support", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:LLVMDialect", - "@llvm-project//mlir:MathDialect", - "@llvm-project//mlir:NVVMDialect", - "@llvm-project//mlir:SCFDialect", - "@llvm-project//mlir:UBDialect", - ], -) - -td_library( - name = "cute_ir_dialect_td", - srcs = [ - CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteAttrs.td", - CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.td", - CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", - CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypes.td", - ], - includes = [CUTE_IR_INCLUDE], - deps = [ - "@llvm-project//mlir:BuiltinDialectTdFiles", - "@llvm-project//mlir:DataLayoutInterfacesTdFiles", - "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - ], -) - -gentbl_cc_library( - name = "cute_ir_dialect_inc_gen", - includes = [CUTE_IR_INCLUDE], - tbl_outs = [ - ( - ["-gen-dialect-decls", "-dialect=cute"], - CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h.inc", - ), - ( - ["-gen-dialect-defs", "-dialect=cute"], - CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.td", - deps = [":cute_ir_dialect_td"], -) - -gentbl_cc_library( - name = "cute_ir_type_interfaces_inc_gen", - includes = [CUTE_IR_INCLUDE], - tbl_outs = [ - ( - ["-gen-type-interface-decls"], - CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h.inc", - ), - ( - ["-gen-type-interface-defs"], - CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", - deps = [":cute_ir_dialect_td"], -) - -# The header alone: the attribute, type and operation class declarations it -# includes are generated by the building tree, which adds the compiler's own. -cc_library( - name = "cute_ir_dialect_header", - hdrs = [CUTE_IR_INCLUDE + "/cute_ir/Dialect/Cute/IR/CuteDialect.h"], - includes = [CUTE_IR_INCLUDE], - deps = [ - ":cute_ir_dialect_inc_gen", - ":cute_ir_type_interfaces_inc_gen", - ], -) diff --git a/third_party/cute_ir/cutegen_dynamic_divisibility.patch b/third_party/cute_ir/cutegen_dynamic_divisibility.patch deleted file mode 100644 index 4f479d60..00000000 --- a/third_party/cute_ir/cutegen_dynamic_divisibility.patch +++ /dev/null @@ -1,243 +0,0 @@ ---- a/cutlass_compiler/cutegen/include/cutegen/dynamic.hpp -+++ b/cutlass_compiler/cutegen/include/cutegen/dynamic.hpp -@@ -73,13 +73,20 @@ - * @brief Width (in bits) of the associated dynamic integer - */ - int32_t width = default_dynamic_int_bitwidth(); -+ /** -+ * @brief Known positive divisor of the associated dynamic integer; 1 when -+ * nothing is known. Spelled `div=N` in the dynamic leaf's properties. -+ */ -+ static_int_t divisibility = 1; - /// Constructor - constexpr dynamic_int_properties_t() = default; - explicit dynamic_int_properties_t(int32_t w); -+ dynamic_int_properties_t(int32_t w, static_int_t div); - /// Equality operators - bool operator==(const dynamic_int_properties_t& rhs) const; - bool operator!=(const dynamic_int_properties_t& rhs) const; - dynamic_int_properties_t with_width(int32_t w) const; -+ dynamic_int_properties_t with_divisibility(static_int_t div) const; - bool is_default() const; - bool are_compatible(static_int_t v) const; - static_int_t minimum() const; -@@ -475,11 +482,25 @@ - static std::string to_str(const dynamic_t& d) - { - std::string s("?"); -- if(d.get_width() != default_dynamic_int_bitwidth()) -+ const bool has_width = d.get_width() != default_dynamic_int_bitwidth(); -+ const bool has_div = d.get_properties().divisibility != 1; -+ if(has_width || has_div) - { - s.append("{"); -- s.append("i"); -- s.append(std::to_string(d.get_width())); -+ if(has_width) -+ { -+ s.append("i"); -+ s.append(std::to_string(d.get_width())); -+ } -+ if(has_div) -+ { -+ if(has_width) -+ { -+ s.append(" "); -+ } -+ s.append("div="); -+ s.append(std::to_string(d.get_properties().divisibility)); -+ } - s.append("}"); - } - return s; -@@ -553,6 +574,22 @@ - return; - } - } -+ else if(kw.value() == "div") -+ { -+ auto eq = parse_type(); -+ if(!eq.has_value()) -+ { -+ // Error: missing '=' after 'div' -+ return; -+ } -+ auto div = parse_type(); -+ if(!div.has_value() || div.value() <= 0) -+ { -+ // Error: no positive value for the divisor -+ return; -+ } -+ prop.divisibility = div.value(); -+ } - else - { - // Error: an unexpected keyword was found -@@ -572,6 +609,7 @@ - static void encode_value(const dynamic_t& d, TBuffer& buffer) - { - encode(buffer, d.get_width()); -+ encode(buffer, d.get_properties().divisibility); - } - }; - -@@ -586,7 +624,10 @@ - if(!w.has_value() || - (widths.end() == std::find(widths.begin(), widths.end(), w.value()))) - return std::nullopt; -- return dynamic_t(w.value()); -+ const auto div = decode(it, end); -+ if(!div.has_value() || div.value() <= 0) -+ return std::nullopt; -+ return dynamic_t(dynamic_int_properties_t(w.value(), div.value())); - } - }; - -@@ -816,45 +857,72 @@ - struct property_policy_c_t - { - using int_t = static_int_t; -+ // The divisibility a result keeps: a sum or difference is a multiple of -+ // the gcd of its operands' divisors, a product of their product, and a -+ // static operand counts as its own absolute value. -+ static static_int_t gcd_div(static_int_t a, static_int_t b) -+ { -+ a = a < 0 ? -a : a; -+ b = b < 0 ? -b : b; -+ while(b != 0) -+ { -+ static_int_t t = a % b; -+ a = b; -+ b = t; -+ } -+ return a == 0 ? 1 : a; -+ } -+ static dynamic_int_properties_t with_div(int32_t width, static_int_t div) -+ { -+ return dynamic_int_properties_t(width, div <= 0 ? 1 : div); -+ } - //-------------------------------------------------------------------------- - // resolve_add() - static arith_resolve_t resolve_add(const dynamic_t& lhs, const dynamic_t& rhs) - { - auto res_width = std::max(lhs.get_width(), rhs.get_width()); -- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); -+ auto div = gcd_div(lhs.get_properties().divisibility, -+ rhs.get_properties().divisibility); -+ return arith_op_desc_t(with_div(res_width, div), res_width); - } - template , bool> = true> - static arith_resolve_t resolve_add(const dynamic_t& lhs, TInt rhs) - { - auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); -- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); -+ auto div = gcd_div(lhs.get_properties().divisibility, static_int_t(rhs)); -+ return arith_op_desc_t(with_div(res_width, div), res_width); - } - static arith_resolve_t resolve_sub(const dynamic_t& lhs, const dynamic_t& rhs) - { - auto res_width = std::max(lhs.get_width(), rhs.get_width()); -- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); -+ auto div = gcd_div(lhs.get_properties().divisibility, -+ rhs.get_properties().divisibility); -+ return arith_op_desc_t(with_div(res_width, div), res_width); - } - template , bool> = true> - static arith_resolve_t resolve_sub(const dynamic_t& lhs, TInt rhs) - { - auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); -- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); -+ auto div = gcd_div(lhs.get_properties().divisibility, static_int_t(rhs)); -+ return arith_op_desc_t(with_div(res_width, div), res_width); - } - template , bool> = true> - static arith_resolve_t resolve_sub(TInt lhs, const dynamic_t& rhs) - { - auto res_width = std::max(rhs.get_width(), in_range(lhs) ? 32 : 64); -- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); -+ auto div = gcd_div(rhs.get_properties().divisibility, static_int_t(lhs)); -+ return arith_op_desc_t(with_div(res_width, div), res_width); - } - //-------------------------------------------------------------------------- - // resolve_mul() - static arith_resolve_t resolve_mul(const dynamic_t& lhs, const dynamic_t& rhs) - { - auto res_width = std::max(lhs.get_width(), rhs.get_width()); -- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); -+ auto div = lhs.get_properties().divisibility * rhs.get_properties().divisibility; -+ return arith_op_desc_t(with_div(res_width, div), res_width); - } - template , bool> = true> -@@ -870,7 +938,9 @@ - return src_operand_index_t(0); - } - auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); -- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); -+ auto div = lhs.get_properties().divisibility * -+ static_int_t(rhs < 0 ? -rhs : rhs); -+ return arith_op_desc_t(with_div(res_width, div), res_width); - } - //-------------------------------------------------------------------------- - // resolve_mod() -@@ -939,7 +1009,9 @@ - return src_operand_index_t(0); - } - auto res_width = std::max(lhs.get_width(), in_range(rhs) ? 32 : 64); -- return arith_op_desc_t(dynamic_int_properties_t(res_width), res_width); -+ auto d = lhs.get_properties().divisibility; -+ auto r = static_int_t(rhs < 0 ? -rhs : rhs); -+ return arith_op_desc_t(with_div(res_width, d % r == 0 ? d / r : 1), res_width); - } - template , bool> = true> -@@ -1259,12 +1331,18 @@ - - inline dynamic_int_properties_t::dynamic_int_properties_t(int32_t w) : - width(w) -+{ -+} -+ -+inline dynamic_int_properties_t::dynamic_int_properties_t(int32_t w, static_int_t div) : -+ width(w), -+ divisibility(div) - { - } - - inline bool dynamic_int_properties_t::operator==(const dynamic_int_properties_t& rhs) const - { -- return (width == rhs.width); -+ return (width == rhs.width) && (divisibility == rhs.divisibility); - } - - inline bool dynamic_int_properties_t::operator!=(const dynamic_int_properties_t& rhs) const -@@ -1284,9 +1362,16 @@ - return r; - } - -+inline dynamic_int_properties_t dynamic_int_properties_t::with_divisibility(static_int_t div) const -+{ -+ dynamic_int_properties_t r = *this; -+ r.divisibility = div; -+ return r; -+} -+ - inline bool dynamic_int_properties_t::are_compatible(static_int_t v) const - { -- return in_range_of_width(v, width); -+ return in_range_of_width(v, width) && (v % divisibility == 0); - } - - inline static_int_t dynamic_int_properties_t::minimum() const -@@ -1313,7 +1398,7 @@ - - inline bool dynamic_int_properties_t::is_constrained() const - { -- return false; -+ return divisibility != 1; - } - - inline dynamic_t::dynamic_t(int32_t width_) : diff --git a/third_party/cute_ir/repo.bzl b/third_party/cute_ir/repo.bzl deleted file mode 100644 index 6beb5df0..00000000 --- a/third_party/cute_ir/repo.bzl +++ /dev/null @@ -1,22 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -def repo(): - # The cutlass_compiler/ tree of the CUTLASS v4.8.0 release: the open-source - # part of the CuTe dialect that mlir/dialects/cute_ir builds on. - git_repository( - name = "cute_ir", - remote = "https://github.com/NVIDIA/cutlass.git", - commit = "098de2a652cf8f00fd70b2df54051c7eccbb855a", - build_file = Label("//third_party/cute_ir:cute_ir.bazel"), - strip_prefix = "cutlass_compiler", - patches = [ - # `cute.slice` upstream only infers tuple and layout results; the - # MoE kernels slice memrefs and coordinate tensors with explicit - # result types. - Label("//third_party/cute_ir:slice_tensor.patch"), - # The DSL spells a dynamic leaf it knows to be a multiple of N as - # `?{div=N}`; the release's cutegen reads only the width. - Label("//third_party/cute_ir:cutegen_dynamic_divisibility.patch"), - ], - patch_args = ["-p2"], - ) diff --git a/third_party/cute_ir/slice_tensor.patch b/third_party/cute_ir/slice_tensor.patch deleted file mode 100644 index 173a795b..00000000 --- a/third_party/cute_ir/slice_tensor.patch +++ /dev/null @@ -1,45 +0,0 @@ -diff --git a/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td b/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td ---- a/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td -+++ b/cutlass_compiler/cute_ir/include/cute_ir/Dialect/Cute/IR/CuteOps.td -@@ -2990,7 +2990,7 @@ - - def Cute_SliceOp : Cute_Op<"slice", [ - Pure, -- DeclareOpInterfaceMethods -+ DeclareOpInterfaceMethods - ]> { - let cuteSpecGroup = "LayoutAlgebra"; - let summary = "Slice a value, keeping modes where the coordinate is `_`"; -@@ -3154,11 +3154,11 @@ - }]; - - let arguments = (ins -- Cute_Arg:$input, -+ Cute_Arg:$input, - Cute_Arg:$coord - ); - let results = (outs -- Cute_Arg:$result -+ Cute_Arg:$result - ); - let assemblyFormat = [{ `(` operands `)` attr-dict `:` type(operands) }]; - } -diff --git a/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp b/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp ---- a/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp -+++ b/cutlass_compiler/cute_ir/lib/Dialect/IR/CuteOps.cpp -@@ -1327,6 +1327,15 @@ - }); - } - -+LogicalResult SliceOp::refineReturnTypes( -+ MLIRContext *, std::optional, ValueRange, DictionaryAttr, -+ PropertyRef, RegionRange, SmallVectorImpl &) { -+ // Tensor views are private CuTe-DSL types. Their result type is supplied -+ // explicitly by the front end, so no public layout-algebra refinement is -+ // required here. -+ return success(); -+} -+ - //===----------------------------------------------------------------------===// - // DiceOp (cute.dice) - //===----------------------------------------------------------------------===// diff --git a/third_party/cutlass/cutlass.bazel b/third_party/cutlass/cutlass.bazel deleted file mode 100644 index e009a6d4..00000000 --- a/third_party/cutlass/cutlass.bazel +++ /dev/null @@ -1,111 +0,0 @@ -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") -load("@rules_cc//cc:defs.bzl", "cc_library") - -td_library( - name = "dialect_td", - srcs = [ - "include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", - "include/cute_ir/Dialect/Cute/IR/CuteDialect.td", - "include/cute_ir/Dialect/Cute/IR/CuteOps.td", - "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", - "include/cute_ir/Dialect/Cute/IR/CuteTypes.td", - ], - includes = ["include"], - deps = [ - "@llvm-project//mlir:BuiltinDialectTdFiles", - "@llvm-project//mlir:DataLayoutInterfacesTdFiles", - "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - ], -) - -gentbl_cc_library( - name = "dialect_inc_gen", - strip_include_prefix = ".", - tbl_outs = [ - ( - ["-gen-dialect-decls", "-dialect=cute"], - "include/cute_ir/Dialect/Cute/IR/CuteDialect.h.inc", - ), - ( - ["-gen-dialect-defs", "-dialect=cute"], - "include/cute_ir/Dialect/Cute/IR/CuteDialect.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/cute_ir/Dialect/Cute/IR/CuteDialect.td", - deps = [":dialect_td"], -) - -gentbl_cc_library( - name = "type_interfaces_inc_gen", - strip_include_prefix = ".", - tbl_outs = [ - ( - ["-gen-type-interface-decls"], - "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h.inc", - ), - ( - ["-gen-type-interface-defs"], - "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.td", - deps = [":dialect_td"], -) - -gentbl_cc_library( - name = "attrs_inc_gen", - strip_include_prefix = ".", - tbl_outs = [ - ( - ["-gen-attrdef-decls", "-attrdefs-dialect=cute"], - "include/cute_ir/Dialect/Cute/IR/CuteAttrs.h.inc", - ), - ( - ["-gen-attrdef-defs", "-attrdefs-dialect=cute"], - "include/cute_ir/Dialect/Cute/IR/CuteAttrs.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/cute_ir/Dialect/Cute/IR/CuteAttrs.td", - deps = [":dialect_td"], -) - -gentbl_cc_library( - name = "ops_inc_gen", - strip_include_prefix = ".", - tbl_outs = [ - ( - ["-gen-op-decls"], - "include/cute_ir/Dialect/Cute/IR/CuteOps.h.inc", - ), - ( - ["-gen-op-defs"], - "include/cute_ir/Dialect/Cute/IR/CuteOps.cpp.inc", - ), - ], - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/cute_ir/Dialect/Cute/IR/CuteOps.td", - deps = [":dialect_td"], -) - -cc_library( - name = "headers", - hdrs = [ - hdr for hdr in [ - "include/cute_ir/Dialect/Cute/IR/CuteDialect.h", - "include/cute_ir/Dialect/Cute/IR/CuteOps.h", - "include/cute_ir/Dialect/Cute/IR/CuteTypeInterfaces.h", - ] - ], - strip_include_prefix = "include", - deps = [ - ":attrs_inc_gen", - ":dialect_inc_gen", - ":ops_inc_gen", - ":type_interfaces_inc_gen", - ], -) diff --git a/third_party/flashattn/BUILD.bazel b/third_party/flashattn/BUILD.bazel new file mode 100644 index 00000000..3f32c61c --- /dev/null +++ b/third_party/flashattn/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") + +filegroup( + name = "libflashattn", + visibility = ["//visibility:public"], + srcs = ["flashattn/lib/libflashattn.so"], +) + +cc_library( + name = "c", + hdrs = ["flashattn/include/capi.h"], + visibility = ["//visibility:public"], +) diff --git a/third_party/flashattn/repo.bzl b/third_party/flashattn/repo.bzl new file mode 100644 index 00000000..654620cc --- /dev/null +++ b/third_party/flashattn/repo.bzl @@ -0,0 +1,13 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + +def repo(): + http_archive( + name = "flashattn", + url = "https://github.com/zml/flash-attention/releases/download/v0.0.6-rc7/flashattn.tar.gz", + sha256 = "6b8195a5ab285404fcb084122c1c32f342ff695040b8652c9144afd2289b71b3", + build_file = "//:third_party/flashattn/BUILD.bazel", + ) + + #new_local_repository(name = "flashattn", build_file="//:third_party/flashattn/BUILD.bazel", path="/home/corendos/flashattn/") + diff --git a/third_party/flashinfer_cutlass_moe/BUILD.bazel b/third_party/flashinfer_cutlass_moe/BUILD.bazel deleted file mode 100644 index b2ddeede..00000000 --- a/third_party/flashinfer_cutlass_moe/BUILD.bazel +++ /dev/null @@ -1,4 +0,0 @@ -exports_files([ - "flashinfer_cutlass_moe.BUILD.bazel", - "repo.bzl", -]) diff --git a/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel b/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel deleted file mode 100644 index 780ac0d2..00000000 --- a/third_party/flashinfer_cutlass_moe/flashinfer_cutlass_moe.BUILD.bazel +++ /dev/null @@ -1,33 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -package(default_visibility = ["//visibility:public"]) - -filegroup( - name = "lib_sm100", - srcs = ["lib/libflashinfer_cutlass_moe_sm100.so"], -) - -filegroup( - name = "lib_sm103", - srcs = ["lib/libflashinfer_cutlass_moe_sm103.so"], -) - -filegroup( - name = "lib_sm120", - srcs = ["lib/libflashinfer_cutlass_moe_sm120.so"], -) - -filegroup( - name = "libraries", - srcs = [ - ":lib_sm100", - ":lib_sm103", - ":lib_sm120", - ], -) - -cc_library( - name = "c", - hdrs = ["include/flashinfer_cutlass_moe.h"], - includes = ["include"], -) diff --git a/third_party/flashinfer_cutlass_moe/repo.bzl b/third_party/flashinfer_cutlass_moe/repo.bzl deleted file mode 100644 index 2f897c2d..00000000 --- a/third_party/flashinfer_cutlass_moe/repo.bzl +++ /dev/null @@ -1,38 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -_RELEASE = "cutlass-moe-v0.2.0" - -_ASSETS = { - "amd64": { - "sha256": "5c3a29da0fd91e2281d046eb2e467a7a40aeb24a0be8ba0487006356d45fecc4", - "url": "https://github.com/zml/flashinfer/releases/download/{release}/flashinfer-cutlass-moe_linux-amd64.tar.gz", - }, - "arm64": { - "sha256": "60b6fdbd193e11c24589add1eb206f59a3900cde4760e53719d6b2add492343e", - "url": "https://github.com/zml/flashinfer/releases/download/{release}/flashinfer-cutlass-moe_linux-arm64.tar.gz", - }, -} - -def _archive(name, arch): - asset = _ASSETS[arch] - - http_archive( - name = name, - urls = [ - asset["url"].format(release = _RELEASE), - ], - sha256 = asset["sha256"], - build_file = Label( - "//third_party/flashinfer_cutlass_moe:flashinfer_cutlass_moe.BUILD.bazel", - ), - ) - -def repo(): - _archive( - name = "flashinfer_cutlass_moe_linux_amd64", - arch = "amd64", - ) - _archive( - name = "flashinfer_cutlass_moe_linux_arm64", - arch = "arm64", - ) diff --git a/third_party/flydsl/atomic_exchange.patch b/third_party/flydsl/atomic_exchange.patch deleted file mode 100644 index 7de76db1..00000000 --- a/third_party/flydsl/atomic_exchange.patch +++ /dev/null @@ -1,25 +0,0 @@ -diff --git a/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td b/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td ---- a/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td -+++ b/include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td -@@ -43,7 +43,8 @@ - I32EnumAttrCase<"And", 3, "and">, - I32EnumAttrCase<"Or", 4, "or">, - I32EnumAttrCase<"Inc", 5, "inc">, -- I32EnumAttrCase<"Dec", 6, "dec"> -+ I32EnumAttrCase<"Dec", 6, "dec">, -+ I32EnumAttrCase<"Exchange", 7, "exchange"> - ]>; - def Fly_AtomicOpAttr : Fly_EnumAttr {} - -diff --git a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp ---- a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp -+++ b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp -@@ -228,6 +228,8 @@ - return isFloat ? std::nullopt : std::optional(LLVM::AtomicBinOp::uinc_wrap); - case AtomicOp::Dec: - return isFloat ? std::nullopt : std::optional(LLVM::AtomicBinOp::udec_wrap); -+ case AtomicOp::Exchange: -+ return LLVM::AtomicBinOp::xchg; - } - return std::nullopt; - } diff --git a/third_party/flydsl/flydsl.bazel b/third_party/flydsl/flydsl.bazel deleted file mode 100644 index 7c8cb7e9..00000000 --- a/third_party/flydsl/flydsl.bazel +++ /dev/null @@ -1,423 +0,0 @@ -load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "td_library") -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -package( - default_visibility = ["//visibility:public"], - features = [ - "-parse_headers", - "-use_header_modules", - ], - licenses = ["notice"], -) - -exports_files(["LICENSE"]) - -td_library( - name = "td_files", - srcs = glob(["include/flydsl/**/*.td"]), - includes = ["include"], - deps = [ - "@llvm-project//mlir:ArithOpsTdFiles", - "@llvm-project//mlir:BuiltinDialectBytecodeTdFiles", - "@llvm-project//mlir:BytecodeOpInterfaceTdFiles", - "@llvm-project//mlir:FunctionInterfacesTdFiles", - "@llvm-project//mlir:GPUOpsTdFiles", - "@llvm-project//mlir:InferTypeOpInterfaceTdFiles", - "@llvm-project//mlir:LLVMOpsTdFiles", - "@llvm-project//mlir:OpBaseTdFiles", - "@llvm-project//mlir:PassBaseTdFiles", - "@llvm-project//mlir:ROCDLOpsTdFiles", - "@llvm-project//mlir:SCFTdFiles", - "@llvm-project//mlir:SideEffectInterfacesTdFiles", - "@llvm-project//mlir:VectorOpsTdFiles", - ], -) - -gentbl_cc_library( - name = "fly_dialect_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/Fly/IR/FlyDialect.cpp.inc": ["--gen-dialect-defs"], - "include/flydsl/Dialect/Fly/IR/FlyDialect.h.inc": ["--gen-dialect-decls"], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/Fly/IR/FlyDialect.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_interfaces_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/Fly/IR/FlyAttrInterfaces.cpp.inc": ["--gen-attr-interface-defs"], - "include/flydsl/Dialect/Fly/IR/FlyAttrInterfaces.h.inc": ["--gen-attr-interface-decls"], - "include/flydsl/Dialect/Fly/IR/FlyOpInterfaces.cpp.inc": ["--gen-op-interface-defs"], - "include/flydsl/Dialect/Fly/IR/FlyOpInterfaces.h.inc": ["--gen-op-interface-decls"], - "include/flydsl/Dialect/Fly/IR/FlyTypeInterfaces.cpp.inc": ["--gen-type-interface-defs"], - "include/flydsl/Dialect/Fly/IR/FlyTypeInterfaces.h.inc": ["--gen-type-interface-decls"], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/Fly/IR/FlyInterfaces.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_types_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/Fly/IR/FlyTypeConstraints.cpp.inc": [ - "--gen-type-constraint-defs", - "--typedefs-dialect=fly", - ], - "include/flydsl/Dialect/Fly/IR/FlyTypeConstraints.h.inc": [ - "--gen-type-constraint-decls", - "--typedefs-dialect=fly", - ], - "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.cpp.inc": [ - "--gen-typedef-defs", - "--typedefs-dialect=fly", - ], - "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.h.inc": [ - "--gen-typedef-decls", - "--typedefs-dialect=fly", - ], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/Fly/IR/FlyTypeDefs.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_attrs_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/Fly/IR/FlyAttrConstraints.cpp.inc": [ - "--gen-attr-constraint-defs", - "--attrdefs-dialect=fly", - ], - "include/flydsl/Dialect/Fly/IR/FlyAttrConstraints.h.inc": [ - "--gen-attr-constraint-decls", - "--attrdefs-dialect=fly", - ], - "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.cpp.inc": [ - "--gen-attrdef-defs", - "--attrdefs-dialect=fly", - ], - "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.h.inc": [ - "--gen-attrdef-decls", - "--attrdefs-dialect=fly", - ], - "include/flydsl/Dialect/Fly/IR/FlyEnums.cpp.inc": ["--gen-enum-defs"], - "include/flydsl/Dialect/Fly/IR/FlyEnums.h.inc": ["--gen-enum-decls"], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/Fly/IR/FlyAttrDefs.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_ops_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/Fly/IR/FlyOps.cpp.inc": ["--gen-op-defs"], - "include/flydsl/Dialect/Fly/IR/FlyOps.h.inc": ["--gen-op-decls"], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/Fly/IR/FlyOps.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_passes_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/Fly/Transforms/Passes.h.inc": [ - "--gen-pass-decls", - "--name=Fly", - ], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/Fly/Transforms/Passes.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_int_tuple_lowering_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/Fly/Transforms/IntTupleLowering.cpp.inc": ["--gen-rewriters"], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/Fly/Transforms/IntTupleLowering.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_layout_lowering_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/Fly/Transforms/LayoutLowering.cpp.inc": ["--gen-rewriters"], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/Fly/Transforms/LayoutLowering.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_memref_lowering_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/Fly/Transforms/MemrefLowering.cpp.inc": ["--gen-rewriters"], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/Fly/Transforms/MemrefLowering.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_rocdl_dialect_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/FlyROCDL/IR/Dialect.cpp.inc": ["--gen-dialect-defs"], - "include/flydsl/Dialect/FlyROCDL/IR/Dialect.h.inc": ["--gen-dialect-decls"], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/FlyROCDL/IR/Dialect.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_rocdl_atom_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/FlyROCDL/IR/Atom.cpp.inc": [ - "--gen-typedef-defs", - "--typedefs-dialect=fly_rocdl", - ], - "include/flydsl/Dialect/FlyROCDL/IR/Atom.h.inc": [ - "--gen-typedef-decls", - "--typedefs-dialect=fly_rocdl", - ], - "include/flydsl/Dialect/FlyROCDL/IR/AtomStateEnums.cpp.inc": [ - "--gen-enum-defs", - "--typedefs-dialect=fly_rocdl", - ], - "include/flydsl/Dialect/FlyROCDL/IR/AtomStateEnums.h.inc": [ - "--gen-enum-decls", - "--typedefs-dialect=fly_rocdl", - ], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/FlyROCDL/IR/Atom.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_rocdl_attrs_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.cpp.inc": [ - "--gen-attrdef-defs", - "--attrdefs-dialect=fly_rocdl", - ], - "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.h.inc": [ - "--gen-attrdef-decls", - "--attrdefs-dialect=fly_rocdl", - ], - "include/flydsl/Dialect/FlyROCDL/IR/AttrEnums.cpp.inc": [ - "--gen-enum-defs", - "--attrdefs-dialect=fly_rocdl", - ], - "include/flydsl/Dialect/FlyROCDL/IR/AttrEnums.h.inc": [ - "--gen-enum-decls", - "--attrdefs-dialect=fly_rocdl", - ], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/FlyROCDL/IR/AttrDefs.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_rocdl_ops_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/FlyROCDL/IR/Ops.cpp.inc": ["--gen-op-defs"], - "include/flydsl/Dialect/FlyROCDL/IR/Ops.h.inc": ["--gen-op-decls"], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/FlyROCDL/IR/Ops.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_to_rocdl_passes_inc_gen", - tbl_outs = { - "include/flydsl/Conversion/FlyToROCDL/Passes.h.inc": [ - "--gen-pass-decls", - "--name=FlyToROCDL", - ], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Conversion/FlyToROCDL/Passes.td", - deps = [":td_files"], -) - -gentbl_cc_library( - name = "fly_rocdl_passes_inc_gen", - tbl_outs = { - "include/flydsl/Dialect/FlyROCDL/Transforms/Passes.h.inc": [ - "--gen-pass-decls", - "--name=FlyROCDL", - ], - }, - tblgen = "@llvm-project//mlir:mlir-tblgen", - td_file = "include/flydsl/Dialect/FlyROCDL/Transforms/Passes.td", - deps = [":td_files"], -) - -cc_library( - name = "FlyDialect", - srcs = [ - "lib/Dialect/Fly/IR/FlyAttrDefs.cpp", - "lib/Dialect/Fly/IR/FlyDialect.cpp", - "lib/Dialect/Fly/IR/FlyOps.cpp", - "lib/Dialect/Fly/IR/FlyTypeDefs.cpp", - "lib/Dialect/Fly/IR/FlyUniversalOps.cpp", - "lib/Dialect/Fly/Utils/IntTupleUtils.cpp", - "lib/Dialect/Fly/Utils/IntUtils.cpp", - "lib/Dialect/Fly/Utils/NormalForm.cpp", - "lib/Dialect/Fly/Utils/PointerUtils.cpp", - ], - hdrs = glob( - [ - "include/flydsl/Dialect/Fly/IR/*.h", - "include/flydsl/Dialect/Fly/Utils/*.h", - "include/flydsl/Dialect/Fly/Utils/*.inc", - ], - exclude = ["include/flydsl/Dialect/Fly/IR/FlyLLVMTranslation.h"], - ), - includes = ["include"], - deps = [ - ":fly_attrs_inc_gen", - ":fly_dialect_inc_gen", - ":fly_interfaces_inc_gen", - ":fly_ops_inc_gen", - ":fly_types_inc_gen", - "@llvm-project//llvm:Support", - "@llvm-project//llvm:ir_headers", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:BytecodeOpInterface", - "@llvm-project//mlir:FuncDialect", - "@llvm-project//mlir:GPUDialect", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:InferTypeOpInterface", - "@llvm-project//mlir:LLVMDialect", - "@llvm-project//mlir:SCFDialect", - "@llvm-project//mlir:SideEffectInterfaces", - ], -) - -cc_library( - name = "FlyTransforms", - srcs = glob(["lib/Dialect/Fly/Transforms/*.cpp"]), - hdrs = glob(["include/flydsl/Dialect/Fly/Transforms/*.h"]), - includes = ["include"], - deps = [ - ":FlyDialect", - ":fly_int_tuple_lowering_inc_gen", - ":fly_layout_lowering_inc_gen", - ":fly_memref_lowering_inc_gen", - ":fly_passes_inc_gen", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:ArithUtils", - "@llvm-project//mlir:FuncDialect", - "@llvm-project//mlir:FuncTransforms", - "@llvm-project//mlir:GPUDialect", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:LLVMDialect", - "@llvm-project//mlir:Pass", - "@llvm-project//mlir:SCFDialect", - "@llvm-project//mlir:SCFTransforms", - "@llvm-project//mlir:Transforms", - "@llvm-project//mlir:UBDialect", - "@llvm-project//mlir:VectorDialect", - ], -) - -cc_library( - name = "FlyROCDLDialect", - srcs = [ - "lib/Dialect/FlyROCDL/CDNA3/CopyAtom.cpp", - "lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp", - "lib/Dialect/FlyROCDL/CDNA4/CopyAtom.cpp", - "lib/Dialect/FlyROCDL/CDNA4/MmaAtom.cpp", - "lib/Dialect/FlyROCDL/CDNA5/CopyAtom.cpp", - "lib/Dialect/FlyROCDL/CDNA5/TdmAtomBuilder.cpp", - "lib/Dialect/FlyROCDL/CDNA5/TdmGeometry.cpp", - "lib/Dialect/FlyROCDL/Dialect.cpp", - "lib/Dialect/FlyROCDL/GFX11/MmaAtom.cpp", - "lib/Dialect/FlyROCDL/GFX120X/MmaAtom.cpp", - "lib/Dialect/FlyROCDL/GFX1250/CopyAtom.cpp", - "lib/Dialect/FlyROCDL/GFX1250/MmaAtom.cpp", - "lib/Dialect/FlyROCDL/GFX1250/MmaAtomScale.cpp", - "lib/Dialect/FlyROCDL/Ops.cpp", - "lib/Dialect/FlyROCDL/Transforms/ClusterAttr.cpp", - "lib/Dialect/FlyROCDL/Transforms/ExpandOps.cpp", - ], - hdrs = glob([ - "include/flydsl/Dialect/FlyROCDL/**/*.h", - "lib/Dialect/FlyROCDL/**/*.h", - ]), - includes = ["include"], - deps = [ - ":FlyDialect", - ":fly_rocdl_atom_inc_gen", - ":fly_rocdl_attrs_inc_gen", - ":fly_rocdl_dialect_inc_gen", - ":fly_rocdl_ops_inc_gen", - ":fly_rocdl_passes_inc_gen", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:LLVMDialect", - "@llvm-project//mlir:Pass", - "@llvm-project//mlir:ROCDLDialect", - "@llvm-project//mlir:SCFDialect", - "@llvm-project//mlir:VectorDialect", - ], -) - -cc_library( - name = "FlyToROCDL", - srcs = [ - "lib/Conversion/FlyToROCDL/FastMathOps.cpp", - "lib/Conversion/FlyToROCDL/FlyToROCDL.cpp", - ], - hdrs = glob(["include/flydsl/Conversion/**/*.h"]), - includes = ["include"], - deps = [ - ":FlyDialect", - ":FlyROCDLDialect", - ":fly_to_rocdl_passes_inc_gen", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:ArithDialect", - "@llvm-project//mlir:ArithUtils", - "@llvm-project//mlir:ControlFlowTransforms", - "@llvm-project//mlir:FuncDialect", - "@llvm-project//mlir:FuncTransforms", - "@llvm-project//mlir:GPUDialect", - "@llvm-project//mlir:IR", - "@llvm-project//mlir:LLVMDialect", - "@llvm-project//mlir:MathDialect", - "@llvm-project//mlir:Pass", - "@llvm-project//mlir:ROCDLDialect", - "@llvm-project//mlir:SCFDialect", - "@llvm-project//mlir:SCFTransforms", - "@llvm-project//mlir:Transforms", - "@llvm-project//mlir:VectorDialect", - ], -) - -# ZML addition: the per-type accessors FlyDSL's Python bindings use to derive -# tiled TV layouts and MMA tile sizes. They live under lib/Bindings/Python but -# depend only on the dialect (no nanobind). -cc_library( - name = "TiledOpTraits", - srcs = ["lib/Bindings/Python/TiledOpTraits.cpp"], - hdrs = ["lib/Bindings/Python/TiledOpTraits.h"], - includes = ["lib/Bindings/Python"], - deps = [ - ":FlyDialect", - "@llvm-project//llvm:Support", - "@llvm-project//mlir:IR", - ], -) diff --git a/third_party/flydsl/mlir_legacy_assembly.patch b/third_party/flydsl/mlir_legacy_assembly.patch deleted file mode 100644 index 0739e489..00000000 --- a/third_party/flydsl/mlir_legacy_assembly.patch +++ /dev/null @@ -1,39 +0,0 @@ ---- a/include/flydsl/Dialect/Fly/IR/FlyDialect.td -+++ b/include/flydsl/Dialect/Fly/IR/FlyDialect.td -@@ -19,6 +19,11 @@ - - let useDefaultTypePrinterParser = 1; - let useDefaultAttributePrinterParser = 1; -+ -+ // ZML custom-call inputs use Fly assembly printed by a separately pinned -+ // MLIR/Fly build. Keep accepting its inherent attributes in attr-dict until -+ // the producer and consumer can migrate to prop-dict together. -+ let useStrictPropertiesInAssemblyFormat = 0; - } - - class Fly_Type traits = []> -@@ -38,7 +43,10 @@ - } - - class Fly_EnumAttr traits = []> -- : EnumAttr; -+ : EnumAttr { -+ // Keep the Fly text ABI independent of MLIR's default enum delimiters. -+ let assemblyFormat = "$value"; -+} - - class Fly_Op traits = []> - : Op { -diff --git a/include/flydsl/Dialect/FlyROCDL/IR/Ops.td b/include/flydsl/Dialect/FlyROCDL/IR/Ops.td -index 2b745065..875d8a69 100644 ---- a/include/flydsl/Dialect/FlyROCDL/IR/Ops.td -+++ b/include/flydsl/Dialect/FlyROCDL/IR/Ops.td -@@ -44,7 +44,7 @@ class FlyROCDL_MakeTiledTdmAtomOp - let results = (outs AnyType:$atom, AnyType:$coordTensor); - - let assemblyFormat = [{ -- $tensor `,` $smemLayout `,` $tiler attr-dict `:` type(operands) -+ $tensor `,` $smemLayout `,` $tiler prop-dict attr-dict `:` type(operands) - }]; - } - diff --git a/third_party/flydsl/mlir_property_ref.patch b/third_party/flydsl/mlir_property_ref.patch deleted file mode 100644 index 145bcc1b..00000000 --- a/third_party/flydsl/mlir_property_ref.patch +++ /dev/null @@ -1,128 +0,0 @@ -diff --git a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp b/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp -index 80d2fb3b..7981077c 100644 ---- a/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp -+++ b/lib/Conversion/FlyToROCDL/FlyToROCDL.cpp -@@ -13,6 +13,7 @@ - #include "mlir/Dialect/SCF/Transforms/Patterns.h" - #include "mlir/Dialect/Vector/IR/VectorOps.h" - #include "mlir/IR/BuiltinAttributes.h" -+#include "mlir/IR/BuiltinOps.h" - #include "mlir/IR/BuiltinTypes.h" - #include "mlir/Pass/Pass.h" - #include "mlir/Transforms/DialectConversion.h" -@@ -85,10 +86,10 @@ static unsigned getLLVMAlignment(fly::PointerType ptrTy) { - // chosen to avoid collisions with existing globals in the module. Shared by - // the static-LDS `make_ptr` and dynamic-LDS `get_dyn_shared` lowerings. - static LLVM::GlobalOp createLDSGlobal(ConversionPatternRewriter &rewriter, -- gpu::GPUModuleOp moduleOp, Location loc, StringRef prefix, -+ Block *moduleBody, Location loc, StringRef prefix, - int64_t nbytes, int64_t align, unsigned addrSpace) { - llvm::StringSet<> existingNames; -- for (auto globalOp : moduleOp.getBody()->getOps()) -+ for (auto globalOp : moduleBody->getOps()) - existingNames.insert(globalOp.getSymName()); - - unsigned counter = 0; -@@ -96,7 +97,7 @@ static LLVM::GlobalOp createLDSGlobal(ConversionPatternRewriter &rewriter, - prefix, [&](StringRef candidate) { return existingNames.contains(candidate); }, counter); - - OpBuilder::InsertionGuard guard(rewriter); -- rewriter.setInsertionPointToStart(moduleOp.getBody()); -+ rewriter.setInsertionPointToStart(moduleBody); - - auto arrayTy = LLVM::LLVMArrayType::get(IntegerType::get(rewriter.getContext(), 8), nbytes); - auto globalOp = LLVM::GlobalOp::create(rewriter, loc, arrayTy, -@@ -149,12 +150,17 @@ public: - return rewriter.notifyMatchFailure( - op, "shared make_ptr requires allocBytes, allocAlign in dictAttrs"); - -- auto moduleOp = op->getParentOfType(); -- if (!moduleOp) -+ Block *moduleBody = nullptr; -+ if (auto gpuModule = op->getParentOfType()) { -+ moduleBody = gpuModule.getBody(); -+ } else if (auto builtinModule = op->getParentOfType()) { -+ moduleBody = builtinModule.getBody(); -+ } -+ if (!moduleBody) - return op->emitError("shared make_ptr must be inside a gpu.module"); - - LLVM::GlobalOp global = -- createLDSGlobal(rewriter, moduleOp, loc, "__shared_alloc", allocBytesAttr.getInt(), -+ createLDSGlobal(rewriter, moduleBody, loc, "__shared_alloc", allocBytesAttr.getInt(), - allocAlignAttr.getInt(), /*addrSpace=*/3); - rewriter.replaceOpWithNewOp(op, global); - return success(); -@@ -191,15 +197,20 @@ public: - auto flyPtrTy = cast(op.getResult().getType()); - unsigned addrSpace = mapAttrToLLVMAddressSpace(flyPtrTy.getAddressSpace()); - -- auto moduleOp = op->getParentOfType(); -- if (!moduleOp) -+ Block *moduleBody = nullptr; -+ if (auto gpuModule = op->getParentOfType()) { -+ moduleBody = gpuModule.getBody(); -+ } else if (auto builtinModule = op->getParentOfType()) { -+ moduleBody = builtinModule.getBody(); -+ } -+ if (!moduleBody) - return op->emitError("get_dyn_shared must be inside a gpu.module"); - - // Dynamic shared memory has a single logical region per kernel; reuse an - // existing `[0 x i8]` global if one is already present so multiple - // `get_dyn_shared` ops resolve to the same base symbol. - LLVM::GlobalOp sharedGlobal; -- for (auto globalOp : moduleOp.getBody()->getOps()) { -+ for (auto globalOp : moduleBody->getOps()) { - if (auto arrayType = dyn_cast(globalOp.getType())) { - if (globalOp.getAddrSpace() == addrSpace && arrayType.getNumElements() == 0 && - globalOp.getAlignment().value_or(0) == 1024) { -@@ -209,7 +220,7 @@ public: - } - } - if (!sharedGlobal) { -- sharedGlobal = createLDSGlobal(rewriter, moduleOp, loc, "__dynamic_shared", -+ sharedGlobal = createLDSGlobal(rewriter, moduleBody, loc, "__dynamic_shared", - /*nbytes=*/0, /*align=*/1024, addrSpace); - } - -@@ -810,10 +821,6 @@ public: - if (Value tok = op.getAsyncToken()) - asyncTokenType = tok.getType(); - -- // There are two relevant builder signatures in this MLIR: -- // - (kernel, ..., asyncTokenType, asyncDependencies, clusterSize) -- // - (kernel, ..., asyncObject, clusterSize) -- // Pick the one that matches the original op structure. - if (Value asyncObj = adaptor.getAsyncObject()) { - if (!adaptor.getAsyncDependencies().empty()) - return rewriter.notifyMatchFailure( -@@ -821,13 +828,14 @@ public: - - rewriter.replaceOpWithNewOp( - op, kernelRef, grid, block, adaptor.getDynamicSharedMemorySize(), -- adaptor.getKernelOperands(), asyncObj, clusterSize); -+ adaptor.getKernelOperands(), asyncTokenType, ValueRange{}, asyncObj, clusterSize); - return success(); - } - - rewriter.replaceOpWithNewOp( - op, kernelRef, grid, block, adaptor.getDynamicSharedMemorySize(), -- adaptor.getKernelOperands(), asyncTokenType, adaptor.getAsyncDependencies(), clusterSize); -+ adaptor.getKernelOperands(), asyncTokenType, adaptor.getAsyncDependencies(), -+ /*asyncObject=*/nullptr, clusterSize); - return success(); - } - }; -diff --git a/lib/Dialect/Fly/Transforms/LayoutLowering.cpp b/lib/Dialect/Fly/Transforms/LayoutLowering.cpp ---- a/lib/Dialect/Fly/Transforms/LayoutLowering.cpp -+++ b/lib/Dialect/Fly/Transforms/LayoutLowering.cpp -@@ -2994,7 +2994,7 @@ - // A value reached twice on one walk (cyclic chain) cannot be bounded here. - if (!visited.insert(def).second) - return std::nullopt; -- auto pop = llvm::make_scope_exit([&] { visited.erase(def); }); -+ auto pop = llvm::scope_exit([&] { visited.erase(def); }); - - // Launch coordinates: ask the op for its own range. Every gpu index op - // implements InferIntRangeInterface, and that implementation already prefers diff --git a/third_party/flydsl/repo.bzl b/third_party/flydsl/repo.bzl deleted file mode 100644 index 144e5af5..00000000 --- a/third_party/flydsl/repo.bzl +++ /dev/null @@ -1,17 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -def repo(): - git_repository( - name = "flydsl", - remote = "https://github.com/ROCm/FlyDSL.git", - commit = "1941889400621f1fc3d8f03bada1a4e7380cdf7e", - build_file = "//third_party/flydsl:flydsl.bazel", - patches = [ - "//third_party/flydsl:atomic_exchange.patch", - "//third_party/flydsl:mlir_property_ref.patch", - "//third_party/flydsl:mlir_legacy_assembly.patch", - "//third_party/flydsl:vector_atomic.patch", - "//third_party/flydsl:xf32_mfma.patch", - ], - patch_args = ["-p1"], - ) diff --git a/third_party/flydsl/vector_atomic.patch b/third_party/flydsl/vector_atomic.patch deleted file mode 100644 index edd23c40..00000000 --- a/third_party/flydsl/vector_atomic.patch +++ /dev/null @@ -1,44 +0,0 @@ -diff --git a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp ---- a/lib/Dialect/Fly/IR/FlyUniversalOps.cpp -+++ b/lib/Dialect/Fly/IR/FlyUniversalOps.cpp -@@ -35,28 +35,36 @@ - return FxLayout(FxShape(FxC(1), FxC(getBitSize())), FxStride(FxC(1), FxC(1))); - } - -+static int32_t getAtomicValueBitWidth(Type type) { -+ if (auto vectorType = dyn_cast(type)) { -+ return vectorType.getNumElements() * -+ vectorType.getElementType().getIntOrFloatBitWidth(); -+ } -+ return type.getIntOrFloatBitWidth(); -+} -+ - bool CopyOpUniversalAtomicType::isStatic() const { return true; } - - Value CopyOpUniversalAtomicType::rebuildStaticValue(OpBuilder &builder, Location loc, - Value currentValue) const { - if (currentValue && isa(currentValue.getDefiningOp())) - return nullptr; -- int32_t bits = getValType().getIntOrFloatBitWidth(); -+ int32_t bits = getAtomicValueBitWidth(getValType()); - return MakeCopyAtomOp::create(builder, loc, CopyAtomType::get(*this, bits), bits); - } - - Attribute CopyOpUniversalAtomicType::getThrLayout() const { return FxLayout(FxC(1), FxC(1)); } - - Attribute CopyOpUniversalAtomicType::getThrBitLayoutSrc() const { -- int32_t bits = getValType().getIntOrFloatBitWidth(); -+ int32_t bits = getAtomicValueBitWidth(getValType()); - return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); - } - Attribute CopyOpUniversalAtomicType::getThrBitLayoutDst() const { -- int32_t bits = getValType().getIntOrFloatBitWidth(); -+ int32_t bits = getAtomicValueBitWidth(getValType()); - return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); - } - Attribute CopyOpUniversalAtomicType::getThrBitLayoutRef() const { -- int32_t bits = getValType().getIntOrFloatBitWidth(); -+ int32_t bits = getAtomicValueBitWidth(getValType()); - return FxLayout(FxShape(FxC(1), FxC(bits)), FxStride(FxC(1), FxC(1))); - } - diff --git a/third_party/flydsl/xf32_mfma.patch b/third_party/flydsl/xf32_mfma.patch deleted file mode 100644 index bc6126a0..00000000 --- a/third_party/flydsl/xf32_mfma.patch +++ /dev/null @@ -1,25 +0,0 @@ -diff --git a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp ---- a/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp -+++ b/lib/Dialect/FlyROCDL/CDNA3/MmaAtom.cpp -@@ -109,8 +109,12 @@ - static bool isBF8(Type ty) { return isa(ty); } - - static Type getMfmaABType(MLIRContext *ctx, Type elemTy, int32_t mn, int32_t k = 0) { -- if (elemTy.isF32()) -+ if (elemTy.isF32()) { -+ // v_mfma_f32_32x32x4_xf32 consumes two packed FP32 inputs per lane. -+ if (mn == 32 && k == 4) -+ return VectorType::get({2}, Float32Type::get(ctx)); - return Float32Type::get(ctx); -+ } - if (elemTy.isF16()) - return VectorType::get({mn * k / 64}, Float16Type::get(ctx)); - if (elemTy.isBF16()) { -@@ -179,6 +183,7 @@ - DISPATCH_MFMA_SSA(4, 1, elemTyA.isF32(), mfma_f32_4x4x1f32) - DISPATCH_MFMA_SSA(32, 2, elemTyA.isF32(), mfma_f32_32x32x2f32) - DISPATCH_MFMA_SSA(16, 4, elemTyA.isF32(), mfma_f32_16x16x4f32) -+ DISPATCH_MFMA_SSA(32, 4, elemTyA.isF32(), mfma_f32_32x32x4_xf32) - - DISPATCH_MFMA_SSA(32, 4, elemTyA.isF16(), mfma_f32_32x32x4f16) - DISPATCH_MFMA_SSA(16, 4, elemTyA.isF16(), mfma_f32_16x16x4f16) diff --git a/third_party/grpc/grpc++_unsecure-visibility.patch b/third_party/grpc/grpc++_unsecure-visibility.patch deleted file mode 100644 index 3b4013ab..00000000 --- a/third_party/grpc/grpc++_unsecure-visibility.patch +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/BUILD b/BUILD -index 0e4bda7e16..dbe1708421 100644 ---- a/BUILD -+++ b/BUILD -@@ -1236,7 +1236,7 @@ grpc_cc_library( - "avoid_dep", - "nofixdeps", - ], -- visibility = ["//bazel:grpc++_unsecure"], -+ visibility = ["//visibility:public"], - deps = [ - "channel_arg_names", - "global_callback_hook", diff --git a/third_party/grpc/plugin-flags.patch b/third_party/grpc/plugin-flags.patch deleted file mode 100644 index b69a77f5..00000000 --- a/third_party/grpc/plugin-flags.patch +++ /dev/null @@ -1,19 +0,0 @@ -# Match XLA's cc_grpc_library extension for services_namespace. ---- a/bazel/cc_grpc_library.bzl -+++ b/bazel/cc_grpc_library.bzl -@@ -29,6 +29,7 @@ - allow_deprecated = False, - use_external = False, # @unused - grpc_only = False, -+ plugin_flags = [], - **kwargs): - """Generates C++ grpc classes for services defined in a proto file. - -@@ -110,6 +111,7 @@ - well_known_protos = well_known_protos, - generate_mocks = generate_mocks, - allow_deprecated = allow_deprecated, -+ flags = plugin_flags, - **kwargs - ) - diff --git a/third_party/grpc/protobuf-35-upb-load.patch b/third_party/grpc/protobuf-35-upb-load.patch deleted file mode 100644 index 7311b981..00000000 --- a/third_party/grpc/protobuf-35-upb-load.patch +++ /dev/null @@ -1,14 +0,0 @@ -diff --git a/bazel/grpc_build_system.bzl b/bazel/grpc_build_system.bzl -index e0d47a29e1..e33f40d758 100644 ---- a/bazel/grpc_build_system.bzl -+++ b/bazel/grpc_build_system.bzl -@@ -31,7 +31,8 @@ load("@build_bazel_apple_support//rules:universal_binary.bzl", "universal_binary - load("@build_bazel_rules_apple//apple:ios.bzl", "ios_unit_test") - load("@build_bazel_rules_apple//apple/testing/default_runner:ios_test_runner.bzl", "ios_test_runner") - load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") --load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_proto_library", "upb_proto_reflection_library") -+load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", upb_proto_library = "upb_c_proto_library") -+load("@com_google_protobuf//bazel:upb_proto_reflection_library.bzl", "upb_proto_reflection_library") - load("@rules_cc//cc:cc_binary.bzl", "cc_binary") - load("@rules_cc//cc:cc_library.bzl", "cc_library") - load("@rules_cc//cc:cc_test.bzl", "cc_test") diff --git a/third_party/iree/fix-added-token-matching.patch b/third_party/iree/fix-added-token-matching.patch deleted file mode 100644 index 2a418780..00000000 --- a/third_party/iree/fix-added-token-matching.patch +++ /dev/null @@ -1,123 +0,0 @@ -diff --git a/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c b/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c -index 47d2a04c4d..66f1474c81 100644 ---- a/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c -+++ b/runtime/src/iree/tokenizer/format/huggingface/tokenizer_json.c -@@ -48,11 +48,12 @@ static const iree_string_view_t kTopLevelAllowedKeys[] = { - - // Builds special_tokens collections from added_tokens. - // --// ALL added tokens with `normalized=true` are included (not just those with --// `special=true`) because they must be matched BEFORE the segmenter runs. --// Without this, ByteLevel segmentation transforms the input before these --// tokens can match, breaking tokenizers like GPT-NeoX that have multi-space --// tokens (e.g., " " at ID 50276). -+// ALL added tokens are included (regardless of their `special` flag) because -+// they must be matched BEFORE the segmenter runs, mirroring HuggingFace's -+// AddedVocabulary. Without this, ByteLevel segmentation transforms the input -+// before these tokens can match, breaking tokenizers like GPT-NeoX that have -+// multi-space tokens (e.g., " " at ID 50276) and dropping non-special added -+// tokens such as `` or ``. - // - // Tokens are split by their normalized flag: - // - out_special_tokens: normalized=false, matched in raw input before -@@ -70,22 +71,10 @@ static iree_status_t iree_tokenizer_huggingface_build_special_tokens( - iree_tokenizer_special_tokens_initialize(out_special_tokens); - iree_tokenizer_special_tokens_initialize(out_special_tokens_post_norm); - -- // Count tokens to process: special tokens OR tokens with normalized=true. -- // Non-special tokens with normalized=true must be matched before -- // segmentation. -- iree_host_size_t match_count = 0; -- for (iree_host_size_t i = 0; i < added_tokens->count; ++i) { -- const iree_tokenizer_huggingface_added_token_t* token = -- iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); -- bool is_special = iree_any_bit_set( -- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL); -- bool is_normalized = iree_any_bit_set( -- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_NORMALIZED); -- // Include if special OR if normalized (needs pre-segmentation matching). -- if (is_special || is_normalized) { -- ++match_count; -- } -- } -+ // All added tokens are matched before segmentation, mirroring HuggingFace's -+ // AddedVocabulary which matches every added token regardless of its `special` -+ // flag. They are routed by their `normalized` flag below. -+ iree_host_size_t match_count = added_tokens->count; - - // Early exit if nothing to process. - if (match_count == 0) { -@@ -108,45 +97,38 @@ static iree_status_t iree_tokenizer_huggingface_build_special_tokens( - i < added_tokens->count && iree_status_is_ok(status); ++i) { - const iree_tokenizer_huggingface_added_token_t* token = - iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); -- bool is_special = iree_any_bit_set( -- token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL); - bool is_normalized = iree_any_bit_set( - token->flags, IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_NORMALIZED); - -- // Process if special OR if normalized (needs pre-segmentation matching). -- if (is_special || is_normalized) { -- iree_string_view_t content = -- iree_tokenizer_huggingface_added_token_content(added_tokens, token); -- -- // Convert HuggingFace flags to special_tokens flags. -- iree_tokenizer_special_token_flags_t special_flags = -- IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE; -- if (iree_any_bit_set( -- token->flags, -- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_LSTRIP)) { -- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_LSTRIP; -- } -- if (iree_any_bit_set( -- token->flags, -- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_RSTRIP)) { -- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_RSTRIP; -- } -- if (iree_any_bit_set( -- token->flags, -- IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SINGLE_WORD)) { -- special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_SINGLE_WORD; -- } -+ iree_string_view_t content = -+ iree_tokenizer_huggingface_added_token_content(added_tokens, token); - -- // Route to appropriate builder based on normalized flag. -- if (is_normalized) { -- // normalized=true: match after normalization but before segmentation. -- status = iree_tokenizer_special_tokens_builder_add( -- &builder_post_norm, content, token->id, special_flags); -- } else { -- // normalized=false (default): match before normalization. -- status = iree_tokenizer_special_tokens_builder_add( -- &builder_pre_norm, content, token->id, special_flags); -- } -+ // Convert HuggingFace flags to special_tokens flags. -+ iree_tokenizer_special_token_flags_t special_flags = -+ IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE; -+ if (iree_any_bit_set(token->flags, -+ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_LSTRIP)) { -+ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_LSTRIP; -+ } -+ if (iree_any_bit_set(token->flags, -+ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_RSTRIP)) { -+ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_RSTRIP; -+ } -+ if (iree_any_bit_set( -+ token->flags, -+ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SINGLE_WORD)) { -+ special_flags |= IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_SINGLE_WORD; -+ } -+ -+ // Route to appropriate builder based on normalized flag. -+ if (is_normalized) { -+ // normalized=true: match after normalization but before segmentation. -+ status = iree_tokenizer_special_tokens_builder_add( -+ &builder_post_norm, content, token->id, special_flags); -+ } else { -+ // normalized=false (default): match before normalization. -+ status = iree_tokenizer_special_tokens_builder_add( -+ &builder_pre_norm, content, token->id, special_flags); - } - } - diff --git a/third_party/iree/fix-gemma4-encode.patch b/third_party/iree/fix-gemma4-encode.patch deleted file mode 100644 index 18f493e6..00000000 --- a/third_party/iree/fix-gemma4-encode.patch +++ /dev/null @@ -1,149 +0,0 @@ -From 96151c5c077dc56c00e2b608c4f39b5553089515 Mon Sep 17 00:00:00 2001 -From: Benjamin Rabier -Date: Thu, 16 Apr 2026 09:07:02 +0000 -Subject: [PATCH] Fix tokenization - -Signed-off-by: Benjamin Rabier ---- - runtime/src/iree/tokenizer/tokenizer.c | 13 ++- - .../iree/tokenizer/tokenizer_encode_test.cc | 100 ++++++++++++++++++ - 2 files changed, 109 insertions(+), 4 deletions(-) - -diff --git a/runtime/src/iree/tokenizer/tokenizer.c b/runtime/src/iree/tokenizer/tokenizer.c -index ff9fe6242e..37df477c9d 100644 ---- a/runtime/src/iree/tokenizer/tokenizer.c -+++ b/runtime/src/iree/tokenizer/tokenizer.c -@@ -2606,10 +2606,15 @@ static iree_status_t iree_tokenizer_encode_state_pump( - state->segmenter_view_start = state->read_position; - } - } -- // When a forced-complete drains the ring, exit partial mode so fresh -- // text after the special token goes through the normal segmenter path. -- if (force_complete && segment_complete && -- state->read_position >= state->write_position) { -+ // When a forced-complete segment finishes, retire the entire ring region -+ // that was just fed directly to the model. Unlike the partial reclaim path -+ // above, a non-partial completion resets the model to SEGMENT_START, so -+ // state_reclaim() returns 0 even though all bytes were fully processed. -+ // If we leave the ring positions unchanged, the same bytes are re-encoded -+ // on the next pump iteration and a pending special token can never flush. -+ if (force_complete && segment_complete) { -+ state->read_position = state->write_position; -+ state->segmenter_view_start = state->write_position; - state->has_partial_segment = false; - } - if (reclaimed > 0 || tokens_written > 0) { -diff --git a/runtime/src/iree/tokenizer/tokenizer_encode_test.cc b/runtime/src/iree/tokenizer/tokenizer_encode_test.cc -index a28811fe9b..0eb3f68dc8 100644 ---- a/runtime/src/iree/tokenizer/tokenizer_encode_test.cc -+++ b/runtime/src/iree/tokenizer/tokenizer_encode_test.cc -@@ -3152,6 +3152,106 @@ TEST_F(TokenizerSpecialTokenStreamingTest, FinalizeDoesNotSilentlyDropPartial) { - iree_tokenizer_free(tokenizer); - } - -+// Regression test for a deferred pre-norm special token that causes an infinite -+// loop when the pump enters partial-segment mode. -+// -+// The bug: after a force-complete of ring content (triggered by a pending -+// special token), state_reclaim() returns 0 because the model resets to -+// SEGMENT_START on non-partial completion. The old code required -+// read_position >= write_position to clear partial mode and advance positions, -+// but since reclaim never advanced read_position, this condition was always -+// false — creating an infinite loop re-encoding the same bytes. -+// -+// This test reproduces the real-world trigger: a Replace normalizer that -+// expands " " → "▁" (1 byte → 3 bytes). With a Split segmenter on space -+// (MergedWithPrevious), each word+space is a small segment, but the normalizer -+// expansion means N bytes of input produce up to 3N bytes of normalized output. -+// When enough text with spaces precedes a special token, the normalized output -+// overflows the ring buffer's logical capacity, forcing partial-segment mode. -+// The pending special token then triggers force_complete, exposing the bug. -+// -+// With a 128-byte ring (64-byte logical capacity), ~40 bytes of space-heavy -+// input normalizes to >64 bytes, reliably triggering the overflow. -+TEST_F(TokenizerSpecialTokenStreamingTest, -+ DeferredSpecialTokenDoesNotReplayForcedCompleteSegment) { -+ // ▁ (U+2581) = 0xE2 0x96 0x81 (3 bytes in UTF-8). -+ static const char kMetaspace[] = "\xE2\x96\x81"; -+ -+ ScopedVocabBuilder vocab_builder(257); -+ for (int i = 0; i < 256; ++i) { -+ char byte_token[2] = {static_cast(i), '\0'}; -+ vocab_builder.AddToken(i, byte_token); -+ } -+ // Add ▁ as a token so the BPE model can encode normalized spaces. -+ vocab_builder.AddToken(256, kMetaspace); -+ vocab_builder.AddToken(50256, "<|tool>"); -+ ScopedVocab vocab = vocab_builder.Build(); -+ -+ iree_tokenizer_special_tokens_builder_t st_builder; -+ iree_tokenizer_special_tokens_builder_initialize(iree_allocator_system(), -+ &st_builder); -+ IREE_ASSERT_OK(iree_tokenizer_special_tokens_builder_add( -+ &st_builder, IREE_SV("<|tool>"), 50256, -+ IREE_TOKENIZER_SPECIAL_TOKEN_FLAG_NONE)); -+ iree_tokenizer_special_tokens_t special_tokens; -+ IREE_ASSERT_OK(iree_tokenizer_special_tokens_builder_build( -+ &st_builder, iree_allocator_system(), &special_tokens)); -+ iree_tokenizer_special_tokens_builder_deinitialize(&st_builder); -+ -+ // Replace normalizer: " " → "▁" (1→3 byte expansion, matching Gemma4). -+ iree_tokenizer_normalizer_t* replace_normalizer = nullptr; -+ IREE_ASSERT_OK(iree_tokenizer_normalizer_replace_allocate( -+ IREE_SV(" "), iree_make_string_view(kMetaspace, 3), -+ iree_allocator_system(), &replace_normalizer)); -+ -+ // Split on literal space with MergedWithPrevious (matching Gemma4 config). -+ iree_tokenizer_segmenter_t* segmenter = CreateSplitSegmenterWithBehavior( -+ " ", IREE_TOKENIZER_UTIL_REGEX_SPLIT_MERGED_WITH_PREVIOUS); -+ ASSERT_NE(segmenter, nullptr); -+ -+ ScopedBuilder builder; -+ iree_tokenizer_builder_set_normalizer(builder.get(), replace_normalizer); -+ iree_tokenizer_builder_set_segmenter(builder.get(), segmenter); -+ iree_tokenizer_builder_set_model(builder.get(), CreateBPEModel(vocab.get())); -+ iree_tokenizer_builder_set_special_tokens(builder.get(), &special_tokens); -+ iree_tokenizer_builder_set_vocab(builder.get(), vocab.release()); -+ -+ iree_tokenizer_t* tokenizer = BuildTokenizer(builder.get()); -+ ASSERT_NE(tokenizer, nullptr); -+ -+ // Normal text with many spaces. Each space expands from 1→3 bytes after -+ // normalization. With a 128-byte ring (64-byte logical capacity), even this -+ // short text overflows the ring due to normalizer expansion, forcing -+ // partial-segment mode. The special token then triggers force_complete. -+ std::string input = -+ "A knight of powder-horn and shot " -+ "Once fill'd his bag as I would not " -+ "Unless the feelings of my breast " -+ "By poverty were sorely press'd " -+ "With birds and squirrels for the spits " -+ "Of certain gormandizing cits " -+ "With merry heart the fellow went "; -+ input += "<|tool>d"; -+ -+ IREE_ASSERT_OK_AND_ASSIGN(auto large_tokens, -+ EncodeWithBufferSize(tokenizer, input, 1024)); -+ IREE_ASSERT_OK_AND_ASSIGN(auto small_tokens, -+ EncodeWithBufferSize(tokenizer, input, 256)); -+ -+ EXPECT_EQ(small_tokens, large_tokens) -+ << "Small-buffer streaming must match large-buffer streaming when a " -+ "deferred special token follows text with normalizer expansion"; -+ -+ // Verify the special token and trailing byte are present at the end. -+ ASSERT_GE(small_tokens.size(), 2u); -+ EXPECT_EQ(small_tokens[small_tokens.size() - 2], 50256u); -+ EXPECT_EQ(small_tokens[small_tokens.size() - 1], -+ static_cast('d')); -+ -+ iree_tokenizer_special_tokens_deinitialize(&special_tokens); -+ iree_tokenizer_free(tokenizer); -+} -+ - //===----------------------------------------------------------------------===// - // Strip Normalizer + Special Token Integration Tests - //===----------------------------------------------------------------------===// --- -2.53.0 - diff --git a/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch b/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch deleted file mode 100644 index 194a7b87..00000000 --- a/third_party/iree/fix-normalizer-lifetime-and-boundaries.patch +++ /dev/null @@ -1,74 +0,0 @@ -diff --git a/runtime/src/iree/tokenizer/normalizer/sequence.c b/runtime/src/iree/tokenizer/normalizer/sequence.c ---- a/runtime/src/iree/tokenizer/normalizer/sequence.c -+++ b/runtime/src/iree/tokenizer/normalizer/sequence.c -@@ -188,6 +188,14 @@ - "%" PRIhsz " exceeds maximum %" PRIu32, - flattened_count, - IREE_TOKENIZER_NORMALIZER_SEQUENCE_MAX_DEPTH); -+ } -+ -+ // Allocate before transferring ownership, including flattening nested -+ // sequences. On failure the caller must retain all original children. -+ iree_tokenizer_normalizer_sequence_t* normalizer = NULL; -+ if (flattened_count != 1) { -+ IREE_RETURN_IF_ERROR(iree_allocator_malloc( -+ allocator, sizeof(*normalizer), (void**)&normalizer)); - } - - // Build flattened children array, transferring ownership as we go. -@@ -232,18 +240,6 @@ - state_size = iree_host_align(state_size, iree_max_align_t); - child_state_offsets[i] = state_size; - state_size += iree_tokenizer_normalizer_state_size(flattened_children[i]); -- } -- -- // Allocate the sequence normalizer. -- iree_tokenizer_normalizer_sequence_t* normalizer = NULL; -- iree_status_t status = iree_allocator_malloc(allocator, sizeof(*normalizer), -- (void**)&normalizer); -- if (!iree_status_is_ok(status)) { -- // Cleanup: we've taken ownership of the children, so free them on failure. -- for (iree_host_size_t i = 0; i < flattened_count; ++i) { -- iree_tokenizer_normalizer_free(flattened_children[i]); -- } -- return status; - } - - // Initialize base. -@@ -566,6 +562,15 @@ - if (tile_size > IREE_TOKENIZER_NORMALIZER_SEQUENCE_MIN_BATCH) { - // Limit tile to what we can handle in scratch. - tile_size = IREE_TOKENIZER_NORMALIZER_SEQUENCE_MIN_BATCH; -+ // Never split a UTF-8 codepoint between child process calls. -+ while (tile_size > 0 && -+ ((uint8_t)remaining_input.data[tile_size] & 0xC0) == 0x80) { -+ --tile_size; -+ } -+ } -+ iree_tokenizer_normalizer_flags_t tile_flags = flags; -+ if (tile_size < remaining_input.size) { -+ tile_flags &= ~IREE_TOKENIZER_NORMALIZER_FLAG_SEGMENT_END; - } - - // Process one tile through all children. -@@ -574,7 +579,7 @@ - iree_host_size_t tile_consumed = 0; - iree_host_size_t tile_written = 0; - IREE_RETURN_IF_ERROR(iree_tokenizer_normalizer_sequence_process_tile( -- state, tile_input, scratch, scratch_size, remaining_output, flags, -+ state, tile_input, scratch, scratch_size, remaining_output, tile_flags, - &tile_consumed, &tile_written)); - - if (tile_consumed == 0 && tile_written == 0) { -diff --git a/runtime/src/iree/tokenizer/normalizer/strip.c b/runtime/src/iree/tokenizer/normalizer/strip.c ---- a/runtime/src/iree/tokenizer/normalizer/strip.c -+++ b/runtime/src/iree/tokenizer/normalizer/strip.c -@@ -353,7 +353,7 @@ - } - - // Phase 3: Handle segment end. -- if (is_segment_end) { -+ if (is_segment_end && in_ptr == in_end) { - // Pending whitespace is trailing - discard it. - iree_tokenizer_normalizer_strip_clear_pending(state); - state->at_start = true; diff --git a/third_party/iree/futex-public-api-header.patch b/third_party/iree/futex-public-api-header.patch deleted file mode 100644 index 7623f422..00000000 --- a/third_party/iree/futex-public-api-header.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/runtime/src/iree/base/threading/futex.h b/runtime/src/iree/base/threading/futex.h -index 9b42d3cfd0..f2509b77e9 100644 ---- a/runtime/src/iree/base/threading/futex.h -+++ b/runtime/src/iree/base/threading/futex.h -@@ -30,11 +30,8 @@ - - #include - --#include "iree/base/attributes.h" --#include "iree/base/config.h" --#include "iree/base/status.h" -+#include "iree/base/api.h" - #include "iree/base/target_platform.h" --#include "iree/base/time.h" - - //===----------------------------------------------------------------------===// - // Platform detection diff --git a/third_party/iree/laguna-tokenizer-compat.patch b/third_party/iree/laguna-tokenizer-compat.patch deleted file mode 100644 index c978c591..00000000 --- a/third_party/iree/laguna-tokenizer-compat.patch +++ /dev/null @@ -1,78 +0,0 @@ -diff --git a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c ---- a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c -+++ b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c -@@ -217,0 +218,10 @@ -+ // Some code-oriented tokenizers use this lookahead split only to attach -+ // single-newline runs to the next segment. The IREE regex compiler does not -+ // support lookahead; the following ByteLevel/GPT split still preserves the -+ // input bytes, so treat this grouping hint as a no-op instead of rejecting -+ // the tokenizer at load time. -+ if (iree_string_view_equal( -+ pattern, IREE_SV("(?:\\r?\\n)+(?!\\r?\\n)")) && -+ behavior == IREE_TOKENIZER_UTIL_REGEX_SPLIT_MERGED_WITH_NEXT && !invert) { -+ return iree_ok_status(); -+ } -diff --git a/runtime/src/iree/tokenizer/format/huggingface/model_json.c b/runtime/src/iree/tokenizer/format/huggingface/model_json.c ---- a/runtime/src/iree/tokenizer/format/huggingface/model_json.c -+++ b/runtime/src/iree/tokenizer/format/huggingface/model_json.c -@@ -444,0 +445,46 @@ -+static char iree_tokenizer_ascii_upper(char c) { -+ return c >= 'a' && c <= 'z' ? (char)(c - ('a' - 'A')) : c; -+} -+ -+static bool iree_tokenizer_string_view_contains_ascii_case( -+ iree_string_view_t value, iree_string_view_t needle) { -+ if (needle.size == 0 || needle.size > value.size) return false; -+ for (iree_host_size_t i = 0; i <= value.size - needle.size; ++i) { -+ bool match = true; -+ for (iree_host_size_t j = 0; j < needle.size; ++j) { -+ if (iree_tokenizer_ascii_upper(value.data[i + j]) != -+ iree_tokenizer_ascii_upper(needle.data[j])) { -+ match = false; -+ break; -+ } -+ } -+ if (match) return true; -+ } -+ return false; -+} -+ -+static bool iree_tokenizer_huggingface_find_added_unk_token( -+ const iree_tokenizer_huggingface_added_tokens_t* added_tokens, -+ iree_tokenizer_token_id_t* out_token_id) { -+ *out_token_id = IREE_TOKENIZER_TOKEN_ID_INVALID; -+ if (!added_tokens) return false; -+ for (iree_host_size_t i = 0; i < added_tokens->count; ++i) { -+ const iree_tokenizer_huggingface_added_token_t* token = -+ iree_tokenizer_huggingface_added_tokens_get(added_tokens, i); -+ if (!iree_any_bit_set( -+ token->flags, -+ IREE_TOKENIZER_HUGGINGFACE_ADDED_TOKEN_FLAG_SPECIAL)) { -+ continue; -+ } -+ iree_string_view_t content = -+ iree_tokenizer_huggingface_added_token_content(added_tokens, token); -+ if (!iree_tokenizer_string_view_contains_ascii_case(content, -+ IREE_SV("unk"))) { -+ continue; -+ } -+ *out_token_id = token->id; -+ return true; -+ } -+ return false; -+} -+ -@@ -584,3 +630,10 @@ -- status = iree_make_status(IREE_STATUS_NOT_FOUND, -- "unk_token '%.*s' not found in vocabulary", -- (int)unk_length, unk_buffer); -+ // Some tokenizer.json files keep a stale default `unk_token` in the BPE -+ // model section while declaring the real unknown token in added_tokens -+ // (for example, custom bracketed forms such as `〈|UNK|〉`). Prefer that -+ // explicit special added token before failing the whole tokenizer load. -+ if (!iree_tokenizer_huggingface_find_added_unk_token(added_tokens, -+ &unk_token_id)) { -+ status = iree_make_status(IREE_STATUS_NOT_FOUND, -+ "unk_token '%.*s' not found in vocabulary", -+ (int)unk_length, unk_buffer); -+ } diff --git a/third_party/iree/match-hf-tokenizer.patch b/third_party/iree/match-hf-tokenizer.patch deleted file mode 100644 index 4cbab21a..00000000 --- a/third_party/iree/match-hf-tokenizer.patch +++ /dev/null @@ -1,1643 +0,0 @@ -diff --git a/runtime/src/iree/tokenizer/model/bpe.c b/runtime/src/iree/tokenizer/model/bpe.c -index 309f1e261f..02fb055eee 100644 ---- a/runtime/src/iree/tokenizer/model/bpe.c -+++ b/runtime/src/iree/tokenizer/model/bpe.c -@@ -179,16 +179,17 @@ iree_status_t iree_tokenizer_bpe_model_allocate( - // Backtracking path capacities. - // Segments up to this size use the O(n) backtracking algorithm. - // Larger segments fall back to the O(n log L) window+heap path. -- // Capped at 4095 so the bitfield (ceil_div(4096, 64) = 64 words) fits in -- // a single uint64_t dirty mask for O(1) amortized per-segment init. -+ // Capped at 32768 to keep exact backtracking state bounded while covering -+ // large normalized prompts that still need HuggingFace-compatible -+ // suffix/prefix merge validation. - iree_host_size_t min_backtrack = 0; - if (!iree_host_size_checked_mul(16, model->max_token_length, - &min_backtrack)) { - // Overflow implies very large token length; clamp to maximum. -- min_backtrack = 4095; -+ min_backtrack = 32768; - } -- if (min_backtrack < 2048) min_backtrack = 2048; -- if (min_backtrack > 4095) min_backtrack = 4095; -+ if (min_backtrack < 32768) min_backtrack = 32768; -+ if (min_backtrack > 32768) min_backtrack = 32768; - model->max_backtrack_segment_bytes = min_backtrack; - // Stack holds at most one token per byte (worst case: all single-byte - // tokens). -@@ -311,6 +312,7 @@ static void iree_tokenizer_bpe_model_destroy( - iree_tokenizer_vocab_trie_free(model->trie); - iree_tokenizer_vocab_merge_hash_free(model->merge_hash); - iree_allocator_free(allocator, model->backtrack_tables.slab); -+ iree_allocator_free(allocator, model->backtrack_tables.alternate_suffix_slab); - iree_allocator_free(allocator, model); - - IREE_TRACE_ZONE_END(z0); -diff --git a/runtime/src/iree/tokenizer/model/bpe_backtrack.c b/runtime/src/iree/tokenizer/model/bpe_backtrack.c -index 7f2cab797c..02b272838f 100644 ---- a/runtime/src/iree/tokenizer/model/bpe_backtrack.c -+++ b/runtime/src/iree/tokenizer/model/bpe_backtrack.c -@@ -81,10 +81,9 @@ typedef struct iree_tokenizer_bpe_validation_frame_t { - static bool iree_tokenizer_bpe_is_valid_token_pair( - const iree_tokenizer_bpe_model_t* model, uint32_t token1, uint32_t token2, - uint32_t deferred_merge_rank) { -- const iree_tokenizer_vocab_t* vocab = model->vocab; -- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; - const iree_tokenizer_bpe_split_entry_t* split_table = - model->backtrack_tables.split_table; -+ const iree_tokenizer_vocab_t* vocab = model->vocab; - - iree_tokenizer_bpe_validation_frame_t - stack[IREE_TOKENIZER_BPE_PAIR_VALIDATION_STACK_CAPACITY]; -@@ -111,7 +110,7 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( - (int32_t)frame->token1, - (int32_t)frame->token2); - if (merge.result_id >= 0) { -- uint32_t combined_rank = effective_rank[(uint32_t)merge.result_id]; -+ uint32_t combined_rank = merge.rank + 1; - // A merge is invalid if it should have fired (rank < limit) UNLESS - // it was intentionally deferred for a better suffix merge. When - // suffix_blocked triggers, it reports the rank of the blocked token; -@@ -140,7 +139,8 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( - // Need to split at least one token. Determine order based on rank. - // Higher-ranked tokens (formed later in BPE) are split first. - bool try_token1_first = -- effective_rank[frame->token1] > effective_rank[frame->token2]; -+ iree_tokenizer_bpe_token_formation_rank(model, frame->token1) > -+ iree_tokenizer_bpe_token_formation_rank(model, frame->token2); - - if (try_token1_first) { - frame->side = -@@ -172,16 +172,26 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( - if (split.left_id != decompose_token) { - // Non-base token: split_table provides (left_id, right_id) directly. - // By construction, merge(left_id, right_id) == decompose_token. -+ uint32_t decomposition_rank = -+ iree_tokenizer_bpe_token_formation_rank(model, decompose_token); -+ iree_tokenizer_merge_hash_result_t split_merge = -+ iree_tokenizer_vocab_merge_hash_lookup( -+ model->merge_hash, (int32_t)split.left_id, -+ (int32_t)split.right_id); -+ if (split_merge.result_id == (int32_t)decompose_token) { -+ decomposition_rank = split_merge.rank + 1; -+ } -+ - uint32_t new_limit; - uint32_t new_token1, new_token2; - if (frame->side == 1) { - new_token1 = split.right_id; - new_token2 = frame->token2; -- new_limit = effective_rank[frame->token1]; -+ new_limit = decomposition_rank; - } else { - new_token1 = frame->token1; - new_token2 = split.left_id; -- new_limit = effective_rank[frame->token2] + 1; -+ new_limit = decomposition_rank + 1; - } - - if (stack_count < IREE_TOKENIZER_BPE_PAIR_VALIDATION_STACK_CAPACITY) { -@@ -243,16 +253,17 @@ static bool iree_tokenizer_bpe_is_valid_token_pair( - if (merge.result_id != (int32_t)decompose_token) continue; - - // Valid decomposition found. Compute new limit and push child frame. -+ uint32_t decomposition_rank = merge.rank + 1; - uint32_t new_limit; - uint32_t new_token1, new_token2; - if (frame->side == 1) { - new_token1 = (uint32_t)right_id; - new_token2 = frame->token2; -- new_limit = effective_rank[frame->token1]; -+ new_limit = decomposition_rank; - } else { - new_token1 = frame->token1; - new_token2 = (uint32_t)left_id; -- new_limit = effective_rank[frame->token2] + 1; -+ new_limit = decomposition_rank + 1; - } - - // Save resume position for backtracking. -@@ -381,160 +392,202 @@ static int32_t iree_tokenizer_bpe_find_first_token_at( - static int32_t iree_tokenizer_bpe_token_at_position( - const iree_tokenizer_bpe_model_t* model, const uint8_t* data, - iree_host_size_t size, iree_host_size_t* out_length) { -- int32_t token = iree_tokenizer_bpe_single_char_token(model, data[0]); -- if (token >= 0) { -- if (out_length) *out_length = 1; -- return token; -+ return iree_tokenizer_bpe_base_token_at_position(model, data, size, -+ out_length); -+} -+ -+static bool iree_tokenizer_bpe_is_suffix_blocked( -+ const iree_tokenizer_bpe_model_t* model, int32_t token, -+ const uint8_t* remaining_data, iree_host_size_t remaining_size, -+ const char* suffix_data, iree_host_size_t suffix_length, -+ uint32_t* out_blocking_rank); -+static bool iree_tokenizer_bpe_is_suffix_blocked_impl( -+ const iree_tokenizer_bpe_model_t* model, int32_t token, -+ const uint8_t* remaining_data, iree_host_size_t remaining_size, -+ const char* suffix_data, iree_host_size_t suffix_length, -+ uint32_t* out_blocking_rank, int depth); -+ -+// Recursive implementation for iree_tokenizer_bpe_is_token_consumed_rightward. -+// -+// |ready_rank_plus_one| tracks when the right compound is available. Base -+// tokens are available at rank 0; a compound built by an internal merge at -+// rank N is available starting at N+1, so an outer merge at rank R can use it -+// iff ready_rank_plus_one <= R. -+static bool iree_tokenizer_bpe_is_token_consumed_rightward_impl( -+ const iree_tokenizer_bpe_model_t* model, int32_t token, -+ const uint8_t* remaining_data, iree_host_size_t remaining_size, -+ iree_host_size_t token_end, uint32_t max_rank, int depth) { -+ if (token_end >= remaining_size) return false; -+ if (depth >= 8) return false; -+ -+ const iree_tokenizer_bpe_split_entry_t* split_table = -+ model->backtrack_tables.split_table; -+ const bool byte_level = -+ iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); -+ -+ iree_tokenizer_trie_cursor_t cursor; -+ iree_tokenizer_trie_cursor_reset(&cursor, model->trie); -+ -+ for (iree_host_size_t i = token_end; i < remaining_size; ++i) { -+ if (!iree_tokenizer_bpe_trie_advance_byte(&cursor, remaining_data[i], -+ byte_level)) { -+ break; -+ } -+ -+ int32_t compound_token = iree_tokenizer_trie_cursor_token_id(&cursor); -+ if (compound_token < 0) continue; -+ -+ iree_host_size_t compound_end = i + 1; -+ iree_tokenizer_merge_hash_result_t merge = -+ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, token, -+ compound_token); -+ if (!iree_tokenizer_merge_hash_result_is_valid(merge) || -+ merge.rank >= max_rank) { -+ continue; -+ } -+ -+ uint32_t compound = (uint32_t)compound_token; -+ bool compound_is_base = split_table[compound].left_id == compound; -+ uint32_t compound_ready_rank = -+ iree_tokenizer_bpe_token_formation_rank(model, compound); -+ if (compound_ready_rank > merge.rank) continue; -+ if (!compound_is_base && -+ !iree_tokenizer_bpe_is_first_token_reachable(model, compound)) { -+ continue; -+ } -+ -+ if (!compound_is_base && compound_end < remaining_size) { -+ uint32_t rightmost_base = -+ iree_tokenizer_bpe_rightmost_base_token(model, compound); -+ uint32_t right_consumed = -+ iree_tokenizer_bpe_right_boundary_consumed_rank(model, compound); -+ if (right_consumed > 0 && -+ iree_tokenizer_bpe_is_token_consumed_rightward_impl( -+ model, (int32_t)rightmost_base, remaining_data, remaining_size, -+ compound_end, right_consumed - 1, depth + 1)) { -+ continue; -+ } -+ } -+ -+ uint32_t result_token = (uint32_t)merge.result_id; -+ if (compound_end < remaining_size && depth < 4) { -+ uint32_t blocking_rank = 0; -+ if (iree_tokenizer_bpe_is_suffix_blocked_impl( -+ model, (int32_t)result_token, remaining_data + compound_end, -+ remaining_size - compound_end, NULL, 0, &blocking_rank, -+ depth + 1)) { -+ continue; -+ } -+ } -+ -+ // merge(token, compound) would consume token if the compound is not itself -+ // consumed to the right before that merge rank. -+ if (!iree_tokenizer_bpe_is_token_consumed_rightward_impl( -+ model, compound_token, remaining_data, remaining_size, compound_end, -+ merge.rank, depth + 1)) { -+ return true; -+ } - } -- return iree_tokenizer_bpe_find_first_token_at(model, data, size, 8, -- out_length); -+ return false; - } - - // Checks whether a token at a given position is consumed by a rightward merge - // at rank lower than |max_rank|. This means merge(token, following) exists and - // fires before any merge at |max_rank| would. - // --// To verify the rightward merge actually fires, also checks that the following --// token isn't itself consumed by an even lower-rank merge. This handles chains --// like: merge(c,h) would consume c, but merge(h,X) at lower rank consumes h --// first — so merge(c,h) can't fire and c survives. -+// The following token may itself be a compound formed by lower-rank rightward -+// merges (for example t+ry after r+y has formed). This bounded search mirrors -+// the short compound lookahead used by suffix-blocking preemption. - static bool iree_tokenizer_bpe_is_token_consumed_rightward( - const iree_tokenizer_bpe_model_t* model, int32_t token, - const uint8_t* remaining_data, iree_host_size_t remaining_size, - iree_host_size_t token_end, uint32_t max_rank) { -- if (token_end >= remaining_size) return false; -- iree_host_size_t following_length = 1; -- int32_t following_token = iree_tokenizer_bpe_token_at_position( -- model, remaining_data + token_end, remaining_size - token_end, -- &following_length); -- if (following_token < 0) return false; -- -- iree_tokenizer_merge_hash_result_t merge = -- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, token, -- following_token); -- if (!iree_tokenizer_merge_hash_result_is_valid(merge) || -- merge.rank >= max_rank) { -- return false; -- } -- -- // merge(token, following) would fire. Verify following is available: if -- // merge(following, next_after) fires at even lower rank, following is -- // consumed and merge(token, following) can't fire. -- iree_host_size_t next_after_end = token_end + following_length; -- if (next_after_end < remaining_size) { -- int32_t next_after_token = iree_tokenizer_bpe_token_at_position( -- model, remaining_data + next_after_end, remaining_size - next_after_end, -- NULL); -- if (next_after_token >= 0) { -- iree_tokenizer_merge_hash_result_t following_merge = -- iree_tokenizer_vocab_merge_hash_lookup( -- model->merge_hash, following_token, next_after_token); -- if (iree_tokenizer_merge_hash_result_is_valid(following_merge) && -- following_merge.rank < merge.rank) { -- return false; // Following consumed first, token survives. -- } -- } -- } -- return true; -+ return iree_tokenizer_bpe_is_token_consumed_rightward_impl( -+ model, token, remaining_data, remaining_size, token_end, max_rank, 0); - } - - // Checks if a suffix merge is preempted by lower-rank merges in the following - // input. Returns true if the suffix merge would NOT actually fire because the - // prefix token would be consumed first. - // --// Two preemption paths: --// Direct: merge(prefix, next) at rank < suffix_merge_rank, where the next --// token is available (not consumed by a lower-rank rightward merge). --// Compound: merge(prefix, compound) where compound is built by progressively --// merging characters: next+third->c1, c1+fourth->c2, etc. This handles --// cases where the direct merge can't fire but the prefix is still consumed --// via a deeper compound (e.g., merge(l, ish) when i+s fires first). --// --// For non-byte-level tokenizers (e.g., SentencePiece), uses trie lookup to find --// tokens when single-char lookup fails. This handles multi-byte base tokens --// like metaspace (E2 96 81). -+// This delegates to the same bounded rightward-consumption search used to -+// validate prefix availability. That search handles direct and compound merges -+// while checking that each compound component is actually available at the rank -+// where it would be used. - static bool iree_tokenizer_bpe_is_suffix_merge_preempted( - const iree_tokenizer_bpe_model_t* model, int32_t prefix_token, - uint32_t suffix_merge_rank, const uint8_t* remaining_data, - iree_host_size_t remaining_size, iree_host_size_t prefix_end) { -- if (prefix_end >= remaining_size) return false; -- -- iree_host_size_t next_length = 1; -- int32_t next_token = iree_tokenizer_bpe_token_at_position( -- model, remaining_data + prefix_end, remaining_size - prefix_end, -- &next_length); -- if (next_token < 0) return false; -- -- // Direct preemption: merge(prefix, next) at rank < suffix_merge_rank. -- // Only valid if next is actually available (not consumed rightward). -- iree_tokenizer_merge_hash_result_t prefix_next = -- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, prefix_token, -- next_token); -- if (iree_tokenizer_merge_hash_result_is_valid(prefix_next) && -- prefix_next.rank < suffix_merge_rank) { -- if (!iree_tokenizer_bpe_is_token_consumed_rightward( -- model, next_token, remaining_data, remaining_size, -- prefix_end + next_length, prefix_next.rank)) { -+ return iree_tokenizer_bpe_is_token_consumed_rightward( -+ model, prefix_token, remaining_data, remaining_size, prefix_end, -+ suffix_merge_rank); -+} -+ -+static bool iree_tokenizer_bpe_suffix_already_collected( -+ const uint32_t* suffixes, const uint32_t* suffix_consumed_at, -+ iree_host_size_t suffix_count, uint32_t suffix, uint32_t consumed_at) { -+ for (iree_host_size_t i = 0; i < suffix_count; ++i) { -+ if (suffixes[i] == suffix && suffix_consumed_at[i] == consumed_at) { - return true; - } - } -+ return false; -+} - -- // Compound preemption: build progressively longer compounds from the -- // characters following the prefix, checking merge(prefix, compound) at each -- // step. This catches cases where the direct merge can't fire but the prefix -- // merges with a multi-character compound that forms at lower rank. -- // -- // Example (Whisper "establish"): -- // prefix=l, next=i, merge(l,i) can't fire (i consumed by i+s at rank 15) -- // compound: i+s->is (rank 15), is+h->ish (rank 486) -- // merge(l, ish) at rank 1677 < suffix_merge_rank -> preempted -- iree_host_size_t compound_end = prefix_end + next_length; -- iree_host_size_t extension_length = 1; -- int32_t extension_token = -1; -- if (compound_end < remaining_size) { -- extension_token = iree_tokenizer_bpe_token_at_position( -- model, remaining_data + compound_end, remaining_size - compound_end, -- &extension_length); -- } -- if (extension_token < 0) return false; -+static void iree_tokenizer_bpe_collect_reachable_suffixes( -+ const iree_tokenizer_bpe_model_t* model, uint32_t token, -+ uint32_t* suffixes, uint32_t* suffix_consumed_at, -+ iree_host_size_t* suffix_count, iree_host_size_t suffix_capacity, -+ int depth) { -+ if (*suffix_count >= suffix_capacity || depth >= 16) return; - -- iree_tokenizer_merge_hash_result_t compound_merge = -- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, next_token, -- extension_token); -- if (!iree_tokenizer_merge_hash_result_is_valid(compound_merge) || -- compound_merge.rank >= suffix_merge_rank) { -- return false; -- } -+ const iree_tokenizer_vocab_t* vocab = model->vocab; -+ const iree_tokenizer_bpe_split_entry_t* split_table = -+ model->backtrack_tables.split_table; - -- // Build the compound progressively, checking at each level. -- int32_t compound_token = compound_merge.result_id; -- compound_end += extension_length; -- for (int depth = 0; depth < 4; ++depth) { -- iree_tokenizer_merge_hash_result_t prefix_compound = -- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, prefix_token, -- compound_token); -- if (iree_tokenizer_merge_hash_result_is_valid(prefix_compound) && -- prefix_compound.rank < suffix_merge_rank) { -- return true; -+ if (split_table[token].left_id == token) return; // Base token. -+ -+ iree_string_view_t token_text = -+ iree_tokenizer_vocab_token_text(vocab, (int32_t)token); -+ -+ for (iree_host_size_t split_pos = 1; -+ split_pos < token_text.size && *suffix_count < suffix_capacity; -+ ++split_pos) { -+ iree_string_view_t left_text = -+ iree_make_string_view(token_text.data, split_pos); -+ iree_string_view_t right_text = iree_make_string_view( -+ token_text.data + split_pos, token_text.size - split_pos); -+ -+ int32_t left_id = iree_tokenizer_vocab_lookup(vocab, left_text); -+ int32_t right_id = iree_tokenizer_vocab_lookup(vocab, right_text); -+ if (left_id < 0 || right_id < 0) continue; -+ -+ iree_tokenizer_merge_hash_result_t merge = -+ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, left_id, -+ right_id); -+ if (merge.result_id != (int32_t)token) continue; -+ -+ if (!iree_tokenizer_bpe_is_decomposition_reachable( -+ model, (uint32_t)left_id, (uint32_t)right_id, token)) { -+ continue; - } - -- // Extend compound by one more character. -- if (compound_end >= remaining_size) break; -- extension_token = iree_tokenizer_bpe_token_at_position( -- model, remaining_data + compound_end, remaining_size - compound_end, -- &extension_length); -- if (extension_token < 0) break; -- -- compound_merge = iree_tokenizer_vocab_merge_hash_lookup( -- model->merge_hash, compound_token, extension_token); -- if (!iree_tokenizer_merge_hash_result_is_valid(compound_merge) || -- compound_merge.rank >= suffix_merge_rank) { -- break; -+ uint32_t consumed_at = merge.rank + 1; -+ if (!iree_tokenizer_bpe_suffix_already_collected( -+ suffixes, suffix_consumed_at, *suffix_count, (uint32_t)right_id, -+ consumed_at)) { -+ suffixes[*suffix_count] = (uint32_t)right_id; -+ suffix_consumed_at[*suffix_count] = consumed_at; -+ ++*suffix_count; - } -- compound_token = compound_merge.result_id; -- compound_end += extension_length; -+ -+ // Walk deeper suffixes of the right child. This generalizes the old -+ // rightmost split_table walk to alternate reachable decompositions, e.g. -+ // recon can split as rec+on or re+con and both suffixes can matter. -+ iree_tokenizer_bpe_collect_reachable_suffixes( -+ model, (uint32_t)right_id, suffixes, suffix_consumed_at, suffix_count, -+ suffix_capacity, depth + 1); - } -- return false; - } - - // Checks if a candidate token is "suffix-blocked" - meaning a proper suffix of -@@ -566,21 +619,25 @@ static bool iree_tokenizer_bpe_is_suffix_merge_preempted( - // This allows pair validation to accept pairs whose merge rank >= - // blocking_rank, since those merges were intentionally deferred for better - // suffix merges. --static bool iree_tokenizer_bpe_is_suffix_blocked( -+static bool iree_tokenizer_bpe_is_suffix_blocked_impl( - const iree_tokenizer_bpe_model_t* model, int32_t token, - const uint8_t* remaining_data, iree_host_size_t remaining_size, - const char* suffix_data, iree_host_size_t suffix_length, -- uint32_t* out_blocking_rank) { -+ uint32_t* out_blocking_rank, int depth) { - *out_blocking_rank = 0; -+ if (depth >= 8) return false; - if (remaining_size == 0 && suffix_length == 0) return false; - - const iree_tokenizer_bpe_split_entry_t* split_table = - model->backtrack_tables.split_table; -- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; -+ const uint64_t* token_has_reachable_alternates = -+ model->backtrack_tables.token_has_reachable_alternates; -+ const iree_tokenizer_vocab_t* vocab = model->vocab; - const bool byte_level = - iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); - -- uint32_t token_rank = effective_rank[(uint32_t)token]; -+ uint32_t token_rank = -+ iree_tokenizer_bpe_token_formation_rank(model, (uint32_t)token); - if (token_rank <= 1) return false; // Base token, no suffix to check. - - // Collect all rightmost suffixes with their consumption ranks. -@@ -601,29 +658,64 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( - uint32_t suffix_consumed_at[32]; - iree_host_size_t suffix_count = 0; - -+ bool needs_alternate_suffixes = false; - for (uint32_t current = (uint32_t)token; -- suffix_count < 32 && split_table[current].left_id != current; -+ suffix_count < IREE_ARRAYSIZE(suffixes) && -+ split_table[current].left_id != current; - current = split_table[current].right_id) { -- // Check if this decomposition is actually reachable via BPE. If the -- // split_table decomposition is blocked by a boundary merge, skip this -- // suffix - it won't exist in the actual BPE token sequence. -+ if (iree_any_bit_set(token_has_reachable_alternates[current / 64], -+ 1ull << (current % 64))) { -+ needs_alternate_suffixes = true; -+ break; -+ } - uint32_t left = split_table[current].left_id; - uint32_t right = split_table[current].right_id; - bool reachable = iree_tokenizer_bpe_is_decomposition_reachable( - model, left, right, current); - if (!reachable) { -- // This decomposition is blocked. The actual BPE path uses a different -- // decomposition, so we can't rely on this suffix chain. Stop collecting -- // suffixes here - the actual suffixes depend on the alternate path which -- // we don't have easy access to. -+ needs_alternate_suffixes = true; - break; - } - suffixes[suffix_count] = right; -- suffix_consumed_at[suffix_count] = effective_rank[current]; -+ suffix_consumed_at[suffix_count] = -+ iree_tokenizer_bpe_token_formation_rank(model, current); - suffix_count++; -+ iree_tokenizer_bpe_collect_reachable_suffixes( -+ model, right, suffixes, suffix_consumed_at, &suffix_count, -+ IREE_ARRAYSIZE(suffixes), 0); -+ } -+ if (needs_alternate_suffixes) { -+ suffix_count = 0; -+ if (model->backtrack_tables.alternate_suffix_offsets) { -+ uint32_t token_id = (uint32_t)token; -+ uint32_t start = -+ model->backtrack_tables.alternate_suffix_offsets[token_id]; -+ uint32_t end = -+ model->backtrack_tables.alternate_suffix_offsets[token_id + 1]; -+ for (uint32_t i = start; -+ i < end && suffix_count < IREE_ARRAYSIZE(suffixes); ++i) { -+ suffixes[suffix_count] = -+ model->backtrack_tables.alternate_suffixes[i]; -+ suffix_consumed_at[suffix_count] = -+ model->backtrack_tables.alternate_suffix_consumed_at[i]; -+ ++suffix_count; -+ } -+ iree_host_size_t loaded_suffix_count = suffix_count; -+ for (iree_host_size_t i = 0; i < loaded_suffix_count; ++i) { -+ iree_tokenizer_bpe_collect_reachable_suffixes( -+ model, suffixes[i], suffixes, suffix_consumed_at, &suffix_count, -+ IREE_ARRAYSIZE(suffixes), 0); -+ } -+ } else { -+ iree_tokenizer_bpe_collect_reachable_suffixes( -+ model, (uint32_t)token, suffixes, suffix_consumed_at, &suffix_count, -+ IREE_ARRAYSIZE(suffixes), 0); -+ } -+ } else { -+ IREE_ASSERT(suffix_count < IREE_ARRAYSIZE(suffixes), -+ "suffix chain depth exceeds capacity"); - } -- IREE_ASSERT(suffix_count < IREE_ARRAYSIZE(suffixes), -- "suffix chain depth exceeds capacity"); -+ - if (suffix_count == 0) return false; - - // Walk the trie to enumerate all prefix tokens of the remaining input. -@@ -652,18 +744,33 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( - - // Skip if merge rank is too high to matter. - uint32_t merge_effective_rank = merge.rank + 1; -+ uint32_t suffix_available_at = -+ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); -+ uint32_t prefix_available_at = -+ iree_tokenizer_bpe_token_formation_rank(model, -+ (uint32_t)prefix_token); -+ if (merge_effective_rank < suffix_available_at) continue; -+ if (merge_effective_rank < prefix_available_at) continue; - if (merge_effective_rank >= token_rank) continue; - - // Skip if the suffix is consumed before this merge could fire. - if (merge_effective_rank >= suffix_consumed_at[s]) continue; - -- // Skip when suffix == prefix (repeating pattern case). -- // When the suffix token equals the prefix token, we have a repeating -- // pattern like consecutive metaspaces. In this case, any shorter token -- // would also face suffix blocking by the same pattern, creating an -- // infinite loop. The suffix blocking optimization doesn't help here -- // because there's no "better" tokenization to defer to. -- if ((int32_t)suffixes[s] == prefix_token) continue; -+ // If the prefix token would itself be split to enable a better suffix -+ // merge with following bytes, it is not available for this boundary -+ // merge. This handles chains such as ge|uffer|t where b+uffer would -+ // form "buffer" before ge+b, except "uffer" is blocked by er+t. -+ if (i + 1 < remaining_size && depth < 7 && -+ iree_tokenizer_bpe_token_formation_rank( -+ model, (uint32_t)prefix_token) > 1) { -+ uint32_t prefix_blocking_rank = 0; -+ if (iree_tokenizer_bpe_is_suffix_blocked_impl( -+ model, prefix_token, remaining_data + i + 1, -+ remaining_size - (i + 1), NULL, 0, &prefix_blocking_rank, -+ depth + 1)) { -+ continue; -+ } -+ } - - // Check if the prefix would be consumed by lower-rank merges first. - if (iree_tokenizer_bpe_is_suffix_merge_preempted( -@@ -678,7 +785,8 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( - // the rightmost base of the prefix merges with the following - // character at a rank lower than the prefix's internal consumption - // of that base, the prefix cannot form here. -- if (effective_rank[(uint32_t)prefix_token] > 1 && -+ if (iree_tokenizer_bpe_token_formation_rank( -+ model, (uint32_t)prefix_token) > 1 && - i + 1 < remaining_size) { - uint32_t rightmost_base = iree_tokenizer_bpe_rightmost_base_token( - model, (uint32_t)prefix_token); -@@ -725,14 +833,18 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( - - // Skip if merge rank is too high to matter. - uint32_t merge_effective_rank = merge.rank + 1; -+ uint32_t suffix_available_at = -+ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); -+ uint32_t prefix_available_at = -+ iree_tokenizer_bpe_token_formation_rank(model, -+ (uint32_t)prefix_token); -+ if (merge_effective_rank < suffix_available_at) continue; -+ if (merge_effective_rank < prefix_available_at) continue; - if (merge_effective_rank >= token_rank) continue; - - // Skip if the token suffix is consumed before this merge could fire. - if (merge_effective_rank >= suffix_consumed_at[s]) continue; - -- // Skip when suffix == prefix (repeating pattern case). -- if ((int32_t)suffixes[s] == prefix_token) continue; -- - // Suffixed tokens consume all remaining input + suffix bytes, so no - // preemption check is needed. The prefix_token represents the complete - // remaining segment (remaining_data + partial suffix) and nothing -@@ -745,6 +857,16 @@ static bool iree_tokenizer_bpe_is_suffix_blocked( - return false; - } - -+static bool iree_tokenizer_bpe_is_suffix_blocked( -+ const iree_tokenizer_bpe_model_t* model, int32_t token, -+ const uint8_t* remaining_data, iree_host_size_t remaining_size, -+ const char* suffix_data, iree_host_size_t suffix_length, -+ uint32_t* out_blocking_rank) { -+ return iree_tokenizer_bpe_is_suffix_blocked_impl( -+ model, token, remaining_data, remaining_size, suffix_data, suffix_length, -+ out_blocking_rank, 0); -+} -+ - //===----------------------------------------------------------------------===// - // BPE Backtracking: Core Algorithm - //===----------------------------------------------------------------------===// -@@ -774,18 +896,11 @@ void iree_tokenizer_bpe_backtrack_encode( - iree_tokenizer_bpe_state_pair_cache(state, model); - const uint32_t pair_cache_mask = (uint32_t)(model->pair_cache_capacity - 1); - -- // Reset only the bitfield words dirtied by the previous segment's -- // backtracking. On the first call, dirty_mask is pre-set to all-ones by -- // state_initialize, so this performs the equivalent of a full init. -- // On subsequent calls with no backtracking, dirty_mask is 0 and this -- // loop body executes zero times. -- uint64_t dirty = state->backtrack.dirty_mask; -- while (dirty) { -- iree_host_size_t word_index = -- (iree_host_size_t)iree_math_count_trailing_zeros_u64(dirty); -- bitfield[word_index] = UINT64_MAX; -- dirty &= dirty - 1; // Clear lowest set bit. -- } -+ // Reset the reachability bitfield for this segment. The exact backtracking -+ // path supports more than 64 bitfield words, so a single dirty mask is not -+ // expressive enough. -+ memset(bitfield, 0xFF, -+ model->backtrack_bitfield_capacity * sizeof(uint64_t)); - state->backtrack.dirty_mask = 0; - - iree_host_size_t stack_count = 0; -@@ -984,7 +1099,8 @@ void iree_tokenizer_bpe_backtrack_encode( - // fires first, blocking the path to "hello" as a single token. - bool token_unreachable = - token_raw_length > 1 && -- model->backtrack_tables.effective_rank[(uint32_t)token] > 1 && -+ iree_tokenizer_bpe_token_formation_rank(model, (uint32_t)token) > -+ 1 && - !iree_any_bit_set( - model->backtrack_tables.token_reachable[(uint32_t)token / 64], - 1ull << ((uint32_t)token % 64)); -diff --git a/runtime/src/iree/tokenizer/model/bpe_encode.c b/runtime/src/iree/tokenizer/model/bpe_encode.c -index bf140ae97c..4fa090026f 100644 ---- a/runtime/src/iree/tokenizer/model/bpe_encode.c -+++ b/runtime/src/iree/tokenizer/model/bpe_encode.c -@@ -280,39 +280,29 @@ static iree_status_t iree_tokenizer_bpe_encode_segment( - byte_position < segment.size; ++byte_position) { - uint8_t input_byte = (uint8_t)segment.data[byte_position]; - -- // Look up single-byte token. For ByteLevel mode, this is the UTF-8 -- // encoding of the ByteLevel codepoint (e.g., space -> "Ġ"). -- int32_t token_id = model->byte_to_token[input_byte]; -+ // Seed with the shortest base vocabulary token. Some base symbols (for -+ // example SentencePiece's U+2581 metaspace) span multiple UTF-8 bytes, -+ // and byte-fallback vocabularies may also have tokens for each raw byte. -+ // The BPE heap must start from the base symbol, not the byte fallback. - iree_host_size_t token_byte_length = 1; -+ int32_t token_id = iree_tokenizer_bpe_base_token_at_position( -+ model, (const uint8_t*)segment.data + byte_position, -+ segment.size - byte_position, &token_byte_length); - if (token_id < 0) { -- // No direct single-byte token. Try a multi-byte trie match first: -- // some base vocabulary tokens (e.g., SentencePiece's ▁ = U+2581, -- // 3 bytes) span multiple raw bytes but are never produced by a merge. -- // The trie walk finds the longest such token; effective_rank filtering -- // inside backtrack_longest_match ensures we only accept tokens that -- // participate in BPE (not stray added tokens). -- iree_host_size_t trie_raw_length = 0; -- iree_tokenizer_bpe_backtrack_longest_match( -- model, (const uint8_t*)segment.data + byte_position, -- segment.size - byte_position, &token_id, &trie_raw_length); -- if (token_id >= 0) { -- token_byte_length = trie_raw_length; -- } else { -- // No trie match. Fall back to byte-fallback (<0xNN>) or UNK. -- // For FUSE_UNK, check the last token in the window (not yet emitted) -- // before falling back to last_emitted_token_id. -- int32_t previous_token = -- (state->window.count > 0) -- ? iree_tokenizer_bpe_window_at(state, model, -- state->window.count - 1) -- ->token_id -- : state->last_emitted_token_id; -- token_id = iree_tokenizer_bpe_handle_unknown_byte(model, input_byte, -- previous_token); -- if (token_id < 0) { -- // Byte was fused with previous UNK. Skip to next byte. -- continue; -- } -+ // No trie match. Fall back to byte-fallback (<0xNN>) or UNK. For -+ // FUSE_UNK, check the last token in the window (not yet emitted) -+ // before falling back to last_emitted_token_id. -+ int32_t previous_token = -+ (state->window.count > 0) -+ ? iree_tokenizer_bpe_window_at(state, model, -+ state->window.count - 1) -+ ->token_id -+ : state->last_emitted_token_id; -+ token_id = iree_tokenizer_bpe_handle_unknown_byte(model, input_byte, -+ previous_token); -+ if (token_id < 0) { -+ // Byte was fused with previous UNK. Skip to next byte. -+ continue; - } - } - -@@ -620,13 +610,14 @@ iree_host_size_t iree_tokenizer_bpe_state_reclaim( - window[index].end_byte -= (uint32_t)committed; - } - -- // Adjust heap entry byte positions. -- // After BYTE_LOOP processes all segment bytes, apply_pending_merges has -- // drained most entries. Any remaining entries reference current window tokens -- // (left_start_byte >= committed). Stale entries that would underflow are -- // discarded by lazy invalidation when popped. -- for (iree_host_size_t i = 0; i < bpe_state->heap.size; ++i) { -- bpe_state->heap.entries[i].left_start_byte -= (uint32_t)committed; -+ // Rebuild heap entries for the rebased window. The heap can contain stale -+ // entries for tokens that were already emitted; subtracting |committed| from -+ // those would underflow their start byte and leave noisy stale entries. The -+ // window is bounded, so rebuilding is cheap and keeps all held-back merges -+ // visible after reclaim. -+ iree_tokenizer_bpe_heap_reset(&bpe_state->heap); -+ for (iree_host_size_t i = 0; i + 1 < bpe_state->window.count; ++i) { -+ iree_tokenizer_bpe_maybe_add_merge(bpe_state, model, i); - } - - return committed; -diff --git a/runtime/src/iree/tokenizer/model/bpe_internal.h b/runtime/src/iree/tokenizer/model/bpe_internal.h -index 69546df3f0..4e9873e653 100644 ---- a/runtime/src/iree/tokenizer/model/bpe_internal.h -+++ b/runtime/src/iree/tokenizer/model/bpe_internal.h -@@ -45,50 +45,42 @@ typedef struct iree_tokenizer_bpe_split_entry_t { - uint32_t right_id; - } iree_tokenizer_bpe_split_entry_t; - --// Backtracking stack entry: token ID and the byte position where it starts. --// Bit-packed backtrack entry to maintain 8-byte size while supporting large --// vocabs. start_byte is bounded by max_backtrack_segment_bytes (<=4095), so --// we use 12 bits for it and 20 bits for deferred_merge_rank (supports 1M --// tokens, covering all realistic vocabularies). --#define IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS 12 --#define IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK \ -- ((1u << IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS) - 1) --#define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS 20 -+// Backtracking stack entry: token ID, byte start, and deferred merge rank. -+// Keep deferred_merge_rank as a full 19-bit value (Gemma-sized vocabularies -+// need it) and store start_byte separately so exact backtracking can cover -+// larger normalized segments without falling back to the approximate heap path. -+#define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS 19 - #define IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX \ - ((1u << IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_BITS) - 1) - - typedef struct iree_tokenizer_bpe_backtrack_entry_t { - int32_t token_id; -- // Packed field: lower 12 bits = start_byte, upper 20 bits = deferred_rank. -+ uint32_t start_byte; - // The deferred_merge_rank is the effective_rank that was active when this - // token was pushed. It tells pair validation at the next position which - // merges were intentionally deferred due to suffix blocking. -- uint32_t start_byte_and_deferred_rank; -+ uint32_t deferred_rank; - } iree_tokenizer_bpe_backtrack_entry_t; - - static inline uint32_t iree_tokenizer_bpe_backtrack_entry_start_byte( - const iree_tokenizer_bpe_backtrack_entry_t* entry) { -- return entry->start_byte_and_deferred_rank & -- IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK; -+ return entry->start_byte; - } - - static inline uint32_t iree_tokenizer_bpe_backtrack_entry_deferred_rank( - const iree_tokenizer_bpe_backtrack_entry_t* entry) { -- return entry->start_byte_and_deferred_rank >> -- IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS; -+ return entry->deferred_rank; - } - - static inline void iree_tokenizer_bpe_backtrack_entry_set( - iree_tokenizer_bpe_backtrack_entry_t* entry, int32_t token_id, - uint32_t start_byte, uint32_t deferred_rank) { - entry->token_id = token_id; -- // Cap deferred_rank at 20-bit max (supports vocabs up to 1M tokens). - if (deferred_rank > IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX) { - deferred_rank = IREE_TOKENIZER_BPE_BACKTRACK_DEFERRED_RANK_MAX; - } -- entry->start_byte_and_deferred_rank = -- (start_byte & IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_MASK) | -- (deferred_rank << IREE_TOKENIZER_BPE_BACKTRACK_START_BYTE_BITS); -+ entry->start_byte = start_byte; -+ entry->deferred_rank = deferred_rank; - } - - //===----------------------------------------------------------------------===// -@@ -101,11 +93,19 @@ static inline void iree_tokenizer_bpe_backtrack_entry_set( - typedef struct iree_tokenizer_bpe_backtrack_tables_t { - // Single allocation for all tables below. - void* slab; -+ void* alternate_suffix_slab; - iree_tokenizer_bpe_split_entry_t* split_table; // Inverse merge table. - // Longest proper prefix token, or UINT32_MAX. - uint32_t* next_prefix_match; - uint32_t* effective_rank; // 0 = non-participant, >0 = BPE-reachable. - uint64_t* token_reachable; // Precomputed reachability bits. -+ // Tokens with more than one reachable decomposition. Suffix blocking must -+ // inspect alternate decompositions for these tokens instead of only walking -+ // split_table's right spine. -+ uint64_t* token_has_reachable_alternates; -+ uint32_t* alternate_suffix_offsets; // vocab_capacity + 1 entries. -+ uint32_t* alternate_suffixes; -+ uint32_t* alternate_suffix_consumed_at; - } iree_tokenizer_bpe_backtrack_tables_t; - - //===----------------------------------------------------------------------===// -@@ -118,10 +118,9 @@ typedef struct iree_tokenizer_bpe_backtrack_state_t { - iree_host_size_t stack_count; // Tokens in the stack. - // Next token to emit from stack. - iree_host_size_t emit_index; -- // Dirty mask for lazy bitfield reset. Each bit corresponds to a uint64_t -- // word in the backtrack bitfield that was modified (cleared) during the -- // previous segment's backtracking. On segment start, only these words are -- // reset to UINT64_MAX, giving O(backtracks) init instead of O(capacity). -+ // Retained for state layout compatibility with older code paths. The -+ // backtracking bitfield is reset as a whole at segment start because the -+ // exact path now supports more than 64 bitfield words. - uint64_t dirty_mask; - } iree_tokenizer_bpe_backtrack_state_t; - -@@ -210,6 +209,22 @@ typedef struct iree_tokenizer_bpe_model_t { - iree_host_size_t pair_cache_offset; - } iree_tokenizer_bpe_model_t; - -+static inline uint32_t iree_tokenizer_bpe_token_formation_rank( -+ const iree_tokenizer_bpe_model_t* model, uint32_t token) { -+ iree_tokenizer_bpe_split_entry_t split = -+ model->backtrack_tables.split_table[token]; -+ if (split.left_id == token) return 0; // Base token. -+ -+ iree_tokenizer_merge_hash_result_t merge = -+ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, -+ (int32_t)split.left_id, -+ (int32_t)split.right_id); -+ if (merge.result_id == (int32_t)token) { -+ return merge.rank + 1; -+ } -+ return model->backtrack_tables.effective_rank[token]; -+} -+ - //===----------------------------------------------------------------------===// - // Segment Context (State-Side) - //===----------------------------------------------------------------------===// -@@ -589,9 +604,9 @@ static inline bool iree_tokenizer_bpe_bitfield_is_set( - - static inline void iree_tokenizer_bpe_bitfield_clear( - uint64_t* bitfield, uint64_t* dirty_mask, iree_host_size_t position) { -+ (void)dirty_mask; - iree_host_size_t word_index = position / 64; - bitfield[word_index] &= ~((uint64_t)1 << (position % 64)); -- *dirty_mask |= (uint64_t)1 << word_index; - } - - //===----------------------------------------------------------------------===// -@@ -679,6 +694,73 @@ iree_tokenizer_bpe_backtrack_longest_match( - } - } - -+// Finds the shortest base vocabulary token matching raw input bytes at data[0]. -+// BPE starts from base symbols and applies merge rules by rank. For -+// SentencePiece-style tokenizers, some base symbols (for example U+2581 -+// metaspace) span multiple UTF-8 bytes and cannot be found through byte_to_token. -+// The window/heap path must seed those base symbols, not the longest reachable -+// merged token, otherwise rank-priority merges can be skipped. -+static inline int32_t iree_tokenizer_bpe_base_token_at_position( -+ const iree_tokenizer_bpe_model_t* model, const uint8_t* data, -+ iree_host_size_t size, iree_host_size_t* out_raw_length) { -+ if (size == 0) return -1; -+ -+ int32_t byte_token = model->byte_to_token[data[0]]; -+ -+ const bool byte_level = -+ iree_all_bits_set(model->flags, IREE_TOKENIZER_BPE_FLAG_BYTE_LEVEL_INPUT); -+ -+ iree_tokenizer_trie_cursor_t cursor; -+ iree_tokenizer_trie_cursor_reset(&cursor, model->trie); -+ -+ int32_t fallback_base_token = -1; -+ iree_host_size_t fallback_base_length = 0; -+ for (iree_host_size_t raw_consumed = 0; raw_consumed < size; -+ ++raw_consumed) { -+ if (!iree_tokenizer_bpe_trie_advance_byte(&cursor, data[raw_consumed], -+ byte_level)) { -+ break; -+ } -+ -+ int32_t token = iree_tokenizer_trie_cursor_token_id(&cursor); -+ if (token < 0) continue; -+ -+ const uint32_t token_id = (uint32_t)token; -+ if (token_id >= model->vocab_capacity) continue; -+ -+ iree_tokenizer_bpe_split_entry_t split = -+ model->backtrack_tables.split_table[token_id]; -+ const bool is_base_token = -+ split.left_id == token_id && split.right_id == token_id; -+ const bool is_matchable = -+ model->backtrack_tables.effective_rank[token_id] > 0; -+ if (is_base_token && is_matchable) { -+ iree_string_view_t token_text = -+ iree_tokenizer_vocab_token_text(model->vocab, token); -+ if (raw_consumed == 0 && data[0] >= 0x80 && token_text.size == 1 && -+ raw_consumed + 1 < size) { -+ fallback_base_token = token; -+ fallback_base_length = raw_consumed + 1; -+ continue; -+ } -+ if (out_raw_length) *out_raw_length = raw_consumed + 1; -+ return token; -+ } -+ } -+ -+ if (fallback_base_token >= 0) { -+ if (out_raw_length) *out_raw_length = fallback_base_length; -+ return fallback_base_token; -+ } -+ -+ if (byte_token >= 0) { -+ if (out_raw_length) *out_raw_length = 1; -+ return byte_token; -+ } -+ -+ return -1; -+} -+ - //===----------------------------------------------------------------------===// - // Table Construction (bpe_tables.c) - //===----------------------------------------------------------------------===// -diff --git a/runtime/src/iree/tokenizer/model/bpe_tables.c b/runtime/src/iree/tokenizer/model/bpe_tables.c -index c44021d3e1..480d7cc4e8 100644 ---- a/runtime/src/iree/tokenizer/model/bpe_tables.c -+++ b/runtime/src/iree/tokenizer/model/bpe_tables.c -@@ -13,6 +13,28 @@ - #include "iree/tokenizer/vocab/vocab_merge_hash.h" - #include "iree/tokenizer/vocab/vocab_trie.h" - -+static bool iree_tokenizer_bpe_is_repeated_initial_symbol( -+ iree_string_view_t text) { -+ if (text.size < 2) return false; -+ -+ bool all_newlines = true; -+ bool all_tabs = true; -+ for (iree_host_size_t i = 0; i < text.size; ++i) { -+ all_newlines &= text.data[i] == '\n'; -+ all_tabs &= text.data[i] == '\t'; -+ } -+ if (all_newlines || all_tabs) return true; -+ -+ static const char kMetaspace[] = "\xE2\x96\x81"; -+ if (text.size % (sizeof(kMetaspace) - 1) != 0) return false; -+ for (iree_host_size_t i = 0; i < text.size; i += sizeof(kMetaspace) - 1) { -+ if (memcmp(text.data + i, kMetaspace, sizeof(kMetaspace) - 1) != 0) { -+ return false; -+ } -+ } -+ return true; -+} -+ - //===----------------------------------------------------------------------===// - // BPE Backtracking: First Token Reachability - //===----------------------------------------------------------------------===// -@@ -56,7 +78,6 @@ uint32_t iree_tokenizer_bpe_right_boundary_consumed_rank( - const iree_tokenizer_bpe_model_t* model, uint32_t token) { - const iree_tokenizer_bpe_split_entry_t* split_table = - model->backtrack_tables.split_table; -- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; - - // Walk down the right spine until we find where the rightmost base token - // is consumed (i.e., where the right child is a base token). -@@ -64,7 +85,7 @@ uint32_t iree_tokenizer_bpe_right_boundary_consumed_rank( - uint32_t right = split_table[token].right_id; - if (split_table[right].left_id == right) { - // Right child is a base token - it's consumed at this token's merge. -- return effective_rank[token]; -+ return iree_tokenizer_bpe_token_formation_rank(model, token); - } - // Recurse into right subtree. - token = right; -@@ -81,7 +102,6 @@ static uint32_t iree_tokenizer_bpe_left_boundary_consumed_rank( - const iree_tokenizer_bpe_model_t* model, uint32_t token) { - const iree_tokenizer_bpe_split_entry_t* split_table = - model->backtrack_tables.split_table; -- const uint32_t* effective_rank = model->backtrack_tables.effective_rank; - - // Walk down the left spine until we find where the leftmost base token - // is consumed (i.e., where the left child is a base token). -@@ -89,7 +109,7 @@ static uint32_t iree_tokenizer_bpe_left_boundary_consumed_rank( - uint32_t left = split_table[token].left_id; - if (split_table[left].left_id == left) { - // Left child is a base token - it's consumed at this token's merge. -- return effective_rank[token]; -+ return iree_tokenizer_bpe_token_formation_rank(model, token); - } - // Recurse into left subtree. - token = left; -@@ -121,6 +141,67 @@ bool iree_tokenizer_bpe_is_first_token_reachable( - 1ull << (token % 64)); - } - -+static bool iree_tokenizer_bpe_decomposition_has_suffix_prefix_block( -+ const iree_tokenizer_bpe_model_t* model, uint32_t left, uint32_t right, -+ uint32_t token, uint32_t token_rank) { -+ const iree_tokenizer_bpe_split_entry_t* split_table = -+ model->backtrack_tables.split_table; -+ -+ uint32_t suffixes[32]; -+ uint32_t suffix_consumed_at[32]; -+ iree_host_size_t suffix_count = 0; -+ for (uint32_t current = left; -+ suffix_count < IREE_ARRAYSIZE(suffixes) && -+ split_table[current].left_id != current; -+ current = split_table[current].right_id) { -+ suffixes[suffix_count] = split_table[current].right_id; -+ suffix_consumed_at[suffix_count] = -+ iree_tokenizer_bpe_token_formation_rank(model, current); -+ suffix_count++; -+ } -+ if (suffix_count == 0) return false; -+ -+ uint32_t prefixes[32]; -+ uint32_t prefix_consumed_at[32]; -+ iree_host_size_t prefix_count = 0; -+ prefixes[prefix_count] = right; -+ prefix_consumed_at[prefix_count] = UINT32_MAX; -+ ++prefix_count; -+ for (uint32_t current = right; -+ prefix_count < IREE_ARRAYSIZE(prefixes) && -+ split_table[current].left_id != current; -+ current = split_table[current].left_id) { -+ prefixes[prefix_count] = split_table[current].left_id; -+ prefix_consumed_at[prefix_count] = -+ iree_tokenizer_bpe_token_formation_rank(model, current); -+ ++prefix_count; -+ } -+ -+ for (iree_host_size_t p = 0; p < prefix_count; ++p) { -+ for (iree_host_size_t s = 0; s < suffix_count; ++s) { -+ iree_tokenizer_merge_hash_result_t merge = -+ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, -+ (int32_t)suffixes[s], -+ (int32_t)prefixes[p]); -+ if (!iree_tokenizer_merge_hash_result_is_valid(merge)) continue; -+ if ((uint32_t)merge.result_id == token) continue; -+ -+ uint32_t merge_effective_rank = merge.rank + 1; -+ uint32_t suffix_available_at = -+ iree_tokenizer_bpe_token_formation_rank(model, suffixes[s]); -+ uint32_t prefix_available_at = -+ iree_tokenizer_bpe_token_formation_rank(model, prefixes[p]); -+ if (merge_effective_rank < suffix_available_at) continue; -+ if (merge_effective_rank < prefix_available_at) continue; -+ if (merge_effective_rank >= token_rank) continue; -+ if (merge_effective_rank >= suffix_consumed_at[s]) continue; -+ if (merge_effective_rank > prefix_consumed_at[p]) continue; -+ return true; -+ } -+ } -+ return false; -+} -+ - // Checks if a specific decomposition (left + right) of a token is reachable. - // This is separate from is_first_token_reachable to allow checking alternative - // decompositions when a token has multiple merges producing it. -@@ -128,6 +209,22 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( - const iree_tokenizer_bpe_model_t* model, uint32_t left, uint32_t right, - uint32_t token) { - const uint32_t* effective_rank = model->backtrack_tables.effective_rank; -+ uint32_t decomposition_rank = effective_rank[token]; -+ iree_tokenizer_merge_hash_result_t decomposition_merge = -+ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, (int32_t)left, -+ (int32_t)right); -+ if (decomposition_merge.result_id == (int32_t)token) { -+ decomposition_rank = decomposition_merge.rank + 1; -+ } -+ -+ uint32_t left_available_at = -+ iree_tokenizer_bpe_token_formation_rank(model, left); -+ uint32_t right_available_at = -+ iree_tokenizer_bpe_token_formation_rank(model, right); -+ if (decomposition_rank < left_available_at || -+ decomposition_rank < right_available_at) { -+ return false; -+ } - - // Check for blocking merge at the split boundary. - // -@@ -158,17 +255,18 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( - if (boundary_merge.result_id >= 0 && - (uint32_t)boundary_merge.result_id != token) { - // There's a different merge at this boundary. It blocks only if it fires -- // before EITHER boundary token is consumed by internal merges. -- uint32_t boundary_rank = effective_rank[(uint32_t)boundary_merge.result_id]; -+ // before the left boundary token is consumed by an internal merge and -+ // before or tied with the right boundary token's internal merge. Ties are -+ // resolved left-to-right: an internal merge in the left token starts before -+ // this boundary and wins the tie, while an internal merge in the right -+ // token starts after this boundary and loses the tie. -+ uint32_t boundary_rank = boundary_merge.rank + 1; - uint32_t left_boundary_consumed = - iree_tokenizer_bpe_right_boundary_consumed_rank(model, left); - uint32_t right_boundary_consumed = - iree_tokenizer_bpe_left_boundary_consumed_rank(model, right); -- uint32_t min_boundary_consumed = -- left_boundary_consumed < right_boundary_consumed -- ? left_boundary_consumed -- : right_boundary_consumed; -- if (boundary_rank < min_boundary_consumed) { -+ if (boundary_rank < left_boundary_consumed && -+ boundary_rank <= right_boundary_consumed) { - return false; // Blocking merge fires before either boundary is consumed. - } - } -@@ -180,7 +278,74 @@ bool iree_tokenizer_bpe_is_decomposition_reachable( - iree_tokenizer_bpe_is_first_token_reachable(model, left); - bool right_reachable = - iree_tokenizer_bpe_is_first_token_reachable(model, right); -- return left_reachable && right_reachable; -+ if (!left_reachable || !right_reachable) return false; -+ -+ return !iree_tokenizer_bpe_decomposition_has_suffix_prefix_block( -+ model, left, right, token, decomposition_rank); -+} -+ -+static bool iree_tokenizer_bpe_table_suffix_already_collected( -+ const uint32_t* suffixes, const uint32_t* suffix_consumed_at, -+ iree_host_size_t suffix_count, uint32_t suffix, uint32_t consumed_at) { -+ for (iree_host_size_t i = 0; i < suffix_count; ++i) { -+ if (suffixes[i] == suffix && suffix_consumed_at[i] == consumed_at) { -+ return true; -+ } -+ } -+ return false; -+} -+ -+static void iree_tokenizer_bpe_table_collect_reachable_suffixes( -+ const iree_tokenizer_bpe_model_t* model, uint32_t token, -+ uint32_t* suffixes, uint32_t* suffix_consumed_at, -+ iree_host_size_t* suffix_count, iree_host_size_t suffix_capacity, -+ int depth) { -+ if (*suffix_count >= suffix_capacity || depth >= 16) return; -+ -+ const iree_tokenizer_vocab_t* vocab = model->vocab; -+ const iree_tokenizer_bpe_split_entry_t* split_table = -+ model->backtrack_tables.split_table; -+ -+ if (split_table[token].left_id == token) return; // Base token. -+ -+ iree_string_view_t token_text = -+ iree_tokenizer_vocab_token_text(vocab, (int32_t)token); -+ -+ for (iree_host_size_t split_pos = 1; -+ split_pos < token_text.size && *suffix_count < suffix_capacity; -+ ++split_pos) { -+ iree_string_view_t left_text = -+ iree_make_string_view(token_text.data, split_pos); -+ iree_string_view_t right_text = iree_make_string_view( -+ token_text.data + split_pos, token_text.size - split_pos); -+ -+ int32_t left_id = iree_tokenizer_vocab_lookup(vocab, left_text); -+ int32_t right_id = iree_tokenizer_vocab_lookup(vocab, right_text); -+ if (left_id < 0 || right_id < 0) continue; -+ -+ iree_tokenizer_merge_hash_result_t merge = -+ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, left_id, -+ right_id); -+ if (merge.result_id != (int32_t)token) continue; -+ -+ if (!iree_tokenizer_bpe_is_decomposition_reachable( -+ model, (uint32_t)left_id, (uint32_t)right_id, token)) { -+ continue; -+ } -+ -+ uint32_t consumed_at = merge.rank + 1; -+ if (!iree_tokenizer_bpe_table_suffix_already_collected( -+ suffixes, suffix_consumed_at, *suffix_count, (uint32_t)right_id, -+ consumed_at)) { -+ suffixes[*suffix_count] = (uint32_t)right_id; -+ suffix_consumed_at[*suffix_count] = consumed_at; -+ ++*suffix_count; -+ } -+ -+ iree_tokenizer_bpe_table_collect_reachable_suffixes( -+ model, (uint32_t)right_id, suffixes, suffix_consumed_at, suffix_count, -+ suffix_capacity, depth + 1); -+ } - } - - //===----------------------------------------------------------------------===// -@@ -213,6 +378,7 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( - iree_host_size_t next_prefix_offset = 0; - iree_host_size_t effective_rank_offset = 0; - iree_host_size_t token_reachable_offset = 0; -+ iree_host_size_t token_has_reachable_alternates_offset = 0; - IREE_RETURN_AND_END_ZONE_IF_ERROR( - z0, - IREE_STRUCT_LAYOUT( -@@ -222,7 +388,9 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( - IREE_STRUCT_FIELD(vocab_capacity, uint32_t, &next_prefix_offset), - IREE_STRUCT_FIELD(vocab_capacity, uint32_t, &effective_rank_offset), - IREE_STRUCT_FIELD(reachable_words, uint64_t, -- &token_reachable_offset))); -+ &token_reachable_offset), -+ IREE_STRUCT_FIELD(reachable_words, uint64_t, -+ &token_has_reachable_alternates_offset))); - - IREE_RETURN_AND_END_ZONE_IF_ERROR( - z0, iree_allocator_malloc(model->allocator, slab_size, -@@ -237,6 +405,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( - (uint32_t*)(slab + effective_rank_offset); - model->backtrack_tables.token_reachable = - (uint64_t*)(slab + token_reachable_offset); -+ model->backtrack_tables.token_has_reachable_alternates = -+ (uint64_t*)(slab + token_has_reachable_alternates_offset); - - // Initialize: base tokens map to (self, self), rank 0, no prefix. - for (iree_host_size_t i = 0; i < vocab_capacity; ++i) { -@@ -309,6 +479,30 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( - } - } - -+ // Gemma-style BPE vocabularies contain repeated initial-symbol tokens -+ // (metaspace, newline, and tab runs) whose merge entries are not a -+ // rank-ordered DAG: longer runs can appear before the shorter runs needed to -+ // build them. HuggingFace still matches these as direct vocabulary symbols. -+ // Treat them as base tokens so reachability and suffix blocking do not try -+ // to validate impossible synthetic merge trees. -+ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { -+ iree_tokenizer_token_attr_t attrs = -+ iree_tokenizer_vocab_token_attrs(vocab, (int32_t)token_id); -+ if (attrs & IREE_TOKENIZER_TOKEN_ATTR_SPECIAL) continue; -+ -+ iree_string_view_t text = -+ iree_tokenizer_vocab_token_text(vocab, (int32_t)token_id); -+ if (!iree_tokenizer_bpe_is_repeated_initial_symbol(text)) continue; -+ -+ model->backtrack_tables.split_table[token_id].left_id = -+ (uint32_t)token_id; -+ model->backtrack_tables.split_table[token_id].right_id = -+ (uint32_t)token_id; -+ if (model->backtrack_tables.effective_rank[token_id] == 0) { -+ model->backtrack_tables.effective_rank[token_id] = 1; -+ } -+ } -+ - // Mark multi-byte merge participants as matchable. - // - // Some base vocabulary tokens (e.g., SentencePiece's ▁ character, U+2581) -@@ -433,6 +627,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( - // and no blocking boundary merge. - memset(model->backtrack_tables.token_reachable, 0, - reachable_words * sizeof(uint64_t)); -+ memset(model->backtrack_tables.token_has_reachable_alternates, 0, -+ reachable_words * sizeof(uint64_t)); - - // Initialize: mark single-codepoint base tokens as reachable. - for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { -@@ -448,7 +644,8 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( - codepoint_count++; - } - } -- if (codepoint_count <= 1) { -+ if (codepoint_count <= 1 || -+ iree_tokenizer_bpe_is_repeated_initial_symbol(token_text)) { - model->backtrack_tables.token_reachable[token_id / 64] |= - 1ull << (token_id % 64); - } -@@ -491,36 +688,10 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( - continue; - } - -- // Check for blocking merge at the boundary. -- uint32_t left_boundary = -- iree_tokenizer_bpe_rightmost_base_token(model, merge.left_id); -- uint32_t right_boundary = -- iree_tokenizer_bpe_leftmost_base_token(model, merge.right_id); -- -- iree_tokenizer_merge_hash_result_t boundary_merge = -- iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, -- (int32_t)left_boundary, -- (int32_t)right_boundary); -- if (boundary_merge.result_id >= 0 && -- (uint32_t)boundary_merge.result_id != (uint32_t)result_id) { -- // There's a different merge at this boundary. It blocks only if it -- // fires before EITHER boundary token is consumed by internal merges. -- uint32_t boundary_rank = -- model->backtrack_tables -- .effective_rank[(uint32_t)boundary_merge.result_id]; -- uint32_t left_boundary_consumed = -- iree_tokenizer_bpe_right_boundary_consumed_rank(model, -- merge.left_id); -- uint32_t right_boundary_consumed = -- iree_tokenizer_bpe_left_boundary_consumed_rank(model, -- merge.right_id); -- uint32_t min_boundary_consumed = -- left_boundary_consumed < right_boundary_consumed -- ? left_boundary_consumed -- : right_boundary_consumed; -- if (boundary_rank < min_boundary_consumed) { -- continue; // Blocking merge fires first. -- } -+ if (!iree_tokenizer_bpe_is_decomposition_reachable( -+ model, (uint32_t)merge.left_id, (uint32_t)merge.right_id, -+ (uint32_t)result_id)) { -+ continue; - } - - // This decomposition is reachable! -@@ -549,6 +720,175 @@ iree_status_t iree_tokenizer_bpe_build_backtrack_tables( - } - } - -+ // Mark tokens that have reachable decompositions besides the canonical -+ // split_table decomposition. Most tokens have exactly one reachable path and -+ // can use the cheap right-spine suffix walk during encode; ambiguous tokens -+ // need the slower all-splits suffix collection for HuggingFace parity. -+ for (iree_host_size_t rank = 0; rank < merge_count; ++rank) { -+ iree_tokenizer_merge_t merge = iree_tokenizer_vocab_merge(vocab, rank); -+ iree_tokenizer_merge_hash_result_t merge_result = -+ iree_tokenizer_vocab_merge_hash_lookup(model->merge_hash, -+ merge.left_id, merge.right_id); -+ int32_t result_id = merge_result.result_id; -+ if (result_id < 0 || (iree_host_size_t)result_id >= vocab_capacity) { -+ continue; -+ } -+ -+ uint32_t result = (uint32_t)result_id; -+ if (!iree_any_bit_set(model->backtrack_tables.token_reachable[result / 64], -+ 1ull << (result % 64))) { -+ continue; -+ } -+ if (!iree_tokenizer_bpe_is_decomposition_reachable( -+ model, (uint32_t)merge.left_id, (uint32_t)merge.right_id, -+ result)) { -+ continue; -+ } -+ -+ iree_tokenizer_bpe_split_entry_t canonical = -+ model->backtrack_tables.split_table[result]; -+ if (canonical.left_id != (uint32_t)merge.left_id || -+ canonical.right_id != (uint32_t)merge.right_id) { -+ model->backtrack_tables.token_has_reachable_alternates[result / 64] |= -+ 1ull << (result % 64); -+ } -+ } -+ -+ // Propagate the ambiguity bit to any token whose right-spine suffixes pass -+ // through an ambiguous token. Those parent tokens also need the precomputed -+ // all-splits suffix list during suffix blocking. -+ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { -+ for (uint32_t current = (uint32_t)token_id; -+ model->backtrack_tables.split_table[current].left_id != current; -+ current = model->backtrack_tables.split_table[current].right_id) { -+ if (iree_any_bit_set( -+ model->backtrack_tables -+ .token_has_reachable_alternates[current / 64], -+ 1ull << (current % 64))) { -+ model->backtrack_tables -+ .token_has_reachable_alternates[token_id / 64] |= -+ 1ull << (token_id % 64); -+ break; -+ } -+ } -+ } -+ -+ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { -+ uint32_t cheap_suffixes[32]; -+ uint32_t cheap_consumed_at[32]; -+ iree_host_size_t cheap_count = 0; -+ for (uint32_t current = (uint32_t)token_id; -+ cheap_count < IREE_ARRAYSIZE(cheap_suffixes) && -+ model->backtrack_tables.split_table[current].left_id != current; -+ current = model->backtrack_tables.split_table[current].right_id) { -+ cheap_suffixes[cheap_count] = -+ model->backtrack_tables.split_table[current].right_id; -+ cheap_consumed_at[cheap_count] = -+ iree_tokenizer_bpe_token_formation_rank(model, current); -+ ++cheap_count; -+ } -+ -+ uint32_t all_suffixes[32]; -+ uint32_t all_consumed_at[32]; -+ iree_host_size_t all_count = 0; -+ iree_tokenizer_bpe_table_collect_reachable_suffixes( -+ model, (uint32_t)token_id, all_suffixes, all_consumed_at, &all_count, -+ IREE_ARRAYSIZE(all_suffixes), 0); -+ -+ bool differs = cheap_count != all_count; -+ for (iree_host_size_t i = 0; i < all_count && !differs; ++i) { -+ bool found = false; -+ for (iree_host_size_t j = 0; j < cheap_count; ++j) { -+ if (all_suffixes[i] == cheap_suffixes[j] && -+ all_consumed_at[i] == cheap_consumed_at[j]) { -+ found = true; -+ break; -+ } -+ } -+ differs = !found; -+ } -+ if (differs) { -+ model->backtrack_tables.token_has_reachable_alternates[token_id / 64] |= -+ 1ull << (token_id % 64); -+ } -+ } -+ -+ iree_host_size_t total_alternate_suffixes = 0; -+ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; ++token_id) { -+ if (!iree_any_bit_set( -+ model->backtrack_tables -+ .token_has_reachable_alternates[token_id / 64], -+ 1ull << (token_id % 64))) { -+ continue; -+ } -+ uint32_t suffixes[32]; -+ uint32_t suffix_consumed_at[32]; -+ iree_host_size_t suffix_count = 0; -+ iree_tokenizer_bpe_table_collect_reachable_suffixes( -+ model, (uint32_t)token_id, suffixes, suffix_consumed_at, &suffix_count, -+ IREE_ARRAYSIZE(suffixes), 0); -+ total_alternate_suffixes += suffix_count; -+ } -+ -+ if (total_alternate_suffixes > 0) { -+ iree_host_size_t alt_slab_size = 0; -+ iree_host_size_t alt_offsets_offset = 0; -+ iree_host_size_t alt_suffixes_offset = 0; -+ iree_host_size_t alt_consumed_offset = 0; -+ IREE_RETURN_AND_END_ZONE_IF_ERROR( -+ z0, -+ IREE_STRUCT_LAYOUT( -+ /*base_size=*/0, &alt_slab_size, -+ IREE_STRUCT_FIELD(vocab_capacity + 1, uint32_t, -+ &alt_offsets_offset), -+ IREE_STRUCT_FIELD(total_alternate_suffixes, uint32_t, -+ &alt_suffixes_offset), -+ IREE_STRUCT_FIELD(total_alternate_suffixes, uint32_t, -+ &alt_consumed_offset))); -+ -+ IREE_RETURN_AND_END_ZONE_IF_ERROR( -+ z0, iree_allocator_malloc(model->allocator, alt_slab_size, -+ &model->backtrack_tables -+ .alternate_suffix_slab)); -+ -+ uint8_t* alt_slab = -+ (uint8_t*)model->backtrack_tables.alternate_suffix_slab; -+ model->backtrack_tables.alternate_suffix_offsets = -+ (uint32_t*)(alt_slab + alt_offsets_offset); -+ model->backtrack_tables.alternate_suffixes = -+ (uint32_t*)(alt_slab + alt_suffixes_offset); -+ model->backtrack_tables.alternate_suffix_consumed_at = -+ (uint32_t*)(alt_slab + alt_consumed_offset); -+ -+ iree_host_size_t write_index = 0; -+ for (iree_host_size_t token_id = 0; token_id < vocab_capacity; -+ ++token_id) { -+ model->backtrack_tables.alternate_suffix_offsets[token_id] = -+ (uint32_t)write_index; -+ if (!iree_any_bit_set( -+ model->backtrack_tables -+ .token_has_reachable_alternates[token_id / 64], -+ 1ull << (token_id % 64))) { -+ continue; -+ } -+ -+ uint32_t suffixes[32]; -+ uint32_t suffix_consumed_at[32]; -+ iree_host_size_t suffix_count = 0; -+ iree_tokenizer_bpe_table_collect_reachable_suffixes( -+ model, (uint32_t)token_id, suffixes, suffix_consumed_at, -+ &suffix_count, IREE_ARRAYSIZE(suffixes), 0); -+ for (iree_host_size_t i = 0; i < suffix_count; ++i) { -+ model->backtrack_tables.alternate_suffixes[write_index] = suffixes[i]; -+ model->backtrack_tables.alternate_suffix_consumed_at[write_index] = -+ suffix_consumed_at[i]; -+ ++write_index; -+ } -+ } -+ model->backtrack_tables.alternate_suffix_offsets[vocab_capacity] = -+ (uint32_t)write_index; -+ } -+ - IREE_TRACE_ZONE_END(z0); - return iree_ok_status(); - } -diff --git a/runtime/src/iree/tokenizer/model/bpe_test.cc b/runtime/src/iree/tokenizer/model/bpe_test.cc -index f2b82260d5..5bac3f0f44 100644 ---- a/runtime/src/iree/tokenizer/model/bpe_test.cc -+++ b/runtime/src/iree/tokenizer/model/bpe_test.cc -@@ -82,13 +82,13 @@ TEST_F(BPEModelTest, StateSizeIsReasonable) { - // tokens. For max_token_length=1, vocab_capacity=1: - // window: 2 * 12 = 24 - // heap: 3 * 8 = 24 -- // backtrack stack: 2048 * 8 = 16,384 -- // backtrack bitfield: 33 * 8 = 264 -+ // backtrack stack: 32768 * 12 = 393,216 -+ // backtrack bitfield: 513 * 8 = 4,104 - // pair cache: 4096 * 8 = 32,768 - // word cache: 0 (disabled, vocab < 256) - // struct: ~128 -- // total: ~49,592 -- EXPECT_LE(state_size, 65536u); -+ // total: ~430,240 -+ EXPECT_LE(state_size, 512 * 1024u); - } - - //===----------------------------------------------------------------------===// -@@ -2738,6 +2738,43 @@ TEST_F(BPEModelTest, MergePriorityWithPrecedingToken) { - /*expect_pending_after_encode=*/false); - } - -+// Gemma-style tokenizers use U+2581 metaspace as a multi-byte base token. Very -+// long segments bypass the backtracking path and use the window/heap encoder, -+// which must still seed base tokens before applying rank-priority merges. -+TEST_F(BPEModelTest, LongSegmentMergePriorityAfterMultiByteBaseToken) { -+ ScopedVocabBuilder builder; -+ -+ builder.AddToken(0, "▁"); -+ builder.AddToken(1, "z"); -+ builder.AddToken(2, "m"); -+ builder.AddToken(3, "l"); -+ builder.AddToken(4, "x"); -+ builder.AddToken(5, "▁z"); -+ builder.AddToken(6, "ml"); -+ builder.AddToken(7, "▁zm"); -+ builder.AddToken(8, "zm"); -+ -+ builder.AddMerge(0, 1); // Rank 0: ▁ + z -> ▁z -+ builder.AddMerge(2, 3); // Rank 1: m + l -> ml -+ builder.AddMerge(5, 2); // Rank 2: ▁z + m -> ▁zm -+ builder.AddMerge(0, 8); // Rank 3: ▁ + zm -> ▁zm (alternate path) -+ builder.AddMerge(1, 2); // Rank 4: z + m -> zm -+ -+ CreateModel(builder); -+ -+ std::string input = "▁zml"; -+ input.append(9000, 'x'); -+ auto tokens = EncodeAndFinalize(model(), input, -+ /*expect_pending_after_encode=*/false); -+ -+ ASSERT_EQ(tokens.size(), 9002u); -+ EXPECT_EQ(tokens[0], 5); // ▁z -+ EXPECT_EQ(tokens[1], 6); // ml -+ for (size_t i = 2; i < tokens.size(); ++i) { -+ EXPECT_EQ(tokens[i], 4); -+ } -+} -+ - // Tests ByteLevel transformation of multi-byte UTF-8 sequences. - // This is the Qwen unicode bug scenario where 'él' (UTF-8: c3 a9 6c) should - // transform to 'él' (UTF-8: c3 83 c2 a9 6c) and match as a single token. -diff --git a/runtime/src/iree/tokenizer/tokenizer.c b/runtime/src/iree/tokenizer/tokenizer.c -index 2299f0de7e..76011353f9 100644 ---- a/runtime/src/iree/tokenizer/tokenizer.c -+++ b/runtime/src/iree/tokenizer/tokenizer.c -@@ -1571,6 +1571,19 @@ static iree_status_t iree_tokenizer_encode_ring_as_segment( - state->transform_buffer.data, physical_write); - } - -+ if (is_partial) { -+ iree_host_size_t incomplete_tail = -+ iree_unicode_utf8_incomplete_tail_length( -+ (const char*)state->transform_buffer.data + physical_read, -+ data_length); -+ data_length -= incomplete_tail; -+ if (data_length == 0) { -+ *out_tokens_written = 0; -+ *out_segment_complete = false; -+ return iree_ok_status(); -+ } -+ } -+ - iree_tokenizer_segment_t segment = { - .start = physical_read, - .end = physical_read + data_length, -diff --git a/runtime/src/iree/tokenizer/tokenizer.h b/runtime/src/iree/tokenizer/tokenizer.h -index 098e6fef87..40db2bb459 100644 ---- a/runtime/src/iree/tokenizer/tokenizer.h -+++ b/runtime/src/iree/tokenizer/tokenizer.h -@@ -32,16 +32,17 @@ extern "C" { - // Maximum transform buffer allocation. - // Recommended values: - // 8KB: Minimal memory, suitable for most English text --// 16KB: Good balance for mixed content and cache locality -+// 64KB: Keeps a 32KB logical ring so large metaspace-normalized prompts can -+// stay on the exact BPE path instead of the approximate streaming path. - // - // The transform buffer determines the maximum segment size that the segmenter - // pipeline processes. Larger buffers allow longer segments, but segments that --// exceed L1 data cache (typically 32KB) cause DFA cache misses during --// multi-stage pre-tokenization. 16KB (8KB logical capacity after ring buffer --// halving) keeps the working set cache-friendly for segmenters with multiple --// children (e.g., DeepSeek V3's 4-stage Sequence pre-tokenizer). -+// exceed L1 data cache (typically 32KB) can cause DFA cache misses during -+// multi-stage pre-tokenization. 64KB (32KB logical capacity after ring buffer -+// halving) favors HuggingFace-compatible exact BPE for long normalized -+// segments. - #ifndef IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE --#define IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE (16 * 1024) -+#define IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE (64 * 1024) - #endif - - // Normalizer expansion factor for transform buffer sizing. diff --git a/third_party/iree/mistral-right-aligned-numbers.patch b/third_party/iree/mistral-right-aligned-numbers.patch deleted file mode 100644 index ca6e3fe5..00000000 --- a/third_party/iree/mistral-right-aligned-numbers.patch +++ /dev/null @@ -1,303 +0,0 @@ -diff --git a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c ---- a/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c -+++ b/runtime/src/iree/tokenizer/format/huggingface/segmenter_json.c -@@ -225,6 +225,29 @@ - return iree_ok_status(); - } - -+ // Tekken groups numbers from the right using a positive lookahead. Match the -+ // entire number run with the DFA, then subdivide it in the Split segmenter. -+ // Restrict this lowering to the exact known pattern and isolated behavior. -+ bool right_aligned_numbers = false; -+ if (behavior == IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED && !invert && -+ iree_string_view_equal( -+ pattern, -+ IREE_SV("[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*" -+ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|" -+ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+" -+ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|" -+ "\\p{N}{1,3}(?=(?:\\p{N}{3})*(?!\\p{N}))|\\p{N}{1,2}|" -+ " ?[^\\s\\p{L}\\p{N}]+[\\r\\n/" -+ "]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"))) { -+ pattern = IREE_SV( -+ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*" -+ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|" -+ "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+" -+ "[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}+|" -+ " ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"); -+ right_aligned_numbers = true; -+ } -+ - // Compile the regex pattern to a DFA. - iree_tokenizer_regex_dfa_t dfa; - uint8_t* dfa_data = NULL; -@@ -239,8 +262,13 @@ - } - - // Create the Split segmenter. On failure, free DFA data. -- status = iree_tokenizer_segmenter_split_allocate( -- dfa, dfa_data, behavior, invert, allocator, out_segmenter); -+ if (right_aligned_numbers) { -+ status = iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( -+ dfa, dfa_data, allocator, out_segmenter); -+ } else { -+ status = iree_tokenizer_segmenter_split_allocate( -+ dfa, dfa_data, behavior, invert, allocator, out_segmenter); -+ } - if (!iree_status_is_ok(status)) { - iree_allocator_free(allocator, dfa_data); - } -diff --git a/runtime/src/iree/tokenizer/segmenter/BUILD.bazel b/runtime/src/iree/tokenizer/segmenter/BUILD.bazel ---- a/runtime/src/iree/tokenizer/segmenter/BUILD.bazel -+++ b/runtime/src/iree/tokenizer/segmenter/BUILD.bazel -@@ -188,6 +188,7 @@ - hdrs = ["split.h"], - deps = [ - "//runtime/src/iree/base", -+ "//runtime/src/iree/base/internal:unicode", - "//runtime/src/iree/tokenizer:segmenter", - "//runtime/src/iree/tokenizer/regex:exec", - ], -diff --git a/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt b/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt ---- a/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt -+++ b/runtime/src/iree/tokenizer/segmenter/CMakeLists.txt -@@ -201,6 +201,7 @@ - "split.c" - DEPS - iree::base -+ iree::base::internal::unicode - iree::tokenizer::regex::exec - iree::tokenizer::segmenter - PUBLIC -diff --git a/runtime/src/iree/tokenizer/segmenter/split.c b/runtime/src/iree/tokenizer/segmenter/split.c ---- a/runtime/src/iree/tokenizer/segmenter/split.c -+++ b/runtime/src/iree/tokenizer/segmenter/split.c -@@ -7,6 +7,8 @@ - #include "iree/tokenizer/segmenter/split.h" - - #include -+ -+#include "iree/base/internal/unicode.h" - - //===----------------------------------------------------------------------===// - // Types -@@ -19,6 +21,7 @@ - bool invert; - // false = regex mode, true = literal mode. - bool is_literal; -+ bool right_aligned_numbers; - union { - struct { - iree_tokenizer_regex_dfa_t dfa; -@@ -361,7 +364,46 @@ - iree_tokenizer_regex_split_behavior_t behavior; - bool invert; - bool absolute; -+ bool right_aligned_numbers; -+ iree_string_view_t input; - } iree_tokenizer_split_callback_context_t; -+ -+// The caller has recognized a number-only match. Count codepoints, rather than -+// bytes, so mixed Unicode number categories retain the same grouping as \p{N}. -+static iree_status_t iree_tokenizer_split_emit_number_match( -+ iree_tokenizer_split_callback_context_t* context, -+ iree_tokenizer_regex_match_t match, iree_string_view_t text) { -+ iree_host_size_t position = 0; -+ iree_host_size_t count = 0; -+ while (position < text.size) { -+ iree_unicode_utf8_decode(text, &position); -+ ++count; -+ } -+ -+ iree_tokenizer_split_emit(context->emitter, context->last_emit_end, -+ match.start, context->absolute); -+ position = 0; -+ iree_host_size_t start = match.start; -+ iree_host_size_t group_size = (count - 1) % 3 + 1; -+ while (position < text.size) { -+ for (iree_host_size_t i = 0; i < group_size; ++i) { -+ iree_unicode_utf8_decode(text, &position); -+ } -+ iree_host_size_t end = match.start + position; -+ // finalize() can retry the same match after partially filling the output. -+ if (end > context->last_emit_end) { -+ iree_tokenizer_split_emit(context->emitter, start, end, -+ context->absolute); -+ if (context->emitter->full) { -+ return iree_status_from_code(IREE_STATUS_RESOURCE_EXHAUSTED); -+ } -+ } -+ start = end; -+ group_size = 3; -+ } -+ context->last_emit_end = match.end; -+ return iree_ok_status(); -+} - - // Processes each regex match inline: computes behavior result, emits segments, - // and commits state. Returns RESOURCE_EXHAUSTED if output capacity is -@@ -370,6 +412,20 @@ - void* user_data, iree_tokenizer_regex_match_t match) { - iree_tokenizer_split_callback_context_t* context = - (iree_tokenizer_split_callback_context_t*)user_data; -+ -+ if (context->right_aligned_numbers && match.end > match.start) { -+ iree_host_size_t base = context->emitter->chunk_base; -+ if (match.start < base || match.end - base > context->input.size) { -+ return iree_make_status(IREE_STATUS_INTERNAL, -+ "number match outside retained input"); -+ } -+ iree_string_view_t text = iree_make_string_view( -+ context->input.data + match.start - base, match.end - match.start); -+ iree_host_size_t position = 0; -+ if (iree_unicode_is_number(iree_unicode_utf8_decode(text, &position))) { -+ return iree_tokenizer_split_emit_number_match(context, match, text); -+ } -+ } - - // Compute behavior result (pure, no side effects). - // Gap is the region between the last emit and the current match start. -@@ -590,6 +646,7 @@ - segmenter->behavior = behavior; - segmenter->invert = invert; - segmenter->is_literal = false; -+ segmenter->right_aligned_numbers = false; - segmenter->regex.dfa = dfa; - segmenter->regex.dfa_data = dfa_data; - -@@ -605,6 +662,17 @@ - } - IREE_TRACE_ZONE_END(z0); - return status; -+} -+ -+iree_status_t iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( -+ iree_tokenizer_regex_dfa_t dfa, uint8_t* dfa_data, -+ iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter) { -+ IREE_RETURN_IF_ERROR(iree_tokenizer_segmenter_split_allocate( -+ dfa, dfa_data, IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, false, allocator, -+ out_segmenter)); -+ ((iree_tokenizer_segmenter_split_t*)*out_segmenter)->right_aligned_numbers = -+ true; -+ return iree_ok_status(); - } - - iree_status_t iree_tokenizer_segmenter_split_literal_allocate( -@@ -641,6 +709,7 @@ - segmenter->behavior = behavior; - segmenter->invert = invert; - segmenter->is_literal = true; -+ segmenter->right_aligned_numbers = false; - segmenter->literal.pattern_data = pattern_data; - segmenter->literal.pattern = - iree_make_string_view((const char*)pattern_data, pattern.size); -@@ -739,6 +808,8 @@ - .behavior = self->behavior, - .invert = self->invert, - .absolute = false, -+ .right_aligned_numbers = segmenter->right_aligned_numbers, -+ .input = input, - }; - - // Feed pattern matcher; matches are processed inline via callback. -@@ -882,6 +953,8 @@ - .behavior = self->behavior, - .invert = self->invert, - .absolute = false, -+ .right_aligned_numbers = segmenter->right_aligned_numbers, -+ .input = remaining_input, - }; - - iree_host_size_t input_end = chunk_base + remaining_input.size; -diff --git a/runtime/src/iree/tokenizer/segmenter/split.h b/runtime/src/iree/tokenizer/segmenter/split.h ---- a/runtime/src/iree/tokenizer/segmenter/split.h -+++ b/runtime/src/iree/tokenizer/segmenter/split.h -@@ -81,6 +81,14 @@ - iree_tokenizer_regex_split_behavior_t behavior, bool invert, - iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter); - -+// Like ISOLATED splitting, but subdivides number-only matches into groups of -+// three Unicode numbers from the right. The DFA must match entire number runs -+// separately from other text. This implements Tekken's numeric lookahead -+// without requiring general positive lookahead support in the regex compiler. -+iree_status_t iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( -+ iree_tokenizer_regex_dfa_t dfa, uint8_t* dfa_data, -+ iree_allocator_t allocator, iree_tokenizer_segmenter_t** out_segmenter); -+ - // Allocates a segmenter that splits text on literal string matches. - // Unlike the regex version, this uses efficient byte-by-byte substring search. - // The |pattern| is copied and owned by the segmenter. -diff --git a/runtime/src/iree/tokenizer/segmenter/split_test.cc b/runtime/src/iree/tokenizer/segmenter/split_test.cc ---- a/runtime/src/iree/tokenizer/segmenter/split_test.cc -+++ b/runtime/src/iree/tokenizer/segmenter/split_test.cc -@@ -28,7 +28,8 @@ - public: - ScopedSplitSegmenter(const char* pattern, - iree_tokenizer_regex_split_behavior_t behavior, -- bool invert = false) { -+ bool invert = false, -+ bool right_aligned_numbers = false) { - iree_tokenizer_regex_compile_error_t error = {0}; - status_ = iree_tokenizer_regex_compile_and_load( - iree_make_cstring_view(pattern), -@@ -37,8 +38,13 @@ - if (!iree_status_is_ok(status_)) return; - - iree_tokenizer_segmenter_t* raw = nullptr; -- status_ = iree_tokenizer_segmenter_split_allocate( -- dfa_, dfa_storage_, behavior, invert, iree_allocator_system(), &raw); -+ if (right_aligned_numbers) { -+ status_ = iree_tokenizer_segmenter_split_right_aligned_numbers_allocate( -+ dfa_, dfa_storage_, iree_allocator_system(), &raw); -+ } else { -+ status_ = iree_tokenizer_segmenter_split_allocate( -+ dfa_, dfa_storage_, behavior, invert, iree_allocator_system(), &raw); -+ } - if (!iree_status_is_ok(status_)) return; - - segmenter_ = ScopedSegmenter(raw); -@@ -72,6 +78,44 @@ - IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED); - IREE_ASSERT_OK(segmenter.status()); - ASSERT_NE(segmenter.get(), nullptr); -+} -+ -+TEST(SplitSegmenterTest, RightAlignedNumbers) { -+ ScopedSplitSegmenter segmenter("\\p{N}+|\\p{L}+|\\s+", -+ IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, -+ false, true); -+ IREE_ASSERT_OK(segmenter.status()); -+ auto input = IREE_SV("a1 12 123 1234 12345 123456 1234567z"); -+ std::vector expected = { -+ "a", "1", " ", "12", " ", "123", " ", "1", "234", " ", -+ "12", "345", " ", "123", "456", " ", "1", "234", "567", "z"}; -+ TestWithAllChunkSizes(segmenter.get(), input, expected, false); -+ TestLimitedOutputCapacity(segmenter.get(), input, expected); -+ TestZeroCapacityOutput(segmenter.get(), input); -+} -+ -+TEST(SplitSegmenterTest, RightAlignedUnicodeNumbers) { -+ ScopedSplitSegmenter segmenter("\\p{N}+|\\p{L}+|\\s+", -+ IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, -+ false, true); -+ IREE_ASSERT_OK(segmenter.status()); -+ // Nd, Nl and No, including mixed UTF-8 lengths within a number run. -+ auto input = IREE_SV("١٢٣٤ 12345 Ⅷ²3४"); -+ std::vector expected = {"١", "٢٣٤", " ", "12", -+ "345", " ", "Ⅷ", "²3४"}; -+ TestWithAllChunkSizesUtf8(segmenter.get(), input, expected, false); -+ TestLimitedOutputCapacity(segmenter.get(), input, expected); -+} -+ -+TEST(SplitSegmenterTest, RightAlignedNumbersLongRun) { -+ ScopedSplitSegmenter segmenter( -+ "\\p{N}+", IREE_TOKENIZER_UTIL_REGEX_SPLIT_ISOLATED, false, true); -+ IREE_ASSERT_OK(segmenter.status()); -+ std::string input(1000, '1'); -+ std::vector expected(334, "111"); -+ expected[0] = "1"; -+ TestLimitedOutputCapacity(segmenter.get(), -+ iree_make_cstring_view(input.c_str()), expected); - } - - TEST(SplitSegmenterTest, StateSizeIsReasonable) { diff --git a/third_party/iree/repo.bzl b/third_party/iree/repo.bzl deleted file mode 100644 index ee89fb6d..00000000 --- a/third_party/iree/repo.bzl +++ /dev/null @@ -1,27 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") -load("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") - -def repo(): - git_repository( - name = "iree", - remote = "https://github.com/iree-org/iree.git", - commit = "4d4e97d00f099a21f38eeff26f82a6d9e3643a11", - sparse_checkout_patterns = [ - "runtime/src/**", - "build_tools/bazel/**", - "build_tools/third_party/libbacktrace/**", - "BUILD.bazel", - ], - patches = [ - "//third_party/iree:tokenizer-only.patch", - "//third_party/iree:fix-added-token-matching.patch", - "//third_party/iree:match-hf-tokenizer.patch", - "//third_party/iree:laguna-tokenizer-compat.patch", - "//third_party/iree:fix-normalizer-lifetime-and-boundaries.patch", - "//third_party/iree:mistral-right-aligned-numbers.patch", - ], - patch_args = ["-p1"], - ) - - # Use this if you want to use a local copy of IREE instead of the git repository. Make sure to update the path to point to your local IREE checkout. - #new_local_repository(name = "iree", build_file="//:third_party/iree/BUILD.bazel", path="/home/corendos/iree/") diff --git a/third_party/iree/tokenizer-only.patch b/third_party/iree/tokenizer-only.patch deleted file mode 100644 index c6be27da..00000000 --- a/third_party/iree/tokenizer-only.patch +++ /dev/null @@ -1,50 +0,0 @@ -diff --git a/build_tools/bazel/build_defs.oss.bzl b/build_tools/bazel/build_defs.oss.bzl ---- a/build_tools/bazel/build_defs.oss.bzl -+++ b/build_tools/bazel/build_defs.oss.bzl -@@ -4,24 +4,35 @@ - # See https://llvm.org/LICENSE.txt for license information. - # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception - --load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "gentbl_filegroup", "td_library") -+# load("@llvm-project//mlir:tblgen.bzl", "gentbl_cc_library", "gentbl_filegroup", "td_library") - - # All load statements must come first in Starlark. - load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") --load("@rules_shell//shell:sh_test.bzl", "sh_test") --load( -- "//build_tools/bazel:iree_cc_fuzz.bzl", -- _iree_cc_fuzz = "iree_cc_fuzz", -- _iree_compiler_cc_fuzz = "iree_compiler_cc_fuzz", -- _iree_runtime_cc_fuzz = "iree_runtime_cc_fuzz", --) -+# load("@rules_shell//shell:sh_test.bzl", "sh_test") -+# load( -+# "//build_tools/bazel:iree_cc_fuzz.bzl", -+# _iree_cc_fuzz = "iree_cc_fuzz", -+# _iree_compiler_cc_fuzz = "iree_compiler_cc_fuzz", -+# _iree_runtime_cc_fuzz = "iree_runtime_cc_fuzz", -+# ) - - """Common Bazel definitions for IREE.""" - - # Re-export fuzz rules for external use. --iree_cc_fuzz = _iree_cc_fuzz --iree_compiler_cc_fuzz = _iree_compiler_cc_fuzz --iree_runtime_cc_fuzz = _iree_runtime_cc_fuzz -+# iree_cc_fuzz = _iree_cc_fuzz -+# iree_compiler_cc_fuzz = _iree_compiler_cc_fuzz -+# iree_runtime_cc_fuzz = _iree_runtime_cc_fuzz -+ -+def generic_pass(**kwargs): -+ pass -+ -+gentbl_cc_library = generic_pass -+gentbl_filegroup = generic_pass -+td_library = generic_pass -+sh_test = generic_pass -+iree_runtime_cc_fuzz = generic_pass -+iree_compiler_cc_fuzz = generic_pass -+iree_cc_fuzz = generic_pass - - def defaulting_select(selector): - """Pass through to select() with special semantics when converting to CMake. diff --git a/third_party/libvaxis/BUILD.bazel b/third_party/libvaxis/BUILD.bazel deleted file mode 100644 index 8b137891..00000000 --- a/third_party/libvaxis/BUILD.bazel +++ /dev/null @@ -1 +0,0 @@ - diff --git a/third_party/libvaxis/fixes.patch b/third_party/libvaxis/fixes.patch deleted file mode 100644 index effbb2aa..00000000 --- a/third_party/libvaxis/fixes.patch +++ /dev/null @@ -1,202 +0,0 @@ -diff --git a/src/Loop.zig b/src/Loop.zig ---- a/src/Loop.zig -+++ b/src/Loop.zig -@@ -352,7 +352,7 @@ - } - }, - .cap_da1 => { -- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); -+ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); - vx.queries_done.store(true, .unordered); - }, - .mouse => |mouse| { -@@ -484,7 +484,7 @@ - vx.caps.program_status = true; - }, - .cap_da1 => { -- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); -+ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); - vx.queries_done.store(true, .unordered); - }, - .winsize => |winsize| { -diff --git a/src/Vaxis.zig b/src/Vaxis.zig ---- a/src/Vaxis.zig -+++ b/src/Vaxis.zig -@@ -139,14 +139,13 @@ - } - } - --/// resets enabled features, sends cursor to home and clears below cursor --pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { -- // always show the cursor on state reset -+/// Resets enabled terminal features without clearing screen content. -+/// Use this for non-interactive / single-shot rendering where the -+/// output should remain visible after exit. -+pub fn resetModes(self: *Vaxis, tty: *std.Io.Writer) !void { - tty.writeAll(ctlseqs.show_cursor) catch {}; - tty.writeAll(ctlseqs.sgr_reset) catch {}; - if (self.screen.cursor_shape != .default) { -- // In many terminals, `.default` will set to the configured cursor shape. Others, it will -- // change to a blinking block. - tty.print(ctlseqs.cursor_shape, .{@backingInt(Cell.CursorShape.default)}) catch {}; - } - if (self.state.kitty_keyboard) { -@@ -159,6 +158,32 @@ - if (self.state.bracketed_paste) { - try self.setBracketedPaste(tty, false); - } -+ if (self.state.color_scheme_updates) { -+ try tty.writeAll(ctlseqs.color_scheme_reset); -+ self.state.color_scheme_updates = false; -+ } -+ if (self.state.in_band_resize) { -+ try tty.writeAll(ctlseqs.in_band_resize_reset); -+ self.state.in_band_resize = false; -+ } -+ if (self.state.changed_default_fg) { -+ try tty.writeAll(ctlseqs.osc10_reset); -+ self.state.changed_default_fg = false; -+ } -+ if (self.state.changed_default_bg) { -+ try tty.writeAll(ctlseqs.osc11_reset); -+ self.state.changed_default_bg = false; -+ } -+ if (self.state.changed_cursor_color) { -+ try tty.writeAll(ctlseqs.osc12_reset); -+ self.state.changed_cursor_color = false; -+ } -+ try tty.flush(); -+} -+ -+/// resets enabled features, sends cursor to home and clears below cursor -+pub fn resetState(self: *Vaxis, tty: *std.Io.Writer) !void { -+ self.resetModes(tty) catch {}; - if (self.state.alt_screen) { - try tty.writeAll(ctlseqs.home); - try tty.writeAll(ctlseqs.erase_below_cursor); -@@ -171,27 +196,6 @@ - } - try tty.writeAll(ctlseqs.erase_below_cursor); - } -- if (self.state.color_scheme_updates) { -- try tty.writeAll(ctlseqs.color_scheme_reset); -- self.state.color_scheme_updates = false; -- } -- if (self.state.in_band_resize) { -- try tty.writeAll(ctlseqs.in_band_resize_reset); -- self.state.in_band_resize = false; -- } -- if (self.state.changed_default_fg) { -- try tty.writeAll(ctlseqs.osc10_reset); -- self.state.changed_default_fg = false; -- } -- if (self.state.changed_default_bg) { -- try tty.writeAll(ctlseqs.osc11_reset); -- self.state.changed_default_bg = false; -- } -- if (self.state.changed_cursor_color) { -- try tty.writeAll(ctlseqs.osc12_reset); -- self.state.changed_cursor_color = false; -- } -- - try tty.flush(); - } - -@@ -1101,19 +1105,19 @@ - switch (format) { - .rgb => { - try tty.print( -- "\x1b_Gf=24,s={d},v={d},i={d},t={c};{s}\x1b\\", -+ "\x1b_Gf=24,s={d},v={d},i={d},t={c},q=2;{s}\x1b\\", - .{ width, height, id, medium_char, encoded }, - ); - }, - .rgba => { - try tty.print( -- "\x1b_Gf=32,s={d},v={d},i={d},t={c};{s}\x1b\\", -+ "\x1b_Gf=32,s={d},v={d},i={d},t={c},q=2;{s}\x1b\\", - .{ width, height, id, medium_char, encoded }, - ); - }, - .png => { - try tty.print( -- "\x1b_Gf=100,i={d},t={c};{s}\x1b\\", -+ "\x1b_Gf=100,i={d},t={c},q=2;{s}\x1b\\", - .{ id, medium_char, encoded }, - ); - }, -@@ -1149,7 +1153,7 @@ - - if (bytes.len < 4096) { - try tty.print( -- "\x1b_Gf={d},s={d},v={d},i={d};{s}\x1b\\", -+ "\x1b_Gf={d},s={d},v={d},i={d},q=2;{s}\x1b\\", - .{ - fmt, - width, -@@ -1162,14 +1166,14 @@ - var n: usize = 4096; - - try tty.print( -- "\x1b_Gf={d},s={d},v={d},i={d},m=1;{s}\x1b\\", -+ "\x1b_Gf={d},s={d},v={d},i={d},m=1,q=2;{s}\x1b\\", - .{ fmt, width, height, id, bytes[0..n] }, - ); - while (n < bytes.len) : (n += 4096) { - const end: usize = @min(n + 4096, bytes.len); - const m: u2 = if (end == bytes.len) 0 else 1; - try tty.print( -- "\x1b_Gm={d};{s}\x1b\\", -+ "\x1b_Gm={d},q=2;{s}\x1b\\", - .{ - m, - bytes[n..end], -diff --git a/src/c_api.zig b/src/c_api.zig ---- a/src/c_api.zig -+++ b/src/c_api.zig -@@ -965,7 +965,7 @@ - .cap_multi_cursor => vx.caps.multi_cursor = true, - .cap_program_status => vx.caps.program_status = true, - .cap_da1 => { -- std.Io.futexWake(vx.io, std.atomic.Value(u32), &vx.query_futex, 10); -+ std.Io.futexWake(vx.io, u32, &vx.query_futex.raw, 10); - vx.queries_done.store(true, .unordered); - }, - .winsize => { -diff --git a/src/ctlseqs.zig b/src/ctlseqs.zig ---- a/src/ctlseqs.zig -+++ b/src/ctlseqs.zig -@@ -140,8 +140,8 @@ - pub const osc52_clipboard_request = "\x1b]52;c;?\x1b\\"; - - // Kitty graphics --pub const kitty_graphics_clear = "\x1b_Ga=d\x1b\\"; --pub const kitty_graphics_preamble = "\x1b_Ga=p,i={d}"; -+pub const kitty_graphics_clear = "\x1b_Ga=d,q=2\x1b\\"; -+pub const kitty_graphics_preamble = "\x1b_Ga=p,q=2,i={d}"; - pub const kitty_graphics_closing = ",C=1\x1b\\"; - - // Color control sequences -diff --git a/src/vxfw/Padding.zig b/src/vxfw/Padding.zig ---- a/src/vxfw/Padding.zig -+++ b/src/vxfw/Padding.zig -@@ -6,7 +6,7 @@ - const vxfw = @import("vxfw.zig"); - - const Padding = @This(); --const PadValues = struct { -+pub const PadValues = struct { - left: u16 = 0, - right: u16 = 0, - top: u16 = 0, -@@ -54,10 +54,6 @@ - - pub fn draw(self: *const Padding, ctx: vxfw.DrawContext) Allocator.Error!vxfw.Surface { - const pad = self.padding; -- if (pad.left > 0 or pad.right > 0) -- std.debug.assert(ctx.max.width != null); -- if (pad.top > 0 or pad.bottom > 0) -- std.debug.assert(ctx.max.height != null); - const inner_min: vxfw.Size = .{ - .width = ctx.min.width -| (pad.right + pad.left), - .height = ctx.min.height -| (pad.top + pad.bottom), diff --git a/third_party/libvaxis/repo.bzl b/third_party/libvaxis/repo.bzl deleted file mode 100644 index e10982a4..00000000 --- a/third_party/libvaxis/repo.bzl +++ /dev/null @@ -1,11 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -def repo(): - git_repository( - name = "libvaxis", - remote = "https://github.com/rockorager/libvaxis.git", - commit = "14b7684272234a3d66316c0d1bbdc876d4e8997c", - build_file = "//third_party/libvaxis:libvaxis.bazel", - patches = ["//third_party/libvaxis:fixes.patch"], - patch_args = ["-p1"], - ) diff --git a/third_party/linenoise/repo.bzl b/third_party/linenoise/repo.bzl index 1d7a1678..7837ac67 100644 --- a/third_party/linenoise/repo.bzl +++ b/third_party/linenoise/repo.bzl @@ -1,7 +1,7 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") def repo(): - git_repository( + new_git_repository( name = "linenoise", remote = "https://github.com/antirez/linenoise.git", commit = "dc83cc373ac2058030eb3cf5e404959a26fef112", diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel b/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel new file mode 100644 index 00000000..7f1750b4 --- /dev/null +++ b/third_party/modules/aspect_bazel_lib/2.8.1.1/MODULE.bazel @@ -0,0 +1,78 @@ +"aspect-build/bazel-lib" + +module( + name = "aspect_bazel_lib", + version = "2.8.1.1", + bazel_compatibility = [">=6.0.0"], + compatibility_level = 1, +) + +# Lower-bounds (minimum) versions for direct runtime dependencies +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "platforms", version = "0.0.8") + +# 0.5.4 is the first version with bzlmod support +bazel_dep(name = "stardoc", version = "0.6.2", repo_name = "io_bazel_stardoc") + +bazel_lib_toolchains = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "toolchains") +bazel_lib_toolchains.copy_directory() +bazel_lib_toolchains.copy_to_directory() +bazel_lib_toolchains.jq() +bazel_lib_toolchains.yq() +bazel_lib_toolchains.coreutils() +bazel_lib_toolchains.tar() +bazel_lib_toolchains.zstd() +bazel_lib_toolchains.expand_template() +bazel_lib_toolchains.bats() +use_repo(bazel_lib_toolchains, "bats_toolchains", "bsd_tar_toolchains", "copy_directory_toolchains", "copy_to_directory_toolchains", "coreutils_toolchains", "expand_template_toolchains", "jq_toolchains", "yq_toolchains", "zstd_toolchains") + +register_toolchains( + "@copy_directory_toolchains//:all", + "@copy_to_directory_toolchains//:all", + "@jq_toolchains//:all", + "@yq_toolchains//:all", + "@coreutils_toolchains//:all", + "@expand_template_toolchains//:all", + "@bats_toolchains//:all", + "@bsd_tar_toolchains//:all", + "@zstd_toolchains//:all", +) + +####### Dev dependencies ######## + +# To allow /tools to be built from source +# NOTE: when publishing to BCR, we patch this to be dev_dependency, as we publish pre-built binaries +# along with our releases. + +bazel_dep( + name = "gazelle", + version = "0.36.0", + dev_dependency = True, +) +bazel_dep( + name = "rules_go", + version = "0.46.0", + repo_name = "io_bazel_rules_go", + dev_dependency = True, +) + +go_deps = use_extension( + "@gazelle//:extensions.bzl", + "go_deps", + dev_dependency = True, +) +go_deps.from_file(go_mod = "//:go.mod") +use_repo( + go_deps, + "com_github_bmatcuk_doublestar_v4", + "org_golang_x_exp", + "org_golang_x_sys", +) + +host = use_extension("@aspect_bazel_lib//lib:extensions.bzl", "host", dev_dependency = True) +host.host() +use_repo(host, "aspect_bazel_lib_host") + +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.5.0", dev_dependency = True) +bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) +bazel_dep(name = "bazel_features", version = "0.2.0", dev_dependency = True) diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch new file mode 100644 index 00000000..96aa8dfc --- /dev/null +++ b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/0001-fix-add-repo-mapping-to-tar-archive.patch @@ -0,0 +1,50 @@ +From 5dbe6a2604e440b684add7b44531898acc8631b4 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Sun, 8 Sep 2024 21:17:29 +0200 +Subject: [PATCH] fix: add repo mapping to tar archive + +When using bzlmod, runfiles lookup will fail without it. +--- + lib/private/tar.bzl | 17 +++++++++++++---- + 1 file changed, 13 insertions(+), 4 deletions(-) + +diff --git a/lib/private/tar.bzl b/lib/private/tar.bzl +index 733ff60..29434f6 100644 +--- a/lib/private/tar.bzl ++++ b/lib/private/tar.bzl +@@ -147,12 +147,19 @@ def _tar_impl(ctx): + args.add(ctx.file.mtree, format = "@%s") + inputs.append(ctx.file.mtree) + ++ src_runfiles = [] ++ for src in ctx.attr.srcs: ++ src_di = src[DefaultInfo] ++ if getattr(src_di.files_to_run, "repo_mapping_manifest", None) != None: ++ src_runfiles.append(depset( ++ direct = [src_di.files_to_run.repo_mapping_manifest], ++ transitive = [src_di.default_runfiles.files], ++ )) ++ else: ++ src_runfiles.append(src_di.default_runfiles.files) + ctx.actions.run( + executable = bsdtar.tarinfo.binary, +- inputs = depset(direct = inputs, transitive = [bsdtar.default.files] + [ +- src[DefaultInfo].default_runfiles.files +- for src in ctx.attr.srcs +- ]), ++ inputs = depset(direct = inputs, transitive = [bsdtar.default.files] + src_runfiles), + outputs = [out], + arguments = [args], + mnemonic = "Tar", +@@ -234,6 +241,8 @@ def _mtree_impl(ctx): + workspace_name = str(ctx.workspace_name) + + content.add(_mtree_line(runfiles_dir, type = "dir")) ++ if getattr(default_info.files_to_run, "repo_mapping_manifest", None) != None: ++ content.add(_mtree_line("{}/_repo_mapping".format(runfiles_dir), type = "file", content = default_info.files_to_run.repo_mapping_manifest.path)) + content.add_all( + s.default_runfiles.files, + expand_directories = True, +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch new file mode 100644 index 00000000..ffe1c1d3 --- /dev/null +++ b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/go_dev_dep.patch @@ -0,0 +1,27 @@ +diff --git a/MODULE.bazel b/MODULE.bazel +index e63fa5b..9d78a88 100644 +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -50,19 +50,19 @@ use_repo(host, "aspect_bazel_lib_host") + bazel_dep( + name = "gazelle", + version = "0.36.0", +- # In released versions: dev_dependency = True ++ dev_dependency = True, + ) + bazel_dep( + name = "rules_go", + version = "0.46.0", + repo_name = "io_bazel_rules_go", +- # In released versions: dev_dependency = True ++ dev_dependency = True, + ) + + go_deps = use_extension( + "@gazelle//:extensions.bzl", + "go_deps", +- # In released versions: dev_dependency = True ++ dev_dependency = True, + ) + go_deps.from_file(go_mod = "//:go.mod") + use_repo( diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch new file mode 100644 index 00000000..fe11475e --- /dev/null +++ b/third_party/modules/aspect_bazel_lib/2.8.1.1/patches/module_dot_bazel_version.patch @@ -0,0 +1,14 @@ +=================================================================== +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -1,9 +1,9 @@ + "aspect-build/bazel-lib" + + module( + name = "aspect_bazel_lib", +- version = "0.0.0", ++ version = "2.8.1", + bazel_compatibility = [">=6.0.0"], + compatibility_level = 1, + ) + diff --git a/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json b/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json new file mode 100644 index 00000000..3b7d500e --- /dev/null +++ b/third_party/modules/aspect_bazel_lib/2.8.1.1/source.json @@ -0,0 +1,11 @@ +{ + "integrity": "sha256-aINU7mvuunGUJD1z6wmSuaEujt7u7FtlRPS1MaMRIjc=", + "strip_prefix": "bazel-lib-2.8.1", + "url": "https://github.com/aspect-build/bazel-lib/releases/download/v2.8.1/bazel-lib-v2.8.1.tar.gz", + "patches": { + "go_dev_dep.patch": "sha256-DTc/hk+etl4D50M0BLRik2vHbrgDb6rds+Dj4xphWb4=", + "module_dot_bazel_version.patch": "sha256-cEMv6bY7Sc5dERv8YG0lq45zuZCsVPNn4oAN9aOkf40=", + "0001-fix-add-repo-mapping-to-tar-archive.patch": "" + }, + "patch_strip": 1 +} diff --git a/third_party/modules/aspect_bazel_lib/metadata.json b/third_party/modules/aspect_bazel_lib/metadata.json new file mode 100644 index 00000000..db6bde15 --- /dev/null +++ b/third_party/modules/aspect_bazel_lib/metadata.json @@ -0,0 +1,25 @@ +{ + "homepage": "https://docs.aspect.build/rules/aspect_bazel_lib", + "maintainers": [ + { + "name": "Alex Eagle", + "email": "alex@aspect.dev", + "github": "alexeagle" + }, + { + "name": "Derek Cormier", + "email": "derek@aspect.dev", + "github": "kormide" + } + ], + "repository": [ + "github:aspect-build/bazel-lib" + ], + "versions": [ + "2.8.1.1" + ], + "yanked_versions": { + "1.31.0": "1.31.0 has a breaking change to the default yq version", + "1.34.2": "The version field has a leading 'v' due to a release automation bug" + } +} diff --git a/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel b/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel new file mode 100644 index 00000000..8f8806c6 --- /dev/null +++ b/third_party/modules/libxev/20240825.0-dbe2291/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20240825.0-dbe2291", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/libvaxis/libvaxis.bazel b/third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel similarity index 51% rename from third_party/libvaxis/libvaxis.bazel rename to third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel index 122e25a1..872ef7fb 100644 --- a/third_party/libvaxis/libvaxis.bazel +++ b/third_party/modules/libxev/20240825.0-dbe2291/overlay/BUILD.bazel @@ -1,17 +1,13 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( - name = "vaxis", + name = "xev", srcs = glob([ "src/*.zig", - "src/vxfw/*.zig", - "src/widgets/*.zig", - "src/widgets/terminal/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", ]), main = "src/main.zig", - deps = [ - "@zigimg", - "@uucode", - ], visibility = ["//visibility:public"], ) diff --git a/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel b/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/libxev/20240825.0-dbe2291/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20240825.0-dbe2291/source.json b/third_party/modules/libxev/20240825.0-dbe2291/source.json new file mode 100644 index 00000000..81950288 --- /dev/null +++ b/third_party/modules/libxev/20240825.0-dbe2291/source.json @@ -0,0 +1,9 @@ +{ + "strip_prefix": "libxev-dbe22910a43e9e8ec9948d3cbd73d8488a074967", + "url": "https://github.com/mitchellh/libxev/archive/dbe22910a43e9e8ec9948d3cbd73d8488a074967.tar.gz", + "integrity": "sha256-JM70bkRUyuAQXKV1piJ6I0IClzCrXV43sdLfIKVC9Lo=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + } +} diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel b/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel new file mode 100644 index 00000000..99daae70 --- /dev/null +++ b/third_party/modules/libxev/20240910.0-a2d9b31/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20240910.0-a2d9b31", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel b/third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel new file mode 100644 index 00000000..872ef7fb --- /dev/null +++ b/third_party/modules/libxev/20240910.0-a2d9b31/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "src/main.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel b/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/libxev/20240910.0-a2d9b31/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20240910.0-a2d9b31/source.json b/third_party/modules/libxev/20240910.0-a2d9b31/source.json new file mode 100644 index 00000000..c56bc6a8 --- /dev/null +++ b/third_party/modules/libxev/20240910.0-a2d9b31/source.json @@ -0,0 +1,9 @@ +{ + "strip_prefix": "libxev-a2d9b3154f1e5ed463c9f2fb98a0d739057796ce", + "url": "https://github.com/zml/libxev/archive/a2d9b3154f1e5ed463c9f2fb98a0d739057796ce.tar.gz", + "integrity": "sha256-xdMjrGSB3telt52iRqHgem0WSRPjfGlTKBRyDMXLlaQ=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + } +} diff --git a/third_party/modules/libxev/20241119.0-6afcde9/MODULE.bazel b/third_party/modules/libxev/20241119.0-6afcde9/MODULE.bazel new file mode 100644 index 00000000..c9b03066 --- /dev/null +++ b/third_party/modules/libxev/20241119.0-6afcde9/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20241119.0-6afcde9", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241119.0-6afcde9/overlay/BUILD.bazel b/third_party/modules/libxev/20241119.0-6afcde9/overlay/BUILD.bazel new file mode 100644 index 00000000..7c0c4ce9 --- /dev/null +++ b/third_party/modules/libxev/20241119.0-6afcde9/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "main2.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20241119.0-6afcde9/overlay/MODULE.bazel b/third_party/modules/libxev/20241119.0-6afcde9/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/libxev/20241119.0-6afcde9/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20241119.0-6afcde9/overlay/main2.zig b/third_party/modules/libxev/20241119.0-6afcde9/overlay/main2.zig new file mode 100644 index 00000000..4166925a --- /dev/null +++ b/third_party/modules/libxev/20241119.0-6afcde9/overlay/main2.zig @@ -0,0 +1,22 @@ +const builtin = @import("builtin"); +const root = @import("root"); + +const main = @import("src/main.zig"); + +pub const ThreadPool = main.ThreadPool; +pub const stream = main.stream; + +pub const Options = struct { + linux_backend: main.Backend = .epoll, +}; + +pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; + +const default: main.Backend = switch (builtin.os.tag) { + .ios, .macos => .kqueue, + .linux => options.linux_backend, + .wasi => .wasi_poll, + .windows => .iocp, + else => @compileError("Unsupported OS"), +}; +pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241119.0-6afcde9/patches/128.patch b/third_party/modules/libxev/20241119.0-6afcde9/patches/128.patch new file mode 100644 index 00000000..6643419d --- /dev/null +++ b/third_party/modules/libxev/20241119.0-6afcde9/patches/128.patch @@ -0,0 +1,119 @@ +From 0d1c2f8258072148459d3114b9ccaf43c02e0958 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Tue, 19 Nov 2024 16:14:14 +0100 +Subject: [PATCH] backend/epoll: implement eventfd wakeup notification + +Tries to mimic what happens in backend/kqueue. + +Closes #4 +--- + src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 42 insertions(+) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index ae4ec7d..f44d326 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -21,6 +21,12 @@ pub const Loop = struct { + + fd: posix.fd_t, + ++ /// The eventfd that this epoll queue always has a filter for. Writing ++ /// an empty message to this eventfd can be used to wake up the loop ++ /// at any time. Waking up the loop via this eventfd won't trigger any ++ /// particular completion, it just forces tick to cycle. ++ eventfd: xev.Async, ++ + /// The number of active completions. This DOES NOT include completions that + /// are queued in the submissions queue. + active: usize = 0, +@@ -56,8 +62,12 @@ pub const Loop = struct { + } = .{}, + + pub fn init(options: xev.Options) !Loop { ++ var eventfd = try xev.Async.init(); ++ errdefer eventfd.deinit(); ++ + var res: Loop = .{ + .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), ++ .eventfd = eventfd, + .thread_pool = options.thread_pool, + .thread_pool_completions = undefined, + .cached_now = undefined, +@@ -68,6 +78,7 @@ pub const Loop = struct { + + pub fn deinit(self: *Loop) void { + posix.close(self.fd); ++ self.eventfd.deinit(); + } + + /// Run the event loop. See RunMode documentation for details on modes. +@@ -262,9 +273,26 @@ pub const Loop = struct { + // Initialize + if (!self.flags.init) { + self.flags.init = true; ++ + if (self.thread_pool != null) { + self.thread_pool_completions.init(); + } ++ ++ var ev: linux.epoll_event = .{ ++ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, ++ .data = .{ .ptr = 0 }, ++ }; ++ posix.epoll_ctl( ++ self.fd, ++ linux.EPOLL.CTL_ADD, ++ self.eventfd.fd, ++ &ev, ++ ) catch |err| { ++ // We reset initialization because we can't do anything ++ // safely unless we get this mach port registered! ++ self.flags.init = false; ++ return err; ++ }; + } + + // Submit all the submissions. We copy the submission queue so that +@@ -369,6 +397,10 @@ pub const Loop = struct { + + // Process all our events and invoke their completion handlers + for (events[0..n]) |ev| { ++ // Zero data values are internal events that we do nothing ++ // on such as the eventfd wakeup. ++ if (ev.data.ptr == 0) continue; ++ + const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); + + // We get the fd and mark this as in progress we can properly +@@ -415,6 +447,7 @@ pub const Loop = struct { + const pool = self.thread_pool orelse return error.ThreadPoolRequired; + + // Setup our completion state so that thread_perform can do stuff ++ c.task_loop = self; + c.task_completions = &self.thread_pool_completions; + c.task = .{ .callback = Loop.thread_perform }; + +@@ -436,6 +469,14 @@ pub const Loop = struct { + + // Add to our completion queue + c.task_completions.push(c); ++ ++ // Wake up our main loop ++ c.task_loop.wakeup() catch {}; ++ } ++ ++ /// Sends an empty message to this loop's eventfd so that it wakes up. ++ fn wakeup(self: *Loop) !void { ++ try self.eventfd.notify(); + } + + fn start(self: *Loop, completion: *Completion) void { +@@ -800,6 +841,7 @@ pub const Completion = struct { + /// reliable way to get access to the loop and shouldn't be used + /// except internally. + task: ThreadPool.Task = undefined, ++ task_loop: *Loop = undefined, + task_completions: *Loop.TaskCompletionQueue = undefined, + task_result: Result = undefined, + diff --git a/third_party/modules/libxev/20241119.0-6afcde9/source.json b/third_party/modules/libxev/20241119.0-6afcde9/source.json new file mode 100644 index 00000000..de3c2417 --- /dev/null +++ b/third_party/modules/libxev/20241119.0-6afcde9/source.json @@ -0,0 +1,14 @@ +{ + "strip_prefix": "libxev-690c76fd792f001c5776716f1e7b04be2cc50b52", + "url": "https://github.com/zml/libxev/archive/690c76fd792f001c5776716f1e7b04be2cc50b52.tar.gz", + "integrity": "sha256-DV66ic8PcRnG3EdimswCleiHo/dDztgebz/1EY5XDXg=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "", + "main2.zig": "" + }, + "patches": { + "128.patch": "" + }, + "patch_strip": 1 +} diff --git a/third_party/modules/libxev/20241208.0-db6a52b/MODULE.bazel b/third_party/modules/libxev/20241208.0-db6a52b/MODULE.bazel new file mode 100644 index 00000000..ce556ab1 --- /dev/null +++ b/third_party/modules/libxev/20241208.0-db6a52b/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20241208.0-db6a52b", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.0-db6a52b/overlay/BUILD.bazel b/third_party/modules/libxev/20241208.0-db6a52b/overlay/BUILD.bazel new file mode 100644 index 00000000..7c0c4ce9 --- /dev/null +++ b/third_party/modules/libxev/20241208.0-db6a52b/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "main2.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20241208.0-db6a52b/overlay/MODULE.bazel b/third_party/modules/libxev/20241208.0-db6a52b/overlay/MODULE.bazel new file mode 100644 index 00000000..ce556ab1 --- /dev/null +++ b/third_party/modules/libxev/20241208.0-db6a52b/overlay/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20241208.0-db6a52b", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.0-db6a52b/overlay/main2.zig b/third_party/modules/libxev/20241208.0-db6a52b/overlay/main2.zig new file mode 100644 index 00000000..4166925a --- /dev/null +++ b/third_party/modules/libxev/20241208.0-db6a52b/overlay/main2.zig @@ -0,0 +1,22 @@ +const builtin = @import("builtin"); +const root = @import("root"); + +const main = @import("src/main.zig"); + +pub const ThreadPool = main.ThreadPool; +pub const stream = main.stream; + +pub const Options = struct { + linux_backend: main.Backend = .epoll, +}; + +pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; + +const default: main.Backend = switch (builtin.os.tag) { + .ios, .macos => .kqueue, + .linux => options.linux_backend, + .wasi => .wasi_poll, + .windows => .iocp, + else => @compileError("Unsupported OS"), +}; +pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241208.0-db6a52b/patches/128.patch b/third_party/modules/libxev/20241208.0-db6a52b/patches/128.patch new file mode 100644 index 00000000..6db29cc8 --- /dev/null +++ b/third_party/modules/libxev/20241208.0-db6a52b/patches/128.patch @@ -0,0 +1,348 @@ +From 7b345e8d423f6cb1907f0dcba1d3c2f82e4c6b4d Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Tue, 19 Nov 2024 16:14:14 +0100 +Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification + +Tries to mimic what happens in backend/kqueue. + +Closes #4 +--- + src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 42 insertions(+) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index ae4ec7d..f44d326 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -21,6 +21,12 @@ pub const Loop = struct { + + fd: posix.fd_t, + ++ /// The eventfd that this epoll queue always has a filter for. Writing ++ /// an empty message to this eventfd can be used to wake up the loop ++ /// at any time. Waking up the loop via this eventfd won't trigger any ++ /// particular completion, it just forces tick to cycle. ++ eventfd: xev.Async, ++ + /// The number of active completions. This DOES NOT include completions that + /// are queued in the submissions queue. + active: usize = 0, +@@ -56,8 +62,12 @@ pub const Loop = struct { + } = .{}, + + pub fn init(options: xev.Options) !Loop { ++ var eventfd = try xev.Async.init(); ++ errdefer eventfd.deinit(); ++ + var res: Loop = .{ + .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), ++ .eventfd = eventfd, + .thread_pool = options.thread_pool, + .thread_pool_completions = undefined, + .cached_now = undefined, +@@ -68,6 +78,7 @@ pub const Loop = struct { + + pub fn deinit(self: *Loop) void { + posix.close(self.fd); ++ self.eventfd.deinit(); + } + + /// Run the event loop. See RunMode documentation for details on modes. +@@ -262,9 +273,26 @@ pub const Loop = struct { + // Initialize + if (!self.flags.init) { + self.flags.init = true; ++ + if (self.thread_pool != null) { + self.thread_pool_completions.init(); + } ++ ++ var ev: linux.epoll_event = .{ ++ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, ++ .data = .{ .ptr = 0 }, ++ }; ++ posix.epoll_ctl( ++ self.fd, ++ linux.EPOLL.CTL_ADD, ++ self.eventfd.fd, ++ &ev, ++ ) catch |err| { ++ // We reset initialization because we can't do anything ++ // safely unless we get this mach port registered! ++ self.flags.init = false; ++ return err; ++ }; + } + + // Submit all the submissions. We copy the submission queue so that +@@ -369,6 +397,10 @@ pub const Loop = struct { + + // Process all our events and invoke their completion handlers + for (events[0..n]) |ev| { ++ // Zero data values are internal events that we do nothing ++ // on such as the eventfd wakeup. ++ if (ev.data.ptr == 0) continue; ++ + const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); + + // We get the fd and mark this as in progress we can properly +@@ -415,6 +447,7 @@ pub const Loop = struct { + const pool = self.thread_pool orelse return error.ThreadPoolRequired; + + // Setup our completion state so that thread_perform can do stuff ++ c.task_loop = self; + c.task_completions = &self.thread_pool_completions; + c.task = .{ .callback = Loop.thread_perform }; + +@@ -436,6 +469,14 @@ pub const Loop = struct { + + // Add to our completion queue + c.task_completions.push(c); ++ ++ // Wake up our main loop ++ c.task_loop.wakeup() catch {}; ++ } ++ ++ /// Sends an empty message to this loop's eventfd so that it wakes up. ++ fn wakeup(self: *Loop) !void { ++ try self.eventfd.notify(); + } + + fn start(self: *Loop, completion: *Completion) void { +@@ -800,6 +841,7 @@ pub const Completion = struct { + /// reliable way to get access to the loop and shouldn't be used + /// except internally. + task: ThreadPool.Task = undefined, ++ task_loop: *Loop = undefined, + task_completions: *Loop.TaskCompletionQueue = undefined, + task_result: Result = undefined, + + +From 40dc25fa922fe436418f9c01acd9b76a74593ddb Mon Sep 17 00:00:00 2001 +From: Corentin Godeau +Date: Tue, 14 Jan 2025 14:43:54 +0000 +Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being + awaken again + +--- + src/backend/epoll.zig | 11 +++++++---- + 1 file changed, 7 insertions(+), 4 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index f44d326..f84c687 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -280,7 +280,7 @@ pub const Loop = struct { + + var ev: linux.epoll_event = .{ + .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, +- .data = .{ .ptr = 0 }, ++ .data = .{ .fd = self.eventfd.fd }, + }; + posix.epoll_ctl( + self.fd, +@@ -397,9 +397,12 @@ pub const Loop = struct { + + // Process all our events and invoke their completion handlers + for (events[0..n]) |ev| { +- // Zero data values are internal events that we do nothing +- // on such as the eventfd wakeup. +- if (ev.data.ptr == 0) continue; ++ // Handle wakeup eventfd ++ if (ev.data.fd == self.eventfd.fd) { ++ var buffer: u64 = undefined; ++ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; ++ continue; ++ } + + const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); + + +From c082ed1270a59d4276d7ac6752833b6730fa73d5 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:47:42 +0000 +Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait + +Since eventfd is now implemented. +--- + src/backend/epoll.zig | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index f84c687..e3eee20 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -375,9 +375,7 @@ pub const Loop = struct { + const timeout: i32 = if (wait_rem == 0) 0 else timeout: { + // If we have a timer, we want to set the timeout to our next + // timer value. If we have no timer, we wait forever. +- // TODO: do not wait 100ms here, use an eventfd for our +- // thread pool to wake us up. +- const t = self.timers.peek() orelse break :timeout 100; ++ const t = self.timers.peek() orelse break :timeout -1; + + // Determine the time in milliseconds. + const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + + +From e8f23275bf736bcc39cb0c9e4617f6e1338c5880 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:48:27 +0000 +Subject: [PATCH 4/5] epoll,kqueue: dispatch close in threadpool + +Close might block, so dispatch it inside a threadpool. +--- + src/backend/epoll.zig | 15 +++++++++++++-- + src/backend/kqueue.zig | 6 +++++- + src/watcher/stream.zig | 16 ++++++++++++++++ + 3 files changed, 34 insertions(+), 3 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index e3eee20..0f4a2ac 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -700,6 +700,13 @@ pub const Loop = struct { + }, + + .close => |v| res: { ++ if (completion.flags.threadpool) { ++ if (self.thread_schedule(completion)) |_| ++ return ++ else |err| ++ break :res .{ .close = err }; ++ } ++ + posix.close(v.fd); + break :res .{ .close = {} }; + }, +@@ -909,7 +916,6 @@ pub const Completion = struct { + // This should never happen because we always do these synchronously + // or in another location. + .cancel, +- .close, + .noop, + .shutdown, + .timer, +@@ -1015,6 +1021,11 @@ pub const Completion = struct { + err, + }; + }, ++ ++ .close => |*op| res: { ++ posix.close(op.fd); ++ break :res .{ .close = {} }; ++ }, + }; + } + +@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ + Unknown, + }; + +-pub const CloseError = posix.EpollCtlError || error{ ++pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ + Unknown, + }; + +diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig +index 456f64f..9914309 100644 +--- a/src/backend/kqueue.zig ++++ b/src/backend/kqueue.zig +@@ -1102,7 +1102,6 @@ pub const Completion = struct { + fn perform(self: *Completion, ev_: ?*const Kevent) Result { + return switch (self.op) { + .cancel, +- .close, + .noop, + .timer, + .shutdown, +@@ -1232,6 +1231,11 @@ pub const Completion = struct { + + break :res .{ .proc = 0 }; + }, ++ ++ .close => |*op| res: { ++ posix.close(op.fd); ++ break :res .{ .close = {} }; ++ }, + }; + } + +diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig +index 7f5df6f..bc95282 100644 +--- a/src/watcher/stream.zig ++++ b/src/watcher/stream.zig +@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options + }).callback, + }; + ++ // If we're dup-ing, then we ask the backend to manage the fd. ++ switch (xev.backend) { ++ .io_uring, ++ .wasi_poll, ++ .iocp, ++ => {}, ++ ++ .epoll => { ++ c.flags.threadpool = true; ++ }, ++ ++ .kqueue => { ++ c.flags.threadpool = true; ++ }, ++ } ++ + loop.add(c); + } + }; + +From 378a3968b8820aef5a6ab64d1893d022b1b6d70c Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:59:18 +0000 +Subject: [PATCH 5/5] epoll: don't count immediate actions + +If an immediate action is dispatched, the loop might block +on epoll_wait even though only one action was requested. +--- + src/backend/epoll.zig | 15 ++++++++++++++- + 1 file changed, 14 insertions(+), 1 deletion(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index 0f4a2ac..fb7e59e 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -297,6 +297,7 @@ pub const Loop = struct { + + // Submit all the submissions. We copy the submission queue so that + // any resubmits don't cause an infinite loop. ++ var wait_rem: usize = @intCast(wait); + var queued = self.submissions; + self.submissions = .{}; + while (queued.pop()) |c| { +@@ -304,6 +305,19 @@ pub const Loop = struct { + // This usually means that we switched them to be deleted or + // something. + if (c.flags.state != .adding) continue; ++ ++ // These operations happen synchronously. Ensure they are ++ // decremented from wait_rem. ++ switch (c.op) { ++ .cancel, ++ // should noop be counted? ++ // .noop, ++ .shutdown, ++ .timer, ++ => wait_rem -|= 1, ++ else => {}, ++ } ++ + self.start(c); + } + +@@ -322,7 +336,6 @@ pub const Loop = struct { + + // Wait and process events. We only do this if we have any active. + var events: [1024]linux.epoll_event = undefined; +- var wait_rem: usize = @intCast(wait); + while (self.active > 0 and (wait == 0 or wait_rem > 0)) { + self.update_now(); + const now_timer: Operation.Timer = .{ .next = self.cached_now }; diff --git a/third_party/modules/libxev/20241208.0-db6a52b/source.json b/third_party/modules/libxev/20241208.0-db6a52b/source.json new file mode 100644 index 00000000..f08c9517 --- /dev/null +++ b/third_party/modules/libxev/20241208.0-db6a52b/source.json @@ -0,0 +1,14 @@ +{ + "strip_prefix": "libxev-db6a52bafadf00360e675fefa7926e8e6c0e9931", + "url": "https://github.com/zml/libxev/archive/db6a52bafadf00360e675fefa7926e8e6c0e9931.tar.gz", + "integrity": "sha256-4GT5wkfkZnIjNv20yDiWEzHAhbIiwHHJfS7A4u/LoNQ=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "", + "main2.zig": "" + }, + "patches": { + "128.patch": "" + }, + "patch_strip": 1 +} diff --git a/third_party/modules/libxev/20241208.1-db6a52b/MODULE.bazel b/third_party/modules/libxev/20241208.1-db6a52b/MODULE.bazel new file mode 100644 index 00000000..ea8c55d7 --- /dev/null +++ b/third_party/modules/libxev/20241208.1-db6a52b/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20241208.1-db6a52b", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.1-db6a52b/overlay/BUILD.bazel b/third_party/modules/libxev/20241208.1-db6a52b/overlay/BUILD.bazel new file mode 100644 index 00000000..7c0c4ce9 --- /dev/null +++ b/third_party/modules/libxev/20241208.1-db6a52b/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "main2.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20241208.1-db6a52b/overlay/MODULE.bazel b/third_party/modules/libxev/20241208.1-db6a52b/overlay/MODULE.bazel new file mode 100644 index 00000000..ce556ab1 --- /dev/null +++ b/third_party/modules/libxev/20241208.1-db6a52b/overlay/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20241208.0-db6a52b", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.1-db6a52b/overlay/main2.zig b/third_party/modules/libxev/20241208.1-db6a52b/overlay/main2.zig new file mode 100644 index 00000000..4166925a --- /dev/null +++ b/third_party/modules/libxev/20241208.1-db6a52b/overlay/main2.zig @@ -0,0 +1,22 @@ +const builtin = @import("builtin"); +const root = @import("root"); + +const main = @import("src/main.zig"); + +pub const ThreadPool = main.ThreadPool; +pub const stream = main.stream; + +pub const Options = struct { + linux_backend: main.Backend = .epoll, +}; + +pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; + +const default: main.Backend = switch (builtin.os.tag) { + .ios, .macos => .kqueue, + .linux => options.linux_backend, + .wasi => .wasi_poll, + .windows => .iocp, + else => @compileError("Unsupported OS"), +}; +pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241208.1-db6a52b/patches/128.patch b/third_party/modules/libxev/20241208.1-db6a52b/patches/128.patch new file mode 100644 index 00000000..361bc9d1 --- /dev/null +++ b/third_party/modules/libxev/20241208.1-db6a52b/patches/128.patch @@ -0,0 +1,323 @@ +From 8927108937e23ce2ad547a310dabd976ddcaa39d Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Tue, 19 Nov 2024 16:14:14 +0100 +Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification + +Tries to mimic what happens in backend/kqueue. + +Closes #4 +--- + src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 42 insertions(+) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index ae4ec7d..f44d326 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -21,6 +21,12 @@ pub const Loop = struct { + + fd: posix.fd_t, + ++ /// The eventfd that this epoll queue always has a filter for. Writing ++ /// an empty message to this eventfd can be used to wake up the loop ++ /// at any time. Waking up the loop via this eventfd won't trigger any ++ /// particular completion, it just forces tick to cycle. ++ eventfd: xev.Async, ++ + /// The number of active completions. This DOES NOT include completions that + /// are queued in the submissions queue. + active: usize = 0, +@@ -56,8 +62,12 @@ pub const Loop = struct { + } = .{}, + + pub fn init(options: xev.Options) !Loop { ++ var eventfd = try xev.Async.init(); ++ errdefer eventfd.deinit(); ++ + var res: Loop = .{ + .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), ++ .eventfd = eventfd, + .thread_pool = options.thread_pool, + .thread_pool_completions = undefined, + .cached_now = undefined, +@@ -68,6 +78,7 @@ pub const Loop = struct { + + pub fn deinit(self: *Loop) void { + posix.close(self.fd); ++ self.eventfd.deinit(); + } + + /// Run the event loop. See RunMode documentation for details on modes. +@@ -262,9 +273,26 @@ pub const Loop = struct { + // Initialize + if (!self.flags.init) { + self.flags.init = true; ++ + if (self.thread_pool != null) { + self.thread_pool_completions.init(); + } ++ ++ var ev: linux.epoll_event = .{ ++ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, ++ .data = .{ .ptr = 0 }, ++ }; ++ posix.epoll_ctl( ++ self.fd, ++ linux.EPOLL.CTL_ADD, ++ self.eventfd.fd, ++ &ev, ++ ) catch |err| { ++ // We reset initialization because we can't do anything ++ // safely unless we get this mach port registered! ++ self.flags.init = false; ++ return err; ++ }; + } + + // Submit all the submissions. We copy the submission queue so that +@@ -369,6 +397,10 @@ pub const Loop = struct { + + // Process all our events and invoke their completion handlers + for (events[0..n]) |ev| { ++ // Zero data values are internal events that we do nothing ++ // on such as the eventfd wakeup. ++ if (ev.data.ptr == 0) continue; ++ + const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); + + // We get the fd and mark this as in progress we can properly +@@ -415,6 +447,7 @@ pub const Loop = struct { + const pool = self.thread_pool orelse return error.ThreadPoolRequired; + + // Setup our completion state so that thread_perform can do stuff ++ c.task_loop = self; + c.task_completions = &self.thread_pool_completions; + c.task = .{ .callback = Loop.thread_perform }; + +@@ -436,6 +469,14 @@ pub const Loop = struct { + + // Add to our completion queue + c.task_completions.push(c); ++ ++ // Wake up our main loop ++ c.task_loop.wakeup() catch {}; ++ } ++ ++ /// Sends an empty message to this loop's eventfd so that it wakes up. ++ fn wakeup(self: *Loop) !void { ++ try self.eventfd.notify(); + } + + fn start(self: *Loop, completion: *Completion) void { +@@ -800,6 +841,7 @@ pub const Completion = struct { + /// reliable way to get access to the loop and shouldn't be used + /// except internally. + task: ThreadPool.Task = undefined, ++ task_loop: *Loop = undefined, + task_completions: *Loop.TaskCompletionQueue = undefined, + task_result: Result = undefined, + + +From bf6c19eb1c06d1fb1b3cb5a66ea2ce8f743da5ba Mon Sep 17 00:00:00 2001 +From: Corentin Godeau +Date: Tue, 14 Jan 2025 14:43:54 +0000 +Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being + awaken again + +--- + src/backend/epoll.zig | 11 +++++++---- + 1 file changed, 7 insertions(+), 4 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index f44d326..f84c687 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -280,7 +280,7 @@ pub const Loop = struct { + + var ev: linux.epoll_event = .{ + .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, +- .data = .{ .ptr = 0 }, ++ .data = .{ .fd = self.eventfd.fd }, + }; + posix.epoll_ctl( + self.fd, +@@ -397,9 +397,12 @@ pub const Loop = struct { + + // Process all our events and invoke their completion handlers + for (events[0..n]) |ev| { +- // Zero data values are internal events that we do nothing +- // on such as the eventfd wakeup. +- if (ev.data.ptr == 0) continue; ++ // Handle wakeup eventfd ++ if (ev.data.fd == self.eventfd.fd) { ++ var buffer: u64 = undefined; ++ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; ++ continue; ++ } + + const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); + + +From c0ae753a9874bf78a48bf25b4362b6cca4de1da1 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:47:42 +0000 +Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait + +Since eventfd is now implemented. +--- + src/backend/epoll.zig | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index f84c687..e3eee20 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -375,9 +375,7 @@ pub const Loop = struct { + const timeout: i32 = if (wait_rem == 0) 0 else timeout: { + // If we have a timer, we want to set the timeout to our next + // timer value. If we have no timer, we wait forever. +- // TODO: do not wait 100ms here, use an eventfd for our +- // thread pool to wake us up. +- const t = self.timers.peek() orelse break :timeout 100; ++ const t = self.timers.peek() orelse break :timeout -1; + + // Determine the time in milliseconds. + const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + + +From 84e75a10ec1c10ea2a13edfcd8a31e582458fc88 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:48:27 +0000 +Subject: [PATCH 4/5] epoll: dispatch close in threadpool + +Close might block, so dispatch it inside a threadpool. +--- + src/backend/epoll.zig | 15 +++++++++++++-- + src/watcher/stream.zig | 16 ++++++++++++++++ + 2 files changed, 29 insertions(+), 2 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index e3eee20..0f4a2ac 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -700,6 +700,13 @@ pub const Loop = struct { + }, + + .close => |v| res: { ++ if (completion.flags.threadpool) { ++ if (self.thread_schedule(completion)) |_| ++ return ++ else |err| ++ break :res .{ .close = err }; ++ } ++ + posix.close(v.fd); + break :res .{ .close = {} }; + }, +@@ -909,7 +916,6 @@ pub const Completion = struct { + // This should never happen because we always do these synchronously + // or in another location. + .cancel, +- .close, + .noop, + .shutdown, + .timer, +@@ -1015,6 +1021,11 @@ pub const Completion = struct { + err, + }; + }, ++ ++ .close => |*op| res: { ++ posix.close(op.fd); ++ break :res .{ .close = {} }; ++ }, + }; + } + +@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ + Unknown, + }; + +-pub const CloseError = posix.EpollCtlError || error{ ++pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ + Unknown, + }; + +diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig +index 7f5df6f..bc95282 100644 +--- a/src/watcher/stream.zig ++++ b/src/watcher/stream.zig +@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options + }).callback, + }; + ++ // If we're dup-ing, then we ask the backend to manage the fd. ++ switch (xev.backend) { ++ .io_uring, ++ .wasi_poll, ++ .iocp, ++ => {}, ++ ++ .epoll => { ++ c.flags.threadpool = true; ++ }, ++ ++ .kqueue => { ++ c.flags.threadpool = true; ++ }, ++ } ++ + loop.add(c); + } + }; + +From 5bc3ad9f23a03ff3af7a4ec33c8bfa4f3cc6bae9 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:59:18 +0000 +Subject: [PATCH 5/5] epoll: don't count immediate actions + +If an immediate action is dispatched, the loop might block +on epoll_wait even though only one action was requested. +--- + src/backend/epoll.zig | 15 ++++++++++++++- + 1 file changed, 14 insertions(+), 1 deletion(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index 0f4a2ac..fb7e59e 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -297,6 +297,7 @@ pub const Loop = struct { + + // Submit all the submissions. We copy the submission queue so that + // any resubmits don't cause an infinite loop. ++ var wait_rem: usize = @intCast(wait); + var queued = self.submissions; + self.submissions = .{}; + while (queued.pop()) |c| { +@@ -304,6 +305,19 @@ pub const Loop = struct { + // This usually means that we switched them to be deleted or + // something. + if (c.flags.state != .adding) continue; ++ ++ // These operations happen synchronously. Ensure they are ++ // decremented from wait_rem. ++ switch (c.op) { ++ .cancel, ++ // should noop be counted? ++ // .noop, ++ .shutdown, ++ .timer, ++ => wait_rem -|= 1, ++ else => {}, ++ } ++ + self.start(c); + } + +@@ -322,7 +336,6 @@ pub const Loop = struct { + + // Wait and process events. We only do this if we have any active. + var events: [1024]linux.epoll_event = undefined; +- var wait_rem: usize = @intCast(wait); + while (self.active > 0 and (wait == 0 or wait_rem > 0)) { + self.update_now(); + const now_timer: Operation.Timer = .{ .next = self.cached_now }; diff --git a/third_party/modules/libxev/20241208.1-db6a52b/source.json b/third_party/modules/libxev/20241208.1-db6a52b/source.json new file mode 100644 index 00000000..f08c9517 --- /dev/null +++ b/third_party/modules/libxev/20241208.1-db6a52b/source.json @@ -0,0 +1,14 @@ +{ + "strip_prefix": "libxev-db6a52bafadf00360e675fefa7926e8e6c0e9931", + "url": "https://github.com/zml/libxev/archive/db6a52bafadf00360e675fefa7926e8e6c0e9931.tar.gz", + "integrity": "sha256-4GT5wkfkZnIjNv20yDiWEzHAhbIiwHHJfS7A4u/LoNQ=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "", + "main2.zig": "" + }, + "patches": { + "128.patch": "" + }, + "patch_strip": 1 +} diff --git a/third_party/modules/libxev/20241208.2-db6a52b/MODULE.bazel b/third_party/modules/libxev/20241208.2-db6a52b/MODULE.bazel new file mode 100644 index 00000000..1d593b7d --- /dev/null +++ b/third_party/modules/libxev/20241208.2-db6a52b/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20241208.2-db6a52b", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.2-db6a52b/overlay/BUILD.bazel b/third_party/modules/libxev/20241208.2-db6a52b/overlay/BUILD.bazel new file mode 100644 index 00000000..7c0c4ce9 --- /dev/null +++ b/third_party/modules/libxev/20241208.2-db6a52b/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "main2.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20241208.2-db6a52b/overlay/MODULE.bazel b/third_party/modules/libxev/20241208.2-db6a52b/overlay/MODULE.bazel new file mode 100644 index 00000000..ce556ab1 --- /dev/null +++ b/third_party/modules/libxev/20241208.2-db6a52b/overlay/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20241208.0-db6a52b", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20241208.2-db6a52b/overlay/main2.zig b/third_party/modules/libxev/20241208.2-db6a52b/overlay/main2.zig new file mode 100644 index 00000000..4166925a --- /dev/null +++ b/third_party/modules/libxev/20241208.2-db6a52b/overlay/main2.zig @@ -0,0 +1,22 @@ +const builtin = @import("builtin"); +const root = @import("root"); + +const main = @import("src/main.zig"); + +pub const ThreadPool = main.ThreadPool; +pub const stream = main.stream; + +pub const Options = struct { + linux_backend: main.Backend = .epoll, +}; + +pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; + +const default: main.Backend = switch (builtin.os.tag) { + .ios, .macos => .kqueue, + .linux => options.linux_backend, + .wasi => .wasi_poll, + .windows => .iocp, + else => @compileError("Unsupported OS"), +}; +pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20241208.2-db6a52b/patches/128.patch b/third_party/modules/libxev/20241208.2-db6a52b/patches/128.patch new file mode 100644 index 00000000..18349433 --- /dev/null +++ b/third_party/modules/libxev/20241208.2-db6a52b/patches/128.patch @@ -0,0 +1,359 @@ +From 7b345e8d423f6cb1907f0dcba1d3c2f82e4c6b4d Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Tue, 19 Nov 2024 16:14:14 +0100 +Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification + +Tries to mimic what happens in backend/kqueue. + +Closes #4 +--- + src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 42 insertions(+) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index ae4ec7d..f44d326 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -21,6 +21,12 @@ pub const Loop = struct { + + fd: posix.fd_t, + ++ /// The eventfd that this epoll queue always has a filter for. Writing ++ /// an empty message to this eventfd can be used to wake up the loop ++ /// at any time. Waking up the loop via this eventfd won't trigger any ++ /// particular completion, it just forces tick to cycle. ++ eventfd: xev.Async, ++ + /// The number of active completions. This DOES NOT include completions that + /// are queued in the submissions queue. + active: usize = 0, +@@ -56,8 +62,12 @@ pub const Loop = struct { + } = .{}, + + pub fn init(options: xev.Options) !Loop { ++ var eventfd = try xev.Async.init(); ++ errdefer eventfd.deinit(); ++ + var res: Loop = .{ + .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), ++ .eventfd = eventfd, + .thread_pool = options.thread_pool, + .thread_pool_completions = undefined, + .cached_now = undefined, +@@ -68,6 +78,7 @@ pub const Loop = struct { + + pub fn deinit(self: *Loop) void { + posix.close(self.fd); ++ self.eventfd.deinit(); + } + + /// Run the event loop. See RunMode documentation for details on modes. +@@ -262,9 +273,26 @@ pub const Loop = struct { + // Initialize + if (!self.flags.init) { + self.flags.init = true; ++ + if (self.thread_pool != null) { + self.thread_pool_completions.init(); + } ++ ++ var ev: linux.epoll_event = .{ ++ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, ++ .data = .{ .ptr = 0 }, ++ }; ++ posix.epoll_ctl( ++ self.fd, ++ linux.EPOLL.CTL_ADD, ++ self.eventfd.fd, ++ &ev, ++ ) catch |err| { ++ // We reset initialization because we can't do anything ++ // safely unless we get this mach port registered! ++ self.flags.init = false; ++ return err; ++ }; + } + + // Submit all the submissions. We copy the submission queue so that +@@ -369,6 +397,10 @@ pub const Loop = struct { + + // Process all our events and invoke their completion handlers + for (events[0..n]) |ev| { ++ // Zero data values are internal events that we do nothing ++ // on such as the eventfd wakeup. ++ if (ev.data.ptr == 0) continue; ++ + const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); + + // We get the fd and mark this as in progress we can properly +@@ -415,6 +447,7 @@ pub const Loop = struct { + const pool = self.thread_pool orelse return error.ThreadPoolRequired; + + // Setup our completion state so that thread_perform can do stuff ++ c.task_loop = self; + c.task_completions = &self.thread_pool_completions; + c.task = .{ .callback = Loop.thread_perform }; + +@@ -436,6 +469,14 @@ pub const Loop = struct { + + // Add to our completion queue + c.task_completions.push(c); ++ ++ // Wake up our main loop ++ c.task_loop.wakeup() catch {}; ++ } ++ ++ /// Sends an empty message to this loop's eventfd so that it wakes up. ++ fn wakeup(self: *Loop) !void { ++ try self.eventfd.notify(); + } + + fn start(self: *Loop, completion: *Completion) void { +@@ -800,6 +841,7 @@ pub const Completion = struct { + /// reliable way to get access to the loop and shouldn't be used + /// except internally. + task: ThreadPool.Task = undefined, ++ task_loop: *Loop = undefined, + task_completions: *Loop.TaskCompletionQueue = undefined, + task_result: Result = undefined, + +-- +2.39.5 (Apple Git-154) + +From 40dc25fa922fe436418f9c01acd9b76a74593ddb Mon Sep 17 00:00:00 2001 +From: Corentin Godeau +Date: Tue, 14 Jan 2025 14:43:54 +0000 +Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being + awaken again + +--- + src/backend/epoll.zig | 11 +++++++---- + 1 file changed, 7 insertions(+), 4 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index f44d326..f84c687 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -280,7 +280,7 @@ pub const Loop = struct { + + var ev: linux.epoll_event = .{ + .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, +- .data = .{ .ptr = 0 }, ++ .data = .{ .fd = self.eventfd.fd }, + }; + posix.epoll_ctl( + self.fd, +@@ -397,9 +397,12 @@ pub const Loop = struct { + + // Process all our events and invoke their completion handlers + for (events[0..n]) |ev| { +- // Zero data values are internal events that we do nothing +- // on such as the eventfd wakeup. +- if (ev.data.ptr == 0) continue; ++ // Handle wakeup eventfd ++ if (ev.data.fd == self.eventfd.fd) { ++ var buffer: u64 = undefined; ++ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; ++ continue; ++ } + + const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); + +-- +2.39.5 (Apple Git-154) + +From c082ed1270a59d4276d7ac6752833b6730fa73d5 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:47:42 +0000 +Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait + +Since eventfd is now implemented. +--- + src/backend/epoll.zig | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index f84c687..e3eee20 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -375,9 +375,7 @@ pub const Loop = struct { + const timeout: i32 = if (wait_rem == 0) 0 else timeout: { + // If we have a timer, we want to set the timeout to our next + // timer value. If we have no timer, we wait forever. +- // TODO: do not wait 100ms here, use an eventfd for our +- // thread pool to wake us up. +- const t = self.timers.peek() orelse break :timeout 100; ++ const t = self.timers.peek() orelse break :timeout -1; + + // Determine the time in milliseconds. + const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + +-- +2.39.5 (Apple Git-154) + +From e8f23275bf736bcc39cb0c9e4617f6e1338c5880 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:48:27 +0000 +Subject: [PATCH 4/5] epoll,kqueue: dispatch close in threadpool + +Close might block, so dispatch it inside a threadpool. +--- + src/backend/epoll.zig | 15 +++++++++++++-- + src/backend/kqueue.zig | 6 +++++- + src/watcher/stream.zig | 16 ++++++++++++++++ + 3 files changed, 34 insertions(+), 3 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index e3eee20..0f4a2ac 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -700,6 +700,13 @@ pub const Loop = struct { + }, + + .close => |v| res: { ++ if (completion.flags.threadpool) { ++ if (self.thread_schedule(completion)) |_| ++ return ++ else |err| ++ break :res .{ .close = err }; ++ } ++ + posix.close(v.fd); + break :res .{ .close = {} }; + }, +@@ -909,7 +916,6 @@ pub const Completion = struct { + // This should never happen because we always do these synchronously + // or in another location. + .cancel, +- .close, + .noop, + .shutdown, + .timer, +@@ -1015,6 +1021,11 @@ pub const Completion = struct { + err, + }; + }, ++ ++ .close => |*op| res: { ++ posix.close(op.fd); ++ break :res .{ .close = {} }; ++ }, + }; + } + +@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ + Unknown, + }; + +-pub const CloseError = posix.EpollCtlError || error{ ++pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ + Unknown, + }; + +diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig +index 456f64f..9914309 100644 +--- a/src/backend/kqueue.zig ++++ b/src/backend/kqueue.zig +@@ -1102,7 +1102,6 @@ pub const Completion = struct { + fn perform(self: *Completion, ev_: ?*const Kevent) Result { + return switch (self.op) { + .cancel, +- .close, + .noop, + .timer, + .shutdown, +@@ -1232,6 +1231,11 @@ pub const Completion = struct { + + break :res .{ .proc = 0 }; + }, ++ ++ .close => |*op| res: { ++ posix.close(op.fd); ++ break :res .{ .close = {} }; ++ }, + }; + } + +diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig +index 7f5df6f..bc95282 100644 +--- a/src/watcher/stream.zig ++++ b/src/watcher/stream.zig +@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options + }).callback, + }; + ++ // If we're dup-ing, then we ask the backend to manage the fd. ++ switch (xev.backend) { ++ .io_uring, ++ .wasi_poll, ++ .iocp, ++ => {}, ++ ++ .epoll => { ++ c.flags.threadpool = true; ++ }, ++ ++ .kqueue => { ++ c.flags.threadpool = true; ++ }, ++ } ++ + loop.add(c); + } + }; +-- +2.39.5 (Apple Git-154) + +From 378a3968b8820aef5a6ab64d1893d022b1b6d70c Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:59:18 +0000 +Subject: [PATCH 5/5] epoll: don't count immediate actions + +If an immediate action is dispatched, the loop might block +on epoll_wait even though only one action was requested. +--- + src/backend/epoll.zig | 15 ++++++++++++++- + 1 file changed, 14 insertions(+), 1 deletion(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index 0f4a2ac..fb7e59e 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -297,6 +297,7 @@ pub const Loop = struct { + + // Submit all the submissions. We copy the submission queue so that + // any resubmits don't cause an infinite loop. ++ var wait_rem: usize = @intCast(wait); + var queued = self.submissions; + self.submissions = .{}; + while (queued.pop()) |c| { +@@ -304,6 +305,19 @@ pub const Loop = struct { + // This usually means that we switched them to be deleted or + // something. + if (c.flags.state != .adding) continue; ++ ++ // These operations happen synchronously. Ensure they are ++ // decremented from wait_rem. ++ switch (c.op) { ++ .cancel, ++ // should noop be counted? ++ // .noop, ++ .shutdown, ++ .timer, ++ => wait_rem -|= 1, ++ else => {}, ++ } ++ + self.start(c); + } + +@@ -322,7 +336,6 @@ pub const Loop = struct { + + // Wait and process events. We only do this if we have any active. + var events: [1024]linux.epoll_event = undefined; +- var wait_rem: usize = @intCast(wait); + while (self.active > 0 and (wait == 0 or wait_rem > 0)) { + self.update_now(); + const now_timer: Operation.Timer = .{ .next = self.cached_now }; +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/modules/libxev/20241208.2-db6a52b/source.json b/third_party/modules/libxev/20241208.2-db6a52b/source.json new file mode 100644 index 00000000..f08c9517 --- /dev/null +++ b/third_party/modules/libxev/20241208.2-db6a52b/source.json @@ -0,0 +1,14 @@ +{ + "strip_prefix": "libxev-db6a52bafadf00360e675fefa7926e8e6c0e9931", + "url": "https://github.com/zml/libxev/archive/db6a52bafadf00360e675fefa7926e8e6c0e9931.tar.gz", + "integrity": "sha256-4GT5wkfkZnIjNv20yDiWEzHAhbIiwHHJfS7A4u/LoNQ=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "", + "main2.zig": "" + }, + "patches": { + "128.patch": "" + }, + "patch_strip": 1 +} diff --git a/third_party/modules/libxev/20250124.0-31eed4e/MODULE.bazel b/third_party/modules/libxev/20250124.0-31eed4e/MODULE.bazel new file mode 100644 index 00000000..6102ba67 --- /dev/null +++ b/third_party/modules/libxev/20250124.0-31eed4e/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20250124.0-31eed4e", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20250124.0-31eed4e/overlay/BUILD.bazel b/third_party/modules/libxev/20250124.0-31eed4e/overlay/BUILD.bazel new file mode 100644 index 00000000..7c0c4ce9 --- /dev/null +++ b/third_party/modules/libxev/20250124.0-31eed4e/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "main2.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20250124.0-31eed4e/overlay/MODULE.bazel b/third_party/modules/libxev/20250124.0-31eed4e/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/libxev/20250124.0-31eed4e/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250124.0-31eed4e/overlay/main2.zig b/third_party/modules/libxev/20250124.0-31eed4e/overlay/main2.zig new file mode 100644 index 00000000..4166925a --- /dev/null +++ b/third_party/modules/libxev/20250124.0-31eed4e/overlay/main2.zig @@ -0,0 +1,22 @@ +const builtin = @import("builtin"); +const root = @import("root"); + +const main = @import("src/main.zig"); + +pub const ThreadPool = main.ThreadPool; +pub const stream = main.stream; + +pub const Options = struct { + linux_backend: main.Backend = .epoll, +}; + +pub const options: Options = if (@hasDecl(root, "xev_options")) root.xev_options else .{}; + +const default: main.Backend = switch (builtin.os.tag) { + .ios, .macos => .kqueue, + .linux => options.linux_backend, + .wasi => .wasi_poll, + .windows => .iocp, + else => @compileError("Unsupported OS"), +}; +pub usingnamespace default.Api(); diff --git a/third_party/modules/libxev/20250124.0-31eed4e/patches/128.patch b/third_party/modules/libxev/20250124.0-31eed4e/patches/128.patch new file mode 100644 index 00000000..18349433 --- /dev/null +++ b/third_party/modules/libxev/20250124.0-31eed4e/patches/128.patch @@ -0,0 +1,359 @@ +From 7b345e8d423f6cb1907f0dcba1d3c2f82e4c6b4d Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Tue, 19 Nov 2024 16:14:14 +0100 +Subject: [PATCH 1/5] backend/epoll: implement eventfd wakeup notification + +Tries to mimic what happens in backend/kqueue. + +Closes #4 +--- + src/backend/epoll.zig | 42 ++++++++++++++++++++++++++++++++++++++++++ + 1 file changed, 42 insertions(+) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index ae4ec7d..f44d326 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -21,6 +21,12 @@ pub const Loop = struct { + + fd: posix.fd_t, + ++ /// The eventfd that this epoll queue always has a filter for. Writing ++ /// an empty message to this eventfd can be used to wake up the loop ++ /// at any time. Waking up the loop via this eventfd won't trigger any ++ /// particular completion, it just forces tick to cycle. ++ eventfd: xev.Async, ++ + /// The number of active completions. This DOES NOT include completions that + /// are queued in the submissions queue. + active: usize = 0, +@@ -56,8 +62,12 @@ pub const Loop = struct { + } = .{}, + + pub fn init(options: xev.Options) !Loop { ++ var eventfd = try xev.Async.init(); ++ errdefer eventfd.deinit(); ++ + var res: Loop = .{ + .fd = try posix.epoll_create1(std.os.linux.EPOLL.CLOEXEC), ++ .eventfd = eventfd, + .thread_pool = options.thread_pool, + .thread_pool_completions = undefined, + .cached_now = undefined, +@@ -68,6 +78,7 @@ pub const Loop = struct { + + pub fn deinit(self: *Loop) void { + posix.close(self.fd); ++ self.eventfd.deinit(); + } + + /// Run the event loop. See RunMode documentation for details on modes. +@@ -262,9 +273,26 @@ pub const Loop = struct { + // Initialize + if (!self.flags.init) { + self.flags.init = true; ++ + if (self.thread_pool != null) { + self.thread_pool_completions.init(); + } ++ ++ var ev: linux.epoll_event = .{ ++ .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, ++ .data = .{ .ptr = 0 }, ++ }; ++ posix.epoll_ctl( ++ self.fd, ++ linux.EPOLL.CTL_ADD, ++ self.eventfd.fd, ++ &ev, ++ ) catch |err| { ++ // We reset initialization because we can't do anything ++ // safely unless we get this mach port registered! ++ self.flags.init = false; ++ return err; ++ }; + } + + // Submit all the submissions. We copy the submission queue so that +@@ -369,6 +397,10 @@ pub const Loop = struct { + + // Process all our events and invoke their completion handlers + for (events[0..n]) |ev| { ++ // Zero data values are internal events that we do nothing ++ // on such as the eventfd wakeup. ++ if (ev.data.ptr == 0) continue; ++ + const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); + + // We get the fd and mark this as in progress we can properly +@@ -415,6 +447,7 @@ pub const Loop = struct { + const pool = self.thread_pool orelse return error.ThreadPoolRequired; + + // Setup our completion state so that thread_perform can do stuff ++ c.task_loop = self; + c.task_completions = &self.thread_pool_completions; + c.task = .{ .callback = Loop.thread_perform }; + +@@ -436,6 +469,14 @@ pub const Loop = struct { + + // Add to our completion queue + c.task_completions.push(c); ++ ++ // Wake up our main loop ++ c.task_loop.wakeup() catch {}; ++ } ++ ++ /// Sends an empty message to this loop's eventfd so that it wakes up. ++ fn wakeup(self: *Loop) !void { ++ try self.eventfd.notify(); + } + + fn start(self: *Loop, completion: *Completion) void { +@@ -800,6 +841,7 @@ pub const Completion = struct { + /// reliable way to get access to the loop and shouldn't be used + /// except internally. + task: ThreadPool.Task = undefined, ++ task_loop: *Loop = undefined, + task_completions: *Loop.TaskCompletionQueue = undefined, + task_result: Result = undefined, + +-- +2.39.5 (Apple Git-154) + +From 40dc25fa922fe436418f9c01acd9b76a74593ddb Mon Sep 17 00:00:00 2001 +From: Corentin Godeau +Date: Tue, 14 Jan 2025 14:43:54 +0000 +Subject: [PATCH 2/5] backend/epoll: read the wakeup eventfd to avoid being + awaken again + +--- + src/backend/epoll.zig | 11 +++++++---- + 1 file changed, 7 insertions(+), 4 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index f44d326..f84c687 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -280,7 +280,7 @@ pub const Loop = struct { + + var ev: linux.epoll_event = .{ + .events = linux.EPOLL.IN | linux.EPOLL.RDHUP, +- .data = .{ .ptr = 0 }, ++ .data = .{ .fd = self.eventfd.fd }, + }; + posix.epoll_ctl( + self.fd, +@@ -397,9 +397,12 @@ pub const Loop = struct { + + // Process all our events and invoke their completion handlers + for (events[0..n]) |ev| { +- // Zero data values are internal events that we do nothing +- // on such as the eventfd wakeup. +- if (ev.data.ptr == 0) continue; ++ // Handle wakeup eventfd ++ if (ev.data.fd == self.eventfd.fd) { ++ var buffer: u64 = undefined; ++ _ = posix.read(self.eventfd.fd, std.mem.asBytes(&buffer)) catch {}; ++ continue; ++ } + + const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.data.ptr))); + +-- +2.39.5 (Apple Git-154) + +From c082ed1270a59d4276d7ac6752833b6730fa73d5 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:47:42 +0000 +Subject: [PATCH 3/5] epoll: use infinite timeout for epoll_wait + +Since eventfd is now implemented. +--- + src/backend/epoll.zig | 4 +--- + 1 file changed, 1 insertion(+), 3 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index f84c687..e3eee20 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -375,9 +375,7 @@ pub const Loop = struct { + const timeout: i32 = if (wait_rem == 0) 0 else timeout: { + // If we have a timer, we want to set the timeout to our next + // timer value. If we have no timer, we wait forever. +- // TODO: do not wait 100ms here, use an eventfd for our +- // thread pool to wake us up. +- const t = self.timers.peek() orelse break :timeout 100; ++ const t = self.timers.peek() orelse break :timeout -1; + + // Determine the time in milliseconds. + const ms_now = @as(u64, @intCast(self.cached_now.tv_sec)) * std.time.ms_per_s + +-- +2.39.5 (Apple Git-154) + +From e8f23275bf736bcc39cb0c9e4617f6e1338c5880 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:48:27 +0000 +Subject: [PATCH 4/5] epoll,kqueue: dispatch close in threadpool + +Close might block, so dispatch it inside a threadpool. +--- + src/backend/epoll.zig | 15 +++++++++++++-- + src/backend/kqueue.zig | 6 +++++- + src/watcher/stream.zig | 16 ++++++++++++++++ + 3 files changed, 34 insertions(+), 3 deletions(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index e3eee20..0f4a2ac 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -700,6 +700,13 @@ pub const Loop = struct { + }, + + .close => |v| res: { ++ if (completion.flags.threadpool) { ++ if (self.thread_schedule(completion)) |_| ++ return ++ else |err| ++ break :res .{ .close = err }; ++ } ++ + posix.close(v.fd); + break :res .{ .close = {} }; + }, +@@ -909,7 +916,6 @@ pub const Completion = struct { + // This should never happen because we always do these synchronously + // or in another location. + .cancel, +- .close, + .noop, + .shutdown, + .timer, +@@ -1015,6 +1021,11 @@ pub const Completion = struct { + err, + }; + }, ++ ++ .close => |*op| res: { ++ posix.close(op.fd); ++ break :res .{ .close = {} }; ++ }, + }; + } + +@@ -1277,7 +1288,7 @@ pub const AcceptError = posix.EpollCtlError || error{ + Unknown, + }; + +-pub const CloseError = posix.EpollCtlError || error{ ++pub const CloseError = posix.EpollCtlError || ThreadPoolError || error{ + Unknown, + }; + +diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig +index 456f64f..9914309 100644 +--- a/src/backend/kqueue.zig ++++ b/src/backend/kqueue.zig +@@ -1102,7 +1102,6 @@ pub const Completion = struct { + fn perform(self: *Completion, ev_: ?*const Kevent) Result { + return switch (self.op) { + .cancel, +- .close, + .noop, + .timer, + .shutdown, +@@ -1232,6 +1231,11 @@ pub const Completion = struct { + + break :res .{ .proc = 0 }; + }, ++ ++ .close => |*op| res: { ++ posix.close(op.fd); ++ break :res .{ .close = {} }; ++ }, + }; + } + +diff --git a/src/watcher/stream.zig b/src/watcher/stream.zig +index 7f5df6f..bc95282 100644 +--- a/src/watcher/stream.zig ++++ b/src/watcher/stream.zig +@@ -80,6 +80,22 @@ pub fn Closeable(comptime xev: type, comptime T: type, comptime options: Options + }).callback, + }; + ++ // If we're dup-ing, then we ask the backend to manage the fd. ++ switch (xev.backend) { ++ .io_uring, ++ .wasi_poll, ++ .iocp, ++ => {}, ++ ++ .epoll => { ++ c.flags.threadpool = true; ++ }, ++ ++ .kqueue => { ++ c.flags.threadpool = true; ++ }, ++ } ++ + loop.add(c); + } + }; +-- +2.39.5 (Apple Git-154) + +From 378a3968b8820aef5a6ab64d1893d022b1b6d70c Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Fri, 17 Jan 2025 20:59:18 +0000 +Subject: [PATCH 5/5] epoll: don't count immediate actions + +If an immediate action is dispatched, the loop might block +on epoll_wait even though only one action was requested. +--- + src/backend/epoll.zig | 15 ++++++++++++++- + 1 file changed, 14 insertions(+), 1 deletion(-) + +diff --git a/src/backend/epoll.zig b/src/backend/epoll.zig +index 0f4a2ac..fb7e59e 100644 +--- a/src/backend/epoll.zig ++++ b/src/backend/epoll.zig +@@ -297,6 +297,7 @@ pub const Loop = struct { + + // Submit all the submissions. We copy the submission queue so that + // any resubmits don't cause an infinite loop. ++ var wait_rem: usize = @intCast(wait); + var queued = self.submissions; + self.submissions = .{}; + while (queued.pop()) |c| { +@@ -304,6 +305,19 @@ pub const Loop = struct { + // This usually means that we switched them to be deleted or + // something. + if (c.flags.state != .adding) continue; ++ ++ // These operations happen synchronously. Ensure they are ++ // decremented from wait_rem. ++ switch (c.op) { ++ .cancel, ++ // should noop be counted? ++ // .noop, ++ .shutdown, ++ .timer, ++ => wait_rem -|= 1, ++ else => {}, ++ } ++ + self.start(c); + } + +@@ -322,7 +336,6 @@ pub const Loop = struct { + + // Wait and process events. We only do this if we have any active. + var events: [1024]linux.epoll_event = undefined; +- var wait_rem: usize = @intCast(wait); + while (self.active > 0 and (wait == 0 or wait_rem > 0)) { + self.update_now(); + const now_timer: Operation.Timer = .{ .next = self.cached_now }; +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/modules/libxev/20250124.0-31eed4e/patches/141.patch b/third_party/modules/libxev/20250124.0-31eed4e/patches/141.patch new file mode 100644 index 00000000..70cee31c --- /dev/null +++ b/third_party/modules/libxev/20250124.0-31eed4e/patches/141.patch @@ -0,0 +1,35 @@ +From 283a911561385e0cc9278dfd350bdc5c5132c9c4 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Wed, 5 Feb 2025 11:17:01 +0100 +Subject: [PATCH] backend/kqueue: fix empty udata crash + +On certain occasions, internal kqueue events are dispatched with an empty udata field. +This causes a crash when trying to access the udata field in the event handler. + +This patch adds a check to ensure that the udata field is not empty before accessing it +just like in the other events consumer. +--- + src/backend/kqueue.zig | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/src/backend/kqueue.zig b/src/backend/kqueue.zig +index 456f64f..7d919ce 100644 +--- a/src/backend/kqueue.zig ++++ b/src/backend/kqueue.zig +@@ -206,11 +206,15 @@ pub const Loop = struct { + // event list to zero length) because it was leading to + // memory corruption we need to investigate. + for (events[0..completed]) |ev| { +- const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.udata))); ++ // Zero udata values are internal events that we do nothing ++ // on such as the mach port wakeup. ++ if (ev.udata == 0) continue; + + // We handle deletions separately. + if (ev.flags & posix.system.EV_DELETE != 0) continue; + ++ const c: *Completion = @ptrFromInt(@as(usize, @intCast(ev.udata))); ++ + // If EV_ERROR is set, then submission failed for this + // completion. We get the syscall errorcode from data and + // store it. diff --git a/third_party/modules/libxev/20250124.0-31eed4e/source.json b/third_party/modules/libxev/20250124.0-31eed4e/source.json new file mode 100644 index 00000000..77af3d8f --- /dev/null +++ b/third_party/modules/libxev/20250124.0-31eed4e/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "libxev-31eed4e337fed7b0149319e5cdbb62b848c24fbd", + "url": "https://github.com/zml/libxev/archive/31eed4e337fed7b0149319e5cdbb62b848c24fbd.tar.gz", + "integrity": "sha256-VHP90NTytIZ8UZsYRKOOxN490/I6yv6ec40sP8y5MJ8=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "", + "main2.zig": "" + }, + "patches": { + "128.patch": "", + "141.patch": "" + }, + "patch_strip": 1 +} diff --git a/third_party/modules/libxev/20250222.0-07bcffa/MODULE.bazel b/third_party/modules/libxev/20250222.0-07bcffa/MODULE.bazel new file mode 100644 index 00000000..424792ed --- /dev/null +++ b/third_party/modules/libxev/20250222.0-07bcffa/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20250222.0-07bcffa", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20250222.0-07bcffa/overlay/BUILD.bazel b/third_party/modules/libxev/20250222.0-07bcffa/overlay/BUILD.bazel new file mode 100644 index 00000000..872ef7fb --- /dev/null +++ b/third_party/modules/libxev/20250222.0-07bcffa/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "src/main.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20250222.0-07bcffa/overlay/MODULE.bazel b/third_party/modules/libxev/20250222.0-07bcffa/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/libxev/20250222.0-07bcffa/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250222.0-07bcffa/source.json b/third_party/modules/libxev/20250222.0-07bcffa/source.json new file mode 100644 index 00000000..1483ad30 --- /dev/null +++ b/third_party/modules/libxev/20250222.0-07bcffa/source.json @@ -0,0 +1,9 @@ +{ + "strip_prefix": "libxev-07bcffa0f63054152a9883baa42bce5faad297e6", + "url": "https://github.com/mitchellh/libxev/archive/07bcffa0f63054152a9883baa42bce5faad297e6.tar.gz", + "integrity": "sha256-H80IoKcTeJFwpL5i4S9zyMuyWaQCK173ppoAO/CYXjc=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + } +} diff --git a/third_party/modules/libxev/20250313.0-5773f46/MODULE.bazel b/third_party/modules/libxev/20250313.0-5773f46/MODULE.bazel new file mode 100644 index 00000000..bfe80aeb --- /dev/null +++ b/third_party/modules/libxev/20250313.0-5773f46/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20250313.0-5773f46", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/libxev/20250313.0-5773f46/overlay/BUILD.bazel b/third_party/modules/libxev/20250313.0-5773f46/overlay/BUILD.bazel new file mode 100644 index 00000000..872ef7fb --- /dev/null +++ b/third_party/modules/libxev/20250313.0-5773f46/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "src/main.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20250313.0-5773f46/overlay/MODULE.bazel b/third_party/modules/libxev/20250313.0-5773f46/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/libxev/20250313.0-5773f46/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250313.0-5773f46/source.json b/third_party/modules/libxev/20250313.0-5773f46/source.json new file mode 100644 index 00000000..65786582 --- /dev/null +++ b/third_party/modules/libxev/20250313.0-5773f46/source.json @@ -0,0 +1,10 @@ +{ + "strip_prefix": "libxev-5773f46de3937e848db3ff87577add8ef7e8222d", + "url": "https://github.com/mitchellh/libxev/archive/5773f46de3937e848db3ff87577add8ef7e8222d.tar.gz", + "integrity": "sha256-k9cyG0DPhEKwM+/dKiLFT9WeM+c2kCokN8VftOriSxI=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + }, + "patch_strip": 1 +} diff --git a/third_party/modules/libxev/20250718.0-9f785d2/MODULE.bazel b/third_party/modules/libxev/20250718.0-9f785d2/MODULE.bazel new file mode 120000 index 00000000..c8a52909 --- /dev/null +++ b/third_party/modules/libxev/20250718.0-9f785d2/MODULE.bazel @@ -0,0 +1 @@ +./overlay/MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20250718.0-9f785d2/overlay/BUILD.bazel b/third_party/modules/libxev/20250718.0-9f785d2/overlay/BUILD.bazel new file mode 100644 index 00000000..872ef7fb --- /dev/null +++ b/third_party/modules/libxev/20250718.0-9f785d2/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "src/main.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20250718.0-9f785d2/overlay/MODULE.bazel b/third_party/modules/libxev/20250718.0-9f785d2/overlay/MODULE.bazel new file mode 100644 index 00000000..82907dd7 --- /dev/null +++ b/third_party/modules/libxev/20250718.0-9f785d2/overlay/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20250718.0-9f785d2", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20250613.0-567662a") diff --git a/third_party/modules/libxev/20250718.0-9f785d2/source.json b/third_party/modules/libxev/20250718.0-9f785d2/source.json new file mode 100644 index 00000000..daf74448 --- /dev/null +++ b/third_party/modules/libxev/20250718.0-9f785d2/source.json @@ -0,0 +1,11 @@ +{ + "strip_prefix": "libxev-9f785d202ddccf7a625e799250579253977978b6", + "url": "https://github.com/mitchellh/libxev/archive/9f785d202ddccf7a625e799250579253977978b6.tar.gz", + "integrity": "sha256-HaExxsAsATjH/pCP9NMo/EjVrG2KVX3SiTtZzmS8R90=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + }, + "patch_strip": 1 +} + diff --git a/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel b/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel new file mode 120000 index 00000000..c8a52909 --- /dev/null +++ b/third_party/modules/libxev/20251010.0-9f785d2/MODULE.bazel @@ -0,0 +1 @@ +./overlay/MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel b/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel new file mode 100644 index 00000000..872ef7fb --- /dev/null +++ b/third_party/modules/libxev/20251010.0-9f785d2/overlay/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "xev", + srcs = glob([ + "src/*.zig", + "src/backend/*.zig", + "src/linux/*.zig", + "src/watcher/*.zig", + ]), + main = "src/main.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel b/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel new file mode 100644 index 00000000..66f5a554 --- /dev/null +++ b/third_party/modules/libxev/20251010.0-9f785d2/overlay/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "libxev", + version = "20251010.0-9f785d2", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "0.12.1") diff --git a/third_party/modules/libxev/20251010.0-9f785d2/source.json b/third_party/modules/libxev/20251010.0-9f785d2/source.json new file mode 100644 index 00000000..daf74448 --- /dev/null +++ b/third_party/modules/libxev/20251010.0-9f785d2/source.json @@ -0,0 +1,11 @@ +{ + "strip_prefix": "libxev-9f785d202ddccf7a625e799250579253977978b6", + "url": "https://github.com/mitchellh/libxev/archive/9f785d202ddccf7a625e799250579253977978b6.tar.gz", + "integrity": "sha256-HaExxsAsATjH/pCP9NMo/EjVrG2KVX3SiTtZzmS8R90=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + }, + "patch_strip": 1 +} + diff --git a/third_party/modules/libxev/metadata.json b/third_party/modules/libxev/metadata.json new file mode 100644 index 00000000..1fd34d18 --- /dev/null +++ b/third_party/modules/libxev/metadata.json @@ -0,0 +1,27 @@ +{ + "homepage": "https://github.com/mitchellh/libxev", + "maintainers": [ + { + "email": "bzlmod@zml.ai", + "github": "zml", + "name": "ZML Engineering Team" + } + ], + "repository": [ + "github:mitchellh/libxev" + ], + "versions": [ + "20240825.0-dbe2291", + "20240910.0-a2d9b31", + "20241119.0-6afcde9", + "20241208.0-db6a52b", + "20241208.1-db6a52b", + "20241208.2-db6a52b", + "20250124.0-31eed4e", + "20250222.0-07bcffa", + "20250313.0-5773f46", + "20250716.0-9f785d2", + "20251010.0-9f785d2" + ], + "yanked_versions": {} +} diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel b/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel new file mode 100644 index 00000000..6bf421a8 --- /dev/null +++ b/third_party/modules/llvm-raw/20240823.0-f142f8a/MODULE.bazel @@ -0,0 +1,10 @@ +module( + name = "llvm-raw", + version = "20240823.0-f142f8a", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") +bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") diff --git a/third_party/cloud_accelerator_diagnostics/BUILD.bazel b/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/BUILD.bazel similarity index 100% rename from third_party/cloud_accelerator_diagnostics/BUILD.bazel rename to third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/BUILD.bazel diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl new file mode 100644 index 00000000..f247ed4b --- /dev/null +++ b/third_party/modules/llvm-raw/20240823.0-f142f8a/overlay/utils/bazel/extension.bzl @@ -0,0 +1,28 @@ +load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list(mandatory = True), + }, + ), + }, +) diff --git a/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json b/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json new file mode 100644 index 00000000..6edc73dc --- /dev/null +++ b/third_party/modules/llvm-raw/20240823.0-f142f8a/source.json @@ -0,0 +1,10 @@ +{ + "strip_prefix": "llvm-project-f142f8afe21bceb00fb495468aa0b5043e98c419", + "url": "https://github.com/llvm/llvm-project/archive/f142f8afe21bceb00fb495468aa0b5043e98c419.tar.gz", + "integrity": "sha256-8ftYOnMGq99igt2I1n9SOZpYLq79+F7HiKL/7lOwcHs=", + "overlay": { + "BUILD.bazel": "", + "MODULE.bazel": "", + "utils/bazel/extension.bzl": "" + } +} diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel b/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel new file mode 100644 index 00000000..c9130b80 --- /dev/null +++ b/third_party/modules/llvm-raw/20240919.0-94c024a/MODULE.bazel @@ -0,0 +1,10 @@ +module( + name = "llvm-raw", + version = "20240919.0-94c024a", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") +bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") diff --git a/third_party/cuda_tile/BUILD.bazel b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/BUILD.bazel similarity index 100% rename from third_party/cuda_tile/BUILD.bazel rename to third_party/modules/llvm-raw/20240919.0-94c024a/overlay/BUILD.bazel diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl new file mode 100644 index 00000000..f247ed4b --- /dev/null +++ b/third_party/modules/llvm-raw/20240919.0-94c024a/overlay/utils/bazel/extension.bzl @@ -0,0 +1,28 @@ +load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list(mandatory = True), + }, + ), + }, +) diff --git a/third_party/modules/llvm-raw/20240919.0-94c024a/source.json b/third_party/modules/llvm-raw/20240919.0-94c024a/source.json new file mode 100644 index 00000000..2aec201a --- /dev/null +++ b/third_party/modules/llvm-raw/20240919.0-94c024a/source.json @@ -0,0 +1,10 @@ +{ + "strip_prefix": "llvm-project-94c024adedcb53059c29d7c2d62982053b60e86a", + "url": "https://github.com/llvm/llvm-project/archive/94c024adedcb53059c29d7c2d62982053b60e86a.tar.gz", + "integrity": "sha256-IEzt6quG8GXvZMs4id0ukt3UqPXVtrwctLJ2aU+2p5g=", + "overlay": { + "BUILD.bazel": "", + "MODULE.bazel": "", + "utils/bazel/extension.bzl": "" + } +} diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/MODULE.bazel b/third_party/modules/llvm-raw/20241022.0-6c4267f/MODULE.bazel new file mode 100644 index 00000000..b3ed250a --- /dev/null +++ b/third_party/modules/llvm-raw/20241022.0-6c4267f/MODULE.bazel @@ -0,0 +1,11 @@ +module( + name = "llvm-raw", + version = "20241022.0-6c4267f", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") +bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") +bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/cute_ir/BUILD.bazel b/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/BUILD.bazel similarity index 100% rename from third_party/cute_ir/BUILD.bazel rename to third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/BUILD.bazel diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/utils/bazel/extension.bzl new file mode 100644 index 00000000..f247ed4b --- /dev/null +++ b/third_party/modules/llvm-raw/20241022.0-6c4267f/overlay/utils/bazel/extension.bzl @@ -0,0 +1,28 @@ +load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list(mandatory = True), + }, + ), + }, +) diff --git a/third_party/modules/llvm-raw/20241022.0-6c4267f/source.json b/third_party/modules/llvm-raw/20241022.0-6c4267f/source.json new file mode 100644 index 00000000..f2286dc8 --- /dev/null +++ b/third_party/modules/llvm-raw/20241022.0-6c4267f/source.json @@ -0,0 +1,10 @@ +{ + "strip_prefix": "llvm-project-6c4267fb1779bc5550bb413f33250f9365acfbc6", + "url": "https://github.com/llvm/llvm-project/archive/6c4267fb1779bc5550bb413f33250f9365acfbc6.tar.gz", + "integrity": "sha256-cBDuj+hiRvq8rtvtIfqawr0lQuDSrWFypEgeApT981Q=", + "overlay": { + "BUILD.bazel": "", + "MODULE.bazel": "", + "utils/bazel/extension.bzl": "" + } +} diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/MODULE.bazel b/third_party/modules/llvm-raw/20250102.0-f739aa4/MODULE.bazel new file mode 100644 index 00000000..102b0a38 --- /dev/null +++ b/third_party/modules/llvm-raw/20250102.0-f739aa4/MODULE.bazel @@ -0,0 +1,11 @@ +module( + name = "llvm-raw", + version = "20250102.0-f739aa4", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") +bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") +bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/flydsl/BUILD.bazel b/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/BUILD.bazel similarity index 100% rename from third_party/flydsl/BUILD.bazel rename to third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/BUILD.bazel diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/utils/bazel/extension.bzl new file mode 100644 index 00000000..f247ed4b --- /dev/null +++ b/third_party/modules/llvm-raw/20250102.0-f739aa4/overlay/utils/bazel/extension.bzl @@ -0,0 +1,28 @@ +load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list(mandatory = True), + }, + ), + }, +) diff --git a/third_party/modules/llvm-raw/20250102.0-f739aa4/source.json b/third_party/modules/llvm-raw/20250102.0-f739aa4/source.json new file mode 100644 index 00000000..146f68ab --- /dev/null +++ b/third_party/modules/llvm-raw/20250102.0-f739aa4/source.json @@ -0,0 +1,10 @@ +{ + "strip_prefix": "llvm-project-f739aa4004165dc64d3a1f418d5ad3c84886f01a", + "url": "https://github.com/llvm/llvm-project/archive/f739aa4004165dc64d3a1f418d5ad3c84886f01a.tar.gz", + "integrity": "sha256-hdoTTnugRO9Q68AJ0aV6h/sOLbIIoEZQ7y/lZOlWSqc=", + "overlay": { + "BUILD.bazel": "", + "MODULE.bazel": "", + "utils/bazel/extension.bzl": "" + } +} diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/MODULE.bazel b/third_party/modules/llvm-raw/20250117.0-bf17016/MODULE.bazel new file mode 100644 index 00000000..b1171a0a --- /dev/null +++ b/third_party/modules/llvm-raw/20250117.0-bf17016/MODULE.bazel @@ -0,0 +1,11 @@ +module( + name = "llvm-raw", + version = "20250117.0-bf17016", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") +bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") +bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/iree/BUILD.bazel b/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/BUILD.bazel similarity index 100% rename from third_party/iree/BUILD.bazel rename to third_party/modules/llvm-raw/20250117.0-bf17016/overlay/BUILD.bazel diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/utils/bazel/extension.bzl new file mode 100644 index 00000000..f247ed4b --- /dev/null +++ b/third_party/modules/llvm-raw/20250117.0-bf17016/overlay/utils/bazel/extension.bzl @@ -0,0 +1,28 @@ +load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list(mandatory = True), + }, + ), + }, +) diff --git a/third_party/modules/llvm-raw/20250117.0-bf17016/source.json b/third_party/modules/llvm-raw/20250117.0-bf17016/source.json new file mode 100644 index 00000000..851e85aa --- /dev/null +++ b/third_party/modules/llvm-raw/20250117.0-bf17016/source.json @@ -0,0 +1,10 @@ +{ + "strip_prefix": "llvm-project-bf17016a92bc8a23d2cdd2b51355dd4eb5019c68", + "url": "https://github.com/llvm/llvm-project/archive/bf17016a92bc8a23d2cdd2b51355dd4eb5019c68.tar.gz", + "integrity": "sha256-ugnxLlAZ9aylMbFzMnXwoQsYHW+JTesaRhDgF/drFyo=", + "overlay": { + "BUILD.bazel": "", + "MODULE.bazel": "", + "utils/bazel/extension.bzl": "" + } +} diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/MODULE.bazel b/third_party/modules/llvm-raw/20250201.0-386af4a/MODULE.bazel new file mode 100644 index 00000000..c213afe0 --- /dev/null +++ b/third_party/modules/llvm-raw/20250201.0-386af4a/MODULE.bazel @@ -0,0 +1,11 @@ +module( + name = "llvm-raw", + version = "20250201.0-386af4a", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.11") +bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") +bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") +bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/zio/BUILD.bazel b/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/BUILD.bazel similarity index 100% rename from third_party/zio/BUILD.bazel rename to third_party/modules/llvm-raw/20250201.0-386af4a/overlay/BUILD.bazel diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/utils/bazel/extension.bzl new file mode 100644 index 00000000..f247ed4b --- /dev/null +++ b/third_party/modules/llvm-raw/20250201.0-386af4a/overlay/utils/bazel/extension.bzl @@ -0,0 +1,28 @@ +load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list(mandatory = True), + }, + ), + }, +) diff --git a/third_party/modules/llvm-raw/20250201.0-386af4a/source.json b/third_party/modules/llvm-raw/20250201.0-386af4a/source.json new file mode 100644 index 00000000..e3a7a1ac --- /dev/null +++ b/third_party/modules/llvm-raw/20250201.0-386af4a/source.json @@ -0,0 +1,10 @@ +{ + "strip_prefix": "llvm-project-386af4a5c64ab75eaee2448dc38f2e34a40bfed0", + "url": "https://github.com/llvm/llvm-project/archive/386af4a5c64ab75eaee2448dc38f2e34a40bfed0.tar.gz", + "integrity": "sha256-S3UqLfteTlXPmBKjU7zn5DfghS9O72SSn0jXRzIIDFA=", + "overlay": { + "BUILD.bazel": "", + "MODULE.bazel": "", + "utils/bazel/extension.bzl": "" + } +} diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/MODULE.bazel b/third_party/modules/llvm-raw/20250217.0-0e779ad/MODULE.bazel new file mode 100644 index 00000000..245f034b --- /dev/null +++ b/third_party/modules/llvm-raw/20250217.0-0e779ad/MODULE.bazel @@ -0,0 +1,11 @@ +module( + name = "llvm-raw", + version = "20250217.0-0e779ad", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.11") +bazel_dep(name = "zstd", version = "1.5.6", repo_name = "llvm_zstd") +bazel_dep(name = "zlib", version = "1.3.1.bcr.3", repo_name = "llvm_zlib") +bazel_dep(name = "rules_python", version = "0.29.0") diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/BUILD.bazel b/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/BUILD.bazel new file mode 100644 index 00000000..e69de29b diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/MODULE.bazel b/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/utils/bazel/extension.bzl b/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/utils/bazel/extension.bzl new file mode 100644 index 00000000..f247ed4b --- /dev/null +++ b/third_party/modules/llvm-raw/20250217.0-0e779ad/overlay/utils/bazel/extension.bzl @@ -0,0 +1,28 @@ +load("//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list(mandatory = True), + }, + ), + }, +) diff --git a/third_party/modules/llvm-raw/20250217.0-0e779ad/source.json b/third_party/modules/llvm-raw/20250217.0-0e779ad/source.json new file mode 100644 index 00000000..6add20e8 --- /dev/null +++ b/third_party/modules/llvm-raw/20250217.0-0e779ad/source.json @@ -0,0 +1,10 @@ +{ + "strip_prefix": "llvm-project-0e779ad4998ef65907502101c5b82ede05ddfa4e", + "url": "https://github.com/llvm/llvm-project/archive/0e779ad4998ef65907502101c5b82ede05ddfa4e.tar.gz", + "integrity": "sha256-1cJWCy2c487XlRET8rXR6kKGZWePTcsfuHgOsSGcphU=", + "overlay": { + "BUILD.bazel": "", + "MODULE.bazel": "", + "utils/bazel/extension.bzl": "" + } +} diff --git a/third_party/modules/llvm-raw/metadata.json b/third_party/modules/llvm-raw/metadata.json new file mode 100644 index 00000000..32713e58 --- /dev/null +++ b/third_party/modules/llvm-raw/metadata.json @@ -0,0 +1,22 @@ +{ + "homepage": "https://github.com/llvm/llvm-project", + "maintainers": [ + { + "email": "bzlmod@zml.ai", + "github": "zml", + "name": "ZML Engineering Team" + } + ], + "repository": [ + "github:llvm/llvm-project" + ], + "versions": [ + "20240823.0-f142f8a", + "20240919.0-94c024a", + "20241022.0-6c4267f", + "20250102.0-f739aa4", + "20250117.0-bf17016", + "20250201.0-386af4a", + ], + "yanked_versions": {} +} diff --git a/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel b/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel new file mode 100644 index 00000000..723c6a98 --- /dev/null +++ b/third_party/modules/rules_zig/20240904.0-010da15/MODULE.bazel @@ -0,0 +1,68 @@ +module( + name = "rules_zig", + version = "20240904.0-010da15", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9", dev_dependency = True) +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240904.0-010da15/source.json b/third_party/modules/rules_zig/20240904.0-010da15/source.json new file mode 100644 index 00000000..253af196 --- /dev/null +++ b/third_party/modules/rules_zig/20240904.0-010da15/source.json @@ -0,0 +1,5 @@ +{ + "type": "git_repository", + "remote": "https://github.com/zml/rules_zig.git", + "commit": "010da15abb4335479778d6b4fb2ca752a0ab80e3" +} diff --git a/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel b/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel new file mode 100644 index 00000000..4ded2bb6 --- /dev/null +++ b/third_party/modules/rules_zig/20240909.0-37f17ff/MODULE.bazel @@ -0,0 +1,68 @@ +module( + name = "rules_zig", + version = "20240909.0-37f17ff", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9", dev_dependency = True) +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240909.0-37f17ff/source.json b/third_party/modules/rules_zig/20240909.0-37f17ff/source.json new file mode 100644 index 00000000..19d05aef --- /dev/null +++ b/third_party/modules/rules_zig/20240909.0-37f17ff/source.json @@ -0,0 +1,5 @@ +{ + "type": "git_repository", + "remote": "https://github.com/zml/rules_zig.git", + "commit": "37f17ffd2f6d04fc4bf9b583f9a1b117856a88d1" +} diff --git a/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel b/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel new file mode 100644 index 00000000..e288be56 --- /dev/null +++ b/third_party/modules/rules_zig/20240912.0-41bfe84/MODULE.bazel @@ -0,0 +1,68 @@ +module( + name = "rules_zig", + version = "20240912.0-41bfe84", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240912.0-41bfe84/source.json b/third_party/modules/rules_zig/20240912.0-41bfe84/source.json new file mode 100644 index 00000000..3d576fb1 --- /dev/null +++ b/third_party/modules/rules_zig/20240912.0-41bfe84/source.json @@ -0,0 +1,5 @@ +{ + "type": "git_repository", + "remote": "https://github.com/zml/rules_zig.git", + "commit": "41bfe84e4d9a43cbe55281dddc80b683cc6fc6eb" +} diff --git a/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel b/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel new file mode 100644 index 00000000..f2ff768e --- /dev/null +++ b/third_party/modules/rules_zig/20240913.0-1957d05/MODULE.bazel @@ -0,0 +1,68 @@ +module( + name = "rules_zig", + version = "20240913.0-1957d05", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20240913.0-1957d05/source.json b/third_party/modules/rules_zig/20240913.0-1957d05/source.json new file mode 100644 index 00000000..9da3ae38 --- /dev/null +++ b/third_party/modules/rules_zig/20240913.0-1957d05/source.json @@ -0,0 +1,5 @@ +{ + "type": "git_repository", + "remote": "https://github.com/zml/rules_zig.git", + "commit": "1957d0572193fb859e721a0fab8bd8f0fb57f3ff" +} diff --git a/third_party/modules/rules_zig/20250314.0-b9739c6/MODULE.bazel b/third_party/modules/rules_zig/20250314.0-b9739c6/MODULE.bazel new file mode 100644 index 00000000..392e8aff --- /dev/null +++ b/third_party/modules/rules_zig/20250314.0-b9739c6/MODULE.bazel @@ -0,0 +1,68 @@ +module( + name = "rules_zig", + version = "20250314.0-b9739c6", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250314.0-b9739c6/source.json b/third_party/modules/rules_zig/20250314.0-b9739c6/source.json new file mode 100644 index 00000000..ceea72c3 --- /dev/null +++ b/third_party/modules/rules_zig/20250314.0-b9739c6/source.json @@ -0,0 +1,5 @@ +{ + "strip_prefix": "rules_zig-b9739c615ce62b64ee595ac4bcd9ee7cc06b0422", + "url": "https://github.com/zml/rules_zig/archive/b9739c615ce62b64ee595ac4bcd9ee7cc06b0422.tar.gz", + "integrity": "sha256-p1rYD9gvYS3DgBjN6jo1pB86Kr4k2wojKjVpYRTZ3Us=" +} diff --git a/third_party/modules/rules_zig/20250519.0-233b207/MODULE.bazel b/third_party/modules/rules_zig/20250519.0-233b207/MODULE.bazel new file mode 100644 index 00000000..b5dacc58 --- /dev/null +++ b/third_party/modules/rules_zig/20250519.0-233b207/MODULE.bazel @@ -0,0 +1,68 @@ +module( + name = "rules_zig", + version = "20250519.0-233b207", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250519.0-233b207/source.json b/third_party/modules/rules_zig/20250519.0-233b207/source.json new file mode 100644 index 00000000..6082ccd8 --- /dev/null +++ b/third_party/modules/rules_zig/20250519.0-233b207/source.json @@ -0,0 +1,5 @@ +{ + "strip_prefix": "rules_zig-233b207499826715a3bf1a6eeea6e3052ed49542", + "url": "https://github.com/zml/rules_zig/archive/233b207499826715a3bf1a6eeea6e3052ed49542.tar.gz", + "integrity": "sha256-4gloFoSMPkWCV0Jxikqs1UkfOgQL+NmoXMMnVhtgVvg=" +} diff --git a/third_party/modules/rules_zig/20250530.0-5084f1f/MODULE.bazel b/third_party/modules/rules_zig/20250530.0-5084f1f/MODULE.bazel new file mode 100644 index 00000000..06b1e94e --- /dev/null +++ b/third_party/modules/rules_zig/20250530.0-5084f1f/MODULE.bazel @@ -0,0 +1,68 @@ +module( + name = "rules_zig", + version = "20250530.0-5084f1f", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250530.0-5084f1f/source.json b/third_party/modules/rules_zig/20250530.0-5084f1f/source.json new file mode 100644 index 00000000..081142c0 --- /dev/null +++ b/third_party/modules/rules_zig/20250530.0-5084f1f/source.json @@ -0,0 +1,5 @@ +{ + "strip_prefix": "rules_zig-5084f1f3a151dfc697900274c19813a4260217c6", + "url": "https://github.com/zml/rules_zig/archive/5084f1f3a151dfc697900274c19813a4260217c6.tar.gz", + "integrity": "sha256-KpZkIDnKG0sXtRhsE0SDoj3QeeHRKQCRjm6LzxmueRA=" +} diff --git a/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel b/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel new file mode 100644 index 00000000..4189634c --- /dev/null +++ b/third_party/modules/rules_zig/20250613.0-567662a/MODULE.bazel @@ -0,0 +1,68 @@ +module( + name = "rules_zig", + version = "20250613.0-567662a", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250613.0-567662a/source.json b/third_party/modules/rules_zig/20250613.0-567662a/source.json new file mode 100644 index 00000000..4dbdafee --- /dev/null +++ b/third_party/modules/rules_zig/20250613.0-567662a/source.json @@ -0,0 +1,5 @@ +{ + "strip_prefix": "rules_zig-567662a3ce5e87894950d56c69d8de5ad6e0b5f0", + "url": "https://github.com/zml/rules_zig/archive/567662a3ce5e87894950d56c69d8de5ad6e0b5f0.tar.gz", + "integrity": "sha256-PjFpDJXO0BGx2CAZ54Ppv6d4TwTDOWS+mVangKQFvZc=" +} diff --git a/third_party/modules/rules_zig/20250714.0-b14a4f1/MODULE.bazel b/third_party/modules/rules_zig/20250714.0-b14a4f1/MODULE.bazel new file mode 100644 index 00000000..69a0dc9b --- /dev/null +++ b/third_party/modules/rules_zig/20250714.0-b14a4f1/MODULE.bazel @@ -0,0 +1,68 @@ +module( + name = "rules_zig", + version = "20250714.0-b14a4f1", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) diff --git a/third_party/modules/rules_zig/20250714.0-b14a4f1/source.json b/third_party/modules/rules_zig/20250714.0-b14a4f1/source.json new file mode 100644 index 00000000..5eb58847 --- /dev/null +++ b/third_party/modules/rules_zig/20250714.0-b14a4f1/source.json @@ -0,0 +1,5 @@ +{ + "strip_prefix": "rules_zig-b14a4f1bba1a766311f75513e922cc8c2179e1e3", + "url": "https://github.com/zml/rules_zig/archive/b14a4f1bba1a766311f75513e922cc8c2179e1e3.tar.gz", + "integrity": "sha256-XEdYlvjs54SVoLGM4x9J+uofCZk2UIO1nbsqwV+Blqs=" +} diff --git a/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel b/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel new file mode 100644 index 00000000..3c64eb32 --- /dev/null +++ b/third_party/modules/rules_zig/20250821.0-be53625/MODULE.bazel @@ -0,0 +1,75 @@ +module( + name = "rules_zig", + version = "20250821.0-be53625", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") + +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) + +# Hack to get around a cc_common.link limitation. +# See https://github.com/bazelbuild/bazel/pull/23838 +cc_common_link = use_repo_rule("//zig:extensions.bzl", "cc_common_link") + +cc_common_link(name = "build_bazel_rules_android") diff --git a/third_party/modules/rules_zig/20250821.0-be53625/source.json b/third_party/modules/rules_zig/20250821.0-be53625/source.json new file mode 100644 index 00000000..959b2dbc --- /dev/null +++ b/third_party/modules/rules_zig/20250821.0-be53625/source.json @@ -0,0 +1,5 @@ +{ + "strip_prefix": "rules_zig-be53625afb13e73856ee4cab38d5aad9f86f63ef", + "url": "https://github.com/zml/rules_zig/archive/be53625afb13e73856ee4cab38d5aad9f86f63ef.tar.gz", + "integrity": "sha256-IsqEl3BHW/vttyHpVBx5WzO3an/K3eZZx8RJk7nVx8s=" +} diff --git a/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel b/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel new file mode 100644 index 00000000..7620c53c --- /dev/null +++ b/third_party/modules/rules_zig/20250827.0-35b6d57/MODULE.bazel @@ -0,0 +1,75 @@ +module( + name = "rules_zig", + version = "20250827.0-35b6d57", + compatibility_level = 1, +) + +bazel_dep(name = "aspect_bazel_lib", version = "2.8.1") +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "platforms", version = "0.0.10") + +zig = use_extension("//zig:extensions.bzl", "zig") +zig.index(file = "//zig/private:versions.json") +use_repo(zig, "zig_toolchains") + +register_toolchains("@rules_zig//zig/target:all") + +register_toolchains("@zig_toolchains//:all") + +zig_dev = use_extension( + "//zig:extensions.bzl", + "zig", + dev_dependency = True, +) +zig_dev.toolchain(zig_version = "0.13.0") +zig_dev.toolchain(zig_version = "0.12.1") +zig_dev.toolchain(zig_version = "0.12.0") +zig_dev.toolchain(zig_version = "0.11.0") + +bazel_dep(name = "rules_cc", version = "0.0.9") + +bazel_dep(name = "stardoc", version = "0.7.0", dev_dependency = True, repo_name = "io_bazel_stardoc") +bazel_dep(name = "gazelle", version = "0.38.0", dev_dependency = True, repo_name = "bazel_gazelle") +bazel_dep(name = "bazel_skylib_gazelle_plugin", version = "1.7.1", dev_dependency = True) +bazel_dep( + name = "buildifier_prebuilt", + version = "7.3.1", + dev_dependency = True, +) +bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True) +bazel_dep(name = "rules_python", version = "0.35.0", dev_dependency = True) +bazel_dep( + name = "rules_bazel_integration_test", + version = "0.25.0", + dev_dependency = True, +) + +bazel_binaries = use_extension( + "@rules_bazel_integration_test//:extensions.bzl", + "bazel_binaries", + dev_dependency = True, +) + +# NOTE: Keep in sync with WORKSPACE. +bazel_binaries.download(version_file = "//:.bazelversion") +bazel_binaries.download(version = "7.0.0") +use_repo( + bazel_binaries, + "bazel_binaries", + "bazel_binaries_bazelisk", + "build_bazel_bazel_.bazelversion", + "build_bazel_bazel_7_0_0", +) + +# TODO[AH] Should be an implicit transitive dependency through rules_bazel_integration_test. +# However, if we do not include it explicitly, then the runfiles resolution for +# cgrindel_bazel_starlib/shlib/lib/message.sh fails in +# rules_bazel_integration_test/tools/update_deleted_packages.sh when invoked +# through the rules_multirun target //util:update. +bazel_dep(name = "cgrindel_bazel_starlib", version = "0.21.0", dev_dependency = True) + +# Hack to get around a cc_common.link limitation. +# See https://github.com/bazelbuild/bazel/pull/23838 +cc_common_link = use_repo_rule("//zig:extensions.bzl", "cc_common_link") + +cc_common_link(name = "build_bazel_rules_android") diff --git a/third_party/modules/rules_zig/20250827.0-35b6d57/source.json b/third_party/modules/rules_zig/20250827.0-35b6d57/source.json new file mode 100644 index 00000000..e42d62ba --- /dev/null +++ b/third_party/modules/rules_zig/20250827.0-35b6d57/source.json @@ -0,0 +1,5 @@ +{ + "strip_prefix": "rules_zig-35b6d57e94f3eb08e978c1133314606f4f38e216", + "url": "https://github.com/zml/rules_zig/archive/35b6d57e94f3eb08e978c1133314606f4f38e216.tar.gz", + "integrity": "sha256-FDnAqynTD2LB3W/IaBgocmnLz8CA9nyHZYYntC4plUU=" +} diff --git a/third_party/modules/rules_zig/metadata.json b/third_party/modules/rules_zig/metadata.json new file mode 100644 index 00000000..e7937f1e --- /dev/null +++ b/third_party/modules/rules_zig/metadata.json @@ -0,0 +1,24 @@ +{ + "homepage": "https://github.com/zml/rules_zig", + "maintainers": [ + { + "email": "bzlmod@zml.ai", + "github": "zml", + "name": "ZML Engineering Team" + } + ], + "repository": ["github:zml/rules_zig"], + "versions": [ + "20240904.0-010da15", + "20240909.0-37f17ff", + "20240912.0-41bfe84", + "20240913.0-1957d05", + "20250314.0-b9739c6", + "20250519.0-233b207", + "20250613.0-567662a", + "20250714.0-b14a4f1", + "20250821.0-be53625", + "20250827.0-35b6d57" + ], + "yanked_versions": {} +} diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel new file mode 100644 index 00000000..af94d93f --- /dev/null +++ b/third_party/modules/sentencepiece/20240618.0-d7ace0a/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "sentencepiece", + version = "20240618.0-d7ace0a", + compatibility_level = 1, +) + +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel new file mode 100644 index 00000000..43639e89 --- /dev/null +++ b/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/BUILD.bazel @@ -0,0 +1,7 @@ +load("@rules_proto//proto:defs.bzl", "proto_library") + +proto_library( + name = "sentencepiece_model_proto", + srcs = ["src/sentencepiece_model.proto"], + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel b/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/sentencepiece/20240618.0-d7ace0a/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json b/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json new file mode 100644 index 00000000..588c9bcb --- /dev/null +++ b/third_party/modules/sentencepiece/20240618.0-d7ace0a/source.json @@ -0,0 +1,9 @@ +{ + "strip_prefix": "sentencepiece-d7ace0a4df17af17d73dc8fe6ff92cfd8d08e2b8", + "url": "https://github.com/google/sentencepiece/archive/d7ace0a4df17af17d73dc8fe6ff92cfd8d08e2b8.tar.gz", + "integrity": "sha256-bt8QFNNIpqcJoCNMUrmWfUf0uNobtk8pW+XbHL+ZRLg=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + } +} diff --git a/third_party/modules/sentencepiece/metadata.json b/third_party/modules/sentencepiece/metadata.json new file mode 100644 index 00000000..2c9758c9 --- /dev/null +++ b/third_party/modules/sentencepiece/metadata.json @@ -0,0 +1,17 @@ +{ + "homepage": "https://github.com/google/sentencepiece", + "maintainers": [ + { + "email": "bzlmod@zml.ai", + "github": "zml", + "name": "ZML Engineering Team" + } + ], + "repository": [ + "google/sentencepiece" + ], + "versions": [ + "20240618.0-d7ace0a" + ], + "yanked_versions": {} +} diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel b/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel new file mode 100644 index 00000000..29e45e04 --- /dev/null +++ b/third_party/modules/stablehlo/20240829.0-54aa1a5/MODULE.bazel @@ -0,0 +1,15 @@ +module( + name = "stablehlo", + version = "20240829.0-54aa1a5", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "llvm-raw", version = "20240823.0-94c024a") + +llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") +llvm.configure( + targets = ["AArch64", "X86", "NVPTX"], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel b/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/stablehlo/20240829.0-54aa1a5/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch b/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch new file mode 100644 index 00000000..82ca0c4d --- /dev/null +++ b/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/0001-Remove-duplicated-symbols-in-StablehloApi.h.patch @@ -0,0 +1,83 @@ +From f11d9bd7639f63cba681caa39cea27af114a4a71 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 2 Sep 2024 23:02:28 +0200 +Subject: [PATCH 1/2] Remove duplicated symbols in StablehloApi.h + +This causes C compilers to choke. + +Refs #2494 +--- + stablehlo/integrations/c/StablehloApi.cpp | 2 +- + stablehlo/integrations/c/StablehloApi.h | 15 +-------------- + stablehlo/integrations/python/StablehloApi.cpp | 2 +- + 3 files changed, 3 insertions(+), 16 deletions(-) + +diff --git a/stablehlo/integrations/c/StablehloApi.cpp b/stablehlo/integrations/c/StablehloApi.cpp +index 8d922198..9b41ff21 100644 +--- a/stablehlo/integrations/c/StablehloApi.cpp ++++ b/stablehlo/integrations/c/StablehloApi.cpp +@@ -98,7 +98,7 @@ MlirLogicalResult stablehloSerializePortableArtifact( + return mlirLogicalResultSuccess(); + } + +-MlirLogicalResult stablehloDeserializePortableArtifact( ++MlirLogicalResult stablehloDeserializePortableArtifactToBytecode( + MlirStringRef artifactStr, MlirStringCallback callback, void *userData) { + mlir::detail::CallbackOstream stream(callback, userData); + if (failed(mlir::stablehlo::deserializePortableArtifact(unwrap(artifactStr), +diff --git a/stablehlo/integrations/c/StablehloApi.h b/stablehlo/integrations/c/StablehloApi.h +index 4c542508..f94dca8d 100644 +--- a/stablehlo/integrations/c/StablehloApi.h ++++ b/stablehlo/integrations/c/StablehloApi.h +@@ -76,16 +76,6 @@ MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( + MlirStringRef moduleStr, MlirStringRef targetVersion, + MlirStringCallback callback, void* userData); + +-// Write a StableHLO program expressed as a string (either prettyprinted MLIR +-// module or MLIR bytecode) to a portable artifact. +-// Can fail if `moduleStr` cannot be parsed, or if it cannot be expressed in the +-// `targetVersion` version of StableHLO, e.g. if it's using new or removed +-// features, or if it involves unsupported dialects. +-// Returns false on failure. +-MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( +- MlirModule moduleStr, MlirStringRef targetVersion, +- MlirStringCallback callback, void* userData); +- + // Read a StableHLO program from a portable artifact, returning the module as + // MLIR bytecode. Note, this bytecode returned is not a portable artifact, + // and has the stability of returning textual assembly format. Bytecode is +@@ -93,7 +83,7 @@ MLIR_CAPI_EXPORTED MlirLogicalResult stablehloSerializePortableArtifact( + // Can fail if `artifactStr` cannot be expressed in the current version of + // StableHLO, e.g. if it's using incompatible features. + // Returns false on failure. +-MLIR_CAPI_EXPORTED MlirLogicalResult stablehloDeserializePortableArtifact( ++MLIR_CAPI_EXPORTED MlirLogicalResult stablehloDeserializePortableArtifactAsBytecode( + MlirStringRef artifactStr, MlirStringCallback callback, void* userData); + + // Read a StableHLO program from a portable artifact, returning the module as +@@ -109,9 +99,6 @@ MLIR_CAPI_EXPORTED MlirModule stablehloDeserializePortableArtifact( + + // Call the Interpreter, returns MlirArrayAttr of dense element + // MlirAttribute results +-MLIR_CAPI_EXPORTED MlirModule stablehloDeserializePortableArtifact( +- MlirStringRef artifactStr, MlirContext ctx); +- + // Entrypoint for calling the StableHLO reference interpreter. + // Returns an array attribute of dense element attributes for results. + // Sets error code to non-zero on failure. +diff --git a/stablehlo/integrations/python/StablehloApi.cpp b/stablehlo/integrations/python/StablehloApi.cpp +index 46a640e1..4229ef76 100644 +--- a/stablehlo/integrations/python/StablehloApi.cpp ++++ b/stablehlo/integrations/python/StablehloApi.cpp +@@ -213,7 +213,7 @@ void AddPortableApi(py::module &m) { + "deserialize_portable_artifact_str", + [](std::string_view artifact) -> py::bytes { + StringWriterHelper accumulator; +- if (mlirLogicalResultIsFailure(stablehloDeserializePortableArtifact( ++ if (mlirLogicalResultIsFailure(stablehloDeserializePortableArtifactToBytecode( + toMlirStringRef(artifact), accumulator.getMlirStringCallback(), + accumulator.getUserData()))) { + PyErr_SetString(PyExc_ValueError, "failed to deserialize module"); +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch b/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch new file mode 100644 index 00000000..5240a14c --- /dev/null +++ b/third_party/modules/stablehlo/20240829.0-54aa1a5/patches/temporary.patch @@ -0,0 +1,401 @@ +diff --ruN a/stablehlo/BUILD.bazel b/stablehlo/BUILD.bazel +--- stablehlo/BUILD.bazel ++++ stablehlo/BUILD.bazel +@@ -340,6 +340,21 @@ + ], + ) + ++gentbl_cc_library( ++ name = "stablehlo_create_compatibility_expander_inc_gen", ++ tbl_outs = [ ++ ( ++ ["--gen-rewriters"], ++ "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.h.inc", ++ ), ++ ], ++ tblgen = "@llvm-project//mlir:mlir-tblgen", ++ td_file = "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td", ++ deps = [ ++ ":stablehlo_ops_td_files", ++ ], ++) ++ + cc_library( + name = "interpreter_ops", + srcs = [ +@@ -1086,6 +1101,7 @@ + "stablehlo/transforms/StablehloAggressiveSimplification.cpp", + "stablehlo/transforms/StablehloCanonicalizeDynamism.cpp", + "stablehlo/transforms/StablehloConvertToSignless.cpp", ++ "stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp", + "stablehlo/transforms/StablehloLegalizeCompositeToCall.cpp", + "stablehlo/transforms/StablehloLegalizeDeprecatedOps.cpp", + "stablehlo/transforms/StablehloLegalizeQDQToQuantizedOp.cpp", +@@ -1109,6 +1125,7 @@ + ":chlo_ops", + ":chlo_rewriters_inc_gen", + ":linalg_passes", ++ ":stablehlo_create_compatibility_expander_inc_gen", + ":stablehlo_legalize_deprecated_ops_inc_gen", + ":stablehlo_ops", + ":stablehlo_ops_inc_gen", +diff --ruN a/stablehlo/stablehlo/dialect/Version.cpp b/stablehlo/stablehlo/dialect/Version.cpp +--- stablehlo/stablehlo/dialect/Version.cpp ++++ stablehlo/stablehlo/dialect/Version.cpp +@@ -82,7 +82,7 @@ + case CompatibilityRequirement::WEEK_4: + return Version(1, 3, 0); // v1.3.0 - Jul 15, 2024 + case CompatibilityRequirement::WEEK_12: +- return Version(1, 0, 0); // v1.0.0 - May 14, 2024 ++ return Version(1, 1, 0); // v1.1.0 - May 30, 2024 + case CompatibilityRequirement::MAX: + return Version::getMinimumVersion(); + } +diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir +--- stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir ++++ stablehlo/stablehlo/tests/transforms/stablehlo_create_compatibility_expander.mlir +@@ -0,0 +1,43 @@ ++// RUN: stablehlo-opt %s -verify-diagnostics -split-input-file -allow-unregistered-dialect --stablehlo-create-compatibility-expander='target=1.0.0' | FileCheck %s --check-prefixes=CHECK ++// RUN: stablehlo-opt %s -verify-diagnostics -split-input-file --stablehlo-create-compatibility-expander='target=1.6.0' | FileCheck %s --check-prefixes=CHECK-NO-DOWNGRADE ++ ++// ----- ++ ++// CHECK-LABEL @tan_op_non_complex ++// CHECK: %[[sine0:.*]] = stablehlo.sine %arg0 : tensor<4xf64> ++// CHECK-NEXT: %[[cosine1:.*]] = stablehlo.cosine %arg0 : tensor<4xf64> ++// CHECK-NEXT: %[[div2:.*]] = stablehlo.divide %[[sine0]], %[[cosine1]] : tensor<4xf64> ++// CHECK-NEXT: return %[[div2]] : tensor<4xf64> ++func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { ++ // CHECK-NO-DOWNGRADE: stablehlo.tan %arg0 : tensor<4xf64> ++ %1 = stablehlo.tan %arg0 : tensor<4xf64> ++ func.return %1 : tensor<4xf64> ++} ++ ++// ----- ++ ++// CHECK-LABEL: @tan_op_complex ++// CHECK: %[[cst:.*]] = stablehlo.constant dense<1.000000e+00> : tensor<4xf64> ++// CHECK: %[[complex:.*]] = stablehlo.complex %arg0, %arg1 : tensor<4xcomplex> ++// CHECK: %[[real:.*]] = stablehlo.real %[[complex]] : (tensor<4xcomplex>) -> tensor<4xf64> ++// CHECK: %[[sine:.*]] = stablehlo.sine %[[real]] : tensor<4xf64> ++// CHECK: %[[cosine:.*]] = stablehlo.cosine %[[real]] : tensor<4xf64> ++// CHECK: %[[divide1:.*]] = stablehlo.divide %[[sine]], %[[cosine]] : tensor<4xf64> ++// CHECK: %[[imag:.*]] = stablehlo.imag %[[complex]] : (tensor<4xcomplex>) -> tensor<4xf64> ++// CHECK: %[[tanh:.*]] = stablehlo.tanh %[[imag]] : tensor<4xf64> ++// CHECK: %[[complex2:.*]] = stablehlo.complex %[[divide1]], %[[tanh]] : tensor<4xcomplex> ++// CHECK: %[[multiply:.*]] = stablehlo.multiply %[[divide1]], %[[tanh]] : tensor<4xf64> ++// CHECK: %[[negate:.*]] = stablehlo.negate %[[multiply]] : tensor<4xf64> ++// CHECK: %[[complex3:.*]] = stablehlo.complex %[[cst]], %[[negate]] : tensor<4xcomplex> ++// CHECK: %[[divide2:.*]] = stablehlo.divide %[[complex2]], %[[complex3]] : tensor<4xcomplex> ++// CHECK: %[[real2:.*]] = stablehlo.real %[[divide2]] : (tensor<4xcomplex>) -> tensor<4xf64> ++// CHECK: %[[imag2:.*]] = stablehlo.imag %[[divide2]] : (tensor<4xcomplex>) -> tensor<4xf64> ++// CHECK: return %[[real2]], %[[imag2]] : tensor<4xf64>, tensor<4xf64> ++func.func @tan_op_complex(%arg0: tensor<4xf64>, %arg1: tensor<4xf64>) -> (tensor<4xf64>, tensor<4xf64>) { ++ %0 = stablehlo.complex %arg0, %arg1 : tensor<4xcomplex> ++ // CHECK-NO-DOWNGRADE: stablehlo.tan %0 : tensor<4xcomplex> ++ %1 = stablehlo.tan %0 : tensor<4xcomplex> ++ %2 = stablehlo.real %1 : (tensor<4xcomplex>) -> tensor<4xf64> ++ %3 = stablehlo.imag %1 : (tensor<4xcomplex>) -> tensor<4xf64> ++ func.return %2, %3 : tensor<4xf64>, tensor<4xf64> ++} +diff --ruN a/stablehlo/stablehlo/transforms/CMakeLists.txt b/stablehlo/stablehlo/transforms/CMakeLists.txt +--- stablehlo/stablehlo/transforms/CMakeLists.txt ++++ stablehlo/stablehlo/transforms/CMakeLists.txt +@@ -20,6 +20,10 @@ + mlir_tablegen(ChloDecompositionPatterns.h.inc --gen-rewriters) + add_public_tablegen_target(ChloDecompositionPatternsIncGen) + ++set(LLVM_TARGET_DEFINITIONS StablehloCreateCompatibilityExpanderPatterns.td) ++mlir_tablegen(StablehloCreateCompatibilityExpanderPatterns.h.inc --gen-rewriters) ++add_public_tablegen_target(StablehloCreateCompatibilityExpanderPatternsIncGen) ++ + set(LLVM_TARGET_DEFINITIONS StablehloLegalizeDeprecatedOpsPatterns.td) + mlir_tablegen(StablehloLegalizeDeprecatedOpsPatterns.h.inc --gen-rewriters) + add_public_tablegen_target(StablehloLegalizeDeprecatedOpsPatternsIncGen) +@@ -27,6 +31,7 @@ + set(LLVM_TARGET_DEFINITIONS VhloToVersionPatterns.td) + mlir_tablegen(VhloToVersionPatterns.h.inc --gen-rewriters) + add_public_tablegen_target(VhloToVersionPatterns) ++ + + add_mlir_dialect_library(StablehloPasses + PARTIAL_SOURCES_INTENDED +@@ -37,6 +42,7 @@ + StablehloAggressiveSimplification.cpp + StablehloCanonicalizeDynamism.cpp + StablehloConvertToSignless.cpp ++ StablehloCreateCompatibilityExpander.cpp + StablehloLegalizeCompositeToCall.cpp + StablehloLegalizeDeprecatedOps.cpp + StablehloLegalizeQuantToMath.cpp +@@ -53,6 +59,7 @@ + StablehloLegalizeDeprecatedOpsPatternsIncGen + PassesIncGen + VhloToVersionPatterns ++ StablehloCreateCompatibilityExpanderPatternsIncGen + + LINK_LIBS PUBLIC + ChloOps +diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/transforms/Passes.h +--- stablehlo/stablehlo/transforms/Passes.h ++++ stablehlo/stablehlo/transforms/Passes.h +@@ -25,6 +25,7 @@ + #include "mlir/Pass/Pass.h" + #include "mlir/Support/LogicalResult.h" + #include "mlir/Transforms/DialectConversion.h" ++#include "stablehlo/dialect/Version.h" + + namespace mlir { + namespace stablehlo { +@@ -96,6 +97,12 @@ + void populateShapeToStablehloPatterns(MLIRContext *context, + RewritePatternSet *patterns); + ++/// Collection of patterns to create compatibility expander for StableHLO ++/// operations. ++void populateStablehloCreateCompatibilityExpanderPatterns( ++ RewritePatternSet *patterns, MLIRContext *context, ++ vhlo::Version targetVersion); ++ + //// Additional pass constructors //// + + std::unique_ptr> createStablehloRefineArgumentsPass( +diff --ruN a/stablehlo/stablehlo/transforms/Passes.td b/stablehlo/stablehlo/transforms/Passes.td +--- stablehlo/stablehlo/transforms/Passes.td ++++ stablehlo/stablehlo/transforms/Passes.td +@@ -292,3 +292,51 @@ + "mlir::stablehlo::StablehloDialect", + ]; + } ++ ++def StablehloCreateCompatibilityExpanderPass : Pass<"stablehlo-create-compatibility-expander", "mlir::func::FuncOp"> { ++ let summary = "Create compatibility expander for StableHLO operations."; ++ ++ let description = [{ ++ StableHLO ops gets updates or new op is introduced in the latest versions. ++ This opt-in pass expands backward compatibility with older StableHLO ++ versions by decomposing newer StableHLO operations into equivalent ++ operations supported by those older versions. ++ ++ Why is this an opt-in pass? ++ ++ Occasionally, StableHLO op enhancements are used to greatly simplify the ++ handling of certain common patterns in the OpenXLA ecosystem. This ++ includes things like TanOp, which has high framework and compiler support, ++ as well as gather/scatter batching dimensions, which can be represented ++ using slices, but makes sharding much more difficult. For this category of ++ new features, we do not offer automatic downgrade, since it may throw away ++ important information used in subsequent optimizations. This pass can be ++ used to expand these ops based on a target version to maximize compatibility ++ at the expense of potentially less optimal compilation. ++ ++ ```mlir ++ func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { ++ %1 = stablehlo.tan %arg0 : tensor<4xf64> ++ func.return %1 : tensor<4xf64> ++ } ++ ``` ++ ++ will become: ++ ++ ```mlir ++ func.func @tan_op_non_complex(%arg0: tensor<4xf64>) -> tensor<4xf64> { ++ %0 = stablehlo.sine %arg0 : tensor<4xf64> ++ %1 = stablehlo.cosine %arg0 : tensor<4xf64> ++ %2 = stablehlo.divide %0, %1 : tensor<4xf64> ++ return %2 : tensor<4xf64> ++ } ++ ``` ++ }]; ++ let options = [ ++ Option<"targetVersionOption", "target", "std::string", "", ++ "The target version. Must be a version of the form #.#.#.">, ++ ]; ++ let dependentDialects = [ ++ "mlir::stablehlo::StablehloDialect", ++ ]; ++} +diff --ruN a/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp b/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp +--- stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp ++++ stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpander.cpp +@@ -0,0 +1,128 @@ ++/* Copyright 2024 The StableHLO Authors. All Rights Reserved. ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ http://www.apache.org/licenses/LICENSE-2.0 ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++#include ++ ++#include ++ ++#include "llvm/ADT/APFloat.h" ++#include "llvm/Support/ErrorHandling.h" ++#include "mlir/Dialect/Func/IR/FuncOps.h" ++#include "mlir/IR/BuiltinAttributes.h" ++#include "mlir/IR/PatternMatch.h" ++#include "mlir/Support/LLVM.h" ++#include "mlir/Transforms/DialectConversion.h" ++#include "mlir/Transforms/GreedyPatternRewriteDriver.h" ++#include "stablehlo/dialect/StablehloOps.h" ++#include "stablehlo/dialect/Version.h" ++#include "stablehlo/transforms/Passes.h" ++ ++namespace mlir { ++namespace stablehlo { ++#define GEN_PASS_DEF_STABLEHLOCREATECOMPATIBILITYEXPANDERPASS ++#include "stablehlo/transforms/Passes.h.inc" ++ ++namespace { ++ ++//===----------------------------------------------------------------------===// ++// Helpers. ++//===----------------------------------------------------------------------===// ++ ++// Creates a constant with all ones. ++static Value createConstantWithAllOnes(OpBuilder &b, Location loc, Value val) { ++ auto shapedTy = dyn_cast(val.getType()); ++ if (!shapedTy) llvm_unreachable("Unsupported shaped type."); ++ ++ mlir::DenseElementsAttr elementsAttr = ++ mlir::DenseElementsAttr::get(shapedTy, 1.0); ++ ++ return b.create(loc, val.getType(), ++ elementsAttr); ++} ++ ++// Check user-specified target version. ++vhlo::Version validateTargetVersion(llvm::StringRef versionRef) { ++ auto failOrVersion = vhlo::Version::fromString(versionRef); ++ if (failed(failOrVersion)) { ++ assert(!versionRef.empty() && ++ "No target version specified. Target version must be of the form " ++ "`#.#.#`."); ++ assert(versionRef.empty() && ++ "Invalid target version argument. Target version must be of the " ++ "form `#.#.#`."); ++ } ++ vhlo::Version targetVersion = *failOrVersion; ++ assert((vhlo::Version::getMinimumVersion() <= targetVersion) && ++ "target version is less than minimum supported."); ++ assert((targetVersion <= vhlo::Version::getCurrentVersion()) && ++ "target version is greater than current version."); ++ return targetVersion; ++} ++ ++//===----------------------------------------------------------------------===// ++// Pass ++//===----------------------------------------------------------------------===// ++ ++struct StablehloCreateCompatibilityExpanderPass ++ : public impl::StablehloCreateCompatibilityExpanderPassBase< ++ StablehloCreateCompatibilityExpanderPass> { ++ StablehloCreateCompatibilityExpanderPass() ++ : StablehloCreateCompatibilityExpanderPassBase< ++ StablehloCreateCompatibilityExpanderPass>() {} ++ StablehloCreateCompatibilityExpanderPass( ++ const StablehloCreateCompatibilityExpanderPassOptions &opts) ++ : StablehloCreateCompatibilityExpanderPassBase< ++ StablehloCreateCompatibilityExpanderPass>(opts) {} ++ ++ public: ++ LogicalResult initialize(MLIRContext *context) override { ++ auto targetVersion = validateTargetVersion(targetVersionOption); ++ ++ config.useTopDownTraversal = true; ++ RewritePatternSet patterns_(context); ++ populateStablehloCreateCompatibilityExpanderPatterns(&patterns_, context, ++ targetVersion); ++ patterns = std::move(patterns_); ++ return success(); ++ } ++ ++ void runOnOperation() override { ++ auto func = getOperation(); ++ if (failed(applyPatternsAndFoldGreedily(func, patterns, config))) { ++ func.emitError( ++ "Failed to converge StableHLOCreateCompatibilityExpanderPass in ") ++ << config.maxIterations << " iterations"; ++ signalPassFailure(); ++ } ++ } ++ ++ private: ++ FrozenRewritePatternSet patterns; ++ GreedyRewriteConfig config; ++}; ++ ++#include "stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.h.inc" ++ ++} // namespace ++ ++void populateStablehloCreateCompatibilityExpanderPatterns( ++ RewritePatternSet *patterns, MLIRContext *context, ++ vhlo::Version targetVersion) { ++ // StableHLO TanOp is introduced in v1.4.0. ++ if (targetVersion < vhlo::Version(1, 4, 0)) { ++ patterns->add(context); ++ patterns->add(context); ++ } ++} ++ ++} // namespace stablehlo ++} // namespace mlir +diff --ruN a/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td b/stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td +--- stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td ++++ stablehlo/stablehlo/transforms/StablehloCreateCompatibilityExpanderPatterns.td +@@ -0,0 +1,47 @@ ++/* Copyright 2022 The StableHLO Authors. ++ ++Licensed under the Apache License, Version 2.0 (the "License"); ++you may not use this file except in compliance with the License. ++You may obtain a copy of the License at ++ ++ http://www.apache.org/licenses/LICENSE-2.0 ++ ++Unless required by applicable law or agreed to in writing, software ++distributed under the License is distributed on an "AS IS" BASIS, ++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++See the License for the specific language governing permissions and ++limitations under the License. ++==============================================================================*/ ++ ++include "mlir/IR/OpBase.td" ++include "stablehlo/dialect/StablehloOps.td" ++ ++def ComplexElementType : Type< ++ CPred<"isa(cast($_self).getElementType())">, ++ "Complex element type">; ++ ++def NonComplexElementType : Type< ++ CPred<"!isa(cast($_self).getElementType())">, ++ "Non-complex element type">; ++ ++def createConstantWithAllOnes : NativeCodeCall<"createConstantWithAllOnes($_builder, $_loc, $0)">; ++ ++// Express `tan` as ++// sine(x) / cosine(x) ++def TanOp_CompatiblityExpander : Pat<(StableHLO_TanOp NonComplexElementType:$input), ++ (StableHLO_DivOp ++ (StableHLO_SineOp $input), ++ (StableHLO_CosineOp $input) ++ )>; ++ ++// Express `tan(a + bi)` as ++// (tan(a) + i tanh(b)) / (1 - i tan(a) * tanh(b)) ++def TanOp_ComplexElementType_CompatiblityExpander : Pat<(StableHLO_TanOp ComplexElementType:$input), ++ (StableHLO_DivOp ++ (StableHLO_ComplexOp ++ (StableHLO_TanOp:$tan (StableHLO_RealOp $input)), ++ (StableHLO_TanhOp:$tanh (StableHLO_ImagOp $input))), ++ (StableHLO_ComplexOp ++ (createConstantWithAllOnes $tan), ++ (StableHLO_NegOp (StableHLO_MulOp $tan, $tanh))) ++ )>; + diff --git a/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json b/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json new file mode 100644 index 00000000..6d271a53 --- /dev/null +++ b/third_party/modules/stablehlo/20240829.0-54aa1a5/source.json @@ -0,0 +1,12 @@ +{ + "strip_prefix": "stablehlo-54aa1a57178251981da616b877dda1a88d840d11", + "url": "https://github.com/openxla/stablehlo/archive/54aa1a57178251981da616b877dda1a88d840d11.tar.gz", + "integrity": "sha256-PFXXs3vQ+WlMiCC8twbGEFbsBEF/67Z4qtmq8CtUSUU=", + "overlay": { + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-Remove-duplicated-symbols-in-StablehloApi.h.patch": "" + } +} diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel b/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel new file mode 100644 index 00000000..7cbfb936 --- /dev/null +++ b/third_party/modules/stablehlo/20240917.0-78c753a/MODULE.bazel @@ -0,0 +1,15 @@ +module( + name = "stablehlo", + version = "20240917.0-78c753a", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "llvm-raw", version = "20240919.0-94c024a") + +llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") +llvm.configure( + targets = ["AArch64", "X86", "NVPTX"], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel b/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/stablehlo/20240917.0-78c753a/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20240917.0-78c753a/source.json b/third_party/modules/stablehlo/20240917.0-78c753a/source.json new file mode 100644 index 00000000..cd677bd8 --- /dev/null +++ b/third_party/modules/stablehlo/20240917.0-78c753a/source.json @@ -0,0 +1,8 @@ +{ + "strip_prefix": "stablehlo-78c753ad13ad8205cacc5fcc12418c1ac97276c7", + "url": "https://github.com/openxla/stablehlo/archive/78c753ad13ad8205cacc5fcc12418c1ac97276c7.tar.gz", + "integrity": "sha256-2qFcPLXs7glf7RFDUiqbKXiBBamW7n8a30egR4Yu/bc=", + "overlay": { + "MODULE.bazel": "" + } +} diff --git a/third_party/modules/stablehlo/20241021.0-1c0b606/MODULE.bazel b/third_party/modules/stablehlo/20241021.0-1c0b606/MODULE.bazel new file mode 100644 index 00000000..637e0e76 --- /dev/null +++ b/third_party/modules/stablehlo/20241021.0-1c0b606/MODULE.bazel @@ -0,0 +1,15 @@ +module( + name = "stablehlo", + version = "20241021.0-1c0b606", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "llvm-raw", version = "20241022.0-6c4267f") + +llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") +llvm.configure( + targets = ["AArch64", "X86", "NVPTX"], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20241021.0-1c0b606/overlay/MODULE.bazel b/third_party/modules/stablehlo/20241021.0-1c0b606/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/stablehlo/20241021.0-1c0b606/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20241021.0-1c0b606/source.json b/third_party/modules/stablehlo/20241021.0-1c0b606/source.json new file mode 100644 index 00000000..e67596eb --- /dev/null +++ b/third_party/modules/stablehlo/20241021.0-1c0b606/source.json @@ -0,0 +1,8 @@ +{ + "strip_prefix": "stablehlo-f7f8e4e35296deeff2e12e39421ac8d9599ba340", + "url": "https://github.com/openxla/stablehlo/archive/f7f8e4e35296deeff2e12e39421ac8d9599ba340.tar.gz", + "integrity": "sha256-/chY6BYV7KcrXIuqXTNIUknISCiA1JYhWE8gG5cEeV4=", + "overlay": { + "MODULE.bazel": "" + } +} diff --git a/third_party/modules/stablehlo/20241220.0-38bb2f9/MODULE.bazel b/third_party/modules/stablehlo/20241220.0-38bb2f9/MODULE.bazel new file mode 100644 index 00000000..a045302f --- /dev/null +++ b/third_party/modules/stablehlo/20241220.0-38bb2f9/MODULE.bazel @@ -0,0 +1,15 @@ +module( + name = "stablehlo", + version = "20241220.0-38bb2f9", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "llvm-raw", version = "20250102.0-f739aa4") + +llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") +llvm.configure( + targets = ["AArch64", "X86", "NVPTX"], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20241220.0-38bb2f9/overlay/MODULE.bazel b/third_party/modules/stablehlo/20241220.0-38bb2f9/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/stablehlo/20241220.0-38bb2f9/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20241220.0-38bb2f9/source.json b/third_party/modules/stablehlo/20241220.0-38bb2f9/source.json new file mode 100644 index 00000000..9370183f --- /dev/null +++ b/third_party/modules/stablehlo/20241220.0-38bb2f9/source.json @@ -0,0 +1,8 @@ +{ + "strip_prefix": "stablehlo-38bb2f9bf63b714e8a49fe34a478139058ee1660", + "url": "https://github.com/openxla/stablehlo/archive/38bb2f9bf63b714e8a49fe34a478139058ee1660.tar.gz", + "integrity": "sha256-TuaG8q6WCdJGl4ECpu8YO8ORYcRo0NSff3xHKVj8Ixw=", + "overlay": { + "MODULE.bazel": "" + } +} diff --git a/third_party/modules/stablehlo/20250117.0-c125b32/MODULE.bazel b/third_party/modules/stablehlo/20250117.0-c125b32/MODULE.bazel new file mode 100644 index 00000000..2aabd323 --- /dev/null +++ b/third_party/modules/stablehlo/20250117.0-c125b32/MODULE.bazel @@ -0,0 +1,15 @@ +module( + name = "stablehlo", + version = "20250117.0-c125b32", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "llvm-raw", version = "20250117.0-bf17016") + +llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") +llvm.configure( + targets = ["AArch64", "X86", "NVPTX"], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20250117.0-c125b32/overlay/MODULE.bazel b/third_party/modules/stablehlo/20250117.0-c125b32/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/stablehlo/20250117.0-c125b32/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20250117.0-c125b32/source.json b/third_party/modules/stablehlo/20250117.0-c125b32/source.json new file mode 100644 index 00000000..fd8426be --- /dev/null +++ b/third_party/modules/stablehlo/20250117.0-c125b32/source.json @@ -0,0 +1,8 @@ +{ + "strip_prefix": "stablehlo-c125b3284819fec57120231cf4430657dab7b881", + "url": "https://github.com/openxla/stablehlo/archive/c125b3284819fec57120231cf4430657dab7b881.tar.gz", + "integrity": "sha256-h4iBDLVbK2JZRDY7qdmNdns25AoVjpLekKdmsETEDf8=", + "overlay": { + "MODULE.bazel": "" + } +} diff --git a/third_party/modules/stablehlo/20250201.0-7775e3e/MODULE.bazel b/third_party/modules/stablehlo/20250201.0-7775e3e/MODULE.bazel new file mode 100644 index 00000000..7e9accd0 --- /dev/null +++ b/third_party/modules/stablehlo/20250201.0-7775e3e/MODULE.bazel @@ -0,0 +1,15 @@ +module( + name = "stablehlo", + version = "20250201.0-7775e3e", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "llvm-raw", version = "20250201.0-386af4a") + +llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") +llvm.configure( + targets = ["AArch64", "X86", "NVPTX"], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20250201.0-7775e3e/overlay/MODULE.bazel b/third_party/modules/stablehlo/20250201.0-7775e3e/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/stablehlo/20250201.0-7775e3e/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20250201.0-7775e3e/source.json b/third_party/modules/stablehlo/20250201.0-7775e3e/source.json new file mode 100644 index 00000000..e3e17ea2 --- /dev/null +++ b/third_party/modules/stablehlo/20250201.0-7775e3e/source.json @@ -0,0 +1,8 @@ +{ + "strip_prefix": "stablehlo-7775e3e2f98fe458f227cdc41aec567535d1c796", + "url": "https://github.com/openxla/stablehlo/archive/7775e3e2f98fe458f227cdc41aec567535d1c796.tar.gz", + "integrity": "sha256-HM/MIs3FeN4PxcYgFsjj04tVJ4u4usGiYGSa9BwT5bg=", + "overlay": { + "MODULE.bazel": "" + } +} diff --git a/third_party/modules/stablehlo/20250217.0-4598975/MODULE.bazel b/third_party/modules/stablehlo/20250217.0-4598975/MODULE.bazel new file mode 100644 index 00000000..8c462655 --- /dev/null +++ b/third_party/modules/stablehlo/20250217.0-4598975/MODULE.bazel @@ -0,0 +1,15 @@ +module( + name = "stablehlo", + version = "20250217.0-4598975", + compatibility_level = 1, +) + +bazel_dep(name = "bazel_skylib", version = "1.7.1") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "llvm-raw", version = "20250217.0-0e779ad") + +llvm = use_extension("@llvm-raw//utils/bazel:extension.bzl", "llvm") +llvm.configure( + targets = ["AArch64", "X86", "NVPTX"], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/stablehlo/20250217.0-4598975/overlay/MODULE.bazel b/third_party/modules/stablehlo/20250217.0-4598975/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/stablehlo/20250217.0-4598975/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/stablehlo/20250217.0-4598975/source.json b/third_party/modules/stablehlo/20250217.0-4598975/source.json new file mode 100644 index 00000000..6988aba7 --- /dev/null +++ b/third_party/modules/stablehlo/20250217.0-4598975/source.json @@ -0,0 +1,8 @@ +{ + "strip_prefix": "stablehlo-459897561d365ef97caba46984847f9184d472ec", + "url": "https://github.com/openxla/stablehlo/archive/459897561d365ef97caba46984847f9184d472ec.tar.gz", + "integrity": "sha256-J2KXzmX3uAJiwuMLl06n8dyb9l1TCzGZ62IULyAUcec=", + "overlay": { + "MODULE.bazel": "" + } +} diff --git a/third_party/modules/stablehlo/metadata.json b/third_party/modules/stablehlo/metadata.json new file mode 100644 index 00000000..d0e5fef6 --- /dev/null +++ b/third_party/modules/stablehlo/metadata.json @@ -0,0 +1,22 @@ +{ + "homepage": "https://github.com/openxla/stablehlo", + "maintainers": [ + { + "email": "bzlmod@zml.ai", + "github": "zml", + "name": "ZML Engineering Team" + } + ], + "repository": [ + "github:openxla/stablehlo" + ], + "versions": [ + "20240829.0-54aa1a5", + "20240917.0-78c753a", + "20241021.0-1c0b606", + "20241220.0-38bb2f9", + "20250117.0-c125b32", + "20250201.0-7775e3e", + ], + "yanked_versions": {} +} diff --git a/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel b/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel new file mode 100644 index 00000000..739fe835 --- /dev/null +++ b/third_party/modules/xla/20240902.0-d18cd64/MODULE.bazel @@ -0,0 +1,34 @@ +module( + name = "xla", + version = "20240902.0-d18cd64", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +bazel_dep(name = "stablehlo", version = "20240829.0-54aa1a5") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", +) diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel b/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20240902.0-d18cd64/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl b/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl new file mode 100644 index 00000000..50c1df12 --- /dev/null +++ b/third_party/modules/xla/20240902.0-d18cd64/overlay/tsl.bzl @@ -0,0 +1,13 @@ +load("//third_party:repo.bzl", "tf_vendored") + +def _tsl_impl(mctx): + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl b/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl new file mode 100644 index 00000000..87c035b7 --- /dev/null +++ b/third_party/modules/xla/20240902.0-d18cd64/overlay/workspace.bzl @@ -0,0 +1,52 @@ +load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "@tsl//third_party/grpc:generate_cc_env_fix.patch", + "@tsl//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", + "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch new file mode 100644 index 00000000..0f58f107 --- /dev/null +++ b/third_party/modules/xla/20240902.0-d18cd64/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch @@ -0,0 +1,27 @@ +From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 18 Mar 2024 17:17:34 +0100 +Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension + +--- + xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +index c821916ad..89a596123 100644 +--- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +@@ -16,8 +16,10 @@ limitations under the License. + #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + ++#ifdef __cplusplus + #include + #include ++#endif + + #include "xla/backends/profiler/plugin/profiler_c_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/xla/20240902.0-d18cd64/source.json b/third_party/modules/xla/20240902.0-d18cd64/source.json new file mode 100644 index 00000000..359f9b12 --- /dev/null +++ b/third_party/modules/xla/20240902.0-d18cd64/source.json @@ -0,0 +1,14 @@ +{ + "strip_prefix": "xla-d18cd64b7cd61a2ade10089665ac104f639101b1", + "url": "https://github.com/openxla/xla/archive/d18cd64b7cd61a2ade10089665ac104f639101b1.tar.gz", + "integrity": "sha256-EtKhjU91STBceUmg0TUE6cPeRkeSz3LI2S1i3EFMj/E=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" + } +} diff --git a/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel b/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel new file mode 100644 index 00000000..bd3a0046 --- /dev/null +++ b/third_party/modules/xla/20240919.0-1b18dd6/MODULE.bazel @@ -0,0 +1,34 @@ +module( + name = "xla", + version = "20240919.0-1b18dd6", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +bazel_dep(name = "stablehlo", version = "20240917.0-78c753a") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", +) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel b/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20240919.0-1b18dd6/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl b/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl new file mode 100644 index 00000000..50c1df12 --- /dev/null +++ b/third_party/modules/xla/20240919.0-1b18dd6/overlay/tsl.bzl @@ -0,0 +1,13 @@ +load("//third_party:repo.bzl", "tf_vendored") + +def _tsl_impl(mctx): + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl b/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl new file mode 100644 index 00000000..87c035b7 --- /dev/null +++ b/third_party/modules/xla/20240919.0-1b18dd6/overlay/workspace.bzl @@ -0,0 +1,52 @@ +load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "@tsl//third_party/grpc:generate_cc_env_fix.patch", + "@tsl//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", + "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch new file mode 100644 index 00000000..0f58f107 --- /dev/null +++ b/third_party/modules/xla/20240919.0-1b18dd6/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch @@ -0,0 +1,27 @@ +From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 18 Mar 2024 17:17:34 +0100 +Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension + +--- + xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +index c821916ad..89a596123 100644 +--- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +@@ -16,8 +16,10 @@ limitations under the License. + #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + ++#ifdef __cplusplus + #include + #include ++#endif + + #include "xla/backends/profiler/plugin/profiler_c_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/xla/20240919.0-1b18dd6/source.json b/third_party/modules/xla/20240919.0-1b18dd6/source.json new file mode 100644 index 00000000..9853728a --- /dev/null +++ b/third_party/modules/xla/20240919.0-1b18dd6/source.json @@ -0,0 +1,14 @@ +{ + "strip_prefix": "xla-d391119197eab771a84c1f8a59a7f50b7da4b43d", + "url": "https://github.com/openxla/xla/archive/d391119197eab771a84c1f8a59a7f50b7da4b43d.tar.gz", + "integrity": "sha256-ToPcvPhoGX6Ny3yLx6kJlQSwj7k5Xy4T+BjIp3DAj/s=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" + } +} diff --git a/third_party/modules/xla/20241025.0-4663f04/MODULE.bazel b/third_party/modules/xla/20241025.0-4663f04/MODULE.bazel new file mode 100644 index 00000000..22019c9e --- /dev/null +++ b/third_party/modules/xla/20241025.0-4663f04/MODULE.bazel @@ -0,0 +1,34 @@ +module( + name = "xla", + version = "20241025.0-4663f04", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +bazel_dep(name = "stablehlo", version = "20241021.0-1c0b606") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", +) diff --git a/third_party/modules/xla/20241025.0-4663f04/overlay/MODULE.bazel b/third_party/modules/xla/20241025.0-4663f04/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20241025.0-4663f04/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20241025.0-4663f04/overlay/tsl.bzl b/third_party/modules/xla/20241025.0-4663f04/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20241025.0-4663f04/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20241025.0-4663f04/overlay/workspace.bzl b/third_party/modules/xla/20241025.0-4663f04/overlay/workspace.bzl new file mode 100644 index 00000000..87c035b7 --- /dev/null +++ b/third_party/modules/xla/20241025.0-4663f04/overlay/workspace.bzl @@ -0,0 +1,52 @@ +load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "@tsl//third_party/grpc:generate_cc_env_fix.patch", + "@tsl//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", + "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20241025.0-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20241025.0-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch new file mode 100644 index 00000000..0f58f107 --- /dev/null +++ b/third_party/modules/xla/20241025.0-4663f04/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch @@ -0,0 +1,27 @@ +From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 18 Mar 2024 17:17:34 +0100 +Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension + +--- + xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +index c821916ad..89a596123 100644 +--- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +@@ -16,8 +16,10 @@ limitations under the License. + #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + ++#ifdef __cplusplus + #include + #include ++#endif + + #include "xla/backends/profiler/plugin/profiler_c_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/xla/20241025.0-4663f04/source.json b/third_party/modules/xla/20241025.0-4663f04/source.json new file mode 100644 index 00000000..33327387 --- /dev/null +++ b/third_party/modules/xla/20241025.0-4663f04/source.json @@ -0,0 +1,14 @@ +{ + "strip_prefix": "xla-4663f04d2653d686833f9c306bd0d899c3127358", + "url": "https://github.com/openxla/xla/archive/4663f04d2653d686833f9c306bd0d899c3127358.tar.gz", + "integrity": "sha256-tQRn0TXkPcmKpCcIuTYM10XNB1qC/d6++4rA1mlnnKw=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" + } +} diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/MODULE.bazel b/third_party/modules/xla/20250103.0-5f1fe6a/MODULE.bazel new file mode 100644 index 00000000..5261911d --- /dev/null +++ b/third_party/modules/xla/20250103.0-5f1fe6a/MODULE.bazel @@ -0,0 +1,34 @@ +module( + name = "xla", + version = "20250103.0-5f1fe6a", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +bazel_dep(name = "stablehlo", version = "20241220.0-38bb2f9") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", +) diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/MODULE.bazel b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/tsl.bzl b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/overlay/workspace.bzl b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/workspace.bzl new file mode 100644 index 00000000..87c035b7 --- /dev/null +++ b/third_party/modules/xla/20250103.0-5f1fe6a/overlay/workspace.bzl @@ -0,0 +1,52 @@ +load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "@tsl//third_party/grpc:generate_cc_env_fix.patch", + "@tsl//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", + "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250103.0-5f1fe6a/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch new file mode 100644 index 00000000..0f58f107 --- /dev/null +++ b/third_party/modules/xla/20250103.0-5f1fe6a/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch @@ -0,0 +1,27 @@ +From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 18 Mar 2024 17:17:34 +0100 +Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension + +--- + xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +index c821916ad..89a596123 100644 +--- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +@@ -16,8 +16,10 @@ limitations under the License. + #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + ++#ifdef __cplusplus + #include + #include ++#endif + + #include "xla/backends/profiler/plugin/profiler_c_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/xla/20250103.0-5f1fe6a/source.json b/third_party/modules/xla/20250103.0-5f1fe6a/source.json new file mode 100644 index 00000000..690bbb4b --- /dev/null +++ b/third_party/modules/xla/20250103.0-5f1fe6a/source.json @@ -0,0 +1,14 @@ +{ + "strip_prefix": "xla-5f1fe6a2f35756238e4a0a0c4e754d6454296efe", + "url": "https://github.com/openxla/xla/archive/5f1fe6a2f35756238e4a0a0c4e754d6454296efe.tar.gz", + "integrity": "sha256-0+kdYIla4F63oPfACtw6LmmMLdKTFlBBrgvc9PmxFoI=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" + } +} diff --git a/third_party/modules/xla/20250122.0-cc075be/MODULE.bazel b/third_party/modules/xla/20250122.0-cc075be/MODULE.bazel new file mode 100644 index 00000000..fd99777d --- /dev/null +++ b/third_party/modules/xla/20250122.0-cc075be/MODULE.bazel @@ -0,0 +1,34 @@ +module( + name = "xla", + version = "20250122.0-cc075be", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.9") +bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +bazel_dep(name = "stablehlo", version = "20250117.0-c125b32") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", +) diff --git a/third_party/modules/xla/20250122.0-cc075be/overlay/MODULE.bazel b/third_party/modules/xla/20250122.0-cc075be/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250122.0-cc075be/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250122.0-cc075be/overlay/tsl.bzl b/third_party/modules/xla/20250122.0-cc075be/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250122.0-cc075be/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250122.0-cc075be/overlay/workspace.bzl b/third_party/modules/xla/20250122.0-cc075be/overlay/workspace.bzl new file mode 100644 index 00000000..87c035b7 --- /dev/null +++ b/third_party/modules/xla/20250122.0-cc075be/overlay/workspace.bzl @@ -0,0 +1,52 @@ +load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "@tsl//third_party/grpc:generate_cc_env_fix.patch", + "@tsl//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", + "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250122.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250122.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch new file mode 100644 index 00000000..0f58f107 --- /dev/null +++ b/third_party/modules/xla/20250122.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch @@ -0,0 +1,27 @@ +From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 18 Mar 2024 17:17:34 +0100 +Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension + +--- + xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +index c821916ad..89a596123 100644 +--- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +@@ -16,8 +16,10 @@ limitations under the License. + #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + ++#ifdef __cplusplus + #include + #include ++#endif + + #include "xla/backends/profiler/plugin/profiler_c_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/xla/20250122.0-cc075be/source.json b/third_party/modules/xla/20250122.0-cc075be/source.json new file mode 100644 index 00000000..4946d805 --- /dev/null +++ b/third_party/modules/xla/20250122.0-cc075be/source.json @@ -0,0 +1,14 @@ +{ + "strip_prefix": "xla-cc075beb6148c2777da2b6749c63830856ee6c2a", + "url": "https://github.com/openxla/xla/archive/cc075beb6148c2777da2b6749c63830856ee6c2a.tar.gz", + "integrity": "sha256-oB8S38WZKEXBtZ6rARd0oL9SAtSmfsPM2xTuj3ylexc=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" + } +} diff --git a/third_party/modules/xla/20250123.0-cc075be/MODULE.bazel b/third_party/modules/xla/20250123.0-cc075be/MODULE.bazel new file mode 100644 index 00000000..6a94e649 --- /dev/null +++ b/third_party/modules/xla/20250123.0-cc075be/MODULE.bazel @@ -0,0 +1,34 @@ +module( + name = "xla", + version = "20250123.0-cc075be", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.2.1", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-02-01", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +bazel_dep(name = "stablehlo", version = "20250117.0-c125b32") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", +) diff --git a/third_party/modules/xla/20250123.0-cc075be/overlay/MODULE.bazel b/third_party/modules/xla/20250123.0-cc075be/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250123.0-cc075be/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250123.0-cc075be/overlay/tsl.bzl b/third_party/modules/xla/20250123.0-cc075be/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250123.0-cc075be/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250123.0-cc075be/overlay/workspace.bzl b/third_party/modules/xla/20250123.0-cc075be/overlay/workspace.bzl new file mode 100644 index 00000000..87c035b7 --- /dev/null +++ b/third_party/modules/xla/20250123.0-cc075be/overlay/workspace.bzl @@ -0,0 +1,52 @@ +load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "@tsl//third_party/grpc:generate_cc_env_fix.patch", + "@tsl//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", + "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250123.0-cc075be/patches/0001-bazel-migration-to-bazel-8.0.1.patch b/third_party/modules/xla/20250123.0-cc075be/patches/0001-bazel-migration-to-bazel-8.0.1.patch new file mode 100644 index 00000000..221fa393 --- /dev/null +++ b/third_party/modules/xla/20250123.0-cc075be/patches/0001-bazel-migration-to-bazel-8.0.1.patch @@ -0,0 +1,42 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-6.5.0 ++8.0.1 +\ No newline at end of file +diff --git a/third_party/tsl/third_party/gpus/cuda_configure.bzl b/third_party/tsl/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/tsl/third_party/gpus/cuda_configure.bzl ++++ b/third_party/tsl/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/xla/20250123.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250123.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch new file mode 100644 index 00000000..0f58f107 --- /dev/null +++ b/third_party/modules/xla/20250123.0-cc075be/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch @@ -0,0 +1,27 @@ +From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 18 Mar 2024 17:17:34 +0100 +Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension + +--- + xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +index c821916ad..89a596123 100644 +--- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +@@ -16,8 +16,10 @@ limitations under the License. + #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + ++#ifdef __cplusplus + #include + #include ++#endif + + #include "xla/backends/profiler/plugin/profiler_c_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/xla/20250123.0-cc075be/source.json b/third_party/modules/xla/20250123.0-cc075be/source.json new file mode 100644 index 00000000..02869dd7 --- /dev/null +++ b/third_party/modules/xla/20250123.0-cc075be/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "xla-cc075beb6148c2777da2b6749c63830856ee6c2a", + "url": "https://github.com/openxla/xla/archive/cc075beb6148c2777da2b6749c63830856ee6c2a.tar.gz", + "integrity": "sha256-oB8S38WZKEXBtZ6rARd0oL9SAtSmfsPM2xTuj3ylexc=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.0.1.patch": "", + "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" + } +} diff --git a/third_party/modules/xla/20250204-0-6789523/MODULE.bazel b/third_party/modules/xla/20250204-0-6789523/MODULE.bazel new file mode 100644 index 00000000..7d2ebfb7 --- /dev/null +++ b/third_party/modules/xla/20250204-0-6789523/MODULE.bazel @@ -0,0 +1,34 @@ +module( + name = "xla", + version = "20250204-0-6789523", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +bazel_dep(name = "stablehlo", version = "20250201.0-7775e3e") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", +) diff --git a/third_party/modules/xla/20250204-0-6789523/overlay/MODULE.bazel b/third_party/modules/xla/20250204-0-6789523/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250204-0-6789523/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250204-0-6789523/overlay/tsl.bzl b/third_party/modules/xla/20250204-0-6789523/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250204-0-6789523/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250204-0-6789523/overlay/workspace.bzl b/third_party/modules/xla/20250204-0-6789523/overlay/workspace.bzl new file mode 100644 index 00000000..87c035b7 --- /dev/null +++ b/third_party/modules/xla/20250204-0-6789523/overlay/workspace.bzl @@ -0,0 +1,52 @@ +load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "@tsl//third_party/grpc:generate_cc_env_fix.patch", + "@tsl//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", + "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250204-0-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch b/third_party/modules/xla/20250204-0-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch new file mode 100644 index 00000000..a29cbd9e --- /dev/null +++ b/third_party/modules/xla/20250204-0-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-6.5.0 ++8.0.1 +\ No newline at end of file +diff --git a/third_party/tsl/third_party/gpus/cuda_configure.bzl b/third_party/tsl/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/tsl/third_party/gpus/cuda_configure.bzl ++++ b/third_party/tsl/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204-0-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250204-0-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch new file mode 100644 index 00000000..4b6a65ce --- /dev/null +++ b/third_party/modules/xla/20250204-0-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch @@ -0,0 +1,26 @@ +From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 18 Mar 2024 17:17:34 +0100 +Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension + +--- + xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +index c821916ad..89a596123 100644 +--- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +@@ -16,8 +16,10 @@ limitations under the License. + #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + ++#ifdef __cplusplus + #include + #include ++#endif + + #include "xla/backends/profiler/plugin/profiler_c_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204-0-6789523/source.json b/third_party/modules/xla/20250204-0-6789523/source.json new file mode 100644 index 00000000..ab3abf73 --- /dev/null +++ b/third_party/modules/xla/20250204-0-6789523/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "xla-6789523d0a42c2cf83309a263f417122c4c63811", + "url": "https://github.com/openxla/xla/archive/6789523d0a42c2cf83309a263f417122c4c63811.tar.gz", + "integrity": "sha256-7z0MdXQBoGApSYpTvYjKFmg18tGkjJ3P/PPQ487cPc8=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.0.1.patch": "", + "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" + } +} diff --git a/third_party/modules/xla/20250204.1-6789523/MODULE.bazel b/third_party/modules/xla/20250204.1-6789523/MODULE.bazel new file mode 100644 index 00000000..f0ad4f19 --- /dev/null +++ b/third_party/modules/xla/20250204.1-6789523/MODULE.bazel @@ -0,0 +1,34 @@ +module( + name = "xla", + version = "20250204.1-6789523", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +bazel_dep(name = "stablehlo", version = "20250217.0-4598975") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", +) diff --git a/third_party/modules/xla/20250204.1-6789523/overlay/MODULE.bazel b/third_party/modules/xla/20250204.1-6789523/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250204.1-6789523/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250204.1-6789523/overlay/tsl.bzl b/third_party/modules/xla/20250204.1-6789523/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250204.1-6789523/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250204.1-6789523/overlay/workspace.bzl b/third_party/modules/xla/20250204.1-6789523/overlay/workspace.bzl new file mode 100644 index 00000000..87c035b7 --- /dev/null +++ b/third_party/modules/xla/20250204.1-6789523/overlay/workspace.bzl @@ -0,0 +1,52 @@ +load("@tsl//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("@tsl//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("@tsl//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("@tsl//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("@tsl//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "@tsl//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "@tsl//third_party/grpc:generate_cc_env_fix.patch", + "@tsl//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "@tsl//third_party/systemlibs:BUILD": "bazel/BUILD", + "@tsl//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "@tsl//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["@tsl//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "@tsl//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "@tsl//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "@tsl//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250204.1-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch b/third_party/modules/xla/20250204.1-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch new file mode 100644 index 00000000..a29cbd9e --- /dev/null +++ b/third_party/modules/xla/20250204.1-6789523/patches/0001-bazel-migration-to-bazel-8.0.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-6.5.0 ++8.0.1 +\ No newline at end of file +diff --git a/third_party/tsl/third_party/gpus/cuda_configure.bzl b/third_party/tsl/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/tsl/third_party/gpus/cuda_configure.bzl ++++ b/third_party/tsl/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204.1-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch b/third_party/modules/xla/20250204.1-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch new file mode 100644 index 00000000..4b6a65ce --- /dev/null +++ b/third_party/modules/xla/20250204.1-6789523/patches/0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch @@ -0,0 +1,26 @@ +From 4db5de34f70d991fedbe28915c8239b97ba7a064 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Mon, 18 Mar 2024 17:17:34 +0100 +Subject: [PATCH 3/3] [PJRT C API] Ensure C compliance for Profiler Extension + +--- + xla/pjrt/c/pjrt_c_api_profiler_extension.h | 2 ++ + 1 file changed, 2 insertions(+) + +diff --git a/xla/pjrt/c/pjrt_c_api_profiler_extension.h b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +index c821916ad..89a596123 100644 +--- a/xla/pjrt/c/pjrt_c_api_profiler_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_profiler_extension.h +@@ -16,8 +16,10 @@ limitations under the License. + #ifndef XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + #define XLA_PJRT_C_PJRT_C_API_PROFILER_EXTENSION_H_ + ++#ifdef __cplusplus + #include + #include ++#endif + + #include "xla/backends/profiler/plugin/profiler_c_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250204.1-6789523/source.json b/third_party/modules/xla/20250204.1-6789523/source.json new file mode 100644 index 00000000..ab3abf73 --- /dev/null +++ b/third_party/modules/xla/20250204.1-6789523/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "xla-6789523d0a42c2cf83309a263f417122c4c63811", + "url": "https://github.com/openxla/xla/archive/6789523d0a42c2cf83309a263f417122c4c63811.tar.gz", + "integrity": "sha256-7z0MdXQBoGApSYpTvYjKFmg18tGkjJ3P/PPQ487cPc8=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.0.1.patch": "", + "0003-PJRT-C-API-Ensure-C-compliance-for-Profiler-Extensio.patch": "" + } +} diff --git a/third_party/modules/xla/20250317.0-71c67e2/MODULE.bazel b/third_party/modules/xla/20250317.0-71c67e2/MODULE.bazel new file mode 100644 index 00000000..19a58980 --- /dev/null +++ b/third_party/modules/xla/20250317.0-71c67e2/MODULE.bazel @@ -0,0 +1,37 @@ +module( + name = "xla", + version = "20250317.0-71c67e2", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl", "python_version_repo") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "llvm-raw", + "stablehlo", +) + +llvm_configure = use_repo_rule("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") +llvm_configure(name = "llvm-project") diff --git a/third_party/modules/xla/20250317.0-71c67e2/overlay/MODULE.bazel b/third_party/modules/xla/20250317.0-71c67e2/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250317.0-71c67e2/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250317.0-71c67e2/overlay/tsl.bzl b/third_party/modules/xla/20250317.0-71c67e2/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250317.0-71c67e2/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250317.0-71c67e2/overlay/workspace.bzl b/third_party/modules/xla/20250317.0-71c67e2/overlay/workspace.bzl new file mode 100644 index 00000000..192e06f8 --- /dev/null +++ b/third_party/modules/xla/20250317.0-71c67e2/overlay/workspace.bzl @@ -0,0 +1,60 @@ +load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("//third_party/triton:workspace.bzl", triton = "repo") +load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + pybind11_bazel() + triton() + llvm("llvm-raw") + stablehlo() + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "//third_party/grpc:generate_cc_env_fix.patch", + "//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", + "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250317.0-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250317.0-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/modules/xla/20250317.0-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250317.0-71c67e2/source.json b/third_party/modules/xla/20250317.0-71c67e2/source.json new file mode 100644 index 00000000..c239d180 --- /dev/null +++ b/third_party/modules/xla/20250317.0-71c67e2/source.json @@ -0,0 +1,14 @@ +{ + "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", + "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", + "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "" + } +} diff --git a/third_party/modules/xla/20250317.1-71c67e2/MODULE.bazel b/third_party/modules/xla/20250317.1-71c67e2/MODULE.bazel new file mode 100644 index 00000000..516cef14 --- /dev/null +++ b/third_party/modules/xla/20250317.1-71c67e2/MODULE.bazel @@ -0,0 +1,37 @@ +module( + name = "xla", + version = "20250317.1-71c67e2", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl", "python_version_repo") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "llvm-raw", + "stablehlo", +) + +llvm_configure = use_repo_rule("@llvm-raw//utils/bazel:configure.bzl", "llvm_configure") +llvm_configure(name = "llvm-project") diff --git a/third_party/modules/xla/20250317.1-71c67e2/overlay/MODULE.bazel b/third_party/modules/xla/20250317.1-71c67e2/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250317.1-71c67e2/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250317.1-71c67e2/overlay/tsl.bzl b/third_party/modules/xla/20250317.1-71c67e2/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250317.1-71c67e2/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250317.1-71c67e2/overlay/workspace.bzl b/third_party/modules/xla/20250317.1-71c67e2/overlay/workspace.bzl new file mode 100644 index 00000000..192e06f8 --- /dev/null +++ b/third_party/modules/xla/20250317.1-71c67e2/overlay/workspace.bzl @@ -0,0 +1,60 @@ +load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("//third_party/triton:workspace.bzl", triton = "repo") +load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + pybind11_bazel() + triton() + llvm("llvm-raw") + stablehlo() + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "//third_party/grpc:generate_cc_env_fix.patch", + "//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", + "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250317.1-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250317.1-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/modules/xla/20250317.1-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch new file mode 100644 index 00000000..58e3874d --- /dev/null +++ b/third_party/modules/xla/20250317.1-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -0,0 +1,131 @@ +From 367df40470c00b9a4f83e3c5bc5553e7b0878351 Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Wed, 5 Feb 2025 19:25:03 +0100 +Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt + +PR: https://github.com/openxla/xla/pull/13420 +--- + xla/pjrt/c/BUILD | 5 ++++ + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 16 ++++++++++++ + xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 35 +++++++++++++++++++++++++-- + 3 files changed, 54 insertions(+), 2 deletions(-) + +diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD +index ad2ed95bce..0e7c35c30f 100644 +--- a/xla/pjrt/c/BUILD ++++ b/xla/pjrt/c/BUILD +@@ -69,7 +69,12 @@ cc_library( + ":pjrt_c_api_wrapper_impl", + "//xla/ffi:execution_context", + "//xla/ffi:type_id_registry", ++ "//xla/ffi:ffi_api", ++ "//xla/ffi/api:c_api", ++ "//xla/ffi/api:ffi", ++ "//xla/service:custom_call_target_registry", + "@com_google_absl//absl/status", ++ "@com_google_absl//absl/strings:str_format", + ], + ) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index a33bd4aa9c..3309194538 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -66,12 +66,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); + // Adds a user data to the execute context. + typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); + ++struct PJRT_FFI_Register_Handler_Args { ++ size_t struct_size; ++ const char* target_name; ++ size_t target_name_size; ++ int api_version; // 0 for an untyped call, 1 -- for typed ++ void* handler; ++ const char* platform_name; ++ size_t platform_name_size; ++}; ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); ++ ++// Registers an FFI call handler for a specific platform. ++typedef PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args); ++ + typedef struct PJRT_FFI_Extension { + size_t struct_size; + PJRT_Extension_Type type; + PJRT_Extension_Base* next; + PJRT_FFI_TypeID_Register* type_id_register; + PJRT_FFI_UserData_Add* user_data_add; ++ PJRT_FFI_Register_Handler* register_handler; + } PJRT_FFI; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +index 0375b39d0b..3527a0756e 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc ++++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +@@ -13,15 +13,20 @@ See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" ++#include + + #include "absl/status/status.h" ++#include "absl/strings/str_format.h" ++#include "xla/ffi/api/c_api.h" ++#include "xla/ffi/api/ffi.h" + #include "xla/ffi/execution_context.h" +-#include "xla/ffi/type_id_registry.h" ++ #include "xla/ffi/type_id_registry.h" ++#include "xla/ffi/ffi_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" + #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" + #include "xla/pjrt/c/pjrt_c_api_helpers.h" + #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/service/custom_call_target_registry.h" + + namespace pjrt { + +@@ -55,6 +60,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { + return nullptr; + } + ++static PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args) { ++ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( ++ "PJRT_FFI_Register_Handler_Args", ++ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); ++ std::string target_name(args->target_name, args->target_name_size); ++ std::string platform_name(args->platform_name, args->platform_name_size); ++ switch (args->api_version) { ++ case 0: ++ xla::CustomCallTargetRegistry::Global()->Register( ++ target_name, args->handler, platform_name); ++ return nullptr; ++ case 1: ++ xla::ffi::Ffi::RegisterStaticHandler( ++ xla::ffi::GetXlaFfiApi(), target_name, platform_name, ++ reinterpret_cast(args->handler)); ++ return nullptr; ++ default: ++ return new PJRT_Error{absl::UnimplementedError( ++ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " ++ "Supported versions are 0 and 1.", ++ args->api_version))}; ++ } ++} ++ + PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + return { + /*struct_size=*/PJRT_FFI_Extension_STRUCT_SIZE, +@@ -62,6 +92,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + /*next=*/next, + /*type_id_register=*/PJRT_FFI_TypeID_Register, + /*user_data_add=*/PJRT_FFI_UserData_Add, ++ /*register_handler=*/PJRT_FFI_Register_Handler, + }; + } + +-- +2.43.0 diff --git a/third_party/modules/xla/20250317.1-71c67e2/source.json b/third_party/modules/xla/20250317.1-71c67e2/source.json new file mode 100644 index 00000000..5577d247 --- /dev/null +++ b/third_party/modules/xla/20250317.1-71c67e2/source.json @@ -0,0 +1,15 @@ +{ + "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", + "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", + "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", + "overlay": { + "tsl.bzl": "", + "workspace.bzl": "", + "MODULE.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" + } +} diff --git a/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel b/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel new file mode 100644 index 00000000..042d899c --- /dev/null +++ b/third_party/modules/xla/20250317.2-71c67e2/MODULE.bazel @@ -0,0 +1,37 @@ +module( + name = "xla", + version = "20250317.2-71c67e2", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.17.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.29.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") + +tsl = use_extension("//:tsl.bzl", "tsl") +use_repo(tsl, "tsl", "python_version_repo") + +xla_workspace = use_extension("//:workspace.bzl", "xla_workspace") +use_repo( + xla_workspace, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "llvm-raw", + "stablehlo", +) + +llvm_configure = use_extension("//:llvm.bzl", "llvm_configure") +use_repo(llvm_configure, "llvm-project") diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/MODULE.bazel b/third_party/modules/xla/20250317.2-71c67e2/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250317.2-71c67e2/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/llvm.bzl b/third_party/modules/xla/20250317.2-71c67e2/overlay/llvm.bzl new file mode 100644 index 00000000..57e77361 --- /dev/null +++ b/third_party/modules/xla/20250317.2-71c67e2/overlay/llvm.bzl @@ -0,0 +1,13 @@ +load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_configure_impl(mctx): + _llvm_configure(name = "llvm-project") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm_configure = module_extension( + implementation = _llvm_configure_impl, +) diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/tsl.bzl b/third_party/modules/xla/20250317.2-71c67e2/overlay/tsl.bzl new file mode 100644 index 00000000..dc25c9d6 --- /dev/null +++ b/third_party/modules/xla/20250317.2-71c67e2/overlay/tsl.bzl @@ -0,0 +1,19 @@ +load("//third_party:repo.bzl", "tf_vendored") +load("//third_party/py:python_init_repositories.bzl", "python_init_repositories") + +def _tsl_impl(mctx): + python_init_repositories( + requirements = { + "3.11": "//:requirements_lock_3_11.txt", + }, + ) + tf_vendored(name = "tsl", relpath = "third_party/tsl") + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["tsl"], + root_module_direct_dev_deps = [], + ) + +tsl = module_extension( + implementation = _tsl_impl, +) diff --git a/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl b/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl new file mode 100644 index 00000000..192e06f8 --- /dev/null +++ b/third_party/modules/xla/20250317.2-71c67e2/overlay/workspace.bzl @@ -0,0 +1,60 @@ +load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("//third_party/triton:workspace.bzl", triton = "repo") +load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _xla_workspace_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + pybind11_bazel() + triton() + llvm("llvm-raw") + stablehlo() + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "//third_party/grpc:generate_cc_env_fix.patch", + "//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", + "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla_workspace = module_extension( + implementation = _xla_workspace_impl, +) diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/modules/xla/20250317.2-71c67e2/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250317.2-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250317.2-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch new file mode 100644 index 00000000..58e3874d --- /dev/null +++ b/third_party/modules/xla/20250317.2-71c67e2/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -0,0 +1,131 @@ +From 367df40470c00b9a4f83e3c5bc5553e7b0878351 Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Wed, 5 Feb 2025 19:25:03 +0100 +Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt + +PR: https://github.com/openxla/xla/pull/13420 +--- + xla/pjrt/c/BUILD | 5 ++++ + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 16 ++++++++++++ + xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 35 +++++++++++++++++++++++++-- + 3 files changed, 54 insertions(+), 2 deletions(-) + +diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD +index ad2ed95bce..0e7c35c30f 100644 +--- a/xla/pjrt/c/BUILD ++++ b/xla/pjrt/c/BUILD +@@ -69,7 +69,12 @@ cc_library( + ":pjrt_c_api_wrapper_impl", + "//xla/ffi:execution_context", + "//xla/ffi:type_id_registry", ++ "//xla/ffi:ffi_api", ++ "//xla/ffi/api:c_api", ++ "//xla/ffi/api:ffi", ++ "//xla/service:custom_call_target_registry", + "@com_google_absl//absl/status", ++ "@com_google_absl//absl/strings:str_format", + ], + ) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index a33bd4aa9c..3309194538 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -66,12 +66,28 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); + // Adds a user data to the execute context. + typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); + ++struct PJRT_FFI_Register_Handler_Args { ++ size_t struct_size; ++ const char* target_name; ++ size_t target_name_size; ++ int api_version; // 0 for an untyped call, 1 -- for typed ++ void* handler; ++ const char* platform_name; ++ size_t platform_name_size; ++}; ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, handler); ++ ++// Registers an FFI call handler for a specific platform. ++typedef PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args); ++ + typedef struct PJRT_FFI_Extension { + size_t struct_size; + PJRT_Extension_Type type; + PJRT_Extension_Base* next; + PJRT_FFI_TypeID_Register* type_id_register; + PJRT_FFI_UserData_Add* user_data_add; ++ PJRT_FFI_Register_Handler* register_handler; + } PJRT_FFI; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +index 0375b39d0b..3527a0756e 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc ++++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +@@ -13,15 +13,20 @@ See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" ++#include + + #include "absl/status/status.h" ++#include "absl/strings/str_format.h" ++#include "xla/ffi/api/c_api.h" ++#include "xla/ffi/api/ffi.h" + #include "xla/ffi/execution_context.h" +-#include "xla/ffi/type_id_registry.h" ++ #include "xla/ffi/type_id_registry.h" ++#include "xla/ffi/ffi_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" + #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" + #include "xla/pjrt/c/pjrt_c_api_helpers.h" + #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/service/custom_call_target_registry.h" + + namespace pjrt { + +@@ -55,6 +60,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { + return nullptr; + } + ++static PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args) { ++ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( ++ "PJRT_FFI_Register_Handler_Args", ++ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); ++ std::string target_name(args->target_name, args->target_name_size); ++ std::string platform_name(args->platform_name, args->platform_name_size); ++ switch (args->api_version) { ++ case 0: ++ xla::CustomCallTargetRegistry::Global()->Register( ++ target_name, args->handler, platform_name); ++ return nullptr; ++ case 1: ++ xla::ffi::Ffi::RegisterStaticHandler( ++ xla::ffi::GetXlaFfiApi(), target_name, platform_name, ++ reinterpret_cast(args->handler)); ++ return nullptr; ++ default: ++ return new PJRT_Error{absl::UnimplementedError( ++ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " ++ "Supported versions are 0 and 1.", ++ args->api_version))}; ++ } ++} ++ + PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + return { + /*struct_size=*/PJRT_FFI_Extension_STRUCT_SIZE, +@@ -62,6 +92,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + /*next=*/next, + /*type_id_register=*/PJRT_FFI_TypeID_Register, + /*user_data_add=*/PJRT_FFI_UserData_Add, ++ /*register_handler=*/PJRT_FFI_Register_Handler, + }; + } + +-- +2.43.0 diff --git a/third_party/modules/xla/20250317.2-71c67e2/source.json b/third_party/modules/xla/20250317.2-71c67e2/source.json new file mode 100644 index 00000000..9d579abc --- /dev/null +++ b/third_party/modules/xla/20250317.2-71c67e2/source.json @@ -0,0 +1,16 @@ +{ + "strip_prefix": "xla-71c67e2a4f40267115a0d4ea7c36748bbe7e750e", + "url": "https://github.com/openxla/xla/archive/71c67e2a4f40267115a0d4ea7c36748bbe7e750e.tar.gz", + "integrity": "sha256-j6D1MC7+WsbZ+Ve3hPmDlCzZX1yV6RIP2BWDgQcbcYc=", + "overlay": { + "llvm.bzl": "", + "MODULE.bazel": "", + "tsl.bzl": "", + "workspace.bzl": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "" + } +} diff --git a/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel b/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel new file mode 100644 index 00000000..74c9eaad --- /dev/null +++ b/third_party/modules/xla/20250527.0-cb67f2f/MODULE.bazel @@ -0,0 +1,52 @@ +module( + name = "xla", + version = "20250527.0-cb67f2f", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.39.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") +bazel_dep(name = "rules_shell", version = "0.4.1") +bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") + +workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") +use_repo( + workspace_private, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "python_version_repo", + "tsl", +) + +workspace_public = use_extension("//:xla.bzl", "xla") +use_repo( + workspace_public, + "llvm-raw", + "stablehlo", + "triton", +) + +llvm = use_extension("//:llvm.bzl", "llvm") +llvm.configure( + targets = [ + "AArch64", + "AMDGPU", + "NVPTX", + "X86", + ], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/MODULE.bazel b/third_party/modules/xla/20250527.0-cb67f2f/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250527.0-cb67f2f/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/llvm.bzl b/third_party/modules/xla/20250527.0-cb67f2f/overlay/llvm.bzl new file mode 100644 index 00000000..b4a2fe4f --- /dev/null +++ b/third_party/modules/xla/20250527.0-cb67f2f/overlay/llvm.bzl @@ -0,0 +1,30 @@ +load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list( + default = [], + ), + }, + ), + }, +) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl b/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl new file mode 100644 index 00000000..b234d122 --- /dev/null +++ b/third_party/modules/xla/20250527.0-cb67f2f/overlay/workspace_private.bzl @@ -0,0 +1,71 @@ +load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/py:python_repo.bzl", "python_repository") +load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("//third_party/triton:workspace.bzl", triton = "repo") +load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("//third_party:repo.bzl", "tf_vendored") +load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _workspace_private_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_vendored(name = "tsl", relpath = "third_party/tsl") + pybind11_bazel() + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "//third_party/grpc:generate_cc_env_fix.patch", + "//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", + "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + python_repository( + name = "python_version_repo", + requirements_versions = ["3.11"], + requirements_locks = ["//:requirements_lock_3_11.txt"], + local_wheel_workspaces = [], + local_wheel_dist_folder = None, + default_python_version = None, + local_wheel_inclusion_list = ["*"], + local_wheel_exclusion_list = [], + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +workspace_private = module_extension( + implementation = _workspace_private_impl, +) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/overlay/xla.bzl b/third_party/modules/xla/20250527.0-cb67f2f/overlay/xla.bzl new file mode 100644 index 00000000..f14bf2a9 --- /dev/null +++ b/third_party/modules/xla/20250527.0-cb67f2f/overlay/xla.bzl @@ -0,0 +1,17 @@ +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/triton:workspace.bzl", triton = "repo") + +def _xla_impl(mctx): + triton() + llvm("llvm-raw") + stablehlo() + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla = module_extension( + implementation = _xla_impl, +) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/modules/xla/20250527.0-cb67f2f/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch new file mode 100644 index 00000000..938ef401 --- /dev/null +++ b/third_party/modules/xla/20250527.0-cb67f2f/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -0,0 +1,135 @@ +From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Tue, 27 May 2025 11:48:17 +0200 +Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt + +PR: https://github.com/openxla/xla/pull/13420 +--- + xla/pjrt/c/BUILD | 5 +++++ + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ + xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- + 3 files changed, 57 insertions(+), 1 deletion(-) + +diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD +index 79f18fa0bc..0f33dd8a6e 100644 +--- a/xla/pjrt/c/BUILD ++++ b/xla/pjrt/c/BUILD +@@ -69,8 +69,13 @@ cc_library( + ":pjrt_c_api_wrapper_impl", + "//xla/ffi:execution_context", + "//xla/ffi:type_id_registry", ++ "//xla/ffi:ffi_api", ++ "//xla/ffi/api:c_api", ++ "//xla/ffi/api:ffi", ++ "//xla/service:custom_call_target_registry", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/strings:str_format", + ], + ) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index 995a2c7e50..b8f10bc2f7 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); + // Adds a user data to the execute context. + typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); + ++typedef enum PJRT_FFI_Handler_TraitsBits { ++ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, ++} PJRT_FFI_Handler_TraitsBits; ++ ++struct PJRT_FFI_Register_Handler_Args { ++ size_t struct_size; ++ const char* target_name; ++ size_t target_name_size; ++ int api_version; // 0 for an untyped call, 1 -- for typed ++ void* handler; ++ const char* platform_name; ++ size_t platform_name_size; ++ PJRT_FFI_Handler_TraitsBits traits; ++}; ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); ++ ++// Registers an FFI call handler for a specific platform. ++typedef PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args); ++ + typedef struct PJRT_FFI_Extension { + PJRT_Extension_Base base; + PJRT_FFI_TypeID_Register* type_id_register; + PJRT_FFI_UserData_Add* user_data_add; ++ PJRT_FFI_Register_Handler* register_handler; + } PJRT_FFI; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +index 5fa88eab33..763270331b 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc ++++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" ++#include + + #include "absl/status/status.h" ++#include "absl/strings/str_format.h" + #include "absl/strings/string_view.h" ++#include "xla/ffi/api/c_api.h" + #include "xla/ffi/execution_context.h" + #include "xla/ffi/type_id_registry.h" ++#include "xla/ffi/ffi_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" + #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" + #include "xla/pjrt/c/pjrt_c_api_helpers.h" + #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/service/custom_call_target_registry.h" + + namespace pjrt { + +@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { + return nullptr; + } + ++static PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args) { ++ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( ++ "PJRT_FFI_Register_Handler_Args", ++ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); ++ std::string target_name(args->target_name, args->target_name_size); ++ std::string platform_name(args->platform_name, args->platform_name_size); ++ switch (args->api_version) { ++ case 0: ++ xla::CustomCallTargetRegistry::Global()->Register( ++ target_name, args->handler, platform_name); ++ return nullptr; ++ case 1: ++ xla::ffi::Ffi::RegisterStaticHandler( ++ xla::ffi::GetXlaFfiApi(), target_name, platform_name, ++ reinterpret_cast(args->handler)); ++ return nullptr; ++ default: ++ return new PJRT_Error{absl::UnimplementedError( ++ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " ++ "Supported versions are 0 and 1.", ++ args->api_version))}; ++ } ++} ++ + PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + return { + PJRT_Extension_Base{ +@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + }, + /*type_id_register=*/PJRT_FFI_TypeID_Register, + /*user_data_add=*/PJRT_FFI_UserData_Add, ++ /*register_handler=*/PJRT_FFI_Register_Handler, + }; + } + +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/modules/xla/20250527.0-cb67f2f/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch b/third_party/modules/xla/20250527.0-cb67f2f/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch new file mode 100644 index 00000000..4173db86 --- /dev/null +++ b/third_party/modules/xla/20250527.0-cb67f2f/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch @@ -0,0 +1,186 @@ +From 2a62bc5df9774810313142eb0c9390aab3cd18f8 Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Thu, 29 May 2025 08:02:32 +0200 +Subject: [PATCH] Revert "Add optional allowOtherDialects field to + stablehlo.serialize_portable_artifact." + +Commit: https://github.com/openxla/xla/commit/e7137a383809a24875a95237b1d1f6485acdf710 + +Issue: C does not support default arguments with ZigTranslateC +--- + third_party/stablehlo/temporary.patch | 151 ++++---------------------- + 1 file changed, 21 insertions(+), 130 deletions(-) + +diff --git a/third_party/stablehlo/temporary.patch b/third_party/stablehlo/temporary.patch +index 6e1fd159f9..d17c141b18 100755 +--- a/third_party/stablehlo/temporary.patch ++++ b/third_party/stablehlo/temporary.patch +@@ -1,3 +1,23 @@ ++diff --ruN a/stablehlo/stablehlo/dialect/StablehloOps.cpp b/stablehlo/stablehlo/dialect/StablehloOps.cpp ++--- stablehlo/stablehlo/dialect/StablehloOps.cpp +++++ stablehlo/stablehlo/dialect/StablehloOps.cpp ++@@ -511,12 +511,10 @@ ++ void CustomCallOp::getEffects( ++ SmallVectorImpl>& ++ effects) { ++- // Note: `has_side_effect` "defaults" to `false` but isn't required to exist. ++- // This semantic contradiction means, in practical terms, that the attribute ++- // won't exist by default but should be *treated* as `false` if missing. ++- // `getHasSideEffect()` abstracts this nuance away and returns `false` by ++- // default, whereas `getHasSideEffectAttr()` may return a null attribute. ++- if (!getHasSideEffect()) return; +++ // CustomCall has "all possible effects" unless the has_side_effect is present +++ // and set to false. +++ auto hasSideEffect = getHasSideEffectAttr(); +++ if (hasSideEffect && !hasSideEffect.getValue()) return; ++ effects.emplace_back(MemoryEffects::Allocate::get()); ++ effects.emplace_back(MemoryEffects::Free::get()); ++ effects.emplace_back(MemoryEffects::Write::get()); + diff --ruN a/stablehlo/stablehlo/dialect/StablehloOps.h b/stablehlo/stablehlo/dialect/StablehloOps.h + --- stablehlo/stablehlo/dialect/StablehloOps.h + +++ stablehlo/stablehlo/dialect/StablehloOps.h +@@ -82,135 +102,6 @@ diff --ruN a/stablehlo/stablehlo/dialect/StablehloOps.td b/stablehlo/stablehlo/d + ]> { + let summary = "Recv operation"; + let description = [{ +-diff --ruN a/stablehlo/stablehlo/dialect/TypeInference.cpp b/stablehlo/stablehlo/dialect/TypeInference.cpp +---- stablehlo/stablehlo/dialect/TypeInference.cpp +-+++ stablehlo/stablehlo/dialect/TypeInference.cpp +-@@ -879,7 +879,8 @@ +- +- auto replicaIds = replicaGroups.getValues(); +- +-- llvm::SmallSet replicaIdsSeen; +-+ // Large programs can have many replicas, use a set with efficient lookup. +-+ llvm::DenseSet replicaIdsSeen; +- for (int64_t replicaId : replicaIds) { +- // Replica groups are stored in a 2D tensor. If the op supports non-uniform +- // groups, null replica IDs are stored as -1. +-@@ -1841,6 +1842,7 @@ +- /*allGroupsMustHaveSameSize=*/true, +- /*useGlobalDeviceIds=*/false, splitCount))) +- return failure(); +-+ +- for (const Value& operand : operands) { +- auto operandType = cast(operand.getType()); +- +-@@ -3562,6 +3564,19 @@ +- DenseIntElementsAttr replicaGroups, +- int64_t channelId, bool useGlobalDeviceIds, +- ValueRange results) { +-+ // all_gather_i3, all_gather_c2, all_gather_c4 +-+ if (failed(verifyReplicaGroups(location, replicaGroups, +-+ /*allGroupsMustHaveSameSize=*/true, +-+ useGlobalDeviceIds, +-+ /*expectedGroupSize=*/std::nullopt))) +-+ return failure(); +-+ +-+ // all_gather_c5 +-+ if (useGlobalDeviceIds && channelId < 0) +-+ return emitOptionalError( +-+ location, +-+ "channel_id cannot be negative when useGlobalDeviceIds is set"); +-+ +- for (const auto& [operand, result] : llvm::zip(operands, results)) { +- auto operandType = cast(operand.getType()); +- auto resultType = cast(result.getType()); +-@@ -3576,19 +3591,6 @@ +- return emitOptionalError( +- location, +- "dimension size of operand at 'all_gather_dim' cannot be zero"); +-- +-- // all_gather_i3, all_gather_c2, all_gather_c4 +-- if (failed(verifyReplicaGroups(location, replicaGroups, +-- /*allGroupsMustHaveSameSize=*/true, +-- useGlobalDeviceIds, +-- /*expectedGroupSize=*/std::nullopt))) +-- return failure(); +-- +-- // all_gather_c5 +-- if (useGlobalDeviceIds && channelId < 0) +-- return emitOptionalError( +-- location, +-- "channel_id cannot be negative when useGlobalDeviceIds is set"); +- +- // all_gather_c6 +- if (resultType.getRank() != operandType.getRank()) +-@@ -3788,7 +3790,7 @@ +- "but instead it is of rank ", replicaGroupType.getRank()); +- +- auto replicaIds = replicaGroups.getValues(); +-- llvm::SmallSet replicaIdsSeen; +-+ llvm::DenseSet replicaIdsSeen; +- for (int64_t replicaId : replicaIds) { +- // collective_broadcast_c2 +- // We only check that is is not negative, as it is impossible +-diff --ruN a/stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp +---- stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp +-+++ stablehlo/stablehlo/integrations/c/StablehloDialectApi.cpp +-@@ -78,10 +78,11 @@ +- +- MlirLogicalResult stablehloSerializePortableArtifactFromModule( +- MlirModule moduleStr, MlirStringRef targetVersion, +-- MlirStringCallback callback, void *userData) { +-+ MlirStringCallback callback, void *userData, bool allowOtherDialects) { +- mlir::detail::CallbackOstream stream(callback, userData); +- if (failed(mlir::stablehlo::serializePortableArtifact( +-- unwrap(moduleStr), unwrap(targetVersion), stream))) +-+ unwrap(moduleStr), unwrap(targetVersion), stream, +-+ allowOtherDialects))) +- return mlirLogicalResultFailure(); +- return mlirLogicalResultSuccess(); +- } +-diff --ruN a/stablehlo/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/stablehlo/integrations/c/StablehloDialectApi.h +---- stablehlo/stablehlo/integrations/c/StablehloDialectApi.h +-+++ stablehlo/stablehlo/integrations/c/StablehloDialectApi.h +-@@ -92,7 +92,8 @@ +- stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, +- MlirStringRef targetVersion, +- MlirStringCallback callback, +-- void* userData); +-+ void* userData, +-+ bool allowOtherDialects = false); +- +- // Read a StableHLO program from a portable artifact, returning the module as +- // MLIR bytecode. Note, this bytecode returned is not a portable artifact, +-diff --ruN a/stablehlo/stablehlo/integrations/python/StablehloApi.cpp b/stablehlo/stablehlo/integrations/python/StablehloApi.cpp +---- stablehlo/stablehlo/integrations/python/StablehloApi.cpp +-+++ stablehlo/stablehlo/integrations/python/StablehloApi.cpp +-@@ -102,20 +102,22 @@ +- // +- m.def( +- "serialize_portable_artifact", +-- [](MlirModule module, std::string_view target) -> nb::bytes { +-+ [](MlirModule module, std::string_view target, +-+ bool allowOtherDialects) -> nb::bytes { +- StringWriterHelper accumulator; +- if (mlirLogicalResultIsFailure( +- stablehloSerializePortableArtifactFromModule( +- module, toMlirStringRef(target), +- accumulator.getMlirStringCallback(), +-- accumulator.getUserData()))) { +-+ accumulator.getUserData(), allowOtherDialects))) { +- throw nb::value_error("failed to serialize module"); +- } +- +- std::string serialized = accumulator.toString(); +- return nb::bytes(serialized.data(), serialized.size()); +- }, +-- nb::arg("module"), nb::arg("target")); +-+ nb::arg("module"), nb::arg("target"), +-+ nb::arg("allow_other_dialects") = false); +- +- m.def( +- "deserialize_portable_artifact", + diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir + --- stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir + +++ stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless.mlir +@@ -223,4 +114,4 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_convert_to_signless. + %7 = builtin.unrealized_conversion_cast %6 : memref to memref + func.return %7 : memref + } +- ++ +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/modules/xla/20250527.0-cb67f2f/source.json b/third_party/modules/xla/20250527.0-cb67f2f/source.json new file mode 100644 index 00000000..86bdd83a --- /dev/null +++ b/third_party/modules/xla/20250527.0-cb67f2f/source.json @@ -0,0 +1,17 @@ +{ + "strip_prefix": "xla-cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3", + "url": "https://github.com/openxla/xla/archive/cb67f2f7ce4787f63f5fc80dc5c30cd3dee8f4e3.tar.gz", + "integrity": "sha256-SDAAOY6cjcCQ5e1JMob5G4+BYHk7spDb5zZEDrVeA4I=", + "overlay": { + "llvm.bzl": "", + "MODULE.bazel": "", + "workspace_private.bzl": "", + "xla.bzl": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", + "0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch": "" + } +} diff --git a/third_party/modules/xla/20250612.0-6e48cbb/MODULE.bazel b/third_party/modules/xla/20250612.0-6e48cbb/MODULE.bazel new file mode 100644 index 00000000..46eb7d18 --- /dev/null +++ b/third_party/modules/xla/20250612.0-6e48cbb/MODULE.bazel @@ -0,0 +1,52 @@ +module( + name = "xla", + version = "20250612.0-6e48cbb", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.39.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") +bazel_dep(name = "rules_shell", version = "0.4.1") +bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") + +workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") +use_repo( + workspace_private, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "python_version_repo", + "tsl", +) + +workspace_public = use_extension("//:xla.bzl", "xla") +use_repo( + workspace_public, + "llvm-raw", + "stablehlo", + "triton", +) + +llvm = use_extension("//:llvm.bzl", "llvm") +llvm.configure( + targets = [ + "AArch64", + "AMDGPU", + "NVPTX", + "X86", + ], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/MODULE.bazel b/third_party/modules/xla/20250612.0-6e48cbb/overlay/MODULE.bazel new file mode 100644 index 00000000..74c9eaad --- /dev/null +++ b/third_party/modules/xla/20250612.0-6e48cbb/overlay/MODULE.bazel @@ -0,0 +1,52 @@ +module( + name = "xla", + version = "20250527.0-cb67f2f", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.39.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") +bazel_dep(name = "rules_shell", version = "0.4.1") +bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") + +workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") +use_repo( + workspace_private, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "python_version_repo", + "tsl", +) + +workspace_public = use_extension("//:xla.bzl", "xla") +use_repo( + workspace_public, + "llvm-raw", + "stablehlo", + "triton", +) + +llvm = use_extension("//:llvm.bzl", "llvm") +llvm.configure( + targets = [ + "AArch64", + "AMDGPU", + "NVPTX", + "X86", + ], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/llvm.bzl b/third_party/modules/xla/20250612.0-6e48cbb/overlay/llvm.bzl new file mode 100644 index 00000000..b4a2fe4f --- /dev/null +++ b/third_party/modules/xla/20250612.0-6e48cbb/overlay/llvm.bzl @@ -0,0 +1,30 @@ +load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list( + default = [], + ), + }, + ), + }, +) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/workspace_private.bzl b/third_party/modules/xla/20250612.0-6e48cbb/overlay/workspace_private.bzl new file mode 100644 index 00000000..b234d122 --- /dev/null +++ b/third_party/modules/xla/20250612.0-6e48cbb/overlay/workspace_private.bzl @@ -0,0 +1,71 @@ +load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/py:python_repo.bzl", "python_repository") +load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("//third_party/triton:workspace.bzl", triton = "repo") +load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls") +load("//third_party:repo.bzl", "tf_vendored") +load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _workspace_private_impl(mctx): + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_vendored(name = "tsl", relpath = "third_party/tsl") + pybind11_bazel() + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "b956598d8cbe168b5ee717b5dafa56563eb5201a947856a6688bbeac9cac4e1f", + strip_prefix = "grpc-b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd", + system_build_file = "//third_party/systemlibs:grpc.BUILD", + patch_file = [ + "//third_party/grpc:generate_cc_env_fix.patch", + "//third_party/grpc:register_go_toolchain.patch", + ], + system_link_files = { + "//third_party/systemlibs:BUILD.bazel": "bazel/BUILD.bazel", + "//third_party/systemlibs:grpc.BUILD": "src/compiler/BUILD", + "//third_party/systemlibs:grpc.bazel.grpc_deps.bzl": "bazel/grpc_deps.bzl", + "//third_party/systemlibs:grpc.bazel.grpc_extra_deps.bzl": "bazel/grpc_extra_deps.bzl", + "//third_party/systemlibs:grpc.bazel.cc_grpc_library.bzl": "bazel/cc_grpc_library.bzl", + "//third_party/systemlibs:grpc.bazel.generate_cc.bzl": "bazel/generate_cc.bzl", + "//third_party/systemlibs:grpc.bazel.protobuf.bzl": "bazel/protobuf.bzl", + }, + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/b54a5b338637f92bfcf4b0bc05e0f57a5fd8fadd.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["//third_party/protobuf:protobuf.patch"], + sha256 = "f66073dee0bc159157b0bd7f502d7d1ee0bc76b3c1eac9836927511bdc4b3fc1", + strip_prefix = "protobuf-3.21.9", + system_build_file = "//third_party/systemlibs:protobuf.BUILD", + system_link_files = { + "//third_party/systemlibs:protobuf.bzl": "protobuf.bzl", + "//third_party/systemlibs:protobuf_deps.bzl": "protobuf_deps.bzl", + }, + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/v3.21.9.zip"), + ) + python_repository( + name = "python_version_repo", + requirements_versions = ["3.11"], + requirements_locks = ["//:requirements_lock_3_11.txt"], + local_wheel_workspaces = [], + local_wheel_dist_folder = None, + default_python_version = None, + local_wheel_inclusion_list = ["*"], + local_wheel_exclusion_list = [], + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +workspace_private = module_extension( + implementation = _workspace_private_impl, +) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/overlay/xla.bzl b/third_party/modules/xla/20250612.0-6e48cbb/overlay/xla.bzl new file mode 100644 index 00000000..f14bf2a9 --- /dev/null +++ b/third_party/modules/xla/20250612.0-6e48cbb/overlay/xla.bzl @@ -0,0 +1,17 @@ +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/triton:workspace.bzl", triton = "repo") + +def _xla_impl(mctx): + triton() + llvm("llvm-raw") + stablehlo() + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla = module_extension( + implementation = _xla_impl, +) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250612.0-6e48cbb/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/modules/xla/20250612.0-6e48cbb/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250612.0-6e48cbb/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250612.0-6e48cbb/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch new file mode 100644 index 00000000..938ef401 --- /dev/null +++ b/third_party/modules/xla/20250612.0-6e48cbb/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -0,0 +1,135 @@ +From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Tue, 27 May 2025 11:48:17 +0200 +Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt + +PR: https://github.com/openxla/xla/pull/13420 +--- + xla/pjrt/c/BUILD | 5 +++++ + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ + xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- + 3 files changed, 57 insertions(+), 1 deletion(-) + +diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD +index 79f18fa0bc..0f33dd8a6e 100644 +--- a/xla/pjrt/c/BUILD ++++ b/xla/pjrt/c/BUILD +@@ -69,8 +69,13 @@ cc_library( + ":pjrt_c_api_wrapper_impl", + "//xla/ffi:execution_context", + "//xla/ffi:type_id_registry", ++ "//xla/ffi:ffi_api", ++ "//xla/ffi/api:c_api", ++ "//xla/ffi/api:ffi", ++ "//xla/service:custom_call_target_registry", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/strings:str_format", + ], + ) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index 995a2c7e50..b8f10bc2f7 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); + // Adds a user data to the execute context. + typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); + ++typedef enum PJRT_FFI_Handler_TraitsBits { ++ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, ++} PJRT_FFI_Handler_TraitsBits; ++ ++struct PJRT_FFI_Register_Handler_Args { ++ size_t struct_size; ++ const char* target_name; ++ size_t target_name_size; ++ int api_version; // 0 for an untyped call, 1 -- for typed ++ void* handler; ++ const char* platform_name; ++ size_t platform_name_size; ++ PJRT_FFI_Handler_TraitsBits traits; ++}; ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); ++ ++// Registers an FFI call handler for a specific platform. ++typedef PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args); ++ + typedef struct PJRT_FFI_Extension { + PJRT_Extension_Base base; + PJRT_FFI_TypeID_Register* type_id_register; + PJRT_FFI_UserData_Add* user_data_add; ++ PJRT_FFI_Register_Handler* register_handler; + } PJRT_FFI; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +index 5fa88eab33..763270331b 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc ++++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" ++#include + + #include "absl/status/status.h" ++#include "absl/strings/str_format.h" + #include "absl/strings/string_view.h" ++#include "xla/ffi/api/c_api.h" + #include "xla/ffi/execution_context.h" + #include "xla/ffi/type_id_registry.h" ++#include "xla/ffi/ffi_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" + #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" + #include "xla/pjrt/c/pjrt_c_api_helpers.h" + #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/service/custom_call_target_registry.h" + + namespace pjrt { + +@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { + return nullptr; + } + ++static PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args) { ++ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( ++ "PJRT_FFI_Register_Handler_Args", ++ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); ++ std::string target_name(args->target_name, args->target_name_size); ++ std::string platform_name(args->platform_name, args->platform_name_size); ++ switch (args->api_version) { ++ case 0: ++ xla::CustomCallTargetRegistry::Global()->Register( ++ target_name, args->handler, platform_name); ++ return nullptr; ++ case 1: ++ xla::ffi::Ffi::RegisterStaticHandler( ++ xla::ffi::GetXlaFfiApi(), target_name, platform_name, ++ reinterpret_cast(args->handler)); ++ return nullptr; ++ default: ++ return new PJRT_Error{absl::UnimplementedError( ++ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " ++ "Supported versions are 0 and 1.", ++ args->api_version))}; ++ } ++} ++ + PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + return { + PJRT_Extension_Base{ +@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + }, + /*type_id_register=*/PJRT_FFI_TypeID_Register, + /*user_data_add=*/PJRT_FFI_UserData_Add, ++ /*register_handler=*/PJRT_FFI_Register_Handler, + }; + } + +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/modules/xla/20250612.0-6e48cbb/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch b/third_party/modules/xla/20250612.0-6e48cbb/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch new file mode 100644 index 00000000..27d52df5 --- /dev/null +++ b/third_party/modules/xla/20250612.0-6e48cbb/patches/0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch @@ -0,0 +1,1855 @@ +From 4c0819ac9fb9dfc6156ae4de83fb29e987ade780 Mon Sep 17 00:00:00 2001 +From: Corentin Godeau +Date: Mon, 16 Jun 2025 14:27:47 +0000 +Subject: [PATCH] Update Stablehlo patch + +--- + third_party/stablehlo/temporary.patch | 841 +++++++++++++------------- + 1 file changed, 432 insertions(+), 409 deletions(-) + mode change 100755 => 100644 third_party/stablehlo/temporary.patch + +diff --git a/third_party/stablehlo/temporary.patch b/third_party/stablehlo/temporary.patch +old mode 100755 +new mode 100644 +index f54e8341ba..6b25e78467 +--- a/third_party/stablehlo/temporary.patch ++++ b/third_party/stablehlo/temporary.patch +@@ -1,7 +1,47 @@ +-diff --ruN a/stablehlo/stablehlo/dialect/VhloAttrs.td b/stablehlo/stablehlo/dialect/VhloAttrs.td +---- stablehlo/stablehlo/dialect/VhloAttrs.td +-+++ stablehlo/stablehlo/dialect/VhloAttrs.td +-@@ -45,14 +45,6 @@ ++From c48aa0650bf27b81c42799f9218dd90779e36e3b Mon Sep 17 00:00:00 2001 ++From: Corentin Godeau ++Date: Mon, 16 Jun 2025 14:08:48 +0000 ++Subject: [PATCH] Remove default argument that is not valid C ++ ++--- ++ stablehlo/dialect/Version.h | 2 +- ++ stablehlo/dialect/VhloAttrs.td | 14 +- ++ stablehlo/dialect/VhloDialect.td | 1 + ++ stablehlo/dialect/VhloTypes.h | 24 +- ++ .../integrations/c/StablehloDialectApi.h | 2 +- ++ .../stablehlo_aggressive_folder.mlir | 37 +- ++ .../stablehlo_aggressive_simplification.mlir | 2 +- ++ .../transforms/stablehlo_refine_shapes.mlir | 10 +- ++ .../stablehlo_legalize_to_vhlo_mixed.mlir | 38 +- ++ .../tests/vhlo/vhlo_attributes_invalid.mlir | 13 +- ++ stablehlo/tools/StablehloTranslateMain.cpp | 8 +- ++ stablehlo/transforms/Passes.h | 7 +- ++ .../transforms/StablehloLegalizeToVhlo.cpp | 48 +- ++ .../transforms/StablehloRefineShapes.cpp | 4 + ++ .../transforms/VhloLegalizeToStablehlo.cpp | 100 ++- ++ stablehlo/transforms/VhloToVersion.cpp | 38 +- ++ .../StablehloAggressiveFolder.cpp | 816 ++++++++++-------- ++ ...ablehloAggressiveSimplificationPatterns.td | 6 +- ++ 18 files changed, 696 insertions(+), 474 deletions(-) ++ ++diff --git a/stablehlo/dialect/Version.h b/stablehlo/dialect/Version.h ++index eb35ad52..6aaef985 100644 ++--- a/stablehlo/dialect/Version.h +++++ b/stablehlo/dialect/Version.h ++@@ -38,7 +38,7 @@ class Version { ++ static FailureOr fromString(llvm::StringRef versionRef); ++ ++ /// Return a Version representing the current VHLO dialect version. ++- static Version getCurrentVersion() { return Version(1, 10, 10); } +++ static Version getCurrentVersion() { return Version(1, 11, 0); } ++ ++ /// Return a Version representing the minimum supported VHLO dialect version. ++ static Version getMinimumVersion() { return Version(0, 9, 0); } ++diff --git a/stablehlo/dialect/VhloAttrs.td b/stablehlo/dialect/VhloAttrs.td ++index bf75ad27..ab48630a 100644 ++--- a/stablehlo/dialect/VhloAttrs.td +++++ b/stablehlo/dialect/VhloAttrs.td ++@@ -45,14 +45,6 @@ class VHLO_AttrDef + def VHLO_ArrayAttrV1 : VHLO_AttrDef<"ArrayV1", "0.9.0", "current"> { + let mnemonic = "array_v1"; + let parameters = (ins ArrayRefParameter<"mlir::Attribute">:$value); +@@ -16,7 +56,7 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloAttrs.td b/stablehlo/stablehlo/dial + let assemblyFormat = "`<` custom($value) `>`"; + } + +-@@ -75,9 +67,9 @@ ++@@ -75,9 +67,9 @@ def VHLO_DictionaryAttrV1 : VHLO_AttrDef<"DictionaryV1", "0.9.0", "current"> { + LogicalResult DictionaryV1Attr::verify( + llvm::function_ref errFn, + ArrayRef> value) { +@@ -29,36 +69,48 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloAttrs.td b/stablehlo/stablehlo/dial + return success(); + } + }]; +-diff --ruN a/stablehlo/stablehlo/dialect/VhloTypes.h b/stablehlo/stablehlo/dialect/VhloTypes.h +---- stablehlo/stablehlo/dialect/VhloTypes.h +-+++ stablehlo/stablehlo/dialect/VhloTypes.h +-@@ -27,20 +27,23 @@ ++diff --git a/stablehlo/dialect/VhloDialect.td b/stablehlo/dialect/VhloDialect.td ++index 4e50cb32..edd220e3 100644 ++--- a/stablehlo/dialect/VhloDialect.td +++++ b/stablehlo/dialect/VhloDialect.td ++@@ -49,6 +49,7 @@ def VHLO_Dialect : Dialect { ++ 1.8.0: Introduce `f4E2M1FN`, `f6E2M3FN`, `f6E3M2FN` and `f8E8M0FNU` types. ++ 1.9.0: Add `ResultAccuracy` attribute to `exp` op. ++ 1.10.0: Add `ResultAccuracy` attribute to `cbrt`, `cosine`, `exponential`, `exponential_minus_one`, `log`, `log_plus_one`, `logistic`, `rsqrt`, `sine`, `sqrt`, `tan` and `tanh` ops. +++ 1.11.0: Allow (de)serializing VHLO programs mixed with potentially unstable dialects. ++ }]; ++ ++ let useDefaultAttributePrinterParser = 0; ++diff --git a/stablehlo/dialect/VhloTypes.h b/stablehlo/dialect/VhloTypes.h ++index e5aee254..f3a8d68d 100644 ++--- a/stablehlo/dialect/VhloTypes.h +++++ b/stablehlo/dialect/VhloTypes.h ++@@ -27,20 +27,23 @@ limitations under the License. + namespace mlir { + namespace vhlo { + + -class VhloTypeConverterBase : public TypeConverter { +-- public: +-- VhloTypeConverterBase() : TypeConverter(){}; +-- +-- virtual ~VhloTypeConverterBase() = default; +-- +-- virtual Attribute convertEncoding(Attribute attr) const = 0; +--}; +- +- // This class is used to manage conversions between VHLO and Builtin +- // dialects. +--class VhloTypeConverter : public VhloTypeConverterBase { +++ +++// This class is used to manage conversions between VHLO and Builtin +++// dialects. + +class VhloTypeConverter : public TypeConverter { + public: +-- VhloTypeConverter() : VhloTypeConverterBase() {} ++- VhloTypeConverterBase() : TypeConverter(){}; + + VhloTypeConverter() : TypeConverter(), allowOtherDialects(false) {} + + VhloTypeConverter(bool allowOtherDialects) + + : TypeConverter(), allowOtherDialects(allowOtherDialects) {} +-+ ++ ++- virtual ~VhloTypeConverterBase() = default; + + virtual ~VhloTypeConverter() = default; +-+ +-+ virtual Attribute convertEncoding(Attribute attr) const = 0; +-+ ++ ++ virtual Attribute convertEncoding(Attribute attr) const = 0; ++-}; ++ ++-// This class is used to manage conversions between VHLO and Builtin ++-// dialects. ++-class VhloTypeConverter : public VhloTypeConverterBase { ++- public: ++- VhloTypeConverter() : VhloTypeConverterBase() {} + + Attribute convertUnknownAttribute(Attribute attr) const { + + if (allowOtherDialects) return attr; + + return {}; +@@ -66,7 +118,7 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloTypes.h b/stablehlo/stablehlo/diale + + // A subclass can call this method to add conversions from VHLO -> Builtin + // types. Note that conversions are applied in reverse order, with the most +-@@ -58,6 +61,9 @@ ++@@ -58,6 +61,9 @@ class VhloTypeConverter : public VhloTypeConverterBase { + + // Mark unrealized casts as legal. Useful for dialect mixing. + void addUnrealizedMaterializations(); +@@ -76,19 +128,34 @@ diff --ruN a/stablehlo/stablehlo/dialect/VhloTypes.h b/stablehlo/stablehlo/diale + }; + + // Autogenerated VHLO type printers and parsers. +-diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir +---- stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir +-+++ stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir ++diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h ++index 385156bf..24d11c1d 100644 ++--- a/stablehlo/integrations/c/StablehloDialectApi.h +++++ b/stablehlo/integrations/c/StablehloDialectApi.h ++@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, ++ MlirStringRef targetVersion, ++ MlirStringCallback callback, ++ void* userData, ++- bool allowOtherDialects = false); +++ bool allowOtherDialects); ++ ++ // Read a StableHLO program from a portable artifact, returning the module as ++ // MLIR bytecode. Note, this bytecode returned is not a portable artifact, ++diff --git a/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir b/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir ++index 758965d0..c5239671 100644 ++--- a/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir +++++ b/stablehlo/tests/transforms/stablehlo_aggressive_folder.mlir + @@ -1,4 +1,4 @@ + -// RUN: stablehlo-opt --stablehlo-aggressive-folder --split-input-file --verify-diagnostics %s | FileCheck %s + +// RUN: stablehlo-opt --stablehlo-aggressive-folder=fold-op-element-limit=100 --split-input-file --verify-diagnostics %s | FileCheck %s + + //////// + // AddOp +-@@ -42,6 +42,21 @@ ++@@ -41,6 +41,21 @@ func.func @broadcast_in_dim_fold_splat(%arg0: tensor<3x3xi32>) ++ + // ----- + +- //////// +++//////// + +// ClampOp + + + +// CHECK-LABEL: func.func @clamp_fold +@@ -103,18 +170,13 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.ml + + + +// ----- + + +-+//////// ++ //////// + // CompareOp + +- // CHECK-LABEL: func.func @compare_folds +-@@ -98,6 +113,26 @@ +- // CHECK-DAG: [[R3:%.+]] = stablehlo.constant dense<{{\[\[0, 1, 2, 11, 12\], \[3, 4, 5, 13, 14\]\]}}> : tensor<2x5xi32> +- // CHECK-NEXT: return [[R0]], [[R1]], [[R2]], [[R3]] +- return %0, %1, %2, %3 : tensor<6xi32>, tensor<3xi32>, tensor<3x3xi32>, tensor<2x5xi32> +-+} +-+ +-+// ----- +-+ ++@@ -102,6 +117,26 @@ func.func @concatenate_fold() -> (tensor<6xi32>, tensor<3xi32>, tensor<3x3xi32>, ++ ++ // ----- ++ + +//////// + +// DivOp + + +@@ -131,32 +193,37 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_folder.ml + + %1 = stablehlo.divide %cst_1, %cst_1 : tensor + + %2 = stablehlo.divide %cst_2, %cst_2 : tensor + + return %0, %1, %2 : tensor, tensor, tensor +- } +++} +++ +++// ----- +++ ++ //////// ++ // MulOp + +- // ----- +-diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir +---- stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir +-+++ stablehlo/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir ++diff --git a/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir b/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir ++index 4921a224..a1eed944 100644 ++--- a/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir +++++ b/stablehlo/tests/transforms/stablehlo_aggressive_simplification.mlir + @@ -1,4 +1,4 @@ + -// RUN: stablehlo-opt --stablehlo-aggressive-simplification --allow-unregistered-dialect --split-input-file %s | FileCheck %s + +// RUN: stablehlo-opt --stablehlo-aggressive-simplification=fold-op-element-limit=100 --allow-unregistered-dialect --split-input-file %s | FileCheck %s + + ///////// + // AddOp +-diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir b/stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir +---- stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir +-+++ stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir +-@@ -521,16 +521,16 @@ ++diff --git a/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir b/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir ++index b262bf09..01b5dd8f 100644 ++--- a/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir +++++ b/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir ++@@ -521,16 +521,16 @@ func.func @eval_slice_zerodim() -> tensor<0x2x1xi64> { + // ----- + + // CHECK-LABEL: func @eval_slice_zerorank + -func.func @eval_slice_zerorank() -> tensor { + - // CHECK: [[RESULT:%.*]] = stablehlo.constant dense<3.300000e+01> : tensor +-- // CHECK: return [[RESULT]] +-- %0 = stablehlo.constant dense<33.0> : tensor + +func.func @eval_slice_zerorank() -> tensor { + + // CHECK: [[RESULT:%.*]] = stablehlo.constant dense<33> : tensor +-+ // CHECK: return [[RESULT]] ++ // CHECK: return [[RESULT]] ++- %0 = stablehlo.constant dense<33.0> : tensor + + %0 = stablehlo.constant dense<33> : tensor + %1 = "stablehlo.slice"(%0) { + start_indices = array, +@@ -169,9 +236,10 @@ diff --ruN a/stablehlo/stablehlo/tests/transforms/stablehlo_refine_shapes.mlir b + } + + // ----- +-diff --ruN a/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir b/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir +---- stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir +-+++ stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir ++diff --git a/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir b/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir ++index 6340898e..a5d344ac 100644 ++--- a/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir +++++ b/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mlir + @@ -5,15 +5,9 @@ + // about what constitutes a good test! The CHECK should be + // minimized and named to reflect the test intent. +@@ -190,14 +258,10 @@ diff --ruN a/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mli + // RUN: diff %t.0 %t.1 + + // CHECK-LABEL: vhlo.func_v1 @op_other( +-@@ -26,6 +20,34 @@ +- func.func @op_other(%arg0: tensor) -> tensor { +- %0 = arith.addf %arg0, %arg0 : tensor +- return %0 : tensor +-+} +-+ +-+// ----- +-+ ++@@ -30,6 +24,34 @@ func.func @op_other(%arg0: tensor) -> tensor { ++ ++ // ----- ++ + +// CHECK-LABEL: vhlo.func_v1 @func_attributes( + +// CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !vhlo.tensor_v1) -> (!vhlo.tensor_v1) { + +// CHECK: "vhlo.return_v1"(%[[VAL_0]]) : (!vhlo.tensor_v1) -> () +@@ -222,12 +286,17 @@ diff --ruN a/stablehlo/stablehlo/tests/vhlo/stablehlo_legalize_to_vhlo_mixed.mli + + vhlo.mixed_dict = {affine_map = affine_map<(d0) -> (d0)>, str_attr = "STR_ATTR"} + +} { + + return %arg0 : tensor +- } +- +- // ----- +-diff --ruN a/stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir b/stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir +---- stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir +-+++ stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir +++} +++ +++// ----- +++ ++ // CHECK-LABEL: vhlo.func_v1 @op_shlo( ++ // CHECK-SAME: %[[VAL_0:[0-9]+|[a-zA-Z$._-][a-zA-Z0-9$._-]*]]: !vhlo.tensor_v1) -> (!vhlo.tensor_v1) { ++ // CHECK: %[[VAL_1:.*]] = "vhlo.add_v1"(%[[VAL_0]], %[[VAL_0]]) : (!vhlo.tensor_v1, !vhlo.tensor_v1) -> !vhlo.tensor_v1 ++diff --git a/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir b/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir ++index b73d1005..f4330e67 100644 ++--- a/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir +++++ b/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir + @@ -1,17 +1,8 @@ + // RUN: stablehlo-opt --vhlo-to-version=target=1.9.0 -verify-diagnostics --split-input-file %s + +@@ -248,10 +317,11 @@ diff --ruN a/stablehlo/stablehlo/tests/vhlo/vhlo_attributes_invalid.mlir b/stabl + } { + return + } +-diff --ruN a/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp +---- stablehlo/stablehlo/tools/StablehloTranslateMain.cpp +-+++ stablehlo/stablehlo/tools/StablehloTranslateMain.cpp +-@@ -76,6 +76,11 @@ ++diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp ++index e3171b2d..fdf0d6a9 100644 ++--- a/stablehlo/tools/StablehloTranslateMain.cpp +++++ b/stablehlo/tools/StablehloTranslateMain.cpp ++@@ -76,6 +76,11 @@ llvm::cl::opt targetOption( + "target", llvm::cl::desc("Target version for serialization"), + llvm::cl::init("")); + +@@ -263,7 +333,7 @@ diff --ruN a/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/st + llvm::cl::opt argsOption( + "args", llvm::cl::desc("Arguments to pass to the interpreter"), + llvm::cl::init("")); +-@@ -317,7 +322,8 @@ ++@@ -317,7 +322,8 @@ TranslateFromMLIRRegistration serializeRegistration( + return module.emitError("failed to strip debuginfo"); + } + +@@ -273,10 +343,11 @@ diff --ruN a/stablehlo/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/st + }, + [](DialectRegistry ®istry) { + mlir::registerAllDialects(registry); +-diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/transforms/Passes.h +---- stablehlo/stablehlo/transforms/Passes.h +-+++ stablehlo/stablehlo/transforms/Passes.h +-@@ -35,6 +35,7 @@ ++diff --git a/stablehlo/transforms/Passes.h b/stablehlo/transforms/Passes.h ++index 0e4406d2..a5a4aa27 100644 ++--- a/stablehlo/transforms/Passes.h +++++ b/stablehlo/transforms/Passes.h ++@@ -35,6 +35,7 @@ limitations under the License. + #include "mlir/Transforms/DialectConversion.h" + #include "stablehlo/dialect/StablehloOps.h" + #include "stablehlo/dialect/Version.h" +@@ -284,7 +355,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/trans + + namespace mlir { + namespace stablehlo { +-@@ -54,17 +55,17 @@ ++@@ -54,17 +55,17 @@ void populateStablehloRefineShapesPatterns(MLIRContext *context, + // Populates StableHLO ops to VHLO ops rewriting patterns. + void populateStablehloToVhloPatterns(MLIRContext *context, + RewritePatternSet *patterns, +@@ -305,10 +376,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/Passes.h b/stablehlo/stablehlo/trans + + /// Collection of rewrite patterns for lowering of CHLO ops to StableHLO and + /// Shape ops. +-diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp +---- stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp +-+++ stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp +-@@ -57,7 +57,7 @@ ++diff --git a/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stablehlo/transforms/StablehloLegalizeToVhlo.cpp ++index e768ded6..25903399 100644 ++--- a/stablehlo/transforms/StablehloLegalizeToVhlo.cpp +++++ b/stablehlo/transforms/StablehloLegalizeToVhlo.cpp ++@@ -57,7 +57,7 @@ namespace { + class StablehloToVhloTypeConverter : public vhlo::VhloTypeConverter { + public: + StablehloToVhloTypeConverter(bool allowOtherDialects) +@@ -317,7 +389,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable + LLVM_DEBUG( + llvm::dbgs() + << "[StablehloToVhloTypeConverter] Creating with allowOtherDialects: " +-@@ -82,7 +82,15 @@ ++@@ -82,7 +82,15 @@ class StablehloToVhloTypeConverter : public vhlo::VhloTypeConverter { + return vhlo::TokenV1Type::get(token.getContext()); + }); + addBuiltinToVhloConversions(); +@@ -334,7 +406,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable + } + + Attribute convertEncoding(Attribute attr) const final { +-@@ -114,7 +122,7 @@ ++@@ -114,7 +122,7 @@ class StablehloToVhloTypeConverter : public vhlo::VhloTypeConverter { + return vhlo::Name##Version##Attr::get(attr.getContext(), vhloValue.value()) + + Attribute convertGeneric(Attribute stablehloAttr, +@@ -343,7 +415,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable + LLVM_DEBUG(llvm::dbgs() << "Convert generic: " << stablehloAttr << '\n'); + + // Handle StableHLO attributes. +-@@ -241,6 +249,10 @@ ++@@ -241,6 +249,10 @@ Attribute convertGeneric(Attribute stablehloAttr, + return vhlo::TypeV1Attr::get(attr.getContext(), vhloType); + } + +@@ -354,7 +426,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable + LLVM_DEBUG(llvm::dbgs() << "Failed to convert: " << stablehloAttr << '\n'); + return {}; // Failed to convert attribute. + } +-@@ -268,13 +280,15 @@ ++@@ -268,13 +280,15 @@ SpecialResult notSpecial() { return SpecialResult::NOT_SPECIAL; } + Attribute convertBool(const ConversionPattern& pattern, int64_t stablehloDim) { + auto stablehloType = IntegerType::get(pattern.getContext(), 1); + auto stablehloAttr = IntegerAttr::get(stablehloType, stablehloDim); +@@ -372,7 +444,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable + } + + Attribute convertInts(const ConversionPattern& pattern, +-@@ -282,7 +296,8 @@ ++@@ -282,7 +296,8 @@ Attribute convertInts(const ConversionPattern& pattern, + auto stablehloType = RankedTensorType::get( + stablehloDims.size(), IntegerType::get(pattern.getContext(), 64)); + auto stablehloAttr = DenseIntElementsAttr::get(stablehloType, stablehloDims); +@@ -382,7 +454,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable + } + + Attribute convertSymbol(const ConversionPattern& pattern, +-@@ -290,7 +305,7 @@ ++@@ -290,7 +305,7 @@ Attribute convertSymbol(const ConversionPattern& pattern, + auto stablehloSymbolAttr = dyn_cast(stablehloAttr); + if (!stablehloSymbolAttr) return {}; + return convertGeneric(stablehloSymbolAttr.getAttr(), +@@ -391,7 +463,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable + } + + SpecialResult convertChannelHandle(const ConversionPattern& pattern, +-@@ -445,15 +460,15 @@ ++@@ -445,15 +460,15 @@ SpecialResult convertDotAlgorithm(const ConversionPattern& pattern, + vhloAttrs.emplace_back( + StringAttr::get(pattern.getContext(), "lhs_precision_type"), + convertGeneric(TypeAttr::get(attr.getLhsPrecisionType()), +@@ -410,7 +482,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable + + // Components + auto vhloLhsComponentCount = convertInt(pattern, attr.getLhsComponentCount()); +-@@ -712,7 +727,9 @@ ++@@ -712,7 +727,9 @@ LogicalResult addDefaults(const OpConversionPattern& pattern, + auto addDefaultAttr = [&](StringRef vhloName, Attribute stablehloAttr) { + vhloAttrs.emplace_back( + StringAttr::get(pattern.getContext(), vhloName), +@@ -421,7 +493,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable + }; + if constexpr (std::is_same::value) { + if (!stablehloOp.getSymVisibilityAttr()) +-@@ -987,8 +1004,9 @@ ++@@ -987,8 +1004,9 @@ class StablehloToVhloOpConverter : public OpConversionPattern { + case SpecialResult::SPECIAL_FAILURE: + return failure(); + case SpecialResult::NOT_SPECIAL: +@@ -433,7 +505,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable + if (!vhloAttr) return failure(); + vhloAttrs.push_back({stablehloAttr.getName(), vhloAttr}); + break; +-@@ -1075,14 +1093,14 @@ ++@@ -1075,14 +1093,14 @@ struct StablehloLegalizeToVhloPass + } + + private: +@@ -450,10 +522,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloLegalizeToVhlo.cpp b/stable + populateStablehloToVhloPatterns< + #define GET_OP_LIST + #include "stablehlo/dialect/StablehloOps.cpp.inc" +-diff --ruN a/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp +---- stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp +-+++ stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp +-@@ -16,6 +16,7 @@ ++diff --git a/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehlo/transforms/StablehloRefineShapes.cpp ++index 64eaa3c0..4b585b46 100644 ++--- a/stablehlo/transforms/StablehloRefineShapes.cpp +++++ b/stablehlo/transforms/StablehloRefineShapes.cpp ++@@ -16,6 +16,7 @@ limitations under the License. + + #include + #include +@@ -461,7 +534,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehl + #include + #include + +-@@ -1038,8 +1039,11 @@ ++@@ -1038,8 +1039,11 @@ LogicalResult applyShapeRefinementPatterns(func::FuncOp func, + // Populate additional patterns for StableHLO extensions. + state.addAdditionalPatterns(patterns); + +@@ -473,10 +546,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/StablehloRefineShapes.cpp b/stablehl + + // The folding patterns implement partial evaluation of shape computations + // which is a critical part of implementing type refinement for ops like +-diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp +---- stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp +-+++ stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp +-@@ -57,7 +57,10 @@ ++diff --git a/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stablehlo/transforms/VhloLegalizeToStablehlo.cpp ++index 5662ad9d..41973a0e 100644 ++--- a/stablehlo/transforms/VhloLegalizeToStablehlo.cpp +++++ b/stablehlo/transforms/VhloLegalizeToStablehlo.cpp ++@@ -57,7 +57,10 @@ namespace { + + class VhloToStablehloTypeConverter : public vhlo::VhloTypeConverter { + public: +@@ -488,7 +562,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + addConversion([](Type type) -> Type { return type; }); + addConversion([](vhlo::TokenV1Type token) -> Type { + LLVM_DEBUG(llvm::dbgs() << "Converting TokenType\n"); +-@@ -90,7 +93,7 @@ ++@@ -90,7 +93,7 @@ class VhloToStablehloTypeConverter : public vhlo::VhloTypeConverter { + return stablehlo::Name##Attr::get(attr.getContext(), stablehloValue.value()) + + Attribute convertGeneric(Attribute vhloAttr, +@@ -497,18 +571,18 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + LLVM_DEBUG(llvm::dbgs() << "Converting attr " << vhloAttr); + if (auto vhloAttrs = dyn_cast(vhloAttr)) { + SmallVector stablehloAttrs; +-@@ -189,6 +192,10 @@ +- // All VHLO attributes must have counterparts in StableHLO. ++@@ -190,6 +193,10 @@ Attribute convertGeneric(Attribute vhloAttr, + return {}; + } +-+ ++ + + // Fall back to type converter for unknown attributes. + + auto unknownAttr = typeConverter->convertUnknownAttribute(vhloAttr); + + if (unknownAttr) return unknownAttr; +- +++ + // This should be unreachable unless program is a mix of VHLO and other + // dialects, e.g. due to user edits to textual assembly format. +-@@ -229,7 +236,7 @@ ++ return {}; ++@@ -229,7 +236,7 @@ bool isNoneType(Attribute vhloAttr) { + } + + LogicalResult convertTypeAttr(Attribute vhloAttr, Type& result, +@@ -517,7 +591,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + auto stablehloAttr = convertGeneric(vhloAttr, typeConverter); + if (!stablehloAttr || !isa(stablehloAttr)) return failure(); + result = cast(stablehloAttr).getValue(); +-@@ -244,7 +251,7 @@ ++@@ -244,7 +251,7 @@ LogicalResult convertInt(Attribute vhloAttr, int64_t& result) { + } + + LogicalResult convertInts(Attribute vhloAttr, +@@ -526,7 +600,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + SmallVector& result) { + auto vhloTensorAttr = dyn_cast(vhloAttr); + if (!vhloTensorAttr) return failure(); +-@@ -256,7 +263,7 @@ ++@@ -256,7 +263,7 @@ LogicalResult convertInts(Attribute vhloAttr, + } + + Attribute convertSymbol(Attribute vhloAttr, +@@ -535,7 +609,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + auto vhloStringAttr = dyn_cast(vhloAttr); + if (!vhloStringAttr) return {}; + auto stablehloStringAttr = dyn_cast_or_null( +-@@ -267,7 +274,7 @@ ++@@ -267,7 +274,7 @@ Attribute convertSymbol(Attribute vhloAttr, + + template + Attribute convertChannelHandle(OpType vhloOp, +@@ -544,7 +618,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + int64_t channelId, channelType; + if (failed(convertInt(vhloOp.getChannelId(), channelId)) || + failed(convertInt(vhloOp.getChannelType(), channelType))) +-@@ -277,7 +284,7 @@ ++@@ -277,7 +284,7 @@ Attribute convertChannelHandle(OpType vhloOp, + } + + Attribute convertChannelId(Attribute vhloAttr, +@@ -553,7 +627,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + int64_t channelId; + if (failed(convertInt(vhloAttr, channelId))) return {}; + return stablehlo::ChannelHandleAttr::get(vhloAttr.getContext(), channelId, +-@@ -285,8 +292,8 @@ ++@@ -285,8 +292,8 @@ Attribute convertChannelId(Attribute vhloAttr, + } + + template +@@ -564,7 +638,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + int64_t stablehloInputBatchDimension, stablehloInputFeatureDimension; + SmallVector stablehloInputSpatialDimensions; + int64_t stablehloKernelInputFeatureDimension, +-@@ -323,7 +330,7 @@ ++@@ -323,7 +330,7 @@ Attribute convertConvDimensionNumbers(OpType vhloOp, + } + + Attribute convertCustomCallCalledComputations( +@@ -573,7 +647,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + if (auto vhloArrayAttr = dyn_cast(vhloAttr)) { + SmallVector stablehloAttrs; + for (auto vhloAttr : vhloArrayAttr.getValue()) { +-@@ -336,8 +343,8 @@ ++@@ -336,8 +343,8 @@ Attribute convertCustomCallCalledComputations( + return {}; + } + +@@ -584,7 +658,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + Type lhsPrecisionType, rhsPrecisionType, accumulationType; + if (isNoneType(vhloOp.getLhsComponentCount())) { + // All must be nonetype +-@@ -373,8 +380,8 @@ ++@@ -373,8 +380,8 @@ FailureOr convertDotAlgorithm(vhlo::DotGeneralOpV2 vhloOp, + numPrimitiveOperations, allowImpreciseAccumulation); + } + +@@ -595,7 +669,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + SmallVector stablehloLhsBatchingDimensions, + stablehloRhsBatchingDimensions, stablehloLhsContractingDimensions, + stablehloRhsContractingDimensions; +-@@ -394,13 +401,13 @@ ++@@ -394,13 +401,13 @@ Attribute convertDotDimensionNumbers(vhlo::DotGeneralOpV2 vhloOp, + } + + Attribute convertFuncCallee(Attribute vhloAttr, +@@ -612,7 +686,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + SmallVector stablehloOffsetDims, stablehloCollapsedSliceDims, + stablehloOperandBatchingDims, stablehloStartIndicesBatchingDims, + stablehloStartIndexMap; +-@@ -423,8 +430,8 @@ ++@@ -423,8 +430,8 @@ Attribute convertGatherDimensionNumbers(OpType vhloOp, + stablehloStartIndexMap, stablehloIndexVectorDim); + } + +@@ -623,7 +697,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + SmallVector stablehloUpdateWindowDims, stablehloInsertedWindowDims, + stablehloInputBatchingDims, stablehloScatterIndicesBatchingDims, + stablehloScatterDimsToOperandDims; +-@@ -463,10 +470,11 @@ ++@@ -463,10 +470,11 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, + VhloOpTy vhloOp, + SmallVector& vhloAttrs, + SmallVector& stablehloAttrs) { +@@ -637,7 +711,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + if (!stablehloAttr) return failure(); + stablehloAttrs.emplace_back( + StringAttr::get(pattern.getContext(), "dimension_numbers"), +-@@ -480,7 +488,7 @@ ++@@ -480,7 +488,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, + if constexpr (std::is_same::value) { + // Dot Dimension Numbers + auto stablehloDotDimAttr = +@@ -646,7 +720,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + if (!stablehloDotDimAttr) return failure(); + stablehloAttrs.emplace_back( + StringAttr::get(pattern.getContext(), "dot_dimension_numbers"), +-@@ -489,8 +497,7 @@ ++@@ -489,8 +497,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, + "lhs_contracting_dimensions", "rhs_contracting_dimensions"); + + // Dot Algorithm +@@ -656,7 +730,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + if (failed(stablehloDotAlgorithmAttr)) return failure(); + if (stablehloDotAlgorithmAttr.value()) + stablehloAttrs.emplace_back( +-@@ -503,8 +510,7 @@ ++@@ -503,8 +510,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, + } + if constexpr (std::is_same::value || + std::is_same::value) { +@@ -666,7 +740,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + if (!stablehloAttr) return failure(); + stablehloAttrs.emplace_back( + StringAttr::get(pattern.getContext(), "dimension_numbers"), +-@@ -514,8 +520,7 @@ ++@@ -514,8 +520,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, + "start_index_map", "index_vector_dim"); + } + if constexpr (std::is_same::value) { +@@ -676,7 +750,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + if (!stablehloAttr) return failure(); + stablehloAttrs.emplace_back( + StringAttr::get(pattern.getContext(), "scatter_dimension_numbers"), +-@@ -526,8 +531,7 @@ ++@@ -526,8 +531,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, + } + if constexpr (std::is_same::value || + std::is_same::value) { +@@ -686,7 +760,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + if (!stablehloAttr) return failure(); + stablehloAttrs.emplace_back( + StringAttr::get(pattern.getContext(), "channel_handle"), stablehloAttr); +-@@ -537,7 +541,7 @@ ++@@ -537,7 +541,7 @@ LogicalResult implodeSpecial(const OpConversionPattern& pattern, + } + + template +@@ -695,7 +769,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + StringAttr vhloName, Attribute vhloAttr, + SmallVector& stablehloAttrs) { + auto tensorAttr = dyn_cast(vhloAttr); +-@@ -556,15 +560,15 @@ ++@@ -556,15 +560,15 @@ SpecialResult convertDenseArray(const TypeConverter* typeConverter, + } + + SpecialResult convertDenseBoolArray( +@@ -715,7 +789,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + return convertDenseArray( + typeConverter, vhloName, vhloAttr, stablehloAttrs); + } +-@@ -575,7 +579,8 @@ ++@@ -575,7 +579,8 @@ SpecialResult convertSpecial(const OpConversionPattern& pattern, + SmallVector& stablehloAttrs) { + StringAttr stablehloName = vhloName; + Attribute stablehloAttr; +@@ -725,7 +799,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + + if constexpr (std::is_same::value || + std::is_same::value || +-@@ -585,7 +590,7 @@ ++@@ -585,7 +590,7 @@ SpecialResult convertSpecial(const OpConversionPattern& pattern, + std::is_same::value) { + if (vhloName == "channel_id") { + stablehloName = StringAttr::get(pattern.getContext(), "channel_handle"); +@@ -734,7 +808,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + if (!stablehloAttr) return specialFailure(); + } + if (vhloName == "use_global_device_ids") { +-@@ -597,20 +602,20 @@ ++@@ -597,20 +602,20 @@ SpecialResult convertSpecial(const OpConversionPattern& pattern, + } + if constexpr (std::is_same::value) { + if (vhloName == "called_computations") { +@@ -759,7 +833,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + if (!stablehloAttr) return specialFailure(); + } + } +-@@ -760,8 +765,8 @@ ++@@ -760,8 +765,8 @@ bool isDefaultResultAccuracyAttribute(Attribute vhloAttr) { + template + bool isSplatTensor(const ConversionPattern& pattern, Attribute vhloAttr, + T splatValue) { +@@ -770,7 +844,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + return attr && attr.isSplat() && + attr.template getSplatValue() == splatValue; + } +-@@ -977,8 +982,9 @@ ++@@ -977,8 +982,9 @@ class VhloToStablehloOpConverter : public OpConversionPattern { + case SpecialResult::SPECIAL_FAILURE: + return failure(); + case SpecialResult::NOT_SPECIAL: +@@ -782,7 +856,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + if (!stablehloAttr) return failure(); + stablehloAttrs.push_back({vhloAttr.getName(), stablehloAttr}); + break; +-@@ -1056,7 +1062,7 @@ ++@@ -1056,7 +1062,7 @@ struct ReconcileUnrealizedConversionCasts + template + void populateVhloToStablehloPatterns(MLIRContext* context, + RewritePatternSet* patterns, +@@ -791,7 +865,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + patterns + ->add>...>( + *converter, context); +-@@ -1104,7 +1110,7 @@ ++@@ -1104,7 +1110,7 @@ struct VhloLegalizeToStablehloPass + + void populateVhloToStablehloPatterns(MLIRContext* context, + RewritePatternSet* patterns, +@@ -800,10 +874,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloLegalizeToStablehlo.cpp b/stable + populateVhloToStablehloPatterns< + #define GET_OP_LIST + #include "stablehlo/dialect/StablehloOps.cpp.inc" +-diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stablehlo/transforms/VhloToVersion.cpp +---- stablehlo/stablehlo/transforms/VhloToVersion.cpp +-+++ stablehlo/stablehlo/transforms/VhloToVersion.cpp +-@@ -56,15 +56,23 @@ ++diff --git a/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/transforms/VhloToVersion.cpp ++index dfbf2877..7818e601 100644 ++--- a/stablehlo/transforms/VhloToVersion.cpp +++++ b/stablehlo/transforms/VhloToVersion.cpp ++@@ -56,15 +56,23 @@ namespace { + + // Currently there are no type-to-version conversions so this class + // simply validates that all types are from the VHLO dialect. +@@ -833,7 +908,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable + }; + + // Check user-specified target version. Emit error if invalid. +-@@ -111,6 +119,10 @@ ++@@ -111,6 +119,10 @@ bool isLegalVersion(VersionedInterface& interface, const Version& target) { + LogicalResult isLegalType(Type type, const Version& targetVersion); + + LogicalResult isLegalAttribute(const Attribute& attr, Version targetVersion) { +@@ -844,7 +919,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable + auto attrInterface = dyn_cast(attr); + if (!attrInterface || !isLegalVersion(attrInterface, targetVersion)) { + LLVM_DEBUG(llvm::dbgs() << "failed to legalize attribute " << attr +-@@ -119,10 +131,11 @@ ++@@ -119,10 +131,11 @@ LogicalResult isLegalAttribute(const Attribute& attr, Version targetVersion) { + } + + // Recursively check attrs if VHLO attr is a container +@@ -857,7 +932,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable + if (auto arrAttr = dyn_cast(attr)) { + return success(llvm::all_of( + arrAttr.getValue(), [&](std::pair entry) { +-@@ -146,6 +159,10 @@ ++@@ -146,6 +159,10 @@ LogicalResult isLegalAttribute(const Attribute& attr, Version targetVersion) { + } + + LogicalResult isLegalType(Type type, const Version& targetVersion) { +@@ -868,7 +943,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable + // All valid VHLO types must have versioned type interface. + auto typeInterface = dyn_cast(type); + if (!typeInterface || !isLegalVersion(typeInterface, targetVersion)) { +-@@ -170,10 +187,11 @@ ++@@ -170,10 +187,11 @@ LogicalResult isLegalType(Type type, const Version& targetVersion) { + return failure(); + return isLegalType(ranked.getElementType(), targetVersion); + } +@@ -881,7 +956,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable + if (auto quant = dyn_cast(type)) + return success( + succeeded(isLegalType(quant.getStorageType(), targetVersion)) && +-@@ -213,7 +231,7 @@ ++@@ -213,7 +231,7 @@ bool isLegalLocation(Location loc, const Version& targetVersion) { + bool isLegalOperation(Operation* op, const Version& targetVersion) { + // Validate op + auto opInterface = dyn_cast(op); +@@ -890,7 +965,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable + if (!isLegalVersion(opInterface, targetVersion)) return false; + LLVM_DEBUG(llvm::dbgs() << "Legal op version for target. " << op << '\n'); + +-@@ -454,7 +472,7 @@ ++@@ -454,7 +472,7 @@ struct AllReduceOpV2ToV1 : public OpRewritePattern { + namespace stablehlo { + void populateVhloToVersionPatterns(MLIRContext* context, + RewritePatternSet* patterns, +@@ -899,10 +974,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/VhloToVersion.cpp b/stablehlo/stable + vhlo::populateWithGenerated(*patterns); + patterns->add(context); + patterns->add(context); +-diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp b/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp +---- stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp +-+++ stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp +-@@ -19,6 +19,7 @@ ++diff --git a/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp b/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp ++index 9cfac0ba..09bfb783 100644 ++--- a/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp +++++ b/stablehlo/transforms/optimization/StablehloAggressiveFolder.cpp ++@@ -19,6 +19,7 @@ limitations under the License. + #include + #include + #include +@@ -910,7 +986,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + #include + + #include "llvm/ADT/APInt.h" +-@@ -47,6 +48,7 @@ ++@@ -47,6 +48,7 @@ limitations under the License. + #include "mlir/IR/TypeUtilities.h" + #include "mlir/IR/Types.h" + #include "mlir/IR/Value.h" +@@ -918,7 +994,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + #include "mlir/Interfaces/SideEffectInterfaces.h" + #include "mlir/Pass/Pass.h" + #include "mlir/Rewrite/FrozenRewritePatternSet.h" +-@@ -98,29 +100,88 @@ ++@@ -98,29 +100,88 @@ LogicalResult validateStaticShapeResult(PatternRewriter& rewriter, + return success(); + } + +@@ -982,8 +1058,8 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + if (res) return cast(res); + + return nullptr; +-+} +-+ ++ } ++ + + + +/// Binary constant folder that used a generic folder function to handle both + +/// ints and floats. +@@ -1005,8 +1081,8 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + if (!res) return rewriter.notifyMatchFailure(op, "folding failed"); + + + + return res; +- } +- +++} +++ + template + -LogicalResult evalConvertHelper(PatternRewriter& rewriter, OpType op, +@@ -1014,7 +1090,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + DenseIntOrFPElementsAttr elements, Type resType, + CalculationT&& calculate) { + auto result = constFoldCastOp( + elements, resType, calculate); + +@@ -1036,7 +1112,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + DenseIntOrFPElementsAttr elements, + RankedTensorType resultType) { + auto oldType = getElementTypeOrSelf(elements); +-@@ -153,7 +215,7 @@ ++@@ -153,7 +215,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, + if (auto newFloatType = dyn_cast(newType)) { + // Float -> Float + const auto& targetSemantics = newFloatType.getFloatSemantics(); +@@ -1045,7 +1121,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + rewriter, op, elements, resultType, + [&targetSemantics](const APFloat& operand, bool& castStatus) { + bool losesInfo; +-@@ -167,7 +229,7 @@ ++@@ -167,7 +229,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, + } + + // Float -> Int +@@ -1054,7 +1130,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + rewriter, op, elements, resultType, + [&newBitWidth, &isNewTypeUnsigned](const APFloat& operand, + bool& castStatus) { +-@@ -186,7 +248,7 @@ ++@@ -186,7 +248,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, + + if (auto newFloatType = dyn_cast(newType)) { + // Int -> Float +@@ -1063,7 +1139,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + rewriter, op, elements, resultType, + [&newFloatType, &isOldTypeUnsigned](const APInt& operand, + bool& /*castStatus*/) { +-@@ -199,64 +261,12 @@ ++@@ -199,7 +261,7 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, + } + + // Int -> Int +@@ -1072,10 +1148,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + rewriter, op, elements, resultType, + [&newBitWidth, &isOldTypeUnsigned](const APInt& operand, + bool& /*castStatus*/) { +- return APSInt(operand, isOldTypeUnsigned).extOrTrunc(newBitWidth); ++@@ -207,58 +269,6 @@ LogicalResult evalConvert(PatternRewriter& rewriter, OpType op, + }); +--} +-- ++ } ++ + -// The patterns below implement partial evaluation of shape computations which + -// is a critical part of implementing type refinement for ops like + -// dynamic_broadcast_in_dim, dynamic_iota and dynamic_reshape whose shape +@@ -1126,10 +1202,12 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + - rewriter.replaceOpWithNewOp(op, + - getTensorAttr(resultType, result)); + - return success(); +- } +- ++-} ++- + template +-@@ -275,29 +285,18 @@ ++ struct FoldOpRewritePattern : OpRewritePattern { ++ FoldOpRewritePattern(MLIRContext* context, ++@@ -275,29 +285,18 @@ struct FoldOpRewritePattern : OpRewritePattern { + ArrayRef generatedNames = {}) = delete; + + const StablehloAggressiveFolderPassOptions& options; +@@ -1154,9 +1232,8 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + - rewriter.replaceOpWithNewOp(op, res); + - return success(); + - } +-- ++ + - return failure(); +-+ + + LogicalResult validateElementCountForFold(PatternRewriter& rewriter, + + Operation* op, + + ShapedType resultType) const { +@@ -1171,31 +1248,21 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + } + }; + +-@@ -318,100 +317,102 @@ ++@@ -318,100 +317,102 @@ struct ShapeOpRewritePattern : public FoldOpRewritePattern { + } + }; + + -struct EvalAddOpShapePattern : public FoldOpRewritePattern { + - using FoldOpRewritePattern::FoldOpRewritePattern; +-- +-- LogicalResult matchAndRewrite(AddOp op, +-- PatternRewriter& rewriter) const override { +-- return evalElementwise(rewriter, op, +-- [&](APSInt lhs, APSInt rhs) { return lhs + rhs; }); +-- } +--}; +-- +--struct EvalAndOpPattern : public FoldOpRewritePattern { +-- using FoldOpRewritePattern::FoldOpRewritePattern; +-- +-- LogicalResult matchAndRewrite(AndOp op, +-- PatternRewriter& rewriter) const override { + +struct FoldAddOpPattern final + + : public ShapeOpRewritePattern { + + using ShapeOpRewritePattern::ShapeOpRewritePattern; +-+ ++ ++- LogicalResult matchAndRewrite(AddOp op, + + LogicalResult matchAndRewrite(mlir::stablehlo::AddOp op, +-+ PatternRewriter& rewriter) const override { ++ PatternRewriter& rewriter) const override { ++- return evalElementwise(rewriter, op, ++- [&](APSInt lhs, APSInt rhs) { return lhs + rhs; }); + + if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) + + return failure(); + + +@@ -1203,14 +1270,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + if (failed(res)) return failure(); + + rewriter.replaceOpWithNewOp(op, res.value()); + + return success(); +-+ } +-+}; +-+ ++ } ++ }; ++ ++-struct EvalAndOpPattern : public FoldOpRewritePattern { ++- using FoldOpRewritePattern::FoldOpRewritePattern; + +struct FoldAndOpPattern : public ShapeOpRewritePattern { + + using ShapeOpRewritePattern::ShapeOpRewritePattern; +-+ ++ ++- LogicalResult matchAndRewrite(AndOp op, + + LogicalResult matchAndRewrite(mlir::stablehlo::AndOp op, +-+ PatternRewriter& rewriter) const override { ++ PatternRewriter& rewriter) const override { + + // TODO: Support more int types + auto resultType = op.getType(); + if (!resultType.getElementType().isInteger(1)) +@@ -1219,24 +1289,14 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + - return evalElementwise(rewriter, op, [&](APSInt lhsInt, APSInt rhsInt) { + - return getAPSInt(resultType.getElementType(), lhsInt != 0 && rhsInt != 0); + - }); +-- } + + auto res = foldBinaryOpIntOrFloat(rewriter, op, FoldAnd{}); + + if (failed(res)) return failure(); + + rewriter.replaceOpWithNewOp(op, res.value()); + + return success(); +-+ } +-+ +-+ struct FoldAnd { +-+ APInt operator()(APInt lhs, APInt rhs) const { +-+ return APInt(lhs.getBitWidth(), !lhs.isZero() && !rhs.isZero()); +-+ } +-+ std::optional operator()(APFloat lhs, APFloat rhs) const { +-+ return std::nullopt; +-+ } +-+ }; +- }; ++ } ++-}; + +- // Pattern: broadcast_in_dim(splat, _) -> constant(splat) ++-// Pattern: broadcast_in_dim(splat, _) -> constant(splat) + -struct FoldBroadcastInDimSplatPattern final + - : FoldOpRewritePattern { + - using FoldOpRewritePattern::FoldOpRewritePattern; +@@ -1251,14 +1311,22 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + - op, SplatElementsAttr::get(op.getType(), + - cstAttr.getSplatValue())); + - return success(); +-- } +++ struct FoldAnd { +++ APInt operator()(APInt lhs, APInt rhs) const { +++ return APInt(lhs.getBitWidth(), !lhs.isZero() && !rhs.isZero()); ++ } + - return failure(); + - } +--}; +-- +++ std::optional operator()(APFloat lhs, APFloat rhs) const { +++ return std::nullopt; +++ } +++ }; ++ }; ++ + -struct EvalBroadcastInDimOpPattern + - : public FoldOpRewritePattern { + - using FoldOpRewritePattern::FoldOpRewritePattern; +++// Pattern: broadcast_in_dim(splat, _) -> constant(splat) + +struct FoldBroadcastInDimOpSplatPattern + + : public ShapeOpRewritePattern { + + using ShapeOpRewritePattern::ShapeOpRewritePattern; +@@ -1267,8 +1335,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + PatternRewriter& rewriter) const override { + auto resultType = op.getType(); + - if (failed(validateStaticShapeResult(rewriter, op, resultType))) +-- return failure(); +-- +++ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || +++ failed(validateShapeFoldDtype(rewriter, op, resultType))) ++ return failure(); ++ + - auto operandType = op.getOperand().getType(); + - if (operandType.getRank() != 0) + - return rewriter.notifyMatchFailure(op, "expected 0-dimensional type"); +@@ -1277,52 +1347,34 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + - if (failed(hlo::matchInts(op.getOperand(), operand))) + - return rewriter.notifyMatchFailure(op, "expected constant operands"); + - auto scalar = operand[0]; +-- +++ SplatElementsAttr cstAttr; +++ matchPattern(op.getOperand(), m_Constant(&cstAttr)); +++ if (!cstAttr) return rewriter.notifyMatchFailure(op, "operand not splat"); ++ + - rewriter.replaceOpWithNewOp( + - op, getTensorAttr(op.getType(), scalar)); +-- return success(); +-- } +--}; +-- +++ rewriter.replaceOpWithNewOp( +++ op, SplatElementsAttr::get(op.getType(), +++ cstAttr.getSplatValue())); ++ return success(); ++ } ++ }; ++ + -struct EvalClampOpPattern : public FoldOpRewritePattern { + - using FoldOpRewritePattern::FoldOpRewritePattern; +-- +++struct FoldCompareOpPattern : public ShapeOpRewritePattern { +++ using ShapeOpRewritePattern::ShapeOpRewritePattern; ++ + - LogicalResult matchAndRewrite(ClampOp op, +-- PatternRewriter& rewriter) const override { +++ LogicalResult matchAndRewrite(CompareOp op, ++ PatternRewriter& rewriter) const override { + - return evalElementwise(rewriter, op, + - [&](APSInt min, APSInt operand, APSInt max) { + - if (operand < min) return min; + - if (max < operand) return max; + - return operand; + - }); +-- } +--}; +-- +--struct EvalCompareOpPattern : public FoldOpRewritePattern { +-- using FoldOpRewritePattern::FoldOpRewritePattern; +-+ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || +-+ failed(validateShapeFoldDtype(rewriter, op, resultType))) +-+ return failure(); +-+ +-+ SplatElementsAttr cstAttr; +-+ matchPattern(op.getOperand(), m_Constant(&cstAttr)); +-+ if (!cstAttr) return rewriter.notifyMatchFailure(op, "operand not splat"); +-+ +-+ rewriter.replaceOpWithNewOp( +-+ op, SplatElementsAttr::get(op.getType(), +-+ cstAttr.getSplatValue())); +-+ return success(); +-+ } +-+}; +-+ +-+struct FoldCompareOpPattern : public ShapeOpRewritePattern { +-+ using ShapeOpRewritePattern::ShapeOpRewritePattern; +- +- LogicalResult matchAndRewrite(CompareOp op, +- PatternRewriter& rewriter) const override { +- auto resultType = op.getType(); +-- auto kind = op.getCompareType(); +-- return evalElementwise(rewriter, op, [&](APInt lhs, APInt rhs) { +++ auto resultType = op.getType(); + + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) + + return failure(); + + +@@ -1332,15 +1384,23 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + if (failed(res)) return failure(); + + rewriter.replaceOpWithNewOp(op, res.value()); + + return success(); +-+ } +-+ ++ } ++-}; ++ ++-struct EvalCompareOpPattern : public FoldOpRewritePattern { ++- using FoldOpRewritePattern::FoldOpRewritePattern; + + struct FoldCompare { + + FoldCompare(ComparisonDirection direction, + + std::optional kind) + + : direction(direction), kind(kind) {} + + ComparisonDirection direction; + + std::optional kind; +-+ ++ ++- LogicalResult matchAndRewrite(CompareOp op, ++- PatternRewriter& rewriter) const override { ++- auto resultType = op.getType(); ++- auto kind = op.getCompareType(); ++- return evalElementwise(rewriter, op, [&](APInt lhs, APInt rhs) { + + // TODO: Enable float folding. + + std::optional operator()(APFloat lhs, APFloat rhs) { + + return std::nullopt; +@@ -1352,7 +1412,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + case ComparisonDirection::EQ: + result = lhs == rhs; + break; +-@@ -431,9 +432,9 @@ ++@@ -431,9 +432,9 @@ struct EvalCompareOpPattern : public FoldOpRewritePattern { + result = kind == ComparisonType::SIGNED ? lhs.slt(rhs) : lhs.ult(rhs); + break; + } +@@ -1365,7 +1425,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + }; + + ////////////////////////////////// +-@@ -441,16 +442,15 @@ ++@@ -441,16 +442,15 @@ struct EvalCompareOpPattern : public FoldOpRewritePattern { + ///////////////////////////////// + + struct FoldConcatenateOpPattern final +@@ -1387,7 +1447,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + return failure(); + + // Fold concatenate when all inputs are constants. +-@@ -466,6 +466,7 @@ ++@@ -466,6 +466,7 @@ struct FoldConcatenateOpPattern final + int64_t{1}, std::multiplies<>{}); + + SmallVector newElems; +@@ -1395,7 +1455,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + newElems.reserve(numElems); + + for (int64_t i = 0; i != topSize; ++i) { +-@@ -485,31 +486,7 @@ ++@@ -485,31 +486,7 @@ struct FoldConcatenateOpPattern final + int64_t foldOpElementLimit; + }; + +@@ -1428,20 +1488,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + using ShapeOpRewritePattern::ShapeOpRewritePattern; + + LogicalResult matchAndRewrite(ConvertOp op, +-@@ -532,28 +509,50 @@ ++@@ -532,28 +509,50 @@ struct EvalConvertOpPattern : public ShapeOpRewritePattern { + return rewriter.notifyMatchFailure( + op, "expected constant integer or float operand"); + + - return evalConvert(rewriter, op, elements, resultType); +-- } +--}; +-- +++ return foldConvert(rewriter, op, elements, resultType); ++ } ++ }; ++ + -struct EvalDivOpPattern : public FoldOpRewritePattern { + - using FoldOpRewritePattern::FoldOpRewritePattern; +-+ return foldConvert(rewriter, op, elements, resultType); +-+ } +-+}; +-+ + +struct FoldDivOpPattern : public ShapeOpRewritePattern { + + using ShapeOpRewritePattern::ShapeOpRewritePattern; + +@@ -1449,12 +1506,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + PatternRewriter& rewriter) const override { + - return evalElementwise(rewriter, op, + - [&](APSInt lhs, APSInt rhs) { return lhs / rhs; }); +-- } +--}; +-- +--struct EvalGetDimensionSizeOpPattern +-- : public FoldOpRewritePattern { +-- using FoldOpRewritePattern::FoldOpRewritePattern; + + auto resultType = op.getType(); + + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) + + return failure(); +@@ -1464,7 +1515,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + if (failed(res)) return failure(); + + rewriter.replaceOpWithNewOp(op, res.value()); + + return success(); +-+ } ++ } + + + + struct FoldDivide { + + FoldDivide(bool isUnsignedInt) +@@ -1479,8 +1530,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + static APInt foldUint(APInt lhs, APInt rhs) { return lhs.udiv(rhs); } + + static APInt foldSint(APInt lhs, APInt rhs) { return lhs.sdiv(rhs); } + + }; +-+}; +-+ ++ }; ++ ++-struct EvalGetDimensionSizeOpPattern ++- : public FoldOpRewritePattern { ++- using FoldOpRewritePattern::FoldOpRewritePattern; + +struct FoldGetDimensionSizeOpPattern + + : public ShapeOpRewritePattern { + + using ShapeOpRewritePattern::ShapeOpRewritePattern; +@@ -1494,7 +1548,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + return failure(); + + auto operandType = op.getOperand().getType(); +-@@ -567,86 +566,187 @@ ++@@ -567,86 +566,187 @@ struct EvalGetDimensionSizeOpPattern + } + }; + +@@ -1549,11 +1603,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + - return evalElementwise(rewriter, op, [&](APSInt lhs, APSInt rhs) { + - return lhs >= rhs ? lhs : rhs; + - }); +-- } +--}; +-- +--struct EvalMinOpPattern : public FoldOpRewritePattern { +-- using FoldOpRewritePattern::FoldOpRewritePattern; + + auto resultType = op.getType(); + + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) + + return failure(); +@@ -1563,9 +1612,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + if (failed(res)) return failure(); + + rewriter.replaceOpWithNewOp(op, res.value()); + + return success(); +-+ } +-+}; +-+ ++ } ++ }; ++ ++-struct EvalMinOpPattern : public FoldOpRewritePattern { ++- using FoldOpRewritePattern::FoldOpRewritePattern; + +struct FoldMinOpPattern : public ShapeOpRewritePattern { + + using ShapeOpRewritePattern::ShapeOpRewritePattern; + +@@ -1574,11 +1625,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + - return evalElementwise(rewriter, op, [&](APSInt lhs, APSInt rhs) { + - return lhs <= rhs ? lhs : rhs; + - }); +-- } +--}; +-- +--struct FoldMulOpPattern final : FoldOpRewritePattern { +-- using FoldOpRewritePattern::FoldOpRewritePattern; + + auto resultType = op.getType(); + + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) + + return failure(); +@@ -1588,19 +1634,35 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + if (failed(res)) return failure(); + + rewriter.replaceOpWithNewOp(op, res.value()); + + return success(); +-+ } +-+}; +-+ ++ } ++ }; ++ ++-struct FoldMulOpPattern final : FoldOpRewritePattern { ++- using FoldOpRewritePattern::FoldOpRewritePattern; + +// Clamp is folded using Min and Max folders. + +struct FoldClampOpPattern : public ShapeOpRewritePattern { + + using ShapeOpRewritePattern::ShapeOpRewritePattern; +-+ ++ ++- LogicalResult matchAndRewrite(mlir::stablehlo::MulOp op, + + LogicalResult matchAndRewrite(ClampOp op, +-+ PatternRewriter& rewriter) const override { ++ PatternRewriter& rewriter) const override { ++- TypedAttr lhsAttr; ++- matchPattern(op.getLhs(), m_Constant(&lhsAttr)); ++- ++- TypedAttr rhsAttr; ++- matchPattern(op.getRhs(), m_Constant(&rhsAttr)); ++- ++- if (TypedAttr res; ++- lhsAttr && rhsAttr && ++- (res = foldBinaryOpIntOrFloat(lhsAttr, rhsAttr, std::multiplies<>{}))) { ++- rewriter.replaceOpWithNewOp(op, res); ++- return success(); ++- } + + auto resultType = op.getType(); + + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) + + return failure(); +-+ ++ ++- return failure(); + + TypedAttr minAttr, operandAttr, maxAttr; + + matchPattern(op.getMin(), m_Constant(&minAttr)); + + matchPattern(op.getOperand(), m_Constant(&operandAttr)); +@@ -1620,43 +1682,19 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + if (!res) return rewriter.notifyMatchFailure(op, "failed to fold clamp"); + + rewriter.replaceOpWithNewOp(op, res); + + return success(); +-+ } +-+}; +-+ +-+struct FoldMulOpPattern final : ShapeOpRewritePattern { +-+ using ShapeOpRewritePattern::ShapeOpRewritePattern; ++ } ++ }; + +- LogicalResult matchAndRewrite(mlir::stablehlo::MulOp op, +- PatternRewriter& rewriter) const override { +-- TypedAttr lhsAttr; +-- matchPattern(op.getLhs(), m_Constant(&lhsAttr)); +-- +-- TypedAttr rhsAttr; +-- matchPattern(op.getRhs(), m_Constant(&rhsAttr)); +-- +-- if (TypedAttr res; +-- lhsAttr && rhsAttr && +-- (res = foldBinaryOpIntOrFloat(lhsAttr, rhsAttr, std::multiplies<>{}))) { +-- rewriter.replaceOpWithNewOp(op, res); +-- return success(); +-- } +-- +-- return failure(); +-- } +--}; +-- + -struct EvalMulOpPattern : public FoldOpRewritePattern { + - using FoldOpRewritePattern::FoldOpRewritePattern; +-- +++struct FoldMulOpPattern final : ShapeOpRewritePattern { +++ using ShapeOpRewritePattern::ShapeOpRewritePattern; ++ + - LogicalResult matchAndRewrite(MulOp op, +-- PatternRewriter& rewriter) const override { +++ LogicalResult matchAndRewrite(mlir::stablehlo::MulOp op, ++ PatternRewriter& rewriter) const override { + - return evalElementwise(rewriter, op, + - [&](APSInt lhs, APSInt rhs) { return lhs * rhs; }); +-- } +--}; +-- +--struct EvalOrOpPattern : public FoldOpRewritePattern { +-- using FoldOpRewritePattern::FoldOpRewritePattern; + + if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) + + return failure(); + + +@@ -1664,9 +1702,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + if (failed(res)) return failure(); + + rewriter.replaceOpWithNewOp(op, res.value()); + + return success(); +-+ } +-+}; +-+ ++ } ++ }; ++ ++-struct EvalOrOpPattern : public FoldOpRewritePattern { ++- using FoldOpRewritePattern::FoldOpRewritePattern; + +struct FoldOrOpPattern : public ShapeOpRewritePattern { + + using ShapeOpRewritePattern::ShapeOpRewritePattern; + +@@ -1680,16 +1720,11 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + - return evalElementwise(rewriter, op, [&](APSInt lhsInt, APSInt rhsInt) { + - return getAPSInt(resultType.getElementType(), lhsInt != 0 || rhsInt != 0); + - }); +-- } +--}; +-- +--struct EvalRemOpPattern : public FoldOpRewritePattern { +-- using FoldOpRewritePattern::FoldOpRewritePattern; + + auto res = foldBinaryOpIntOrFloat(rewriter, op, FoldOr{}); + + if (failed(res)) return failure(); + + rewriter.replaceOpWithNewOp(op, res.value()); + + return success(); +-+ } ++ } + + + + struct FoldOr { + + APInt operator()(APInt lhs, APInt rhs) const { +@@ -1699,8 +1734,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + return std::nullopt; + + } + + }; +-+}; +-+ ++ }; ++ ++-struct EvalRemOpPattern : public FoldOpRewritePattern { ++- using FoldOpRewritePattern::FoldOpRewritePattern; + +struct FoldRemOpPattern : public ShapeOpRewritePattern { + + using ShapeOpRewritePattern::ShapeOpRewritePattern; + +@@ -1708,10 +1745,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + PatternRewriter& rewriter) const override { + - return evalElementwise(rewriter, op, + - [&](APSInt lhs, APSInt rhs) { return lhs % rhs; }); +-- } +--}; +-- +--struct EvalReshapeOpPattern : public ShapeOpRewritePattern { + + auto resultType = op.getType(); + + if (failed(validateShapeFoldDtype(rewriter, op, resultType))) + + return failure(); +@@ -1721,7 +1754,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + if (failed(res)) return failure(); + + rewriter.replaceOpWithNewOp(op, res.value()); + + return success(); +-+ } ++ } + + + + struct FoldRem { + + FoldRem(bool isUnsignedInt) +@@ -1736,14 +1769,15 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + static APInt foldUint(APInt lhs, APInt rhs) { return lhs.urem(rhs); } + + static APInt foldSint(APInt lhs, APInt rhs) { return lhs.srem(rhs); } + + }; +-+}; +-+ ++ }; ++ ++-struct EvalReshapeOpPattern : public ShapeOpRewritePattern { + +// Pattern: reshape(cst, shape) -> cst + +struct FoldReshapeOpPattern : public ShapeOpRewritePattern { + using ShapeOpRewritePattern::ShapeOpRewritePattern; + + LogicalResult matchAndRewrite(ReshapeOp op, +-@@ -656,7 +756,6 @@ ++@@ -656,7 +756,6 @@ struct EvalReshapeOpPattern : public ShapeOpRewritePattern { + failed(validateShapeFoldDtype(rewriter, op, resultType))) + return failure(); + +@@ -1751,7 +1785,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + DenseIntOrFPElementsAttr attr; + if (!matchPattern(op.getOperand(), m_Constant(&attr))) + return rewriter.notifyMatchFailure(op, "expected constant operand"); +-@@ -665,53 +764,98 @@ ++@@ -665,53 +764,98 @@ struct EvalReshapeOpPattern : public ShapeOpRewritePattern { + } + }; + +@@ -1764,16 +1798,14 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + PatternRewriter& rewriter) const override { + auto resultType = op.getType(); + - if (failed(validateStaticShapeResult(rewriter, op, resultType))) +-- return failure(); +-- +++ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || +++ failed(validateShapeFoldDtype(rewriter, op, resultType))) ++ return failure(); ++ + - SmallVector pred, onTrue, onFalse; + - if (failed(hlo::matchInts(op.getPred(), pred)) || + - failed(hlo::matchInts(op.getOnTrue(), onTrue)) || + - failed(hlo::matchInts(op.getOnFalse(), onFalse))) +-+ if (failed(validateStaticShapeResult(rewriter, op, resultType)) || +-+ failed(validateShapeFoldDtype(rewriter, op, resultType))) +-+ return failure(); +-+ + + DenseIntElementsAttr predAttr; + + DenseElementsAttr onTrueAttr, onFalseAttr; + + matchPattern(op.getPred(), m_Constant(&predAttr)); +@@ -1783,14 +1815,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + return rewriter.notifyMatchFailure(op, "expected constant operands"); + + - SmallVector result; ++- for (auto [predEl, onTrueEl, onFalseEl] : ++- llvm::zip(pred, onTrue, onFalse)) { ++- result.push_back(predEl != 0 ? onTrueEl : onFalseEl); + + // Optimization, handle splat predicate + + if (isa(predAttr)) { + + auto pred = predAttr.getSplatValue(); + + rewriter.replaceOpWithNewOp( + + op, pred.isZero() ? onFalseAttr : onTrueAttr); + + return success(); +-+ } +-+ ++ } ++ + + // TODO: Enable float folding. + + if (op.getType().getElementType().isFloat()) + + return rewriter.notifyMatchFailure(op, "float select not supported yet"); +@@ -1800,27 +1835,17 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + return failure(); + + + + SmallVector result; +- for (auto [predEl, onTrueEl, onFalseEl] : +-- llvm::zip(pred, onTrue, onFalse)) { +-- result.push_back(predEl != 0 ? onTrueEl : onFalseEl); +-- } +-- +++ for (auto [predEl, onTrueEl, onFalseEl] : + + llvm::zip(predAttr.getValues(), onTrueAttr.getValues(), + + onFalseAttr.getValues())) { + + result.push_back(!predEl.isZero() ? onTrueEl : onFalseEl); + + } + rewriter.replaceOpWithNewOp( + - op, getTensorAttr(op.getType(), result)); +-- return success(); +-- } +--}; +-- +--struct EvalSignOpPattern : public FoldOpRewritePattern { +-- using FoldOpRewritePattern::FoldOpRewritePattern; + + op, DenseIntElementsAttr::get(resultType, result)); + + +-+ return success(); +-+ } ++ return success(); ++ } + + + + struct FoldSelect { + + std::optional operator()(APFloat pred, APFloat onTrue, +@@ -1832,8 +1857,10 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + return pred != 0 ? onTrue : onFalse; + + } + + }; +-+}; +-+ ++ }; ++ ++-struct EvalSignOpPattern : public FoldOpRewritePattern { ++- using FoldOpRewritePattern::FoldOpRewritePattern; + +struct FoldSignOpPattern : public ShapeOpRewritePattern { + + using ShapeOpRewritePattern::ShapeOpRewritePattern; + +@@ -1881,7 +1908,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + }; + + template +-@@ -749,13 +893,14 @@ ++@@ -749,13 +893,14 @@ DenseElementsAttr sliceType(SliceOp& op, const RangeType& data) { + ArrayRef(result)); + } + +@@ -1899,7 +1926,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + return failure(); + + auto operand = op.getOperand(); +-@@ -784,36 +929,18 @@ ++@@ -784,36 +929,18 @@ struct EvalSliceOpPattern : public FoldOpRewritePattern { + }; + + struct FoldSubtractOpPattern final +@@ -1930,14 +1957,13 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + - + -struct EvalSubtractOpPattern : public FoldOpRewritePattern { + - using FoldOpRewritePattern::FoldOpRewritePattern; +-- +++ if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) +++ return failure(); ++ + - LogicalResult matchAndRewrite(SubtractOp op, + - PatternRewriter& rewriter) const override { + - return evalElementwise(rewriter, op, + - [&](APSInt lhs, APSInt rhs) { return lhs - rhs; }); +-+ if (failed(validateShapeFoldDtype(rewriter, op, op.getType()))) +-+ return failure(); +-+ + + auto res = foldBinaryOpIntOrFloat(rewriter, op, std::minus<>{}); + + if (failed(res)) return failure(); + + rewriter.replaceOpWithNewOp(op, res.value()); +@@ -1945,23 +1971,35 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + } + }; + +-@@ -823,42 +950,38 @@ ++@@ -823,42 +950,38 @@ struct FoldSqrtOpPattern + + LogicalResult matchAndRewrite(mlir::stablehlo::SqrtOp op, + PatternRewriter& rewriter) const final { + - TypedAttr lhsAttr; + - matchPattern(op.getOperand(), m_Constant(&lhsAttr)); +-- +++ auto res = foldUnaryOpIntOrFloat(rewriter, op, FoldSqrt()); +++ if (failed(res)) return failure(); +++ rewriter.replaceOpWithNewOp(op, res.value()); +++ return success(); +++ } ++ + - if (!lhsAttr) + - return rewriter.notifyMatchFailure(op, "operand not constant"); +-- +++ struct FoldSqrt { +++ std::optional operator()(APFloat operand) { +++ if (operand.getSizeInBits(operand.getSemantics()) == 64) +++ return APFloat(std::sqrt(operand.convertToDouble())); ++ + - if (auto res = constFoldUnaryOp( + - lhsAttr, foldSqrt)) { + - rewriter.replaceOpWithNewOp( + - op, op.getType(), llvm::cast(res)); + - return success(); +-- } +-- +++ if (operand.getSizeInBits(operand.getSemantics()) == 32) +++ return APFloat(sqrtf(operand.convertToFloat())); +++ return std::nullopt; ++ } ++ + - return rewriter.notifyMatchFailure(op, "unable to fold sqrt"); + - } + - +@@ -1973,32 +2011,14 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + - return APFloat(sqrtf(a.convertToFloat())); + - return {}; + - } +--}; +-- +--struct EvalIotaOpPattern : public FoldOpRewritePattern { +-+ auto res = foldUnaryOpIntOrFloat(rewriter, op, FoldSqrt()); +-+ if (failed(res)) return failure(); +-+ rewriter.replaceOpWithNewOp(op, res.value()); +-+ return success(); +-+ } +-+ +-+ struct FoldSqrt { +-+ std::optional operator()(APFloat operand) { +-+ if (operand.getSizeInBits(operand.getSemantics()) == 64) +-+ return APFloat(std::sqrt(operand.convertToDouble())); +-+ +-+ if (operand.getSizeInBits(operand.getSemantics()) == 32) +-+ return APFloat(sqrtf(operand.convertToFloat())); +-+ return std::nullopt; +-+ } +-+ + + // TODO: Enable int folding. + + std::optional operator()(APInt operand) { + + return std::nullopt; + + } + + }; +-+}; +-+ ++ }; ++ ++-struct EvalIotaOpPattern : public FoldOpRewritePattern { + +struct FoldIotaOpPattern : public FoldOpRewritePattern { + using FoldOpRewritePattern::FoldOpRewritePattern; + +@@ -2015,7 +2035,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + + auto elementType = resultType.getElementType(); + +-@@ -929,7 +1052,7 @@ ++@@ -929,7 +1052,7 @@ DenseElementsAttr transposeType(TransposeOp& op, const RangeType& data) { + // transpose(constant) => constant with permuted dimensions + // This covers ranked tensor types with 0 dimensions(zero elements) and 0 + // rank(scalar), as well as splat values. +@@ -2024,7 +2044,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + using FoldOpRewritePattern::FoldOpRewritePattern; + + LogicalResult matchAndRewrite(TransposeOp op, +-@@ -943,6 +1066,7 @@ ++@@ -943,6 +1066,7 @@ struct EvalTransposeOpPattern : public FoldOpRewritePattern { + return rewriter.notifyMatchFailure( + op, "expected constant integer or float operand"); + +@@ -2032,7 +2052,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + DenseElementsAttr resAttr; + if (auto data = els.tryGetValues()) + resAttr = transposeType(op, *data); +-@@ -957,6 +1081,7 @@ ++@@ -957,6 +1081,7 @@ struct EvalTransposeOpPattern : public FoldOpRewritePattern { + } + }; + +@@ -2040,7 +2060,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + struct LowerBoolSplatConstantsIntoReduceOpRegion + : public FoldOpRewritePattern { + using FoldOpRewritePattern::FoldOpRewritePattern; +-@@ -1160,7 +1285,7 @@ ++@@ -1160,7 +1285,7 @@ bool hasNoDeclaredSideEffects(Operation* op) { + return true; + } + +@@ -2049,7 +2069,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + : public FoldOpRewritePattern { + using FoldOpRewritePattern::FoldOpRewritePattern; + +-@@ -1233,23 +1358,16 @@ ++@@ -1233,23 +1358,16 @@ void populateStablehloAggressiveFolderPatterns( + PatternBenefit benefit) { + populateStablehloShapeFolderPatterns(context, patterns, options, benefit); + +@@ -2083,7 +2103,7 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + } + + class StablehloTargetIndependentOptimizationPass { +-@@ -1266,25 +1384,25 @@ ++@@ -1266,25 +1384,25 @@ void populateStablehloShapeFolderPatterns( + MLIRContext* context, RewritePatternSet* patterns, + const StablehloAggressiveFolderPassOptions& options, + PatternBenefit benefit) { +@@ -2128,28 +2148,29 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveFold + } + + void populateStablehloShapeFolderPatterns(MLIRContext* context, +-diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td b/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td +---- stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td +-+++ stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td +-@@ -48,6 +48,8 @@ +- def RankEqual : Constraint< ++diff --git a/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td b/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td ++index 8adb6dbc..3a8edc61 100644 ++--- a/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td +++++ b/stablehlo/transforms/optimization/StablehloAggressiveSimplificationPatterns.td ++@@ -49,6 +49,8 @@ def RankEqual : Constraint< + CPred<"llvm::cast($0.getType()).getRank() == llvm::cast($1.getType()).getRank()">, + "same rank">; +-+ +-+def TensorDimsAllOne : Constraint, "all tensor dims are 1">; + +++def TensorDimsAllOne : Constraint, "all tensor dims are 1">; +++ + def TypesEqual : Constraint, "operands are equal">; + +-@@ -100,6 +102,8 @@ +- def ZeroExtent : AttrConstraint< ++ /////////// ++@@ -101,6 +103,8 @@ def ZeroExtent : AttrConstraint< + CPred<"cast($_self).getNumElements() == 0">, + "is zero extent">; +-+ +-+def AnyStaticShapeIntTensor : StaticShapeTensorOf<[HLO_Int]>; + +++def AnyStaticShapeIntTensor : StaticShapeTensorOf<[HLO_Int]>; +++ + /////////// + //// Native Code Call Utilities +-@@ -503,7 +507,7 @@ ++ ++@@ -503,7 +507,7 @@ def SelectOp_InvertBroadcastPredicateAndSwap + // Must be static shape, otherwise would require broadcasting via + // CHLO_ConstantLike. + def SubtractOp_FoldToZero +@@ -2158,4 +2179,6 @@ diff --ruN a/stablehlo/stablehlo/transforms/optimization/StablehloAggressiveSimp + (StableHLO_ConstantLike<"0"> $operand)>; + + // Pattern: subtract(X, 0) -> X ++-- ++2.48.1 + +-- +2.48.1 + diff --git a/third_party/modules/xla/20250612.0-6e48cbb/source.json b/third_party/modules/xla/20250612.0-6e48cbb/source.json new file mode 100644 index 00000000..6a4e208b --- /dev/null +++ b/third_party/modules/xla/20250612.0-6e48cbb/source.json @@ -0,0 +1,17 @@ +{ + "strip_prefix": "xla-6e48cbb8d33d771c964697e39bfaf678bcc6de31", + "url": "https://github.com/openxla/xla/archive/6e48cbb8d33d771c964697e39bfaf678bcc6de31.tar.gz", + "integrity": "sha256-i9lYvZ2MkzfyVW2Iu3qIucXIgGEhkbwsYXCrUZ6Yze8=", + "overlay": { + "llvm.bzl": "", + "MODULE.bazel": "", + "workspace_private.bzl": "", + "xla.bzl": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", + "0003-Revert-Add-optional-allowOtherDialects-field-to-stab.patch": "" + } +} diff --git a/third_party/modules/xla/20250710.0-22ea002/MODULE.bazel b/third_party/modules/xla/20250710.0-22ea002/MODULE.bazel new file mode 100644 index 00000000..09bc2307 --- /dev/null +++ b/third_party/modules/xla/20250710.0-22ea002/MODULE.bazel @@ -0,0 +1,58 @@ +module( + name = "xla", + version = "20250710.0-22ea002", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.39.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") +bazel_dep(name = "rules_shell", version = "0.4.1") +bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") + +toolchains_private = use_extension("//:toolchains_private.bzl", "toolchains_private") +use_repo( + toolchains_private, + "rules_ml_toolchain", +) + +workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") +use_repo( + workspace_private, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "python_version_repo", + "tsl", +) + +workspace_public = use_extension("//:xla.bzl", "xla") +use_repo( + workspace_public, + "llvm-raw", + "stablehlo", + "triton", +) + +llvm = use_extension("//:llvm.bzl", "llvm") +llvm.configure( + targets = [ + "AArch64", + "AMDGPU", + "NVPTX", + "X86", + ], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/MODULE.bazel b/third_party/modules/xla/20250710.0-22ea002/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250710.0-22ea002/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/llvm.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/llvm.bzl new file mode 100644 index 00000000..b4a2fe4f --- /dev/null +++ b/third_party/modules/xla/20250710.0-22ea002/overlay/llvm.bzl @@ -0,0 +1,30 @@ +load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list( + default = [], + ), + }, + ), + }, +) diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/toolchains_private.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/toolchains_private.bzl new file mode 100644 index 00000000..b8c79070 --- /dev/null +++ b/third_party/modules/xla/20250710.0-22ea002/overlay/toolchains_private.bzl @@ -0,0 +1,21 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def _toolchains_private_impl(mctx): + http_archive( + name = "rules_ml_toolchain", + sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", + strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", + urls = [ + "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", + ], + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +toolchains_private = module_extension( + implementation = _toolchains_private_impl, +) diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/workspace_private.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/workspace_private.bzl new file mode 100644 index 00000000..9dfe0b16 --- /dev/null +++ b/third_party/modules/xla/20250710.0-22ea002/overlay/workspace_private.bzl @@ -0,0 +1,73 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") +load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/py:python_repo.bzl", "python_repository") +load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") +load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _workspace_private_impl(mctx): + http_archive( + name = "rules_ml_toolchain", + sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", + strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", + urls = [ + "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", + ], + ) + + # Use cuda_configure from XLA to make it work with bzlmod. + # A pure bzlmod solution for rules_ml_toolchain is impossible because of the legacy design. + # It relies on a "generate-then-load" pattern that creates a deadlock in Bazel's architecture: + # - Generate: First, it runs a rule to generate a .bzl file containing configuration data. + # - Load: Then, it requires a load() statement to load that same file to continue the setup. + # This fails in bzlmod because Bazel's Loading Phase (when load() statements are processed) happens before + # the Analysis Phase (when repository rules are run). + # This creates a fundamental chicken-and-egg problem: the build tries to load a file that has not been generated yet. + # Without using the official WORKSPACE.bzlmod escape hatch, + # this incompatibility cannot be resolved without modifying the upstream rules. + + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_vendored(name = "tsl", relpath = "third_party/tsl") + pybind11_bazel() + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "afbc5d78d6ba6d509cc6e264de0d49dcd7304db435cbf2d630385bacf49e066c", + strip_prefix = "grpc-1.68.2", + patch_file = [ + "//third_party/grpc:grpc.patch", + ], + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.68.2.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["//third_party/protobuf:protobuf.patch"], + sha256 = "f645e6e42745ce922ca5388b1883ca583bafe4366cc74cf35c3c9299005136e2", + strip_prefix = "protobuf-5.28.3", + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/refs/tags/v5.28.3.zip"), + ) + + python_repository( + name = "python_version_repo", + requirements_versions = ["3.11"], + requirements_locks = ["//:requirements_lock_3_11.txt"], + local_wheel_workspaces = [], + local_wheel_dist_folder = None, + default_python_version = None, + local_wheel_inclusion_list = ["*"], + local_wheel_exclusion_list = [], + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +workspace_private = module_extension( + implementation = _workspace_private_impl, +) diff --git a/third_party/modules/xla/20250710.0-22ea002/overlay/xla.bzl b/third_party/modules/xla/20250710.0-22ea002/overlay/xla.bzl new file mode 100644 index 00000000..f14bf2a9 --- /dev/null +++ b/third_party/modules/xla/20250710.0-22ea002/overlay/xla.bzl @@ -0,0 +1,17 @@ +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/triton:workspace.bzl", triton = "repo") + +def _xla_impl(mctx): + triton() + llvm("llvm-raw") + stablehlo() + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla = module_extension( + implementation = _xla_impl, +) diff --git a/third_party/modules/xla/20250710.0-22ea002/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250710.0-22ea002/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/modules/xla/20250710.0-22ea002/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250710.0-22ea002/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250710.0-22ea002/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch new file mode 100644 index 00000000..938ef401 --- /dev/null +++ b/third_party/modules/xla/20250710.0-22ea002/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -0,0 +1,135 @@ +From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Tue, 27 May 2025 11:48:17 +0200 +Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt + +PR: https://github.com/openxla/xla/pull/13420 +--- + xla/pjrt/c/BUILD | 5 +++++ + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ + xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- + 3 files changed, 57 insertions(+), 1 deletion(-) + +diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD +index 79f18fa0bc..0f33dd8a6e 100644 +--- a/xla/pjrt/c/BUILD ++++ b/xla/pjrt/c/BUILD +@@ -69,8 +69,13 @@ cc_library( + ":pjrt_c_api_wrapper_impl", + "//xla/ffi:execution_context", + "//xla/ffi:type_id_registry", ++ "//xla/ffi:ffi_api", ++ "//xla/ffi/api:c_api", ++ "//xla/ffi/api:ffi", ++ "//xla/service:custom_call_target_registry", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/strings:str_format", + ], + ) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index 995a2c7e50..b8f10bc2f7 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); + // Adds a user data to the execute context. + typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); + ++typedef enum PJRT_FFI_Handler_TraitsBits { ++ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, ++} PJRT_FFI_Handler_TraitsBits; ++ ++struct PJRT_FFI_Register_Handler_Args { ++ size_t struct_size; ++ const char* target_name; ++ size_t target_name_size; ++ int api_version; // 0 for an untyped call, 1 -- for typed ++ void* handler; ++ const char* platform_name; ++ size_t platform_name_size; ++ PJRT_FFI_Handler_TraitsBits traits; ++}; ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); ++ ++// Registers an FFI call handler for a specific platform. ++typedef PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args); ++ + typedef struct PJRT_FFI_Extension { + PJRT_Extension_Base base; + PJRT_FFI_TypeID_Register* type_id_register; + PJRT_FFI_UserData_Add* user_data_add; ++ PJRT_FFI_Register_Handler* register_handler; + } PJRT_FFI; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +index 5fa88eab33..763270331b 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc ++++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" ++#include + + #include "absl/status/status.h" ++#include "absl/strings/str_format.h" + #include "absl/strings/string_view.h" ++#include "xla/ffi/api/c_api.h" + #include "xla/ffi/execution_context.h" + #include "xla/ffi/type_id_registry.h" ++#include "xla/ffi/ffi_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" + #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" + #include "xla/pjrt/c/pjrt_c_api_helpers.h" + #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/service/custom_call_target_registry.h" + + namespace pjrt { + +@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { + return nullptr; + } + ++static PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args) { ++ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( ++ "PJRT_FFI_Register_Handler_Args", ++ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); ++ std::string target_name(args->target_name, args->target_name_size); ++ std::string platform_name(args->platform_name, args->platform_name_size); ++ switch (args->api_version) { ++ case 0: ++ xla::CustomCallTargetRegistry::Global()->Register( ++ target_name, args->handler, platform_name); ++ return nullptr; ++ case 1: ++ xla::ffi::Ffi::RegisterStaticHandler( ++ xla::ffi::GetXlaFfiApi(), target_name, platform_name, ++ reinterpret_cast(args->handler)); ++ return nullptr; ++ default: ++ return new PJRT_Error{absl::UnimplementedError( ++ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " ++ "Supported versions are 0 and 1.", ++ args->api_version))}; ++ } ++} ++ + PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + return { + PJRT_Extension_Base{ +@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + }, + /*type_id_register=*/PJRT_FFI_TypeID_Register, + /*user_data_add=*/PJRT_FFI_UserData_Add, ++ /*register_handler=*/PJRT_FFI_Register_Handler, + }; + } + +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/modules/xla/20250710.0-22ea002/patches/0003-Remove-unconventional-C-code-in-headers.patch b/third_party/modules/xla/20250710.0-22ea002/patches/0003-Remove-unconventional-C-code-in-headers.patch new file mode 100644 index 00000000..6df5b1b3 --- /dev/null +++ b/third_party/modules/xla/20250710.0-22ea002/patches/0003-Remove-unconventional-C-code-in-headers.patch @@ -0,0 +1,124 @@ +From 6078da86a46b6f0d983dccb9ae4f36fc90640247 Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Fri, 11 Jul 2025 14:05:16 +0200 +Subject: [PATCH] zml patch + +--- + third_party/stablehlo/workspace.bzl | 1 + + third_party/stablehlo/zml.patch | 93 +++++++++++++++++++++++++++++ + 2 files changed, 94 insertions(+) + create mode 100644 third_party/stablehlo/zml.patch + +diff --git a/third_party/stablehlo/workspace.bzl b/third_party/stablehlo/workspace.bzl +index d9d5063744..44980948d0 100644 +--- a/third_party/stablehlo/workspace.bzl ++++ b/third_party/stablehlo/workspace.bzl +@@ -15,5 +15,6 @@ def repo(): + urls = tf_mirror_urls("https://github.com/openxla/stablehlo/archive/{commit}.zip".format(commit = STABLEHLO_COMMIT)), + patch_file = [ + "//third_party/stablehlo:temporary.patch", # Autogenerated, don't remove. ++ "//third_party/stablehlo:zml.patch", # Autogenerated, don't remove. + ], + ) +diff --git a/third_party/stablehlo/zml.patch b/third_party/stablehlo/zml.patch +new file mode 100644 +index 0000000000..2a09384582 +--- /dev/null ++++ b/third_party/stablehlo/zml.patch +@@ -0,0 +1,93 @@ ++From e38ab68376dd8a17ebf4469d2c8350f521310182 Mon Sep 17 00:00:00 2001 ++From: Hugo Mano ++Date: Fri, 11 Jul 2025 12:08:35 +0200 ++Subject: [PATCH] zml patch ++ ++--- ++ stablehlo/dialect/Serialization.cpp | 5 ++--- ++ stablehlo/dialect/Serialization.h | 3 +-- ++ stablehlo/integrations/c/StablehloDialectApi.cpp | 3 +-- ++ stablehlo/integrations/c/StablehloDialectApi.h | 2 +- ++ stablehlo/tools/StablehloTranslateMain.cpp | 2 +- ++ 5 files changed, 6 insertions(+), 9 deletions(-) ++ ++diff --git a/stablehlo/dialect/Serialization.cpp b/stablehlo/dialect/Serialization.cpp ++index cb89d673..4370d588 100644 ++--- a/stablehlo/dialect/Serialization.cpp +++++ b/stablehlo/dialect/Serialization.cpp ++@@ -39,8 +39,7 @@ namespace stablehlo { ++ ++ LogicalResult serializePortableArtifact(ModuleOp module, ++ StringRef targetVersion, ++- raw_ostream& os, ++- bool allowOtherDialects) { +++ raw_ostream& os) { ++ MLIRContext* context = module.getContext(); ++ ++ // Convert StableHLO --> VHLO. ++@@ -49,7 +48,7 @@ LogicalResult serializePortableArtifact(ModuleOp module, ++ { ++ PassManager pm(context); ++ StablehloLegalizeToVhloPassOptions options; ++- options.allowOtherDialects = allowOtherDialects; +++ options.allowOtherDialects = false; ++ pm.addPass(stablehlo::createStablehloLegalizeToVhloPass(options)); ++ if (!succeeded(pm.run(module))) { ++ return failure(); ++diff --git a/stablehlo/dialect/Serialization.h b/stablehlo/dialect/Serialization.h ++index 811ca97b..abe95e63 100644 ++--- a/stablehlo/dialect/Serialization.h +++++ b/stablehlo/dialect/Serialization.h ++@@ -34,8 +34,7 @@ namespace stablehlo { ++ // unsupported dialects. ++ LogicalResult serializePortableArtifact(ModuleOp module, ++ StringRef targetVersion, ++- raw_ostream& os, ++- bool allowOtherDialects = false); +++ raw_ostream& os); ++ ++ // Read StableHLO portable artifact ++ // ++diff --git a/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/integrations/c/StablehloDialectApi.cpp ++index 343f8d0b..8f52e4d5 100644 ++--- a/stablehlo/integrations/c/StablehloDialectApi.cpp +++++ b/stablehlo/integrations/c/StablehloDialectApi.cpp ++@@ -81,8 +81,7 @@ MlirLogicalResult stablehloSerializePortableArtifactFromModule( ++ MlirStringCallback callback, void *userData, bool allowOtherDialects) { ++ mlir::detail::CallbackOstream stream(callback, userData); ++ if (failed(mlir::stablehlo::serializePortableArtifact( ++- unwrap(moduleStr), unwrap(targetVersion), stream, ++- allowOtherDialects))) +++ unwrap(moduleStr), unwrap(targetVersion), stream))) ++ return mlirLogicalResultFailure(); ++ return mlirLogicalResultSuccess(); ++ } ++diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h ++index 385156bf..24d11c1d 100644 ++--- a/stablehlo/integrations/c/StablehloDialectApi.h +++++ b/stablehlo/integrations/c/StablehloDialectApi.h ++@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, ++ MlirStringRef targetVersion, ++ MlirStringCallback callback, ++ void* userData, ++- bool allowOtherDialects = false); +++ bool allowOtherDialects); ++ ++ // Read a StableHLO program from a portable artifact, returning the module as ++ // MLIR bytecode. Note, this bytecode returned is not a portable artifact, ++diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp ++index fdf0d6a9..8d5c8752 100644 ++--- a/stablehlo/tools/StablehloTranslateMain.cpp +++++ b/stablehlo/tools/StablehloTranslateMain.cpp ++@@ -323,7 +323,7 @@ TranslateFromMLIRRegistration serializeRegistration( ++ } ++ ++ return stablehlo::serializePortableArtifact( ++- module, targetVersion, os, allowOtherDialectsOption.getValue()); +++ module, targetVersion, os); ++ }, ++ [](DialectRegistry ®istry) { ++ mlir::registerAllDialects(registry); ++-- ++2.39.5 (Apple Git-154) ++ +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/modules/xla/20250710.0-22ea002/source.json b/third_party/modules/xla/20250710.0-22ea002/source.json new file mode 100644 index 00000000..d01c18a3 --- /dev/null +++ b/third_party/modules/xla/20250710.0-22ea002/source.json @@ -0,0 +1,18 @@ +{ + "strip_prefix": "xla-ef07e787ea1303fa2f8d8a175d24d434bfb84107", + "url": "https://github.com/openxla/xla/archive/ef07e787ea1303fa2f8d8a175d24d434bfb84107.tar.gz", + "integrity": "sha256-OL0e3Y+dgu/DM5MV+LsWvtOI9rUGLvXbwIU/9bnHGXc=", + "overlay": { + "toolchains_private.bzl": "", + "llvm.bzl": "", + "MODULE.bazel": "", + "workspace_private.bzl": "", + "xla.bzl": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", + "0003-Remove-unconventional-C-code-in-headers.patch": "" + } +} diff --git a/third_party/modules/xla/20250718.0-6319f0d/MODULE.bazel b/third_party/modules/xla/20250718.0-6319f0d/MODULE.bazel new file mode 100644 index 00000000..6c1bde70 --- /dev/null +++ b/third_party/modules/xla/20250718.0-6319f0d/MODULE.bazel @@ -0,0 +1,58 @@ +module( + name = "xla", + version = "20250718.0-6319f0d", + compatibility_level = 1, +) + +bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.5.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +bazel_dep(name = "rules_apple", version = "3.22.0", repo_name = "build_bazel_rules_apple") +bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "com_google_absl") +bazel_dep(name = "rules_python", version = "0.39.0") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") +bazel_dep(name = "rules_java", version = "7.3.2") +bazel_dep(name = "rules_pkg", version = "0.9.1") +bazel_dep(name = "zlib", version = "1.2.13") +bazel_dep(name = "re2", version = "2024-07-02.bcr.1", repo_name = "com_googlesource_code_re2") +bazel_dep(name = "rules_license", version = "0.0.8") +bazel_dep(name = "rules_shell", version = "0.4.1") +bazel_dep(name = "bazel_features", version = "1.25.0", repo_name = "proto_bazel_features") + +toolchains_private = use_extension("//:toolchains_private.bzl", "toolchains_private") +use_repo( + toolchains_private, + "rules_ml_toolchain", +) + +workspace_private = use_extension("//:workspace_private.bzl", "workspace_private") +use_repo( + workspace_private, + "com_github_grpc_grpc", + "com_google_protobuf", + "local_config_cuda", + "local_config_remote_execution", + "local_config_rocm", + "local_config_tensorrt", + "python_version_repo", + "tsl", +) + +workspace_public = use_extension("//:xla.bzl", "xla") +use_repo( + workspace_public, + "llvm-raw", + "stablehlo", + "triton", +) + +llvm = use_extension("//:llvm.bzl", "llvm") +llvm.configure( + targets = [ + "AArch64", + "AMDGPU", + "NVPTX", + "X86", + ], +) +use_repo(llvm, "llvm-project") diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/MODULE.bazel b/third_party/modules/xla/20250718.0-6319f0d/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/xla/20250718.0-6319f0d/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/llvm.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/llvm.bzl new file mode 100644 index 00000000..b4a2fe4f --- /dev/null +++ b/third_party/modules/xla/20250718.0-6319f0d/overlay/llvm.bzl @@ -0,0 +1,30 @@ +load("@llvm-raw//utils/bazel:configure.bzl", _llvm_configure = "llvm_configure") + +def _llvm_impl(mctx): + _targets = {} + for mod in mctx.modules: + for conf in mod.tags.configure: + for target in conf.targets: + _targets[target] = True + _llvm_configure( + name = "llvm-project", + targets = _targets.keys(), + ) + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +llvm = module_extension( + implementation = _llvm_impl, + tag_classes = { + "configure": tag_class( + attrs = { + "targets": attr.string_list( + default = [], + ), + }, + ), + }, +) diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/toolchains_private.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/toolchains_private.bzl new file mode 100644 index 00000000..b8c79070 --- /dev/null +++ b/third_party/modules/xla/20250718.0-6319f0d/overlay/toolchains_private.bzl @@ -0,0 +1,21 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +def _toolchains_private_impl(mctx): + http_archive( + name = "rules_ml_toolchain", + sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", + strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", + urls = [ + "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", + ], + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +toolchains_private = module_extension( + implementation = _toolchains_private_impl, +) diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/workspace_private.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/workspace_private.bzl new file mode 100644 index 00000000..9dfe0b16 --- /dev/null +++ b/third_party/modules/xla/20250718.0-6319f0d/overlay/workspace_private.bzl @@ -0,0 +1,73 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") +load("//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") +load("//third_party/gpus:cuda_configure.bzl", "cuda_configure") +load("//third_party/gpus:rocm_configure.bzl", "rocm_configure") +load("//third_party/py:python_repo.bzl", "python_repository") +load("//third_party/pybind11_bazel:workspace.bzl", pybind11_bazel = "repo") +load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure") +load("//tools/toolchains/remote:configure.bzl", "remote_execution_configure") + +def _workspace_private_impl(mctx): + http_archive( + name = "rules_ml_toolchain", + sha256 = "fb78d09234528aef2be856820b69b76486829f65e4eb3c7ffaa5803b667fa441", + strip_prefix = "rules_ml_toolchain-f4ad89fa906be2c1374785a79335c8a7dcd49df7", + urls = [ + "https://github.com/zml/rules_ml_toolchain/archive/f4ad89fa906be2c1374785a79335c8a7dcd49df7.tar.gz", + ], + ) + + # Use cuda_configure from XLA to make it work with bzlmod. + # A pure bzlmod solution for rules_ml_toolchain is impossible because of the legacy design. + # It relies on a "generate-then-load" pattern that creates a deadlock in Bazel's architecture: + # - Generate: First, it runs a rule to generate a .bzl file containing configuration data. + # - Load: Then, it requires a load() statement to load that same file to continue the setup. + # This fails in bzlmod because Bazel's Loading Phase (when load() statements are processed) happens before + # the Analysis Phase (when repository rules are run). + # This creates a fundamental chicken-and-egg problem: the build tries to load a file that has not been generated yet. + # Without using the official WORKSPACE.bzlmod escape hatch, + # this incompatibility cannot be resolved without modifying the upstream rules. + + cuda_configure(name = "local_config_cuda") + remote_execution_configure(name = "local_config_remote_execution") + rocm_configure(name = "local_config_rocm") + tensorrt_configure(name = "local_config_tensorrt") + tf_vendored(name = "tsl", relpath = "third_party/tsl") + pybind11_bazel() + tf_http_archive( + name = "com_github_grpc_grpc", + sha256 = "afbc5d78d6ba6d509cc6e264de0d49dcd7304db435cbf2d630385bacf49e066c", + strip_prefix = "grpc-1.68.2", + patch_file = [ + "//third_party/grpc:grpc.patch", + ], + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.68.2.tar.gz"), + ) + tf_http_archive( + name = "com_google_protobuf", + patch_file = ["//third_party/protobuf:protobuf.patch"], + sha256 = "f645e6e42745ce922ca5388b1883ca583bafe4366cc74cf35c3c9299005136e2", + strip_prefix = "protobuf-5.28.3", + urls = tf_mirror_urls("https://github.com/protocolbuffers/protobuf/archive/refs/tags/v5.28.3.zip"), + ) + + python_repository( + name = "python_version_repo", + requirements_versions = ["3.11"], + requirements_locks = ["//:requirements_lock_3_11.txt"], + local_wheel_workspaces = [], + local_wheel_dist_folder = None, + default_python_version = None, + local_wheel_inclusion_list = ["*"], + local_wheel_exclusion_list = [], + ) + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +workspace_private = module_extension( + implementation = _workspace_private_impl, +) diff --git a/third_party/modules/xla/20250718.0-6319f0d/overlay/xla.bzl b/third_party/modules/xla/20250718.0-6319f0d/overlay/xla.bzl new file mode 100644 index 00000000..f14bf2a9 --- /dev/null +++ b/third_party/modules/xla/20250718.0-6319f0d/overlay/xla.bzl @@ -0,0 +1,17 @@ +load("//third_party/llvm:workspace.bzl", llvm = "repo") +load("//third_party/stablehlo:workspace.bzl", stablehlo = "repo") +load("//third_party/triton:workspace.bzl", triton = "repo") + +def _xla_impl(mctx): + triton() + llvm("llvm-raw") + stablehlo() + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], + ) + +xla = module_extension( + implementation = _xla_impl, +) diff --git a/third_party/modules/xla/20250718.0-6319f0d/patches/0001-bazel-migration-to-bazel-8.1.1.patch b/third_party/modules/xla/20250718.0-6319f0d/patches/0001-bazel-migration-to-bazel-8.1.1.patch new file mode 100644 index 00000000..8924cf4b --- /dev/null +++ b/third_party/modules/xla/20250718.0-6319f0d/patches/0001-bazel-migration-to-bazel-8.1.1.patch @@ -0,0 +1,41 @@ +From 6cf475b500521c1b8be06f590fdbc1818f0dc44b Mon Sep 17 00:00:00 2001 +From: Jean-Baptiste Dalido +Date: Mon, 6 Jan 2025 13:33:13 +0100 +Subject: [PATCH] bazel: migration to bazel 8.0.1 + +--- + .bazelversion | 2 +- + third_party/tsl/third_party/gpus/cuda_configure.bzl | 4 ++-- + 2 files changed, 3 insertions(+), 3 deletions(-) + +diff --git a/.bazelversion b/.bazelversion +index f22d756da3..fa5fce04b3 100644 +--- a/.bazelversion ++++ b/.bazelversion +@@ -1 +1 @@ +-7.4.1 ++8.1.1 +\ No newline at end of file +diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl +index d62531152d..71d80a5a99 100644 +--- a/third_party/gpus/cuda_configure.bzl ++++ b/third_party/gpus/cuda_configure.bzl +@@ -33,14 +33,14 @@ NB: DEPRECATED! Use `hermetic/cuda_configure` rule instead. + load( + "@bazel_tools//tools/cpp:lib_cc_configure.bzl", + "escape_string", +- "get_env_var", + ) + load( + "@bazel_tools//tools/cpp:windows_cc_configure.bzl", +- "find_msvc_tool", + "find_vc_path", + "setup_vc_env_vars", + ) ++load("@rules_cc//cc/private/toolchain:windows_cc_configure.bzl", "find_msvc_tool") ++load("@rules_cc//cc/private/toolchain:lib_cc_configure.bzl", "get_env_var") + load("//third_party/clang_toolchain:download_clang.bzl", "download_clang") + load( + "//third_party/remote_config:common.bzl", +-- +2.39.3 (Apple Git-146) diff --git a/third_party/modules/xla/20250718.0-6319f0d/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch b/third_party/modules/xla/20250718.0-6319f0d/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch new file mode 100644 index 00000000..938ef401 --- /dev/null +++ b/third_party/modules/xla/20250718.0-6319f0d/patches/0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch @@ -0,0 +1,135 @@ +From 2ae9bb9d24b569c2c6bfab3c54b428103614944d Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Tue, 27 May 2025 11:48:17 +0200 +Subject: [PATCH 1/8] Added FFI handler registration API to the FFI PjRt + +PR: https://github.com/openxla/xla/pull/13420 +--- + xla/pjrt/c/BUILD | 5 +++++ + xla/pjrt/c/pjrt_c_api_ffi_extension.h | 21 ++++++++++++++++++ + xla/pjrt/c/pjrt_c_api_ffi_internal.cc | 32 ++++++++++++++++++++++++++- + 3 files changed, 57 insertions(+), 1 deletion(-) + +diff --git a/xla/pjrt/c/BUILD b/xla/pjrt/c/BUILD +index 79f18fa0bc..0f33dd8a6e 100644 +--- a/xla/pjrt/c/BUILD ++++ b/xla/pjrt/c/BUILD +@@ -69,8 +69,13 @@ cc_library( + ":pjrt_c_api_wrapper_impl", + "//xla/ffi:execution_context", + "//xla/ffi:type_id_registry", ++ "//xla/ffi:ffi_api", ++ "//xla/ffi/api:c_api", ++ "//xla/ffi/api:ffi", ++ "//xla/service:custom_call_target_registry", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings:string_view", ++ "@com_google_absl//absl/strings:str_format", + ], + ) + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_extension.h b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +index 995a2c7e50..b8f10bc2f7 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_extension.h ++++ b/xla/pjrt/c/pjrt_c_api_ffi_extension.h +@@ -69,10 +69,31 @@ PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_UserData_Add_Args, user_data); + // Adds a user data to the execute context. + typedef PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args); + ++typedef enum PJRT_FFI_Handler_TraitsBits { ++ PJRT_FFI_HANDLER_TRAITS_COMMAND_BUFFER_COMPATIBLE = 1u << 0, ++} PJRT_FFI_Handler_TraitsBits; ++ ++struct PJRT_FFI_Register_Handler_Args { ++ size_t struct_size; ++ const char* target_name; ++ size_t target_name_size; ++ int api_version; // 0 for an untyped call, 1 -- for typed ++ void* handler; ++ const char* platform_name; ++ size_t platform_name_size; ++ PJRT_FFI_Handler_TraitsBits traits; ++}; ++PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Register_Handler_Args, traits); ++ ++// Registers an FFI call handler for a specific platform. ++typedef PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args); ++ + typedef struct PJRT_FFI_Extension { + PJRT_Extension_Base base; + PJRT_FFI_TypeID_Register* type_id_register; + PJRT_FFI_UserData_Add* user_data_add; ++ PJRT_FFI_Register_Handler* register_handler; + } PJRT_FFI; + PJRT_DEFINE_STRUCT_TRAITS(PJRT_FFI_Extension, user_data_add); + +diff --git a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +index 5fa88eab33..763270331b 100644 +--- a/xla/pjrt/c/pjrt_c_api_ffi_internal.cc ++++ b/xla/pjrt/c/pjrt_c_api_ffi_internal.cc +@@ -13,16 +13,20 @@ See the License for the specific language governing permissions and + limitations under the License. + ==============================================================================*/ + +-#include "xla/pjrt/c/pjrt_c_api_ffi_internal.h" ++#include + + #include "absl/status/status.h" ++#include "absl/strings/str_format.h" + #include "absl/strings/string_view.h" ++#include "xla/ffi/api/c_api.h" + #include "xla/ffi/execution_context.h" + #include "xla/ffi/type_id_registry.h" ++#include "xla/ffi/ffi_api.h" + #include "xla/pjrt/c/pjrt_c_api.h" + #include "xla/pjrt/c/pjrt_c_api_ffi_extension.h" + #include "xla/pjrt/c/pjrt_c_api_helpers.h" + #include "xla/pjrt/c/pjrt_c_api_wrapper_impl.h" ++#include "xla/service/custom_call_target_registry.h" + + namespace pjrt { + +@@ -68,6 +72,31 @@ static PJRT_Error* PJRT_FFI_UserData_Add(PJRT_FFI_UserData_Add_Args* args) { + return nullptr; + } + ++static PJRT_Error* PJRT_FFI_Register_Handler( ++ PJRT_FFI_Register_Handler_Args* args) { ++ PJRT_RETURN_IF_ERROR(ActualStructSizeIsGreaterOrEqual( ++ "PJRT_FFI_Register_Handler_Args", ++ PJRT_FFI_Register_Handler_Args_STRUCT_SIZE, args->struct_size)); ++ std::string target_name(args->target_name, args->target_name_size); ++ std::string platform_name(args->platform_name, args->platform_name_size); ++ switch (args->api_version) { ++ case 0: ++ xla::CustomCallTargetRegistry::Global()->Register( ++ target_name, args->handler, platform_name); ++ return nullptr; ++ case 1: ++ xla::ffi::Ffi::RegisterStaticHandler( ++ xla::ffi::GetXlaFfiApi(), target_name, platform_name, ++ reinterpret_cast(args->handler)); ++ return nullptr; ++ default: ++ return new PJRT_Error{absl::UnimplementedError( ++ absl::StrFormat("API version %d not supported for PJRT GPU plugin. " ++ "Supported versions are 0 and 1.", ++ args->api_version))}; ++ } ++} ++ + PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + return { + PJRT_Extension_Base{ +@@ -77,6 +106,7 @@ PJRT_FFI_Extension CreateFfiExtension(PJRT_Extension_Base* next) { + }, + /*type_id_register=*/PJRT_FFI_TypeID_Register, + /*user_data_add=*/PJRT_FFI_UserData_Add, ++ /*register_handler=*/PJRT_FFI_Register_Handler, + }; + } + +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/modules/xla/20250718.0-6319f0d/patches/0003-Remove-unconventional-C-code-in-headers.patch b/third_party/modules/xla/20250718.0-6319f0d/patches/0003-Remove-unconventional-C-code-in-headers.patch new file mode 100644 index 00000000..6df5b1b3 --- /dev/null +++ b/third_party/modules/xla/20250718.0-6319f0d/patches/0003-Remove-unconventional-C-code-in-headers.patch @@ -0,0 +1,124 @@ +From 6078da86a46b6f0d983dccb9ae4f36fc90640247 Mon Sep 17 00:00:00 2001 +From: Hugo Mano +Date: Fri, 11 Jul 2025 14:05:16 +0200 +Subject: [PATCH] zml patch + +--- + third_party/stablehlo/workspace.bzl | 1 + + third_party/stablehlo/zml.patch | 93 +++++++++++++++++++++++++++++ + 2 files changed, 94 insertions(+) + create mode 100644 third_party/stablehlo/zml.patch + +diff --git a/third_party/stablehlo/workspace.bzl b/third_party/stablehlo/workspace.bzl +index d9d5063744..44980948d0 100644 +--- a/third_party/stablehlo/workspace.bzl ++++ b/third_party/stablehlo/workspace.bzl +@@ -15,5 +15,6 @@ def repo(): + urls = tf_mirror_urls("https://github.com/openxla/stablehlo/archive/{commit}.zip".format(commit = STABLEHLO_COMMIT)), + patch_file = [ + "//third_party/stablehlo:temporary.patch", # Autogenerated, don't remove. ++ "//third_party/stablehlo:zml.patch", # Autogenerated, don't remove. + ], + ) +diff --git a/third_party/stablehlo/zml.patch b/third_party/stablehlo/zml.patch +new file mode 100644 +index 0000000000..2a09384582 +--- /dev/null ++++ b/third_party/stablehlo/zml.patch +@@ -0,0 +1,93 @@ ++From e38ab68376dd8a17ebf4469d2c8350f521310182 Mon Sep 17 00:00:00 2001 ++From: Hugo Mano ++Date: Fri, 11 Jul 2025 12:08:35 +0200 ++Subject: [PATCH] zml patch ++ ++--- ++ stablehlo/dialect/Serialization.cpp | 5 ++--- ++ stablehlo/dialect/Serialization.h | 3 +-- ++ stablehlo/integrations/c/StablehloDialectApi.cpp | 3 +-- ++ stablehlo/integrations/c/StablehloDialectApi.h | 2 +- ++ stablehlo/tools/StablehloTranslateMain.cpp | 2 +- ++ 5 files changed, 6 insertions(+), 9 deletions(-) ++ ++diff --git a/stablehlo/dialect/Serialization.cpp b/stablehlo/dialect/Serialization.cpp ++index cb89d673..4370d588 100644 ++--- a/stablehlo/dialect/Serialization.cpp +++++ b/stablehlo/dialect/Serialization.cpp ++@@ -39,8 +39,7 @@ namespace stablehlo { ++ ++ LogicalResult serializePortableArtifact(ModuleOp module, ++ StringRef targetVersion, ++- raw_ostream& os, ++- bool allowOtherDialects) { +++ raw_ostream& os) { ++ MLIRContext* context = module.getContext(); ++ ++ // Convert StableHLO --> VHLO. ++@@ -49,7 +48,7 @@ LogicalResult serializePortableArtifact(ModuleOp module, ++ { ++ PassManager pm(context); ++ StablehloLegalizeToVhloPassOptions options; ++- options.allowOtherDialects = allowOtherDialects; +++ options.allowOtherDialects = false; ++ pm.addPass(stablehlo::createStablehloLegalizeToVhloPass(options)); ++ if (!succeeded(pm.run(module))) { ++ return failure(); ++diff --git a/stablehlo/dialect/Serialization.h b/stablehlo/dialect/Serialization.h ++index 811ca97b..abe95e63 100644 ++--- a/stablehlo/dialect/Serialization.h +++++ b/stablehlo/dialect/Serialization.h ++@@ -34,8 +34,7 @@ namespace stablehlo { ++ // unsupported dialects. ++ LogicalResult serializePortableArtifact(ModuleOp module, ++ StringRef targetVersion, ++- raw_ostream& os, ++- bool allowOtherDialects = false); +++ raw_ostream& os); ++ ++ // Read StableHLO portable artifact ++ // ++diff --git a/stablehlo/integrations/c/StablehloDialectApi.cpp b/stablehlo/integrations/c/StablehloDialectApi.cpp ++index 343f8d0b..8f52e4d5 100644 ++--- a/stablehlo/integrations/c/StablehloDialectApi.cpp +++++ b/stablehlo/integrations/c/StablehloDialectApi.cpp ++@@ -81,8 +81,7 @@ MlirLogicalResult stablehloSerializePortableArtifactFromModule( ++ MlirStringCallback callback, void *userData, bool allowOtherDialects) { ++ mlir::detail::CallbackOstream stream(callback, userData); ++ if (failed(mlir::stablehlo::serializePortableArtifact( ++- unwrap(moduleStr), unwrap(targetVersion), stream, ++- allowOtherDialects))) +++ unwrap(moduleStr), unwrap(targetVersion), stream))) ++ return mlirLogicalResultFailure(); ++ return mlirLogicalResultSuccess(); ++ } ++diff --git a/stablehlo/integrations/c/StablehloDialectApi.h b/stablehlo/integrations/c/StablehloDialectApi.h ++index 385156bf..24d11c1d 100644 ++--- a/stablehlo/integrations/c/StablehloDialectApi.h +++++ b/stablehlo/integrations/c/StablehloDialectApi.h ++@@ -93,7 +93,7 @@ stablehloSerializePortableArtifactFromModule(MlirModule moduleStr, ++ MlirStringRef targetVersion, ++ MlirStringCallback callback, ++ void* userData, ++- bool allowOtherDialects = false); +++ bool allowOtherDialects); ++ ++ // Read a StableHLO program from a portable artifact, returning the module as ++ // MLIR bytecode. Note, this bytecode returned is not a portable artifact, ++diff --git a/stablehlo/tools/StablehloTranslateMain.cpp b/stablehlo/tools/StablehloTranslateMain.cpp ++index fdf0d6a9..8d5c8752 100644 ++--- a/stablehlo/tools/StablehloTranslateMain.cpp +++++ b/stablehlo/tools/StablehloTranslateMain.cpp ++@@ -323,7 +323,7 @@ TranslateFromMLIRRegistration serializeRegistration( ++ } ++ ++ return stablehlo::serializePortableArtifact( ++- module, targetVersion, os, allowOtherDialectsOption.getValue()); +++ module, targetVersion, os); ++ }, ++ [](DialectRegistry ®istry) { ++ mlir::registerAllDialects(registry); ++-- ++2.39.5 (Apple Git-154) ++ +-- +2.39.5 (Apple Git-154) + diff --git a/third_party/modules/xla/20250718.0-6319f0d/source.json b/third_party/modules/xla/20250718.0-6319f0d/source.json new file mode 100644 index 00000000..a34f9a9e --- /dev/null +++ b/third_party/modules/xla/20250718.0-6319f0d/source.json @@ -0,0 +1,18 @@ +{ + "strip_prefix": "xla-6319f0d3bdfd3078e04bb984a759c890b7116484", + "url": "https://github.com/openxla/xla/archive/6319f0d3bdfd3078e04bb984a759c890b7116484.tar.gz", + "integrity": "sha256-0XAMbJ4tn5yFllPGVCSh8st7lgTOUoyeypoudCB8tsY=", + "overlay": { + "toolchains_private.bzl": "", + "llvm.bzl": "", + "MODULE.bazel": "", + "workspace_private.bzl": "", + "xla.bzl": "" + }, + "patch_strip": 1, + "patches": { + "0001-bazel-migration-to-bazel-8.1.1.patch": "", + "0002-Added-FFI-handler-registration-API-to-the-FFI-PjRt.patch": "", + "0003-Remove-unconventional-C-code-in-headers.patch": "" + } +} diff --git a/third_party/modules/xla/metadata.json b/third_party/modules/xla/metadata.json new file mode 100644 index 00000000..4fa86cba --- /dev/null +++ b/third_party/modules/xla/metadata.json @@ -0,0 +1,30 @@ +{ + "homepage": "https://github.com/openxla/xla", + "maintainers": [ + { + "email": "bzlmod@zml.ai", + "github": "zml", + "name": "ZML Engineering Team" + } + ], + "repository": [ + "github:openxla/xla" + ], + "versions": [ + "20240902.0-d18cd64", + "20240919.0-1b18dd6", + "20241025.0-4663f04", + "20250103.0-5f1fe6a", + "20250122.0-cc075be", + "20250123.0-cc075be", + "20250204-0-6789523", + "20250317.0-71c67e2", + "20250317.1-71c67e2", + "20250317.2-71c67e2", + "20250527.0-cb67f2f", + "20250612.0-6e48cbb", + "20250710.0-22ea002", + "20250718.0-6319f0d" + ], + "yanked_versions": {} +} diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel new file mode 100644 index 00000000..4fb39921 --- /dev/null +++ b/third_party/modules/zig-protobuf/20240722.0-c644d11/MODULE.bazel @@ -0,0 +1,8 @@ +module( + name = "zig-protobuf", + version = "20240722.0-c644d11", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel new file mode 100644 index 00000000..832c7591 --- /dev/null +++ b/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") +load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") + +zig_library( + name = "protobuf", + import_name = "protobuf", + main = "src/protobuf.zig", + visibility = ["//visibility:public"], +) + +zig_binary( + name = "generator", + srcs = [ + "bootstrapped-generator/FullName.zig", + "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", + "bootstrapped-generator/google/protobuf/descriptor.pb.zig", + ], + kind = BINARY_KIND.exe, + main = "bootstrapped-generator/main.zig", + visibility = ["//visibility:public"], + deps = [":protobuf"], +) + +proto_lang_toolchain( + name = "zig_toolchain", + command_line = "--zig_out=$(OUT)", + output_files = "multiple", + plugin = ":generator", + plugin_format_flag = "--plugin=protoc-gen-zig=%s", + runtime = ":protobuf", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/zig-protobuf/20240722.0-c644d11/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch b/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch new file mode 100644 index 00000000..67353922 --- /dev/null +++ b/third_party/modules/zig-protobuf/20240722.0-c644d11/patches/0001-USE_MODULES-true.patch @@ -0,0 +1,11 @@ +--- a/bootstrapped-generator/main.zig ++++ b/bootstrapped-generator/main.zig +@@ -6,7 +6,7 @@ const descriptor = @import("google/protobuf/descriptor.pb.zig"); + const mem = std.mem; + const FullName = @import("./FullName.zig").FullName; + +-const USE_MODULES = false; ++const USE_MODULES = true; + + const allocator = std.heap.page_allocator; + diff --git a/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json b/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json new file mode 100644 index 00000000..f985b249 --- /dev/null +++ b/third_party/modules/zig-protobuf/20240722.0-c644d11/source.json @@ -0,0 +1,13 @@ +{ + "strip_prefix": "zig-protobuf-c644d1105caaca4d9a48d3536e253687052f74c8", + "url": "https://github.com/gwenzek/zig-protobuf/archive/c644d1105caaca4d9a48d3536e253687052f74c8.tar.gz", + "integrity": "sha256-KoZp8MLLEQQgS7Uip4sdjw7TYmhzKPdpilQYKp5iYV4=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-USE_MODULES-true.patch": "" + } +} diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/MODULE.bazel b/third_party/modules/zig-protobuf/20250213.0-5304067/MODULE.bazel new file mode 100644 index 00000000..0b9ec156 --- /dev/null +++ b/third_party/modules/zig-protobuf/20250213.0-5304067/MODULE.bazel @@ -0,0 +1,8 @@ +module( + name = "zig-protobuf", + version = "20250213.0-5304067", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/BUILD.bazel new file mode 100644 index 00000000..832c7591 --- /dev/null +++ b/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") +load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") + +zig_library( + name = "protobuf", + import_name = "protobuf", + main = "src/protobuf.zig", + visibility = ["//visibility:public"], +) + +zig_binary( + name = "generator", + srcs = [ + "bootstrapped-generator/FullName.zig", + "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", + "bootstrapped-generator/google/protobuf/descriptor.pb.zig", + ], + kind = BINARY_KIND.exe, + main = "bootstrapped-generator/main.zig", + visibility = ["//visibility:public"], + deps = [":protobuf"], +) + +proto_lang_toolchain( + name = "zig_toolchain", + command_line = "--zig_out=$(OUT)", + output_files = "multiple", + plugin = ":generator", + plugin_format_flag = "--plugin=protoc-gen-zig=%s", + runtime = ":protobuf", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/zig-protobuf/20250213.0-5304067/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-protobuf/20250213.0-5304067/source.json b/third_party/modules/zig-protobuf/20250213.0-5304067/source.json new file mode 100644 index 00000000..e4395bdd --- /dev/null +++ b/third_party/modules/zig-protobuf/20250213.0-5304067/source.json @@ -0,0 +1,9 @@ +{ + "strip_prefix": "zig-protobuf-5304067205135532c0ad57f78243bfe86dc1ad3f", + "url": "https://github.com/gwenzek/zig-protobuf/archive/5304067205135532c0ad57f78243bfe86dc1ad3f.tar.gz", + "integrity": "sha256-cdby+J1CGNOvLRvdeASPNlB7y1q59LYcgjFOooBvIcA=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + } +} diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/MODULE.bazel b/third_party/modules/zig-protobuf/20250318.0-930153e/MODULE.bazel new file mode 100644 index 00000000..40830df4 --- /dev/null +++ b/third_party/modules/zig-protobuf/20250318.0-930153e/MODULE.bazel @@ -0,0 +1,8 @@ +module( + name = "zig-protobuf", + version = "20250318.0-930153e", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/BUILD.bazel new file mode 100644 index 00000000..832c7591 --- /dev/null +++ b/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") +load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") + +zig_library( + name = "protobuf", + import_name = "protobuf", + main = "src/protobuf.zig", + visibility = ["//visibility:public"], +) + +zig_binary( + name = "generator", + srcs = [ + "bootstrapped-generator/FullName.zig", + "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", + "bootstrapped-generator/google/protobuf/descriptor.pb.zig", + ], + kind = BINARY_KIND.exe, + main = "bootstrapped-generator/main.zig", + visibility = ["//visibility:public"], + deps = [":protobuf"], +) + +proto_lang_toolchain( + name = "zig_toolchain", + command_line = "--zig_out=$(OUT)", + output_files = "multiple", + plugin = ":generator", + plugin_format_flag = "--plugin=protoc-gen-zig=%s", + runtime = ":protobuf", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/zig-protobuf/20250318.0-930153e/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-protobuf/20250318.0-930153e/source.json b/third_party/modules/zig-protobuf/20250318.0-930153e/source.json new file mode 100644 index 00000000..fa33ec5e --- /dev/null +++ b/third_party/modules/zig-protobuf/20250318.0-930153e/source.json @@ -0,0 +1,9 @@ +{ + "strip_prefix": "zig-protobuf-930153ec955853ff9fcf8a55ba632b2b8051f731", + "url": "https://github.com/zml/zig-protobuf/archive/930153ec955853ff9fcf8a55ba632b2b8051f731.tar.gz", + "integrity": "sha256-ORRDxK/cLQ2E+c9l2lcj8FgIUHH+uEk/+qWVWuHZMEY=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + } +} diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel new file mode 100644 index 00000000..b71798cc --- /dev/null +++ b/third_party/modules/zig-protobuf/20250716.0-97f1e31/MODULE.bazel @@ -0,0 +1,8 @@ +module( + name = "zig-protobuf", + version = "20250716.0-97f1e31", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20250613.0-567662a") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel new file mode 100644 index 00000000..832c7591 --- /dev/null +++ b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_proto//proto:defs.bzl", "proto_lang_toolchain") +load("@rules_zig//zig:defs.bzl", "BINARY_KIND", "zig_binary", "zig_library") + +zig_library( + name = "protobuf", + import_name = "protobuf", + main = "src/protobuf.zig", + visibility = ["//visibility:public"], +) + +zig_binary( + name = "generator", + srcs = [ + "bootstrapped-generator/FullName.zig", + "bootstrapped-generator/google/protobuf/compiler/plugin.pb.zig", + "bootstrapped-generator/google/protobuf/descriptor.pb.zig", + ], + kind = BINARY_KIND.exe, + main = "bootstrapped-generator/main.zig", + visibility = ["//visibility:public"], + deps = [":protobuf"], +) + +proto_lang_toolchain( + name = "zig_toolchain", + command_line = "--zig_out=$(OUT)", + output_files = "multiple", + plugin = ":generator", + plugin_format_flag = "--plugin=protoc-gen-zig=%s", + runtime = ":protobuf", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel new file mode 100644 index 00000000..b71798cc --- /dev/null +++ b/third_party/modules/zig-protobuf/20250716.0-97f1e31/overlay/MODULE.bazel @@ -0,0 +1,8 @@ +module( + name = "zig-protobuf", + version = "20250716.0-97f1e31", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20250613.0-567662a") +bazel_dep(name = "rules_proto", version = "6.0.0-rc1") diff --git a/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json b/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json new file mode 100644 index 00000000..4975cdb8 --- /dev/null +++ b/third_party/modules/zig-protobuf/20250716.0-97f1e31/source.json @@ -0,0 +1,9 @@ +{ + "strip_prefix": "zig-protobuf-97f1e314d9bbf600fbaadee5bb08c8856e0006de", + "url": "https://github.com/zml/zig-protobuf/archive/97f1e314d9bbf600fbaadee5bb08c8856e0006de.tar.gz", + "integrity": "sha256-iAqsLIKhl15JInJvwMY8NGVll6eUBXzdLW6Xv8wANGY=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + } +} diff --git a/third_party/modules/zig-protobuf/metadata.json b/third_party/modules/zig-protobuf/metadata.json new file mode 100644 index 00000000..4a038360 --- /dev/null +++ b/third_party/modules/zig-protobuf/metadata.json @@ -0,0 +1,20 @@ +{ + "homepage": "https://github.com/gwenzek/zig-protobuf", + "maintainers": [ + { + "email": "bzlmod@zml.ai", + "github": "zml", + "name": "ZML Engineering Team" + } + ], + "repository": [ + "zml/zig-protobuf" + ], + "versions": [ + "20240722.0-c644d11", + "20250213.0-5304067", + "20250318.0-930153e", + "20250716.0-97f1e31", + ], + "yanked_versions": {} +} diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel new file mode 100644 index 00000000..151f3337 --- /dev/null +++ b/third_party/modules/zig-yaml/20240903.0-83d5fdf/MODULE.bazel @@ -0,0 +1,7 @@ +module( + name = "zig-yaml", + version = "20240903.0-83d5fdf", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel new file mode 100644 index 00000000..94897f78 --- /dev/null +++ b/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +zig_library( + name = "zig-yaml", + srcs = glob([ + "src/*.zig", + "src/parse/*.zig", + "src/yaml/*.zig", + ]), + main = "src/yaml.zig", + visibility = ["//visibility:public"], +) diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel b/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/zig-yaml/20240903.0-83d5fdf/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json b/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json new file mode 100644 index 00000000..643aa238 --- /dev/null +++ b/third_party/modules/zig-yaml/20240903.0-83d5fdf/source.json @@ -0,0 +1,9 @@ +{ + "strip_prefix": "zig-yaml-83d5fdfdd6c847b4bd2a8cad73b05a5c6447984b", + "url": "https://github.com/gwenzek/zig-yaml/archive/83d5fdfdd6c847b4bd2a8cad73b05a5c6447984b.tar.gz", + "integrity": "sha256-uusblAIZQgyKlg0qVmuHWD2JG+dMM+N6oKNGZsBdpEk=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + } +} diff --git a/third_party/modules/zig-yaml/metadata.json b/third_party/modules/zig-yaml/metadata.json new file mode 100644 index 00000000..3aa2c43c --- /dev/null +++ b/third_party/modules/zig-yaml/metadata.json @@ -0,0 +1,17 @@ +{ + "homepage": "https://github.com/gwenzek/zig-yaml", + "maintainers": [ + { + "email": "bzlmod@zml.ai", + "github": "zml", + "name": "ZML Engineering Team" + } + ], + "repository": [ + "github:gwenzek/zig-yaml" + ], + "versions": [ + "20240903.0-83d5fdf" + ], + "yanked_versions": {} +} diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel new file mode 100644 index 00000000..a59b2d87 --- /dev/null +++ b/third_party/modules/zigcoro/20240829.0-fc1db29/MODULE.bazel @@ -0,0 +1,8 @@ +module( + name = "zigcoro", + version = "20240829.0-fc1db29", + compatibility_level = 1, +) + +bazel_dep(name = "rules_zig", version = "20240904.0-010da15") +bazel_dep(name = "libxev", version = "20240825.0-dbe2291") diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel new file mode 100644 index 00000000..2aa934ca --- /dev/null +++ b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/BUILD.bazel @@ -0,0 +1,29 @@ +load("@rules_zig//zig:defs.bzl", "zig_library") + +genrule( + name = "libcoro_options_gen", + outs = ["options.zig"], + cmd = """\ +cat < $@ +pub const default_stack_size: usize = 4096; +pub const debug_log_level: usize = 0; +EOF +""", +) + +zig_library( + name = "libcoro_options", + main = ":libcoro_options_gen", +) + +zig_library( + name = "libcoro", + srcs = glob(["src/*.zig"]), + extra_srcs = glob(["src/asm/*.s"]), + main = "src/main.zig", + visibility = ["//visibility:public"], + deps = [ + ":libcoro_options", + "@libxev", + ], +) diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel new file mode 120000 index 00000000..9b599e3a --- /dev/null +++ b/third_party/modules/zigcoro/20240829.0-fc1db29/overlay/MODULE.bazel @@ -0,0 +1 @@ +../MODULE.bazel \ No newline at end of file diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch b/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch new file mode 100644 index 00000000..6072de07 --- /dev/null +++ b/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0001-Handle-generic-function-in-Signature.patch @@ -0,0 +1,48 @@ +From e2416ff5ec9cad5572e6a5edac3f949b75024c00 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Tue, 6 Aug 2024 11:34:35 +0200 +Subject: [PATCH 1/2] Handle generic function in Signature + +--- + src/coro.zig | 7 +++++-- + 1 file changed, 5 insertions(+), 2 deletions(-) + +diff --git a/src/coro.zig b/src/coro.zig +index 9285489..8a34717 100644 +--- a/src/coro.zig ++++ b/src/coro.zig +@@ -266,6 +266,7 @@ const CoroT = struct { + YieldT: type = void, + InjectT: type = void, + ArgsT: type, ++ _ReturnT: type, + + // If the function this signature represents is compile-time known, + // it can be held here. +@@ -273,19 +274,21 @@ const CoroT = struct { + + fn init(comptime Func: anytype, comptime options: CoroT.Options) @This() { + const FuncT = if (@TypeOf(Func) == type) Func else @TypeOf(Func); ++ const ArgsT = options.ArgsT orelse ArgsTuple(FuncT); + return .{ + .Func = FuncT, + .YieldT = options.YieldT, + .InjectT = options.InjectT, +- .ArgsT = options.ArgsT orelse ArgsTuple(FuncT), ++ .ArgsT = ArgsT, + .func_ptr = if (@TypeOf(Func) == type) null else struct { + const val = Func; + }, ++ ._ReturnT = @TypeOf(@call(.auto, Func, @as(ArgsT, undefined))), + }; + } + + pub fn ReturnT(comptime self: @This()) type { +- return @typeInfo(self.Func).Fn.return_type.?; ++ return self._ReturnT; + } + }; + +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch b/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch new file mode 100644 index 00000000..f1e68622 --- /dev/null +++ b/third_party/modules/zigcoro/20240829.0-fc1db29/patches/0002-asyncio-return-error-super-set-in-pread-pwrite.patch @@ -0,0 +1,34 @@ +From d8a4773ee9fa0ae5c5fd91e84c322e75a634a077 Mon Sep 17 00:00:00 2001 +From: Steeve Morin +Date: Tue, 6 Aug 2024 13:14:59 +0200 +Subject: [PATCH 2/2] asyncio: return error super set in pread/pwrite + +--- + src/asyncio.zig | 4 ++-- + 1 file changed, 2 insertions(+), 2 deletions(-) + +diff --git a/src/asyncio.zig b/src/asyncio.zig +index 68cc7d3..8487754 100644 +--- a/src/asyncio.zig ++++ b/src/asyncio.zig +@@ -354,7 +354,7 @@ pub const File = struct { + } + + const PReadResult = xev.ReadError!usize; +- pub fn pread(self: Self, buf: xev.ReadBuffer, offset: u64) PReadResult { ++ pub fn pread(self: Self, buf: xev.ReadBuffer, offset: u64) !usize { + const ResultT = PReadResult; + const Data = struct { + result: ResultT = undefined, +@@ -391,7 +391,7 @@ pub const File = struct { + } + + const PWriteResult = xev.WriteError!usize; +- pub fn pwrite(self: Self, buf: xev.WriteBuffer, offset: u64) PWriteResult { ++ pub fn pwrite(self: Self, buf: xev.WriteBuffer, offset: u64) !usize { + const ResultT = PWriteResult; + const Data = struct { + result: ResultT = undefined, +-- +2.39.3 (Apple Git-146) + diff --git a/third_party/modules/zigcoro/20240829.0-fc1db29/source.json b/third_party/modules/zigcoro/20240829.0-fc1db29/source.json new file mode 100644 index 00000000..17e66d4e --- /dev/null +++ b/third_party/modules/zigcoro/20240829.0-fc1db29/source.json @@ -0,0 +1,14 @@ +{ + "strip_prefix": "zigcoro-fc1db29dee049ce572a516f1586cce0eca4fcb2a", + "url": "https://github.com/rsepassi/zigcoro/archive/fc1db29dee049ce572a516f1586cce0eca4fcb2a.tar.gz", + "integrity": "sha256-2ynCPLSRFPto3HIOhDTBDyUS770PNMI9ioMkj8Z7Y+s=", + "overlay": { + "MODULE.bazel": "", + "BUILD.bazel": "" + }, + "patch_strip": 1, + "patches": { + "0001-Handle-generic-function-in-Signature.patch": "", + "0002-asyncio-return-error-super-set-in-pread-pwrite.patch": "" + } +} diff --git a/third_party/modules/zigcoro/metadata.json b/third_party/modules/zigcoro/metadata.json new file mode 100644 index 00000000..533957ce --- /dev/null +++ b/third_party/modules/zigcoro/metadata.json @@ -0,0 +1,17 @@ +{ + "homepage": "https://github.com/rsepassi/zigcoro", + "maintainers": [ + { + "email": "bzlmod@zml.ai", + "github": "zml", + "name": "ZML Engineering Team" + } + ], + "repository": [ + "github:rsepassi/zigcoro" + ], + "versions": [ + "20240829.0-fc1db29" + ], + "yanked_versions": {} +} diff --git a/third_party/non_module_deps.bzl b/third_party/non_module_deps.bzl index 876b0db2..fbeee624 100644 --- a/third_party/non_module_deps.bzl +++ b/third_party/non_module_deps.bzl @@ -1,46 +1,26 @@ load("//third_party/arocc:repo.bzl", arocc = "repo") -load("//third_party/cloud_accelerator_diagnostics:repo.bzl", cloud_accelerator_diagnostics = "repo") load("//third_party/com_google_sentencepiece:repo.bzl", com_google_sentencepiece = "repo") -load("//third_party/cuda_tile:repo.bzl", cuda_tile = "repo") -load("//third_party/cute_ir:repo.bzl", cute_ir = "repo") -load("//third_party/flashinfer_cutlass_moe:repo.bzl", flashinfer_cutlass_moe = "repo") -load("//third_party/flydsl:repo.bzl", flydsl = "repo") -load("//third_party/iree:repo.bzl", iree = "repo") -load("//third_party/libvaxis:repo.bzl", libvaxis = "repo") -load("//third_party/linenoise:repo.bzl", linenoise = "repo") load("//third_party/mnist:repo.bzl", mnist = "repo") load("//third_party/org_swig_swig:repo.bzl", org_swig_swig = "repo") -load("//third_party/stb:repo.bzl", stb = "repo") load("//third_party/translate-c:repo.bzl", translate_c = "repo") -load("//third_party/uucode:repo.bzl", uucode = "repo") load("//third_party/xla:repo.bzl", xla = "repo") -load("//third_party/zigimg:repo.bzl", zigimg = "repo") -load("//third_party/zio:repo.bzl", zio = "repo") -load("//third_party/zls:repo.bzl", zls = "repo") +load("//third_party/flashattn:repo.bzl", flashattn = "repo") +load("//third_party/linenoise:repo.bzl", linenoise = "repo") def _non_module_deps_impl(mctx): - cloud_accelerator_diagnostics() com_google_sentencepiece() org_swig_swig() mnist() xla() arocc() translate_c() - flashinfer_cutlass_moe() + flashattn() linenoise() - stb() - zigimg() - uucode() - libvaxis() - iree() - cuda_tile() - flydsl() - cute_ir() - zio() - zls() return mctx.extension_metadata( reproducible = True, + root_module_direct_deps = "all", + root_module_direct_dev_deps = [], ) non_module_deps = module_extension( diff --git a/third_party/org_swig_swig/repo.bzl b/third_party/org_swig_swig/repo.bzl index 9a6c55bb..c35659bf 100644 --- a/third_party/org_swig_swig/repo.bzl +++ b/third_party/org_swig_swig/repo.bzl @@ -3,10 +3,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") def repo(): http_archive( name = "org_swig_swig", - urls = [ - "https://downloads.sourceforge.net/project/swig/swig/swig-4.4.1/swig-4.4.1.tar.gz", - "https://prdownloads.sourceforge.net/swig/swig-4.4.1.tar.gz", - ], + url = "http://prdownloads.sourceforge.net/swig/swig-4.4.1.tar.gz", sha256 = "40162a706c56f7592d08fd52ef5511cb7ac191f3593cf07306a0a554c6281fcf", strip_prefix = "swig-4.4.1", build_file = "//:third_party/org_swig_swig/swig.bazel", diff --git a/third_party/rules_distroless/accept-nospace-version-constraints.patch b/third_party/rules_distroless/accept-nospace-version-constraints.patch deleted file mode 100644 index 62973749..00000000 --- a/third_party/rules_distroless/accept-nospace-version-constraints.patch +++ /dev/null @@ -1,28 +0,0 @@ -diff --git a/apt/private/version_constraint.bzl b/apt/private/version_constraint.bzl -index 531fa01..b3a3d30 100644 ---- a/apt/private/version_constraint.bzl -+++ b/apt/private/version_constraint.bzl -@@ -2,11 +2,19 @@ - - load(":version.bzl", version_lib = "version") - -+_VERSION_CONSTRAINT_OPERATORS = ["<<", ">>", "<=", ">=", "="] -+ - def _parse_version_constraint(rawv): -- vconst_i = rawv.find(" ") -- if vconst_i == -1: -- fail('invalid version string %s expected a version constraint ">=", "=", ">=", "<<", ">>"' % rawv) -- return (rawv[:vconst_i], rawv[vconst_i + 1:]) -+ rawv = rawv.strip() -+ for op in _VERSION_CONSTRAINT_OPERATORS: -+ if rawv.startswith(op): -+ version = rawv[len(op):].strip() -+ if version: -+ return (op, version) -+ -+ break -+ -+ fail('invalid version string %s expected a version constraint ">=", "=", "<=", "<<", ">>"' % rawv) - - def _parse_dep(raw): - raw = raw.strip() # remove leading & trailing whitespace diff --git a/third_party/rules_distroless/flat-repositories.patch b/third_party/rules_distroless/flat-repositories.patch deleted file mode 100644 index e4ba73bc..00000000 --- a/third_party/rules_distroless/flat-repositories.patch +++ /dev/null @@ -1,59 +0,0 @@ -diff --git a/apt/private/deb_resolve.bzl b/apt/private/deb_resolve.bzl ---- a/apt/private/deb_resolve.bzl -+++ b/apt/private/deb_resolve.bzl -@@ -46,7 +46,10 @@ - sources = [] - - for src in manifest["sources"]: -- distr, components = src["channel"].split(" ", 1) -+ if src["channel"].endswith("/"): -+ distr, components = src["channel"], "" -+ else: -+ distr, components = src["channel"].split(" ", 1) - for comp in components.split(" "): - # TODO: only support urls before 1.0 - if "urls" in src: -@@ -60,6 +63,7 @@ - urls, - distr, - comp, -+ src.get("archs", manifest["archs"]), - )) - - repository = deb_repository.new(rctx, sources = sources, archs = manifest["archs"]) -diff --git a/apt/private/apt_deb_repository.bzl b/apt/private/apt_deb_repository.bzl ---- a/apt/private/apt_deb_repository.bzl -+++ b/apt/private/apt_deb_repository.bzl -@@ -13,7 +13,7 @@ - return use_netrc(netrc, urls, {}) - - def _fetch_package_index(rctx, urls, dist, comp, arch, integrity): -- target_triple = "{dist}/{comp}/{arch}".format(dist = dist, comp = comp, arch = arch) -+ target_triple = "{dist}/{comp}/{arch}".format(dist = dist.strip("/") or ".", comp = comp, arch = arch) - - # See https://linux.die.net/man/1/xz , https://linux.die.net/man/1/gzip , and https://linux.die.net/man/1/bzip2 - # --keep -> keep the original file (Bazel might be still committing the output to the cache) -@@ -35,7 +35,10 @@ - download = None - for (ext, cmd) in supported_extensions: - output = "{}/Packages{}".format(target_triple, ext) -- dist_url = "{}/dists/{}/{}/binary-{}/Packages{}".format(url, dist, comp, arch, ext) -+ if dist.endswith("/"): -+ dist_url = "{}/{}Packages{}".format(url, dist.lstrip("/"), ext) -+ else: -+ dist_url = "{}/dists/{}/{}/binary-{}/Packages{}".format(url, dist, comp, arch, ext) - auth = {} - if url in base_auth: - auth = {dist_url: base_auth[url]} -@@ -167,7 +170,10 @@ - ) - - for arch in archs: -- for (urls, dist, comp) in sources: -+ for (urls, dist, comp, source_archs) in sources: -+ if arch not in source_archs: -+ continue -+ - # We assume that `url` does not contain a trailing forward slash when passing to - # functions below. If one is present, remove it. Some HTTP servers do not handle - # redirects properly when a path contains "//" diff --git a/third_party/stb/BUILD.bazel b/third_party/stb/BUILD.bazel deleted file mode 100644 index f91653e8..00000000 --- a/third_party/stb/BUILD.bazel +++ /dev/null @@ -1 +0,0 @@ -exports_files(["stb.BUILD"]) diff --git a/third_party/stb/repo.bzl b/third_party/stb/repo.bzl deleted file mode 100644 index 44645c46..00000000 --- a/third_party/stb/repo.bzl +++ /dev/null @@ -1,9 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -def repo(): - git_repository( - name = "stb", - remote = "https://github.com/nothings/stb.git", - build_file = "//third_party/stb:stb.BUILD", - commit = "f1c79c02822848a9bed4315b12c8c8f3761e1296", - ) diff --git a/third_party/stb/stb.BUILD b/third_party/stb/stb.BUILD deleted file mode 100644 index b59ef43f..00000000 --- a/third_party/stb/stb.BUILD +++ /dev/null @@ -1,35 +0,0 @@ -cc_import( - name = "stb_image_write", - hdrs = ["stb_image_write.h"], - includes = ["."], - visibility = ["//visibility:public"], -) - -genrule( - name = "impl", - outs = ["impl.c"], - cmd = """\ -cat < $(@) -#include "stb_image.h" -#include "stb_image_resize2.h" -#include "stb_image_write.h" -EOF -""", -) - -cc_library( - name = "stb", - local_defines = [ - "STB_IMAGE_IMPLEMENTATION", - "STB_IMAGE_RESIZE_IMPLEMENTATION", - "STB_IMAGE_WRITE_IMPLEMENTATION", - ], - srcs = [":impl"], - hdrs = [ - "stb_image.h", - "stb_image_resize2.h", - "stb_image_write.h", - ], - includes = ["."], - visibility = ["//visibility:public"], -) diff --git a/third_party/translate-c/repo.bzl b/third_party/translate-c/repo.bzl index dfd8835d..ab5e9860 100644 --- a/third_party/translate-c/repo.bzl +++ b/third_party/translate-c/repo.bzl @@ -1,11 +1,9 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository") def repo(): - git_repository( + new_git_repository( name = "translate-c", - # Use the ZML GitHub mirror because Codeberg is unreliable. - remote = "https://github.com/zml/translate-c", - # 2.0.0 - commit = "0da7a16c3235b935b82421646076e0657cda21f6", - build_file = Label("//third_party/translate-c:translate-c.bazel"), + remote = "https://codeberg.org/ziglang/translate-c.git", + commit = "d2be2f19ef7c9caa1561d38c96581ac79dd4c654", + build_file = "//third_party/translate-c:translate-c.bazel", ) diff --git a/third_party/translate-c/translate-c.bazel b/third_party/translate-c/translate-c.bazel index 7de2a5eb..f71603e3 100644 --- a/third_party/translate-c/translate-c.bazel +++ b/third_party/translate-c/translate-c.bazel @@ -1,12 +1,4 @@ load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_configure_binary", "zig_library") -load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") - -zig_option(name = "debug_allocations", bool = False) - -zig_options( - name = "build_options", - settings = [":debug_allocations"], -) zig_library( name = "c_builtins", @@ -22,28 +14,21 @@ zig_library( zig_binary( name = "translate-c_raw", - data = glob(["lib/**/*.zig"]), srcs = glob(["src/**/*.zig"]), main = "src/main.zig", linkopts = ["-lc"], + visibility = ["//visibility:public"], deps = [ - ":build_options", ":c_builtins", ":helpers", "@arocc//:aro", ], - visibility = ["//visibility:public"], ) -# TODO(cerisier): Generate 1 translate-c toolchain per target platform -# So that the choice of doing this for a given platform is given to users. -# -# Right now we do this because the arocc source code has the same support -# for platforms than the official zig compiler and in the case of using a -# custom zig version with support for newer chips, arocc cannot be built. zig_configure_binary( name = "translate-c", actual = ":translate-c_raw", - bootstrapped = 0, + no_translate_c = 1, + use_cc_common_link = 0, visibility = ["//visibility:public"], ) diff --git a/third_party/uucode/BUILD.bazel b/third_party/uucode/BUILD.bazel deleted file mode 100644 index 49e3a399..00000000 --- a/third_party/uucode/BUILD.bazel +++ /dev/null @@ -1,4 +0,0 @@ -exports_files([ - "build_config.zig", - "uucode.bazel", -]) diff --git a/third_party/uucode/build_config.zig b/third_party/uucode/build_config.zig deleted file mode 100644 index a4e76bff..00000000 --- a/third_party/uucode/build_config.zig +++ /dev/null @@ -1,16 +0,0 @@ -const config = @import("config.zig"); - -pub const fields = config.fields; -pub const build_components = config.build_components; -pub const get_components = config.get_components; - -pub const tables: []const config.Table = &.{ - .{ - .fields = &.{ - "east_asian_width", - "grapheme_break", - "general_category", - "is_emoji_presentation", - }, - }, -}; diff --git a/third_party/uucode/config-storage-module.patch b/third_party/uucode/config-storage-module.patch deleted file mode 100644 index d390986c..00000000 --- a/third_party/uucode/config-storage-module.patch +++ /dev/null @@ -1,33 +0,0 @@ -diff --git a/src/config.zig b/src/config.zig -index 2cadab4..aa46c5d 100644 ---- a/src/config.zig -+++ b/src/config.zig -@@ -1,5 +1,5 @@ - const std = @import("std"); --const storage = @import("storage.zig"); -+pub const storage = @import("storage.zig"); - const multi_slice = @import("multi_slice.zig"); - pub const quirks = @import("quirks.zig"); - pub const components = @import("components.zig"); -diff --git a/src/generate.zig b/src/generate.zig -index 6f4dcf5..8ef1d83 100644 ---- a/src/generate.zig -+++ b/src/generate.zig -@@ -1,7 +1,7 @@ - const std = @import("std"); --const storage = @import("storage.zig"); - const config = @import("config.zig"); - const build_config = @import("build_config"); -+const storage = config.storage; - const inlineAssert = config.quirks.inlineAssert; - - pub const std_options: std.Options = .{ -@@ -339,7 +339,7 @@ pub fn main(init: std.process.Init.Minimal) !void { - \\const std = @import("std"); - \\const config = @import("config.zig"); - \\const types = config.types; -- \\const storage = @import("storage.zig"); -+ \\const storage = config.storage; - \\const build_config = @import("build_config"); - \\ - \\pub const get_components = build_config.get_components; diff --git a/third_party/uucode/repo.bzl b/third_party/uucode/repo.bzl deleted file mode 100644 index 5f7b1cbf..00000000 --- a/third_party/uucode/repo.bzl +++ /dev/null @@ -1,29 +0,0 @@ -def _uucode_repo_impl(ctx): - ctx.download_and_extract( - url = "https://github.com/jacobsandlund/uucode/archive/{}.tar.gz".format(ctx.attr.commit), - stripPrefix = "uucode-{}".format(ctx.attr.commit), - ) - ctx.patch(ctx.attr.config_storage_patch, strip = 1) - - ctx.symlink(ctx.attr.build_config, "build_config.zig") - ctx.symlink(ctx.attr.build_file, "BUILD.bazel") - -_uucode_repo = repository_rule( - implementation = _uucode_repo_impl, - attrs = { - "commit": attr.string(mandatory = True), - "build_config": attr.label(mandatory = True, allow_single_file = True), - "build_file": attr.label(mandatory = True, allow_single_file = True), - "config_storage_patch": attr.label(mandatory = True, allow_single_file = True), - }, -) - -def repo(): - _uucode_repo( - name = "uucode", - # main branch - commit = "1fb73433bba5d93366c57f23ff2e9d7939746500", - build_config = "//third_party/uucode:build_config.zig", - build_file = "//third_party/uucode:uucode.bazel", - config_storage_patch = "//third_party/uucode:config-storage-module.patch", - ) diff --git a/third_party/uucode/uucode.bazel b/third_party/uucode/uucode.bazel deleted file mode 100644 index 4a3710f2..00000000 --- a/third_party/uucode/uucode.bazel +++ /dev/null @@ -1,91 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") - -# Generator - -zig_library( - name = "_config_gen", - import_name = "config.zig", - main = "src/config.zig", - srcs = [ - "src/components.zig", - "src/ucd_inputs.zig", - "src/fields.zig", - "src/multi_slice.zig", - "src/quirks.zig", - "src/storage.zig", - "src/types.zig", - ], -) - -zig_library( - name = "_build_config_gen", - import_name = "build_config", - main = "build_config.zig", - deps = [":_config_gen"], -) - -zig_binary( - name = "gen_tables", - main = "src/generate.zig", - deps = [":_build_config_gen", ":_config_gen"], -) - -genrule( - name = "tables", - srcs = glob(["ucd/**"]), - outs = ["tables.zig"], - cmd = "OUT=$$PWD/$@ && TOOL=$$PWD/$(execpath :gen_tables) && cd $$(dirname $(location ucd/UnicodeData.txt))/.. && $$TOOL $$OUT", - tools = [":gen_tables"], -) - -# Library - -zig_library( - name = "types", - import_name = "types.zig", - main = "src/types.zig", -) - -zig_library( - name = "config", - import_name = "config.zig", - main = "src/config.zig", - srcs = [ - "src/components.zig", - "src/ucd_inputs.zig", - "src/fields.zig", - "src/multi_slice.zig", - "src/quirks.zig", - "src/storage.zig", - ], - deps = [":types"], -) - -zig_library( - name = "build_config", - import_name = "build_config", - main = "build_config.zig", - deps = [":config"], -) - -zig_library( - name = "tables_lib", - import_name = "tables", - main = ":tables", - deps = [":build_config", ":config"], -) - -zig_library( - name = "uucode", - import_name = "uucode", - main = "src/root.zig", - srcs = [ - "src/ascii.zig", - "src/code_point.zig", - "src/get.zig", - "src/grapheme.zig", - "src/utf8.zig", - ], - deps = [":config", ":tables_lib", ":types"], - visibility = ["//visibility:public"], -) diff --git a/third_party/xla/BUILD.bazel b/third_party/xla/BUILD.bazel index 75f731ff..e69de29b 100644 --- a/third_party/xla/BUILD.bazel +++ b/third_party/xla/BUILD.bazel @@ -1 +0,0 @@ -# Package marker for XLA patches diff --git a/third_party/xla/cuda-root-path-local-defines.patch b/third_party/xla/cuda-root-path-local-defines.patch deleted file mode 100644 index c9ca6aa8..00000000 --- a/third_party/xla/cuda-root-path-local-defines.patch +++ /dev/null @@ -1,25 +0,0 @@ -diff --git a/xla/tsl/platform/default/BUILD b/xla/tsl/platform/default/BUILD -index f01bdfc67e..18a30d44a 100644 ---- a/xla/tsl/platform/default/BUILD -+++ b/xla/tsl/platform/default/BUILD -@@ -82,13 +82,13 @@ cc_library( - if_false = ["@cuda_nvcc//:nvvm"], - if_true = ["@cuda_nvvm//:nvvm"], - )), -- local_defines = [ -- "CUDA_NVCC_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvcc") + "\\\"", -- "CUDA_NVDISASM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvdisasm") + "\\\"", -- "NVIDIA_NVSHMEM_REPO_NAME=\\\"" + get_canonical_repo_name("nvidia_nvshmem") + "\\\"", -- "CUDA_NVVM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvvm") + "\\\"", -- "CUDA_CUDART_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_cudart") + "\\\"", -- ], -+ # local_defines = [ -+ # "CUDA_NVCC_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvcc") + "\\\"", -+ # "CUDA_NVDISASM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvdisasm") + "\\\"", -+ # "NVIDIA_NVSHMEM_REPO_NAME=\\\"" + get_canonical_repo_name("nvidia_nvshmem") + "\\\"", -+ # "CUDA_NVVM_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_nvvm") + "\\\"", -+ # "CUDA_CUDART_REPO_NAME=\\\"" + get_canonical_repo_name("cuda_cudart") + "\\\"", -+ # ], - tags = [ - "manual", - "no_oss", diff --git a/third_party/xla/llvm_raw_only.bzl b/third_party/xla/llvm_raw_only.bzl deleted file mode 100644 index f2758994..00000000 --- a/third_party/xla/llvm_raw_only.bzl +++ /dev/null @@ -1,13 +0,0 @@ -load("@xla//third_party/llvm:workspace.bzl", llvm = "repo") - -def _llvm_raw_only_impl(mctx): - llvm("llvm-raw") - return mctx.extension_metadata( - reproducible = True, - root_module_direct_deps = ["llvm-raw"], - root_module_direct_dev_deps = [], - ) - -llvm_raw_only = module_extension( - implementation = _llvm_raw_only_impl, -) diff --git a/third_party/xla/repo.bzl b/third_party/xla/repo.bzl index 0e1d9620..67740474 100644 --- a/third_party/xla/repo.bzl +++ b/third_party/xla/repo.bzl @@ -4,9 +4,5 @@ def repo(): git_repository( name = "xla", remote = "https://github.com/openxla/xla.git", - commit = "65aed3e0ed7b0f965825676dd27e312965c846c8", - patches = [ - "//third_party/xla:cuda-root-path-local-defines.patch", - ], - patch_args = ["-p1"], + commit = "01da52b9afe3a2e694bd926323a649a6e63a3785", ) diff --git a/third_party/xla/xla.bzl b/third_party/xla/xla.bzl index 6634fa15..43a6629b 100644 --- a/third_party/xla/xla.bzl +++ b/third_party/xla/xla.bzl @@ -1,11 +1,5 @@ load("@xla//third_party:repo.bzl", "tf_http_archive", "tf_mirror_urls", "tf_vendored") -load("@xla//third_party/eigen3:workspace.bzl", eigen3 = "repo") -load("@xla//third_party/farmhash:workspace.bzl", farmhash = "repo") -load("@xla//third_party/highwayhash:workspace.bzl", highwayhash = "repo") -load("@xla//third_party/hwloc:workspace.bzl", hwloc = "repo") load("@xla//third_party/llvm:workspace.bzl", llvm = "repo") -load("@xla//third_party/py/ml_dtypes:workspace.bzl", ml_dtypes = "repo") -load("@xla//third_party/shardy:workspace.bzl", shardy = "repo") load("@xla//third_party/stablehlo:workspace.bzl", stablehlo = "repo") load("@xla//third_party/triton:workspace.bzl", triton = "repo") @@ -30,39 +24,7 @@ simple_files = repository_rule( def _dummy_repos(mctx): simple_files(name = "local_config_cuda", files = { - "BUILD.bazel": """\ -package(default_visibility = ["//visibility:public"]) - -config_setting( - name = "is_cuda_enabled", - define_values = {"local_config_cuda_enabled": "true"}, -) - -config_setting( - name = "is_cuda_compiler_nvcc", - define_values = {"local_config_cuda_compiler_nvcc": "true"}, -) -""", - "cuda/BUILD.bazel": """\ -package(default_visibility = ["//visibility:public"]) - -exports_files(["build_defs.bzl"]) - -filegroup( - name = "build_defs_bzl", - srcs = ["build_defs.bzl"], -) - -config_setting( - name = "using_config_cuda", - define_values = {"local_config_cuda_using_config": "true"}, -) - -config_setting( - name = "FALSE", - define_values = {"local_config_cuda_false": "true"}, -) -""", + "cuda/BUILD.bazel": "", "cuda/build_defs.bzl": _BZL_HELPERS + """\ cuda_library = always_false if_cuda = always_if_false @@ -72,22 +34,7 @@ is_cuda_configured = always_false """, }) simple_files(name = "local_config_rocm", files = { - "BUILD.bazel": "", - "rocm/BUILD.bazel": """\ -package(default_visibility = ["//visibility:public"]) - -exports_files(["build_defs.bzl"]) - -filegroup( - name = "build_defs_bzl", - srcs = ["build_defs.bzl"], -) - -config_setting( - name = "using_hipcc", - define_values = {"local_config_rocm_using_hipcc": "true"}, -) -""", + "rocm/BUILD.bazel": "", "rocm/build_defs.bzl": _BZL_HELPERS + """\ if_rocm = always_if_false if_rocm_is_configured = always_if_false @@ -100,91 +47,23 @@ get_rbe_amdgpu_pool = always_false }) simple_files(name = "local_config_sycl", files = { "BUILD.bazel": "", - "sycl/BUILD.bazel": """\ -package(default_visibility = ["//visibility:public"]) - -exports_files(["build_defs.bzl"]) - -filegroup( - name = "build_defs_bzl", - srcs = ["build_defs.bzl"], -) -""", + "sycl/BUILD.bazel": "", "crosstool/BUILD.bazel": "", - "sycl/build_defs.bzl": """\ -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -""" + _BZL_HELPERS + """\ + "sycl/build_defs.bzl": _BZL_HELPERS + """\ if_sycl = always_if_false if_sycl_is_configured = always_if_false -sycl_library = cc_library """, }) simple_files(name = "local_config_remote_execution", files = { "remote_execution.bzl": """gpu_test_tags = lambda: []""", }) simple_files(name = "local_config_tensorrt", files = { - "BUILD.bazel": """\ -package(default_visibility = ["//visibility:public"]) - -exports_files(["build_defs.bzl"]) - -filegroup( - name = "build_defs_bzl", - srcs = ["build_defs.bzl"], -) -""", "build_defs.bzl": _BZL_HELPERS + """if_tensorrt = always_if_false""", }) simple_files(name = "python_version_repo", files = { "py_version.bzl": """USE_PYWRAP_RULES = False""", }) simple_files(name = "rules_ml_toolchain", files = { - "py/rules_pywrap/BUILD.bazel": """\ -package(default_visibility = ["//visibility:public"]) - -exports_files([ - "pywrap.default.bzl", - "pywrap.impl.bzl", -]) - -filegroup( - name = "pywrap_bzl", - srcs = [ - "pywrap.default.bzl", - "pywrap.impl.bzl", - ], -) -""", - "py/rules_pywrap/pywrap.default.bzl": """\ -def use_pywrap_rules(): - return False - -def _unsupported(*args, **kwargs): - fail("rules_ml_toolchain pywrap rules are not available in this workspace") - -pybind_extension = _unsupported -pywrap_aware_cc_import = _unsupported -pywrap_aware_filegroup = _unsupported -pywrap_aware_genrule = _unsupported -pywrap_binaries = _unsupported -pywrap_common_library = _unsupported -pywrap_library = _unsupported -stripped_cc_info = _unsupported -""", - "py/rules_pywrap/pywrap.impl.bzl": """\ -def _unsupported(*args, **kwargs): - fail("rules_ml_toolchain pywrap rules are not available in this workspace") - -collected_pywrap_infos = _unsupported -generated_common_win_def_file = _unsupported -pybind_extension = _unsupported -python_extension = _unsupported -pywrap_binaries = _unsupported -pywrap_common_library = _unsupported -pywrap_library = _unsupported -stripped_cc_info = _unsupported -""", "third_party/gpus/BUILD.bazel": "", "third_party/gpus/nvidia_common_rules.bzl": """cuda_rpath_flags = lambda *args, **kwargs: []""", "third_party/extensions/sycl_configure.bzl": "", @@ -195,30 +74,17 @@ stripped_cc_info = _unsupported def _xla_impl(mctx): llvm("llvm-raw") stablehlo() - shardy() triton() - farmhash() - highwayhash() - hwloc() - eigen3() - ml_dtypes() tf_http_archive( name = "com_github_grpc_grpc", - sha256 = "41b695614b26652ff9e97ce50cfd4a6c7a3d45a9fe598d1454407746499bbf2c", - strip_prefix = "grpc-1.81.0", + sha256 = "dd6a2fa311ba8441bbefd2764c55b99136ff10f7ea42954be96006a2723d33fc", + strip_prefix = "grpc-1.74.0", patch_file = ["//third_party/grpc:grpc.patch"], - urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.81.0.tar.gz"), + urls = tf_mirror_urls("https://github.com/grpc/grpc/archive/refs/tags/v1.74.0.tar.gz"), ) tf_vendored(name = "tsl", path = "third_party/tsl") - tf_http_archive( - name = "snappy", - build_file = "//third_party:snappy.BUILD", - sha256 = "90f74bc1fbf78a6c56b3c4a082a05103b3a56bb17bca1a27e052ea11723292dc", - strip_prefix = "snappy-1.2.2", - urls = tf_mirror_urls("https://github.com/google/snappy/archive/refs/tags/1.2.2.tar.gz"), - ) _dummy_repos(mctx) return mctx.extension_metadata( diff --git a/third_party/zigimg/BUILD.bazel b/third_party/zigimg/BUILD.bazel deleted file mode 100644 index 8b137891..00000000 --- a/third_party/zigimg/BUILD.bazel +++ /dev/null @@ -1 +0,0 @@ - diff --git a/third_party/zigimg/repo.bzl b/third_party/zigimg/repo.bzl deleted file mode 100644 index a4bb178f..00000000 --- a/third_party/zigimg/repo.bzl +++ /dev/null @@ -1,10 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -def repo(): - git_repository( - name = "zigimg", - remote = "https://github.com/zigimg/zigimg.git", - # Match libvaxis build.zig.zon. - commit = "734fa919fb15b7917d4eb124f53600348f4ffa30", - build_file = Label("//third_party/zigimg:zigimg.bazel"), - ) diff --git a/third_party/zigimg/zigimg.bazel b/third_party/zigimg/zigimg.bazel deleted file mode 100644 index f02d9380..00000000 --- a/third_party/zigimg/zigimg.bazel +++ /dev/null @@ -1,18 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "zigimg", - srcs = glob([ - "src/*.zig", - "src/compressions/*.zig", - "src/compressions/deflate/*.zig", - "src/formats/*.zig", - "src/formats/jpeg/*.zig", - "src/formats/png/*.zig", - "src/formats/tiff/*.zig", - "src/Image/*.zig", - "tests/**/*.zig", - ]), - main = "zigimg.zig", - visibility = ["//visibility:public"], -) diff --git a/third_party/zio/repo.bzl b/third_party/zio/repo.bzl deleted file mode 100644 index 96133ef0..00000000 --- a/third_party/zio/repo.bzl +++ /dev/null @@ -1,9 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -def repo(): - git_repository( - name = "zio", - remote = "https://github.com/lalinsky/zio", - commit = "1b4e9787f8a4a67d91372a603b23258364f89bf0", - build_file = Label("//third_party/zio:zio.bazel"), - ) diff --git a/third_party/zio/zio.bazel b/third_party/zio/zio.bazel deleted file mode 100644 index 39805936..00000000 --- a/third_party/zio/zio.bazel +++ /dev/null @@ -1,46 +0,0 @@ -load("@bazel_skylib//rules:common_settings.bzl", "string_flag") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") -load("@zml//bazel:zig_options.bzl", "zig_option", "zig_options") - -string_flag( - name = "backend", - build_setting_default = "", - values = ["", "poll", "linux", "epoll", "kqueue", "io_uring", "iocp"], -) - -string_flag( - name = "scheduling", - build_setting_default = "work_stealing", - values = ["single_executor", "pinned", "work_stealing"], -) - -string_flag( - name = "resolve_beneath_mode", - build_setting_default = "strict", - values = ["strict", "best_effort"], -) - -zig_option(name = "backend_option", flag = ":backend", nullable = True) -zig_option(name = "scheduling_option", flag = ":scheduling") -zig_option(name = "resolve_beneath_mode_option", flag = ":resolve_beneath_mode") -zig_option(name = "no_hacks", bool = False) -zig_option(name = "scheduler_metrics", bool = False) - -zig_options( - name = "zio_build_options", - settings = [ - ":backend_option", - ":scheduling_option", - ":resolve_beneath_mode_option", - ":no_hacks", - ":scheduler_metrics", - ], -) - -zig_library( - name = "zio", - main = "src/zio.zig", - srcs = glob(["src/**/*.zig"]), - deps = [":zio_build_options"], - visibility = ["//visibility:public"], -) diff --git a/third_party/zls/BUILD.bazel b/third_party/zls/BUILD.bazel index 69cc6128..b47cae3e 100644 --- a/third_party/zls/BUILD.bazel +++ b/third_party/zls/BUILD.bazel @@ -1,36 +1,51 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") -load("@rules_zig//zig/zls:toolchain.bzl", "zls_toolchain") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load(":resolved_toolchain.bzl", "resolved_toolchain") -exports_files(glob(["*.bazel"])) - -zig_library( - name = "build_options", - main = "build_options.zig", - visibility = ["@zls//:__pkg__"], +# This is the ZLS binary toolchain type. Zig rule authors should use this +# target in the `toolchains` parameter in order to access the appropriate ZLS +# binary for the current execution platform. +toolchain_type( + name = "toolchain_type", + visibility = ["//visibility:public"], ) -zig_library( - name = "exe_options", - main = "exe_options.zig", - visibility = ["@zls//:__pkg__"], +resolved_toolchain( + name = "resolved_toolchain", + # Marked manual so that `bazel test //...` passes + # even if no toolchain is registered. + tags = ["manual"], + visibility = ["//visibility:public"], ) -zig_library( - name = "tracy_options", - import_name = "options", - main = "tracy_options.zig", - visibility = ["@zls//:__pkg__"], +exports_files( + [ + "workspace_printer.zig", + "zls_build_runner.zig", + "zls_runner.zig", + "zls.completion.tpl", + ], ) -zls_toolchain( - name = "zls", - bin = "@zls//:zls", - zls_version = "0.17.0", +# Execute `bazel run //util:update_filegroups` to update this target. +filegroup( + name = "all_files", + srcs = [ + ":BUILD.bazel", + ":zls.completion.tpl", + ":zls_build_runner.zig", + ":zls_completion.bzl", + ":zls_runner.zig", + ":zls_toolchain.bzl", + ], + visibility = ["@rules_zig//zig:__pkg__"], ) -toolchain( - name = "toolchain", - target_settings = ["@zig_toolchains//:any_0.17.release"], - toolchain = ":zls", - toolchain_type = "@rules_zig//zig/zls:toolchain_type", +zig_binary( + name = "workspace_printer", + main = "workspace_printer.zig", +) + +zig_binary( + name = "build_runner", + main = "zls_build_runner.zig", ) diff --git a/third_party/zls/build_options.zig b/third_party/zls/build_options.zig deleted file mode 100644 index b50cb2d8..00000000 --- a/third_party/zls/build_options.zig +++ /dev/null @@ -1,5 +0,0 @@ -const std = @import("std"); - -pub const version_string = "0.17.0-dev+eab2be0fd744"; -pub const version = std.SemanticVersion.parse(version_string) catch unreachable; -pub const minimum_runtime_zig_version_string = "0.17.0"; diff --git a/third_party/zls/diffz.bazel b/third_party/zls/diffz.bazel deleted file mode 100644 index fb4b85fa..00000000 --- a/third_party/zls/diffz.bazel +++ /dev/null @@ -1,8 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "diffz", - srcs = glob(["**/*.zig"]), - main = "DiffMatchPatch.zig", - visibility = ["@zls//:__pkg__"], -) diff --git a/third_party/zls/exe_options.zig b/third_party/zls/exe_options.zig deleted file mode 100644 index eaad61b5..00000000 --- a/third_party/zls/exe_options.zig +++ /dev/null @@ -1,3 +0,0 @@ -pub const enable_failing_allocator = false; -pub const enable_failing_allocator_likelihood: u32 = 256; -pub const debug_gpa = false; diff --git a/third_party/zls/known_folders.bazel b/third_party/zls/known_folders.bazel deleted file mode 100644 index 234db4b0..00000000 --- a/third_party/zls/known_folders.bazel +++ /dev/null @@ -1,8 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library") - -zig_library( - name = "known-folders", - srcs = glob(["**/*.zig"]), - main = "known-folders.zig", - visibility = ["@zls//:__pkg__"], -) diff --git a/third_party/zls/lsp.bazel b/third_party/zls/lsp.bazel deleted file mode 100644 index b7c6b6dd..00000000 --- a/third_party/zls/lsp.bazel +++ /dev/null @@ -1,51 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") - -# @embedFile resolves a module's root file without parsing it as Zig. -genrule( - name = "meta_model_source", - srcs = ["metaModel.json"], - outs = ["meta_model.zig"], - cmd = "cp $(<) $(@)", -) - -zig_library( - name = "meta-model", - main = ":meta_model.zig", -) - -zig_binary( - name = "codegen", - srcs = glob(["src/codegen/*.zig"]), - extra_srcs = ["src/codegen/config.zon"], - main = "src/codegen/codegen.zig", - deps = [":meta-model"], -) - -genrule( - name = "generate_types", - outs = ["lsp_types.zig"], - cmd = "$(location :codegen) $(@)", - tools = [":codegen"], -) - -zig_library( - name = "parser", - main = "src/parser.zig", -) - -zig_library( - name = "types", - main = ":lsp_types.zig", - deps = [":parser"], -) - -zig_library( - name = "lsp", - srcs = glob(["src/*.zig"]), - main = "src/lsp.zig", - visibility = ["@zls//:__pkg__"], - deps = [ - ":parser", - ":types", - ], -) diff --git a/third_party/zls/repo.bzl b/third_party/zls/repo.bzl deleted file mode 100644 index 27f05c5c..00000000 --- a/third_party/zls/repo.bzl +++ /dev/null @@ -1,28 +0,0 @@ -load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") - -def repo(): - # ZLS 0.17 development revision; dependencies match its build.zig.zon. - git_repository( - name = "zls", - remote = "https://github.com/zigtools/zls.git", - commit = "eab2be0fd74443a27662da809b771c4ad0d2afcf", - build_file = Label("//third_party/zls:zls.bazel"), - ) - git_repository( - name = "zls_lsp", - remote = "https://github.com/zigtools/lsp-kit.git", - commit = "d148676ace1eaecf3c703ce7d350d0e94e502ed7", - build_file = Label("//third_party/zls:lsp.bazel"), - ) - git_repository( - name = "zls_diffz", - remote = "https://github.com/ziglibs/diffz.git", - commit = "aac8aa99c436ab8277b0711922aad062c0167b12", - build_file = Label("//third_party/zls:diffz.bazel"), - ) - git_repository( - name = "zls_known_folders", - remote = "https://github.com/ziglibs/known-folders.git", - commit = "6da0d0c41b78b9ed2d34fa364fcb81b5ebece6c4", - build_file = Label("//third_party/zls:known_folders.bazel"), - ) diff --git a/third_party/zls/resolved_toolchain.bzl b/third_party/zls/resolved_toolchain.bzl new file mode 100644 index 00000000..c5e38e63 --- /dev/null +++ b/third_party/zls/resolved_toolchain.bzl @@ -0,0 +1,24 @@ +"""This module implements an alias rule to the resolved toolchain.""" + +DOC = """\ +Exposes a concrete toolchain which is the result of Bazel resolving the +toolchain for the execution or target platform. +Workaround for https://github.com/bazelbuild/bazel/issues/14009 +""" + +# Forward all the providers +def _resolved_toolchain_impl(ctx): + toolchain_info = ctx.toolchains["//third_party/zls:toolchain_type"] + return [ + toolchain_info, + toolchain_info.default, + toolchain_info.zlstoolchaininfo, + ] + +# Copied from java_toolchain_alias +# https://cs.opensource.google/bazel/bazel/+/master:tools/jdk/java_toolchain_alias.bzl +resolved_toolchain = rule( + implementation = _resolved_toolchain_impl, + toolchains = ["//third_party/zls:toolchain_type"], + doc = DOC, +) diff --git a/third_party/zls/string_replace_reader.zig b/third_party/zls/string_replace_reader.zig new file mode 100644 index 00000000..3b68bf67 --- /dev/null +++ b/third_party/zls/string_replace_reader.zig @@ -0,0 +1,28 @@ +const std = @import("std"); + +pub const StringReplaceReader = struct { + interface: std.Io.Reader = .{ + .buffer = &.{}, + .vtable = &.{ + .stream = stream, + }, + .seek = 0, + .end = 0, + }, + in: *std.Io.Reader, + pattern: []const u8, + replace_by: []const u8, + + pub fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { + const self: *StringReplaceReader = @fieldParentPtr("interface", r); + if (std.mem.eql(u8, try self.in.peek(self.pattern.len), self.pattern)) { + self.in.toss(self.pattern.len); + try w.writeAll(self.replace_by); + return self.replace_by.len; + } + return self.in.streamDelimiterLimit(w, self.pattern[0], limit) catch |err| switch (err) { + std.Io.Reader.StreamDelimiterLimitError.StreamTooLong => return 0, + else => |e| return e, + }; + } +}; diff --git a/third_party/zls/tracy_options.zig b/third_party/zls/tracy_options.zig deleted file mode 100644 index 01dfcb1d..00000000 --- a/third_party/zls/tracy_options.zig +++ /dev/null @@ -1,3 +0,0 @@ -pub const enable = false; -pub const enable_allocation = false; -pub const enable_callstack = false; diff --git a/third_party/zls/workspace_printer.zig b/third_party/zls/workspace_printer.zig new file mode 100644 index 00000000..cbd8e130 --- /dev/null +++ b/third_party/zls/workspace_printer.zig @@ -0,0 +1,39 @@ +const std = @import("std"); + +pub fn main(init: std.process.Init) !void { + const arena = init.arena; + const io = init.io; + const args = try init.minimal.args.toSlice(arena.allocator()); + + const build_workspace_directory = init.environ_map.get("BUILD_WORKSPACE_DIRECTORY").?; + var output_base_result = try std.process.run(arena.allocator(), init.io, .{ + .argv = &.{ + "bazel", + "info", + "execution_root", + }, + .cwd = .{ .path = build_workspace_directory }, + }); + + var output = try std.Io.Dir.cwd().readFileAlloc( + io, + args[1], + arena.allocator(), + .unlimited, + ); + output = try std.mem.replaceOwned( + u8, + arena.allocator(), + output, + "@@__BUILD_WORKSPACE_DIRECTORY__@@", + build_workspace_directory, + ); + output = try std.mem.replaceOwned( + u8, + arena.allocator(), + output, + "@@__BAZEL_EXECUTION_ROOT__@@", + std.mem.trimEnd(u8, output_base_result.stdout, "\n"), + ); + try std.Io.File.stdout().writeStreamingAll(io, output); +} diff --git a/third_party/zls/zls.bazel b/third_party/zls/zls.bazel deleted file mode 100644 index 56ccbff8..00000000 --- a/third_party/zls/zls.bazel +++ /dev/null @@ -1,53 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") - -zig_binary( - name = "config_gen", - extra_srcs = ["src/tools/config.json"], - main = "src/tools/config_gen.zig", -) - -genrule( - name = "generate_version_data", - srcs = ["src/tools/langref.html.in"], - outs = ["version_data.zig"], - cmd = "$(location :config_gen) --langref-version master --langref-path $(<) --generate-version-data $(@)", - tools = [":config_gen"], -) - -zig_library( - name = "version_data", - main = ":version_data.zig", -) - -zig_library( - name = "tracy", - main = "src/tracy.zig", - deps = ["@zml//third_party/zls:tracy_options"], -) - -zig_library( - name = "zls_lib", - srcs = glob(["src/**/*.zig"]), - import_name = "zls", - main = "src/zls.zig", - deps = [ - ":tracy", - ":version_data", - "@zls_diffz//:diffz", - "@zls_lsp//:lsp", - "@zml//third_party/zls:build_options", - ], -) - -zig_binary( - name = "zls", - srcs = glob(["src/**/*.zig"]), - main = "src/main.zig", - visibility = ["//visibility:public"], - deps = [ - ":tracy", - ":zls_lib", - "@zls_known_folders//:known-folders", - "@zml//third_party/zls:exe_options", - ], -) diff --git a/third_party/zls/zls.completion.tpl b/third_party/zls/zls.completion.tpl new file mode 100644 index 00000000..b3fb688e --- /dev/null +++ b/third_party/zls/zls.completion.tpl @@ -0,0 +1,10 @@ +{ + "dependencies": {}, + "modules": [ + @@PACKAGES@@ + ], + "include_dirs": [], + "top_level_steps": [], + "available_options": {}, + "c_macros": [] +} diff --git a/third_party/zls/zls_build_runner.zig b/third_party/zls/zls_build_runner.zig new file mode 100644 index 00000000..ce82a66d --- /dev/null +++ b/third_party/zls/zls_build_runner.zig @@ -0,0 +1,14 @@ +const std = @import("std"); + +pub fn main(init: std.process.Init) !void { + const build_workspace_directory = init.environ_map.get("BUILD_WORKSPACE_DIRECTORY").?; + var child = try std.process.spawn(init.io, .{ + .argv = &.{ + "bazel", + "run", + "@@__TARGET__@@", + }, + .cwd = .{ .path = build_workspace_directory }, + }); + _ = try child.wait(init.io); +} diff --git a/third_party/zls/zls_completion.bzl b/third_party/zls/zls_completion.bzl new file mode 100644 index 00000000..9028c386 --- /dev/null +++ b/third_party/zls/zls_completion.bzl @@ -0,0 +1,71 @@ +"""Implementation of the zls_completion macro.""" + +load("@aspect_bazel_lib//lib:utils.bzl", "utils") +load("@bazel_skylib//rules:expand_template.bzl", "expand_template") +load("@rules_zig//zig:defs.bzl", "zig_binary") +load(":zls_write_build_config.bzl", "zls_write_build_config") +load(":zls_write_runner_zig_src.bzl", "zls_write_runner_zig_src") + +def zls_completion(name, deps, **kwargs): + """Entry point for ZLS completion. + + Args: + name: The name of the completion target. + deps: The List of Zig modules to include for completion. + **kwargs: Additional keyword arguments passed to the `zig_binary` + """ + + # Generate the ZLS BuildConfig file. + # It contains the list of Zig packages alongside their main Zig file paths. + build_config = name + ".build_config" + zls_write_build_config( + name = build_config, + out = name + ".build_config.json", + deps = deps, + ) + + # Create a target that will be invoked by ZLS using our customer build_runner. + build_config_printer = "{}.print_build_config".format(name) + zig_binary( + name = build_config_printer, + main = Label("//third_party/zls:workspace_printer.zig"), + data = [ + ":{}".format(build_config), + ], + args = [ + "$(location :{})".format(build_config), + ], + visibility = ["//visibility:private"], + ) + + # Generate the Zig build runner that will be used by ZLS to query the build config. + expand_template( + name = name + ".build_runner", + out = name + ".build_runner.zig", + substitutions = { + "@@__TARGET__@@": str(utils.to_label(build_config_printer)), + }, + template = Label(":zls_build_runner.zig"), + ) + + # Generate the Zig source file for the ZLS runner binary which embeds the + # rlocationpath of all runtime dependencies of the ZLS runner binary. + zls_write_runner_zig_src( + name = name + ".runner", + out = name + ".runner.zig", + build_runner = ":" + name + ".build_runner.zig", + ) + + zig_binary( + name = name, + main = name + ".runner", + data = [ + "@rules_zig//zig:resolved_toolchain", + "@zml//third_party/zls:resolved_toolchain", + name + ".build_runner", + ], + deps = [ + "@rules_zig//zig/runfiles", + ], + **kwargs + ) diff --git a/third_party/zls/zls_runner.zig b/third_party/zls/zls_runner.zig new file mode 100644 index 00000000..c0808d9e --- /dev/null +++ b/third_party/zls/zls_runner.zig @@ -0,0 +1,109 @@ +/// Custom ZLS launcher. +/// +/// Sets up paths to ZLS dependencies with Bazel runfiles. +/// +/// This file is used as a template by `zls_write_runner_zig_src.bzl`. +const std = @import("std"); + +const bazel_builtin = @import("bazel_builtin"); +const runfiles = @import("runfiles"); + +fn getRandomFilename(io: std.Io, buf: []u8, extension: []const u8) ![]const u8 { + const now = std.Io.Clock.real.now(io); + return std.fmt.bufPrint(buf, "/tmp/{d}{s}", .{ now.nanoseconds, extension }) catch @panic("OOM"); +} + +const Config = struct { + /// Override the Zig library path. Will be automatically resolved using the 'zig_exe_path'. + zig_lib_path: ?[]const u8 = null, + + /// Specify the path to the Zig executable (not the directory). If unset, zig is looked up in `PATH`. e.g. `/path/to/zig-templeos-armless-1.0.0/zig`. + zig_exe_path: ?[]const u8 = null, + + /// Specify a custom build runner to resolve build system information. + build_runner_path: ?[]const u8 = null, + + /// Path to a directory that will be used as zig's cache. Will default to `${KnownFolders.Cache}/zls`. + global_cache_path: ?[]const u8 = null, +}; + +pub fn main(init: std.process.Init) !void { + const arena = init.arena; + const io = init.io; + + var r_ = try runfiles.Runfiles.create(.{ .allocator = arena.allocator(), .io = io, .environ_map = init.environ_map, .argv = init.minimal.args }) orelse + return error.RunfilesNotFound; + const r = r_.withSourceRepo(bazel_builtin.current_repository); + + const zls_bin_rpath = "@@__ZLS_BIN_RPATH__@@"; + var zls_bin_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const zls_bin_path = try r.rlocation(zls_bin_rpath, &zls_bin_path_buf) orelse + return error.RLocationNotFound; + + const zig_exe_rpath = "@@__ZIG_EXE_RPATH__@@"; + var zig_exe_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const zig_exe_path = blk2: { + if (zig_exe_rpath[0] == '/') { + break :blk2 zig_exe_rpath; + } else { + break :blk2 try r.rlocation(zig_exe_rpath, &zig_exe_path_buf) orelse + return error.RLocationNotFound; + } + }; + + const zig_lib_path = "@@__ZIG_LIB_PATH__@@"; + const zig_lib_computed_path = blk: { + if (zig_lib_path[0] == '/') { + break :blk zig_lib_path; + } else { + break :blk null; + } + }; + + const zls_build_runner_rpath = "@@__ZLS_BUILD_RUNNER_RPATH__@@"; + var zls_build_runner_path_buf: [std.Io.Dir.max_path_bytes]u8 = undefined; + const zls_build_runner_path = try r.rlocation(zls_build_runner_rpath, &zls_build_runner_path_buf) orelse + return error.RLocationNotFound; + + const global_cache_path = "@@__GLOBAL_CACHE_PATH__@@"; + + const config: Config = .{ + .zig_exe_path = zig_exe_path, + .zig_lib_path = zig_lib_computed_path, + .build_runner_path = zls_build_runner_path, + .global_cache_path = global_cache_path, + }; + + var buf: [std.Io.Dir.max_name_bytes]u8 = undefined; + const tmp_file_path = try getRandomFilename(io, &buf, ".json"); + + { + var tmp_file = try std.Io.Dir.createFileAbsolute(io, tmp_file_path, .{}); + defer tmp_file.close(io); + + var out_buf: [4096]u8 = undefined; + var tmp_file_writer = tmp_file.writer(io, &out_buf); + + const formatter = std.json.fmt(config, .{ .whitespace = .indent_2 }); + + try tmp_file_writer.interface.print("{f}", .{formatter}); + try tmp_file_writer.interface.flush(); + } + + const args = init.minimal.args.toSlice(arena.allocator()) catch return error.ArgsAllocFailed; + + const exec_args_len = args.len - 1 + 3; // Skip args[0] and add "zls" + --config-path + tmp_file_path + var exec_args = try arena.allocator().alloc([]const u8, exec_args_len); + + exec_args[0] = zls_bin_path; // "zls" + + @memcpy(exec_args[1 .. exec_args.len - 2], args[1..]); + @memcpy(exec_args[exec_args.len - 2 ..], &[_][]const u8{ + "--config-path", + tmp_file_path, + }); + var child = try std.process.spawn(io, .{ + .argv = exec_args, + }); + _ = try child.wait(io); +} diff --git a/third_party/zls/zls_toolchain.bzl b/third_party/zls/zls_toolchain.bzl new file mode 100644 index 00000000..d83e8f1a --- /dev/null +++ b/third_party/zls/zls_toolchain.bzl @@ -0,0 +1,123 @@ +"""Implementation of the zls_toolchain rule.""" + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +_VERSION = "0.16.0-dev.215+0e289f5c" + +_ZLS_PLATFORMS = { + "x86_64-linux": struct( + compatible_with = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", + ], + ), + "x86_64-macos": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:x86_64", + ], + ), + "aarch64-macos": struct( + compatible_with = [ + "@platforms//os:macos", + "@platforms//cpu:aarch64", + ], + ), +} + +_ZLS_VERSIONS = { + "x86_64-linux": "026a0ce9ea06b276319210509fa2a2edd7acf4e68da83b1a704c06ae26671548", + "x86_64-macos": "061845ec346620b914063763e8e39f9a374d2817c4ae976ea3eeb577a6e1cf32", + "aarch64-macos": "fb711e068e2ec77b3979458ff72f18311fa7df74644a8cf43985c0f88cae5c66", +} + +ZlsToolchainInfo = provider( + doc = "Provide info for executing zls", + fields = { + "bin": "Executable ZLS binary", + }, +) + +def _zls_toolchain_impl(ctx): + binary = ctx.file.bin + + default_info = DefaultInfo( + files = depset([binary]), + runfiles = ctx.runfiles(files = [binary]), + ) + zlstoolchaininfo = ZlsToolchainInfo( + bin = binary, + ) + toolchain_info = platform_common.ToolchainInfo( + default = default_info, + zlstoolchaininfo = zlstoolchaininfo, + ) + + return [ + default_info, + zlstoolchaininfo, + toolchain_info, + ] + +zls_toolchain = rule( + implementation = _zls_toolchain_impl, + attrs = { + "bin": attr.label( + executable = True, + allow_single_file = True, + cfg = "exec", + ), + }, +) + +def _zls_toolchains_repos_impl(mctx): + for platform, _ in _ZLS_PLATFORMS.items(): + http_archive( + name = "zls_{}".format(platform), + url = "https://builds.zigtools.org/zls-{platform}-{version}.tar.xz".format( + version = _VERSION, + platform = platform, + ), + sha256 = _ZLS_VERSIONS[platform], + build_file_content = """\ +load("@zml//third_party/zls:zls_toolchain.bzl", "zls_toolchain") +zls_toolchain(name = "toolchain", bin = "zls") +""", + ) + + zls_toolchains_repo(name = "zls_toolchains") + + return mctx.extension_metadata( + reproducible = True, + root_module_direct_deps = ["zls_toolchains"], + root_module_direct_dev_deps = [], + ) + +zls_toolchains = module_extension( + implementation = _zls_toolchains_repos_impl, +) + +def _zls_toolchains_repo_impl(rctx): + build_content = """# @generated by @zml//third_party/zls:zls_toolchain.bzl""" + + for platform, config in _ZLS_PLATFORMS.items(): + build_content += """ +toolchain( + name = "{platform}_toolchain", + exec_compatible_with = {compatible_with}, + target_compatible_with = {compatible_with}, + toolchain = "@zls_{platform}//:toolchain", + toolchain_type = "@zml//third_party/zls:toolchain_type", +) +""".format( + platform = platform, + compatible_with = config.compatible_with, + ) + + rctx.file("BUILD.bazel", build_content) + +zls_toolchains_repo = repository_rule( + _zls_toolchains_repo_impl, + doc = """Creates a repository with toolchain definitions for all known platforms + which can be registered or selected.""", +) diff --git a/third_party/zls/zls_write_build_config.bzl b/third_party/zls/zls_write_build_config.bzl new file mode 100644 index 00000000..fdd20322 --- /dev/null +++ b/third_party/zls/zls_write_build_config.bzl @@ -0,0 +1,176 @@ +"""Generates the ZLS build config file.""" + +load("@rules_cc//cc:find_cc_toolchain.bzl", "use_cc_toolchain") +load("@rules_cc//cc/common:cc_common.bzl", "cc_common") +load("@rules_cc//cc/common:cc_info.bzl", "CcInfo") +load("@rules_zig//zig/private:cc_helper.bzl", "need_translate_c") +load("@rules_zig//zig/private/common:translate_c.bzl", "zig_translate_c") +load("@rules_zig//zig/private/common:zig_cache.bzl", "zig_cache_output") +load("@rules_zig//zig/private/common:zig_lib_dir.bzl", "zig_lib_dir") +load("@rules_zig//zig/private/providers:zig_module_info.bzl", "ZigModuleInfo", "zig_module_info") +load( + "@rules_zig//zig/private/providers:zig_target_info.bzl", + "zig_target_platform", +) + +def _zls_construct_zig_module_info_impl(target, ctx): + """Aspect that constructs ZigModuleInfo for zig_binary and zig_library rules.""" + if ZigModuleInfo in target: + return [] + + if ctx.rule.kind not in ("zig_binary", "zig_static_library", "zig_shared_library"): + return [] + + cdeps = [] + zdeps = [] + for dep in ctx.rule.attr.deps: + if ZigModuleInfo in dep: + zdeps.append(dep[ZigModuleInfo]) + elif CcInfo in dep: + cdeps.append(dep[CcInfo]) + + root_module_is_only_dep = len(ctx.rule.attr.deps) == 1 and ZigModuleInfo in ctx.rule.attr.deps[0] + if root_module_is_only_dep: + root_module = ctx.rule.attr.deps[0][ZigModuleInfo] + else: + root_module = zig_module_info( + name = ctx.rule.attr.name, + canonical_name = target.label.name, + main = ctx.rule.file.main, + srcs = ctx.rule.files.srcs, + extra_srcs = ctx.rule.files.extra_srcs, + deps = zdeps, # [bazel_builtin_module(ctx)], + cdeps = cdeps, + zigopts = [], + ) + + return [ + root_module, + ] + +zls_construct_zig_module_info = aspect( + implementation = _zls_construct_zig_module_info_impl, +) + +def format_main_file(main): + prefix = "@@__BUILD_WORKSPACE_DIRECTORY__@@/" + if (main.startswith("bazel-out/") or main.startswith("external/")): + prefix = "@@__BAZEL_EXECUTION_ROOT__@@/" + return prefix + main + +def _zls_write_build_config_impl(ctx): + zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo + zigtargetinfo = ctx.toolchains["@rules_zig//zig/target:toolchain_type"].zigtargetinfo + + c_module_contexts = [] + c_module_inputs = [] + cc_info = cc_common.merge_cc_infos(direct_cc_infos = [dep[ZigModuleInfo].cc_info for dep in ctx.attr.deps if dep[ZigModuleInfo].cc_info]) + if need_translate_c(cc_info): + global_args = ctx.actions.args() + global_args.use_param_file("@%s") + + zig_target_platform( + target = zigtargetinfo, + args = global_args, + ) + + zig_lib_dir( + zigtoolchaininfo = zigtoolchaininfo, + args = global_args, + ) + + zig_cache_output( + zigtoolchaininfo = zigtoolchaininfo, + args = global_args, + ) + + c_module = zig_translate_c( + ctx = ctx, + name = "c", + canonical_name = "c", + zigtoolchaininfo = zigtoolchaininfo, + global_args = global_args, + cc_infos = [cc_info], + ) + + c_module_contexts = [depset( + direct = [c_module.module_context], + transitive = [c_module.transitive_module_contexts], + )] + c_module_inputs = [c_module.transitive_inputs] + + contexts = depset( + direct = [dep[ZigModuleInfo].module_context for dep in ctx.attr.deps], + transitive = [dep[ZigModuleInfo].transitive_module_contexts for dep in ctx.attr.deps] + c_module_contexts, + ) + + modules = { + mod.canonical_name: mod + for mod in contexts.to_list() + } + + output = { + "dependencies": {}, + "modules": { + format_main_file(mod.main): { + "import_table": { + dep.name: format_main_file(modules.get(dep.canonical_name, "").main) + for dep in mod.dependency_mappings + }, + "c_macros": [], + "include_dirs": [], + } + for mod in modules.values() + }, + "compilations": [], + "top_level_steps": [], + "available_options": {}, + } + + config = ctx.outputs.out + ctx.actions.write( + output = config, + content = json.encode(output), + ) + + return [ + DefaultInfo( + files = depset(direct = [config]), + # Not really runfiles but we need to pull the output of translate_c... + runfiles = ctx.runfiles(transitive_files = depset(transitive = c_module_inputs)), + ), + ] + +zls_write_build_config = rule( + implementation = _zls_write_build_config_impl, + attrs = { + "deps": attr.label_list( + providers = [ZigModuleInfo], + mandatory = True, + aspects = [zls_construct_zig_module_info], + ), + "out": attr.output( + mandatory = True, + ), + "_zls_completion_tpl": attr.label( + default = ":zls.completion.tpl", + allow_single_file = True, + ), + "_translate_c": attr.label( + default = Label("@rules_zig//zig/private/common:translate-c"), + cfg = "exec", + executable = True, + ), + "_c_helpers": attr.label( + default = Label("@rules_zig//zig/private/common:helpers"), + ), + "_c_builtins": attr.label( + default = Label("@rules_zig//zig/private/common:c_builtins"), + ), + }, + toolchains = [ + "@rules_zig//zig:toolchain_type", + "@rules_zig//zig/target:toolchain_type", + ] + use_cc_toolchain(mandatory = False), + fragments = ["cpp", "apple"], +) diff --git a/third_party/zls/zls_write_runner_zig_src.bzl b/third_party/zls/zls_write_runner_zig_src.bzl new file mode 100644 index 00000000..ac89e08e --- /dev/null +++ b/third_party/zls/zls_write_runner_zig_src.bzl @@ -0,0 +1,45 @@ +"""Generates the ZLS runner source file.""" + +load("@aspect_bazel_lib//lib:paths.bzl", "to_rlocation_path") + +def _zls_write_runner_zig_src_impl(ctx): + zigtoolchaininfo = ctx.toolchains["@rules_zig//zig:toolchain_type"].zigtoolchaininfo + zlstoolchaininfo = ctx.toolchains["//third_party/zls:toolchain_type"].zlstoolchaininfo + + zls_runner = ctx.outputs.out + ctx.actions.expand_template( + output = zls_runner, + template = ctx.file._runner_tpl, + substitutions = { + "@@__ZIG_EXE_RPATH__@@": zigtoolchaininfo.zig_exe_rpath, + "@@__ZIG_LIB_PATH__@@": zigtoolchaininfo.zig_lib_rpath, + "@@__ZLS_BIN_RPATH__@@": to_rlocation_path(ctx, zlstoolchaininfo.bin), + "@@__ZLS_BUILD_RUNNER_RPATH__@@": to_rlocation_path(ctx, ctx.file.build_runner), + "@@__GLOBAL_CACHE_PATH__@@": zigtoolchaininfo.zig_cache, + }, + ) + + return [ + DefaultInfo(files = depset([zls_runner])), + ] + +zls_write_runner_zig_src = rule( + implementation = _zls_write_runner_zig_src_impl, + attrs = { + "out": attr.output( + mandatory = True, + ), + "build_runner": attr.label( + allow_single_file = True, + mandatory = True, + ), + "_runner_tpl": attr.label( + default = "zls_runner.zig", + allow_single_file = True, + ), + }, + toolchains = [ + "@rules_zig//zig:toolchain_type", + "//third_party/zls:toolchain_type", + ], +) diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel index 844778f5..5dc3d219 100644 --- a/tools/BUILD.bazel +++ b/tools/BUILD.bazel @@ -1,26 +1,7 @@ -load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_python//python:py_library.bzl", "py_library") -load("@rules_shell//shell:sh_binary.bzl", "sh_binary") py_library( name = "zml_utils", srcs = ["zml_utils.py"], visibility = ["//visibility:public"], ) - -sh_binary( - name = "nsys", - srcs = ["nsys.sh"], - visibility = ["//visibility:public"], -) - -sh_binary( - name = "rocprofv3", - srcs = ["rocprofv3.sh"], - data = [ - "@amdrocm-profiler-base//:rocprofv3_tool", - "@libzml_rocm//:sandbox", - ], - target_compatible_with = ["@platforms//os:linux"], - visibility = ["//visibility:public"], -) diff --git a/tools/neuron/BUILD.bazel b/tools/neuron/BUILD.bazel deleted file mode 100644 index 8bfcabed..00000000 --- a/tools/neuron/BUILD.bazel +++ /dev/null @@ -1,51 +0,0 @@ -load("@rules_shell//shell:sh_binary.bzl", "sh_binary") - -sh_binary( - name = "profile", - srcs = ["profile.sh"], - data = ["@libzml_neuron//:sandbox"], - visibility = ["//visibility:public"], -) - -sh_binary( - name = "server", - srcs = ["server.sh"], - args = [ - "/tmp/zml-neuron-explorer", - "3001", - ], - data = ["@libzml_neuron//:sandbox"], - visibility = ["//visibility:public"], -) - -sh_binary( - name = "ingest", - srcs = ["ingest.sh"], - args = [ - "/tmp/zml-neuron-explorer", - "/tmp/zml-neuron-profile", - ], - data = ["@libzml_neuron//:sandbox"], - visibility = ["//visibility:public"], -) - -sh_binary( - name = "summary-txt", - srcs = ["summary.sh"], - data = ["@libzml_neuron//:sandbox"], - visibility = ["//visibility:public"], -) - -sh_binary( - name = "summary-json", - srcs = ["summary.sh"], - data = ["@libzml_neuron//:sandbox"], - visibility = ["//visibility:public"], -) - -sh_binary( - name = "summary-perfetto", - srcs = ["summary.sh"], - data = ["@libzml_neuron//:sandbox"], - visibility = ["//visibility:public"], -) diff --git a/tools/neuron/ingest.sh b/tools/neuron/ingest.sh deleted file mode 100755 index bb03850d..00000000 --- a/tools/neuron/ingest.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -shopt -s nullglob - -neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" -data_path="$1" -default_profile_root="$2" -shift 2 - -mkdir -p "${data_path}" - -ingest_run() { - local run_dir="$1" - local execution_dir="${run_dir}/execution" - local run_id - run_id="$(basename "${run_dir}")" - - if [[ ! -d "${execution_dir}" ]]; then - echo "skipping ${run_dir}: missing execution directory" >&2 - return - fi - - for profile_session in "${execution_dir}"/*/*; do - [[ -f "${profile_session}/ntrace.pb" ]] || continue - - local system_dir - system_dir="$(mktemp -d)" - for file in ntrace.pb trace_info.pb cpu_util.pb host_mem.pb; do - [[ -f "${profile_session}/${file}" ]] && cp "${profile_session}/${file}" "${system_dir}/${file}" - done - - "${neuron_explorer_bin}" view \ - --ingest-only \ - --force \ - --data-path "${data_path}" \ - --display-name "${run_id}:system" \ - -d "${system_dir}" - - rm -rf "${system_dir}" - done - - for neff in "${execution_dir}"/*/*/*.neff; do - local neff_name - local program_name - local ntff_prefix - neff_name="$(basename "${neff}" .neff)" - program_name="$(grep -aEom1 'MODULE_[A-Za-z0-9_]+' "${neff}" | sed -E 's/^MODULE_//; s/_[0-9]+$//')" - if [[ -z "${program_name}" ]]; then - echo "failed to find program name in ${neff}" >&2 - exit 1 - fi - ntff_prefix="$(dirname "${neff}")/${neff_name}" - - if ! compgen -G "${ntff_prefix}_rank_*.ntff" >/dev/null && [[ ! -f "${ntff_prefix}.ntff" ]]; then - "${neuron_explorer_bin}" capture \ - -n "${neff}" \ - -s "${ntff_prefix}.ntff" \ - --num-exec=2 \ - --profile-nth-exec=2 \ - --io-from=neff - fi - - for ntff in "${ntff_prefix}"_rank_*.ntff "${ntff_prefix}".ntff; do - [[ -f "${ntff}" ]] || continue - - local core_suffix - local display_name - core_suffix="$(basename "${ntff}" .ntff | sed -E 's/^.*_rank_([0-9]+).*$/nc\1/; s/^.*_(vnc_[0-9]+)$/\1/')" - display_name="${run_id}:${program_name}_${core_suffix}" - - "${neuron_explorer_bin}" view \ - --ingest-only \ - --force \ - --data-path "${data_path}" \ - --display-name "${display_name}" \ - -n "${neff}" \ - -s "${ntff}" - done - done -} - -[[ $# -eq 0 ]] && set -- "${default_profile_root}" - -for path in "$@"; do - if [[ -d "${path}/execution" ]]; then - ingest_run "${path}" - else - for run_dir in "${path}"/*; do - [[ -d "${run_dir}" ]] && ingest_run "${run_dir}" - done - fi -done diff --git a/tools/neuron/profile.sh b/tools/neuron/profile.sh deleted file mode 100755 index e1118d56..00000000 --- a/tools/neuron/profile.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -profile_root="$1" -shift - -run_dir="${profile_root}/$(date -u '+%Y%m%dT%H%M%SZ')" -execution_dir="${run_dir}/execution" -compile_dir="${run_dir}/compile" - -mkdir -p "${execution_dir}" "${compile_dir}" - -neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" - -xla_flags="--xla_dump_to=${compile_dir} --xla_dump_hlo_as_text --xla_dump_hlo_as_proto --xla_dump_hlo_pass_re=.*" - -set +e -env \ - NEURON_RT_INSPECT_ENABLE=1 \ - NEURON_RT_INSPECT_SYSTEM_PROFILE=1 \ - NEURON_RT_INSPECT_DEVICE_PROFILE=1 \ - NEURON_RT_INSPECT_OUTPUT_DIR="${execution_dir}" \ - SKIP_PJRT_PROFILER=true \ - XLA_IR_DEBUG=1 \ - XLA_HLO_DEBUG=1 \ - XLA_FLAGS="${xla_flags}" \ - "${neuron_explorer_bin}" inspect -o "${execution_dir}" "$@" -status=$? -set -e - -echo "Neuron profile dump path: ${run_dir}" >&2 - -exit "${status}" diff --git a/tools/neuron/server.sh b/tools/neuron/server.sh deleted file mode 100755 index 3d0babe1..00000000 --- a/tools/neuron/server.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -data_path="$1" -port="$2" -neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" - -mkdir -p "${data_path}" - -echo "Neuron Explorer server: http://127.0.0.1:${port}" - -exec "${neuron_explorer_bin}" view \ - --data-path "${data_path}" \ - --port "${port}" diff --git a/tools/neuron/summary.sh b/tools/neuron/summary.sh deleted file mode 100755 index 5a1b1461..00000000 --- a/tools/neuron/summary.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -neuron_explorer_bin="$0.runfiles/+neuron_packages+libzml_neuron/sandbox/bin/neuron-explorer" - -run_dir="$1" -execution_dir="${run_dir}/execution" - -case "$(basename "$0")" in - summary-txt) - output_format="summary-text" - output_file="summary.txt" - ;; - summary-json) - output_format="summary-json" - output_file="summary.json" - ;; - summary-perfetto) - output_format="perfetto" - output_file="summary.pftrace" - ;; -esac - -if [[ "${output_format}" == "summary-text" || "${output_format}" == "summary-json" ]]; then - "${neuron_explorer_bin}" view \ - -d "${execution_dir}" \ - --output-format "${output_format}" \ - >"${run_dir}/${output_file}" -else - "${neuron_explorer_bin}" view \ - -d "${execution_dir}" \ - --output-format "${output_format}" \ - --output-file "${run_dir}/${output_file}" -fi - -echo "Neuron summary path: ${run_dir}/${output_file}" >&2 diff --git a/tools/nsys.sh b/tools/nsys.sh deleted file mode 100755 index b8d38e38..00000000 --- a/tools/nsys.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ $# -lt 1 ]]; then - echo "usage: $0 [args...]" >&2 - exit 1 -fi - -sudo_cmd=() -if [[ "${ZML_PROFILE_NO_SUDO:-0}" != "1" ]]; then - sudo_cmd=(sudo -E) -fi - -declare -a nsys_args -if [[ -n "${ZML_NSYS_ARGS:-}" ]]; then - nsys_args=(${ZML_NSYS_ARGS}) -else - nsys_args=( - profile - -t - cuda,syscall,nvtx,cublas,cublas-verbose,cusparse,cusparse-verbose,cudnn,osrt - --inherit-environment=true - --cuda-memory-usage - true - --cuda-event-trace=false - --backtrace=dwarf - --cuda-graph-trace=node - ) -fi - -exec env SKIP_PJRT_PROFILER=true "${sudo_cmd[@]}" nsys "${nsys_args[@]}" "$@" diff --git a/tools/rocprofv3.sh b/tools/rocprofv3.sh deleted file mode 100755 index cc7e70bb..00000000 --- a/tools/rocprofv3.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if [[ $# -lt 1 ]]; then - echo "usage: $0 [args...]" >&2 - exit 1 -fi - -if [[ -n "${RUNFILES_DIR:-}" ]]; then - runfiles_root="${RUNFILES_DIR}" -elif [[ -d "$0.runfiles" ]]; then - runfiles_root="$0.runfiles" -else - echo "failed to locate Bazel runfiles for sandboxed rocprofv3" >&2 - exit 1 -fi - -rocm_root="${runfiles_root}/+rocm_packages+libzml_rocm/sandbox" -if [[ ! -d "${rocm_root}" ]]; then - echo "sandboxed ROCm runtime root not found at ${rocm_root}" >&2 - exit 1 -fi - -rocprofv3_bin="${rocm_root}/bin/rocprofv3" -if [[ ! -x "${rocprofv3_bin}" ]]; then - echo "sandboxed rocprofv3 not found at ${rocprofv3_bin}" >&2 - exit 1 -fi - -declare -a rocprof_args -if [[ -n "${ZML_ROCPROFV3_ARGS:-}" ]]; then - rocprof_args=(${ZML_ROCPROFV3_ARGS}) -else - rocprof_args=( - --sys-trace - --runtime-trace - --hip-trace - --kernel-trace - --memory-copy-trace - --hip-runtime-trace - --hsa-core-trace - --scratch-memory-trace - --marker-trace - --kernel-rename - --stats - --output-format - pftrace - ) -fi - -exec env SKIP_PJRT_PROFILER=true "${rocprofv3_bin}" --rocm-root "${rocm_root}" "${rocprof_args[@]}" -- "$@" diff --git a/tools/safetensors/BUILD.bazel b/tools/safetensors/BUILD.bazel deleted file mode 100644 index aff4954d..00000000 --- a/tools/safetensors/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") - -zig_binary( - name = "safetensors", - main = "main.zig", - visibility = ["//visibility:public"], - zigopts = ["-lc"], - deps = [ - "//stdx", - "//vfs", - "//zml", - ], -) diff --git a/tools/safetensors/main.zig b/tools/safetensors/main.zig deleted file mode 100644 index cf902eef..00000000 --- a/tools/safetensors/main.zig +++ /dev/null @@ -1,91 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); -const VFS = @import("vfs"); -const zml = @import("zml"); - -const log = std.log.scoped(.safetensors_convert); - -const Args = struct { - output: ?[]const u8 = null, - positional: struct { model: []const u8 }, - - pub const help = - \\Usage: [--output=] - \\ - \\Read safetensors headers and write model.safetensors.full_index.json in the working directory - \\Example: bazel run //tools/safetensors -- --output="model.safetensors.full_index.json" hf://deepseek-ai/DeepSeek-V4.1-Flash - ; -}; - -pub fn main(init: std.process.Init) !void { - const args = stdx.flags.parse(init.minimal.args, Args); - const allocator = init.gpa; - - const start = std.Io.Clock.now(.awake, init.io); - var client: std.http.Client = .{ .allocator = allocator, .io = init.io }; - defer client.deinit(); - try client.initDefaultProxies(allocator, init.environ_map); - - const input = args.positional.model; - - var file_vfs: VFS.File = .init(allocator, init.io, .{ .direct_io = false }); - defer file_vfs.deinit(); - var http_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .http); - defer http_vfs.deinit(); - var https_vfs: VFS.HTTP = try .init(allocator, init.io, &client, .https); - defer https_vfs.deinit(); - var hf_vfs: ?VFS.HF = if (std.mem.startsWith(u8, input, "hf://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (hf_vfs) |*backend| backend.deinit(); - var s3_vfs: ?VFS.S3 = if (std.mem.startsWith(u8, input, "s3://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (s3_vfs) |*backend| backend.deinit(); - var gcs_vfs: ?VFS.GCS = if (std.mem.startsWith(u8, input, "gs://")) - try .auto(allocator, init.io, &client, init.environ_map) - else - null; - defer if (gcs_vfs) |*backend| backend.deinit(); - - var vfs: VFS = try .init(allocator, init.io); - const io = vfs.io(); - defer vfs.deinit(); - try vfs.register("file", file_vfs.io()); - try vfs.register("http", http_vfs.io()); - try vfs.register("https", https_vfs.io()); - if (hf_vfs) |*backend| try vfs.register("hf", backend.io()); - if (s3_vfs) |*backend| try vfs.register("s3", backend.io()); - if (gcs_vfs) |*backend| try vfs.register("gs", backend.io()); - - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - - const repo = try zml.safetensors.resolveModelRepo(io, input); - var registry: zml.safetensors.TensorRegistry = try .fromRepo(allocator, io, repo); - defer registry.deinit(); - const loaded = std.Io.Clock.now(.awake, init.io); - - const index_json_file = if (args.output) |output| - try std.Io.Dir.cwd().createFile(io, output, .{}) - else - try repo.createFile(io, full_index_path, .{}); - const out_buffer = try allocator.alloc(u8, 4096); - defer allocator.free(out_buffer); - var out_writer = index_json_file.writer(io, out_buffer); - try registry.saveAsFullIndexJson(io, allocator, repo, &out_writer.interface); - const done = std.Io.Clock.now(.awake, init.io); - - log.info("Wrote {s}: {d} tensors; load {f}s, write {f}s, total {f}s", .{ - args.output orelse full_index_path, - registry.tensors.count(), - start.durationTo(loaded), - loaded.durationTo(done), - start.durationTo(done), - }); -} - -const full_index_path = "model.safetensors.full_index.json"; diff --git a/tools/workspace_status.sh b/tools/workspace_status.sh deleted file mode 100755 index 05f02bed..00000000 --- a/tools/workspace_status.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -if git diff --quiet; then - echo BUILD_SCM_REVISION "$(git rev-parse --short HEAD)" -else - echo BUILD_SCM_REVISION "dev" -fi - -echo STABLE_ZML_SMI_VERSION "$(git describe --tags --dirty --match 'zml-smi-v*' 2>/dev/null || echo "dev")" diff --git a/tools/xprof/BUILD.bazel b/tools/xprof/BUILD.bazel deleted file mode 100644 index 285f7003..00000000 --- a/tools/xprof/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -load("@rules_python//python/uv:lock.bzl", uv_lock = "lock") -load("@rules_python//python:py_binary.bzl", "py_binary") - -py_binary( - name = "xprof", - srcs = ["main.py"], - main = "main.py", - tags = ["manual"], - visibility = ["//visibility:public"], - deps = ["@xprof//xprof:pkg"], -) - -uv_lock( - name = "requirements", - srcs = ["requirements.in"], - out = "requirements.lock.txt", - args = ["--upgrade"], - tags = ["manual"], -) diff --git a/tools/xprof/main.py b/tools/xprof/main.py deleted file mode 100644 index 79ed3ccf..00000000 --- a/tools/xprof/main.py +++ /dev/null @@ -1,42 +0,0 @@ -import os -import sys -from pathlib import Path - -from xprof import profile_plugin -from xprof.cli.xprof_cli import main as xprof_main - - -def _configure_static_dir() -> None: - static_dir = Path(profile_plugin.__file__).parent / "static" - # Bazel runfiles contain a real directory whose files are symlinks into the - # repository cache. Resolve a file so XProf's traversal check sees the same - # base path as the assets it opens. - resolved_static_dir = (static_dir / "index.html").resolve().parent - os.environ.setdefault("XPROF_STATIC_DIR", str(resolved_static_dir)) - - -def _argv() -> list[str]: - user_args = sys.argv[1:] - if user_args and not user_args[0].startswith("-"): - logdir = user_args[0] - trailing_args = user_args[1:] - else: - logdir = "." - trailing_args = user_args - return [ - sys.argv[0], - "--logdir", - logdir, - "--max_concurrent_worker_requests", - "32", - "--port", - "6006", - "--hide_capture_profile_button", - *trailing_args, - ] - - -if __name__ == "__main__": - _configure_static_dir() - sys.argv = _argv() - raise SystemExit(xprof_main()) diff --git a/tools/xprof/requirements.in b/tools/xprof/requirements.in deleted file mode 100644 index 7a238fea..00000000 --- a/tools/xprof/requirements.in +++ /dev/null @@ -1 +0,0 @@ -xprof==2.23.2 diff --git a/tools/xprof/requirements.lock.txt b/tools/xprof/requirements.lock.txt deleted file mode 100644 index f3ac6542..00000000 --- a/tools/xprof/requirements.lock.txt +++ /dev/null @@ -1,1453 +0,0 @@ -# This file was autogenerated by uv via the following command: -# bazel run //tools/xprof:requirements.update -absl-py==2.5.0 \ - --hash=sha256:0c996f25c0490700fadabe6351630f6111534fa0ae252cc6d2014ea3b141135f \ - --hash=sha256:0f17b89f2a4eaaedc4f28c622998aa690564b3012a396a4ffad0821007fe03ba - # via xprof -aiohappyeyeballs==2.7.1 \ - --hash=sha256:065665c041c42a5938ed220bdcd7230f22527fbec085e1853d2402c8a3615d9d \ - --hash=sha256:9243213661e29250eb41368e5daa826fc017156c3b8a11440826b2e3ed376472 - # via aiohttp -aiohttp==3.14.3 \ - --hash=sha256:03cd2bde3d7f085b64e549c985f4bb928cad7e8ecf5323bfca320db548d81b39 \ - --hash=sha256:041badb8f84396357c4d3ad26de6afd7a32b112f43d3c63045c0c8278cfd2043 \ - --hash=sha256:0a5ff2dfbb9ce645fa5b8ef3e02c6c0b9cc3f6030ff863d0c51fffc50cb5541b \ - --hash=sha256:0fdea2281997af69da84c77ffa6f5938a0285f21fb3887c249d67419ca865b3d \ - --hash=sha256:11fb37ef075669eee52ab1928fbf6e1741fada40409fa309ebde9607a962aebf \ - --hash=sha256:134ac5ddcf61c6fad984b9a5727d83492ada43d63471db20fb73042c13fca62f \ - --hash=sha256:152516815ef926786a0b6ae2b8f1fd2e0c71582dee0b435636865316fd4891b7 \ - --hash=sha256:1576145bdceeb92382d899751e12743a3a5b8e460a841e3e50543859e54864dc \ - --hash=sha256:16100ad3ab8d649fdfbee87602d9d2dcdca9df0b9eda8a1b5fdc0d41f96da559 \ - --hash=sha256:16ea7e24c309fb7c0bbd505d149abe4fe4dccfb8db911db7dbec0921bc889a6f \ - --hash=sha256:18c441d0a8fca6de8d1f546849b9f0ab20d435993e2c5b59562b2fae6be2f929 \ - --hash=sha256:18cb43369747b2ae007bd2655fb8e63a099c2ff1d207962943636dac989b3147 \ - --hash=sha256:1b59533861b70a2185c8f4f350f791f39d64358ef6944ce71c5240c9ec0982c9 \ - --hash=sha256:1c5281acc88b92396f88c7e1e2748f8466689df22b80170e4f51efa712fb47a8 \ - --hash=sha256:1c5ec8fb1bcc31a8466f74aaf26c345d5c386fa4bd08a3f0eb9c7a4a3fe8b5bf \ - --hash=sha256:1caa7b0d05f3e3a36f87788c59e970a7ee1cefcfcbb924a9f138c4a6551c9cb7 \ - --hash=sha256:21c016079415ed3fd676963e9793700a566d85dbbd6bfc564b9b2d209147dcc8 \ - --hash=sha256:2498f0fe69ead802f9675beca44a7c21c62fdaa4ec5145ea1c3ad6edbee29f85 \ - --hash=sha256:25bd2708db6bdf6a6630dd37bdcdfcb47c4434d22ac69c64665b802910140b30 \ - --hash=sha256:270d3dace9ca2f10f0da5d8ebe519b7a310fc6112ed916e32df5866df0888553 \ - --hash=sha256:2e1161602f45a54de2ce0905243a95f58cb42dcd378402f3697f5e0b21e9d2e7 \ - --hash=sha256:2e9878ae68e4a5f1c0abe4dd497dbc3d51946f5837b56759e2a02e78fa90ef86 \ - --hash=sha256:30402d03a7c0ff52bce290b57e564e9079fd9d0cb545c8aba73f86a103162d2e \ - --hash=sha256:33a2d7c28d33797a2e99923dffa63f83d908a19b6bf26cfe80fa790aa5e1a75a \ - --hash=sha256:362a3fd481769cac1a824514bcd86fda51c65e8fe6e051099e008fddde6db17c \ - --hash=sha256:38901a84da3ce22249f6e860bf8f90d141bcab7da090cc398f8bb58c0e44b7da \ - --hash=sha256:39aded8c7f3b935b54aab1d8d73c70ec0ee2d3ec3b943e0e86611bc150ba47f5 \ - --hash=sha256:3a26434dafe408229ff3403458ca58de24fb51936504decac49ce6755f77e59d \ - --hash=sha256:3ae5b3a59436d089b5395d910121a390feed4d00578eb95a0fd1a329fe963100 \ - --hash=sha256:3d4f72af88ac2474bb5bca640030320e3d38a0163a1d7533500e87be458eef71 \ - --hash=sha256:3f42e9b78301f11c8f861746175d8b9c1ccef713fcad9eab396e2f6db8ed4a22 \ - --hash=sha256:42a67efc36300d052fb4508a53e8b6901b9284b599ae63945c377569c5fcc1e1 \ - --hash=sha256:48d67b87db6279c044760787eb01f6413032c2e6f3ba1cafaa492b1c8e578479 \ - --hash=sha256:498c6c623134f8e09a3c4e60bcd607a0b4590dd7dbf08dd40851b27cbb520ccb \ - --hash=sha256:49f7325beb0f85ef4aef5f48f490269575f83e6e2acad00a1d80b807eb027062 \ - --hash=sha256:4e3ac92d90e92773b2362d506068e9a948192bd553e743c5b2429e28527c8661 \ - --hash=sha256:530125ee1163c4219af35dc3aa1206e541e7b31b6efc1a3f93b70a136f65d427 \ - --hash=sha256:5373dc80ad1aa2fb9ad95c83f24eef418bbda3a61375f128e5b0192e4f3f9b32 \ - --hash=sha256:53e5179d8abb5710f8e83ba207c41c8d1261fcffd4616500e15ca2b7a33be10a \ - --hash=sha256:53e7b4ce82b54a8bcc71b3b67a5cbd177ca1d7f592cbc92cd38b7349f73482db \ - --hash=sha256:543906c127fb1d929b95076db19b83fa2d46751006ff1e23b093aa5ac4d8db42 \ - --hash=sha256:54cfcdee2770dac994417cbb0ee1f3eb0e7cb6b30c79bf44f2c02ff79ec5124a \ - --hash=sha256:55bdcc472aafe2de4a253045cc128007a64f1e0264fb675791e132ea5edaa3bd \ - --hash=sha256:56f355e79f71aef2a85c80305cc915f894b170dba76de5fe84f6351939b83c06 \ - --hash=sha256:5895ef58c4620afe02fa16044f023dc4dafec08158f9d08874a46a7dbc0341b8 \ - --hash=sha256:5bcb6ff3fdab1258a192679ff1a05d44f59626430aa05cd1a9d2447423599228 \ - --hash=sha256:5f08ec777f35ee70720233b8b9811d3bb5d728137f30ac91b7457709c3261ac0 \ - --hash=sha256:614c61d478b83953e261d02bb2df750f17227cd33ef8002945bf5aebbde21919 \ - --hash=sha256:617105e2c3018ee38d0c8ce5ee3c84f621a6d8b9f723202aacaff28449ca91ee \ - --hash=sha256:6debfa7312ff9d4c124dc71d72e9a0a4b9e0879e48ba6fcb42bef5c3300289e2 \ - --hash=sha256:7041d52c3a7fa20c9e8c182b534704abb19502c8bdcbde7ab23bfda6f642394f \ - --hash=sha256:70c987b27534f9ae1a723f47ae921571d616da21d3208282bf4c52af5164ac43 \ - --hash=sha256:74ab5b6a9fb13e873e5a90946588baecaf488745e1db1a4a5c433f971f035098 \ - --hash=sha256:78253b573e6ffab5028924fc98bc281aae05445969982a10864bc360dea2016c \ - --hash=sha256:7a75aa63cbf9b21cfaf60dc2657e19df2c2867d91707d653fee171ffeedd1371 \ - --hash=sha256:8800c996b01c2772a783e3e46f3e1abd5823029adca0df54231960de9bfefa5b \ - --hash=sha256:89176250f686cb9853c0fb7ead90e639e915b84a6f43eedc2a4e7ec21f1037f0 \ - --hash=sha256:8a5fd34f7f7410d1730d5c2ba873cacb2eed3fede366feb268a70ba22581ed8f \ - --hash=sha256:8b3b60de05f3dcb6f6a00f818bb2ec781cee4de0645f59ccaf99b1d1823b6100 \ - --hash=sha256:8f2f1c4c032c7cedd7d8da6f54c97b70266c6570c3108d3fdffee7188bb70529 \ - --hash=sha256:9491196535a88924a60afd5b5f434b5b203b6cc616250878dbdb223a8f7844bc \ - --hash=sha256:9aa6e61fdf20105c4144e755bd586008ff450791d67b1c8146fdc15959c4d51c \ - --hash=sha256:9d9edccfe496b476db5f398d97b865e9a6752bcf8aec4eef8390ce20fb64bb41 \ - --hash=sha256:9fc7b5bfec6573f3ae844f457fdde5adeb713f8b8e4a81ad64fc207b49383716 \ - --hash=sha256:a0dc483c00da8b673abbb367eb6f8d8f4bcec30eb58529ea13cb42e7fd2dfa33 \ - --hash=sha256:a3a8296e7ab5c295f53f1041487cb088e1480775aafbf7fe545d93b770a0f96f \ - --hash=sha256:a3e22975f905b89a55a488c2a08f2fdb2186175349e917d48985cc468a3d4c6e \ - --hash=sha256:a4af35c443e0b1a1bd6a8af3f3485d7fda15c142751a00f3ff8090f0b93346fa \ - --hash=sha256:a94dbaae5ae27bd849c93570669bff91e0510f33a80805738e3de72a7be0447b \ - --hash=sha256:ac74facc01463f138b0da5580329cfcc82818dea5656e83ddcd11268fc12ff80 \ - --hash=sha256:ad4c8b7488d745d2ca4838ebd8ae5ba9b56341d30b1da43640e4ce87f9f49646 \ - --hash=sha256:b014a6ed7cf912e787149fdc529166d3ceabac23f26efeea3158c9aba2354e7e \ - --hash=sha256:b20032766aedf6261c7a566585a40867d092ac03a0d81592d5370ef9b054f99b \ - --hash=sha256:b2466434105a4e03113c36ec775cc2ebe6676b62eae326fa670bb607ef788c1c \ - --hash=sha256:b304db572b4368edd8dda8a2274f73156fe15558fca4a917cb8a09fc47af5963 \ - --hash=sha256:ba59d59aba08ac02fc03b0c8983ccd5ee39a199d0552ce9e6d2b4845b34d59ae \ - --hash=sha256:bd52f811e65f6fb634b1047159657c98f52b407f8efec907bcfc09da9a4c0a25 \ - --hash=sha256:bdd0e2834dce1a26c1bbe26464861e16bbe217042cbff619247c11594472518c \ - --hash=sha256:c23ec8ee9d5ab2f5421f9c7fffce208435607af27fd46d4a44e031954352838f \ - --hash=sha256:c39846c3aad97a8530c89d7a3869a8f8e9e3762c6ac0504481e5c80948f7e807 \ - --hash=sha256:c3c200cf9757edd785051dc699c7ecbec22110dbfcb3fefc7a9f9695eda8ea7a \ - --hash=sha256:c7d3a97c678d34fc5b59da671ee9cd630096ddc643e7b5a30d54a2a6f3574d3f \ - --hash=sha256:c8653fd547c93a61aadc612007790f5555cdd18946fa48cf45e26d8ea4ea473d \ - --hash=sha256:cc7cb243a68167172f48c1fd43cee91ec4b1d40cefd190edd43369d1a6bc9c82 \ - --hash=sha256:ccd4893707b3e2a13e39c90d43cf80edf2e4d0457935bcc103bf2346214c3f15 \ - --hash=sha256:cd817772b2fcf2b8c0905795318485f9ec16eae60b29feb7f4c77085311637f0 \ - --hash=sha256:cda5fd5c95ad7a125a2e8464acc78b98b94c475a3780d6aa0aa157c93f470f4d \ - --hash=sha256:cef89a58e628c4efcac3275c2d68083f82426dcdc89c1492a6f654f9f7ea6ab9 \ - --hash=sha256:d1558173930a5a8d3069cee5c92fc91c87c4dbcb099debbb3622053717145a19 \ - --hash=sha256:d6088ec9894113802bddb3c09e974929aed2c7b3a8c456219b8aab4481f1a239 \ - --hash=sha256:d6218d92e450824e9b4881f44e8c09f1853b490f9a64130801024a4793b1b3b0 \ - --hash=sha256:d77640cc618c1d99fc4f8589c0f24a730adfa54eb1e57ef7bf0c8dfb78da898c \ - --hash=sha256:d7d2deec16eeedf55f2c7cf75b521ea3856a5177e123844f8fd0f114ce252cb5 \ - --hash=sha256:db332af25642007330fca8be5c4d194caf2bea7a7fc84415aff3497af5dfee6b \ - --hash=sha256:dd54d0e8717de95939766febac482ac0474d8ac3b048115f9f2b1d23a16e7db4 \ - --hash=sha256:ddcac3c6b382e81f1dd0499199d4136b877beb4cb5ef770bbbfba56c4b8f55d2 \ - --hash=sha256:df82f3787c940c94986b34222d59c9e38843fba85139f36e85255a82ad5355a9 \ - --hash=sha256:dfa68deb2a443bdaa3ea5297b0699c1464f08aef3812b486d1348eee61b07dc0 \ - --hash=sha256:dff9461ec275f22135650d5ba4b4931a11f3958df7dfbb8db630000d4dee0883 \ - --hash=sha256:e1e74298bab6ee0d6e749ed4fd1901c7e604bdda32c03d787a2cc71c46d0433d \ - --hash=sha256:e2667f0bbe7eb6c74eae5e9691441ad186e5845ca3cff63230fc09c4e7514f5d \ - --hash=sha256:e3be98a7c30b8c25d573dafba7171d66dfb05ee6a9070fc46535464ff97700a6 \ - --hash=sha256:e568e14940c09955aa51f4e645b6daa18a581c5dcfcd73744dcc86a856e3ced3 \ - --hash=sha256:e72ee89e28d907a18f46959b4eb0bb06701cc7f8cf4366e00029e2ccfaaf5924 \ - --hash=sha256:e92eb8acc45eb6a9f4935071a77edf5b85cc6f8dfad5cd99e97653c26593cdde \ - --hash=sha256:ea05e1f97ceea523942d9b2a7d7c0359d781d683d6b043f5943a602b14da4787 \ - --hash=sha256:eac645b09bcfdf73df7536331f0678c1086ea250981118ddb5199e17ccef72bb \ - --hash=sha256:eb0495d778817619273c108784292be161a924b9f5ae5cbbc70a2caa6838250b \ - --hash=sha256:ebe8e504f058fe91223351cecd2d9d6946c9d241bb0250d898ffbdf584cc72b0 \ - --hash=sha256:ed099d105449c4f9e84f24af203cd131349d4761d8813fa7e02c32e7128cd910 \ - --hash=sha256:f0f177d1b195b9e06376cfd7d308d8a1b920909a609d03ac82a8c73bbb16d3b9 \ - --hash=sha256:f3d2669fe7dec7fc359ecdb5984b29b50d85d5d00f8c1cb61de4f4a24ee42627 \ - --hash=sha256:f4e05329faa0ea1a404b37de4f034fd2c2defcca06a68dc6745e4e56c88e8a48 \ - --hash=sha256:f53bcd52f585e1ac3e590d61434eb61f9a88c38df041b4ea126d97144344a77b \ - --hash=sha256:f55119f7bf25f49ed210f6096090715da24f2943c62102448915fde3c62877ce \ - --hash=sha256:f631fe87a6f30df5fbe6d79640b25e4cffb38c31c7fb6f10871517b84b0f8c1a \ - --hash=sha256:f8fb78a83c9e5f741ca3a68cfb455c1f5bb83b4e7249a3848b3cd78d0a8563b0 \ - --hash=sha256:fa9467a8113aa69d3d7c55a70ef0b7c636010a40993f3df9d9d0d73b3eb7ef24 \ - --hash=sha256:fd51ebf9d3a00c074df4ede271023f4d2dba289bcc740b88191872716014e3c5 - # via gcsfs -aiosignal==1.4.0 \ - --hash=sha256:053243f8b92b990551949e63930a839ff0cf0b0ebbe0597b0f3fb19e1a0fe82e \ - --hash=sha256:f47eecd9468083c2029cc99945502cb7708b082c232f9aca65da147157b251c7 - # via aiohttp -attrs==26.1.0 \ - --hash=sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309 \ - --hash=sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32 - # via aiohttp -certifi==2026.7.22 \ - --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ - --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 - # via requests -cffi==2.1.1 \ - --hash=sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e \ - --hash=sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66 \ - --hash=sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2 \ - --hash=sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0 \ - --hash=sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6 \ - --hash=sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971 \ - --hash=sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c \ - --hash=sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d \ - --hash=sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9 \ - --hash=sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517 \ - --hash=sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735 \ - --hash=sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80 \ - --hash=sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f \ - --hash=sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1 \ - --hash=sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29 \ - --hash=sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8 \ - --hash=sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c \ - --hash=sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e \ - --hash=sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48 \ - --hash=sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813 \ - --hash=sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac \ - --hash=sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632 \ - --hash=sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6 \ - --hash=sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1 \ - --hash=sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659 \ - --hash=sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688 \ - --hash=sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004 \ - --hash=sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0 \ - --hash=sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062 \ - --hash=sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779 \ - --hash=sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94 \ - --hash=sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50 \ - --hash=sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab \ - --hash=sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac \ - --hash=sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6 \ - --hash=sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676 \ - --hash=sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1 \ - --hash=sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9 \ - --hash=sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf \ - --hash=sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13 \ - --hash=sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e \ - --hash=sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e \ - --hash=sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973 \ - --hash=sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527 \ - --hash=sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72 \ - --hash=sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890 \ - --hash=sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c \ - --hash=sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990 \ - --hash=sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd \ - --hash=sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9 \ - --hash=sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94 \ - --hash=sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3 \ - --hash=sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80 \ - --hash=sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41 \ - --hash=sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5 \ - --hash=sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c \ - --hash=sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a \ - --hash=sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4 \ - --hash=sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e \ - --hash=sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6 \ - --hash=sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98 \ - --hash=sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b \ - --hash=sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1 \ - --hash=sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03 \ - --hash=sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af \ - --hash=sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231 \ - --hash=sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2 \ - --hash=sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3 \ - --hash=sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836 \ - --hash=sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5 \ - --hash=sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399 \ - --hash=sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96 \ - --hash=sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e \ - --hash=sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be \ - --hash=sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf \ - --hash=sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc \ - --hash=sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455 \ - --hash=sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0 \ - --hash=sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12 \ - --hash=sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b \ - --hash=sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7 \ - --hash=sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692 \ - --hash=sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54 \ - --hash=sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3 \ - --hash=sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b \ - --hash=sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be \ - --hash=sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d \ - --hash=sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358 \ - --hash=sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a \ - --hash=sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7 \ - --hash=sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc \ - --hash=sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960 \ - --hash=sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125 \ - --hash=sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb \ - --hash=sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a \ - --hash=sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa \ - --hash=sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf \ - --hash=sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3 \ - --hash=sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4 \ - --hash=sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264 - # via cryptography -charset-normalizer==3.5.1 \ - --hash=sha256:00668ebb0609751758682eb0b5857e7c35b9f00e84dfdef062e103244ec94d45 \ - --hash=sha256:012a22b88a77ca2e59b98ac5889b0deb604147666032f45e6d6e217634d2550d \ - --hash=sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5 \ - --hash=sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b \ - --hash=sha256:0722590aabf9dc6a6c0343d523c05458fa2b5047dbe6302fd526bb570600753f \ - --hash=sha256:07ffd07412fc5d5e84cd8952acf9ff7e4ed7a708e69d1bada19d8ba91711353f \ - --hash=sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5 \ - --hash=sha256:0b2b1b3fa5670c127b246df1d0c059defd41f689a868a3b9d79df9b1cac42d22 \ - --hash=sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5 \ - --hash=sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac \ - --hash=sha256:13e3afe97712e8887cd516e960c63f0b93122971e5b5e4b2622fe7701771e838 \ - --hash=sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90 \ - --hash=sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626 \ - --hash=sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4 \ - --hash=sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369 \ - --hash=sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b \ - --hash=sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e \ - --hash=sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee \ - --hash=sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1 \ - --hash=sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102 \ - --hash=sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8 \ - --hash=sha256:29880d17a8eb0b5cfdfd8944b468322928059aa35f1f5fa8ff22b149ec0b42f8 \ - --hash=sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9 \ - --hash=sha256:2e9cf9253119d8e5d111f05d71626786fd3d6193817316eab1ca088cdb8593cf \ - --hash=sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0 \ - --hash=sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031 \ - --hash=sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e \ - --hash=sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235 \ - --hash=sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072 \ - --hash=sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb \ - --hash=sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c \ - --hash=sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950 \ - --hash=sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2 \ - --hash=sha256:366ec70f5547c640d3ce1985722490f23faf4eb5216a7eeba78277490e78dacb \ - --hash=sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e \ - --hash=sha256:3d27167433c0d5f18dc850f07d0b3816221984fecdc405d6c157a6f0b8f8e9e6 \ - --hash=sha256:3e5e1224c0a6a90e05843e07adfec669edebec17801c67072f51e59561d63c0b \ - --hash=sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2 \ - --hash=sha256:433c5a81eade63b47e522303bad236f59dba55ea6951746f5558355eeed8c75d \ - --hash=sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa \ - --hash=sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2 \ - --hash=sha256:494b70049a4d69aec6e8137c13af4cf8db8c9f9820a1392ac293b0dd2987a818 \ - --hash=sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032 \ - --hash=sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71 \ - --hash=sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96 \ - --hash=sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687 \ - --hash=sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8 \ - --hash=sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3 \ - --hash=sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61 \ - --hash=sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9 \ - --hash=sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1 \ - --hash=sha256:55261ac0d2941c42f196dd576f543d87a8ee03cd6f5e30dfb4d807b2e3b9121a \ - --hash=sha256:56490c595a28b1bb27dfc583e816152a9767721ef58b2c03b13f954d2f707420 \ - --hash=sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4 \ - --hash=sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65 \ - --hash=sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663 \ - --hash=sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f \ - --hash=sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591 \ - --hash=sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a \ - --hash=sha256:5ca0555312ae2fe82715cada7fac375530c2f3349e1eaa1bcb33d0283ac79a18 \ - --hash=sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e \ - --hash=sha256:5e2d0e146dcb57034f8b97dc58d2d512cb90aba253960ce449f695fec6a82c6f \ - --hash=sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7 \ - --hash=sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3 \ - --hash=sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c \ - --hash=sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3 \ - --hash=sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7 \ - --hash=sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96 \ - --hash=sha256:6ba32c4d2abf1d2fe7cf27d280f4cca5664233b0f885549c7761719eb977f486 \ - --hash=sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3 \ - --hash=sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6 \ - --hash=sha256:6e2912d4babbc65196ac13c2f53468dc57fb8b9c25ef913e8c59ddf7c6dc0e1b \ - --hash=sha256:6e5e4d73d588ca5ed09df1b7dcd1b203d1df3c542e3f50d126c947d432b10731 \ - --hash=sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959 \ - --hash=sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9 \ - --hash=sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf \ - --hash=sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8 \ - --hash=sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e \ - --hash=sha256:789b8982559ae28dad2356519f841655756cdcd96616410590ae0b17454ee64f \ - --hash=sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885 \ - --hash=sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0 \ - --hash=sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506 \ - --hash=sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2 \ - --hash=sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0 \ - --hash=sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e \ - --hash=sha256:85de3134b5379856e323ba37c19c9256d39425f7b76a63af52b09fb4664c2e8f \ - --hash=sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e \ - --hash=sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491 \ - --hash=sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a \ - --hash=sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20 \ - --hash=sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449 \ - --hash=sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af \ - --hash=sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c \ - --hash=sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712 \ - --hash=sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7 \ - --hash=sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a \ - --hash=sha256:94fbf1c0c6cc0d3d5e50f9a9313a8cdca90dd696d34b381cd1704f8c9e939f20 \ - --hash=sha256:950f23cb393f85543777b0433f082cddd25b51ab398eac7971146495679efe5f \ - --hash=sha256:96eefc178f8636b9c760c5829345307fd81cfae9ab1e80997dbddeb0f54ee9a3 \ - --hash=sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9 \ - --hash=sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e \ - --hash=sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5 \ - --hash=sha256:994e883d17c559cdfd38c84003c8b27d25424a1077272a17e7cd27bfe0bf57b2 \ - --hash=sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36 \ - --hash=sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263 \ - --hash=sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4 \ - --hash=sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11 \ - --hash=sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a \ - --hash=sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3 \ - --hash=sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375 \ - --hash=sha256:a545775cfe815855ea32d7c27731d79da358ef2055b4a25830231b1622dd18aa \ - --hash=sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d \ - --hash=sha256:a6d095662e73e74f0a49988e0593373e243e3a52e27bfeea0a859e88acf4a0f5 \ - --hash=sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99 \ - --hash=sha256:a951ad59cad9145664a730d3036b40b844e74d2d3683da40111463cd3a83845d \ - --hash=sha256:aa1099b956fb795e686d073568f6dc002a0bb89765ea6d5b055dd7d9bf1b116c \ - --hash=sha256:aa2bb0b37202dca27175591f761108b5d34096ade1191ffe4808bdf6b1571488 \ - --hash=sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6 \ - --hash=sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc \ - --hash=sha256:ac00177c4831ffa650f8609e4bdddd5fe09c03b1c0c47acece7e6ea20421598b \ - --hash=sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f \ - --hash=sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00 \ - --hash=sha256:ae31a1a1db2ee6cc2942fccaf695c934bc7f3db9f2133a3fef1f367cf1a4ab10 \ - --hash=sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598 \ - --hash=sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6 \ - --hash=sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962 \ - --hash=sha256:b54e7e13267d49ffbfe68e25b3cbd774dab38fa37238f71265e91b36146eb21c \ - --hash=sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08 \ - --hash=sha256:ba2f37ee79e6338845261a3c5b1784e5d1acdff2c0785b284f1b633033d136ab \ - --hash=sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573 \ - --hash=sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90 \ - --hash=sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5 \ - --hash=sha256:bd6c173f04743d483881bffa1478d5a4624475b8cd1d2194956a75548e191c18 \ - --hash=sha256:be47f99644b208bff7766314013f9acf57b056b04191d570d68ad14022cf5b1d \ - --hash=sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af \ - --hash=sha256:c1dcc36dcb96abc02236e182d17e0f71430152a6c2c7447421da2d2dc144edea \ - --hash=sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c \ - --hash=sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b \ - --hash=sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6 \ - --hash=sha256:c7b742bf31c88566b4bb6335a7f393bb322e580b6bb98df7bd0c25e6e3519ce8 \ - --hash=sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774 \ - --hash=sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004 \ - --hash=sha256:ce854f5f478050ade5a238731c4ca985a7d3b3cb53ff600a9b5c3b689b5f0a7a \ - --hash=sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a \ - --hash=sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2 \ - --hash=sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2 \ - --hash=sha256:d1ee1e296209fdce05b81b663250eefa02213a2da7b41bf26f7829b8ba3545aa \ - --hash=sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe \ - --hash=sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3 \ - --hash=sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc \ - --hash=sha256:e06efa066f7dbadbc84ebc126a97c452a6451dfcf589d89d788484949e1cf795 \ - --hash=sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d \ - --hash=sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc \ - --hash=sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893 \ - --hash=sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef \ - --hash=sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d \ - --hash=sha256:e9fbdce1e47394b09bc9f26ab117dfc8d6491977a11d86f592bb42c779db2fda \ - --hash=sha256:eb12fb2ba69ffa05f8695f61c69e591dc4b4a12ac3757ac8af8adb259bf56d17 \ - --hash=sha256:eda059b6bc8bc0812d626fd91a7ce01bf583df0a61296eff390fd94141a34e30 \ - --hash=sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7 \ - --hash=sha256:f298e218441525d3794428b4c8b8fb8662c6d3ea79925d4807ee6b9a96a3bca5 \ - --hash=sha256:f5542f9b941279d82d41eb0aa9f98eba36fe4df5c7086c651df7944935b37182 \ - --hash=sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f \ - --hash=sha256:f9b1e28d0e8dbfa858abdba91d6b547beaf2df1a59bec6da6faae7b96a4991a9 \ - --hash=sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada \ - --hash=sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876 \ - --hash=sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a \ - --hash=sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348 \ - --hash=sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3 \ - --hash=sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f \ - --hash=sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0 \ - --hash=sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f - # via requests -cheroot==11.1.2 \ - --hash=sha256:0f6c0ba05c00fbc869fb46b1de4ec2384e1d85418ae963d3bc10ae83b688dbfa \ - --hash=sha256:bfb70c49663f63b0440f2b54dbc6b0d1650e56dfe4e2641f59b2c6f727b44aca - # via xprof -cryptography==50.0.1 \ - --hash=sha256:01f41478cf33fc605a6a089cd56d28b45c6c0b45a1928b61797f2621a04bac71 \ - --hash=sha256:05ba322c4da95b262a212c345af888ef2c37c88c0509756ea00a0e6d68850f23 \ - --hash=sha256:16c5ecd954b3330ebfb6605eca4fd952da8bef376551d5cc264534e3770a9ee6 \ - --hash=sha256:2a93d05e34d5f67fba6f891fe85d929999baa7195e853923ea6d7576c9e68c5e \ - --hash=sha256:2b34d76a652ea2b6faf777c35df230c5637842cd904e04f16230c3f9f03e4361 \ - --hash=sha256:2ebbfb0f1fed745e91796e3e1080a1440423fdae8ece1b995a1d80883a409054 \ - --hash=sha256:30a125032e5642a21ff816e021152bd4e7e94f03eff3f4b7fca41cd22bc3110f \ - --hash=sha256:330fbb252391c596f1ae42c5754449dc924e6ad012dca8efe0d703f9f2d12ec6 \ - --hash=sha256:359e62deae718bce96170e223fdcb6357e4fbd3bb7a3a75f4430763532560e49 \ - --hash=sha256:407fe2b6db00939c05c0e945e9914238f2f0a430974839429dafc82b1ee6bee5 \ - --hash=sha256:42be3bb70596b3abe4ac097b75be223e8b3ab614a0e5de068e3dcc54d71d6149 \ - --hash=sha256:4c4188f7c0cf655be5c06342b817ed0f9595b69ffa2b12026e5353eed29dea88 \ - --hash=sha256:51593d180cf6d179bde5c5d065bed81386b1f381656ae7d042b7ffc87a9895ad \ - --hash=sha256:51afcfceb15597cf2635068e4ac9a56b2abde622edde17f37d85fd7b5306497a \ - --hash=sha256:53e279950892dc102c6b4e52af03ae5ea92fac572a1ddab78ca73a997f62b69f \ - --hash=sha256:55d16b1ef3ee0958d893a977b19777887e546c9954ea81b200c3301a864013f2 \ - --hash=sha256:5dd9bda1c12b4162f6ff568eeb5e0ff956c28d14406e875cfe8a63a2d414ff20 \ - --hash=sha256:5fe002589592ed749ce77fe0695fcbd3500dd61d7d6db5858a7544c612fa8e45 \ - --hash=sha256:5fe939deeb161024a6be98229c953b6591fef1f41214497a78fe793a244c017f \ - --hash=sha256:693c99b49bd37d0d096e4334c10232c77248c415b98d35236094cdf96d57258b \ - --hash=sha256:76de83fbd91ac49c0feaaa983d0748fd7a53176afac5fb3bf7478d244f0eb527 \ - --hash=sha256:79bf008d1f9af6071c797ad133e39915dfee7614f18f18f4db9072eb715064a3 \ - --hash=sha256:804728ce710890870f3aaa344b2e161172d258d768ac139d02cfd9092d0d94e6 \ - --hash=sha256:8921d58f426793c5f1b47f0b59575780de9a095214958d0eb37d909593db8367 \ - --hash=sha256:8df2de9102026855887e4587084f6eabd80ed0f345b8ad8a7ac27ab9bf4723e0 \ - --hash=sha256:9cb3cb952cf5a8abd50c782a98a89d71699715e802fe349704b47f2425b42a94 \ - --hash=sha256:9dde0a357190eb3b1da1bb9ab750e9c85cba82ca5977aa0836cbb94e92611239 \ - --hash=sha256:9ebcdd5519be9b652a46f507817a74591774fc3d6923ac364e4dfa64e36b291b \ - --hash=sha256:a0b1a59e3a089064a0ec309e9428c8e3ae4e161419d20ac33600767e83fc658a \ - --hash=sha256:a255449073358275b64b67d3f595f268bbef70e72b6edb65e0c70c735bf739c9 \ - --hash=sha256:a8f40ea47330e71b594a7e246898f93177c259490c63183dbaf9e571d71ed9a5 \ - --hash=sha256:ac02b07824d4d1001bd4367599f839c19cb171924c796e52c23508ac14c2c0cc \ - --hash=sha256:aed8db4f6d71c51efb89530e12d9464e7bf2923d46c3205dc794a2a93f8c0648 \ - --hash=sha256:b8f852c65863251b9e3a1b8c150ce21e59b522dbb6a7d4bc80e680d38388e986 \ - --hash=sha256:be224a65493ec5b74a158ff22a5522ce4a5ca1e543c647a3a4730d4a09e5f959 \ - --hash=sha256:ca83d00d9e69cd5eb63f2e69c3a5a59e0cecae5ae14c6ae0b35830fe3b37bad0 \ - --hash=sha256:cbf74a81765ee67413503ca6e26dcc4f6f5a519822436cc0a1b97aab6c1b8a17 \ - --hash=sha256:d63ae8f6481fec907ac0f588eee8a90aefde112c633131fe540e5711ddbb5a4e \ - --hash=sha256:e22dfed744bd4002e909464cb23d2f0b05c6f3113a79ef2e9864a53db737c733 \ - --hash=sha256:e2ca8fd1b6b4b82a1c4cb02841d0837e3c12336c2e24b520ab8ab3b969733d8f \ - --hash=sha256:e74591e283fe6eb956416c929eb58262a719fe0311fd9054c62c3350ed8760d8 \ - --hash=sha256:f74455bb086a85d5e81246412602aaa97ed095e504cd40dd261ef50be42205bf \ - --hash=sha256:fb4b9672d389c738b175c4166e78310f8a70358886aacd9173ee03a85ffdc671 \ - --hash=sha256:fc3ed7ebd2a8c96f5b166de0ab9b624996bef3b07bbeb19364dfb78222c22c80 \ - --hash=sha256:fd3718b960d0b5dd213cdf03f3bcb7000e69dda0de8b956061947ff6bcff5558 \ - --hash=sha256:ff838d62ec1bfce4f9ba7fa16f4a7b554cd8d0c299e6be37502161a660c84eef - # via google-auth -decorator==5.3.1 \ - --hash=sha256:4cbcdd55a6efadb9dbea26b858f4fb3264567b52d69ca0d25b721b553f60ea82 \ - --hash=sha256:f47fe6fdbd2edd623ecfe36875d37aba411624e2670dd395dddae1358689bb3c - # via gcsfs -etils[epath]==1.14.0 \ - --hash=sha256:8136e7f4c4173cd0af0ca5481c4475152f0b8686192951eefa60ee8711e1ede4 \ - --hash=sha256:b5df7341f54dbe1405a4450b2741207b4a8c279780402b45f87202b94dfc52b4 - # via xprof -fire==0.7.1 \ - --hash=sha256:3b208f05c736de98fb343310d090dcc4d8c78b2a89ea4f32b837c586270a9cbf \ - --hash=sha256:e43fd8a5033a9001e7e2973bab96070694b9f12f2e0ecf96d4683971b5ab1882 - # via xprof -frozenlist==1.8.0 \ - --hash=sha256:0325024fe97f94c41c08872db482cf8ac4800d80e79222c6b0b7b162d5b13686 \ - --hash=sha256:032efa2674356903cd0261c4317a561a6850f3ac864a63fc1583147fb05a79b0 \ - --hash=sha256:03ae967b4e297f58f8c774c7eabcce57fe3c2434817d4385c50661845a058121 \ - --hash=sha256:06be8f67f39c8b1dc671f5d83aaefd3358ae5cdcf8314552c57e7ed3e6475bdd \ - --hash=sha256:073f8bf8becba60aa931eb3bc420b217bb7d5b8f4750e6f8b3be7f3da85d38b7 \ - --hash=sha256:07cdca25a91a4386d2e76ad992916a85038a9b97561bf7a3fd12d5d9ce31870c \ - --hash=sha256:09474e9831bc2b2199fad6da3c14c7b0fbdd377cce9d3d77131be28906cb7d84 \ - --hash=sha256:0c18a16eab41e82c295618a77502e17b195883241c563b00f0aa5106fc4eaa0d \ - --hash=sha256:0f96534f8bfebc1a394209427d0f8a63d343c9779cda6fc25e8e121b5fd8555b \ - --hash=sha256:102e6314ca4da683dca92e3b1355490fed5f313b768500084fbe6371fddfdb79 \ - --hash=sha256:11847b53d722050808926e785df837353bd4d75f1d494377e59b23594d834967 \ - --hash=sha256:119fb2a1bd47307e899c2fac7f28e85b9a543864df47aa7ec9d3c1b4545f096f \ - --hash=sha256:13d23a45c4cebade99340c4165bd90eeb4a56c6d8a9d8aa49568cac19a6d0dc4 \ - --hash=sha256:154e55ec0655291b5dd1b8731c637ecdb50975a2ae70c606d100750a540082f7 \ - --hash=sha256:168c0969a329b416119507ba30b9ea13688fafffac1b7822802537569a1cb0ef \ - --hash=sha256:17c883ab0ab67200b5f964d2b9ed6b00971917d5d8a92df149dc2c9779208ee9 \ - --hash=sha256:1a7607e17ad33361677adcd1443edf6f5da0ce5e5377b798fba20fae194825f3 \ - --hash=sha256:1a7fa382a4a223773ed64242dbe1c9c326ec09457e6b8428efb4118c685c3dfd \ - --hash=sha256:1aa77cb5697069af47472e39612976ed05343ff2e84a3dcf15437b232cbfd087 \ - --hash=sha256:1b9290cf81e95e93fdf90548ce9d3c1211cf574b8e3f4b3b7cb0537cf2227068 \ - --hash=sha256:20e63c9493d33ee48536600d1a5c95eefc870cd71e7ab037763d1fbb89cc51e7 \ - --hash=sha256:21900c48ae04d13d416f0e1e0c4d81f7931f73a9dfa0b7a8746fb2fe7dd970ed \ - --hash=sha256:229bf37d2e4acdaf808fd3f06e854a4a7a3661e871b10dc1f8f1896a3b05f18b \ - --hash=sha256:2552f44204b744fba866e573be4c1f9048d6a324dfe14475103fd51613eb1d1f \ - --hash=sha256:27c6e8077956cf73eadd514be8fb04d77fc946a7fe9f7fe167648b0b9085cc25 \ - --hash=sha256:28bd570e8e189d7f7b001966435f9dac6718324b5be2990ac496cf1ea9ddb7fe \ - --hash=sha256:294e487f9ec720bd8ffcebc99d575f7eff3568a08a253d1ee1a0378754b74143 \ - --hash=sha256:29548f9b5b5e3460ce7378144c3010363d8035cea44bc0bf02d57f5a685e084e \ - --hash=sha256:2c5dcbbc55383e5883246d11fd179782a9d07a986c40f49abe89ddf865913930 \ - --hash=sha256:2dc43a022e555de94c3b68a4ef0b11c4f747d12c024a520c7101709a2144fb37 \ - --hash=sha256:2f05983daecab868a31e1da44462873306d3cbfd76d1f0b5b69c473d21dbb128 \ - --hash=sha256:33139dc858c580ea50e7e60a1b0ea003efa1fd42e6ec7fdbad78fff65fad2fd2 \ - --hash=sha256:332db6b2563333c5671fecacd085141b5800cb866be16d5e3eb15a2086476675 \ - --hash=sha256:33f48f51a446114bc5d251fb2954ab0164d5be02ad3382abcbfe07e2531d650f \ - --hash=sha256:34187385b08f866104f0c0617404c8eb08165ab1272e884abc89c112e9c00746 \ - --hash=sha256:342c97bf697ac5480c0a7ec73cd700ecfa5a8a40ac923bd035484616efecc2df \ - --hash=sha256:3462dd9475af2025c31cc61be6652dfa25cbfb56cbbf52f4ccfe029f38decaf8 \ - --hash=sha256:39ecbc32f1390387d2aa4f5a995e465e9e2f79ba3adcac92d68e3e0afae6657c \ - --hash=sha256:3e0761f4d1a44f1d1a47996511752cf3dcec5bbdd9cc2b4fe595caf97754b7a0 \ - --hash=sha256:3ede829ed8d842f6cd48fc7081d7a41001a56f1f38603f9d49bf3020d59a31ad \ - --hash=sha256:3ef2d026f16a2b1866e1d86fc4e1291e1ed8a387b2c333809419a2f8b3a77b82 \ - --hash=sha256:405e8fe955c2280ce66428b3ca55e12b3c4e9c336fb2103a4937e891c69a4a29 \ - --hash=sha256:42145cd2748ca39f32801dad54aeea10039da6f86e303659db90db1c4b614c8c \ - --hash=sha256:4314debad13beb564b708b4a496020e5306c7333fa9a3ab90374169a20ffab30 \ - --hash=sha256:433403ae80709741ce34038da08511d4a77062aa924baf411ef73d1146e74faf \ - --hash=sha256:44389d135b3ff43ba8cc89ff7f51f5a0bb6b63d829c8300f79a2fe4fe61bcc62 \ - --hash=sha256:48e6d3f4ec5c7273dfe83ff27c91083c6c9065af655dc2684d2c200c94308bb5 \ - --hash=sha256:494a5952b1c597ba44e0e78113a7266e656b9794eec897b19ead706bd7074383 \ - --hash=sha256:4970ece02dbc8c3a92fcc5228e36a3e933a01a999f7094ff7c23fbd2beeaa67c \ - --hash=sha256:4e0c11f2cc6717e0a741f84a527c52616140741cd812a50422f83dc31749fb52 \ - --hash=sha256:50066c3997d0091c411a66e710f4e11752251e6d2d73d70d8d5d4c76442a199d \ - --hash=sha256:517279f58009d0b1f2e7c1b130b377a349405da3f7621ed6bfae50b10adf20c1 \ - --hash=sha256:54b2077180eb7f83dd52c40b2750d0a9f175e06a42e3213ce047219de902717a \ - --hash=sha256:5500ef82073f599ac84d888e3a8c1f77ac831183244bfd7f11eaa0289fb30714 \ - --hash=sha256:581ef5194c48035a7de2aefc72ac6539823bb71508189e5de01d60c9dcd5fa65 \ - --hash=sha256:59a6a5876ca59d1b63af8cd5e7ffffb024c3dc1e9cf9301b21a2e76286505c95 \ - --hash=sha256:5a3a935c3a4e89c733303a2d5a7c257ea44af3a56c8202df486b7f5de40f37e1 \ - --hash=sha256:5c1c8e78426e59b3f8005e9b19f6ff46e5845895adbde20ece9218319eca6506 \ - --hash=sha256:5d63a068f978fc69421fb0e6eb91a9603187527c86b7cd3f534a5b77a592b888 \ - --hash=sha256:667c3777ca571e5dbeb76f331562ff98b957431df140b54c85fd4d52eea8d8f6 \ - --hash=sha256:6da155091429aeba16851ecb10a9104a108bcd32f6c1642867eadaee401c1c41 \ - --hash=sha256:6dc4126390929823e2d2d9dc79ab4046ed74680360fc5f38b585c12c66cdf459 \ - --hash=sha256:7398c222d1d405e796970320036b1b563892b65809d9e5261487bb2c7f7b5c6a \ - --hash=sha256:74c51543498289c0c43656701be6b077f4b265868fa7f8a8859c197006efb608 \ - --hash=sha256:776f352e8329135506a1d6bf16ac3f87bc25b28e765949282dcc627af36123aa \ - --hash=sha256:778a11b15673f6f1df23d9586f83c4846c471a8af693a22e066508b77d201ec8 \ - --hash=sha256:78f7b9e5d6f2fdb88cdde9440dc147259b62b9d3b019924def9f6478be254ac1 \ - --hash=sha256:799345ab092bee59f01a915620b5d014698547afd011e691a208637312db9186 \ - --hash=sha256:7bf6cdf8e07c8151fba6fe85735441240ec7f619f935a5205953d58009aef8c6 \ - --hash=sha256:8009897cdef112072f93a0efdce29cd819e717fd2f649ee3016efd3cd885a7ed \ - --hash=sha256:80f85f0a7cc86e7a54c46d99c9e1318ff01f4687c172ede30fd52d19d1da1c8e \ - --hash=sha256:8585e3bb2cdea02fc88ffa245069c36555557ad3609e83be0ec71f54fd4abb52 \ - --hash=sha256:878be833caa6a3821caf85eb39c5ba92d28e85df26d57afb06b35b2efd937231 \ - --hash=sha256:8a76ea0f0b9dfa06f254ee06053d93a600865b3274358ca48a352ce4f0798450 \ - --hash=sha256:8b7b94a067d1c504ee0b16def57ad5738701e4ba10cec90529f13fa03c833496 \ - --hash=sha256:8d92f1a84bb12d9e56f818b3a746f3efba93c1b63c8387a73dde655e1e42282a \ - --hash=sha256:908bd3f6439f2fef9e85031b59fd4f1297af54415fb60e4254a95f75b3cab3f3 \ - --hash=sha256:92db2bf818d5cc8d9c1f1fc56b897662e24ea5adb36ad1f1d82875bd64e03c24 \ - --hash=sha256:940d4a017dbfed9daf46a3b086e1d2167e7012ee297fef9e1c545c4d022f5178 \ - --hash=sha256:957e7c38f250991e48a9a73e6423db1bb9dd14e722a10f6b8bb8e16a0f55f695 \ - --hash=sha256:96153e77a591c8adc2ee805756c61f59fef4cf4073a9275ee86fe8cba41241f7 \ - --hash=sha256:96f423a119f4777a4a056b66ce11527366a8bb92f54e541ade21f2374433f6d4 \ - --hash=sha256:97260ff46b207a82a7567b581ab4190bd4dfa09f4db8a8b49d1a958f6aa4940e \ - --hash=sha256:974b28cf63cc99dfb2188d8d222bc6843656188164848c4f679e63dae4b0708e \ - --hash=sha256:9ff15928d62a0b80bb875655c39bf517938c7d589554cbd2669be42d97c2cb61 \ - --hash=sha256:a6483e309ca809f1efd154b4d37dc6d9f61037d6c6a81c2dc7a15cb22c8c5dca \ - --hash=sha256:a88f062f072d1589b7b46e951698950e7da00442fc1cacbe17e19e025dc327ad \ - --hash=sha256:ac913f8403b36a2c8610bbfd25b8013488533e71e62b4b4adce9c86c8cea905b \ - --hash=sha256:adbeebaebae3526afc3c96fad434367cafbfd1b25d72369a9e5858453b1bb71a \ - --hash=sha256:b2a095d45c5d46e5e79ba1e5b9cb787f541a8dee0433836cea4b96a2c439dcd8 \ - --hash=sha256:b3210649ee28062ea6099cfda39e147fa1bc039583c8ee4481cb7811e2448c51 \ - --hash=sha256:b37f6d31b3dcea7deb5e9696e529a6aa4a898adc33db82da12e4c60a7c4d2011 \ - --hash=sha256:b4dec9482a65c54a5044486847b8a66bf10c9cb4926d42927ec4e8fd5db7fed8 \ - --hash=sha256:b4f3b365f31c6cd4af24545ca0a244a53688cad8834e32f56831c4923b50a103 \ - --hash=sha256:b6db2185db9be0a04fecf2f241c70b63b1a242e2805be291855078f2b404dd6b \ - --hash=sha256:b9be22a69a014bc47e78072d0ecae716f5eb56c15238acca0f43d6eb8e4a5bda \ - --hash=sha256:bac9c42ba2ac65ddc115d930c78d24ab8d4f465fd3fc473cdedfccadb9429806 \ - --hash=sha256:bf0a7e10b077bf5fb9380ad3ae8ce20ef919a6ad93b4552896419ac7e1d8e042 \ - --hash=sha256:c23c3ff005322a6e16f71bf8692fcf4d5a304aaafe1e262c98c6d4adc7be863e \ - --hash=sha256:c4c800524c9cd9bac5166cd6f55285957fcfc907db323e193f2afcd4d9abd69b \ - --hash=sha256:c7366fe1418a6133d5aa824ee53d406550110984de7637d65a178010f759c6ef \ - --hash=sha256:c8d1634419f39ea6f5c427ea2f90ca85126b54b50837f31497f3bf38266e853d \ - --hash=sha256:c9a63152fe95756b85f31186bddf42e4c02c6321207fd6601a1c89ebac4fe567 \ - --hash=sha256:cb89a7f2de3602cfed448095bab3f178399646ab7c61454315089787df07733a \ - --hash=sha256:cba69cb73723c3f329622e34bdbf5ce1f80c21c290ff04256cff1cd3c2036ed2 \ - --hash=sha256:cee686f1f4cadeb2136007ddedd0aaf928ab95216e7691c63e50a8ec066336d0 \ - --hash=sha256:cf253e0e1c3ceb4aaff6df637ce033ff6535fb8c70a764a8f46aafd3d6ab798e \ - --hash=sha256:d1eaff1d00c7751b7c6662e9c5ba6eb2c17a2306ba5e2a37f24ddf3cc953402b \ - --hash=sha256:d3bb933317c52d7ea5004a1c442eef86f426886fba134ef8cf4226ea6ee1821d \ - --hash=sha256:d4d3214a0f8394edfa3e303136d0575eece0745ff2b47bd2cb2e66dd92d4351a \ - --hash=sha256:d6a5df73acd3399d893dafc71663ad22534b5aa4f94e8a2fabfe856c3c1b6a52 \ - --hash=sha256:d8b7138e5cd0647e4523d6685b0eac5d4be9a184ae9634492f25c6eb38c12a47 \ - --hash=sha256:db1e72ede2d0d7ccb213f218df6a078a9c09a7de257c2fe8fcef16d5925230b1 \ - --hash=sha256:e25ac20a2ef37e91c1b39938b591457666a0fa835c7783c3a8f33ea42870db94 \ - --hash=sha256:e2de870d16a7a53901e41b64ffdf26f2fbb8917b3e6ebf398098d72c5b20bd7f \ - --hash=sha256:e4a3408834f65da56c83528fb52ce7911484f0d1eaf7b761fc66001db1646eff \ - --hash=sha256:eaa352d7047a31d87dafcacbabe89df0aa506abb5b1b85a2fb91bc3faa02d822 \ - --hash=sha256:eab8145831a0d56ec9c4139b6c3e594c7a83c2c8be25d5bcf2d86136a532287a \ - --hash=sha256:ec3cc8c5d4084591b4237c0a272cc4f50a5b03396a47d9caaf76f5d7b38a4f11 \ - --hash=sha256:edee74874ce20a373d62dc28b0b18b93f645633c2943fd90ee9d898550770581 \ - --hash=sha256:eefdba20de0d938cec6a89bd4d70f346a03108a19b9df4248d3cf0d88f1b0f51 \ - --hash=sha256:ef2b7b394f208233e471abc541cc6991f907ffd47dc72584acee3147899d6565 \ - --hash=sha256:f21f00a91358803399890ab167098c131ec2ddd5f8f5fd5fe9c9f2c6fcd91e40 \ - --hash=sha256:f4be2e3d8bc8aabd566f8d5b8ba7ecc09249d74ba3c9ed52e54dc23a293f0b92 \ - --hash=sha256:f57fb59d9f385710aa7060e89410aeb5058b99e62f4d16b08b91986b9a2140c2 \ - --hash=sha256:f6292f1de555ffcc675941d65fffffb0a5bcd992905015f85d0592201793e0e5 \ - --hash=sha256:f833670942247a14eafbb675458b4e61c82e002a148f49e68257b79296e865c4 \ - --hash=sha256:fa47e444b8ba08fffd1c18e8cdb9a75db1b6a27f17507522834ad13ed5922b93 \ - --hash=sha256:fb30f9626572a76dfe4293c7194a09fb1fe93ba94c7d4f720dfae3b646b45027 \ - --hash=sha256:fe3c58d2f5db5fbd18c2987cba06d51b0529f52bc3a6cdc33d3f4eab725104bd - # via - # aiohttp - # aiosignal -fsspec[gcs]==2026.7.0 \ - --hash=sha256:b57ddbafedfaef7018c1ecab32aa200a9d7ca26b77965f64e48b70061249d279 \ - --hash=sha256:c803c40f4cf860b49dea58ee3e1c33cb9c790520e233537e1340049f89b82a88 - # via - # etils - # gcsfs - # xprof -gcsfs==2026.8.1 \ - --hash=sha256:66e6316690b17e1887a4f4bc0902eee53f5b3fd860cc3f0377d65c5be3313b1f \ - --hash=sha256:738da7dcc74817c7668d032f932598482a23cce1b061c537e54278be44d2fff8 - # via fsspec -google-api-core[grpc]==2.37.0 \ - --hash=sha256:cf58f220aa797f1ffdda52194c4c7d72efeced09297c2976529f8135f6d85b9a \ - --hash=sha256:d84042a0034cce9c4304e17d2e5e9966c2fcb1a851b73799291f687a67470c8b - # via - # google-cloud-core - # google-cloud-storage - # google-cloud-storage-control -google-auth==2.58.0 \ - --hash=sha256:55e30cf15e737de92c5323d78cda8a83fcd57e7ffbaf900c4600039fd60a80fd \ - --hash=sha256:8a9c4645bb4c8e91668fb1934b95ae6a8687084232753639220ba9bf04a1610d - # via - # gcsfs - # google-api-core - # google-auth-oauthlib - # google-cloud-core - # google-cloud-storage - # google-cloud-storage-control -google-auth-oauthlib==1.4.1 \ - --hash=sha256:1a83f5f2a8421dedadaa3caf25b3a710dddf85a33a63144be41c2fc79174b106 \ - --hash=sha256:a1be43ec69fe563ac9b2c4d6fc4334b323b21cbdc59a638b5fa34dd4d5a2a348 - # via gcsfs -google-cloud-core==2.7.0 \ - --hash=sha256:874aaf89765db87a9b911b7a2ca7c5068554868eed9e75c7766affe342a2913d \ - --hash=sha256:c18a250904cfdda021eb3ae8b8238c9f9ca272a4cbbfb5cba946b3fe3022eed1 - # via google-cloud-storage -google-cloud-storage==3.14.1 \ - --hash=sha256:8f0fe2b74bd80ddcbd67247a3b47b81035a0a4f448569ed358fa40a46c940118 \ - --hash=sha256:b24e74b493c60b19b83462933a83bb831e45b6ca4924b0d75eac8e176b58b3a7 - # via - # gcsfs - # xprof -google-cloud-storage-control==1.15.0 \ - --hash=sha256:46c8c801ff0425a1bd59bc28899a8242e15f6bbb06a4f2304cbd169c7d26cb29 \ - --hash=sha256:e3d50276f0aed0c33ebcf3c6221e933889e042df70a56886ddffe59c66889183 - # via gcsfs -google-crc32c==1.8.0 \ - --hash=sha256:014a7e68d623e9a4222d663931febc3033c5c7c9730785727de2a81f87d5bab8 \ - --hash=sha256:01f126a5cfddc378290de52095e2c7052be2ba7656a9f0caf4bcd1bfb1833f8a \ - --hash=sha256:0470b8c3d73b5f4e3300165498e4cf25221c7eb37f1159e221d1825b6df8a7ff \ - --hash=sha256:119fcd90c57c89f30040b47c211acee231b25a45d225e3225294386f5d258288 \ - --hash=sha256:14f87e04d613dfa218d6135e81b78272c3b904e2a7053b841481b38a7d901411 \ - --hash=sha256:17446feb05abddc187e5441a45971b8394ea4c1b6efd88ab0af393fd9e0a156a \ - --hash=sha256:19b40d637a54cb71e0829179f6cb41835f0fbd9e8eb60552152a8b52c36cbe15 \ - --hash=sha256:2a3dc3318507de089c5384cc74d54318401410f82aa65b2d9cdde9d297aca7cb \ - --hash=sha256:3b9776774b24ba76831609ffbabce8cdf6fa2bd5e9df37b594221c7e333a81fa \ - --hash=sha256:3cc0c8912038065eafa603b238abf252e204accab2a704c63b9e14837a854962 \ - --hash=sha256:3d488e98b18809f5e322978d4506373599c0c13e6c5ad13e53bb44758e18d215 \ - --hash=sha256:3ebb04528e83b2634857f43f9bb8ef5b2bbe7f10f140daeb01b58f972d04736b \ - --hash=sha256:450dc98429d3e33ed2926fc99ee81001928d63460f8538f21a5d6060912a8e27 \ - --hash=sha256:4b8286b659c1335172e39563ab0a768b8015e88e08329fa5321f774275fc3113 \ - --hash=sha256:57a50a9035b75643996fbf224d6661e386c7162d1dfdab9bc4ca790947d1007f \ - --hash=sha256:61f58b28e0b21fcb249a8247ad0db2e64114e201e2e9b4200af020f3b6242c9f \ - --hash=sha256:6f35aaffc8ccd81ba3162443fabb920e65b1f20ab1952a31b13173a67811467d \ - --hash=sha256:71734788a88f551fbd6a97be9668a0020698e07b2bf5b3aa26a36c10cdfb27b2 \ - --hash=sha256:864abafe7d6e2c4c66395c1eb0fe12dc891879769b52a3d56499612ca93b6092 \ - --hash=sha256:86cfc00fe45a0ac7359e5214a1704e51a99e757d0272554874f419f79838c5f7 \ - --hash=sha256:87b0072c4ecc9505cfa16ee734b00cd7721d20a0f595be4d40d3d21b41f65ae2 \ - --hash=sha256:87fa445064e7db928226b2e6f0d5304ab4cd0339e664a4e9a25029f384d9bb93 \ - --hash=sha256:89c17d53d75562edfff86679244830599ee0a48efc216200691de8b02ab6b2b8 \ - --hash=sha256:8b3f68782f3cbd1bce027e48768293072813469af6a61a86f6bb4977a4380f21 \ - --hash=sha256:a428e25fb7691024de47fecfbff7ff957214da51eddded0da0ae0e0f03a2cf79 \ - --hash=sha256:b0d1a7afc6e8e4635564ba8aa5c0548e3173e41b6384d7711a9123165f582de2 \ - --hash=sha256:ba6aba18daf4d36ad4412feede6221414692f44d17e5428bdd81ad3fc1eee5dc \ - --hash=sha256:cb5c869c2923d56cb0c8e6bcdd73c009c36ae39b652dbe46a05eb4ef0ad01454 \ - --hash=sha256:d511b3153e7011a27ab6ee6bb3a5404a55b994dc1a7322c0b87b29606d9790e2 \ - --hash=sha256:db3fe8eaf0612fc8b20fa21a5f25bd785bc3cd5be69f8f3412b0ac2ffd49e733 \ - --hash=sha256:e6584b12cb06796d285d09e33f63309a09368b9d806a551d8036a4207ea43697 \ - --hash=sha256:f4b51844ef67d6cf2e9425983274da75f18b1597bb2c998e1c0a0e8d46f8f651 \ - --hash=sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c - # via - # google-cloud-storage - # google-resumable-media -google-resumable-media==2.10.2 \ - --hash=sha256:1de441703cd298d75a419bfdc0066e9fc7b0a1de630df96eea8ce8f5c759358c \ - --hash=sha256:e3cedc827a4ea41e216582d74346f1fb9fceb625a8c3c53912f2ca1d663334d7 - # via google-cloud-storage -googleapis-common-protos[grpc]==1.75.3 \ - --hash=sha256:57c435ac2c68b108999b6db075d9053e4d7a936ba57b4a3d45667b1346f1738a \ - --hash=sha256:a018d2bf098ca9fb6faa08d5bb780e2a2c2f73c566f069761331386c9596d3f2 - # via - # google-api-core - # grpc-google-iam-v1 - # grpcio-status -grpc-google-iam-v1==0.14.5 \ - --hash=sha256:07fd3a9fafb586588e771831fbfc8f6597050181d0c3b45e039d18b8fdc1aab5 \ - --hash=sha256:0f5e680b20aa0a9441e68c769da04d94d70fca4e43751a82d8abb8aa6a7181ca - # via google-cloud-storage-control -grpcio==1.84.0 \ - --hash=sha256:026d757df86c5b7a41de8200b9a2cda454aaa5004cb0c7e3374c66eb82f61499 \ - --hash=sha256:06619ba1515e5ee69fb2a514e95dd8be05ce74cb3928d5b34f87f87c86fe3c27 \ - --hash=sha256:08735e3d08d24ab3132cf87e2e5dea8746cabcc7d676c2b0b7362f195feef9d9 \ - --hash=sha256:0d532ade4486dad9b302ffa4d4683d67561051c26d17c4023322845e9fa10140 \ - --hash=sha256:158c1c11cfb61b4849c3caf4d52de6f5ecd376e14446feb4a90dc95a90d616f5 \ - --hash=sha256:15bb76489e337fc492685c9758e2fd4d4ab516b901ad830dc5a91987decf00be \ - --hash=sha256:19aaf172fc2edbefccce3f6e92c5150975dbe56c45744e9e87cf72ebdf85bfbe \ - --hash=sha256:209414080da8c20af94df1395b635da52dd57b5edc9e917e1deca0dc1c4bb55e \ - --hash=sha256:210e4c32f907045eb8158273e60c6ab69a3947697df6245dbda381f26c59485b \ - --hash=sha256:23e6e8e8a75cff88e0a793bfd3becea03a13e2763ae90c1ff573bc19ca5b429a \ - --hash=sha256:27b8b36200a9fbee6e120246f4a8a41657549107ef19fb2c819c4b2fd524f39a \ - --hash=sha256:28d2609691da93051e998495108bbddd2a9f7a561253bae94828d81290f30c15 \ - --hash=sha256:2c024da73b296f040b8360e60bd73a659b230093684a438da0e1260f34cc724e \ - --hash=sha256:393d8a78bff6731ecc5ad2151a821f8fbc1709b137ebb9c25a4ef399fbdcc914 \ - --hash=sha256:3d6a82c4fc6c85f2fb7572c86bdb86f84c97b6580e5f6599f711800bac48a5d8 \ - --hash=sha256:3de427b05f244ba2c2a9bdc67e7a6731c8340811524ecc4435466549f8af1d17 \ - --hash=sha256:406583b4e8fb2282ebd392e12b963e601c1f82e07125a8c2cb5b144e7e024796 \ - --hash=sha256:4119efa6519871719ad81f33bc95ab87857dcb1c5801f30a6e592f2c41164169 \ - --hash=sha256:42959bd50dd660ffc3f2a9bec15a6da4f9aaa0dda555d59ff2d2e80b908456a8 \ - --hash=sha256:455ed6083353b8e938f1d58c765eab2fbb165731e5b507be30fee344915a2a11 \ - --hash=sha256:465eef3d17e59ad22a556fc0138f7c7c799df426734344daec42c797d49fda99 \ - --hash=sha256:47ecf0d9b81d981f07b61bd89eced9d2582f5eaacc3aaa36ad27f81aef70a27f \ - --hash=sha256:49717e857899f4136d7657bf5aded61ac479110a075438290923a4d86af7cd02 \ - --hash=sha256:4aaeceeb7fa7d824c322d1ec3208c8495c88478a927295553235435fc49043ad \ - --hash=sha256:57dc36a5ab0e676f5f6e171de2917fd0aef73f32a9aaf23956bfe19997a30bd1 \ - --hash=sha256:5933a052946873d01a42119a05420d669bdca436aeba2d1851988ccb12b421c0 \ - --hash=sha256:5deda5b4bf62769eb98c119cca43d40e1231e34846b19db5cdea821d446a2253 \ - --hash=sha256:61386101ecaa096b694d0dd278caf99a56aeec78440cc17e918eef0b50f2d567 \ - --hash=sha256:659728f20fc7a0933ed7b1945435e31014b97ab8a5a7edcbaa70da4794aeb191 \ - --hash=sha256:70bb4ce8be0c5606bec259cbd7152374470396413b7863a658a08c849e6b29ff \ - --hash=sha256:71fd60e6e426d293d0a2f685115ad0a0845117602cf13605a4be7524fb5f7bba \ - --hash=sha256:756ea5c2da00fa65c930284892d2a9706828704ca3ba40b4c51c4834eb39fcfd \ - --hash=sha256:800b7e00d92553313c0463c200087930aa78678ec1d528193aeb50906f55989b \ - --hash=sha256:82da34ae4f639c73ac46e521e00c0a49bf86f717b9fb1f405f133e98731e38dc \ - --hash=sha256:8e1a45d174b6b8589f51dce1cea804aa6c1f72c9c80cba91ae2caabeb6d90540 \ - --hash=sha256:8e3f508d0e9e6236ba2f08d56e33355e434e785e813149a1b8477d3edf69779d \ - --hash=sha256:986e9751d416d7a6eaa2fecdac38da63153d63a4b340ba7d624889c490451500 \ - --hash=sha256:9b73836ba0e16fcbb57c31cf6cbc2907c8d8c790b83679df454b74bd15e0be04 \ - --hash=sha256:9bab4cf571653a8afffb83ce21aa27b51dfe629b526b7b6adec35491fe1fc2ea \ - --hash=sha256:a71d24f40b0cc6798feaa978c7411dc1135b7018e9fc0442db611c139bf58344 \ - --hash=sha256:a9383401d9f116f98cacd4eba6c505a6edb80ba65badfc8e8ed8ae64983bcc44 \ - --hash=sha256:b44f0a0fc7bc6677d38cc80bca1a32814ce6c8f200fb8b3c1a61c9d77eaefbf3 \ - --hash=sha256:b5c6f20d657ae09ae4e30d9d3a21edd13f1219d58cc6f999b9d1bb63be9c1baa \ - --hash=sha256:b61692f0069b3eee2fc8a3a1b7f6c044df9e03fede6ce69b3ca832e1c39f26c5 \ - --hash=sha256:b8c62888c3e49debf37ad9773e3c02f77b0c1e811f8fb0962f2b6c3bbab5b97a \ - --hash=sha256:bd8ea8eb3817b226057cc1c0e7ec4b378dcda52043b972b6ff12b1152178967d \ - --hash=sha256:c5559b492007dc09b4de9b95dab05f0b5e53547aad230cf07e46c7dd017a3be5 \ - --hash=sha256:d0fdd25faece8a1f95e8a3a8006e29701b5cf8dadb4a8132e68f3134637004a5 \ - --hash=sha256:e094dd21f077af8194923fc263cad872eaa1802bb0156fd7e5ae18e99cd86715 \ - --hash=sha256:e41c3993eee896c617dbd8a505085d28b6e84a0445ed9a1f40f95808473cf678 \ - --hash=sha256:e88d304f094f4937bc27ec6a435e218a084168f11ec630c8d5d39b431d08d81d \ - --hash=sha256:e90e3bdf7b5eac005fef631adae9cafde16f922def207b80a7c46b253c18ad20 \ - --hash=sha256:ed2c1493c44d0932f1e55fdb5d1ead658c68288ec5d51b8c4928422d98633ef9 \ - --hash=sha256:edb6f87fc60ff438557291501b3e16c7a77c3b01a52d782cf276dccc7c5dd89c \ - --hash=sha256:efb29f8633bf6630dc89de4fe0353ac3d7e4b70ef7b6e29fb40f00e68c127fa5 \ - --hash=sha256:f6c972474ce691aca74e58d17625450cef153dc4760364cadeb167983ea6d589 \ - --hash=sha256:f6d178ba6dc8e82976c184b65fddde172d054c17237993a3e083efe4f134d55b \ - --hash=sha256:f9a456bdbed52a01c9ab8423bdebab04a5363c78676edc55ab9b58bd13bdf9e1 \ - --hash=sha256:fbdbcd06986ede3ce584083b1dc2afe6808e8943e5cf50ad11183c03aceda25a \ - --hash=sha256:fc66cb50c93554b86db0b6625ab5c6e9051dbf8847c08d93c84918e02e413fb7 \ - --hash=sha256:fff5ef3fe1bba7d6147e5f19e01e5e122ac2c076486887ddcb8d42e663400fbe - # via - # google-api-core - # google-cloud-storage-control - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-status -grpcio-status==1.84.0 \ - --hash=sha256:0c182ca0d6e60acbfd0e14499cf39a155e4827a1c3fd9f7638e49af15a74c30a \ - --hash=sha256:5caf28ba7184b81f618b5f7f094859fd2541bf429d2189bbbcd715c9c2cdcee2 - # via google-api-core -gviz-api==1.10.0 \ - --hash=sha256:846692dd8cc73224fc31b18e41589bd934e1cc05090c6576af4b4b26c2e71b90 \ - --hash=sha256:a05055fed8c279f34f4b496eace7648c7fe9c1b06851e8a36e748541f1adbb05 - # via xprof -idna==3.19 \ - --hash=sha256:5e0811a4383b21dc5838069f801c4fb62113b7447663d2530d2bd6e77b49bf15 \ - --hash=sha256:815e7be7a7806d54abb586dc943addc79e8b2ee16915059658cbeff4b1b43bf4 - # via - # requests - # yarl -jaraco-functools==4.6.0 \ - --hash=sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280 \ - --hash=sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30 - # via cheroot -markupsafe==3.0.3 \ - --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ - --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ - --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ - --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ - --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ - --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ - --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ - --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ - --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ - --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ - --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ - --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ - --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ - --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ - --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ - --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ - --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ - --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ - --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ - --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ - --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ - --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ - --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ - --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ - --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ - --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ - --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ - --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ - --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ - --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ - --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ - --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ - --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ - --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ - --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ - --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ - --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ - --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ - --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ - --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ - --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ - --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ - --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ - --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ - --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ - --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ - --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ - --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ - --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ - --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ - --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ - --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ - --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ - --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ - --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ - --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ - --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ - --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ - --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ - --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ - --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ - --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ - --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ - --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ - --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ - --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ - --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ - --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ - --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ - --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ - --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ - --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ - --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ - --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ - --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ - --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ - --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ - --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ - --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ - --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ - --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ - --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ - --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ - --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ - --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ - --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ - --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ - --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ - --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 - # via werkzeug -more-itertools==11.1.0 \ - --hash=sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d \ - --hash=sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192 - # via - # cheroot - # jaraco-functools -multidict==6.8.0 \ - --hash=sha256:003a3bddb32915c3f67096ea41d24e53edf710edb65a1f5d0c70ab40b0e4d20b \ - --hash=sha256:00be37bde741bf60871082cd347a093218c44886e99231b7516671c70f2c280d \ - --hash=sha256:029897732a9c798737457e382bf84e8c64237eff224a90aea2639f4413c45e4e \ - --hash=sha256:05c2e90c5289c5f7436ba2c25812a5fbdaa1c1bc11c8d8d3bbf64f5cd7c633dd \ - --hash=sha256:071da134651b04a8507dfb331ac0988f376337c2aea59486bf20989fb5b5a64e \ - --hash=sha256:088b04a66b3c1fce6fe4d771ec184a0426262d0b86709c908477b4ac7965df40 \ - --hash=sha256:093167d22a8c95af30f597b8a5686f20a14512989942d4be804d119899caca20 \ - --hash=sha256:0935971bffd0b479fc90c4811ca787703e93fcb6afea939a375dfc80285ab368 \ - --hash=sha256:095f62ea4e7a3be2f6c567ab695ce10e950f2adb905c1bec82281593e0b2d2ad \ - --hash=sha256:0b143d53590e89f43153d81d505a8448d4d57354354385aef8a51d67ffefa27e \ - --hash=sha256:0c1c4debad7337627b86837abdf0237ca3cb3d7e17de7eab0177c263878546d4 \ - --hash=sha256:0eca15d627e942ce186a935061f1568cc46c02e97c419c8da802df2be9f917d8 \ - --hash=sha256:0ef606c15cac6c90279acf34120784b6f36662cbf382defd3955cd8f1115336b \ - --hash=sha256:10456943903744ae1249728161c96bd9d2f7eb5ee17fcc2ffda2dc32e1bb36c7 \ - --hash=sha256:11d71490bf4bbff1141b14b93af419ad68c56b60bea9277fcb3f94dcca4796eb \ - --hash=sha256:122adc7c46ac1e31ecfc7f81b2530533dccafdba70f5d741649f87e336c63384 \ - --hash=sha256:13967dca8b2f33230a1427b52438326bb1c9101a1df22a3309ed3fcbbb3c96f0 \ - --hash=sha256:13e26f59f0eecfc5f67c663ad550ffdaf62c0f657547cde387f6c86af1c9449e \ - --hash=sha256:15db8e6cab5f4cc9241bc56e69fdf3452cf49c10ee3c7977c742e68a275b3786 \ - --hash=sha256:18f0e06360c3e451a3ab800355773c8d125a758238d780c800b0ee5e90ee903c \ - --hash=sha256:1969971900b0871530f9b62280dcc2d75688e74d2a69262bc01faf2b96c78f04 \ - --hash=sha256:1b8986d4313dcee7c932837d16a535f1840b827bac1ea7c5c4c80751d0423794 \ - --hash=sha256:1bdb9b8fba5a9aef673ec90db3f55b1ce743f2fbdea4d37dc04d14ccdfc153ff \ - --hash=sha256:1f57c414be82490bc0e0305fdb834186229b2d9b6a35fa0afd1eb1a772d125ab \ - --hash=sha256:1f66fe6a021173d0d47968491791966b9f3e6d61115f2491744aa0c07a6e67af \ - --hash=sha256:202436df907c15adbb94360296c425ea53cf8968a5d2cff9b5b9790ae1972b33 \ - --hash=sha256:2196ba6df392c3574acadd14ef87550f3611349c8618564de324b806a7a31cee \ - --hash=sha256:22a310ad37672a261e55a8b5e28d0ae08cfb68abb1f46418ccd19835c3b8e836 \ - --hash=sha256:23c9ee89967b6a9b4048acb3b93b660ed714ce9c8bf3bbe652959bc120dc02dc \ - --hash=sha256:2622fe114c0bd66ca5c461859357587f5a5e35ee5ff49fc5643d1bc78dbb41c6 \ - --hash=sha256:26a7aafc992e78872e2c8c1f7248c0e01139cf9020a7781b0c064fa566832712 \ - --hash=sha256:27747162712e85c84598d364425dbf1714ff335bdb6ba3171c4e5081196e8916 \ - --hash=sha256:29631224698de1e42abc8fa7658d830e0aed0029785144b5832b695da5adef2f \ - --hash=sha256:29b6e7bc4442a56cf8e0dc1cabf3fdc77cd533568d6829fc76a1effd2ce332ec \ - --hash=sha256:29be9fd289e9ab8f480996ea2f686e1654b80242033843cb11691688329423f1 \ - --hash=sha256:2ba9933e8f35fe4a70f540b837254c4055da82dc3a9e500a8f95e61498083a15 \ - --hash=sha256:2cc66abb85e2108c9ff8a1c0d20fa260bf690bbb33caef4ff3ecb2c2cbdfff5d \ - --hash=sha256:2cd560498ae8e1bcc955643c1d78eb8e338226d07a983c656ea8c4443d3eec0f \ - --hash=sha256:2f79cc3e8039a8cf5c77e0811b0807953fd52d0863b9b76970b20d696dc64a78 \ - --hash=sha256:2f8a4b0b4d639d525928c7f30de527bfdf9ead6e44a5e8cb9c50aced5e4590cb \ - --hash=sha256:307c1acd812fe897e7fbe10c6758822e8c04be4e7c60a9f54901cdf8b5ab8bc3 \ - --hash=sha256:3126f2a96704505aa4e92a72d6e8a5d7f29d40a987ced8bf69e29d71dfc71fbc \ - --hash=sha256:31e8901637e20ccb3cf8f8848b5d0f7a00462bf5b34f7cf3dcbb2753b18e8b39 \ - --hash=sha256:346ac52e56bcda320c0dcdfdd081947ed7cada33afea4e2284bef7b0733bff9b \ - --hash=sha256:348bb85e2038b40c007383616d73f734869063772372519549ebd7da1723d1a4 \ - --hash=sha256:3533a03e4e789baf6a286e7b0b1b6da3f3d7c3eab569686ee29ee1d8b52e2cb4 \ - --hash=sha256:35977263d9bf506dbc65349f63b3b8c91606d4abc110990945e3b94bc671319c \ - --hash=sha256:397599503b718f0137f26d3f6532d6955069cd2e5917c47ef581495bc2529ff8 \ - --hash=sha256:3bafff8598f0528017ddc74194e5451d5c22d046c98935f8f86247b0f286e4f8 \ - --hash=sha256:3d1f48582686a0a3b81e9b43234766cc96697df72081af3f48107bd3f34d34e5 \ - --hash=sha256:4261863fc8b5ab1b815ede94e592e94c6af5b04616014929057e61859e7382a9 \ - --hash=sha256:43a4b56555bbcf8af161e7c7682bd93eec10f068c95844511864c018c8e5e13b \ - --hash=sha256:45cc39ba50fb0754a4359b90f8229ae08598fe2266abe3521b4e5a9ba916534a \ - --hash=sha256:46029e6e27a3ec0dc55b53f58df82d10f04c5e111f78248279b530bedad2c30a \ - --hash=sha256:48ea524a25a1cd5972cf293bc95713918cba0bcd6fa9b992d906c857c546abe2 \ - --hash=sha256:4ee953a5ebaeed38dc21cc032ed17a9d9782802e00042200497ab4b01b0bf7c0 \ - --hash=sha256:54af1266710cb0f305127ae0b970aff8d208057f8a29cd6e1db99b0114947035 \ - --hash=sha256:560b211fc3bd4a1e1c6de44f6d38113bf5b410dfc89a4c0d2a3c0edbf1a0dfb8 \ - --hash=sha256:563661919f603374c40cf45ffcd25535c12b8954203569a2ab1cee5265871cf4 \ - --hash=sha256:563d6500ca80dac7bba6f48a78e0ffd87e21a7d4d24642c6503a2ddccd70c110 \ - --hash=sha256:59e539c4eb4d3a53b0e630a6ba2b2f2824732b5e73f90e30a280f12fde157b15 \ - --hash=sha256:5bbbb696c8024475b1877d14ce20d5f1cc05b8f6d786cea0fe3aa7fedc02e891 \ - --hash=sha256:5caf684986a2490628f059a99dd107b566a2d34cf947f8eb8387e0500a1f90c5 \ - --hash=sha256:5cd4637ce76312ba1e05eb9c5193fec231f64fee0944e135fa1e951242355b37 \ - --hash=sha256:610c7637bc36b90f39e6c66f710f93d57018f83d53e1e187caaa218c6892b95f \ - --hash=sha256:628ff11e6720f90acd0c305dfa3339f04a783a20de8cda6ac333ba46447261e8 \ - --hash=sha256:62b8e291a4f7edbf7cde7a43d831d893ba443a1b627498b53581943b0e348feb \ - --hash=sha256:6300d5176647145ba1e22991c924fb29743e54b4d7b8bc85a0d3ec0e55e189cb \ - --hash=sha256:64eaeda36ee8d88f9e8616a587a8c66a663283cf6e0dcf013c1ddd8c758e4aef \ - --hash=sha256:658f5a1895b804423d97b22d06fc0d0b171c7c01dcc3aa9c8faf0c0e26a249a5 \ - --hash=sha256:65c85c79f5a2c04fbbc18f006c014674dc5fdf270cb978d8862c82c6f694e60c \ - --hash=sha256:68186a2d4051c8ffd17be33553bea2ec9bbc8ef860fe2980a221d96126296f31 \ - --hash=sha256:68d40b2bace413f3231f5729d3fcfb1837fd31c4907e241b5d43211bfd76f3c2 \ - --hash=sha256:69708fecaa88bcb2341397b49fc95057a835b02a3670c551b37f95dd79e64e3a \ - --hash=sha256:69b3e519a132bb943b0daae15fc8c2168706b17f826481d32a32a5e784b129e3 \ - --hash=sha256:6b62b7e0025aa48dec11e125e655d1157985a5fdcec04b1ad500101ad072b891 \ - --hash=sha256:714597cb5d5e15a8a449d2ae23c45b486a9e8fa33c462c7a33d7f35b65d92943 \ - --hash=sha256:758233648ac47b07c575224c4eadd73c8929c3b4c31e2afcfea935fde1cda735 \ - --hash=sha256:75daa15ca16d6285eb2e104b2f05ee6f8d9836c68da3ce5c85f615a0450eed0e \ - --hash=sha256:77745725125d01fd613b6db043362aa7c6bfbfdb23d45dbfc3d92bf58160af62 \ - --hash=sha256:7941ef106ca1f2c62314a13c7ed913bcf49641f3efdc12864d588e17870920ac \ - --hash=sha256:7a2573d0fd34f361a4a14e54d8cda3a91ac4e55fbf0d719698024f3b09c5b147 \ - --hash=sha256:7a62e302fc8cd6aa8972207e7e951d1fdee7c1dda18568305041d19f0e2c00f5 \ - --hash=sha256:7bb0dad75068fee80fcb60f88569722c199d8656a16706702dc6e3b786819c90 \ - --hash=sha256:7bc7003991ebd368a20d05228137a37b3d3066751f3ea1e4f7b8efe8e752f2f5 \ - --hash=sha256:7d26dc8f070c0ec5579e987fa615ffd6883086106eefdff9e10d160fc5630630 \ - --hash=sha256:8125e60f3c70e323ac07dd8b3635f7b3bbc5c3a9ac04ae5988f668ff7ae28a18 \ - --hash=sha256:8180b635290a75af8478f1b3e9810135381ae24833293fe77b85c1c21ff842ab \ - --hash=sha256:82780eb8bf59e8fb25dd081fde6e058805045d6374a7f2f877effc826ca4434b \ - --hash=sha256:835d5a90b11d1f5f8200ff3cc8316bded76eebebc92436398947a27657e645e7 \ - --hash=sha256:83ff054b04915be5c15680da6c6012474a2cc2bf534129a0e8c6a99f17ba7238 \ - --hash=sha256:8457aff3c12a89a8e1c4674de5c777857fbc429f40fe117a3d29538547cbc364 \ - --hash=sha256:847d6082ae694dc95e548acb201bc100e1cfa96513bc71fdcb86f709dad6c435 \ - --hash=sha256:883284137e25318ed9735b742ae46341a864888fae28e8b6314c4f84da080f08 \ - --hash=sha256:887f9a975996032c686719eb7b3e1e7942fab5079c2b778bbd9afe9a9d78244f \ - --hash=sha256:8890c89d662560e51c55ac1304d6f919b23942abe9ae1127cb1de9aa6132fa52 \ - --hash=sha256:88a6df88567680504ae28bfa7a1f2f64243d91e79a40b2c92ef42efc531e23da \ - --hash=sha256:8d1046b5427dcafe6e8a0e07527dd74f1ee694006160162f53f3a17f15aad3b4 \ - --hash=sha256:8daafaa0b2eb43f76898ced78b1e0fb91b38c4fa50da516c18067f2a2d578c20 \ - --hash=sha256:8dc2d9c3a924ed14166e63650b2cf9f59e7821743bdd50b23802bd97ca09bde5 \ - --hash=sha256:90c10b22860dbd09982d0b8993b66231a861bea2993d4a817ff35273f6ea285a \ - --hash=sha256:91fa75d0a693832106d98f66c849f034f21c828d14437f1fb97d3784aab89e84 \ - --hash=sha256:930c6058047410e3edff445f5a6e4457f2e089042dede00e2d18ce06f3ceae2e \ - --hash=sha256:9442b14eec262a1f74369bbd07e75bc5155105164649a4b9fbc1ebc7b8fb0b14 \ - --hash=sha256:95c27b4f3f04320fc44e338573f40c5c956b504a7fcf081a157fd0b02579311c \ - --hash=sha256:9606f583e7acaf61e7b3f56074e14037b9af7cb194590edfc0114b3ae5931ff7 \ - --hash=sha256:962f18c59a000f30b084ea2e6b8001521bb315efd4e5f10acf9fb36f366b7882 \ - --hash=sha256:9caef53b20a105c0d66518a34be2f71b2783de8d091767575ef86f6ea422236d \ - --hash=sha256:9e37024b41d7a7e7e9cce14b248d54707c21c2a2ea30a47b71bdcefcafec00f2 \ - --hash=sha256:a5a7ee1217949ddd43c6b7bcf70d5c22193bb50e8c695386de5905325e93ce9f \ - --hash=sha256:a5e1583c14775580da05641240ce0d93f36ce3ddef3d5083a827468b0bcfe874 \ - --hash=sha256:a9e246f67ac038568b854ed7c5578e4c6af1f742359901a8fcc3603ff1358df6 \ - --hash=sha256:ab83fdd8cf307353edba9c427c17a3a021c2522d690f5633dd9f72d28b48ccca \ - --hash=sha256:ac746cb365bac1c462da9e3e6ab8904a8efe2217a56b0b2e3d9480f41d2b2602 \ - --hash=sha256:ad474c11d851b6fc97cb625e4822bc0cbd567fc07dc2602e28faec5a36b42bbb \ - --hash=sha256:b03ca066b47b18b205cc080dca6f76cbd159f8cdd33a02a0700164c13b37e463 \ - --hash=sha256:b1cd4d66ce894a45482e1ac2837c31d0bd447df35065e542b60055aa2d00404b \ - --hash=sha256:b25426f9f6ed402835617c8f23609a47045f91ecff365eb6734817e039a8ed25 \ - --hash=sha256:b367c342327717d644db4c0ddb37ceb655c84822215ea0773a3a36911b74b71d \ - --hash=sha256:b7e62b8fc7bd6cad007b9f2e0ad9c8d4854c06350d5f51e1a439dd18b510ecac \ - --hash=sha256:b8b7aa75146266fd3e2a2437cf69ae188688c04ab8665b163d4257b46c1e0c83 \ - --hash=sha256:bb36381e1f9f9d06eba2f10bdd438e5d20c07d5b55e1a3eee30b9f44cbf52316 \ - --hash=sha256:bb8c7da8c861391f7ae48e3593762be2dabe405109e01aec520fbe1a6d15d14b \ - --hash=sha256:bb9a60b7faa5d37c426fa91cf4d6738182a1f2755b9fab7c9c64cd466c4ce51e \ - --hash=sha256:be007d1aee2cbd530347dcafedb400891a3b5f1bd7135f95cf5d5b330b5219ee \ - --hash=sha256:be569fff1d85cd29391c431c5641c8772acb75bbdc61e60a8e82fceb9023d385 \ - --hash=sha256:bea7df027015856ba5d0a88e3b4777ff8cb5c66b58fc108050fe79d4dd9d4d2d \ - --hash=sha256:c0fe437a6d2f36aac2b49517057776575b5bf359df314cca20d230a6e139c089 \ - --hash=sha256:c2b2a96cf1dd99fe7867be4c013314225f4d5786e6685906e29932d42aca6f11 \ - --hash=sha256:c2c5fd0fd39574ccd58e1a52565b341aff522c5c836f1b3eb7605c371e61f52c \ - --hash=sha256:c46a08bf070d6849fed483e9d9833f9d06aecb8382ed985be0b38508b3ae958e \ - --hash=sha256:c5f3a2af441670d80ce5fdf13b6c1b421fc1fc7fc5182d58ac7486738bb2b742 \ - --hash=sha256:c60e50bc5b07faac92fd3a20fa21cc8cf3e3f7204d2867b206c73293ebc19101 \ - --hash=sha256:c68e0c0649d17c2d0339e3674e86a4aeba4a7e6b21c1e394cf947a95433b31d0 \ - --hash=sha256:c9c98d2f0126ba84cb45601eed97ff67ff767e19ae6eb3c31b02827b54d700e5 \ - --hash=sha256:ca52b9ec80851366197577154c862c4c4c7036ca76ae94cef5cb59c5cfeab944 \ - --hash=sha256:cbd86f9787c5e2f5fd27d8b21458222f107347c6731c4e93dde68f554b466a2d \ - --hash=sha256:d0264f8d5cb0a803f650a6a8572dfa0cd1e099a2234c588dc8fb220b415b865f \ - --hash=sha256:d0be2b832435001bc623ca7f1499ca1a853d4f082fb61221a80ce71132f50b26 \ - --hash=sha256:d244cf6b52b5ba1c34c3832f4652a668ebb36d95949b96eed9a1c54d916a90dd \ - --hash=sha256:d2d236b8a44ae91536a12ebcb996bdb31cf27425f36b4d05c87f2ba2716050ba \ - --hash=sha256:d3da668e903c934ed0b587ecacfed6901f6ae6384a6e975887592b61845e78bc \ - --hash=sha256:d6dc7804c50fabd28644d4d18a4b20aad3681b3e64f3acd3182b330ca73f7a32 \ - --hash=sha256:d7e5ba0a0153e35fbce9c51df530c8b4cb0c3012b46a04ff9a048441a269c2ed \ - --hash=sha256:d8a5ac357ac283490a8d1899b0383355fd1f8634b14ba0d59e4c0dd97db85556 \ - --hash=sha256:da1c112c5784ccd9d32cd90be6739fee32644e874eff6ae8f0497cba3e352e58 \ - --hash=sha256:dc911ae6152e455b16a2a1a626aa6cd612fa01efb9d0a4ab3f5cf328b911483d \ - --hash=sha256:e0db3a4d1e264e225037a6023888972c25206a96e016021a5bea41c9a939f2a9 \ - --hash=sha256:e192018b732f7b168e6604cbdf40fa8e05c996693b9eb445a0d8a73f4b77c5d3 \ - --hash=sha256:e37b744849fb631bb52e3dadde35ffeee365a6c41cf71257b5b7acc9cd83fd38 \ - --hash=sha256:e41226ecf607f062fe34a2f4cf64ad3a89e3a0180dc800b463b6b14c06dd10dc \ - --hash=sha256:e418ec99574ca24365ca96546af285c2b021a1a072478a79f0e3cc3b08837154 \ - --hash=sha256:e6ec7d37841609a691b96a10b4fde386c7cd93ebbb939f59c9f23325ee788395 \ - --hash=sha256:e886ef8c9879105fe4fc99417447b3a5f35d1131412ce839470bd2089fe2043f \ - --hash=sha256:e8e1e895e23818d343e4ae7dd95a0a556fdeaf8b471acf1c0a39b93c6f54d478 \ - --hash=sha256:e9dc7b4ff6ef184504b49ef9a4113d49a646653b2ce89f5f48c1f57cdf6ba081 \ - --hash=sha256:ea880d441be7c510106bc56064be39266d948aef94ad4955e8784690019a5d9f \ - --hash=sha256:eabb03dc3e4ed6333ecd1cc9826ec80e7a98b5506deeb832d7260c8e44166d23 \ - --hash=sha256:ec0a4d066356054d569a66e0a94691a2058b680be5e710298f61db11a3c4609f \ - --hash=sha256:edda19aff836ec515caafc09ea53d2ab144a041f09ee9a7cefcbd3ae4e976256 \ - --hash=sha256:f1f4a220db6ed7c8fd16b6d644ffd1f082651693204daf3275e049fadc849e39 \ - --hash=sha256:f25b61a708bd276e8cbb6afcbbf1b8e793a3be70ba0a842d0b8692020f83b706 \ - --hash=sha256:f2fa3d3b1c933d4bcb8fd2018700d5e7235c52f2ab8c88d22286965c5c0f00f8 \ - --hash=sha256:f3071e6515cc63714d014da8f738ae9fa3997c476203f3cd46de380c2376ed7b \ - --hash=sha256:f3a0a31189acf6703307397c6139ddabd734c20c5ef92649fc93e473df6615a3 \ - --hash=sha256:f7eefd0233a7c33ca980a5cfef26f1e9b5e2137839e752a99963696729f12d91 \ - --hash=sha256:f8b09b25e0f4dc2ea9e2adbb1cc3ba11a94d6fa3dd978ae659c8743052e1afbc \ - --hash=sha256:f8d7b66c9e09c0bb0add2b5895e646b62a0849e71155066f215523de6b95cbe6 \ - --hash=sha256:fa6c2880709c84457de104385b704fc28860f27e442ad13966fc4af8e714fe9c \ - --hash=sha256:fc5460940f50dff00731b4132366840ba9685286ea88ea104b661899084f3fea \ - --hash=sha256:fd789a294d8e098528be29b2669b83005ce569339f8cef167fc0274c3115c34c - # via - # aiohttp - # yarl -oauthlib==3.3.1 \ - --hash=sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9 \ - --hash=sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1 - # via requests-oauthlib -opentelemetry-api==1.44.0 \ - --hash=sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a \ - --hash=sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef - # via google-api-core -propcache==0.5.4 \ - --hash=sha256:004e685b315646c410771836e72a44f143bbe624f29653a42687815069a303d5 \ - --hash=sha256:02c0a34f16889cf800f10f0247a564d8ce6eeab6ffcd7c87198f769067eb8432 \ - --hash=sha256:03969626faf0783a592dfa17e28eac06018bd0b44dafae6943d53b92421a7f72 \ - --hash=sha256:03b229037d25b801e7af53fd52b9fc49d9439b036fca1e087e02780631adfa97 \ - --hash=sha256:0951315a6b3142ee2167404d707743f0157c110091342b1aa0accac5cf0e4acf \ - --hash=sha256:0a095db8e15a6020db149ecbed6461939fe74f6acaa3ae8b702a1fe8c38cd983 \ - --hash=sha256:0c889f6fa84957bc7e8b4eab71fd16a0455068d5045e3aa40c733071d2b2fd77 \ - --hash=sha256:0d21d0d2c82bbfeb1677a9711f38df968f9837576102bb4add1bd449d28d88f1 \ - --hash=sha256:10ef33a68a61ce317e095fd2e202a592ea92392b90944a78c993f0d9a73ab06c \ - --hash=sha256:12682126712ddc19b70ff819debbd279e58adf1f0c8f8f8138c18ade2044b284 \ - --hash=sha256:135036c5cfc93864affb0f9af9a27e5d7a71cb7bd745e7b6dbfc2d56cc30e827 \ - --hash=sha256:13e52b6e0bde97dee98ab66552dbff2931649c96f1ac432eac299fe689ec373b \ - --hash=sha256:141fdbd73748db0cf7636035030aaac383d2efde8f34e7bc24594cc776d225b8 \ - --hash=sha256:146f48a9e4812611a7581003b1a39de56c34967046310c4171a68ef908c9a745 \ - --hash=sha256:174507f82d3594622acb1dd2dafecf2d899d6d506335494e7107767bf05f3aae \ - --hash=sha256:1783582065a1f07f9d9ee1e992e13f15d7dc8fb1eb3a7476d43eb3f2e69d26bb \ - --hash=sha256:17a7400cec0256f0a71ae71f9da398f9894c956ff6668a1c9d317b3367316320 \ - --hash=sha256:1b2f3bec4261a94019575481c726c29850f72e27907773c75b1de421e20e9f9d \ - --hash=sha256:1d759d05634f1b038fb625a66662a8c85e5a8fec912da381b5149ddac107482b \ - --hash=sha256:1df8d8561b21465c5dd56110a01caf897e026d065b4b84e98a488209094272ec \ - --hash=sha256:213bb68d9ced5cf2bf717b1071bf2b09b4b04c426256f9fe6d054c60318424c4 \ - --hash=sha256:23278f808cd81d5ada7184a76606b925fb3389c60e1077b2cd7da7b1fcf0553c \ - --hash=sha256:251c63dd46a0659bb875cb254dc4c1e79ee91a847c737cd62373295afc2235dc \ - --hash=sha256:279655a16973f1ee2bd2fe79973137681642fd9ae0d89215bba263726eb0dc3a \ - --hash=sha256:2814ecd8e818f487bee4b0f921bc4d1c176cc5fc71ac0f072d0fa67eda4ac14b \ - --hash=sha256:286867fb156488c251a3721766e380ac4495e4fd6b51aaa1403d89ce7f4359d9 \ - --hash=sha256:2dba2f02d2d5c09ef8a0e6c1a42aeaa451f4be9898cb00b04fe98717da2eb23b \ - --hash=sha256:30cc1cebaf9aef49db06357a50398323ae04d70460c0491837d026ab7d6452ea \ - --hash=sha256:31eb43ba2edc704ab2ec27815315dd8a19def0fb16215be4cfe8d32fe78ffd51 \ - --hash=sha256:350b272b2279f4135a64fc0c304a5d08e28a137c9573442c606152446638a831 \ - --hash=sha256:36c0d9db44b523ef93d03341b1c42d69ff01d673c053d1b1c6c3a363bcaa39ba \ - --hash=sha256:3af0c8642b2da4815d86e631232ac8286e17644fad907c19508aa8e7cb4ba8ad \ - --hash=sha256:3cd3a7edb6b95b9b33998135ebfa18d709da82290fb8f27c858970b5a12c8b56 \ - --hash=sha256:3d605bb239b796e82a81c6709548b2bd460ab73b4590cb0c83de8a2dd9694d0f \ - --hash=sha256:3e413d7a4a9b4866b7a761d6060d434b64d23cd35122eda3b026a0bbe8196b25 \ - --hash=sha256:3eb2e820e8e2101407da93f17c57cbb7d225461955fc60105daaba14cd421ee2 \ - --hash=sha256:3fa15757fea1dfcd5b7745cad9f4638929605531bd4018ab2adff7955f1a403d \ - --hash=sha256:3fc24f209c1b7f7f688b66b98293954f5504279760999b58920ee12dd8471c1d \ - --hash=sha256:4054acf80d40456a0537f2913b349718649d8d6458a14ab7f48d0ce28c30869d \ - --hash=sha256:40e94adb1e7d39ff28a8bd8d8b8fbd1df6b9f40976dbe379134f1ce058e532dd \ - --hash=sha256:420162a77f94eb1cf5ef7893f500016dabd548e73de956785a1dd899cc73006a \ - --hash=sha256:425f8cc86ab5018b4b8d4a23bc8e74d964bd3d757c3702e301aa79be76c53f6c \ - --hash=sha256:44149f46500a0a41b95b4d99c2e586a77319539730607b9892974a092788b111 \ - --hash=sha256:445ee3bfb46e85838387fb3c536a73cc0b994dc192b004e40e170adc54aa2a7e \ - --hash=sha256:45488d1a5f9ab5bd90aaa1ca20f50fe1922b8ffad71a2009d2adf41355897aac \ - --hash=sha256:45bebbe252550fec975ba3b62bc6f931643cfd3b5464ef47619cf3fef154e01c \ - --hash=sha256:45bf2e730ab8905d0527fe05a86500f406e64305c34cc81ebe64b4617cab9760 \ - --hash=sha256:48cb48c5346a97de792254af77715aa2529c2a1ebc5f586aa0aae44a02f1fe57 \ - --hash=sha256:4a1f4f5ffa55dce6307631f3cb2948e117e665966ea512e0d502b16c24f567e7 \ - --hash=sha256:4cfe0a92ae30151869e67a4b5f5e105e4e03ad30b3f38e5211b5bf77d0881993 \ - --hash=sha256:4d86476a935c88963d9b8e1a9a0d38188790e9622169bfbafa173046846709d3 \ - --hash=sha256:4e985382be6d15da8d0c2710a6fa7b9070fc9ecdeefb7f580e88373984ec8be3 \ - --hash=sha256:4f2d880ff60f45898f4acfa152aac8d04e3ee627d90ff4003491bf92239d5757 \ - --hash=sha256:4fbc1a15dc8cd1689508758d626b372b1f09d28d9577667feaf9e6bfcd8efcbc \ - --hash=sha256:50e337653721d20ead710da33bf44487fbe8a0db8782714b60306481e9f95b51 \ - --hash=sha256:53eaa697c4d0422ff4cb714d00231b43352064d97b944033b30c1d57cc506ec0 \ - --hash=sha256:56fc3f7599528db40b1efa0889a620116e2704144495273d66066e8164e45838 \ - --hash=sha256:58134228927cee6c047d626c08e60a81be604a20578a12ce752cc5c9a84d4826 \ - --hash=sha256:594eb4c6ec35e7179b058481f4e9f02521b56de16fa577c4b85c76fb1bf8a9f8 \ - --hash=sha256:5cacf3c9efd09df409dc33654dd077e1c245ba8fb747b0f0236ef41b7c49b589 \ - --hash=sha256:60a64cbccaa11b7760ce705a14ada17ba459e7ca9f23ba587eb013821032d7ef \ - --hash=sha256:62530ca89187827e4a4fe733f971abe81a7542eeea48ff61995f19b64d7199c8 \ - --hash=sha256:62c60aec739ed00124573cce1178138fd690c7676352d67a37328c1cf51d7468 \ - --hash=sha256:69fc35c0779522da366c563e5faf203ffc1f8ff0021d5b1337fa4efa5be73177 \ - --hash=sha256:6af4693716bfb03f1752ef1b30faa593db2c01d5272e9b8564a1549452a979ab \ - --hash=sha256:6c7599df2b57ebeea8de011b5f2f7b85de95e76037d43d34b95e328430275487 \ - --hash=sha256:6e9368e87a3efc285e559131092c5db643eb8e56de4ee42064d5baec22ef2bb5 \ - --hash=sha256:6f0093ac3e9daada202c2082439d414a625c57184727a46e112a3fb2a81cb788 \ - --hash=sha256:7177c43eddf10a0893c4fec52ebb408fdcd7f7d63962caace9180d8f81b14ece \ - --hash=sha256:720cf832eb2d0b0dfee129cb3335a26f6ce3cc45ee1187e8f0731758caa16792 \ - --hash=sha256:770e8209d018175fc0063936fa9583b6d27e88c5ad31543f3383d66080efdd62 \ - --hash=sha256:7a8d5ff04eb1f85698a78d20c62a14676e7b960dcafde09a388d60ad377d355d \ - --hash=sha256:7b9100a93b372418d8688f3f2a3e5b45c64d70ca4d6176e121aca1e3bfc1e32f \ - --hash=sha256:7cc528e760a8af06f2b13e9b9f362cd90c7c718ea61228a96dbd31ba16ed7f47 \ - --hash=sha256:7ffafcbfc7b549ab940047e505c831eabac5e67de53e1bc174adbc5285c55944 \ - --hash=sha256:87a3caecf8095e48dc72f84bfa42e23a848cf410cc9cc13031fba4869b706a21 \ - --hash=sha256:886b59c4d28ca97dd23b025fdfc50a0356be934efbbbca89ad26230067f86fe5 \ - --hash=sha256:8876b39961e33d912afe3c1bee18ee564fdad0206f873cc15d522756b7f50737 \ - --hash=sha256:897d1ddf6716e8f47200f7aad9a0efa6cc7586df66c6defa572f9eab379c078e \ - --hash=sha256:8a1fc236528c457cd739c88abe823da851b7ab645d72792f88658114cc340c12 \ - --hash=sha256:8a235f73d6e020855dc29dff012d920c02ee0feab8d73a24185a7569f4be1161 \ - --hash=sha256:8f911c395cef73c510bac566da9507bb6a43e7763d0c79138dc60ee53f11207e \ - --hash=sha256:96f7c5c15656040ddcbc51e56dc59b58aa25999d743c126abd425b9766ab43e9 \ - --hash=sha256:978f28401afbc76cdc3df9e1717b4229a06b626a1dcc75db4e1f2beb3884c3e9 \ - --hash=sha256:98914de2c4d7f0f9f4a8c6ea4bf05841f4175796941e3ef7d47eb718f22311fb \ - --hash=sha256:9a2a8a50a93dee0268a860a07fa3b4bd968f8ce4dbd794957da772f395368526 \ - --hash=sha256:9cbfff4423eef4cc6cafc021469641a2b835f610b2647a6c5281903e21b8670d \ - --hash=sha256:9e9ab13760aa8b6d0881ae7cb04fd891d8d490cd2554ea8e79bb278399169bcc \ - --hash=sha256:9f3551b8a35c1df3e7ea4d2d86edee15f0dde1bddd434a71744048683544d0ef \ - --hash=sha256:9f86f7259efe2c951f43e57d471c9b41daa5bfc7db9f67189059cf1ae6d77fd9 \ - --hash=sha256:9fb0a5be8d9aa213150e8d8148a42aca4984b285bcad1e69587dc4298edd929b \ - --hash=sha256:a219f0ac59817a9114dd2aa57c13180f993e819ba658c7ddab4b66ed1ee0d370 \ - --hash=sha256:a419ee85e654927baabda3929c03c0cc1112bf472ff0dfd6142f4e3a81ca4162 \ - --hash=sha256:a4d7a54719b67338a305dca2ce6aafe366817df94ddfd4b5514374356f5ca546 \ - --hash=sha256:a5793c7698a53f56f4a1889a4737c7eeb1b7ad0842fa6b1abca22913ff79c8c1 \ - --hash=sha256:a5e8ef588c109725dc713ba69aadcac00a1ef90c2ce9c0a8c7075128f569f47f \ - --hash=sha256:a74bfa37147cc08fb29df10bd9c16f40fa7f860cd3a6d2fff853323a94f6e17f \ - --hash=sha256:ada748108a43d29b7c328ba7db3755327cd94f028bcc1a7ee3f0addcfacd9c38 \ - --hash=sha256:ae58f361bd5dae942717c65d3413b478c70aea9c462599e7b9adad3731db3894 \ - --hash=sha256:b28f41fa3b8c6900457f858ec5b03998f3a6d535fbc1bb2edec5961ea05ec429 \ - --hash=sha256:b3083bfe87f95c756e610bd8025f26cbd1cd4aaa03a422f2d65efb7a97cd53d8 \ - --hash=sha256:b61805357d966680acf68b3b6d49772631ed9df44ebece10ff1460e117a7da8a \ - --hash=sha256:b77c313314524ca9c38fbd70f73515d04597ac58c40c939bc0e71eeb4abff680 \ - --hash=sha256:bee7d3aed13d56f54e681df38c3a23031bc9e3863f687d9d598825c9146acd7d \ - --hash=sha256:c02c0e570c5c7e077b0181a9f3cdb7d4c3617d1cda6b5c95bd5d34022923d82c \ - --hash=sha256:c174bfd1c48a1b51a3078e95586dde718374bac79719ab3541ec9e74aec40574 \ - --hash=sha256:c2ba30a89035b57b73e00475de948521602f543d79ce01db10b04b36c4c76fc8 \ - --hash=sha256:c3e98c55bde2bcf7db3c70d1aed7ae9aa8aebbf19a250c66645cde44cdb8b867 \ - --hash=sha256:c3ef2818d63bc86071e9d2989ae75a1bc32b8f7059cfd9f5abbbee70c32e2ed6 \ - --hash=sha256:c83acbce9f2b5e3f5f5eda9e53d2001fed22fcdfef81274a9e02d8fd53b70a30 \ - --hash=sha256:c9281e922c072158c91974d4589f1dbe0fee6d467f284c28e463f9f5a4d933f4 \ - --hash=sha256:cc07876cfb079b6f6f36d21ce75784ad6c2c6b563eeac0ed26c2fa2669b85df9 \ - --hash=sha256:ccf4f7a79e26bb7efb06ecd50c177833b71df05cbc748701372325e6bcc17f6f \ - --hash=sha256:cdee8205a44d0be91bbac4c41b95d86641b72dfc7aef1279400e4fda3f26a937 \ - --hash=sha256:ceb3e879afac028f93d272c957814695dc5569e4904262dbee92f6c41bd5e4a3 \ - --hash=sha256:d1f5a500bfcbb2c0ab85e98a0dcd70f5899d34efe365a0187700369a79603031 \ - --hash=sha256:d42a9a856a4a6e2f6c10f1318c07e7daa498d6593abe745c71dae4521a26ca39 \ - --hash=sha256:d83b12902eb8bce151259c86c03ba746600b2d994543de46e370cecf96c452f2 \ - --hash=sha256:d8e017eeb7482bed34cdb0d61cf2bcfc88d104bbab296a17cd16a6af8aabc70e \ - --hash=sha256:db3ae52ccc150dbc84704e9d642743897f3e1c54742ff34cacb661e52e3818a9 \ - --hash=sha256:dbab5f5ff6897c81f355d079010cdae85b02e5a0b518b5251523b8ad8ae9ac3c \ - --hash=sha256:dc4242ca653c9b30ab51c5f8193323e7bc0928f897ee9103201e59a43abcb72e \ - --hash=sha256:dcbf346a318a5e30063f547630b02bb787ce2f45b6368d5da143660b6a3835d8 \ - --hash=sha256:dd2ac8f5b643454c2cc6b6118b13da16e88f4a6434fc3ba61aca384029f04f36 \ - --hash=sha256:e1d52a05dc417279f7e5c7618c5dfbbc29923aaf9bc0a5c1802ddcebf54c61a0 \ - --hash=sha256:e6720ba44ad7e72174314d0e1fb0172494cff5c73a3a8a2159c3d2402ff15565 \ - --hash=sha256:e738ab81179510ce79b2eac9a6ecf47feffd9e76d1c72e403005dddb6e36c06c \ - --hash=sha256:e904d4d01f36bd6e197590be1533c44e06058771e0746dd073a8ebb3ef880858 \ - --hash=sha256:e9f165403b81fea7e89c932d89046a1e3d9a3a60e8d7ef2f249dccdcb0982bf5 \ - --hash=sha256:ec6a85f424afa8d23e0d9a094e5dbb6eda01da91c92b9183cd433768247ffc97 \ - --hash=sha256:ee19113bce2f3acd46432050688b70f61acd6857d75abb9ec96341b7e9ced123 \ - --hash=sha256:ef3b928d9c984322b5c44e6964d8dbc653da87d2d8ee1647fa6da43072e650a9 \ - --hash=sha256:f273dcf7149a50527c4fd1f55cfe9eac0f60753f5af544b4c9352578e20c0874 \ - --hash=sha256:f5470694918830da62fac9e69133b53d23b736d7070e587b27a4a2be37e08e68 \ - --hash=sha256:f574e460d1c8a08384a016fdb09ccf3543433263ed6b2f97104f979e64ea57c2 \ - --hash=sha256:f85915e00dcb1cd9f2f890ead064ed40a27df06f0db65be427b29482ae357572 \ - --hash=sha256:fc2461ecc45f17893f8207e73b46ea8ba93e33630e51cf4af3fbc21d47462b1a \ - --hash=sha256:ff6b113f50bc066a698db5d944d2c6dc7507168dd3341e255a8892fd0715a558 - # via - # aiohttp - # yarl -proto-plus==1.28.4 \ - --hash=sha256:4b01341272f8a348db3f003b6143109f83ab43091019d5181b3fcdf500ab32aa \ - --hash=sha256:5ff7ecad828e032a491fcb86947801768e32237f99dd049b649965b892ae9a63 - # via - # google-api-core - # google-cloud-storage-control -protobuf==7.36.1 \ - --hash=sha256:0b53ce95272aad50ad25d7ff03373743209822e8ba42ea7fad27d2bee1547d00 \ - --hash=sha256:39c518c05586c016d7874ff6079ee115bcec1ea5fbb1d177fbf7867ef4c67e44 \ - --hash=sha256:3cf2ee25d006cee57294a1196ea43b37feb78e0dcd1e8af5c1aeddb777655aca \ - --hash=sha256:43d3d37b1eb24c113b9b7d02008cac44e423f00b611b7781ae998d7623972969 \ - --hash=sha256:51139351435d9b43d88a55eaa49fb6f737fbb478fb0cbf2cf694d1a04a9d3363 \ - --hash=sha256:7d951e46b3f963d6c264c367c437921de9d5aedd9c3f9612b9077736b4e3ad5c \ - --hash=sha256:97198b77e369a0abd8e262b8f6c7266c55ddb796a3a12c76d7b8881188ed83aa \ - --hash=sha256:d0f6470f0ce2b84e3feaea2d4b816378b37ba4d4aa08a274305373de93e2d524 - # via - # google-api-core - # google-cloud-storage-control - # googleapis-common-protos - # grpc-google-iam-v1 - # grpcio-status - # proto-plus - # xprof -pyasn1==0.6.4 \ - --hash=sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81 \ - --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b - # via pyasn1-modules -pyasn1-modules==0.4.2 \ - --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ - --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 - # via google-auth -pycparser==3.0 \ - --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ - --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 - # via cffi -requests==2.34.2 \ - --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ - --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed - # via - # gcsfs - # google-api-core - # google-cloud-storage - # requests-oauthlib -requests-oauthlib==2.0.0 \ - --hash=sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36 \ - --hash=sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9 - # via google-auth-oauthlib -setuptools==69.5.1 \ - --hash=sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987 \ - --hash=sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32 - # via xprof -six==1.17.0 \ - --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ - --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 - # via - # gviz-api - # xprof -termcolor==3.3.0 \ - --hash=sha256:348871ca648ec6a9a983a13ab626c0acce02f515b9e1983332b17af7979521c5 \ - --hash=sha256:cf642efadaf0a8ebbbf4bc7a31cec2f9b5f21a9f726f4ccbb08192c9c26f43a5 - # via fire -typing-extensions==4.16.0 \ - --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ - --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 - # via - # aiohttp - # aiosignal - # etils - # grpcio - # opentelemetry-api -urllib3==2.8.0 \ - --hash=sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3 \ - --hash=sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63 - # via - # requests - # xprof -werkzeug==3.1.8 \ - --hash=sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50 \ - --hash=sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44 - # via xprof -xprof==2.23.2 \ - --hash=sha256:38e809a29b8226183143d7adf59321f96f6b09ce5a0db0810162a2d9742187fa \ - --hash=sha256:921bc9b1d28ea78d7c63614e28b88dadbe535cf5b6c07c7b65a829a329635a9e \ - --hash=sha256:cdd3387cc357f08713df10b1032977d01c478690e7601f833de13a8ef763aa7b \ - --hash=sha256:d2b802594cf853c509e502a63403c481e607630a98d9ace4cb5fb275c707b5f5 - # via -r tools/xprof/requirements.in -yarl==1.25.1 \ - --hash=sha256:0136d640dfa9b0523853e411430a99f8a91eca85774c6420285a33b755bc6de3 \ - --hash=sha256:03dd38de09bc213e9a8b29761eec33ee1d5318dac0e49d8af36e4d27830e23a7 \ - --hash=sha256:0a191bfdb30a79b98e5d175d75285f9fcb78bf0e46ba5efda042e1c72071a0de \ - --hash=sha256:0a66db89ea473abeac4b70523cafd94db3772380e565f9d28af7a179b7af71fa \ - --hash=sha256:0ae12ff2b805fa02c4dab838005caef735e39986322698c48588d3beacb65c62 \ - --hash=sha256:0f12afda4eea8c8994a76d4df1875c765194f5fbe8a9d197929ea303caee29ec \ - --hash=sha256:10b2fd95332f0d716d5eee3c9fb2ce8eada19082de7fee83d32e37992fd75c26 \ - --hash=sha256:126a2533570c554719ca40a1288fdee1700b6bc82e7131aa69fa85252d92e651 \ - --hash=sha256:12b6bc4906e11f5e1a1cdcb12296e7afbd366c783cc8073403cd2fb74334e453 \ - --hash=sha256:142c06c4d6a35ee3ec5da08499805e879cb3ca7c1fbfbecb0140fe72403818d6 \ - --hash=sha256:14b79a30a93a3ce2e8832603fd0ab780ada281b0ba5110b519a634f2d7d7d1fc \ - --hash=sha256:17c9877a89fb6e2bca6f9087eb24cd7fb434653946ef5075e470d23d49b52287 \ - --hash=sha256:192a866877a49993949ef1975864ad8728bea28ee810f6abe1a0729c2b500426 \ - --hash=sha256:1ab7618921a93767387a4b83776f751588f5b5ae9bb5bc96620e2e2e00bca868 \ - --hash=sha256:1e80dcf1446e1b080b1932b0d103c464a04112f5bc31f0f983ad418172063cde \ - --hash=sha256:1f51020b2eb8a003c84925638ec63c21a750a4bddd3a22ec8eac6a742dadf1b9 \ - --hash=sha256:1fb2a01ba8cd9c5d2c5dc1ec35e0fc951d04b4f037541d4ac090c993ce58b3d7 \ - --hash=sha256:2239a02249d9326655419e0168a28ca9008938eaab31dc29fc875c217927a6c0 \ - --hash=sha256:23bf5b403c879a54964e0feac7285688e04bb220074878d737d331522da0a5bf \ - --hash=sha256:24ce942011a61953e7d313438038f4d32ff21387b775f58a957f7a07dd55ef95 \ - --hash=sha256:25868beca8b6765f8f7d0e11fe6dd7c66dd4b0793b9500286d20cc92352126a5 \ - --hash=sha256:287e99ff5aa4dc1c7630bfc683ded6f106d756c99dec432a2d7f197a784f51c6 \ - --hash=sha256:29273edf1530e397bd07cb784db1fbe0d2590b77569f2e24679a9c0a2d763b94 \ - --hash=sha256:2b49375d22299b0a834c2bca72f39aaecc270d96fb24c30424899676f487b22a \ - --hash=sha256:30eec96e8a91bd588ce897c9543f6d5d8d34b28fbcba28a4dedf20ebeae9fe57 \ - --hash=sha256:319e070a01db9920fb63761843f96a104c8e2b9427266731810dc1e22595b17c \ - --hash=sha256:35dcbea443fafb3eece757ad4e514560ddeb6c34cfae1582c620d7b293d7feee \ - --hash=sha256:3f4d48a6112712973e676bd792121fee470e432d749177162d9949d5c9460a1b \ - --hash=sha256:3feb99222553a8cbedfa52c2f59dd84c3f50d5b582c728d522caf8d72769a54b \ - --hash=sha256:419f392a1da624877975709e3864dfe833af6cc7671b39318086d456e288380c \ - --hash=sha256:42a66563d8cc056ee32e6191e05097a7b2b3bc302e0bc3133daf8710eb18bd26 \ - --hash=sha256:48796ea00a303961507dc6c8437c4b325a6fc3f95f7c36c71b91ea9a8150963c \ - --hash=sha256:4bd6340d20ae2c7ca719b87b426e808e90743b676d05d4c26c4fb5ca71f41184 \ - --hash=sha256:4ca89e4e21854ed27ec753297dde84b16c9f8e53b14a4866fb44457d643c19f8 \ - --hash=sha256:4d781294bb815ecb5ea57ff6bbf8038e0a31a95fdf3e1788f66e0dc100d64b58 \ - --hash=sha256:4f1c91f5a5980a937ff8e238e98e6897e1ad74a4b1e2c0d68c73b5ffbb3f5c0b \ - --hash=sha256:564fdc7085d2245ab84f88882fdb1d6ac0723124bff6ded35bfb1c00f812630d \ - --hash=sha256:59ba3a6e1aa8cfe5adf4bd270fd965db21955401b7ca6f1696010c55ed4daec2 \ - --hash=sha256:5df89f769cc8ff94c3d7e7603386fba309d25ce5240132d26c15baa8d0e96c4c \ - --hash=sha256:632da579b2d879f6bad20f2cfa35ded1efe2f4f77f8abb26a6234a5b236acd2f \ - --hash=sha256:65b5b2066651b7432d389e9799d979c703bcc6ef44266bb8153ef54e91e4aab3 \ - --hash=sha256:664ec6a520b74a1df2810666eb67695fcb77fa663e6ea0a25aaf2e529cb24dfa \ - --hash=sha256:681c758b0490f9e96b78e5fa8e8dc6e648e9185bb6eaebe73183c33ea0c445f3 \ - --hash=sha256:683e362b8ba453080f7489c66f4ea794e751c35b72e7eab3575ef784c2fbc7fb \ - --hash=sha256:68782fdb4027b8d1eee25ec35e9a6db05e863b899eb0310b3a33b6c3fef55707 \ - --hash=sha256:6efaf45df6a849cef613a03a94c845647456662f85438c886bb67a9c027c8c2c \ - --hash=sha256:71f42c5b9a948c113bbdebfa544598321431d064ff959d32e99b1feb61d68345 \ - --hash=sha256:72849d892954be4d09e569b8b831ac39ce58417fedc767d4308a0fe542018a40 \ - --hash=sha256:72c34ac7ad4314c19362d5ce27626dcc8429bd30bbf8c179f4234078851f9492 \ - --hash=sha256:734f6e5400352ac4254456003d462866c684703570929cff7a7bde015d0cb371 \ - --hash=sha256:75baa6cf9b6d1c52f3e111a130e202fd8cf0a5b3a066c3f73d615e885092e4ec \ - --hash=sha256:77716e245c90f058466a05e6a465bb8600f767a8f4b18b4d40f3aff958e5f73c \ - --hash=sha256:77e5099b99b37f3cf79c246998ca9f7313a78054cd1809ec46bc1afad47e1c4c \ - --hash=sha256:783dd1467083f4d3f7722ad6a313f24c173e7571372738fcb7a6e6d1ba48df25 \ - --hash=sha256:7a5c3115595995779ee21f2567035793911c3802a43c74f3fbb0314929ec67ac \ - --hash=sha256:7c88edaec8c349ad4c5ad4c486a3defcc4b80ceb2f074436ffa0a87caf5e76a6 \ - --hash=sha256:7cb414a73e21a7ab58254926073f2930cb22f5b4314ea4260a687e2b3fd4dce3 \ - --hash=sha256:7d42e7e3ca399555578b4d617e3a6ecf13371b3743a115995fa010c7bf341459 \ - --hash=sha256:7d575b54cb3863ef9bc290ea4b009999d55dc237326131e4853cf33e888fee03 \ - --hash=sha256:7e4de3ac4adbad3d0bc7c6f4360a7dbff5de2f15e3b723be3198074e17fd9c40 \ - --hash=sha256:7efc9f082dfed77c316edffa9deb52888e1bc6789171887cc1f68e06d65465c8 \ - --hash=sha256:80a063f8297fc796296f00f100be520f209b23dc98f93ce8eba6ee7122598209 \ - --hash=sha256:80e47012e730da131c9f059c80936783f9659aae22dc31c03c0595590d11ed54 \ - --hash=sha256:83d4a37e4b95da4d8bda930d6d35b75b4cdadbacbb4980cae290ea3100b5d51d \ - --hash=sha256:83e9f4a25085bd4b7214701a0794ff1f50fc633ffb8bdfebf07abdd81c2db126 \ - --hash=sha256:85a18376073f8a39aa07be34f9fc77e2869aa72c55c441efdd2cf79a0407504d \ - --hash=sha256:87796fedc3ba97ec14fab55acb48584276e6c1e4c1e89c422bda62c838e754a9 \ - --hash=sha256:882569ff613758cac762a457a5d72d6e211b28d4bcfea89d1d71ea942b02eac0 \ - --hash=sha256:8ce4d6ccafb33d39bd78444612d14938ead674c25702ded2ee9c54a47735d225 \ - --hash=sha256:8e7d98cdbb6d71e726f7d525952867096053d1f290dd4e3c50d7d313a136f414 \ - --hash=sha256:8ee202350cf57abf0e9502a41601841019c25d3db7ff52d980aaf31446254059 \ - --hash=sha256:8fb0eb4955adf0579001581f2f71a126e8781ba61bcd120f127b0401163c6c2d \ - --hash=sha256:90c30ed53546da833c700115c0064c22120d1b1560f474699fd31f22dd668233 \ - --hash=sha256:9489e6abf47ba37f332075a91444c7cfedb03e6ce99fbb2f116bfe1ce810da3b \ - --hash=sha256:94d7aa6debf92a1dd14cb5280b083a764169a13cfb23a452111160274ed989f4 \ - --hash=sha256:98d370568f393215d605304cdb77b3d5539bd192c75b623c7304c42c8d6d8273 \ - --hash=sha256:9b1bdaae98bc016825dd3c9d8ee1832f829b3341f9cc6ebd1a1b0a7fef7367cc \ - --hash=sha256:9d693bf4bf534e9ba3ae2780cfd577f5135629f7b5ac653490859d0b77864865 \ - --hash=sha256:9d6ed3d17bccce4c05343e1ca8da13bc5c02c812a4e7282ddd05e8769322d3fc \ - --hash=sha256:9e23c82b63cd7652fc24d33ed6cc17099d607aa3b4fc4ddc75e95062f3d82df4 \ - --hash=sha256:a1daf47cd95a7c3a63456336bc5aaa8c86dd3a47d07ed3d0e76132ae4666a5a1 \ - --hash=sha256:a1e32763e641a1566507d90a8d3b19bfc3cc04a9d4e5ae3e32189874ed4b58a3 \ - --hash=sha256:a2059a2d891bd156bc5184e7ab7a56e78a84dfcfdeac8c501b552533ad1c36ee \ - --hash=sha256:a2ed0ba415ccdf08f14bf544cb78346d0f76086707ffee24921a2c84dbf1305a \ - --hash=sha256:a3faadac7d812ddac258feb57b9846b60c1b437c4f4b9ad42595c6f6fe4390df \ - --hash=sha256:a5877f2255aab518ebe528289037699201d5dc5f045f2396cb30aa02db22f57f \ - --hash=sha256:a78b50b4f7918a3de71105d5c0b93bbc57bb8339a4d03a9dfd449f9068e76f3d \ - --hash=sha256:a8c2b841478068440d8b733005d13a5ef535b9928cbc05f17182d410f32ba449 \ - --hash=sha256:a9ca696eb02e5c02a8afd872ada510eba9b7fe6e68b9572c2e9a9b1941e31e2e \ - --hash=sha256:aa4ed3dd308548f9e707d9caaf005d2d7f8c1e7868f858dfeb47fe76e16b391d \ - --hash=sha256:ab2054c5531af2a9ba7b69b8ec91e4f884420e83a8c5e579b013084cb57e5e5d \ - --hash=sha256:abb1384477f5901d436b5d2e5465954de46ea6098f59163d243660b5c4461d35 \ - --hash=sha256:acae6b45d1ace09b6ba3876da43b88366ef368f73b988c7f57e14231753d4420 \ - --hash=sha256:acfa7e22aa6c6e7a5996a41d275bfa01efa7ea56ab890590280e9063e2cf5c1b \ - --hash=sha256:af4ea5b37403ef4e30f3927eaed540db942bde01d8d3ff083527c0704d1c9c68 \ - --hash=sha256:b10dd0557ba422715b5206b3743192135a6022acca8baec51aa127d0a75db8fe \ - --hash=sha256:b13b88747769537f3d32e89e3a735da10c0a9e35d7322928c701b5f93d3afffd \ - --hash=sha256:b51c159a9794633f5e0db7ecec7b2b6e3734eca1f5d17dc989ff3552a43ff78b \ - --hash=sha256:b5402a340723fa7da00b5cff987ddab61276be6d11251ea71ae02bcac54890d8 \ - --hash=sha256:b7abffdf37af1cec6a2ad69b827aa84320db5894791bc8ed932dc93fb274b7e9 \ - --hash=sha256:b8075fe90bc08e40b8b8a1874fab42ee4c7b56af05c5886e9cc841397f916908 \ - --hash=sha256:bc3ac7bf569f6b64dad04dd7808c7872dae8a97df657856eac05e9b7e3614a85 \ - --hash=sha256:bd0912757081f89b107d6c00b2ff8a194401b0b87eadcf4481de2b865a8fd44f \ - --hash=sha256:bdc8d8b8c22e9e43ac68316b5e6cf083dec537f4ec213cb4aa967b583bc3fa64 \ - --hash=sha256:be80550d9bfe83d9b62398a37081a90434e6df2d978ec345c3d2820de6beddab \ - --hash=sha256:c6f117789d22dce188e5754e8bc65b7e6ebf8cb73963b9fa761f672a5883769d \ - --hash=sha256:ca32926d7d77bcc8838425c4c95e040a3ace1cb7dfdae599013458dcda2607ca \ - --hash=sha256:cce0727fd5ac04d372fa9bbfde9febc2bcf209aadfcf0468e45dec72719895d1 \ - --hash=sha256:d0f1489233a254bb3643d2f05de7d59019254d81daeca6b9162fe9edef57e0c7 \ - --hash=sha256:d1c557dfd5e3db046053a0bdc72261ade790ebe8e2c7a41b36b0ca1f14cb95f3 \ - --hash=sha256:d21f0fa80a02d05299207eeaafef345d812ace96d5306e4ef265e1d419a615fa \ - --hash=sha256:d45673badd08456d0340e9364eddafe1c53a9d2896424294de4d7dd71ad3ee57 \ - --hash=sha256:d5add7b4ca7afeea91d52e4d4e4db3b1fe9885b71f07054560d8c4296b7441a2 \ - --hash=sha256:d5f90e44653c4e0f78501ed9bb7d3fce835a8d62b7c6ed0cb16557534087e743 \ - --hash=sha256:d7306dee25b8a0e737363f347362b875094b4dc4e367311470656ae420fdbf8e \ - --hash=sha256:dbcef5a9119ef653653132cccaf999b30a0af6f33bb0a4ba80bec30056868487 \ - --hash=sha256:def538065f9e4d4cf1ae164bd59aba00dfa84f03923e0de4c3788f252d6bcd17 \ - --hash=sha256:df23df54b5114a17c2d0ef192433e2e5a9f0c5178c32375e90b7cfc965f349d0 \ - --hash=sha256:dfbf531053a0935f2e871bcd4753f90313688772ff8c017f5ea402e315a78c1f \ - --hash=sha256:e029648f9c951db30e98a7d7ec90835db88ec4b32820efe2a9bdc2287e032eb6 \ - --hash=sha256:e07595c7d6f4db270ceede356a1bd1c07a34f1c26f958d1ed0cd7b48e0d2bba3 \ - --hash=sha256:e12c538e00e7c1b286a07061046b90e8124e6a9793efae2c70db6a4aad07faad \ - --hash=sha256:e546fe1d4a93ebc2910f0d768baff19faa09843ab3f2036a67ed6e69fae4419d \ - --hash=sha256:e5637ca8d0bd7fb72648a6c7934af4baaccb697657f7438c9d264fc2abb8b0b1 \ - --hash=sha256:e636b64d24fd9c38053c5e389a1174c66361fa49dcfd220f4dd35b4abde7cb89 \ - --hash=sha256:e7011b8fb8c4054bf0c12e5edc6cd83778b0028e99ce59b18586ed036f92cfdc \ - --hash=sha256:e80f557716fd765439577131e526b8942ffc2c07bdbc5e39fa62f660ba1e963f \ - --hash=sha256:e92b6bcc741b86d67606c40d3cb9c7cc8e6c737f81e31f4a94efc204456c92e3 \ - --hash=sha256:eb96ed1ae6c7d072d60840c0434aef07a2df611812810807fbc54263a6053e9a \ - --hash=sha256:eda19ea5ee88742f47a2340816e6f2d40b53bed3ab5b69794769f36af9f35bb4 \ - --hash=sha256:ef74070ac553c59eb4f04258722066d6c6135b7baa03b2e9f2da65c096e96d98 \ - --hash=sha256:efb01a106f971cb3752856bca2318bbdf7f01bd8823779c461586cbe5ffd5258 \ - --hash=sha256:f074e8d4aa0a5798920ddb6de3d08b228c614ff3724c3e8bd7577f4bafea867b \ - --hash=sha256:f38a70074041d3b7e138e452799f5174198bae5bd5ab2000917badf403908c5f \ - --hash=sha256:f41753a76f4f63927d03a0d8ba8f5ce0f2083bec29a8cfaccc55371b1564b96b \ - --hash=sha256:f53dcd26694f148f738edc052b5a69234833e739f10f4c3287bdfd8ec0f7b326 \ - --hash=sha256:f61964f235a43738bfac50da46fc4254943a7eea3051aeb0b6fc7c992c29fadc \ - --hash=sha256:fe01645169a2112aa1d4ebc3e4c5f029c5c8f97adfc32e5d37c993b39a994d75 - # via aiohttp -zipp==4.1.0 \ - --hash=sha256:25ad4e16390cd314347dd8f1de67a2ac538ae658ed4ab9db16029c07c188e97f \ - --hash=sha256:4cb57381f544315db7688e976e922a2b18cdb513d21cc194eb42232ba2a3e602 - # via etils diff --git a/tools/xspace_to_perfetto/BUILD.bazel b/tools/xspace_to_perfetto/BUILD.bazel deleted file mode 100644 index 5de164d6..00000000 --- a/tools/xspace_to_perfetto/BUILD.bazel +++ /dev/null @@ -1,32 +0,0 @@ -load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_library") - -upb_c_proto_library( - name = "xplane_upb", - deps = ["@tsl//tsl/profiler/protobuf:xplane_proto"], - tags = ["manual"], -) - -zig_library( - name = "xspace_to_perfetto_lib", - tags = ["manual"], - import_name = "tools/xspace_to_perfetto", - main = "xspace_to_perfetto.zig", - visibility = ["//visibility:public"], - zigopts = ["-lc"], - deps = [ - ":xplane_upb", - "//upb", - ], -) - -zig_binary( - name = "xspace_to_perfetto", - main = "cli.zig", - visibility = ["//visibility:public"], - zigopts = ["-lc"], - deps = [ - ":xspace_to_perfetto_lib", - "//stdx", - ], -) diff --git a/tools/xspace_to_perfetto/cli.zig b/tools/xspace_to_perfetto/cli.zig deleted file mode 100644 index f338855a..00000000 --- a/tools/xspace_to_perfetto/cli.zig +++ /dev/null @@ -1,34 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); -const xspace_to_perfetto = @import("tools/xspace_to_perfetto"); - -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; - - var threaded: std.Io.Threaded = .init(allocator, .{}); - defer threaded.deinit(); - const io = threaded.io(); - - var upb_arena = std.heap.ArenaAllocator.init(allocator); - defer upb_arena.deinit(); - - const args = stdx.flags.parseProcessArgs(init.minimal, xspace_to_perfetto.Args); - const input_path = args.positional.input; - const output_path = args.positional.output orelse try std.fmt.allocPrint(allocator, "{s}.trace.json", .{input_path}); - defer if (args.positional.output == null) allocator.free(output_path); - - const stat = try std.Io.Dir.statFile(.cwd(), io, input_path, .{}); - - var file = try std.Io.Dir.openFile(.cwd(), io, input_path, .{ .mode = .read_only }); - defer file.close(io); - - const xspace = try allocator.alloc(u8, stat.size); - defer allocator.free(xspace); - _ = try file.readPositionalAll(io, xspace, 0); - - try xspace_to_perfetto.dumpXSpaceProtoToTraceJsonFile(allocator, io, xspace, output_path); - - var stdout = std.Io.File.stdout().writer(io, &.{}); - try stdout.interface.print("{s}\n", .{output_path}); -} diff --git a/tools/xspace_to_perfetto/xspace_to_perfetto.zig b/tools/xspace_to_perfetto/xspace_to_perfetto.zig deleted file mode 100644 index 46aec099..00000000 --- a/tools/xspace_to_perfetto/xspace_to_perfetto.zig +++ /dev/null @@ -1,712 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const upb = @import("upb"); - -pub const Args = struct { - positional: struct { - input: []const u8, - output: ?[]const u8 = null, - }, - - pub const help = - \\Usage: xspace_to_perfetto [output_trace.json] - \\ - \\Converts an XSpace protobuf profile into trace-viewer JSON. - \\ - ; -}; - -// XPlane schema source: -// ../../xla/third_party/tsl/tsl/profiler/protobuf/xplane.proto -const kHostThreadsPlaneName = "/host:CPU"; -const kGpuPlanePrefix = "/device:GPU:"; -const kTpuPlanePrefix = "/device:TPU:"; -const kCustomPlanePrefix = "/device:CUSTOM:"; -const kXlaAsyncOpLineName = "Async XLA Ops"; -const kOpaqueBytesText = ""; - -const kPicosecondsPerNanosecond: u64 = 1_000; -const kPicosecondsPerMicrosecond: f64 = 1_000_000.0; -const kOutputBufferSizeBytes = 1024 * 1024; -const kHexDigits = "0123456789abcdef"; - -const kFirstDeviceId: u32 = 1; -const kLastDeviceId: u32 = 500; -const kFirstCustomPlaneDeviceId: u32 = kLastDeviceId + 1; -const kMaxCustomPlaneDevicesPerHost: u32 = 200; -const kLastCustomPlaneDeviceId: u32 = kFirstCustomPlaneDeviceId + kMaxCustomPlaneDevicesPerHost - 1; -const kHostThreadsDeviceId: u32 = kLastCustomPlaneDeviceId + 1; - -const StringValue = union(enum) { - text: []const u8, - int64: i64, - uint64: u64, - double: f64, - opaque_bytes, -}; - -const Arg = struct { - name: []const u8, - value: StringValue, -}; - -const StatBehavior = enum { - emit, - step_name, - skip, -}; - -const JsonArrayWriter = struct { - needs_comma: bool = false, - - fn nextItem(self: *JsonArrayWriter, writer: *std.Io.Writer) !void { - if (self.needs_comma) { - try writer.writeByte(','); - } - self.needs_comma = true; - } -}; - -const EventScratch = struct { - args: std.ArrayListUnmanaged(Arg) = .empty, - - fn reset(self: *EventScratch) void { - self.args.clearRetainingCapacity(); - } - - fn deinit(self: *EventScratch, allocator: std.mem.Allocator) void { - self.args.deinit(allocator); - } -}; - -const PlaneContext = struct { - plane: *const c.tensorflow_profiler_XPlane, - device_id: u32, - is_host: bool, - event_metadata_by_id: std.AutoHashMapUnmanaged(i64, *const c.tensorflow_profiler_XEventMetadata) = .{}, - stat_metadata_by_id: std.AutoHashMapUnmanaged(i64, *const c.tensorflow_profiler_XStatMetadata) = .{}, - event_is_internal_by_id: std.AutoHashMapUnmanaged(i64, bool) = .{}, - stat_behavior_by_id: std.AutoHashMapUnmanaged(i64, StatBehavior) = .{}, - - // Metadata caching mirrors the role of XPlaneVisitor lookups in: - // ../../xla/xla/tsl/profiler/utils/xplane_visitor.h - // ../../xla/xla/tsl/profiler/utils/xplane_visitor.cc - fn init(plane: *const c.tensorflow_profiler_XPlane, device_id: u32, is_host: bool) PlaneContext { - return .{ - .plane = plane, - .device_id = device_id, - .is_host = is_host, - }; - } - - fn deinit(self: *PlaneContext, allocator: std.mem.Allocator) void { - self.event_metadata_by_id.deinit(allocator); - self.stat_metadata_by_id.deinit(allocator); - self.event_is_internal_by_id.deinit(allocator); - self.stat_behavior_by_id.deinit(allocator); - } - - fn getEventMetadata( - self: *PlaneContext, - allocator: std.mem.Allocator, - metadata_id: i64, - ) !?*const c.tensorflow_profiler_XEventMetadata { - if (self.event_metadata_by_id.get(metadata_id)) |metadata| { - return metadata; - } - - var metadata: ?*c.tensorflow_profiler_XEventMetadata = null; - if (!c.tensorflow_profiler_XPlane_event_metadata_get(self.plane, metadata_id, &metadata)) { - return null; - } - const resolved = metadata orelse return null; - const result: *const c.tensorflow_profiler_XEventMetadata = resolved; - try self.event_metadata_by_id.put(allocator, metadata_id, result); - return result; - } - - fn getStatMetadata( - self: *PlaneContext, - allocator: std.mem.Allocator, - metadata_id: i64, - ) !?*const c.tensorflow_profiler_XStatMetadata { - if (self.stat_metadata_by_id.get(metadata_id)) |metadata| { - return metadata; - } - - var metadata: ?*c.tensorflow_profiler_XStatMetadata = null; - if (!c.tensorflow_profiler_XPlane_stat_metadata_get(self.plane, metadata_id, &metadata)) { - return null; - } - const resolved = metadata orelse return null; - const result: *const c.tensorflow_profiler_XStatMetadata = resolved; - try self.stat_metadata_by_id.put(allocator, metadata_id, result); - return result; - } - - fn isInternalEvent( - self: *PlaneContext, - allocator: std.mem.Allocator, - metadata_id: i64, - ) !bool { - if (self.event_is_internal_by_id.get(metadata_id)) |is_internal| { - return is_internal; - } - - const is_internal = if (try self.getEventMetadata(allocator, metadata_id)) |metadata| - isInternalEventName(stringView(c.tensorflow_profiler_XEventMetadata_name(metadata))) - else - false; - try self.event_is_internal_by_id.put(allocator, metadata_id, is_internal); - return is_internal; - } - - fn statBehavior( - self: *PlaneContext, - allocator: std.mem.Allocator, - metadata_id: i64, - ) !StatBehavior { - if (self.stat_behavior_by_id.get(metadata_id)) |behavior| { - return behavior; - } - - const behavior = if (try self.getStatMetadata(allocator, metadata_id)) |metadata| - classifyStatName(stringView(c.tensorflow_profiler_XStatMetadata_name(metadata))) - else - .emit; - try self.stat_behavior_by_id.put(allocator, metadata_id, behavior); - return behavior; - } -}; - -pub fn dumpXSpaceProtoToTraceJsonFile( - allocator: std.mem.Allocator, - io: std.Io, - xspace_proto: []const u8, - output_path: []const u8, -) !void { - var upb_backing_arena = std.heap.ArenaAllocator.init(allocator); - defer upb_backing_arena.deinit(); - - var upb_allocator: upb.Allocator = .init(upb_backing_arena.allocator()); - const upb_arena = c.upb_Arena_Init(null, 0, upb_allocator.inner()); - defer c.upb_Arena_Free(upb_arena); - - const xspace = c.tensorflow_profiler_XSpace_parse(xspace_proto.ptr, xspace_proto.len, upb_arena) orelse - return error.MalformedXSpace; - try writeTraceJsonFile(allocator, io, xspace, output_path); -} - -fn writeTraceJsonFile( - allocator: std.mem.Allocator, - io: std.Io, - xspace: *const c.tensorflow_profiler_XSpace, - output_path: []const u8, -) !void { - var output_file = try std.Io.Dir.createFile(.cwd(), io, output_path, .{}); - defer output_file.close(io); - - const output_buffer = try allocator.alloc(u8, kOutputBufferSizeBytes); - defer allocator.free(output_buffer); - - var file_writer = output_file.writer(io, output_buffer); - defer file_writer.interface.flush() catch {}; - - try streamXSpaceToTraceJson(allocator, xspace, &file_writer.interface); - try file_writer.interface.flush(); -} - -fn streamXSpaceToTraceJson( - allocator: std.mem.Allocator, - xspace: *const c.tensorflow_profiler_XSpace, - writer: *std.Io.Writer, -) !void { - // Keep the same trace JSON envelope as the streaming C++ implementation in: - // ../../xla/xla/tsl/profiler/convert/trace_events_to_json.cc - var planes = try collectSelectedPlanes(allocator, xspace); - defer { - for (planes.items) |*plane| { - plane.deinit(allocator); - } - planes.deinit(allocator); - } - - try writer.writeAll("{\"displayTimeUnit\":\"ns\",\"metadata\":{\"highres-ticks\":true},\"traceEvents\":["); - var array_writer: JsonArrayWriter = .{}; - - for (planes.items) |*plane| { - try streamPlaneMetadata(writer, &array_writer, plane); - } - for (planes.items) |*plane| { - try streamPlaneEvents(allocator, writer, &array_writer, plane); - } - - try writer.writeAll("]}"); -} - -fn collectSelectedPlanes( - allocator: std.mem.Allocator, - xspace: *const c.tensorflow_profiler_XSpace, -) !std.ArrayListUnmanaged(PlaneContext) { - // Plane selection intentionally follows: - // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc - var result: std.ArrayListUnmanaged(PlaneContext) = .empty; - errdefer { - for (result.items) |*plane| { - plane.deinit(allocator); - } - result.deinit(allocator); - } - - var plane_count: usize = 0; - const planes = c.tensorflow_profiler_XSpace_planes(xspace, &plane_count) orelse return result; - const plane_slice = planes[0..plane_count]; - - var host_plane: ?*const c.tensorflow_profiler_XPlane = null; - for (plane_slice) |plane_c| { - const plane: *const c.tensorflow_profiler_XPlane = plane_c orelse continue; - if (std.mem.eql(u8, stringView(c.tensorflow_profiler_XPlane_name(plane)), kHostThreadsPlaneName)) { - host_plane = plane; - break; - } - } - - if (host_plane) |plane| { - try result.append(allocator, PlaneContext.init(plane, kHostThreadsDeviceId, true)); - } - - if (try appendPlanesWithPrefix(allocator, &result, plane_slice, kGpuPlanePrefix)) { - return result; - } - if (try appendPlanesWithPrefix(allocator, &result, plane_slice, kTpuPlanePrefix)) { - return result; - } - _ = try appendPlanesWithPrefix(allocator, &result, plane_slice, kCustomPlanePrefix); - return result; -} - -fn appendPlanesWithPrefix( - allocator: std.mem.Allocator, - result: *std.ArrayListUnmanaged(PlaneContext), - planes: []const [*c]const c.tensorflow_profiler_XPlane, - prefix: []const u8, -) !bool { - var found = false; - for (planes) |plane_c| { - const plane: *const c.tensorflow_profiler_XPlane = plane_c orelse continue; - const name = stringView(c.tensorflow_profiler_XPlane_name(plane)); - if (!std.mem.startsWith(u8, name, prefix)) continue; - found = true; - const device_ordinal = wrapI64ToU32(c.tensorflow_profiler_XPlane_id(plane)); - try result.append(allocator, PlaneContext.init(plane, kFirstDeviceId + device_ordinal, false)); - } - return found; -} - -fn streamPlaneMetadata( - writer: *std.Io.Writer, - array_writer: *JsonArrayWriter, - plane: *const PlaneContext, -) !void { - // Metadata event shape matches the resource/process metadata emitted in: - // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc - const plane_name = stringView(c.tensorflow_profiler_XPlane_name(plane.plane)); - if (plane_name.len != 0) { - try array_writer.nextItem(writer); - try writer.writeAll("{\"ph\":\"M\",\"pid\":"); - try writer.print("{d}", .{plane.device_id}); - try writer.writeAll(",\"name\":\"process_name\",\"args\":{\"name\":"); - try writeJsonString(writer, plane_name); - try writer.writeAll("}}"); - } - - try array_writer.nextItem(writer); - try writer.writeAll("{\"ph\":\"M\",\"pid\":"); - try writer.print("{d}", .{plane.device_id}); - try writer.writeAll(",\"name\":\"process_sort_index\",\"args\":{\"sort_index\":"); - try writer.print("{d}", .{plane.device_id}); - try writer.writeAll("}}"); - - var ordinal: u32 = 0; - var line_count: usize = 0; - const lines = c.tensorflow_profiler_XPlane_lines(plane.plane, &line_count) orelse return; - for (lines[0..line_count]) |line_c| { - const line: *const c.tensorflow_profiler_XLine = line_c orelse continue; - const resource_id = try lineResourceId(line); - const resource_name = lineDisplayName(line); - const sort_index = if (plane.is_host) blk: { - ordinal += 1; - break :blk ordinal; - } else resource_id; - try streamResourceMetadata(writer, array_writer, plane.device_id, resource_id, resource_name, sort_index); - } -} - -fn streamResourceMetadata( - writer: *std.Io.Writer, - array_writer: *JsonArrayWriter, - device_id: u32, - resource_id: u32, - resource_name: []const u8, - sort_index: u32, -) !void { - if (resource_name.len != 0) { - try array_writer.nextItem(writer); - try writer.writeAll("{\"ph\":\"M\",\"pid\":"); - try writer.print("{d}", .{device_id}); - try writer.writeAll(",\"tid\":"); - try writer.print("{d}", .{resource_id}); - try writer.writeAll(",\"name\":\"thread_name\",\"args\":{\"name\":"); - try writeJsonString(writer, resource_name); - try writer.writeAll("}}"); - } - - try array_writer.nextItem(writer); - try writer.writeAll("{\"ph\":\"M\",\"pid\":"); - try writer.print("{d}", .{device_id}); - try writer.writeAll(",\"tid\":"); - try writer.print("{d}", .{resource_id}); - try writer.writeAll(",\"name\":\"thread_sort_index\",\"args\":{\"sort_index\":"); - try writer.print("{d}", .{sort_index}); - try writer.writeAll("}}"); -} - -fn streamPlaneEvents( - allocator: std.mem.Allocator, - writer: *std.Io.Writer, - array_writer: *JsonArrayWriter, - plane: *PlaneContext, -) !void { - var scratch: EventScratch = .{}; - defer scratch.deinit(allocator); - - var line_count: usize = 0; - const lines = c.tensorflow_profiler_XPlane_lines(plane.plane, &line_count) orelse return; - for (lines[0..line_count]) |line_c| { - const line: *const c.tensorflow_profiler_XLine = line_c orelse continue; - if (std.mem.eql(u8, lineDisplayName(line), kXlaAsyncOpLineName)) continue; - - const resource_id = try lineResourceId(line); - var event_count: usize = 0; - const events = c.tensorflow_profiler_XLine_events(line, &event_count) orelse continue; - for (events[0..event_count]) |event_c| { - const event: *const c.tensorflow_profiler_XEvent = event_c orelse continue; - try streamEvent(allocator, writer, array_writer, plane, line, event, resource_id, &scratch); - } - } -} - -fn streamEvent( - allocator: std.mem.Allocator, - writer: *std.Io.Writer, - array_writer: *JsonArrayWriter, - plane: *PlaneContext, - line: *const c.tensorflow_profiler_XLine, - event: *const c.tensorflow_profiler_XEvent, - resource_id: u32, - scratch: *EventScratch, -) !void { - const metadata_id = c.tensorflow_profiler_XEvent_metadata_id(event); - if (try plane.isInternalEvent(allocator, metadata_id)) return; - - // Display name, long_name, stat merge order, and step_name override mirror: - // ../../xla/xla/tsl/profiler/convert/xplane_to_trace_events.cc - const metadata = try plane.getEventMetadata(allocator, metadata_id); - const metadata_name = if (metadata) |value| stringView(c.tensorflow_profiler_XEventMetadata_name(value)) else ""; - const metadata_display_name = if (metadata) |value| stringView(c.tensorflow_profiler_XEventMetadata_display_name(value)) else ""; - - var event_name: StringValue = .{ - .text = if (metadata_display_name.len != 0) metadata_display_name else metadata_name, - }; - - scratch.reset(); - const args = &scratch.args; - - if (metadata_display_name.len != 0) { - try putArg(args, allocator, "long_name", .{ .text = metadata_name }); - } - - if (metadata) |value| { - try appendStats(allocator, plane, args, &event_name, metadataStats(value)); - } - try appendStats(allocator, plane, args, &event_name, eventStats(event)); - - std.mem.sort(Arg, args.items, {}, argLessThan); - - try array_writer.nextItem(writer); - try writer.writeAll("{\"ph\":\"X\",\"pid\":"); - try writer.print("{d}", .{plane.device_id}); - try writer.writeAll(",\"tid\":"); - try writer.print("{d}", .{resource_id}); - try writer.writeAll(",\"ts\":"); - var number_buf: [64]u8 = undefined; - try writer.writeAll(formatPicosToMicros(&number_buf, eventTimestampPs(line, event))); - try writer.writeAll(",\"dur\":"); - try writer.writeAll(formatPicosToMicros(&number_buf, eventDurationPs(event))); - try writer.writeAll(",\"name\":"); - try writeJsonStringValue(writer, event_name); - - if (args.items.len != 0) { - try writer.writeAll(",\"args\":{"); - for (args.items, 0..) |arg, index| { - if (index != 0) try writer.writeByte(','); - try writeJsonString(writer, arg.name); - try writer.writeByte(':'); - try writeJsonStringValue(writer, arg.value); - } - try writer.writeByte('}'); - } - - try writer.writeByte('}'); -} - -fn appendStats( - allocator: std.mem.Allocator, - plane: *PlaneContext, - args: *std.ArrayListUnmanaged(Arg), - event_name: *StringValue, - stats: []const [*c]const c.tensorflow_profiler_XStat, -) !void { - // Stat filtering and name lookup correspond to: - // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc - for (stats) |stat_c| { - const stat: *const c.tensorflow_profiler_XStat = stat_c orelse continue; - const value_case = c.tensorflow_profiler_XStat_value_case(stat); - if (value_case == c.tensorflow_profiler_XStat_value_NOT_SET) continue; - - const metadata_id = c.tensorflow_profiler_XStat_metadata_id(stat); - const behavior = try plane.statBehavior(allocator, metadata_id); - if (behavior == .skip) continue; - - const stat_value = try stringValueFromStat(allocator, plane, stat); - if (behavior == .step_name) { - event_name.* = stat_value; - } - - const stat_name = if (try plane.getStatMetadata(allocator, metadata_id)) |metadata| - stringView(c.tensorflow_profiler_XStatMetadata_name(metadata)) - else - ""; - try putArg(args, allocator, stat_name, stat_value); - } -} - -fn putArg( - args: *std.ArrayListUnmanaged(Arg), - allocator: std.mem.Allocator, - name: []const u8, - value: StringValue, -) !void { - for (args.items) |*arg| { - if (std.mem.eql(u8, arg.name, name)) { - arg.value = value; - return; - } - } - try args.append(allocator, .{ .name = name, .value = value }); -} - -fn metadataStats(metadata: *const c.tensorflow_profiler_XEventMetadata) []const [*c]const c.tensorflow_profiler_XStat { - var stat_count: usize = 0; - const stats = c.tensorflow_profiler_XEventMetadata_stats(metadata, &stat_count) orelse return &.{}; - return stats[0..stat_count]; -} - -fn eventStats(event: *const c.tensorflow_profiler_XEvent) []const [*c]const c.tensorflow_profiler_XStat { - var stat_count: usize = 0; - const stats = c.tensorflow_profiler_XEvent_stats(event, &stat_count) orelse return &.{}; - return stats[0..stat_count]; -} - -fn stringValueFromStat( - allocator: std.mem.Allocator, - plane: *PlaneContext, - stat: *const c.tensorflow_profiler_XStat, -) !StringValue { - // Stringification intentionally matches XStatVisitor::ToString(): - // ../../xla/xla/tsl/profiler/utils/xplane_visitor.cc - return switch (c.tensorflow_profiler_XStat_value_case(stat)) { - c.tensorflow_profiler_XStat_value_int64_value => .{ .int64 = c.tensorflow_profiler_XStat_int64_value(stat) }, - c.tensorflow_profiler_XStat_value_uint64_value => .{ .uint64 = c.tensorflow_profiler_XStat_uint64_value(stat) }, - c.tensorflow_profiler_XStat_value_double_value => .{ .double = c.tensorflow_profiler_XStat_double_value(stat) }, - c.tensorflow_profiler_XStat_value_str_value => .{ .text = stringView(c.tensorflow_profiler_XStat_str_value(stat)) }, - c.tensorflow_profiler_XStat_value_bytes_value => .opaque_bytes, - c.tensorflow_profiler_XStat_value_ref_value => blk: { - const ref_metadata_id = std.math.cast(i64, c.tensorflow_profiler_XStat_ref_value(stat)) orelse return error.InvalidStatMetadataId; - const text = if (try plane.getStatMetadata(allocator, ref_metadata_id)) |metadata| - stringView(c.tensorflow_profiler_XStatMetadata_name(metadata)) - else - ""; - break :blk .{ .text = text }; - }, - else => unreachable, - }; -} - -fn writeJsonStringValue(writer: *std.Io.Writer, value: StringValue) !void { - switch (value) { - .text => |text| try writeJsonString(writer, text), - .opaque_bytes => try writeJsonString(writer, kOpaqueBytesText), - .int64 => |number| { - var buffer: [64]u8 = undefined; - const text = try std.fmt.bufPrint(&buffer, "{d}", .{number}); - try writeJsonString(writer, text); - }, - .uint64 => |number| { - var buffer: [64]u8 = undefined; - const text = try std.fmt.bufPrint(&buffer, "{d}", .{number}); - try writeJsonString(writer, text); - }, - .double => |number| { - var buffer: [64]u8 = undefined; - try writeJsonString(writer, formatDouble(&buffer, "%g", number)); - }, - } -} - -fn writeJsonString(writer: *std.Io.Writer, text: []const u8) !void { - try writer.writeByte('"'); - - var chunk_start: usize = 0; - for (text, 0..) |byte, index| { - const escape = switch (byte) { - '"' => "\\\"", - '\\' => "\\\\", - '\n' => "\\n", - '\r' => "\\r", - '\t' => "\\t", - 0x08 => "\\b", - 0x0c => "\\f", - else => null, - }; - if (escape) |escaped| { - if (chunk_start < index) { - try writer.writeAll(text[chunk_start..index]); - } - try writer.writeAll(escaped); - chunk_start = index + 1; - continue; - } - if (byte < 0x20) { - if (chunk_start < index) { - try writer.writeAll(text[chunk_start..index]); - } - try writeAsciiControlEscape(writer, byte); - chunk_start = index + 1; - } - } - - if (chunk_start < text.len) { - try writer.writeAll(text[chunk_start..]); - } - - try writer.writeByte('"'); -} - -fn lineResourceId(line: *const c.tensorflow_profiler_XLine) !u32 { - const display_id = c.tensorflow_profiler_XLine_display_id(line); - const effective_id = if (display_id != 0) display_id else c.tensorflow_profiler_XLine_id(line); - return wrapI64ToU32(effective_id); -} - -fn lineDisplayName(line: *const c.tensorflow_profiler_XLine) []const u8 { - const display_name = stringView(c.tensorflow_profiler_XLine_display_name(line)); - if (display_name.len != 0) return display_name; - return stringView(c.tensorflow_profiler_XLine_name(line)); -} - -fn eventTimestampPs(line: *const c.tensorflow_profiler_XLine, event: *const c.tensorflow_profiler_XEvent) u64 { - const line_timestamp_ns = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XLine_timestamp_ns(line)); - const event_offset_ps = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XEvent_offset_ps(event)); - return line_timestamp_ns *% kPicosecondsPerNanosecond +% event_offset_ps; -} - -fn eventDurationPs(event: *const c.tensorflow_profiler_XEvent) u64 { - const duration_ps = reinterpretSignedAsUnsigned64(c.tensorflow_profiler_XEvent_duration_ps(event)); - return @max(duration_ps, 1); -} - -fn formatPicosToMicros(buffer: *[64]u8, picoseconds: u64) []const u8 { - // The conversion and "%.17g" formatting are kept identical to: - // ../../xla/xla/tsl/profiler/utils/math_utils.h - // ../../xla/xla/tsl/profiler/utils/format_utils.h - const microseconds = @as(f64, @floatFromInt(picoseconds)) / kPicosecondsPerMicrosecond; - return formatDouble(buffer, "%.17g", microseconds); -} - -fn formatDouble(buffer: *[64]u8, comptime fmt: [*:0]const u8, value: f64) []const u8 { - const written = c.snprintf(buffer.ptr, buffer.len, fmt, value); - if (written <= 0 or written >= buffer.len) unreachable; - return buffer[0..@intCast(written)]; -} - -fn stringView(value: c.upb_StringView) []const u8 { - const text = upb.slice(value); - return text orelse ""; -} - -fn reinterpretSignedAsUnsigned64(value: i64) u64 { - const same_bits: u64 = @bitCast(value); - return same_bits; -} - -fn wrapI64ToU32(value: i64) u32 { - const unsigned_value = reinterpretSignedAsUnsigned64(value); - return @truncate(unsigned_value); -} - -fn writeAsciiControlEscape(writer: *std.Io.Writer, byte: u8) !void { - const high_nibble = @as(usize, byte / 16); - const low_nibble = @as(usize, byte % 16); - - try writer.writeAll("\\u00"); - try writer.writeByte(kHexDigits[high_nibble]); - try writer.writeByte(kHexDigits[low_nibble]); -} - -fn argLessThan(_: void, lhs: Arg, rhs: Arg) bool { - return std.mem.order(u8, lhs.name, rhs.name) == .lt; -} - -fn isInternalEventName(name: []const u8) bool { - // Internal host-event filtering is derived from: - // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc - inline for ([_][]const u8{ - "MemoryAllocation", - "MemoryDeallocation", - "PrefetchProduce", - "PrefetchConsume", - "ParallelInterleaveProduce", - "ParallelInterleaveConsume", - "ParallelInterleaveInitializeInput", - "ParallelMapProduce", - "ParallelMapConsume", - "MapAndBatchProduce", - "MapAndBatchConsume", - "ParseExampleProduce", - "ParseExampleConsume", - }) |candidate| { - if (std.mem.eql(u8, name, candidate)) return true; - } - return false; -} - -fn classifyStatName(name: []const u8) StatBehavior { - // Internal stat filtering and step_name handling are derived from: - // ../../xla/xla/tsl/profiler/utils/xplane_schema.cc - if (std.mem.eql(u8, name, "step_name")) return .step_name; - inline for ([_][]const u8{ - "_pt", - "_ct", - "_p", - "_c", - "_r", - "flops", - "program_id", - "symbol_id", - }) |candidate| { - if (std.mem.eql(u8, name, candidate)) return .skip; - } - return .emit; -} diff --git a/tools/zls.sh b/tools/zls.sh index 2b83f7b6..8428ee01 100755 --- a/tools/zls.sh +++ b/tools/zls.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash cd "$(dirname "${BASH_SOURCE[0]}")" cd "$(bazel info workspace)" exec bazel run -- @zml//:completion "${@}" diff --git a/translate-c/BUILD.bazel b/translate-c/BUILD.bazel new file mode 100644 index 00000000..91f3fc51 --- /dev/null +++ b/translate-c/BUILD.bazel @@ -0,0 +1,17 @@ +alias( + name = "translate-c", + actual = "@translate-c", + visibility = ["//visibility:public"], +) + +alias( + name = "helpers", + actual = "@translate-c//:helpers", + visibility = ["//visibility:public"], +) + +alias( + name = "c_builtins", + actual = "@translate-c//:c_builtins", + visibility = ["//visibility:public"], +) diff --git a/upb/BUILD.bazel b/upb/BUILD.bazel index 9189fc8f..ebf973fb 100644 --- a/upb/BUILD.bazel +++ b/upb/BUILD.bazel @@ -3,7 +3,6 @@ load("@rules_zig//zig:defs.bzl", "zig_library") cc_library( name = "upb_c", - tags = ["manual"], hdrs = ["upb.h"], deps = [ "@com_google_protobuf//upb/message:copy", @@ -12,7 +11,6 @@ cc_library( zig_library( name = "upb", - tags = ["manual"], main = "upb.zig", visibility = ["//visibility:public"], deps = [":upb_c"], diff --git a/upb/upb.zig b/upb/upb.zig index f07d003a..b58252d7 100644 --- a/upb/upb.zig +++ b/upb/upb.zig @@ -34,6 +34,7 @@ pub const ParseError = error{ BadUtf8, MaxDepthExceeded, MissingRequired, + UnlinkedSubMessage, Unknown, } || std.mem.Allocator.Error; @@ -101,6 +102,7 @@ pub fn parseEx(comptime UpbType: type, arena: *c.upb_Arena, data: []const u8, op c.kUpb_DecodeStatus_BadUtf8 => ParseError.BadUtf8, c.kUpb_DecodeStatus_MaxDepthExceeded => ParseError.MaxDepthExceeded, c.kUpb_DecodeStatus_MissingRequired => ParseError.MissingRequired, + c.kUpb_DecodeStatus_UnlinkedSubMessage => ParseError.UnlinkedSubMessage, else => ParseError.Unknown, }; } diff --git a/vfs/gcs.zig b/vfs/gcs.zig deleted file mode 100644 index acc7f634..00000000 --- a/vfs/gcs.zig +++ /dev/null @@ -1,1138 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const stdx = @import("stdx"); - -const parallel_read = @import("parallel_read.zig"); -const VFSBase = @import("base.zig").VFSBase; - -const log = std.log.scoped(.@"zml/vfs/gcs"); - -const EndpointUrl = "https://storage.googleapis.com"; -const MetadataUrl = "http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"; - -/// Check if running on Google Compute Engine / Cloud Run host infrastructure, -/// because metadata server probing should only happen on GCP. -/// Mirrors the TPU platform detection approach. -fn isOnGCP(io: std.Io) !bool { - if (builtin.os.tag != .linux) return false; - - const GoogleComputeEngine = "Google Compute Engine"; - var buffer: [GoogleComputeEngine.len]u8 = undefined; - return std.mem.eql( - u8, - GoogleComputeEngine, - try std.Io.Dir.readFile( - .cwd(), - io, - "/sys/devices/virtual/dmi/id/product_name", - &buffer, - ), - ); -} - -const OAuthToken = struct { - // https://developers.google.com/identity/protocols/oauth2#size - const Size = 2048; - - access_token: []const u8, - expires_in: u64, -}; - -const Credentials = union(enum) { - const AuthorizedUser = struct { - type: []const u8, - client_id: []const u8, - client_secret: []const u8, - refresh_token: []const u8, - quota_project_id: ?[]const u8 = null, - account: []const u8, - universe_domain: []const u8, - }; - - const ServiceAccount = struct { - pub const JwtAssertion = struct { - iss: []const u8, - scope: []const u8 = "https://www.googleapis.com/auth/devstorage.read_only", - aud: []const u8 = "https://oauth2.googleapis.com/token", - exp: i64, - iat: i64, - sub: ?[]const u8 = null, - }; - - type: []const u8, - client_email: []const u8, - private_key: []const u8, - token_uri: []const u8, - quota_project_id: ?[]const u8 = null, - - pub fn makeAssertion(self: *const ServiceAccount, io_: std.Io) JwtAssertion { - const now: std.Io.Timestamp = .now(io_, .real); - return .{ - .iss = self.client_email, - .iat = now.toSeconds(), - .exp = now.addDuration(.fromSeconds(3600)).toSeconds(), - }; - } - }; - - authorized_user: AuthorizedUser, - service_account: ServiceAccount, - metadata_server: void, - - pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) std.json.ParseError(@TypeOf(source.*))!Credentials { - return jsonParseFromValue( - allocator, - try std.json.innerParse(std.json.Value, allocator, source, options), - options, - ); - } - - pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Credentials { - return switch (source) { - .object => |obj| blk: { - const obj_type = obj.get("type") orelse return std.json.ParseFromValueError.UnexpectedToken; - if (std.ascii.eqlIgnoreCase(obj_type.string, "authorized_user")) { - break :blk .{ - .authorized_user = try std.json.parseFromValueLeaky( - AuthorizedUser, - allocator, - source, - options, - ), - }; - } else if (std.ascii.eqlIgnoreCase(obj_type.string, "service_account")) { - break :blk .{ - .service_account = try std.json.parseFromValueLeaky( - ServiceAccount, - allocator, - source, - options, - ), - }; - } - log.warn("Unknown credentials type: {s}", .{obj_type.string}); - return std.json.ParseFromValueError.InvalidEnumTag; - }, - else => std.json.ParseFromValueError.UnexpectedToken, - }; - } -}; - -const ReadState = struct { index: usize, objects: [][]const u8 }; - -pub const GCS = struct { - const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - backend: *GCS, - uri: std.Uri, - path: []const u8, - authorization: std.http.Client.Request.Headers.Value, - }; - - const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var req = client.request( - .GET, - job.batch.uri, - .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = job.batch.authorization, - }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }, - ) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout, .too_many_requests => .retry(), - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to read {s}: {s}", .{ job.batch.path, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; - }; - - const Token = struct { - header: []u8, - expires_at: std.Io.Timestamp, - - fn expired(self: *const Token, io_: std.Io) bool { - const now: std.Io.Timestamp = .now(io_, .real); - return now.toSeconds() >= self.expires_at.toSeconds(); - } - }; - - pub const Config = struct { - credentials: ?Credentials = null, - endpoint_uri: std.Uri, - region: []const u8 = "auto", - }; - - const Handle = struct { - pub const Type = enum { file, directory }; - - type: Type, - uri: []const u8, - pos: u64, - size: u64, - - pub fn init(allocator: std.mem.Allocator, type_: Type, path: []const u8, size: u64) !Handle { - return .{ - .type = type_, - .uri = try allocator.dupe(u8, path), - .pos = 0, - .size = size, - }; - } - - pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { - allocator.free(self.uri); - } - }; - - allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - mutex: std.Io.Mutex = .init, - client: *std.http.Client, - config: Config, - token: Token, - read_pool: *ParallelRead.Pool, - handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .empty, - dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, - base: VFSBase, - - pub const InitOpts = struct { - credentials: ?union(enum) { - json: *std.Io.Reader, - metadata_server: void, - } = null, - endpoint_url: []const u8 = "https://storage.googleapis.com", - region: []const u8 = "auto", - read_pool: parallel_read.InitOpts = .{ - .chunk_size = 16 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, - }; - - pub const InitError = error{ - InvalidCredentialJson, - Unexpected, - } || std.mem.Allocator.Error || std.Io.ConcurrentError; - - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, opts: InitOpts) InitError!GCS { - var arena: std.heap.ArenaAllocator = .init(allocator); - errdefer arena.deinit(); - - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - - const config: Config = .{ - .credentials = if (opts.credentials) |creds| switch (creds) { - .json => |reader| blk: { - var json_reader: std.json.Reader = .init(allocator, reader); - defer json_reader.deinit(); - - break :blk std.json.parseFromTokenSourceLeaky( - Credentials, - arena.allocator(), - &json_reader, - .{ .allocate = .alloc_always }, - ) catch return InitError.InvalidCredentialJson; - }, - .metadata_server => .{ .metadata_server = {} }, - } else null, - .endpoint_uri = std.Uri.parse(try arena.allocator().dupe(u8, opts.endpoint_url)) catch return InitError.Unexpected, - .region = try arena.allocator().dupe(u8, opts.region), - }; - - const token: Token = .{ - .header = try arena.allocator().alloc(u8, "Bearer ".len + OAuthToken.Size), - .expires_at = .zero, - }; - - return .{ - .allocator = allocator, - .arena = arena, - .base = .init(inner), - .client = http_client, - .config = config, - .token = token, - .read_pool = read_pool, - }; - } - - pub fn auto(allocator: std.mem.Allocator, inner_io: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !GCS { - var jsonBuffer: [1024]u8 = undefined; - - const credentials_file: ?std.Io.File = if (environ_map.get("GOOGLE_APPLICATION_CREDENTIALS")) |json_path| blk: { - break :blk std.Io.Dir.openFile(.cwd(), inner_io, json_path, .{}) catch |err| { - log.err("Unable to open GCS credentials file {s}, skipping: {t}", .{ json_path, err }); - break :blk null; - }; - } else if (applicationDefaultCredentials(inner_io, environ_map)) |f| - f - else - null; - defer if (credentials_file) |f| f.close(inner_io); - - if (credentials_file) |f| blk: { - var reader = f.reader(inner_io, &jsonBuffer); - return GCS.init(allocator, inner_io, http_client, .{ .credentials = .{ .json = &reader.interface } }) catch |err| switch (err) { - InitError.InvalidCredentialJson => { - log.err("Invalid GCS credentials JSON", .{}); - break :blk; - }, - else => return err, - }; - } - - if (isOnGCP(inner_io) catch false) { - return .init(allocator, inner_io, http_client, .{ .credentials = .{ .metadata_server = {} } }); - } - - return .init(allocator, inner_io, http_client, .{}); - } - - fn applicationDefaultCredentials(io_: std.Io, environ_map: *std.process.Environ.Map) ?std.Io.File { - var buffer: [1024]u8 = undefined; - const path = stdx.Io.Dir.path.bufJoin(&buffer, switch (builtin.os.tag) { - .windows => &.{ environ_map.get("APPDATA") orelse return null, "gcloud", "application_default_credentials.json" }, - else => &.{ environ_map.get("HOME") orelse return null, ".config", "gcloud", "application_default_credentials.json" }, - }) catch unreachable; - return std.Io.Dir.openFile(.cwd(), io_, path, .{}) catch null; - } - - fn refreshMetadataServerToken(client: *std.http.Client, buffer: []u8) !?[]const u8 { - var response_writer: std.Io.Writer = .fixed(buffer); - const result = try client.fetch(.{ - .location = .{ .url = MetadataUrl }, - .method = .GET, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - }, - .extra_headers = &.{ - .{ .name = "Metadata-Flavor", .value = "Google" }, - }, - .response_writer = &response_writer, - }); - - if (result.status != .ok) { - return null; - } - - return response_writer.buffered(); - } - - fn refreshAuthorizedUserToken(client: *std.http.Client, authorized_user: Credentials.AuthorizedUser, buffer: []u8) ![]const u8 { - var response_writer: std.Io.Writer = .fixed(buffer); - const result = try client.fetch(.{ - .location = .{ .url = "https://oauth2.googleapis.com/token" }, - .method = .POST, - .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_id }, .formatQuery), - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), - std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), - }), - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .content_type = .{ .override = "application/x-www-form-urlencoded" }, - }, - .response_writer = &response_writer, - }); - if (result.status != .ok) { - log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); - return error.RequestFailed; - } - return response_writer.buffered(); - } - - fn refreshServiceAccountToken(io_: std.Io, client: *std.http.Client, service_account: Credentials.ServiceAccount, buffer: []u8) ![]const u8 { - _ = io_; - _ = client; - _ = service_account; - _ = buffer; - return error.Unimplemented; - - // const assertion = service_account.makeAssertion(io_); - - // var tmp_buffer: [8 * 1024]u8 = undefined; - - // var buffer_writer: std.Io.Writer = .fixed(buffer); - // var signing_input_buffer: [2 * 1024]u8 = undefined; - // var signing_input_writer: std.Io.Writer = .fixed(signing_input_buffer); - - // const encoded_rs256_header = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9"; - // try signing_input_writer.print("{s}.", .{encoded_rs256_header}); - // std.base64.url_safe_no_pad.Encoder.encodeWriter( - // signing_input_writer, - // try std.fmt.bufPrint(tmp_buffer, "{f}", .{ - // std.json.fmt(assertion, .{}), - // }), - // ); - // const signing_input = signing_input_writer.buffered(); - // _ = signing_input; - - // const result = try client.fetch(.{ - // .location = .{ .url = "https://oauth2.googleapis.com/token" }, - // .method = .POST, - // .payload = try std.fmt.bufPrint(buffer, "grant_type=refresh_token&client_id={f}&client_secret={f}&refresh_token={f}", .{ - // std.fmt.alt(std.Uri.Component{ .raw = service_account.client_id }, .formatQuery), - // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.client_secret }, .formatQuery), - // std.fmt.alt(std.Uri.Component{ .raw = authorized_user.refresh_token }, .formatQuery), - // }), - // .headers = .{ - // .accept_encoding = .{ .override = "identity" }, - // .content_type = .{ .override = "application/x-www-form-urlencoded" }, - // }, - // .response_writer = &response_writer, - // }); - // if (result.status != .ok) { - // log.err("Failed to refresh ADC token: {s}", .{response_writer.buffered()}); - // return error.RequestFailed; - // } - } - - fn refreshToken(self: *GCS) !void { - var buffer: [4 * 1024]u8 = undefined; - const payload = switch (self.config.credentials.?) { - .authorized_user => |authorized_user| try refreshAuthorizedUserToken(self.client, authorized_user, &buffer), - .service_account => |service_account| try refreshServiceAccountToken(self.base.inner, self.client, service_account, &buffer), - .metadata_server => try refreshMetadataServerToken(self.client, &buffer) orelse return error.RequestFailed, - }; - const oauth_token = try std.json.parseFromSlice(OAuthToken, self.allocator, payload, .{ - .allocate = .alloc_if_needed, - .ignore_unknown_fields = true, - }); - defer oauth_token.deinit(); - self.token = .{ - .header = try std.fmt.bufPrint(self.token.header, "Bearer {s}", .{oauth_token.value.access_token}), - .expires_at = std.Io.Clock.now(.real, self.base.inner).addDuration(.fromSeconds(@intCast(oauth_token.value.expires_in))), - }; - } - - fn getOrRefreshToken(self: *GCS) !std.http.Client.Request.Headers.Value { - if (self.config.credentials == null) { - return .omit; - } - - if (self.token.expired(self.base.inner)) { - try self.refreshToken(); - } - return .{ .override = self.token.header }; - } - - pub fn deinit(self: *GCS) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - - var idx: usize = 0; - while (idx < self.handles.len) : (idx += 1) { - const is_closed = for (self.closed_handles.items) |closed_idx| { - if (closed_idx == idx) break true; - } else false; - - if (!is_closed) { - self.handles.at(idx).deinit(self.allocator); - } - } - self.handles.deinit(self.allocator); - self.closed_handles.deinit(self.allocator); - - var it = self.dir_read_states.iterator(); - while (it.next()) |entry| { - for (entry.value_ptr.objects) |obj| { - self.allocator.free(obj); - } - self.allocator.free(entry.value_ptr.objects); - } - self.dir_read_states.deinit(self.allocator); - self.arena.deinit(); - } - - pub fn io(self: *GCS) std.Io { - return .{ - .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, - }), - }; - } - - fn openHandle(self: *GCS) !struct { u32, *Handle } { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - if (self.closed_handles.pop()) |idx| return .{ idx, self.handles.at(idx) }; - - return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; - } - - fn closeHandle(self: *GCS, idx: u32) !void { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - self.handles.at(idx).deinit(self.allocator); - try self.closed_handles.append(self.allocator, idx); - } - - fn getFileHandle(self: *GCS, file: std.Io.File) *Handle { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - return self.handles.at(@intCast(file.handle)); - } - - fn getDirHandle(self: *GCS, dir: std.Io.Dir) *Handle { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - return self.handles.at(@intCast(dir.handle)); - } - - fn resolvePath(self: *GCS, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { - if (std.meta.eql(dir, std.Io.Dir.cwd())) { - return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); - } - - const handle = self.getDirHandle(dir); - - const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); - const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); - - if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); - if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); - - return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); - } - - fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - switch (operation) { - .file_read_streaming => |o| { - const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; - }; - - if (total == 0) { - return .{ .file_read_streaming = error.EndOfStream }; - } - - handle.pos += @intCast(total); - return .{ .file_read_streaming = total }; - }, - .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { - return self.base.inner.vtable.operate(self.base.inner.userdata, operation); - }, - } - } - - fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; - - const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; - - return .{ .handle = @intCast(idx) }; - } - - fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const handle = self.getDirHandle(dir); - - return .{ - .inode = @intCast(@intFromPtr(handle)), - .nlink = 0, - .size = handle.size, - .permissions = .fromMode(0o444), - .kind = .directory, - .atime = null, - .mtime = std.Io.Timestamp.zero, - .ctime = std.Io.Timestamp.zero, - .block_size = 1, - }; - } - - fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, - else => return std.Io.File.OpenError.Unexpected, - }; - - return .{ - .inode = 0, - .nlink = 0, - .size = size, - .permissions = .fromMode(0o444), - .kind = .file, - .atime = null, - .mtime = std.Io.Timestamp.zero, - .ctime = std.Io.Timestamp.zero, - .block_size = 1, - }; - } - - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.PermissionDenied => return std.Io.File.OpenError.PermissionDenied, - else => return std.Io.File.OpenError.Unexpected, - }; - - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; - - return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; - } - - fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - for (dirs) |dir| { - self.closeHandle(@intCast(dir.handle)) catch unreachable; - } - } - - fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - - if (reader.state == .finished) return 0; - - if (reader.state == .reset) { - if (self.dir_read_states.fetchRemove(reader)) |kv| { - for (kv.value.objects) |obj| self.allocator.free(obj); - self.allocator.free(kv.value.objects); - } - - const handle = self.getDirHandle(reader.dir); - const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; - - self.dir_read_states.put(self.allocator, reader, .{ - .index = 0, - .objects = objects, - }) catch return std.Io.Dir.Reader.Error.Unexpected; - - reader.state = if (objects.len > 0) .reading else .finished; - - if (objects.len == 0) return 0; - } - - const state = self.dir_read_states.getPtr(reader) orelse return std.Io.Dir.Reader.Error.Unexpected; - - var count: usize = 0; - while (count < entries.len and state.index < state.objects.len) { - const obj_key = state.objects[state.index]; - const kind: std.Io.File.Kind = if (std.mem.endsWith(u8, obj_key, "/")) .directory else .file; - - const name = if (std.mem.lastIndexOfScalar(u8, std.mem.trimEnd(u8, obj_key, "/"), '/')) |idx| - obj_key[idx + 1 ..] - else - obj_key; - - entries[count] = .{ - .name = std.mem.trimEnd(u8, name, "/"), - .kind = kind, - .inode = state.index, - }; - count += 1; - state.index += 1; - } - - if (state.index >= state.objects.len) { - reader.state = .finished; - } - - return count; - } - - fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const handle = self.getDirHandle(dir); - const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; - return path.len; - } - - fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; - return real_path.len; - } - - fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const handle = self.getFileHandle(file); - - return .{ - .inode = @intCast(file.handle), - .nlink = 0, - .size = handle.size, - .permissions = .fromMode(0o444), - .kind = .file, - .atime = null, - .mtime = std.Io.Timestamp.zero, - .ctime = std.Io.Timestamp.zero, - .block_size = 1, - }; - } - - fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - return self.getFileHandle(file).size; - } - - fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - for (files) |file| { - self.closeHandle(@intCast(file.handle)) catch unreachable; - } - } - - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; - } - - fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const handle = self.getFileHandle(file); - - if (relative_offset >= 0) { - handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); - } else { - const abs_offset: u64 = @intCast(-relative_offset); - if (abs_offset > handle.pos) { - handle.pos = 0; - } else { - handle.pos -= abs_offset; - } - } - } - - fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const handle = self.getFileHandle(file); - handle.pos = absolute_offset; - } - - fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *GCS = @alignCast(@fieldParentPtr("base", VFSBase.as(userdata))); - const handle = self.getFileHandle(file); - const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; - return path.len; - } - - fn getTimestamp(self: *GCS, buf: []u8) ![]const u8 { - const now_ts = std.Io.Clock.now(.real, self.base.inner); - - const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); - const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; - const year_day = epoch_secs.getEpochDay().calculateYearDay(); - const month_day = year_day.calculateMonthDay(); - const day_seconds = epoch_secs.getDaySeconds(); - - return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ - year_day.year, - @backingInt(month_day.month), - month_day.day_index + 1, - day_seconds.getHoursIntoDay(), - day_seconds.getMinutesIntoHour(), - day_seconds.getSecondsIntoMinute(), - }); - } - - fn pathComponents(_: *GCS, path_: []const u8) struct { []const u8, []const u8 } { - const path = std.mem.trim(u8, path_, "/"); - - if (std.mem.indexOfScalar(u8, path, '/')) |idx| { - return .{ path[0..idx], if (idx + 1 < path.len) path[idx + 1 ..] else "" }; - } else { - return .{ path, "" }; - } - } - - fn gcsUri(self: *GCS, path: []const u8) std.Uri { - var uri = self.config.endpoint_uri; - uri.path = .{ .raw = path }; - uri.query = .{ .percent_encoded = "alt=media" }; - return uri; - } - - // fn authHeaders(self: *GCS, _: std.http.Method, _: std.Uri, _: []const u8, authorization_buffer: []u8) !struct { - // authorization: ?[]const u8, - // extra_headers: [3]?std.http.Header, - // extra_len: usize, - // } { - // if (self.config.oauth_access_token == null) { - // if (self.config.googleCredentialsJsonStr) |json| { - // const oauth = try refreshAuthorizedUserTokenFromJson(self.allocator, self.base.inner, json); - // if (self.config.oauth_access_token) |old_token| self.allocator.free(old_token); - // if (self.config.quota_project_id) |old_project| self.allocator.free(old_project); - // self.config.oauth_access_token = oauth.access_token; - // self.config.quota_project_id = oauth.quota_project_id; - // } - // } - - // const token = self.config.oauth_access_token orelse return error.MissingCredentials; - // const authorization = try std.fmt.bufPrint(authorization_buffer, "Bearer {s}", .{token}); - // var extra_headers: [3]?std.http.Header = .{ null, null, null }; - // var extra_len: usize = 0; - // if (self.config.quota_project_id) |project| { - // extra_headers[0] = .{ .name = "x-goog-user-project", .value = project }; - // extra_len = 1; - // } - // return .{ - // .authorization = authorization, - // .extra_headers = extra_headers, - // .extra_len = extra_len, - // }; - // } - - fn listObjectsBody(self: *GCS, bucket: []const u8, key_prefix: []const u8) ![]u8 { - var query_buf: [4096]u8 = undefined; - var query_writer = std.Io.Writer.fixed(&query_buf); - - try query_writer.writeAll("delimiter="); - try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); - try query_writer.writeAll("&max-keys=1000"); - - if (key_prefix.len > 0) { - try query_writer.writeAll("&prefix="); - try std.Uri.Component.percentEncode(&query_writer, key_prefix, gcsEncodeIsValid); - if (!std.mem.endsWith(u8, key_prefix, "/")) { - try std.Uri.Component.percentEncode(&query_writer, "/", gcsEncodeIsValid); - } - } - - var path_buf: [256]u8 = undefined; - var uri = self.config.endpoint_uri; - uri.path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }; - uri.query = .{ .percent_encoded = query_writer.buffered() }; - - var req = try self.client.request(.GET, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = try self.getOrRefreshToken(), - }, - .extra_headers = if (self.quotaProjectId()) |project| - &.{.{ .name = "x-goog-user-project", .value = project }} - else - &.{}, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [2 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .ok) { - log.err("Failed to list object {f}", .{uri}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; - } - - return if (res.head.content_length) |content_len| - try res.reader(&.{}).readAlloc(self.allocator, content_len) - else - try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); - } - - fn listObjects(self: *GCS, prefix: []const u8) ![][]const u8 { - const bucket, const key_prefix = self.pathComponents(prefix); - const body = try self.listObjectsBody(bucket, key_prefix); - defer self.allocator.free(body); - - return try self.parseListObjectsResponse(body, key_prefix); - } - - fn gcsEncodeIsValid(c: u8) bool { - return switch (c) { - 'A'...'Z', 'a'...'z', '0'...'9', '-', '.', '_', '~' => true, - else => false, - }; - } - - fn quotaProjectId(self: *const GCS) ?[]const u8 { - const credentials = self.config.credentials orelse return null; - return switch (credentials) { - .authorized_user => |authorized_user| authorized_user.quota_project_id, - .service_account => |service_account| service_account.quota_project_id, - .metadata_server => null, - }; - } - - fn parseListObjectsResponse(self: *GCS, xml: []const u8, prefix: []const u8) ![][]const u8 { - var results: std.ArrayListUnmanaged([]const u8) = .empty; - errdefer { - for (results.items) |item| self.allocator.free(item); - results.deinit(self.allocator); - } - - inline for (.{ "Key", "Prefix" }) |tag| { - var pos: usize = 0; - while (std.mem.indexOfPos(u8, xml, pos, "<" ++ tag ++ ">")) |start| { - const v_start = start + tag.len + 2; - const end = std.mem.indexOfPos(u8, xml, v_start, "") orelse break; - const val = xml[v_start..end]; - - const is_self = std.mem.eql(u8, std.mem.trimEnd(u8, val, "/"), std.mem.trimEnd(u8, prefix, "/")); - if (val.len > 0 and !is_self) { - try results.append(self.allocator, try self.allocator.dupe(u8, val)); - } - pos = end + tag.len + 3; - } - } - - return try results.toOwnedSlice(self.allocator); - } - - fn fetchSize(self: *GCS, dir: std.Io.Dir, sub_path: []const u8) !u64 { - var path_buffer: [8 * 1024]u8 = undefined; - const path = try self.resolvePath(dir, sub_path, &path_buffer); - const uri = self.gcsUri(path); - var req = try self.client.request(.HEAD, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = try self.getOrRefreshToken(), - }, - .extra_headers = &.{}, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - const res = try req.receiveHead(&redirect_buffer); - - const size = switch (res.head.status.class()) { - .success => res.head.content_length.?, - else => switch (res.head.status) { - .not_found => return error.FileNotFound, - .unauthorized, .forbidden => return error.PermissionDenied, - else => blk: { - log.err("Failed to fetch size for {f}: {s}", .{ uri, res.head.bytes }); - break :blk error.ServerError; - }, - }, - }; - return size; - } - - fn performRead(self: *GCS, handle: *Handle, data: []const []u8, offset: u64) !usize { - const read_size = parallel_read.readSize(handle.size, offset, data); - if (read_size == 0) return 0; - - const uri = self.gcsUri(handle.uri); - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); - - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .backend = self, - .uri = uri, - .path = handle.uri, - .authorization = try self.getOrRefreshToken(), - }; - - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, - }; - } - - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; - - return read_size; - } -}; - -test "GCS parses XML bucket listing objects and common prefixes" { - const xml = - "" ++ - "" ++ - "bucket" ++ - "LC08/01/001/002/" ++ - "LC08/01/001/002/scene_$folder$6" ++ - "LC08/01/001/002/scene/" ++ - "LC08/01/001/002/" ++ - ""; - - var backend: GCS = undefined; - backend.allocator = std.testing.allocator; - - const objects = try backend.parseListObjectsResponse(xml, "LC08/01/001/002/"); - defer { - for (objects) |object| std.testing.allocator.free(object); - std.testing.allocator.free(objects); - } - - try std.testing.expectEqual(@as(usize, 2), objects.len); - try std.testing.expectEqualStrings("LC08/01/001/002/scene_$folder$", objects[0]); - try std.testing.expectEqualStrings("LC08/01/001/002/scene/", objects[1]); -} - -test "GCS splits bucket and object prefix" { - var backend: GCS = undefined; - - const root_bucket, const root_prefix = backend.pathComponents("gcp-public-data-landsat/"); - try std.testing.expectEqualStrings("gcp-public-data-landsat", root_bucket); - try std.testing.expectEqualStrings("", root_prefix); - - const bucket, const prefix = backend.pathComponents("gcp-public-data-landsat/LC08/01"); - try std.testing.expectEqualStrings("gcp-public-data-landsat", bucket); - try std.testing.expectEqualStrings("LC08/01", prefix); -} diff --git a/vfs/parallel_read.zig b/vfs/parallel_read.zig deleted file mode 100644 index 70c9d89d..00000000 --- a/vfs/parallel_read.zig +++ /dev/null @@ -1,211 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -pub const InitOpts = struct { - chunk_size: usize, - num_workers: usize, - queue_capacity: usize, - max_retries: usize, - retry_initial_delay: std.Io.Duration, - retry_max_delay: std.Io.Duration, -}; - -pub const Status = union(enum) { - success, - retry_after: ?std.Io.Duration, - - pub fn retry() Status { - return .{ .retry_after = null }; - } -}; - -pub const BatchState = struct { - pending: std.atomic.Value(u32), - err: std.atomic.Value(u16) = .init(0), - - pub fn failed(batch: *BatchState) bool { - return batch.err.load(.acquire) != 0; - } - - pub fn fail(batch: *BatchState, err: anyerror) void { - _ = batch.err.cmpxchgStrong(0, @intFromError(err), .release, .monotonic); - } - - pub fn anyError(batch: *BatchState) ?anyerror { - const err = batch.err.load(.acquire); - return if (err == 0) null else @errorFromInt(err); - } - - pub fn completeOne(batch: *BatchState, io: std.Io) void { - const previous = batch.pending.fetchSub(1, .release); - std.debug.assert(previous > 0); - if (previous == 1) io.futexWake(u32, &batch.pending.raw, 1); - } - - pub fn waitUncancelable(batch: *BatchState, io: std.Io) void { - while (true) { - const pending = batch.pending.load(.acquire); - if (pending == 0) return; - io.futexWaitUncancelable(u32, &batch.pending.raw, pending); - } - } -}; - -pub fn Pool(comptime Job: type) type { - return struct { - const Self = @This(); - - group: std.Io.Group = .init, - job_queue: std.Io.Queue(Job), - queue_buf: []Job, - client: *std.http.Client, - prng: std.Random.DefaultPrng, - chunk_size: usize, - max_retries: usize, - retry_initial_delay: std.Io.Duration, - retry_max_delay: std.Io.Duration, - - pub fn init(self: *Self, allocator: std.mem.Allocator, io: std.Io, client: *std.http.Client, opts: InitOpts) !void { - stdx.debug.assert(opts.num_workers > 0, "Pool must have at least one worker", .{}); - stdx.debug.assert(opts.chunk_size > 0, "Pool must have a positive download chunk size", .{}); - stdx.debug.assert(opts.queue_capacity > 0, "Pool must have a positive queue capacity", .{}); - stdx.debug.assert(opts.retry_initial_delay.nanoseconds >= 0, "Pool retry initial delay must not be negative", .{}); - stdx.debug.assert(opts.retry_max_delay.nanoseconds >= 0, "Pool retry max delay must not be negative", .{}); - stdx.debug.assert(opts.retry_max_delay.nanoseconds >= opts.retry_initial_delay.nanoseconds, "Pool retry max delay must be at least the initial delay", .{}); - - const queue_buf = try allocator.alloc(Job, opts.queue_capacity); - errdefer allocator.free(queue_buf); - - self.* = .{ - .client = client, - .chunk_size = opts.chunk_size, - .max_retries = opts.max_retries, - .retry_initial_delay = opts.retry_initial_delay, - .retry_max_delay = opts.retry_max_delay, - .prng = .init(blk: { - var seed: u64 = undefined; - io.random(@ptrCast(&seed)); - break :blk seed; - }), - .job_queue = .init(queue_buf), - .queue_buf = queue_buf, - }; - errdefer self.group.cancel(io); - - for (0..opts.num_workers) |_| { - try self.group.concurrent(io, worker, .{ self, io }); - } - } - - pub fn deinit(self: *Self, allocator: std.mem.Allocator, io: std.Io) void { - self.job_queue.close(io); - self.group.cancel(io); - - allocator.free(self.queue_buf); - } - - fn worker(pool: *Self, io: std.Io) std.Io.Cancelable!void { - while (true) { - const job = pool.job_queue.getOne(io) catch break; - - var attempt: usize = 0; - while (true) { - if (job.batch.state.failed()) break; - const status = job.perform(pool.client) catch |err| { - job.batch.state.fail(err); - break; - }; - - switch (status) { - .success => break, - .retry_after => |duration| { - if (attempt >= pool.max_retries) { - job.batch.state.fail(error.RetriesExhausted); - break; - } - - io.sleep(duration orelse pool.backoffDuration(attempt), .awake) catch { - job.batch.state.fail(error.RetriesExhausted); - break; - }; - - attempt += 1; - }, - } - continue; - } - - job.batch.state.completeOne(io); - } - } - - fn backoffDuration(pool: *Self, attempt: usize) std.Io.Duration { - // FullJitter algorithm - const max_delay_ns: i96 = @min( - pool.retry_max_delay.toNanoseconds(), - pool.retry_initial_delay.toNanoseconds() *| (@as(i96, 1) << @as(u7, @intCast(@min(attempt, std.math.maxInt(u7))))), - ); - const delay_ns = pool.prng.random().intRangeAtMost(i96, 0, max_delay_ns); - return .fromNanoseconds(delay_ns); - } - }; -} - -pub const ContentRange = struct { - start: u64, - end: u64, - total: u64, -}; - -pub fn parseContentRange(value: []const u8) ?ContentRange { - const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; - const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; - const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; - - return .{ - .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, - .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, - .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, - }; -} - -pub fn readSize(file_size: u64, offset: u64, data: []const []u8) usize { - if (offset >= file_size) return 0; - - var requested: usize = 0; - for (data) |buf| requested += buf.len; - return @intCast(@min(file_size - offset, requested)); -} - -pub fn readChunk( - reader: *std.Io.Reader, - content_range: ?ContentRange, - file_offset: u64, - data: []const []u8, - chunk_offset: usize, - chunk_len: usize, -) !void { - if (content_range) |cr| { - if (cr.start > file_offset) return error.InvalidContentRange; - if (cr.start < file_offset) { - try reader.discardAll(file_offset - cr.start); - } - } - - var remaining = chunk_len; - var skip = chunk_offset; - - for (data) |buf| { - if (skip >= buf.len) { - skip -= buf.len; - continue; - } - - const destination = buf[skip..][0..@min(remaining, buf.len - skip)]; - try reader.readSliceAll(destination); - remaining -= destination.len; - if (remaining == 0) break; - skip = 0; - } -} diff --git a/vfs/s3.zig b/vfs/s3.zig deleted file mode 100644 index eb849e21..00000000 --- a/vfs/s3.zig +++ /dev/null @@ -1,997 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const parallel_read = @import("parallel_read.zig"); -const VFSBase = @import("base.zig").VFSBase; - -const log = std.log.scoped(.@"zml/vfs/s3"); - -pub const AwsSigV4 = struct { - access_key: ?[]const u8, - secret_key: ?[]const u8, - region: []const u8, - service: []const u8, - - const ALGORITHM = "AWS4-HMAC-SHA256"; - pub const UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; - - pub fn generateAuthHeader( - self: AwsSigV4, - output_buffer: []u8, - method: std.http.Method, - uri: std.Uri, - timestamp: []const u8, - extra_headers: []const std.http.Header, - ) !?[]const u8 { - const access_key = self.access_key orelse return null; - _ = self.secret_key orelse return null; - - const date = timestamp[0..8]; - - var header_buf: [32]std.http.Header = undefined; - var header_count: usize = 0; - - const host = uri.host orelse return error.MissingHost; - const host_header: std.http.Header = blk: { - const host_ = switch (host) { - .raw, .percent_encoded => |h| h, - }; - if (uri.port) |port| { - var host_buf: [128]u8 = undefined; - break :blk .{ .name = "host", .value = try std.fmt.bufPrint(&host_buf, "{s}:{d}", .{ host_, port }) }; - } else { - break :blk .{ .name = "host", .value = host_ }; - } - }; - - header_buf[0] = host_header; - header_buf[1] = .{ .name = "x-amz-content-sha256", .value = UNSIGNED_PAYLOAD }; - header_buf[2] = .{ .name = "x-amz-date", .value = timestamp }; - header_count = 3; - - for (extra_headers) |h| { - if (header_count >= header_buf.len) return error.TooManyHeaders; - header_buf[header_count] = h; - header_count += 1; - } - - const active_headers = header_buf[0..header_count]; - - std.mem.sort(std.http.Header, active_headers, {}, struct { - fn less(_: void, a: std.http.Header, b: std.http.Header) bool { - return std.ascii.lessThanIgnoreCase(a.name, b.name); - } - }.less); - - var sha = std.crypto.hash.sha2.Sha256.init(.{}); - var sha_writer = stdx.crypto.hmacWriter(&sha); - const sw = &sha_writer.interface; - - try sw.print("{s}\n", .{@tagName(method)}); - if (uri.path.isEmpty()) { - try sw.writeAll("/\n"); - } else { - try uri.path.formatPath(sw); - try sw.writeByte('\n'); - } - - if (uri.query) |query| { - try query.formatEscaped(sw); - try sw.writeByte('\n'); - } else { - try sw.writeAll("\n"); - } - - var signed_headers_buf: [512]u8 = undefined; - var sh_writer = std.Io.Writer.fixed(&signed_headers_buf); - for (active_headers, 0..) |h, i| { - for (h.name) |char| try sw.writeByte(std.ascii.toLower(char)); - try sw.print(":{s}\n", .{h.value}); - if (i > 0) _ = try sh_writer.write(";"); - for (h.name) |char| try sh_writer.writeByte(std.ascii.toLower(char)); - } - const signed_headers = sh_writer.buffered(); - - try sw.print("\n{s}\n{s}", .{ signed_headers, UNSIGNED_PAYLOAD }); - - var canonical_request_hash: [32]u8 = undefined; - sha.final(&canonical_request_hash); - - var scope_buf: [128]u8 = undefined; - const scope = try std.fmt.bufPrint(&scope_buf, "{s}/{s}/{s}/aws4_request", .{ date, self.region, self.service }); - - const signing_key = try self.deriveSigningKey(date); - - var h_state = std.crypto.auth.hmac.sha2.HmacSha256.init(&signing_key); - var h_writer = stdx.crypto.hmacWriter(&h_state); - const hw = &h_writer.interface; - - try hw.print("{s}\n{s}\n{s}\n{x}", .{ ALGORITHM, timestamp, scope, &canonical_request_hash }); - - var signature: [32]u8 = undefined; - h_state.final(&signature); - - return try std.fmt.bufPrint(output_buffer, "{s} Credential={s}/{s}, SignedHeaders={s}, Signature={x}", .{ - ALGORITHM, - access_key, - scope, - signed_headers, - &signature, - }); - } - - fn deriveSigningKey(self: AwsSigV4, date: []const u8) ![32]u8 { - var key_buf: [128]u8 = undefined; - const k_init = try std.fmt.bufPrint(&key_buf, "AWS4{s}", .{self.secret_key.?}); - - const k_date = hmacStatic(k_init, date); - const k_region = hmacStatic(&k_date, self.region); - const k_service = hmacStatic(&k_region, self.service); - return hmacStatic(&k_service, "aws4_request"); - } - - fn hmacStatic(key: []const u8, data: []const u8) [32]u8 { - var out: [32]u8 = undefined; - std.crypto.auth.hmac.sha2.HmacSha256.create(&out, data, key); - return out; - } -}; - -const ReadState = struct { index: usize, objects: [][]const u8 }; - -pub const S3 = struct { - const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - backend: *S3, - uri: std.Uri, - url: []const u8, - }; - - const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var timestamp_buf: [16]u8 = undefined; - const timestamp = job.batch.backend.getTimestamp(×tamp_buf) catch unreachable; - - const signer: AwsSigV4 = .{ - .access_key = job.batch.backend.config.access_key, - .secret_key = job.batch.backend.config.secret_key, - .region = job.batch.backend.config.region, - .service = job.batch.backend.config.auth_service, - }; - - var authorization_buffer: [512]u8 = undefined; - const authorization = signer.generateAuthHeader( - &authorization_buffer, - .GET, - job.batch.uri, - timestamp, - &.{.{ .name = "Range", .value = range_header }}, - ) catch |err| { - log.err("Failed to generate auth header: {}", .{err}); - return err; - }; - - var req = client.request(.GET, job.batch.uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - .{ .name = "Range", .value = range_header }, - }, - }) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout, .too_many_requests => .retry(), - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to read {s}: {s}", .{ job.batch.url, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; - }; - - pub const InitOpts = struct { - read_pool: parallel_read.InitOpts = .{ - .chunk_size = 16 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, - }; - - pub const Config = struct { - access_key: ?[]const u8 = null, - secret_key: ?[]const u8 = null, - endpoint_url: []const u8, - region: []const u8, - auth_service: []const u8 = "s3", - }; - - const Handle = struct { - pub const Type = enum { file, directory }; - - type: Type, - uri: []const u8, - pos: u64, - size: u64, - - pub fn init(allocator: std.mem.Allocator, type_: Type, path: []const u8, size: u64) !Handle { - return .{ - .type = type_, - .uri = try allocator.dupe(u8, path), - .pos = 0, - .size = size, - }; - } - - pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { - allocator.free(self.uri); - } - }; - - allocator: std.mem.Allocator, - mutex: std.Io.Mutex = .init, - client: *std.http.Client, - config: Config, - read_pool: *ParallelRead.Pool, - handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .empty, - dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, - base: VFSBase, - - pub fn init( - allocator: std.mem.Allocator, - inner: std.Io, - http_client: *std.http.Client, - config: Config, - opts: InitOpts, - ) !S3 { - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - - return .{ - .allocator = allocator, - .base = .init(inner), - .client = http_client, - .read_pool = read_pool, - .config = .{ - .access_key = if (config.access_key) |k| try allocator.dupe(u8, k) else null, - .secret_key = if (config.secret_key) |k| try allocator.dupe(u8, k) else null, - .endpoint_url = try allocator.dupe(u8, config.endpoint_url), - .region = try allocator.dupe(u8, config.region), - .auth_service = try allocator.dupe(u8, config.auth_service), - }, - }; - } - - pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !S3 { - const access_key = environ_map.get("AWS_ACCESS_KEY_ID"); - const secret_key = environ_map.get("AWS_SECRET_ACCESS_KEY"); - - var endpoint_is_default = false; - const endpoint = environ_map.get("AWS_ENDPOINT_URL") orelse environ_map.get("AWS_ENDPOINT_URL_S3") orelse blk: { - endpoint_is_default = true; - break :blk "https://s3.amazonaws.com"; - }; - - var region_is_default = false; - const region = environ_map.get("AWS_REGION") orelse environ_map.get("AWS_DEFAULT_REGION") orelse blk: { - region_is_default = true; - break :blk "us-east-1"; - }; - - if (access_key == null or secret_key == null or endpoint_is_default or region_is_default) { - var buf: [512]u8 = undefined; - var writer = std.Io.Writer.fixed(&buf); - - if (access_key == null or secret_key == null) { - try writer.writeAll("S3 initialized without full credentials (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY). Access will be unauthenticated."); - } else { - try writer.writeAll("S3 initialized with credentials."); - } - - if (endpoint_is_default) { - try writer.print(" Using default endpoint: {s} (set AWS_ENDPOINT_URL or AWS_ENDPOINT_URL_S3).", .{endpoint}); - } - - if (region_is_default) { - try writer.print(" Using default region: {s} (set AWS_REGION or AWS_DEFAULT_REGION).", .{region}); - } - - log.warn("{s}", .{writer.buffered()}); - } - - return init(allocator, inner, http_client, .{ - .access_key = access_key, - .secret_key = secret_key, - .endpoint_url = endpoint, - .region = region, - }, .{}); - } - - pub fn deinit(self: *S3) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - - var idx: usize = 0; - while (idx < self.handles.len) : (idx += 1) { - const is_closed = for (self.closed_handles.items) |closed_idx| { - if (closed_idx == idx) break true; - } else false; - - if (!is_closed) { - self.handles.at(idx).deinit(self.allocator); - } - } - self.handles.deinit(self.allocator); - self.closed_handles.deinit(self.allocator); - - var it = self.dir_read_states.iterator(); - while (it.next()) |entry| { - for (entry.value_ptr.objects) |obj| { - self.allocator.free(obj); - } - self.allocator.free(entry.value_ptr.objects); - } - self.dir_read_states.deinit(self.allocator); - - if (self.config.access_key) |access_key| self.allocator.free(access_key); - if (self.config.secret_key) |secret_key| self.allocator.free(secret_key); - self.allocator.free(self.config.endpoint_url); - self.allocator.free(self.config.region); - self.allocator.free(self.config.auth_service); - } - - pub fn io(self: *S3) std.Io { - return .{ - .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, - }), - }; - } - - fn openHandle(self: *S3) !struct { u32, *Handle } { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - if (self.closed_handles.pop()) |idx| return .{ idx, self.handles.at(idx) }; - - return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; - } - - fn closeHandle(self: *S3, idx: u32) !void { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - self.handles.at(idx).deinit(self.allocator); - try self.closed_handles.append(self.allocator, idx); - } - - fn getFileHandle(self: *S3, file: std.Io.File) *Handle { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - return self.handles.at(@intCast(file.handle)); - } - - fn getDirHandle(self: *S3, dir: std.Io.Dir) *Handle { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - return self.handles.at(@intCast(dir.handle)); - } - - fn resolvePath(self: *S3, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { - if (std.meta.eql(dir, std.Io.Dir.cwd())) { - return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); - } - - const handle = self.getDirHandle(dir); - - const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); - const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); - - if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); - if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); - - return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); - } - - fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - switch (operation) { - .file_read_streaming => |o| { - const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; - }; - - if (total == 0) { - return .{ .file_read_streaming = error.EndOfStream }; - } - - handle.pos += @intCast(total); - return .{ .file_read_streaming = total }; - }, - .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { - return self.base.inner.vtable.operate(self.base.inner.userdata, operation); - }, - } - } - - fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; - - const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; - - return .{ .handle = @intCast(idx) }; - } - - fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getDirHandle(dir); - - return .{ - .inode = @intCast(@intFromPtr(handle)), - .nlink = 0, - .size = handle.size, - .permissions = .fromMode(0o444), - .kind = .directory, - .atime = null, - .mtime = std.Io.Timestamp.zero, - .ctime = std.Io.Timestamp.zero, - .block_size = 1, - }; - } - - fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.BadPathName => return std.Io.File.OpenError.BadPathName, - else => return std.Io.File.OpenError.Unexpected, - }; - - return .{ - .inode = 0, - .nlink = 0, - .size = size, - .permissions = .fromMode(0o444), - .kind = .file, - .atime = null, - .mtime = std.Io.Timestamp.zero, - .ctime = std.Io.Timestamp.zero, - .block_size = 1, - }; - } - - fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const size = self.fetchSize(dir, sub_path) catch |err| switch (err) { - error.FileNotFound => return std.Io.File.OpenError.FileNotFound, - error.BadPathName => return std.Io.File.OpenError.BadPathName, - else => return std.Io.File.OpenError.Unexpected, - }; - - var path_buffer: [8 * 1024]u8 = undefined; - const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; - - return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; - } - - fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - for (dirs) |dir| { - self.closeHandle(@intCast(dir.handle)) catch unreachable; - } - } - - fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - - if (reader.state == .finished) return 0; - - if (reader.state == .reset) { - if (self.dir_read_states.fetchRemove(reader)) |kv| { - for (kv.value.objects) |obj| self.allocator.free(obj); - self.allocator.free(kv.value.objects); - } - - const handle = self.getDirHandle(reader.dir); - const objects = self.listObjects(handle.uri) catch return std.Io.Dir.Reader.Error.Unexpected; - - self.dir_read_states.put(self.allocator, reader, .{ - .index = 0, - .objects = objects, - }) catch return std.Io.Dir.Reader.Error.Unexpected; - - reader.state = if (objects.len > 0) .reading else .finished; - - if (objects.len == 0) return 0; - } - - const state = self.dir_read_states.getPtr(reader) orelse return std.Io.Dir.Reader.Error.Unexpected; - - var count: usize = 0; - while (count < entries.len and state.index < state.objects.len) { - const obj_key = state.objects[state.index]; - const kind: std.Io.File.Kind = if (std.mem.endsWith(u8, obj_key, "/")) .directory else .file; - - const name = if (std.mem.lastIndexOfScalar(u8, std.mem.trimEnd(u8, obj_key, "/"), '/')) |idx| - obj_key[idx + 1 ..] - else - obj_key; - - entries[count] = .{ - .name = std.mem.trimEnd(u8, name, "/"), - .kind = kind, - .inode = state.index, - }; - count += 1; - state.index += 1; - } - - if (state.index >= state.objects.len) { - reader.state = .finished; - } - - return count; - } - - fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getDirHandle(dir); - const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; - return path.len; - } - - fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; - return real_path.len; - } - - fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - - return .{ - .inode = @intCast(file.handle), - .nlink = 0, - .size = handle.size, - .permissions = .fromMode(0o444), - .kind = .file, - .atime = null, - .mtime = std.Io.Timestamp.zero, - .ctime = std.Io.Timestamp.zero, - .block_size = 1, - }; - } - - fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - return self.getFileHandle(file).size; - } - - fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - for (files) |file| { - self.closeHandle(@intCast(file.handle)) catch unreachable; - } - } - - fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - return self.performRead(handle, data, offset) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; - }; - } - - fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - - if (relative_offset >= 0) { - handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); - } else { - const abs_offset: u64 = @intCast(-relative_offset); - if (abs_offset > handle.pos) { - handle.pos = 0; - } else { - handle.pos -= abs_offset; - } - } - } - - fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - handle.pos = absolute_offset; - } - - fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle = self.getFileHandle(file); - const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; - return path.len; - } - - fn getTimestamp(self: *S3, buf: []u8) ![]const u8 { - const now_ts = std.Io.Clock.now(.real, self.base.inner); - - const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); - const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; - const year_day = epoch_secs.getEpochDay().calculateYearDay(); - const month_day = year_day.calculateMonthDay(); - const day_seconds = epoch_secs.getDaySeconds(); - - return try std.fmt.bufPrint(buf, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ - year_day.year, - @backingInt(month_day.month), - month_day.day_index + 1, - day_seconds.getHoursIntoDay(), - day_seconds.getMinutesIntoHour(), - day_seconds.getSecondsIntoMinute(), - }); - } - - fn pathComponents(self: *S3, path_: []const u8) struct { []const u8, []const u8, []const u8 } { - const endpoint = std.mem.trimEnd(u8, self.config.endpoint_url, "/"); - const path = std.mem.trim(u8, path_, "/"); - - if (std.mem.findScalar(u8, path, '/')) |idx| { - return .{ endpoint, path[0..idx], if (idx + 1 < path.len) path[idx + 1 ..] else "" }; - } else { - return .{ endpoint, path, "" }; - } - } - - fn s3Url(self: *S3, path: []const u8, buf: []u8) ![]const u8 { - const endpoint, const bucket, const key = self.pathComponents(path); - return try std.fmt.bufPrint(buf, "{s}/{s}/{s}", .{ endpoint, bucket, key }); - } - - fn listObjectsBody(self: *S3, endpoint: []const u8, bucket: []const u8, key_prefix: []const u8) ![]u8 { - var query_buf: [4096]u8 = undefined; - var query_writer = std.Io.Writer.fixed(&query_buf); - - try query_writer.writeAll("delimiter="); - try std.Uri.Component.percentEncode(&query_writer, "/", s3EncodeIsValid); - try query_writer.writeAll("&list-type=2&max-keys=1000"); - - if (key_prefix.len > 0) { - try query_writer.writeAll("&prefix="); - try std.Uri.Component.percentEncode(&query_writer, key_prefix, s3EncodeIsValid); - if (!std.mem.endsWith(u8, key_prefix, "/")) { - try std.Uri.Component.percentEncode(&query_writer, "/", s3EncodeIsValid); - } - } - - const endpoint_uri = try std.Uri.parse(endpoint); - - var path_buf: [256]u8 = undefined; - - const uri: std.Uri = .{ - .scheme = endpoint_uri.scheme, - .host = endpoint_uri.host, - .port = endpoint_uri.port, - .path = .{ .percent_encoded = try std.fmt.bufPrint(&path_buf, "/{s}", .{bucket}) }, - .query = .{ .percent_encoded = query_writer.buffered() }, - .fragment = null, - }; - - var timestamp_buf: [16]u8 = undefined; - const timestamp = try self.getTimestamp(×tamp_buf); - - const signer: AwsSigV4 = .{ - .access_key = self.config.access_key, - .secret_key = self.config.secret_key, - .region = self.config.region, - .service = self.config.auth_service, - }; - - var authorization_buffer: [512]u8 = undefined; - const authorization = try signer.generateAuthHeader(&authorization_buffer, .GET, uri, timestamp, &.{}); - - var req = try self.client.request(.GET, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - }, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [2 * 1024]u8 = undefined; - var res = try req.receiveHead(&redirect_buffer); - - if (res.head.status != .ok) { - log.err("Failed to list object {f}", .{uri}); - log.err("{s}", .{res.head.bytes}); - return error.RequestFailed; - } - - return if (res.head.content_length) |content_len| - try res.reader(&.{}).readAlloc(self.allocator, content_len) - else - // When we don't have a content length, put a reasonable limit - try res.reader(&.{}).allocRemaining(self.allocator, .limited(1024 * 1024)); - } - - fn listObjects(self: *S3, prefix: []const u8) ![][]const u8 { - const endpoint, const bucket, const key_prefix = self.pathComponents(prefix); - const body = try self.listObjectsBody(endpoint, bucket, key_prefix); - defer self.allocator.free(body); - - return try self.parseListObjectsResponse(body, key_prefix); - } - - fn s3EncodeIsValid(c: u8) bool { - return switch (c) { - 'A'...'Z', 'a'...'z', '0'...'9', '-', '.', '_', '~' => true, - else => false, - }; - } - - fn parseListObjectsResponse(self: *S3, xml: []const u8, prefix: []const u8) ![][]const u8 { - var results: std.ArrayListUnmanaged([]const u8) = .empty; - errdefer { - for (results.items) |item| self.allocator.free(item); - results.deinit(self.allocator); - } - - inline for (.{ "Key", "Prefix" }) |tag| { - var pos: usize = 0; - while (std.mem.indexOfPos(u8, xml, pos, "<" ++ tag ++ ">")) |start| { - const v_start = start + tag.len + 2; - const end = std.mem.indexOfPos(u8, xml, v_start, "") orelse break; - const val = xml[v_start..end]; - - // Filter out the directory itself (S3 often includes the prefix as a 0-byte key) - const is_self = std.mem.eql(u8, std.mem.trimEnd(u8, val, "/"), std.mem.trimEnd(u8, prefix, "/")); - if (val.len > 0 and !is_self) { - try results.append(self.allocator, try self.allocator.dupe(u8, val)); - } - pos = end + tag.len + 3; - } - } - - return try results.toOwnedSlice(self.allocator); - } - - fn fetchSize(self: *S3, dir: std.Io.Dir, sub_path: []const u8) !u64 { - var path_buffer: [8 * 1024]u8 = undefined; - var url_buf: [8 * 1024]u8 = undefined; - - const path = try self.resolvePath(dir, sub_path, &path_buffer); - const url = try self.s3Url(path, &url_buf); - - const uri = std.Uri.parse(url) catch return error.BadPathName; - - var timestamp_buf: [16]u8 = undefined; - const timestamp = try self.getTimestamp(×tamp_buf); - - const signer: AwsSigV4 = .{ - .access_key = self.config.access_key, - .secret_key = self.config.secret_key, - .region = self.config.region, - .service = self.config.auth_service, - }; - - var authorization_buffer: [512]u8 = undefined; - const authorization = try signer.generateAuthHeader(&authorization_buffer, .HEAD, uri, timestamp, &.{}); - - var req = try self.client.request(.HEAD, uri, .{ - .redirect_behavior = .not_allowed, - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = if (authorization) |auth| .{ .override = auth } else .omit, - }, - .extra_headers = &.{ - .{ .name = "x-amz-date", .value = timestamp }, - .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, - }, - }); - defer req.deinit(); - - try req.sendBodiless(); - - var redirect_buffer: [8 * 1024]u8 = undefined; - const res = try req.receiveHead(&redirect_buffer); - - return switch (res.head.status.class()) { - .success => res.head.content_length.?, - else => switch (res.head.status) { - .not_found => return error.FileNotFound, - else => blk: { - log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); - break :blk error.ServerError; - }, - }, - }; - } - - fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { - const read_size = parallel_read.readSize(handle.size, offset, data); - if (read_size == 0) return 0; - - var url_buf: [8 * 1024]u8 = undefined; - const url = try self.s3Url(handle.uri, &url_buf); - const uri = std.Uri.parse(url) catch return error.BadPathName; - - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); - - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .backend = self, - .uri = uri, - .url = url, - }; - - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, - }; - } - - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; - - return read_size; - } -}; diff --git a/vfs/vfs.zig b/vfs/vfs.zig deleted file mode 100644 index fe626ff9..00000000 --- a/vfs/vfs.zig +++ /dev/null @@ -1,483 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -pub const File = @import("file.zig").File; -pub const GCS = @import("gcs.zig").GCS; -pub const HF = @import("hf.zig").HF; -pub const HTTP = @import("http.zig").HTTP; -pub const S3 = @import("s3.zig").S3; -pub const VFSBase = @import("base.zig").VFSBase; - -const log = std.log.scoped(.@"zml/vfs"); - -const CWD_HANDLE: u32 = 0; - -const VFS = @This(); -const Handle = struct { handle: u32, backend_idx: ?usize, flags: std.Io.File.Flags = .{ .nonblocking = false } }; - -allocator: std.mem.Allocator, -mutex: std.Io.Mutex = .init, - -backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, -handles: stdx.SegmentedList(Handle, 128) = .{}, -closed_handles: std.ArrayList(u32) = .empty, - -base: VFSBase, - -pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { - const base = VFSBase.init(base_io); - - var handles: @FieldType(@This(), "handles") = .{}; - try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); - try handles.append(allocator, .{ .handle = std.posix.STDIN_FILENO, .backend_idx = null }); - try handles.append(allocator, .{ .handle = std.posix.STDOUT_FILENO, .backend_idx = null }); - try handles.append(allocator, .{ .handle = std.posix.STDERR_FILENO, .backend_idx = null }); - - return .{ - .allocator = allocator, - .handles = handles, - .base = base, - }; -} - -pub fn deinit(self: *VFS) void { - self.handles.deinit(self.allocator); - self.closed_handles.deinit(self.allocator); - self.backends.deinit(self.allocator); -} - -pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - try self.backends.put(self.allocator, scheme, backend); -} - -pub fn unregister(self: *VFS, scheme: []const u8) bool { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - return self.backends.remove(scheme); -} - -pub fn io(self: *VFS) std.Io { - return .{ - .userdata = &self.base, - .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, - .dirOpenDir = dirOpenDir, - .dirStat = dirStat, - .dirStatFile = dirStatFile, - .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, - .dirOpenFile = dirOpenFile, - .dirClose = dirClose, - .dirRead = dirRead, - .dirRealPath = dirRealPath, - .dirRealPathFile = dirRealPathFile, - .fileStat = fileStat, - .fileLength = fileLength, - .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, - .fileReadPositional = fileReadPositional, - .fileSeekBy = fileSeekBy, - .fileSeekTo = fileSeekTo, - .fileRealPath = fileRealPath, - }), - }; -} - -fn openHandle(self: *VFS) !struct { u32, *Handle } { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - if (self.closed_handles.pop()) |idx| { - return .{ idx, self.handles.at(idx) }; - } - return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; -} - -fn closeHandle(self: *VFS, idx: u32) !void { - if (idx == CWD_HANDLE) return; - - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - try self.closed_handles.append(self.allocator, idx); -} - -fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { - self.mutex.lockUncancelable(self.base.inner); - const handle = self.handles.at(@intCast(file.handle)); - self.mutex.unlock(self.base.inner); - - return .{ handle, self.getBackend(handle.backend_idx) }; -} - -fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { - self.mutex.lockUncancelable(self.base.inner); - defer self.mutex.unlock(self.base.inner); - - if (std.meta.eql(dir, std.Io.Dir.cwd())) return self.handles.at(CWD_HANDLE); - return self.handles.at(@intCast(dir.handle)); -} - -fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { - if (backend_idx) |idx| return self.backends.entries.items(.key)[idx] else return null; -} - -fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { - if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; -} - -fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { - // A scheme-qualified path (e.g. "hf://owner/model/file") is absolute: it - // is resolved from the scheme's root regardless of `dir`. Without this, - // opening such a path relative to an already-open dir of the same - // backend double-prefixes the dir's path. - if (sub_path) |sp| { - if (std.mem.indexOf(u8, sp, "://") != null) { - const uri = std.Uri.parse(sp) catch return error.VFSNotRegistered; - const backend_idx: usize = for (self.backends.entries.items(.key), 0..) |s, idx| { - if (std.mem.eql(u8, uri.scheme, s)) break idx; - } else return error.VFSNotRegistered; - return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; - } - } - - if (std.meta.eql(dir, std.Io.Dir.cwd())) { - if (sub_path == null) return .{ null, dir, self.base.inner }; - if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; - - const uri = std.Uri.parse(sub_path.?) catch null; - if (uri) |u| { - const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { - if (std.mem.eql(u8, u.scheme, s)) break idx; - } else null; - if (backend_idx == null) return error.VFSNotRegistered; - return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; - } else { - return .{ null, std.Io.Dir.cwd(), self.base.inner }; - } - } else { - const handle = self.getDirHandle(dir); - if (handle.backend_idx) |backend_idx| { - return .{ backend_idx, .{ .handle = @intCast(handle.handle) }, self.getBackend(backend_idx) }; - } else { - return .{ null, .{ .handle = @intCast(handle.handle) }, self.base.inner }; - } - } -} - -fn stripScheme(path: []const u8) []const u8 { - const uri = std.Uri.parse(path) catch return path; - return path[uri.scheme.len + 3 ..]; -} - -fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - switch (operation) { - .file_read_streaming => |o| { - const handle, const backend = self.getFileHandle(o.file); - return backend.vtable.operate(backend.userdata, .{ .file_read_streaming = .{ - .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, - .data = o.data, - } }); - }, - .file_write_streaming => |o| { - const handle, const backend = self.getFileHandle(o.file); - return backend.vtable.operate(backend.userdata, .{ .file_write_streaming = .{ - .file = .{ .handle = @intCast(handle.handle), .flags = handle.flags }, - .header = o.header, - .data = o.data, - .splat = o.splat, - } }); - }, - .device_io_control, .net_receive, .net_read, .net_send, .net_write => { - return self.base.inner.vtable.operate(self.base.inner.userdata, operation); - }, - } -} - -fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for opening dir '{s}' : {any}", .{ sub_path, err }); - return std.Io.Dir.OpenError.Unexpected; - }; - const fs_dir = try backend.vtable.dirOpenDir(backend.userdata, dir_, stripScheme(sub_path), options); - const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = .{ - .handle = @intCast(fs_dir.handle), - .backend_idx = backend_idx, - }; - return .{ .handle = @intCast(idx) }; -} - -fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { - log.err("Failed to lookup backend for dir stat : {any}", .{err}); - return std.Io.Dir.StatError.Unexpected; - }; - return backend.vtable.dirStat(backend.userdata, dir_); -} - -pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir stat file '{s}' : {any}", .{ sub_path, err }); - return std.Io.Dir.StatFileError.Unexpected; - }; - return backend.vtable.dirStatFile(backend.userdata, dir_, stripScheme(sub_path), options); -} - -pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir access '{s}' : {any}", .{ sub_path, err }); - return std.Io.Dir.AccessError.Unexpected; - }; - return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); -} - -pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for dir create file '{s}' : {any}", .{ sub_path, err }); - return std.Io.File.OpenError.Unexpected; - }; - - const file = try backend.vtable.dirCreateFile(backend.userdata, dir_, stripScheme(sub_path), flags); - errdefer file.close(backend); - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .{ - .handle = @intCast(file.handle), - .backend_idx = backend_idx, - .flags = file.flags, - }; - return .{ .handle = @intCast(idx), .flags = file.flags }; -} - -fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { - log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); - return std.Io.File.OpenError.Unexpected; - }; - const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); - errdefer file.close(backend); - const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; - handle.* = .{ - .handle = @intCast(file.handle), - .backend_idx = backend_idx, - .flags = file.flags, - }; - return .{ .handle = @intCast(idx), .flags = handle.flags }; -} - -fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - for (dirs) |dir| { - _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { - log.err("Failed to lookup backend for closing dir : {any}", .{err}); - continue; - }; - backend.vtable.dirClose(backend.userdata, &.{dir_}); - self.closeHandle(@intCast(dir.handle)) catch unreachable; - } -} - -fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - _, const dir_, const backend = self.lookupDir(reader.dir, null) catch |err| { - log.err("Failed to lookup backend for dir real path : {any}", .{err}); - return std.Io.Dir.Reader.Error.Unexpected; - }; - - const original_dir = reader.dir; - reader.dir = dir_; - defer reader.dir = original_dir; - - return backend.vtable.dirRead(backend.userdata, reader, entries); -} - -fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, null) catch |err| { - log.err("Failed to lookup backend for dir real path : {any}", .{err}); - return std.Io.Dir.RealPathError.Unexpected; - }; - - if (self.getScheme(backend_idx)) |s| { - const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer[prefix.len..]); - return prefix.len + path_len; - } else { - return try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer); - } -} - -fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const backend_idx, const dir_, const backend = self.lookupDir(dir, path_name) catch |err| { - log.err("Failed to lookup backend for dir real path file '{s}' : {any}", .{ path_name, err }); - return std.Io.Dir.RealPathFileError.Unexpected; - }; - - if (self.getScheme(backend_idx)) |s| { - const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); - return prefix.len + path_len; - } else { - return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); - } -} - -fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileStat(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); -} - -fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileLength(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }); -} - -fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - for (files) |file| { - const handle, const backend = self.getFileHandle(file); - backend.vtable.fileClose(backend.userdata, &.{.{ .handle = @intCast(handle.handle), .flags = handle.flags }}); - self.closeHandle(@intCast(file.handle)) catch unreachable; - } -} - -fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileWritePositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, header, data, splat, offset); -} - -fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, const src_backend = self.getFileHandle(reader.file); - if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; - - const original_src_io = reader.io; - const original_src_handle = reader.file; - reader.io = src_backend; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer { - reader.io = original_src_io; - reader.file = original_src_handle; - } - - return dst_backend.vtable.fileWriteFileStreaming(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit); -} - -fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const dst_handle, const dst_backend = self.getFileHandle(file); - const src_handle, const src_backend = self.getFileHandle(reader.file); - if (src_backend.userdata != dst_backend.userdata or src_backend.vtable != dst_backend.vtable) return error.Unimplemented; - - const original_src_io = reader.io; - const original_src_handle = reader.file; - reader.io = src_backend; - reader.file = .{ .handle = @intCast(src_handle.handle), .flags = src_handle.flags }; - defer { - reader.io = original_src_io; - reader.file = original_src_handle; - } - - return dst_backend.vtable.fileWriteFilePositional(dst_backend.userdata, .{ .handle = @intCast(dst_handle.handle), .flags = dst_handle.flags }, header, reader, limit, offset); -} - -fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, data, offset); -} - -fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileSeekBy(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, relative_offset); -} - -fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - return backend.vtable.fileSeekTo(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, absolute_offset); -} - -fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { - const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); - const handle, const backend = self.getFileHandle(file); - - if (self.getScheme(handle.backend_idx)) |s| { - const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); - const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer[prefix.len..]); - return prefix.len + path_len; - } else { - return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle), .flags = handle.flags }, out_buffer); - } -} - -test "copy a local file through the VFS" { - const allocator = std.testing.allocator; - const process_io = std.testing.io; - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - const source_data: [8192]u8 = @splat('x'); - try tmp.dir.writeFile(process_io, .{ - .sub_path = "source", - .data = &source_data, - }); - - var path_buffer: [std.Io.Dir.max_path_bytes]u8 = undefined; - const path_len = try tmp.dir.realPath(process_io, &path_buffer); - const source_uri = try std.fmt.allocPrint(allocator, "file://{s}/source", .{path_buffer[0..path_len]}); - defer allocator.free(source_uri); - const destination_uri = try std.fmt.allocPrint(allocator, "file://{s}/destination", .{path_buffer[0..path_len]}); - defer allocator.free(destination_uri); - - var file_backend: File = .init(allocator, process_io, .{}); - defer file_backend.deinit(); - var vfs: VFS = try .init(allocator, process_io); - defer vfs.deinit(); - try vfs.register("file", file_backend.io()); - - const vfs_io = vfs.io(); - { - var source = try std.Io.Dir.openFile(.cwd(), vfs_io, source_uri, .{}); - defer source.close(vfs_io); - var destination = try std.Io.Dir.createFile(.cwd(), vfs_io, destination_uri, .{}); - defer destination.close(vfs_io); - - var reader: std.Io.File.Reader = .initStreaming(source, vfs_io, &.{}); - var copy_buffer: [4096]u8 align(4096) = undefined; - var writer = destination.writer(vfs_io, ©_buffer); - try std.testing.expectEqual(source_data.len, try reader.interface.streamRemaining(&writer.interface)); - try writer.interface.flush(); - } - - const copied = try tmp.dir.readFileAlloc(process_io, "destination", allocator, .limited(source_data.len + 1)); - defer allocator.free(copied); - try std.testing.expectEqualSlices(u8, &source_data, copied); -} diff --git a/zml/BUILD.bazel b/zml/BUILD.bazel index f52ecdd5..f037d6b2 100644 --- a/zml/BUILD.bazel +++ b/zml/BUILD.bazel @@ -1,145 +1,52 @@ -load("@com_google_protobuf//bazel:upb_c_proto_library.bzl", "upb_c_proto_library") +load("@com_google_protobuf//bazel:upb_proto_library.bzl", "upb_c_proto_library") load("@rules_cc//cc:defs.bzl", "cc_library") load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") upb_c_proto_library( name = "xla_data_upb", - tags = ["manual"], deps = ["@xla//xla:xla_data_proto"], ) upb_c_proto_library( name = "xla_compile_options_upb", - tags = ["manual"], deps = ["@xla//xla/pjrt/proto:compile_options_proto"], ) -upb_c_proto_library( - name = "profiler_options", - tags = ["manual"], - deps = ["@tsl//tsl/profiler/protobuf:profiler_options_proto"], -) - cc_library( name = "posix", hdrs = ["posix.h"], - tags = ["manual"], -) - -cc_library( - name = "profiling_traceme_lib", - srcs = ["profiling/traceme_lib.cc"], - hdrs = ["profiling/traceme.h"], - defines = select({ - "//platforms:cuda.enabled": ["ZML_RUNTIME_CUDA"], - "//conditions:default": [], - }) + select({ - "//platforms:rocm.enabled": ["ZML_RUNTIME_ROCM"], - "//conditions:default": [], - }), - linkopts = select({ - "//platforms:rocm.enabled": ["-ldl"], - "//conditions:default": [], - }), - tags = ["manual"], - visibility = ["//visibility:public"], - deps = [ - "//ffi:cc", - "@tsl//tsl/profiler/lib:traceme", - "@tsl//tsl/profiler/protobuf:xplane_proto_cc", - "@xla//xla/tsl/profiler/backends/cpu:host_tracer_utils", - "@xla//xla/tsl/profiler/backends/cpu:traceme_recorder", - "@xla//xla/tsl/profiler/convert:post_process_single_host_xplane", - "@xla//xla/tsl/profiler/utils:time_utils", - "@xla//xla/tsl/profiler/utils:xplane_utils", - ] + select({ - "//platforms:cuda.enabled": [ - "//platforms/cuda:cuda_nvtx_headers", - ], - "//conditions:default": [], - }) + select({ - "//platforms:rocm.enabled": [ - "@amdrocm-profiler-base//:roctx_headers", - ], - "//conditions:default": [], - }), -) - -zig_library( - name = "logo", - srcs = ["logo.zig"], - import_name = "zml/logo", - main = "logo.zig", - visibility = ["//visibility:public"], ) zig_library( name = "zml", srcs = [ - "Compiler.zig", - "Sharding.zig", - "attention/Options.zig", + "attention.zig", "attention/attention.zig", + "attention/flashattn.zig", + "attention/paged_attention.zig", "buffer.zig", - "buffer/HostAccessible.zig", "constants.zig", "dtype.zig", "exe.zig", "floats.zig", "io.zig", - "kernels/flash_attention/flash_attention.zig", - "kernels/flash_attention/tests.zig", - "kernels/moe/api.zig", - "kernels/moe/moe.zig", - "kernels/moe/tests.zig", - "kernels/moe/weights.zig", - "kernels/paged_attention/options.zig", - "kernels/paged_attention/paged_attention.zig", - "logo.zig", "mem.zig", "meta.zig", "mlirx.zig", + "module.zig", "nn.zig", "ops.zig", - "ops/manual_computation.zig", "pjrtx.zig", "platform.zig", - "profiling/profiler.zig", - "profiling/tracer.zig", - "quantization.zig", "safetensors.zig", "shape.zig", "slice.zig", "tensor.zig", "testing.zig", + "tracer.zig", "zml.zig", - "//zml/kernel", - "//zml/kernels/flash_attention/cute", - "//zml/kernels/flash_attention/metal", - "//zml/kernels/flash_attention/nki", - "//zml/kernels/moe/cute", - "//zml/kernels/moe/flashinfer_cutlass", - "//zml/kernels/moe/fly", - "//zml/kernels/moe/metal", - "//zml/kernels/moe/mosaic_tpu", - "//zml/kernels/moe/stablehlo", - "//zml/kernels/moe/tcl", - "//zml/kernels/moe/triton", - "//zml/kernels/paged_attention/cute", - "//zml/kernels/paged_attention/metal", - "//zml/kernels/paged_attention/mosaic_tpu", - "//zml/kernels/paged_attention/stablehlo", - "//zml/kernels/paged_attention/tcl", - "//zml/kernels/paged_attention/triton", ], - data = select({ - "//platforms:neuron.enabled": [ - "//zml/kernels/flash_attention/nki:attention_kernel", - ], - "//conditions:default": [], - }), main = "zml.zig", - tags = ["manual"], visibility = ["//visibility:public"], zigopts = [ "-lc", @@ -147,53 +54,40 @@ zig_library( ], deps = [ ":posix", - ":profiler_options", - ":profiling_traceme_lib", ":xla_compile_options_upb", ":xla_data_upb", "//bazel", - "//dsl/cuda_tile:builder", - "//dsl/cute:builder", - "//dsl/fly:builder", - "//dsl/metal:builder", - "//dsl/mosaic_tpu:builder", - "//dsl/tcl:builder", - "//dsl/triton:builder", - "//ffi:zig", "//mlir", - "//mlir/dialects/func", - "//mlir/dialects/llvm", - "//mlir/dialects/mosaic_tpu", - "//mlir/dialects/nvvm", - "//mlir/dialects/shardy", - "//mlir/dialects/stablehlo", - "//mlir/dialects/ttir", - "//mlir/dialects/vector", + "//mlir/dialects", "//pjrt", "//platforms", - "//platforms/tpu:ragged_paged", "//stdx", - "//tools/xspace_to_perfetto:xspace_to_perfetto_lib", "//upb", - "//vfs", + "//zml/io", "//zml/tokenizer", "@rules_zig//zig/runfiles", - "@zio", ] + select({ - "//platforms:neuron.enabled": [ - "//platforms/neuron:nki_kernel", - "//platforms/neuron:topology", - ], - "//conditions:default": [], - }) + select({ "//platforms:cuda.enabled": [ - "//platforms/cuda/flashinfer_cutlass_moe", + "//platforms/cuda/flashattn", ], "//conditions:default": [], }), ) +# All ZML Tests + zig_test( name = "test", + data = [ + #"aio/torch/simple.pt", + #"aio/torch/simple_test_4.pickle", + ], + test_runner = ":test_runner", deps = ["//zml"], ) + +filegroup( + name = "test_runner", + srcs = ["test_runner.zig"], + visibility = ["//visibility:public"], +) diff --git a/zml/Compiler.zig b/zml/Compiler.zig deleted file mode 100644 index 93eafa8d..00000000 --- a/zml/Compiler.zig +++ /dev/null @@ -1,1096 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const pjrt = @import("pjrt"); -const shardy = @import("mlir/dialects/shardy"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const upb = @import("upb"); -const zio = @import("zio"); - -const Buffer = @import("buffer.zig").Buffer; -const DataType = @import("dtype.zig").DataType; -const Exe = @import("exe.zig").Exe; -const Memory = @import("platform.zig").Memory; -const meta = @import("meta.zig"); -const mlirx = @import("mlirx.zig"); -const ops = @import("ops.zig"); -const pjrtx = @import("pjrtx.zig"); -const Platform = @import("platform.zig").Platform; -const tracer = @import("profiling/tracer.zig"); -const Shape = @import("shape.zig").Shape; -const Sharding = @import("Sharding.zig"); -const Tensor = @import("tensor.zig").Tensor; -const TensorStore = @import("io.zig").TensorStore; - -const Compiler = @This(); -const log = std.log.scoped(.@"zml/Compiler"); - -allocator: std.mem.Allocator, -io: std.Io, -arena: std.heap.ArenaAllocator, - -options: Options, -mlir_registry: *mlir.DialectRegistry, -mlir_ctx: *mlir.Context, -mlir_pass_manager: *mlir.PassManager, -module: *mlir.Module, -platform: *const Platform, -meshes: []const *const Sharding.Mesh, -partitioner: Sharding.Partitioner, - -mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type), - -scopes: stdx.BoundedArray(Scope, 16) = .empty, -manual_computation_depth: usize = 0, -unknown_location: *const mlir.Location, -location: *const mlir.Location, - -channel_id: i64 = 0, -composite_id: i64 = 0, - -threadlocal var _current: ?*Compiler = null; - -var mlir_global_init_mutex: std.Io.Mutex = .init; -var mlir_global_registry: ?*mlir.DialectRegistry = null; - -fn mlirRegistry(io: std.Io) *mlir.DialectRegistry { - mlir_global_init_mutex.lockUncancelable(io); - defer mlir_global_init_mutex.unlock(io); - - if (mlir_global_registry == null) { - mlir.registerPasses("Transforms"); - - const mlir_registry = mlir.DialectRegistry.init() catch unreachable; - func.registerDialect(mlir_registry); - stablehlo.registerDialect(mlir_registry); - shardy.registerDialect(mlir_registry); - func.registerExtensions(mlir_registry); - - mlir_global_registry = mlir_registry; - } - - return mlir_global_registry.?; -} - -pub const Options = struct { - /// Incremented once after successful compilation. - progress: std.Progress.Node = .none, - meshes: []const *const Sharding.Mesh = &.{}, - store: ?*const TensorStore = null, - // If null, will be initialized from the target - partitioner: ?Sharding.Partitioner = null, - /// HACK: Extra bytes added to the device memory size reported to XLA. - /// Some buffers (e.g. engram tables) live in host-pinned memory but are declared to XLA as - /// device memory, because XLA only supports a few GPU operations reading host-pinned memory. - /// XLA then counts them against device memory, leaves no budget for temporary buffers and - /// rematerializes aggressively. Callers set this to the per-device size of such buffers to - /// compensate. Remove once XLA properly supports host-pinned inputs. - extra_device_memory_bytes: u64 = 0, - // Debugging options - program_name: []const u8 = "zml", - xla_dump_to: ?[]const u8 = null, - xla_dump_fusion_visualization: bool = false, - xla_dump_hlo_pass_re: ?[]const u8 = null, - xla_dump_emitter_re: ?[]const u8 = null, -}; - -/// Errors surfaced while lowering and compiling a ZML program. -pub const Error = std.mem.Allocator.Error || - std.Io.Writer.Error || - mlir.Error || - upb.SerializeError || - pjrtx.Client.CompileError || - error{ MissingDeviceInTile, OpenShardingAtBoundaries }; - -pub const Donation = union(enum) { - implicit: u32, - explicit: u32, -}; - -pub const Scope = struct { - compiler: *Compiler, - block: *mlir.Block, - id_to_argument: std.AutoArrayHashMapUnmanaged(Tensor.Id, *const mlir.Value), - id_to_donation: std.AutoArrayHashMapUnmanaged(Tensor.Id, Donation), - id_to_memory: std.AutoArrayHashMapUnmanaged(Tensor.Id, Memory.Kind), - arena: std.heap.ArenaAllocator, - - pub fn initFromBlock(compiler: *Compiler, block: *mlir.Block) Scope { - const arena: std.heap.ArenaAllocator = .init(compiler.allocator); - return .{ - .compiler = compiler, - .block = block, - .id_to_argument = .empty, - .id_to_donation = .empty, - .id_to_memory = .empty, - .arena = arena, - }; - } - - pub fn pop(self: *Scope) void { - const compiler = self.compiler; - std.debug.assert(self == compiler.currentScope()); - var arena = self.arena; - _ = compiler.scopes.pop(); - arena.deinit(); - } - - pub fn registerTensorAsBlockArgument(scope: *Scope, id: Tensor.Id, arg_id: usize) void { - scope.id_to_argument.put(scope.arena.allocator(), id, scope.block.argument(arg_id)) catch @panic("OOM"); - } - - pub fn value(scope: *Scope, x: Tensor) *const mlir.Value { - return scope.id_to_argument.get(x.id) orelse - x._value orelse - @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); - } -}; - -pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, opts: Options) Compiler { - var arena = std.heap.ArenaAllocator.init(allocator); - const mlir_registry = mlirRegistry(io); - var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; - mlir_ctx.loadAllAvailableDialects(); - - const unknown_location: *const mlir.Location = .unknown(mlir_ctx); - const module = mlir.Module.init(unknown_location); - module.operation().setAttributeByName("sym_name", .string(mlir_ctx, opts.program_name)); - - const pass_manager = mlir.PassManager.init(mlir_ctx); - { - var opm = pass_manager.asOpPassManager(); - const passes: []const []const u8 = &.{ - "canonicalize", - "cse", - "canonicalize", - }; - for (passes) |pass| { - opm.addPipeline(pass) catch unreachable; - } - } - - var mlir_known_types: std.enums.EnumArray(DataType, *const mlir.Type) = .initUndefined(); - { - for (0.., &mlir_known_types.values) |i, *mlir_type| { - mlir_type.* = mlirx.Type.fromDType(mlir_ctx, @fromBackingInt(@intCast(i))); - } - } - - var mesh_list: std.ArrayList(*const Sharding.Mesh) = .empty; - mesh_list.appendSlice(arena.allocator(), opts.meshes) catch @panic("OOM"); - if (std.mem.indexOfScalar(*const Sharding.Mesh, mesh_list.items, platform.replicated_mesh) == null) { - mesh_list.append(arena.allocator(), platform.replicated_mesh) catch @panic("OOM"); - } - const meshes = mesh_list.items; - validateMeshes(meshes) catch |err| stdx.debug.panic("Incompatible meshes: {t}", .{err}); - - return .{ - .allocator = allocator, - .io = io, - .arena = arena, - .mlir_registry = mlir_registry, - .mlir_ctx = mlir_ctx, - .mlir_pass_manager = pass_manager, - .mlir_known_types = mlir_known_types, - .module = module, - .platform = platform, - .options = opts, - .partitioner = opts.partitioner orelse .fromTarget(platform.target), - .meshes = meshes, - .location = unknown_location, - .unknown_location = unknown_location, - }; -} - -pub fn deinit(self: *Compiler) void { - if (_current == self) _current = null; - std.debug.assert(self.scopes.len == 0); - self.mlir_pass_manager.deinit(); - self.module.deinit(); - self.mlir_ctx.deinit(); - self.arena.deinit(); -} - -pub fn activate(self: *Compiler) void { - std.debug.assert(_current == null); - _current = self; -} - -pub fn deactivate(self: *Compiler) void { - _ = self; - _current = null; -} - -pub fn current() *Compiler { - return currentOrNull().?; -} - -pub fn currentOrNull() ?*Compiler { - return _current; -} - -pub fn currentScope(self: *Compiler) *Scope { - return &self.scopes.slice()[self.scopes.len - 1]; -} - -pub fn pushBlock(self: *Compiler, block: *mlir.Block) *Scope { - const scope = Scope.initFromBlock(self, block); - self.scopes.appendAssumeCapacity(scope); - return self.currentScope(); -} - -/// Change `Compiler.location` by appending a frame to the current stack of locations -/// Must be followed by a `defer compiler.popLocation` -pub fn pushLocation(self: *Compiler, location: std.builtin.SourceLocation, name: []const u8) void { - var callee: *const mlir.Location = .fromSrc(self.mlir_ctx, location); - if (name.len > 0) callee = callee.named(self.mlir_ctx, name); - self.location = if (self.location == self.unknown_location) callee else .callSite(callee, self.location); -} - -/// see `zml.Compiler.pushLocation` -pub fn pushLocationFmt(self: *Compiler, location: std.builtin.SourceLocation, comptime fmt: []const u8, args: anytype) void { - const name = std.fmt.allocPrint(self.allocator, fmt, args) catch { - return self.pushLocation(location, ""); - }; - defer self.allocator.free(name); - - return self.pushLocation(location, name); -} - -/// Remove last call frame pushed by `zml.Compiler.pushLocation` -pub fn popLocation(self: *Compiler) void { - switch (self.location.inspect()) { - .named => { - // We only pushed a named, when there was no parent location - self.location = self.unknown_location; - return; - }, - .callsite => |callsite| { - self.location = callsite.caller(); - return; - }, - .unknown, .file_line_col, .fused => @panic("unexpected location metadata"), - } -} - -test pushLocation { - const mlir_registry = mlirRegistry(std.testing.io); - var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; - mlir_ctx.loadAllAvailableDialects(); - - const unknown_location: *const mlir.Location = .unknown(mlir_ctx); - - var compiler: Compiler = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - .arena = undefined, - .mlir_registry = mlir_registry, - .mlir_ctx = mlir_ctx, - .mlir_pass_manager = undefined, - .mlir_known_types = undefined, - .module = undefined, - .options = .{}, - .platform = undefined, - .partitioner = undefined, - .meshes = undefined, - .location = unknown_location, - .unknown_location = unknown_location, - }; - - compiler.pushLocation(@src(), "loc1"); - const loc1 = compiler.location; - - compiler.pushLocation(@src(), "loc2"); - compiler.popLocation(); - - try std.testing.expectEqual(loc1, compiler.location); - compiler.popLocation(); - - try std.testing.expectEqual(unknown_location, compiler.location); -} - -pub fn nextChannelId(self: *Compiler) i64 { - self.channel_id += 1; - return self.channel_id; -} - -pub fn nextCompositeId(self: *Compiler) i64 { - self.composite_id += 1; - return self.composite_id; -} - -pub fn mlirType(self: *const Compiler, dt: DataType) *const mlir.Type { - return self.mlir_known_types.get(dt); -} - -pub fn dtype(self: *const Compiler, mlir_type: *const mlir.Type) DataType { - @setRuntimeSafety(false); - for (0.., &self.mlir_known_types.values) |i, known_type| { - if (known_type == mlir_type) return @fromBackingInt(@intCast(i)); - } - std.debug.panic("Can't convert unknown mlir type to dtype: {f}", .{mlir_type}); -} - -pub fn abortOOM(self: *Compiler) noreturn { - _ = self; - @panic("OOM"); -} - -pub fn alloc(self: *Compiler, T: type, n: usize) []T { - return self.arena.allocator().alloc(T, n) catch self.abortOOM(); -} - -pub fn allocPrint(self: *Compiler, comptime fmt: []const u8, args: anytype) []u8 { - return std.fmt.allocPrint(self.arena.allocator(), fmt, args) catch self.abortOOM(); -} - -/// Lookup a mesh by it's name -pub fn mesh(compiler: *const Compiler, name: @EnumLiteral()) *const Sharding.Mesh { - const name_slice = @tagName(name); - for (compiler.meshes) |m| { - if (std.mem.eql(u8, name_slice, m.name)) { - return m; - } - } - if (name == .replicated) return compiler.platform.replicated_mesh; - std.debug.panic( - \\Found no meshes named {s}. - \\Try passing more meshes to `zml.compile`. - \\Known meshes: {f} - , .{ name_slice, stdx.fmt.slice(compiler.meshes) }); -} - -/// Find a mesh that covers all the given axes -pub fn resolveMesh(compiler: *const Compiler, logical_axes: anytype) *const Sharding.Mesh { - if (@TypeOf(logical_axes) != []const Shape.Tag) { - return compiler.resolveMesh(comptime Shape.parseTags(logical_axes)); - } - - var ok_meshe: ?*const Sharding.Mesh = null; - for (compiler.meshes) |shd| { - var covers_all: bool = true; - for (logical_axes) |input_axis| { - var covers_this: bool = false; - for (shd.logical.axes.slice()) |existing_axis| { - if (std.mem.eql(u8, std.mem.span(existing_axis), std.mem.span(input_axis))) { - covers_this = true; - break; - } - } - covers_all = covers_all and covers_this; - } - if (covers_all) { - if (ok_meshe) |first_match| { - std.debug.panic( - \\Found two meshes covering axes: {any}, expected exacty one. - \\- First match: {f} - \\- Second match: {f} - , .{ logical_axes, first_match, shd }); - } - ok_meshe = shd; - } - } - - return ok_meshe orelse std.debug.panic( - \\Found no meshes covering axes: {any}, expected exacty one. - \\Try passing more meshes to `zml.compile`. - \\Known meshes: {f} - , .{ logical_axes, stdx.fmt.slice(compiler.meshes) }); -} - -pub fn Typed(comptime func_: anytype) type { - return struct { - pub fn compile( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - opts: Options, - args: std.meta.ArgsTuple(@TypeOf(func_)), - ) Error!Exe { - return Compiler.compile(allocator, io, platform, func_, args, opts); - } - }; -} - -pub fn compile( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - comptime func_: anytype, - args: std.meta.ArgsTuple(@TypeOf(func_)), - opts: Options, -) Error!Exe { - return switch (platform.io_impl) { - .threaded => compileInternal(allocator, io, platform, func_, args, opts), - .zio => try zio.blockInPlace(struct { - fn call(allocator_: std.mem.Allocator, io_: std.Io, platform_: *const Platform, args_: @TypeOf(args), opts_: Options) Error!Exe { - return try compileInternal(allocator_, io_, platform_, func_, args_, opts_); - } - }.call, .{ allocator, io, platform, args, opts }), - }; -} - -pub fn compileInternal( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - comptime func_: anytype, - args: std.meta.ArgsTuple(@TypeOf(func_)), - opts: Options, -) Error!Exe { - // TODO: Here we have somewhat of a requirement - // Emitting MLIR requires to have the compiler context available at all times using `Compiler.current()`. - // If in the future, we inject an Io that is not thread-based, we might have some surprises. - // - // I think the correct implementation would be to dispatch `emitMlir` to a thread pool, then wait for the result - // asynchronously using the provided Io. For now, we'll simply make that blocking as it's not a big deal but keep - // in mind we might want to revisit that later. - // - // This is NOT compatible with std.Io.Evented. - // It is compatible with zio, hence the blockInPlace above. - - const span_name = try tracer.formatSpanName(allocator, "zml.module.compile", .{ - .program_name = opts.program_name, - .arg_count = args.len, - }); - - defer allocator.free(span_name); - var span = tracer.Span.start(span_name); - defer span.end(); - - var compiler: Compiler = .init(allocator, io, platform, opts); - defer compiler.deinit(); - - var result = try compiler.emitMlir(func_, args); - defer result.output_info.deinit(compiler.allocator); - defer result.input_info.deinit(compiler.allocator); - - try addPartitionerOperations(&compiler); - - _ = result.func.appendTo(compiler.module.body()); - - // Mesh replication is encoded in shardings within one SPMD executable. - const num_partitions = compiler.meshes[0].numDevices(); - const num_replicas = 1; - const num_devices = num_partitions * num_replicas; - - compiler.module.operation().setAttributeByName( - "mhlo.num_partitions", - .int(compiler.mlir_ctx, .i32, num_partitions), - ); - compiler.module.operation().setAttributeByName( - "mhlo.num_replicas", - .int(compiler.mlir_ctx, .i32, num_replicas), - ); - - compiler.mlir_pass_manager.runOnOp(compiler.module.operation()) catch |err| switch (err) { - error.MlirUnexpected => { - compiler.handleCompilationError(io, compiler.arena.allocator(), opts, err, "Failed to canonicalize invalid MLIR"); - @panic("ZML generated invalid mlir. Please open a bug report"); - }, - }; - - const loaded_executable = compiler.compileModuleToPjrtExecutable(opts) catch |err| { - compiler.handleCompilationError(io, compiler.arena.allocator(), opts, err, "Pjrt failed to compile the following MLIR"); - return err; - }; - - log.debug("\n******** ZML generated MLIR ********\n{f}", .{compiler.module.operation()}); - - const exe = try Exe.init( - allocator, - platform, - loaded_executable, - num_devices, - num_partitions, - // This will get copied into exe - result.input_info.items(.shape), - result.output_info.items(.shape), - result.input_info.items(.aliasing_output), - ); - errdefer exe.deinit(); - - opts.progress.completeOne(); - return exe; -} - -/// Declare all logical meshes used by this program -fn addPartitionerOperations(ctx: *Compiler) !void { - const mlir_ctx = ctx.mlir_ctx; - const module = ctx.module; - - switch (ctx.partitioner) { - .gspmd => { - // GSPMD doesn't use meshes, but explicit partitioning operations. - }, - .shardy => { - for (ctx.meshes) |shd| { - _ = shardy.mesh(mlir_ctx, shd.name, shd.sdyMeshAttr(mlir_ctx), .unknown(mlir_ctx)).appendTo(module.body()); - } - }, - } -} - -const EmitMlirResult = struct { - func: *mlir.Operation, - input_info: std.MultiArrayList(TensorInfo), - output_info: std.MultiArrayList(TensorInfo), -}; - -pub const TensorInfo = struct { - id: Tensor.Id, - shape: Shape, - value: *const mlir.Value, - - // Only used for input tensors, stores which output tensor ends up with their buffer - aliasing_output: ?u32 = null, - - pub fn attributes(info: TensorInfo, arena: std.mem.Allocator, compiler: *const Compiler, scope: *const Scope) error{OutOfMemory}!AttributeList { - var attrs: AttributeList = .empty; - - const mlir_ctx = compiler.mlir_ctx; - const mesh_attr = try compiler.tensorShardingAttr(arena, info.shape); - attrs.appendAssumeCapacity(switch (compiler.partitioner) { - .gspmd => .named(mlir_ctx, "mhlo.sharding", mesh_attr), - .shardy => shardy.shardingNamedAttr(mlir_ctx, mesh_attr.isA(shardy.TensorShardingAttr).?), - }); - - const memory = scope.id_to_memory.get(info.id) orelse .device; - if (memory != .device) { - attrs.appendAssumeCapacity(.named( - mlir_ctx, - "mhlo.memory_kind", - .string(mlir_ctx, compiler.platform.memoryKind(memory)), - )); - } - - if (info.aliasing_output) |output_index| { - attrs.appendAssumeCapacity( - .named(mlir_ctx, "tf.aliasing_output", .int(mlir_ctx, .i32, output_index)), - ); - } - - return attrs; - } -}; - -const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); - -fn emitMlir(compiler: *Compiler, comptime func_: anytype, args: std.meta.ArgsTuple(@TypeOf(func_))) !EmitMlirResult { - const module = mlir.Module.init(.unknown(compiler.mlir_ctx)); - errdefer module.deinit(); - - const block = mlir.Block.init(&.{}, &.{}); - errdefer block.deinit(); - - const fn_scope: *Scope = compiler.pushBlock(block); - defer fn_scope.pop(); - - var input_info = try createBlockArguments(compiler, fn_scope, &args); - errdefer input_info.deinit(compiler.allocator); - - var output_info = blk: { - compiler.activate(); - defer compiler.deactivate(); - - var result = @call(.auto, func_, args); - // Output collection can emit replication constraints, so it is part of tracing. - break :blk try collectOutputInfo(compiler, fn_scope, &result); - }; - errdefer output_info.deinit(compiler.allocator); - - // Finalize in a separate function that doesn't depend on `func` type. - return try finalizeMlirFunc(compiler, fn_scope, input_info, output_info); -} - -fn createBlockArguments(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { - const CreateBlockArgumentsCb = struct { - compiler: *Compiler, - scope: *Scope, - - current_argument_id: u32 = 0, - infos: std.MultiArrayList(TensorInfo) = .empty, - - fn cb(ctx: *@This(), tensor: *const Tensor) error{ OutOfMemory, OpenShardingAtBoundaries }!void { - // Declare the program argument as a packed u8 - const og_shape = tensor._shape; - if (og_shape.containsPartitionSpec(.open)) { - log.err("Executable input {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.current_argument_id, og_shape }); - return error.OpenShardingAtBoundaries; - } - const packed_shape = og_shape.packedShape(); - const mlir_ctx = ctx.compiler.mlir_ctx; - const mlir_type = mlirx.Type.rankedTensor(mlir_ctx, packed_shape); - var location: *const mlir.Location = ctx.compiler.unknown_location; - if (ctx.compiler.options.store) |store| { - if (store.id_to_sources.get(tensor.id)) |sources| { - location = location.named(mlir_ctx, sources.tensors[0].name); - } - } - var value = ctx.scope.block.addArgument(mlir_type, location); - - // But immediately create the MLIR value corresponding to the fp4 data. - // That way the user code only sees a proper fp4 tensor. - if (tensor.dtype().bitSizeOf() < 8) { - value = unpack(ctx.compiler, ctx.scope, og_shape, value); - } - const gop = ctx.scope.id_to_argument.getOrPutValue(ctx.scope.arena.allocator(), tensor.id, value) catch @panic("OOM"); - if (gop.found_existing) std.debug.panic("Tensor with id {} has already been used once as an argument", .{tensor.id}); - - // Associate each input argument with their own buffer - ctx.scope.id_to_donation.putNoClobber(ctx.scope.arena.allocator(), tensor.id, .{ .implicit = ctx.current_argument_id }) catch @panic("OOM"); - - defer ctx.current_argument_id += 1; - - try ctx.infos.append(ctx.compiler.allocator, .{ - .id = tensor.id, - .shape = og_shape, - .value = value, - }); - } - }; - - var context: CreateBlockArgumentsCb = .{ .compiler = compiler, .scope = scope }; - errdefer context.infos.deinit(compiler.allocator); - - try meta.visit(CreateBlockArgumentsCb.cb, &context, v); - - return context.infos; -} - -fn collectOutputInfo(compiler: *Compiler, scope: *Scope, v: anytype) error{ OutOfMemory, OpenShardingAtBoundaries }!std.MultiArrayList(TensorInfo) { - const CollectOutputInfoCb = struct { - compiler: *Compiler, - scope: *Scope, - - infos: std.MultiArrayList(TensorInfo) = .empty, - - fn cb(ctx: *@This(), tensor: *const Tensor) !void { - // Outputs with no declared sharding default to replication. Constrain the - // returned value so the partitioner produces a replicated output buffer. - const output = if (tensor.shape()._sharding.mesh != null or tensor.shape().isFullyReplicated()) tensor.* else tensor.replicate(); - const og_shape = output.shape(); - if (og_shape.containsPartitionSpec(.open)) { - log.err("Executable output {d} has open sharding: {f}. Specify a logical mesh axis or .replicated for every dimension.", .{ ctx.infos.len, og_shape }); - return error.OpenShardingAtBoundaries; - } - var value = ctx.scope.id_to_argument.get(output.id) orelse - output._value orelse - @panic("no value found for output tensor"); - - if (tensor.dtype().bitSizeOf() < 8) { - value = repack(ctx.compiler, ctx.scope, og_shape, value); - } - - try ctx.infos.append(ctx.compiler.allocator, .{ - .id = tensor.id, - // const packed_shape = og_shape.packedShape(); - // TODO: clarify why this og_shape and not packedShape() - .shape = og_shape, - .value = value, - }); - } - }; - - var context: CollectOutputInfoCb = .{ .compiler = compiler, .scope = scope }; - errdefer context.infos.deinit(compiler.allocator); - - try meta.visit(CollectOutputInfoCb.cb, &context, v); - return context.infos; -} - -fn finalizeMlirFunc(compiler: *Compiler, fn_scope: *Scope, input_info: std.MultiArrayList(TensorInfo), output_info: std.MultiArrayList(TensorInfo)) error{OutOfMemory}!EmitMlirResult { - var arena_state = std.heap.ArenaAllocator.init(compiler.allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - const mlir_ctx = compiler.mlir_ctx; - - const fn_return = func.returns(mlir_ctx, output_info.items(.value), .unknown(mlir_ctx)); - _ = fn_return.appendTo(fn_scope.block); - - // Resolve donations - for (0.., output_info.items(.id)) |output_index, output_id| { - if (fn_scope.id_to_donation.get(output_id)) |donation| { - const donated_input = switch (donation) { - // don't emit implicit donation since they modify the way the function is called - .implicit => continue, - .explicit => |input| input, - }; - const aliasing_output: *?u32 = &input_info.items(.aliasing_output)[donated_input]; - if (aliasing_output.*) |previous_aliased_output| { - std.debug.panic("Input {d} buffer {} was reused twice with `reuseBuffer` for output {} and output {}. Expected `reuseBuffer` to be called at most once", .{ donated_input, input_info.items(.shape)[donated_input], previous_aliased_output, output_index }); - } - aliasing_output.* = @intCast(output_index); - } - } - - // Input mesh/memory/aliasing attributes - const input_attributes = try arena.alloc(*const mlir.Attribute, input_info.len); - for (0.., input_attributes) |i, *input_attrs| { - const attrs_list = try input_info.get(i).attributes(arena, compiler, fn_scope); - input_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); - } - - // Output mesh/memory attributes - const output_attributes = try arena.alloc(*const mlir.Attribute, output_info.len); - for (0.., output_attributes) |i, *output_attrs| { - const attrs_list = try output_info.get(i).attributes(arena, compiler, fn_scope); - output_attrs.* = .dict(mlir_ctx, attrs_list.constSlice()); - } - - const input_types = try arena.alloc(*const mlir.Type, input_info.len); - for (input_types, 0..) |*input_type, i| input_type.* = fn_scope.block.argument(i).type_(); - - const output_types = try arena.alloc(*const mlir.Type, output_info.len); - for (output_types, output_info.items(.value)) |*output_type, value| output_type.* = value.type_(); - - const mlir_func = func.func(mlir_ctx, .{ - .name = "main", - .args = input_types, - .results = output_types, - .block = fn_scope.block, - .location = .unknown(mlir_ctx), - .args_attributes = input_attributes, - .results_attributes = output_attributes, - .verify = false, - }); - - return .{ - .func = mlir_func, - .input_info = input_info, - .output_info = output_info, - }; -} - -fn unpack(compiler: *Compiler, scope: *Scope, og_shape: Shape, tensor_value: *const mlir.Value) *const mlir.Value { - const true_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape); - const elem_per_bytes = @divExact(8, og_shape.dtype().bitSizeOf()); - const intermediary_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.splitAxis(-1, .{ -1, elem_per_bytes })); - - const bit_cast_op = stablehlo.bitcast_convert( - compiler.mlir_ctx, - tensor_value, - intermediary_type, - .unknown(compiler.mlir_ctx), - ).appendTo(scope.block); - - const reshape_op = stablehlo.reshape( - compiler.mlir_ctx, - bit_cast_op.result(0), - true_type, - .unknown(compiler.mlir_ctx), - ).appendTo(scope.block); - - return reshape_op.result(0); -} - -fn repack(compiler: *Compiler, scope: *Scope, og_shape: Shape, value: *const mlir.Value) *const mlir.Value { - const packed_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.packedShape()); - const elem_per_bytes = @divExact(8, og_shape.dtype().bitSizeOf()); - const intermediary_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, og_shape.splitAxis(-1, .{ -1, elem_per_bytes })); - - const reshape_op = stablehlo.reshape( - compiler.mlir_ctx, - value, - intermediary_type, - .unknown(compiler.mlir_ctx), - ).appendTo(scope.block); - - const bit_cast_op = stablehlo.bitcast_convert( - compiler.mlir_ctx, - reshape_op.result(0), - packed_type, - .unknown(compiler.mlir_ctx), - ).appendTo(scope.block); - - return bit_cast_op.result(0); -} - -fn validateMeshes(meshes: []const *const Sharding.Mesh) !void { - stdx.debug.assert(meshes.len >= 1, "Waiting at leat 1 mesh strategy to be implemented", .{}); - - for (meshes) |m| { - const num_replicas = m.numReplicas(); - if (num_replicas > 1 and !std.mem.eql(u8, "replicated", m.name)) { - // This seems like a performance footgun, rather that something anyone would want to use. - // It's generally better to have meshes that cover all devices, - // even if later it means that some mesh axis is never used. - // Otherwise a "sharded" tensor can in fact be replicated because the underlying mesh has replicas - // TODO(#914): decide if we should forbid this altogether - // currently DirectMemoryWriter tests have replicated meshes depending on the platform they target - log.warn("Mesh({s}) has numReplicas() == {d}. This imply that all tensor put on this mesh will be implicitly replicated {d} times.\n{f}", .{ m.name, num_replicas, num_replicas, m }); - } - log.debug("Using {f}", .{m}); - } - - const first = meshes[0]; - const devices = first.numDevices(); - - for (meshes[1..]) |m| { - if (m.numDevices() != devices) { - // todo: deviceAssignments should also be checked for consistency here, but for simplicity we just check the cardinality numbers - return error.InconsistentMesheCardinality; - } - } -} - -pub fn tensorShardingAttr( - compiler: *const Compiler, - allocator: std.mem.Allocator, - shape: Shape, -) error{OutOfMemory}!*const mlir.Attribute { - const mesh_ = shape._sharding.mesh orelse compiler.platform.replicated_mesh; - return switch (compiler.partitioner) { - .shardy => (try mesh_.sdyShardingAttrForShape(allocator, compiler.mlir_ctx, shape)).attribute(), - .gspmd => mesh_.gspmdShardingAttrForShape(allocator, compiler.mlir_ctx, shape) catch |err| switch (err) { - error.WriteFailed => error.OutOfMemory, // We're writing to memory - error.OutOfMemory => error.OutOfMemory, - error.IncompatibleSharding => @panic("Shape dimensions must be divisible by their mesh partitions"), - }, - }; -} - -fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { - const result = c.upb_Map_Set( - map, - .{ .str_val = upb.stringView(flag) }, - .{ .msg_val = blk: { - const field = try upb.new(c.xla_OptionOverrideProto, upb_arena); - switch (@typeInfo(@TypeOf(value))) { - .bool => c.xla_OptionOverrideProto_set_bool_field(field, value), - .comptime_int, .int => c.xla_OptionOverrideProto_set_int_field(field, @intCast(value)), - .comptime_float, .float => c.xla_OptionOverrideProto_set_double_field(field, @floatCast(value)), - else => c.xla_OptionOverrideProto_set_string_field(field, upb.stringView(value)), - } - break :blk @ptrCast(field); - } }, - upb_arena, - ); - - if (result == false) { - return std.mem.Allocator.Error.OutOfMemory; - } -} - -fn compileModuleToPjrtExecutable(compiler: *Compiler, opts: Options) !*pjrt.LoadedExecutable { - const use_shardy_partitioner = switch (compiler.partitioner) { - .shardy => true, - .gspmd => false, - }; - - const platform = compiler.platform; - const main_mesh = compiler.meshes[0]; - // Mesh replicas are part of the SPMD device set, not XLA program replicas. - const num_partitions: i32 = @intCast(main_mesh.numDevices()); - const num_replicas = 1; - - const device_assignment = try main_mesh.deviceAssignment(compiler.allocator); - defer compiler.allocator.free(device_assignment); - - var arena = std.heap.ArenaAllocator.init(compiler.allocator); - defer arena.deinit(); - var upb_alloc: upb.Allocator = .init(arena.allocator()); - const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); - defer c.upb_Arena_Free(upb_arena); - const options = blk: { - const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); - c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { - const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); - c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); - c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, num_replicas); - c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, num_partitions); - c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, true); - c.xla_ExecutableBuildOptionsProto_set_use_shardy_partitioner(exec_build_options, use_shardy_partitioner); - - // Tell XLA how much device memory PJRT actually made available. - // Without this, the GPU scheduler falls back to 80% of physical - // memory. Programs whose donated inputs and outputs exceed that - // artificial limit get a zero-byte temporary-memory budget and - // can trigger excessive rematerialization. - var device_memory_size: ?u64 = null; - for (platform.devices) |device| { - const bytes_limit = device.memoryStats().bytes_limit orelse { - device_memory_size = null; - break; - }; - device_memory_size = @min(device_memory_size orelse bytes_limit, bytes_limit); - } - if (device_memory_size) |bytes_limit| { - // HACK: See `Options.extra_device_memory_bytes`. - c.xla_ExecutableBuildOptionsProto_set_device_memory_size(exec_build_options, @intCast(bytes_limit + opts.extra_device_memory_bytes)); - } - - c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { - const device_assignment_proto = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); - - c.xla_DeviceAssignmentProto_set_replica_count(device_assignment_proto, num_replicas); - c.xla_DeviceAssignmentProto_set_computation_count(device_assignment_proto, num_partitions); - - const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices( - device_assignment_proto, - @intCast(num_partitions), - upb_arena, - ); - - for (computation_devices[0..@intCast(num_partitions)], 0..) |*computation_device, i| { - computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); - _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids( - computation_device.*, - @intCast(device_assignment[@intCast(i)]), - upb_arena, - ); - } - - break :device_assignment_blk device_assignment_proto; - }); - - break :executable_build_options_blk exec_build_options; - }); - - const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); - switch (platform.target) { - .cuda => { - // NVIDIA recommends these settings - // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_scaled_dot_with_tile_ir", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_enable_subchannel_dequantisation_fusion", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_unsupported_enable_triton_multi_output_fusion", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); - // Add collectives to the default list - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES_KERNEL,COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); - // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings - // if VMM reclaims their backing under memory pressure, causing silent corruption. - // Enable user buffers to assign color 1 and receive the reclaim exemption: - // https://github.com/openxla/xla/pull/46029 - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); - // Enable for both Blackwell+ and Ampere+ - // try setXlaOverrideFlag(overrides_map, "xla_gpu_cudnn_gemm_fusion_level", 2, upb_arena); - }, - .rocm => { - try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_update_mode", "SKIP_TEMP", upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_collective_kernels", "COLLECTIVE_KERNEL_ALL_REDUCE", upb_arena); - // Only capture all-reduce and all-gather collectives in HIP graphs; other collective types execute outside command buffers since they are unsupported in HIP graphs. - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_collectives_command_buffer_filter", "ALLREDUCE,ALLGATHER", upb_arena); - // Add collectives to the default list - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "COLLECTIVES_KERNEL,COLLECTIVES,CUBLAS,CUBLASLT,CUDNN,CUSTOM_CALL,DYNAMIC_SLICE_FUSION,FUSION", upb_arena); - // With SKIP_TEMP, captured NCCL collectives can retain stale physical mappings - // if VMM reclaims their backing under memory pressure, causing silent corruption. - // Enable user buffers to assign color 1 and receive the reclaim exemption: - // https://github.com/openxla/xla/pull/46029 - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_nccl_user_buffers", true, upb_arena); - // Enables fusion to produce output for multiple separate output. - try setXlaOverrideFlag(overrides_map, "xla_gpu_unsupported_enable_triton_multi_output_fusion", true, upb_arena); - // Rearranges operations so that communication between GPUs happens in the background while the compute units are busy doing computation. - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); - }, - .metal => { - try setXlaOverrideFlag(overrides_map, "xla_gpu_ftz", true, upb_arena); - }, - .oneapi => { - // More efficient for the allgather/broadcast implementation of the collective permute. - try setXlaOverrideFlag(overrides_map, "xla_gpu_collective_permute_connected_components", true, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_autotune_level", 0, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "CUBLASLT,CUSTOM_CALL,FUSION", upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_gpu_command_buffer_scheduling_mode", "CONCURRENT", upb_arena); - - // Grouped GEMM still needs a oneDNN lowering. - try setXlaOverrideFlag(overrides_map, "xla_gpu_experimental_use_ragged_dot_grouped_gemm", false, upb_arena); - }, - else => {}, - } - - if (opts.xla_dump_to) |xla_dump_to| { - try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); - try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); - if (opts.xla_dump_fusion_visualization) { - try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); - } - if (opts.xla_dump_hlo_pass_re) |re| { - try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); - } - if (opts.xla_dump_emitter_re) |re| { - try setXlaOverrideFlag(overrides_map, "xla_dump_emitter_re", re, upb_arena); - } - } - - switch (platform.target) { - .rocm, .cuda, .oneapi => if (std.c.getenv("ZML_AUTOTUNE_CACHE_DIR")) |path| { - try setXlaOverrideFlag(overrides_map, "xla_gpu_per_fusion_autotune_cache_dir", std.mem.span(path), upb_arena); - }, - else => {}, - } - - break :blk options; - }; - - const loaded_executable = try pjrtx.Client.compile( - platform.pjrt_client, - platform.pjrt_api, - compiler.allocator, - compiler.io, - compiler.module, - try upb.serialize(options, upb_arena), - ); - errdefer loaded_executable.deinit(); - - return loaded_executable; -} - -fn handleCompilationError(compiler: *Compiler, io: std.Io, allocator: std.mem.Allocator, opts: Options, err: anyerror, msg: []const u8) void { - const module_op = compiler.module.operation().fmt(.{ .debug_info = true, .debug_info_pretty_form = true }); - const truncated_msg = "..."; - var too_big: bool = true; - - const buffer: []u8 = allocator.alloc(u8, 8192) catch { - log.err("{s} ({}):\n{f}", .{ msg, err, module_op }); - return; - }; - defer allocator.free(buffer); - - var buffer_writer: std.Io.Writer = .fixed(buffer); - const module_mlir: []const u8 = module_mlir: { - module_op.format(&buffer_writer) catch { - @memcpy(buffer[8192 - truncated_msg.len ..], truncated_msg); - too_big = true; - break :module_mlir buffer; - }; - too_big = false; - break :module_mlir buffer_writer.buffered(); - }; - - log.err("{s} ({}):\n{s}", .{ msg, err, module_mlir }); - if (!too_big) return; - - if (opts.xla_dump_to == null) { - log.warn("To see the full .mlir, set `zml.Compiler.Options.xla_dump_to`", .{}); - return; - } - const xla_dir = opts.xla_dump_to.?; - const dir = std.Io.Dir.openDirAbsolute(io, xla_dir, .{}) catch dir: { - std.Io.Dir.createDirAbsolute(io, xla_dir, .default_dir) catch |e| { - log.err("failed to dump mlir to {s}: {}", .{ xla_dir, e }); - return; - }; - break :dir std.Io.Dir.openDirAbsolute(io, xla_dir, .{}) catch unreachable; - }; - const filename = std.fmt.allocPrint(allocator, "{s}.mlir", .{opts.program_name}) catch return; - const mlir_file = dir.createFile(io, filename, .{ .truncate = true }) catch |e| { - log.err("failed to dump mlir to {s}/{s}: {}", .{ xla_dir, filename, e }); - return; - }; - - // Reuse the previously allocated buffer - var mlir_file_writer = mlir_file.writer(io, buffer); - module_op.format(&mlir_file_writer.interface) catch |e| { - log.err("Partial mlir dump at: {s}/{s} ({})", .{ xla_dir, filename, e }); - return; - }; - log.warn("Full .mlir at: {s}/{s}", .{ xla_dir, filename }); -} diff --git a/zml/Sharding.zig b/zml/Sharding.zig deleted file mode 100644 index 040216bc..00000000 --- a/zml/Sharding.zig +++ /dev/null @@ -1,3290 +0,0 @@ -//! Explain how to split a buffer across different devices. -const std = @import("std"); -const builtin = @import("builtin"); - -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const pjrt = @import("pjrt"); -const platforms = @import("platforms"); -const shardy = @import("mlir/dialects/shardy"); -const stdx = @import("stdx"); - -const Platform = @import("platform.zig").Platform; -const PlatformDevice = @import("platform.zig").Device; -const Shape = @import("shape.zig").Shape; -const Slice = @import("slice.zig").Slice; -const Target = @import("platform.zig").Target; - -const Sharding = @This(); - -mesh: ?*const Mesh, -partition: Partitioning, - -pub const MAX_MESH_RANK = 4; - -/// Full replication across participating devices. -/// For buffers, each device holds a complete copy. Applied as a graph constraint, -/// this closes every dimension to partitioning. `Shape.init` uses this sharding. -/// Unconstrained graph results use `open` instead. -pub const replicated: Sharding = .{ .mesh = null, .partition = .replicated(Shape.MAX_RANK) }; - -/// Unspecified sharding for graph results. During compilation, the partitioner -/// may infer sharding from surrounding constraints or leave the value replicated. -/// Executable inputs require closed sharding. An executable output with this -/// unspecified sharding defaults to replication. An explicitly declared output -/// sharding containing open dimensions is rejected. -pub const open: Sharding = .{ .mesh = null, .partition = .open(Shape.MAX_RANK) }; - -pub fn get(sharding: Sharding, ax: usize) PartitionSpec { - return sharding.partition.get(ax); -} - -pub fn set(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { - var res = self; - res.partition = res.partition.set(ax, spec); - return res; -} - -pub fn insert(self: Sharding, ax: usize, spec: PartitionSpec) Sharding { - var res = self; - res.partition = res.partition.insert(ax, spec); - return res; -} - -pub fn orderedRemove(self: Sharding, ax: usize) Sharding { - var res = self; - res.partition = res.partition.orderedRemove(ax); - return res; -} - -pub fn toArray(self: Sharding) [Shape.MAX_RANK]PartitionSpec { - var result: [Shape.MAX_RANK]PartitionSpec = undefined; - for (&result, 0..) |*spec, ax| spec.* = self.get(ax); - return result; -} - -pub fn isFullyReplicated(self: Sharding) bool { - for (self.partition.toArray()) |spec| { - if (spec != .replicated and spec != .out_of_bound) return false; - } - return true; -} - -/// An unspecified mesh is compatible with any sharding. On the same mesh, -/// open axes accept any in-bounds partition, while closed axes must match. -pub fn isCompatible(self: Sharding, other: Sharding) bool { - if (self.mesh == null or other.mesh == null) return true; - if (self.mesh != other.mesh) return false; - for (self.partition.toArray(), other.partition.toArray()) |lhs, rhs| { - if (lhs == rhs) continue; - if (lhs == .out_of_bound or rhs == .out_of_bound) return false; - if (lhs != .open and rhs != .open) return false; - } - return true; -} - -test isCompatible { - const tp: Mesh = .initTest(.tp, .{.model}); - const dp: Mesh = .initTest(.dp, .{ .data, .model }); - - const sharded: Sharding = .{ .mesh = &tp, .partition = .init(&.{ .mesh_axis_0, .replicated }) }; - const cases = [_]struct { lhs: Sharding, rhs: Sharding, compatible: bool }{ - .{ .lhs = .replicated, .rhs = .replicated, .compatible = true }, - .{ .lhs = .replicated, .rhs = sharded, .compatible = true }, - .{ .lhs = sharded, .rhs = sharded, .compatible = true }, - .{ .lhs = sharded, .rhs = .{ .mesh = &tp, .partition = .open(2) }, .compatible = true }, - .{ .lhs = sharded.set(1, .open), .rhs = sharded.set(0, .open), .compatible = true }, - .{ .lhs = sharded, .rhs = sharded.set(0, .mesh_axis_1), .compatible = false }, - .{ .lhs = sharded, .rhs = sharded.set(0, .replicated), .compatible = false }, - .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = sharded.partition }, .compatible = false }, - .{ .lhs = sharded, .rhs = .{ .mesh = &dp, .partition = .open(2) }, .compatible = false }, - .{ .lhs = sharded, .rhs = sharded.set(1, .out_of_bound), .compatible = false }, - .{ .lhs = sharded, .rhs = sharded.set(2, .open), .compatible = false }, - }; - for (cases) |case| { - errdefer std.log.err("{f}.isCompatible({f}) expected to be {}, got {}", .{ case.lhs, case.rhs, case.compatible, !case.compatible }); - - try std.testing.expectEqual(case.compatible, case.lhs.isCompatible(case.rhs)); - try std.testing.expectEqual(case.compatible, case.rhs.isCompatible(case.lhs)); - } -} - -/// Formats the declared sharding, not the partitioner's inferred sharding. -/// Without a mesh, prints `{replicated}` for full replication or `{open}` for -/// unspecified graph sharding. -/// -/// With a mesh, entries follow tensor dimension order, e.g. `{data,replicated,?}@dp`: -/// - `data`: split this dimension over the logical mesh axis named `data`. -/// - `replicated`: keep this dimension whole on each shard. -/// - `?`: leave this dimension to partitioner inference during compilation. -/// - `@dp`: the mesh name. -/// Trailing `.out_of_bound` entries are omitted. Any remaining ones print as `!`. -pub fn format(self: Sharding, writer: *std.Io.Writer) !void { - const mesh = self.mesh orelse return writer.writeAll(if (self.isFullyReplicated()) "{replicated}" else "{open}"); - const specs = self.partition.toArray(); - var rank = specs.len; - while (rank > 0 and specs[rank - 1] == .out_of_bound) rank -= 1; - - try writer.writeByte('{'); - for (specs[0..rank], 0..) |spec, i| { - if (i > 0) try writer.writeByte(','); - switch (spec) { - .replicated => try writer.writeAll("replicated"), - .open => try writer.writeByte('?'), - .out_of_bound => try writer.writeByte('!'), - else => try writer.print("{s}", .{mesh.logical.axes.get(spec.meshAxis().?)}), - } - } - try writer.print("}}@{s}", .{mesh.name}); -} - -/// Describes how one tensor dimension is distributed across a logical mesh. -/// A mesh axis splits the dimension. `.replicated` keeps it whole on each shard. -/// In a graph constraint, `.open` lets the partitioner choose either behavior. -/// Executable buffers require closed sharding specs: a mesh axis or `.replicated`. -pub const PartitionSpec = enum(u4) { - mesh_axis_0 = 0, - mesh_axis_1 = 1, - mesh_axis_2 = 2, - mesh_axis_3 = 3, - mesh_axis_4 = 4, - mesh_axis_5 = 5, - mesh_axis_6 = 6, - mesh_axis_7 = 7, - // an 8D mesh seems already a lot, the max we know about is 3D. - replicated = 8, - - // Partitioning don't encode the rank of corresponding tensor, so this is used to mark invalid positions. - out_of_bound = 0xa, - open = 15, - - pub fn sharded(mesh_axis: u3) PartitionSpec { - return @fromBackingInt(@intCast(mesh_axis)); - } - - /// Extract the mesh axis along which we are sharded. Null if not sharded. - pub fn meshAxis(self: PartitionSpec) ?u3 { - const ax = @backingInt(self); - return if (ax < @backingInt(PartitionSpec.replicated)) @intCast(ax) else null; - } - - pub fn isSharded(self: PartitionSpec) bool { - return @backingInt(self) < @backingInt(PartitionSpec.replicated); - } - - pub fn isClosed(self: PartitionSpec) bool { - return @backingInt(self) <= @backingInt(PartitionSpec.replicated); - } - - test isClosed { - try std.testing.expect(PartitionSpec.mesh_axis_2.isClosed()); - try std.testing.expect(!PartitionSpec.open.isClosed()); - - try std.testing.expect(PartitionSpec.replicated.isClosed()); - - try std.testing.expect(!PartitionSpec.out_of_bound.isClosed()); - } -}; - -pub const Partitioner = union(enum) { - shardy, - gspmd, - - pub fn fromTarget(target: Target) Partitioner { - return switch (target) { - .cpu, .cuda, .rocm, .tpu, .oneapi, .neuron, .metal, .furiosa => .shardy, - }; - } -}; - -fn shapeHasAxisPartition(shape: Shape) bool { - for (0..shape.rank()) |ax| { - if (shape.partition(ax).meshAxis()) |_| return true; - } - return false; -} - -pub const DimSharding = union(enum) { - sharded: struct { - dim: i64, - factor: u32, - }, - replicated, -}; - -pub fn sdyPerValueShardingAttr( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - shapes: []const Shape, - mesh: *const Mesh, -) error{OutOfMemory}!*const shardy.TensorShardingPerValueAttr { - const meshe_attrs = try allocator.alloc(*const shardy.TensorShardingAttr, shapes.len); - for (meshe_attrs, shapes) |*attr, shape| { - attr.* = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); - } - return .get(ctx, meshe_attrs); -} - -pub fn sdyManualAxesAttr( - allocator: std.mem.Allocator, - ctx: *mlir.Context, - in_shapes: []const Shape, - out_shapes: []const Shape, - mesh: *const Mesh, -) error{OutOfMemory}!*const shardy.ManualAxesAttr { - var axis_names = std.ArrayList([]const u8).empty; - defer axis_names.deinit(allocator); - - const Collect = struct { - fn appendUnique(list: *std.ArrayList([]const u8), allocator_: std.mem.Allocator, axis_name: []const u8) void { - for (list.items) |existing| { - if (std.mem.eql(u8, existing, axis_name)) return; - } - list.append(allocator_, axis_name) catch unreachable; - } - }; - - for (in_shapes) |shape| { - const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); - for (0..attr.numReplicatedAxes()) |i| { - Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); - } - for (0..attr.numDimensions()) |i| { - const dim = attr.dimension(i); - for (0..dim.numAxes()) |j| { - Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); - } - } - } - for (out_shapes) |shape| { - const attr = try mesh.sdyShardingAttrForShape(allocator, ctx, shape); - for (0..attr.numReplicatedAxes()) |i| { - Collect.appendUnique(&axis_names, allocator, attr.replicatedAxis(i).name()); - } - for (0..attr.numDimensions()) |i| { - const dim = attr.dimension(i); - for (0..dim.numAxes()) |j| { - Collect.appendUnique(&axis_names, allocator, dim.axis(j).name()); - } - } - } - - const axes = try allocator.alloc(*const mlir.StringAttribute, axis_names.items.len); - for (axis_names.items, 0..) |axis_name, i| { - axes[i] = mlir.StringAttribute.init(ctx, axis_name); - } - - return .get(ctx, axes); -} - -pub const Partitioning = packed struct { - _0: PartitionSpec, - _1: PartitionSpec, - _2: PartitionSpec, - _3: PartitionSpec, - _4: PartitionSpec, - _5: PartitionSpec, - _6: PartitionSpec, - _7: PartitionSpec, - - pub const out_of_bound: Partitioning = splat(.out_of_bound); - - const MAX_RANK = Shape.MAX_RANK; - const Vec = @Vector(MAX_RANK, u4); - - pub fn init(specs: []const PartitionSpec) Partitioning { - var res: Partitioning = out_of_bound; - for (0.., specs) |i, spec| { - res = res.set(i, spec); - } - return res; - } - - pub fn replicated(rank_: usize) Partitioning { - return repeat(.replicated, rank_); - } - - pub fn open(rank_: usize) Partitioning { - return repeat(.open, rank_); - } - - pub fn splat(spec: PartitionSpec) Partitioning { - const vec: Vec = @splat(@backingInt(spec)); - return @bitCast(vec); - } - - pub fn repeat(spec: PartitionSpec, rank_: usize) Partitioning { - std.debug.assert(rank_ <= MAX_RANK); - const splatted: Vec = @splat(@backingInt(spec)); - const mask = std.simd.iota(u4, 8) < @as(Vec, @splat(@truncate(rank_))); - return @bitCast(@select(u4, mask, splatted, @as(Vec, @bitCast(out_of_bound)))); - } - - pub fn get(p: Partitioning, ax: usize) PartitionSpec { - std.debug.assert(ax < MAX_RANK); - const pack: u32 = @bitCast(p); - const shift: u5 = @intCast(4 * ax); - return @fromBackingInt(@truncate(pack >> shift)); - } - - pub fn set(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { - std.debug.assert(ax < MAX_RANK); - const pack: u32 = @bitCast(p); - const shift: u5 = @intCast(4 * ax); - const mask = @as(u32, 0xf) << shift; - return @bitCast((pack & ~mask) | (@as(u32, @backingInt(spec)) << shift)); - } - - pub fn rank(p: Partitioning) u8 { - for (0..MAX_RANK) |i_usize| { - const i: u8 = @truncate(i_usize); - if (p.get(i) == .out_of_bound) return i; - } - return MAX_RANK; - } - - /// Inserts a spec, shifting subsequent slots right and discarding the last slot. - pub fn insert(p: Partitioning, ax: usize, spec: PartitionSpec) Partitioning { - std.debug.assert(ax < MAX_RANK); - const pack: u32 = @bitCast(p); - const shift: u5 = @intCast(4 * ax); - const lower_mask = (@as(u32, 1) << shift) - 1; - return @bitCast((pack & lower_mask) | ((pack & ~lower_mask) << 4) | (@as(u32, @backingInt(spec)) << shift)); - } - - /// Removes a spec, shifting subsequent slots left and filling the last with unknown. - pub fn orderedRemove(p: Partitioning, ax: usize) Partitioning { - std.debug.assert(ax < MAX_RANK); - const pack: u32 = @bitCast(p); - const shift: u5 = @intCast(4 * ax); - const lower_mask = (@as(u32, 1) << shift) - 1; - return @bitCast((pack & lower_mask) | ((pack >> 4) & ~lower_mask) | (@as(u32, @backingInt(PartitionSpec.out_of_bound)) << 28)); - } - - pub fn toArray(p: Partitioning) [MAX_RANK]PartitionSpec { - var res: [MAX_RANK]PartitionSpec = undefined; - for (&res, 0..) |*spec, ax| spec.* = p.get(ax); - return res; - } - - pub fn hasUniqueAxes(p: Partitioning) bool { - var used_mesh_axes: std.StaticBitSet(MAX_RANK) = .empty; - for (0..MAX_RANK) |shape_ax| { - const spec = p.get(shape_ax); - if (spec.meshAxis()) |mesh_axis| { - if (used_mesh_axes.isSet(mesh_axis)) return false; - used_mesh_axes.set(mesh_axis); - } - } - return true; - } - - test hasUniqueAxes { - try std.testing.expect(hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_2 }))); - try std.testing.expect(!hasUniqueAxes(.init(&.{ .mesh_axis_0, .mesh_axis_1, .replicated, .mesh_axis_0 }))); - } - - test "packed access and updates" { - try std.testing.expectEqual(32, @bitSizeOf(Partitioning)); - try std.testing.expectEqual(4, @sizeOf(Partitioning)); - inline for (std.meta.tags(PartitionSpec)) |spec| { - const filled = Partitioning.splat(spec); - for (0..MAX_RANK) |ax| { - try std.testing.expectEqual(spec, filled.get(ax)); - const updated = Partitioning.out_of_bound.set(ax, spec); - for (0..MAX_RANK) |i| { - try std.testing.expectEqual(if (i == ax) spec else .out_of_bound, updated.get(i)); - } - } - } - for (0..MAX_RANK + 1) |rank_| { - const parts = Partitioning.replicated(rank_); - for (0..MAX_RANK) |ax| { - try std.testing.expectEqual(if (ax < rank_) PartitionSpec.replicated else .out_of_bound, parts.get(ax)); - } - } - } - - test "insertion and removal at every slot" { - var parts: Partitioning = .out_of_bound; - for (0..MAX_RANK) |ax| parts = parts.set(ax, @fromBackingInt(@intCast(ax))); - const original = parts.toArray(); - for (0..MAX_RANK) |ax| { - var inserted = original; - std.mem.copyBackwards(PartitionSpec, inserted[ax + 1 ..], original[ax .. MAX_RANK - 1]); - inserted[ax] = .open; - try std.testing.expectEqualSlices(PartitionSpec, &inserted, &parts.insert(ax, .open).toArray()); - - var removed = original; - std.mem.copyForwards(PartitionSpec, removed[ax .. MAX_RANK - 1], original[ax + 1 ..]); - removed[MAX_RANK - 1] = .out_of_bound; - try std.testing.expectEqualSlices(PartitionSpec, &removed, &parts.orderedRemove(ax).toArray()); - try std.testing.expectEqual(parts, parts.orderedRemove(ax).insert(ax, parts.get(ax))); - } - try std.testing.expectEqual(Partitioning.out_of_bound, Partitioning.out_of_bound.orderedRemove(0)); - const comptime_parts = comptime Partitioning.out_of_bound.insert(7, .open).orderedRemove(0).set(0, .replicated); - try std.testing.expectEqual(PartitionSpec.replicated, comptime_parts.get(0)); - try std.testing.expectEqual(PartitionSpec.open, comptime_parts.get(6)); - try std.testing.expectEqual(PartitionSpec.out_of_bound, comptime_parts.get(7)); - } - - /// Parse a struct literal into a partitioning object. - /// The struct passed to partitioning must only have fields names present in `tags`. - /// The ordering of tags matter, the return struct will map names to axis as specified by `tags`. - /// The default value is important. - /// In Tensor code unspecified axis will be inferred as `.open` while in shape code, it's `.replicated`. - pub fn parse(tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec, partitioning: anytype) Partitioning { - const logical = if (@TypeOf(partitioning) == Logical) - partitioning - else - // capture all the comptime logic in Logical. - comptime Logical.parse(partitioning); - - return logical.applyUseAll(tags, mesh, default_value); - } - - pub const Logical = union(enum) { - /// Replicates every dimension, including when used as a graph constraint. - replicated, - mapping: struct { - src: Shape.TagsArray, - tgt: Shape.TagsArray, - }, - - const replicated_tag: Shape.Tag = Shape.toTag(.replicated); - const open_tag: Shape.Tag = Shape.toTag(.open); - - pub fn parse(v: anytype) Logical { - const V = @TypeOf(v); - - var src_: Shape.TagsArray = .empty; - var tgt_: Shape.TagsArray = .empty; - - if (comptime stdx.meta.isStruct(V)) { - const field_names = comptime std.meta.fieldNames(V); - stdx.debug.assertComptime(field_names.len <= @TypeOf(src_).max_len, "Too many tags in struct {} ({d})", .{ V, field_names.len }); - inline for (field_names) |field_name| { - const fv = @field(v, field_name); - tgt_.appendAssumeCapacity(@tagName(fv)); - - if (!comptime stdx.meta.isTuple(V)) { - src_.appendAssumeCapacity(field_name); - } - } - return .{ .mapping = .{ .src = src_, .tgt = tgt_ } }; - } - - stdx.debug.compileError("Sharding.Partitioning.parse expects a struct of enum literals, eg {{ .b = .data, .d = .model }}, got: {any}", .{V}); - } - - test "Logical.parse" { - const logical = comptime Logical.parse(.{ .a = .x, .b = .y }); - - try std.testing.expectEqualSlices(Shape.Tag, &.{ "a".ptr, "b".ptr }, logical.mapping.src.constSlice()); - try std.testing.expectEqualSlices(Shape.Tag, &.{ "x".ptr, "y".ptr }, logical.mapping.tgt.constSlice()); - } - - test "Logical.replicated" { - const mesh: Mesh = .initTest(.logical, .{.model}); - const tags: []const Shape.Tag = &.{ "a", "b" }; - const logical: Logical = .replicated; - for ([_]PartitionSpec{ .open, .replicated }) |default| { - try std.testing.expectEqual(Partitioning.replicated(tags.len), logical.applyUseAll(tags, &mesh, default)); - } - const unspecified = Logical.parse(.{}); - try std.testing.expectEqual(Partitioning.open(tags.len), unspecified.applyUseAll(tags, &mesh, .open)); - } - - pub fn apply(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) struct { Partitioning, ?u8 } { - const mapping = switch (logical.*) { - .replicated => return .{ .replicated(tags.len), null }, - .mapping => |*mapping_| mapping_, - }; - - var partition_: Partitioning = .repeat(default_value, tags.len); - var missing_src_axis: ?u8 = null; - - for (0.., mapping.src.slice(), mapping.tgt.slice()) |i, shape_tag, mesh_tag| { - const shape_ax = Shape.axisFromTagMaybe(tags, shape_tag) orelse { - missing_src_axis = @intCast(i); - continue; - }; - - const spec: PartitionSpec = if (mesh_tag == Logical.replicated_tag) - .replicated - else if (mesh_tag == Logical.open_tag) - .open - else spec: { - const mesh_ax = mesh.resolveLogicalAxis(mesh_tag) orelse { - stdx.debug.panic("{f} has no logical axis '{s}'", .{ mesh, mesh_tag }); - }; - break :spec .sharded(@intCast(mesh_ax)); - }; - partition_ = partition_.set(shape_ax, spec); - } - return .{ partition_, missing_src_axis }; - } - - pub fn applyUseAll(logical: *const Logical, tags: []const [*:0]const u8, mesh: *const Mesh, default_value: PartitionSpec) Partitioning { - const p, const missing_shape_axis = logical.apply(tags, mesh, default_value); - - if (missing_shape_axis) |ax| { - std.debug.panic("{f} doesn't have an axis {s} to be partitioned on.", .{ stdx.fmt.stringsZ(tags), logical.mapping.src.get(ax) }); - } - return p; - } - }; -}; - -/// Device as part of a PhysicalMesh. -pub const Device = struct { - /// Unique device identifier in the mesh - id: u32, - - /// Coordinates in the physical mesh - coords: Coords, - - const Coords = [MAX_MESH_RANK]u8; - // Placeholder coords, that would most likely trigger panic if used like that. - // Those are used when creating a mesh, before calling assignCoords - // Private so it's kept as implementation detail - const undefined_coords: Coords = @splat(0xff); - - fn hasValidCoords(device: Device) bool { - return @as(u32, @bitCast(device.coords)) != @as(u32, @bitCast(undefined_coords)); - } -}; - -/// Physical axis tags represent hardware interconnect tiers. -pub const PhysicalAxisTag = enum { - link, // Intra-device / island-local 1D interconnect - link_x, // Torus/Mesh Dim X (TPU, Trainium) - link_y, // Torus/Mesh Dim Y (TPU, Trainium) - link_z, // Torus/Mesh Dim Z (TPU v3/v4/v5) - bus, // PCIe / NUMA (Potential isolation boundary) -}; - -/// Ring geometry style for 1D neighbor-only interconnects. -pub const RingKind = enum { - linear, // Neighbors only, open ends - closed_ring, // Neighbors only, circular (wrap-around) -}; - -/// Mesh geometry style for 2D/3D neighbor-only interconnects. -pub const MeshKind = enum { - grid, // 2D/3D Neighbors only, open ends - torus, // 2D/3D Neighbors only, circular wrap-around -}; - -/// AxisGeometry describes the communication pattern for an axis. -pub const AxisGeometry = union(enum) { - point_to_point: void, // Full all-to-all (NVLink Switch, TPU v5p) - ring: RingKind, // Neighbor-only 1D - mesh: MeshKind, // Neighbor-only 2D/3D - tree: void, // Hierarchical (Standard PCIe) - isolated: void, // No direct communication (AWS Inf2 islands) - - pub fn format(self: AxisGeometry, writer: *std.Io.Writer) !void { - switch (self) { - .point_to_point => try writer.writeAll("P2P"), - .ring => |k| try writer.print("Ring({s})", .{@tagName(k)}), - .mesh => |k| try writer.print("Mesh({s})", .{@tagName(k)}), - .tree => try writer.writeAll("Tree"), - .isolated => try writer.writeAll("Isolated"), - } - } -}; - -/// Represents the hierarchical physical topology of the hardware devices. -pub const PhysicalNode = union(enum) { - /// A branch represents a physical axis - branch: struct { - tag: PhysicalAxisTag, - geometry: AxisGeometry, - children: []PhysicalNode, - }, - /// A leaf represents the actual hardware device - leaf: Device, - - pub fn axis(tag: PhysicalAxisTag, geometry_: AxisGeometry, children: []const PhysicalNode) PhysicalNode { - return .{ - .branch = .{ - .tag = tag, - .geometry = geometry_, - .children = @constCast(children), - }, - }; - } - - pub fn device(device_: PlatformDevice) PhysicalNode { - return .{ - .leaf = .{ - .id = @intCast(device_.id()), - .coords = Device.undefined_coords, - }, - }; - } - - pub fn countDevices(self: PhysicalNode) usize { - return switch (self) { - .leaf => 1, - .branch => |b| { - var sum: usize = 0; - for (b.children) |child| sum += child.countDevices(); - return sum; - }, - }; - } -}; - -/// PhysicalMesh models hardware as a tree of axes. -/// Each branch = one physical axis (link_x/link_y/bus/etc), -/// each leaf = one device. -/// -/// Diagram: -/// link_x -/// ├─ link_y -/// │ └─ device -/// └─ link_y -/// └─ device -/// -/// The DFS axis order defines canonical coordinate ordering. -pub const PhysicalMesh = struct { - /// Explicit topology tree API (canonical form) - pub const Tree = PhysicalNode; - - /// AxisTraversal captures canonical axis order and depth mapping. - pub const AxisTraversal = struct { - pub const Order = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); - pub const DepthByTag = std.EnumArray(PhysicalAxisTag, ?u8); - - /// Order of axes (DFS, first-child descent). - order: Order, - /// Depth (index) of each axis tag in the order. - depth_by_tag: DepthByTag, - - /// Build axis order and depth mapping from a topology tree. - pub fn init(root: PhysicalNode) AxisTraversal { - var order: Order = .empty; - axisOrderNode(root, &order); - - var depth_by_tag: DepthByTag = .initFill(null); - for (order.slice(), 0..) |t, i| { - depth_by_tag.set(t, @intCast(i)); - } - - return .{ - .order = order, - .depth_by_tag = depth_by_tag, - }; - } - - pub fn depth(self: AxisTraversal, tag: PhysicalAxisTag) ?u8 { - return self.depth_by_tag.get(tag); - } - - /// Canonical DFS axis order: - /// - /// Tree: - /// link_x - /// └─ link_y - /// └─ leaf - /// - /// Order = [link_x, link_y] - /// - /// We descend only the first child to keep a stable axis order that - /// depends on structure (axes), not on device count. - fn axisOrderNode(node: PhysicalNode, out: *Order) void { - switch (node) { - .leaf => {}, - .branch => |b| { - out.appendAssumeCapacity(b.tag); - if (b.children.len > 0) axisOrderNode(b.children[0], out); - }, - } - } - }; - - /// AxisInfo exposes the size and geometry for a tag. - pub const AxisInfo = struct { - tag: PhysicalAxisTag, - size: i64, - geometry: AxisGeometry, - }; - - target: Target, - root: PhysicalNode, - axis_traversal: AxisTraversal, - devices_in_canonical_order: []Device, - - /// Build a PhysicalMesh from a tree: - /// - clone the tree into `allocator` - /// - validate geometry invariants - /// - assign coordinates to leaves - /// - compute axis traversal and canonical device order - pub fn fromTree(allocator: std.mem.Allocator, target: Target, root: PhysicalNode) !PhysicalMesh { - const cloned = try cloneNode(allocator, root); - errdefer freeNode(allocator, cloned); - - return try fromOwnedTree(allocator, target, cloned); - } - - fn fromOwnedTree(allocator: std.mem.Allocator, target: Target, root: PhysicalNode) !PhysicalMesh { - try validateGeometry(root); - - var owned_root = root; - var path: Device.Coords = @splat(0); - assignCoords(&owned_root, &path, 0); - - var mesh: PhysicalMesh = .{ - .target = target, - .root = owned_root, - .axis_traversal = .init(owned_root), - .devices_in_canonical_order = undefined, - }; - try mesh.populateDevicesInCanonicalOrder(allocator); - for (mesh.devices_in_canonical_order) |device| { - std.debug.assert(device.hasValidCoords()); - } - return mesh; - } - - pub fn torus1d( - allocator: std.mem.Allocator, - target: Target, - devices: []const PlatformDevice, - ) !Sharding.PhysicalMesh { - const nodes = try allocator.alloc(PhysicalNode, devices.len); - for (nodes, devices) |*node, device| { - node.* = .device(device); - } - - return .fromOwnedTree(allocator, target, .axis(.link_x, .{ .mesh = .torus }, nodes)); - } - - pub fn torus2d( - allocator: std.mem.Allocator, - target: Target, - devices: []const PlatformDevice, - ) !Sharding.PhysicalMesh { - // Find the best split: 4 -> 2x2, 6 -> 2x3, 8 -> 2x4 - var link_x: usize = 1; - var link_y: usize = devices.len; - if (devices.len < 2) return error.NotEnoughDevices; - - for (2..devices.len) |i| { - if (@mod(devices.len, i) != 0) continue; - const j = @divExact(devices.len, i); - if (i <= j) { - link_x = i; - link_y = j; - } else break; - } - - // This only happen if number of devices is 1 or a prime number - if (link_x == 1) return error.NotEnoughDevices; - - const x_axes = try allocator.alloc(PhysicalNode, link_x); - var device_queue = devices; - for (x_axes) |*x_ax| { - const y_devices = try allocator.alloc(PhysicalNode, link_y); - for (y_devices) |*y_dev| { - y_dev.* = .device(device_queue[0]); - device_queue = device_queue[1..]; - } - - x_ax.* = .axis(.link_y, .{ .mesh = .torus }, y_devices); - } - - return .fromOwnedTree(allocator, target, .axis(.link_x, .{ .mesh = .torus }, x_axes)); - } - - pub fn torus3d( - allocator: std.mem.Allocator, - target: Target, - devices: []const PlatformDevice, - ) !Sharding.PhysicalMesh { - if (devices.len < 8) return error.NotEnoughDevices; - if (devices.len > 8) @panic("TODO: adapt logic from torus2D"); - const topology: Sharding.PhysicalMesh.Tree = .axis(.link_x, .{ .mesh = .torus }, &.{ - .axis(.link_y, .{ .mesh = .torus }, &.{ - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[0]), - .device(devices[1]), - }), - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[2]), - .device(devices[3]), - }), - }), - .axis(.link_y, .{ .mesh = .torus }, &.{ - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[4]), - .device(devices[5]), - }), - .axis(.link_z, .{ .mesh = .torus }, &.{ - .device(devices[6]), - .device(devices[7]), - }), - }), - }); - - return .fromTree(allocator, target, topology); - } - - fn populateDevicesInCanonicalOrder(self: *PhysicalMesh, allocator: std.mem.Allocator) !void { - var devices_: std.ArrayList(Device) = .empty; - errdefer devices_.deinit(allocator); - try devicesNodeInto(allocator, self.root, &devices_); - self.devices_in_canonical_order = try devices_.toOwnedSlice(allocator); - - const Order = struct { - mesh: *const PhysicalMesh, - fn lessThan(ctx: @This(), a: Device, b: Device) bool { - const ia = ctx.mesh.linearIndexFromCoords(a.coords); - const ib = ctx.mesh.linearIndexFromCoords(b.coords); - return ia < ib; - } - }; - - std.mem.sort(Device, self.devices_in_canonical_order, Order{ .mesh = self }, Order.lessThan); - } - - fn cloneNode(allocator: std.mem.Allocator, node: PhysicalNode) !PhysicalNode { - return switch (node) { - .leaf => |d| .{ - .leaf = .{ - .id = d.id, - .coords = d.coords, - }, - }, - .branch => |b| blk: { - const children = try allocator.alloc(PhysicalNode, b.children.len); - errdefer allocator.free(children); - for (b.children, 0..) |child, i| { - children[i] = try cloneNode(allocator, child); - } - break :blk .{ - .branch = .{ - .tag = b.tag, - .geometry = b.geometry, - .children = children, - }, - }; - }, - }; - } - - fn freeNode(allocator: std.mem.Allocator, node: PhysicalNode) void { - switch (node) { - .leaf => {}, - .branch => |b| { - for (b.children) |child| freeNode(allocator, child); - allocator.free(b.children); - }, - } - } - - pub fn deinit(self: *PhysicalMesh, allocator: std.mem.Allocator) void { - allocator.free(self.devices_in_canonical_order); - freeNode(allocator, self.root); - self.* = undefined; - } - - fn validateGeometry(node: PhysicalNode) !void { - switch (node) { - .leaf => return, - .branch => |b| { - if (b.children.len == 0) return error.InvalidPhysicalMesh; - - switch (b.geometry) { - .mesh => { - var has_leaf = false; - var has_branch = false; - var expected_tag: ?PhysicalAxisTag = null; - - for (b.children) |child| { - switch (child) { - .leaf => has_leaf = true, - .branch => |cb| { - has_branch = true; - if (expected_tag == null) { - expected_tag = cb.tag; - } else if (cb.tag != expected_tag.?) { - return error.IncompatibleGeometry; - } - }, - } - } - - if (has_leaf and has_branch) return error.IncompatibleGeometry; - }, - .ring => { - if (b.children.len < 2) return error.IncompatibleGeometry; - }, - .point_to_point, .tree, .isolated => {}, - } - - for (b.children) |child| try validateGeometry(child); - }, - } - } - - /// Assign coords to each leaf using DFS path indices. - /// - /// Example (2x2): - /// link_x - /// └─ link_y - /// - /// coords: - /// (0,0) (0,1) - /// (1,0) (1,1) - fn assignCoords(node: *PhysicalNode, path: *Device.Coords, depth: usize) void { - switch (node.*) { - .leaf => |*d| { - // Coords can already been set by caller - if (d.hasValidCoords()) return; - - d.coords = path.*; - }, - .branch => |*b| { - for (b.children, 0..) |*child, i| { - path[depth] = @intCast(i); - assignCoords(child, path, depth + 1); - } - }, - } - } - - pub fn countDevices(self: PhysicalMesh) usize { - return self.root.countDevices(); - } - - fn devicesNodeInto(allocator: std.mem.Allocator, node: PhysicalNode, out: *std.ArrayList(Device)) !void { - switch (node) { - .leaf => |d| try out.append(allocator, d), - .branch => |b| { - for (b.children) |child| { - try devicesNodeInto(allocator, child, out); - } - }, - } - } - - /// Canonical DFS linearization for coordinate consensus. - /// - /// sizes: S0..Sk, coords: C0..Ck - /// linear = (((C0 * S1) + C1) * S2 + C2) ... - pub fn linearIndexFromCoords(self: PhysicalMesh, coords: Device.Coords) usize { - const order = self.axisOrder(); - var idx: usize = 0; - for (order.slice()) |tag| { - const depth = self.axis_traversal.depth(tag); - if (depth) |d| { - const coord = coords[@intCast(d)]; - const size = self.axis(tag); - idx = idx * @as(usize, @intCast(size)) + coord; - } - } - return idx; - } - - pub fn axisInfo(self: PhysicalMesh, tag: PhysicalAxisTag) ?AxisInfo { - return axisInfoNode(self.root, tag); - } - - fn axisInfoNode(node: PhysicalNode, tag: PhysicalAxisTag) ?AxisInfo { - return switch (node) { - .leaf => null, - .branch => |b| { - if (b.tag == tag) { - return .{ - .tag = b.tag, - .size = @intCast(b.children.len), - .geometry = b.geometry, - }; - } - - for (b.children) |child| { - if (axisInfoNode(child, tag)) |info| return info; - } - - return null; - }, - }; - } - - pub fn hasAxis(self: PhysicalMesh, tag: PhysicalAxisTag) bool { - return self.axisInfo(tag) != null; - } - - pub fn axis(self: PhysicalMesh, tag: PhysicalAxisTag) i64 { - if (self.axisInfo(tag)) |info| return info.size; - - return 1; - } - - pub fn axisOrder(self: PhysicalMesh) AxisTraversal.Order { - return self.axis_traversal.order; - } - - pub fn geometry(self: PhysicalMesh, tag: PhysicalAxisTag) ?AxisGeometry { - if (self.axisInfo(tag)) |info| return info.geometry; - - return null; - } - - /// Preferred axis ordering by speed: fastest -> slowest. - /// This is used by Strategy.suggest to match intent to bandwidth. - pub fn shardableAxes(self: PhysicalMesh) []const PhysicalAxisTag { - return switch (self.target) { - .tpu => &.{ .link_x, .link_y, .link_z }, - .neuron => &.{ .link, .link_x, .link_y, .link_z }, - .cuda, .rocm => &.{.link}, - .cpu, .metal, .furiosa => &.{.bus}, - .oneapi => &.{ .link, .bus }, - }; - } - - pub fn isShardable(self: PhysicalMesh, tag: PhysicalAxisTag) bool { - for (self.shardableAxes()) |t| { - if (t == tag) return true; - } - - return false; - } - - pub fn format(self: PhysicalMesh, writer: *std.Io.Writer) !void { - try writer.print("PhysicalMesh(platform={s} shardable_axes={any} num_devices={d})\n", .{ @tagName(self.target), self.shardableAxes(), self.countDevices() }); - try formatNode(self.root, writer, "", 0, true); - } - - fn formatNode(node: PhysicalNode, writer: *std.Io.Writer, prefix: []const u8, depth: u8, is_last: bool) !void { - try writer.print("{s}{s}", .{ prefix, if (is_last) "└── " else "├── " }); - switch (node) { - .leaf => |device| { - try writer.print("Device(id={d}, coords={any})\n", .{ device.id, device.coords[0..depth] }); - }, - .branch => |b| { - try writer.print("[{s}] 0..{d} ({s})\n", .{ @tagName(b.tag), b.children.len, @tagName(b.geometry) }); - - var buf: [256]u8 = undefined; - const new_prefix = std.fmt.bufPrint(&buf, "{s}{s}", .{ prefix, if (is_last) " " else "│ " }) catch unreachable; - - for (b.children, 0..) |child, i| { - const child_is_last = (i == b.children.len - 1); - try formatNode(child, writer, new_prefix, depth +| 1, child_is_last); - } - }, - } - } - - /// CoordsTopology builds a mesh from PJRT `coords` attributes. - const CoordsTopology = struct { - const CoordLayout = struct { - rank: usize, - axis_sizes: [MAX_MESH_RANK]usize, - }; - - const IndexedCoord = struct { - placement: Device, - linear: usize, - }; - - fn parseDevice(device: PlatformDevice) !Device { - const api = device.platform.pjrt_api; - const coords_attr = device.pjrt_desc.attribute(api, "coords") orelse return error.MissingDeviceCoords; - const coords: Device.Coords = coords: switch (coords_attr) { - .int64list => |values| { - var coords: Device.Coords = @splat(0); - if (values.len == 0 or values.len > MAX_MESH_RANK) return error.InvalidDeviceCoords; - for (coords[0..values.len], values) |*coord, value| { - if (value < 0) return error.InvalidDeviceCoords; - coord.* = @intCast(value); - } - break :coords coords; - }, - else => return error.InvalidDeviceCoords, - }; - - return .{ .id = device.id(), .coords = coords }; - } - - pub fn collect(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) ![]Device { - const placements = try allocator.alloc(Device, platform_devices.len); - for (platform_devices, 0..) |device, i| { - placements[i] = try parseDevice(device); - } - return placements; - } - - pub fn layout(placements: []const Device, max_rank: usize) !CoordLayout { - if (placements.len == 0) return error.InvalidPhysicalMesh; - - const rank = placements[0].coords.len; - if (rank == 0) return error.InvalidDeviceCoords; - if (rank > max_rank) return error.UnsupportedDeviceCoordsRank; - - for (placements[1..]) |coord_placement| { - if (coord_placement.coords.len != rank) return error.InvalidDeviceCoordsRank; - } - - var axis_sizes: [MAX_MESH_RANK]usize = @splat(1); - for (0..rank) |ax_i| { - var max_coord: usize = 0; - for (placements) |coord_placement| { - max_coord = @max(max_coord, coord_placement.coords[ax_i]); - } - axis_sizes[ax_i] = max_coord + 1; - } - - return .{ - .rank = rank, - .axis_sizes = axis_sizes, - }; - } - - fn linearIndex(coords: Device.Coords, rank: usize, axis_sizes: []const usize) usize { - var linear: usize = 0; - var stride: usize = 1; - var ax = rank; - while (ax > 0) { - ax -= 1; - linear += coords[ax] * stride; - stride *= axis_sizes[ax]; - } - return linear; - } - - pub fn sorted(allocator: std.mem.Allocator, placements: []const Device, rank: usize, axis_sizes: []const usize) ![]IndexedCoord { - const indexed = try allocator.alloc(IndexedCoord, placements.len); - for (placements, indexed) |coord_placement, *out| { - out.* = .{ - .placement = coord_placement, - .linear = linearIndex(coord_placement.coords, rank, axis_sizes), - }; - } - - const SortCtx = struct { - fn lessThan(_: @This(), a: IndexedCoord, b: IndexedCoord) bool { - return a.linear < b.linear; - } - }; - std.mem.sort(IndexedCoord, indexed, SortCtx{}, SortCtx.lessThan); - - for (indexed[1..], 1..) |entry, i| { - if (entry.linear == indexed[i - 1].linear) return error.InvalidDeviceTopology; - } - - return indexed; - } - - pub fn axisStrides(axis_sizes: []const usize) ![MAX_MESH_RANK]usize { - var strides: [MAX_MESH_RANK]usize = @splat(0); - var stride: usize = 1; - - var i = axis_sizes.len; - while (i > 0) { - i -= 1; - strides[i] = stride; - stride = std.math.mul(usize, stride, axis_sizes[i]) catch return error.InvalidDeviceTopology; - } - - return strides; - } - - pub fn buildMeshNode( - allocator: std.mem.Allocator, - indexed: []const IndexedCoord, - axis_sizes: []const usize, - axis_tags: []const PhysicalAxisTag, - axis_strides: []const usize, - depth: usize, - base_offset: usize, - ) !PhysicalNode { - if (depth == axis_sizes.len) { - const device = &indexed[base_offset].placement; - return .{ .leaf = device.* }; - } - - const children = try allocator.alloc(PhysicalNode, axis_sizes[depth]); - var built: usize = 0; - errdefer { - for (children[0..built]) |child| freeNode(allocator, child); - allocator.free(children); - } - - for (children, 0..) |*child, i| { - child.* = try buildMeshNode( - allocator, - indexed, - axis_sizes, - axis_tags, - axis_strides, - depth + 1, - base_offset + i * axis_strides[depth], - ); - built += 1; - } - - return .{ - .branch = .{ - .tag = axis_tags[depth], - .geometry = .{ .mesh = .torus }, - .children = children, - }, - }; - } - }; - - pub fn auto(allocator: std.mem.Allocator, target: Target, platform_devices: []const PlatformDevice) !PhysicalMesh { - const root = try switch (target) { - .cpu => cpu(allocator, platform_devices), - .cuda, .rocm => gpu(allocator, platform_devices), - .tpu => tpu(allocator, platform_devices), - .neuron => return neuron(allocator, platform_devices), - .oneapi, .metal, .furiosa => cpu(allocator, platform_devices), - }; - errdefer freeNode(allocator, root); - - // We check a posteriori that the device id is a valid offset into platform.devices. - // This is generally true because platform.devices is sorted by id and ids are 0-N. - // For platforms where this isn't true like Neuron, - // the platform specific builder need to correctly update the ids so that it's true. - const mesh = try fromOwnedTree(allocator, target, root); - for (mesh.devices_in_canonical_order) |pl| { - const device = platform_devices[pl.id]; - std.debug.assert(pl.hasValidCoords()); - std.debug.assert(device.id() == pl.id); - } - - return mesh; - } - - pub fn cpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { - const nodes = try allocator.alloc(PhysicalNode, platform_devices.len); - - for (nodes, platform_devices) |*n, d| n.* = .device(d); - - return .{ - .branch = .{ - .tag = .bus, - .geometry = .tree, - .children = nodes, - }, - }; - } - - pub fn gpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { - const placements = try CoordsTopology.collect(allocator, platform_devices); - defer allocator.free(placements); - - const layout = try CoordsTopology.layout(placements, MAX_MESH_RANK); - const indexed = try CoordsTopology.sorted(allocator, placements, layout.rank, layout.axis_sizes[0..layout.rank]); - defer allocator.free(indexed); - - const nodes = try allocator.alloc(PhysicalNode, platform_devices.len); - for (nodes, indexed, 0..) |*n, entry, i| { - const linear_coord: Device.Coords = .{ @intCast(i), 0, 0, 0 }; - n.* = .{ .leaf = .{ .id = entry.placement.id, .coords = linear_coord } }; - } - - return .{ - .branch = .{ - .tag = .link, - .geometry = .point_to_point, - .children = nodes, - }, - }; - } - - pub fn tpu(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !Tree { - if (platform_devices.len == 0) return error.InvalidPhysicalMesh; - - const placements = try CoordsTopology.collect(allocator, platform_devices); - defer allocator.free(placements); - - const layout = try CoordsTopology.layout(placements, 4); - var rank = layout.rank; - var axis_sizes = layout.axis_sizes; - - while (rank > 1 and axis_sizes[rank - 1] == 1) : (rank -= 1) {} - const indexed = try CoordsTopology.sorted(allocator, placements, rank, axis_sizes[0..rank]); - defer allocator.free(indexed); - - var total_devices: usize = 1; - for (axis_sizes[0..rank]) |axis_size| { - total_devices = std.math.mul(usize, total_devices, axis_size) catch return error.InvalidDeviceTopology; - } - if (total_devices != indexed.len) return error.InvalidDeviceTopology; - for (indexed, 0..) |entry, i| { - if (entry.linear != i) return error.InvalidDeviceTopology; - } - - const axis_tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; - const axis_strides = try CoordsTopology.axisStrides(axis_sizes[0..rank]); - - return CoordsTopology.buildMeshNode(allocator, indexed, axis_sizes[0..rank], axis_tags[0..rank], axis_strides[0..rank], 0, 0); - } - - pub fn neuron(allocator: std.mem.Allocator, platform_devices: []const PlatformDevice) !PhysicalMesh { - if (comptime !platforms.isEnabled(.neuron)) { - return error.UnsupportedPlatform; - } - - const neuron_topology = @import("platforms/neuron/topology"); - - const nc_ids = try allocator.alloc(usize, platform_devices.len); - defer allocator.free(nc_ids); - for (platform_devices, nc_ids) |device, *nc_id| { - nc_id.* = @intCast(device.localHardwareId()); - } - - var topology = try neuron_topology.visibleMeshFromNcIds(allocator, nc_ids); - defer topology.deinit(allocator); - - var axis_tags: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK) = .empty; - var axis_sizes: stdx.BoundedArray(usize, MAX_MESH_RANK) = .empty; - var axis_geometries: stdx.BoundedArray(AxisGeometry, MAX_MESH_RANK) = .empty; - - for (topology.axes()) |mesh_axis| { - axis_tags.appendAssumeCapacity(switch (mesh_axis.fabric) { - .core => .link, - .chip_x => .link_x, - .chip_y => .link_y, - .chip_z => .link_z, - }); - axis_sizes.appendAssumeCapacity(mesh_axis.size); - axis_geometries.appendAssumeCapacity(switch (mesh_axis.geometry) { - .linear_ring => .{ .ring = .linear }, - .closed_ring => .{ .ring = .closed_ring }, - .torus => .{ .mesh = .torus }, - .point_to_point => .point_to_point, - }); - } - - var coords_buf: [MAX_MESH_RANK]usize = @splat(0); - var next_device: usize = 0; - - const Node = struct { - fn build( - allocator_: std.mem.Allocator, - platform_devices_: []const PlatformDevice, - placements: []const neuron_topology.Placement, - axis_tags_: []const PhysicalAxisTag, - axis_sizes_: []const usize, - axis_geometries_: []const AxisGeometry, - depth: usize, - coords_buf_: *[MAX_MESH_RANK]usize, - next_device_: *usize, - ) !PhysicalNode { - if (depth == axis_tags_.len) { - const placement_ = placements[next_device_.*]; - next_device_.* += 1; - - var coords: Device.Coords = @splat(0); - for (coords_buf_[0..axis_tags_.len], 0..) |coord, i| coords[i] = @intCast(coord); - - return .{ .leaf = .{ .id = @intCast(placement_.nc_id), .coords = coords } }; - } - - const children = try allocator_.alloc(PhysicalNode, axis_sizes_[depth]); - var built: usize = 0; - errdefer { - for (children[0..built]) |child| freeNode(allocator_, child); - allocator_.free(children); - } - - for (children, 0..) |*child, i| { - coords_buf_[depth] = i; - child.* = try @This().build( - allocator_, - platform_devices_, - placements, - axis_tags_, - axis_sizes_, - axis_geometries_, - depth + 1, - coords_buf_, - next_device_, - ); - built += 1; - } - - return .{ - .branch = .{ - .tag = axis_tags_[depth], - .geometry = axis_geometries_[depth], - .children = children, - }, - }; - } - }; - - const root = try Node.build( - allocator, - platform_devices, - topology.placements, - axis_tags.slice(), - axis_sizes.slice(), - axis_geometries.slice(), - 0, - &coords_buf, - &next_device, - ); - const mesh: PhysicalMesh = try fromOwnedTree(allocator, .neuron, root); - for (mesh.devices_in_canonical_order) |*dev| { - // In Node.build we use nc_id as the Device id. - // But now we want to use the id to get the offset into platform.devices. - const dev_nc_id = dev.id; - for (0.., topology.placements) |order, placement_| { - if (placement_.nc_id == dev_nc_id) { - dev.id = @intCast(order); - std.debug.assert(platform_devices[order].localHardwareId() == dev_nc_id); - break; - } - } - } - return mesh; - } -}; - -pub const LogicalAxisIntent = enum { - /// Needs the fastest possible path (e.g. Weight mesh / All-Reduce). - /// Requires high-bandwidth, low-latency interconnects (Cores/NVLink). - high_bandwidth, - - /// Needs reliable throughput but can handle some latency (e.g. Shuffling / All-to-All). - /// Suitable for chip-to-chip interconnects (NeuronLink/ICI). - balanced, - - /// Can tolerate lower bandwidth and higher latency (e.g. Batch mesh). - /// Suitable for system bus or network (PCIe/Ethernet). - low_bandwidth, -}; - -/// Semantic axes, their communication intents, and optional device-count constraints. -pub const LogicalMesh = struct { - pub const Axes = stdx.BoundedArray(Shape.Tag, MAX_MESH_RANK); - pub const Intents = stdx.BoundedArray(LogicalAxisIntent, MAX_MESH_RANK); - - axes: Axes, - intents: Intents, - dimensions: [MAX_MESH_RANK]i32, - - /// Create a logical mesh. - /// * names of the logical axes: .{ .a, .b }, - /// * intents: intent for each axis - /// * dimensions: dimension for each axis, -1 infers the size from underlying topology - pub fn init(names: anytype, intents: []const LogicalAxisIntent, dimensions: []const i32) !LogicalMesh { - if (@TypeOf(names) != []const Shape.Tag) { - const parsed_names: []const Shape.Tag = comptime Shape.parseTags(names); - return .init(parsed_names, intents, dimensions); - } - - if (names.len == 0 or names.len > MAX_MESH_RANK or names.len != intents.len or names.len != dimensions.len) @panic("invalid logical mesh"); - - var result: LogicalMesh = .{ .axes = .init(names), .intents = .init(intents), .dimensions = @splat(-1) }; - for (names, dimensions, 0..) |name, dimension, i| { - if (dimension != -1 and dimension <= 0) return error.InvalidLogicalDimension; - if (std.mem.findScalar(Shape.Tag, names[0..i], name)) |_| { - @panic("invalid logical mesh"); - } - - result.dimensions[i] = dimension; - } - return result; - } - - pub fn mesh(axes_: anytype) LogicalMesh { - const T = @TypeOf(axes_); - - var axes: Axes = .empty; - var intents: Intents = .empty; - - inline for (comptime std.meta.fieldNames(T)) |field_name| { - const value = @field(axes_, field_name); - axes.appendAssumeCapacity(Shape.toTag(field_name)); - intents.appendAssumeCapacity(intentFromValue(value)); - } - - if (axes.len == 0) { - stdx.debug.panic("LogicalMesh must have at least one axis defined", .{}); - } - - return .{ .axes = axes, .intents = intents, .dimensions = @splat(-1) }; - } - - pub fn hasAxis(m: *const LogicalMesh, logical_ax: Shape.Tag) ?u3 { - return @truncate(std.mem.findScalar(Shape.Tag, m.axes.slice(), logical_ax) orelse return null); - } - - pub fn intent(self: LogicalMesh, tag: Shape.Tag) ?LogicalAxisIntent { - const target = std.mem.span(tag); - for (self.axes.slice(), self.intents.slice()) |t, i| { - if (std.mem.eql(u8, std.mem.span(t), target)) return i; - } - - return null; - } - - fn intentFromValue(v: anytype) LogicalAxisIntent { - const V = @TypeOf(v); - if (V == LogicalAxisIntent) return v; - - if (V == @EnumLiteral()) { - return @field(LogicalAxisIntent, @tagName(v)); - } - - stdx.debug.compileError("LogicalMesh intent must be LogicalAxisIntent, got {}", .{V}); - } - - pub fn format(self: LogicalMesh, writer: *std.Io.Writer) !void { - try writer.writeAll("LogicalMesh("); - for (self.axes.slice(), self.intents.slice(), 0..) |axis, int, i| { - try writer.print(" {s}={s}", .{ axis, @tagName(int) }); - if (self.dimensions[i] != -1) try writer.print("[{d}]", .{self.dimensions[i]}); - } - try writer.writeAll(")"); - } -}; - -pub const PhysicalList = stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK); - -pub const Mesh = struct { - pub const zml_no_meta_visit: void = {}; - - name: []const u8, - physical: *const PhysicalMesh, - logical: LogicalMesh, - - grid: DeviceGrid, - - /// Compact binding table: logical axis -> physical axes - bindings: stdx.BoundedArray(PhysicalList, MAX_MESH_RANK), - - pub fn shardableDim(mesh: *const Mesh, dim: i64, logical_axis: anytype, must_divide: i64) DimSharding { - const partitions = mesh.numPartitionsForLogicalAxis(logical_axis); - - if (@mod(dim, partitions) == 0) { - return if (@mod(must_divide, dim) == 0) .{ .sharded = .{ .dim = dim, .factor = 1 } } else .replicated; - } - - const gcd: i64 = @intCast(std.math.gcd(@as(u64, @intCast(dim)), @as(u64, @intCast(partitions)))); - const repeat_factor: u32 = @intCast(@divExact(partitions, gcd)); - const materialized_dim = std.math.mul(i64, dim, @as(i64, repeat_factor)) catch return .replicated; - if (@mod(must_divide, materialized_dim) == 0) { - return .{ .sharded = .{ .dim = materialized_dim, .factor = repeat_factor } }; - } else { - return .replicated; - } - } - - pub fn binding(self: *const Mesh, mesh_axis: usize) []const PhysicalAxisTag { - return self.bindings.constSlice()[mesh_axis].constSlice(); - } - - pub fn resolveLogicalAxis(self: *const Mesh, tag: Shape.Tag) ?u8 { - if (@inComptime()) { - for (0.., self.logical.axes.slice()) |i, logical_axis| { - if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(tag))) return @truncate(i); - } - } else { - for (0.., self.logical.axes.slice()) |i, logical_axis| { - if (logical_axis == tag) return @truncate(i); - } - } - return null; - } - - pub fn init( - name_: []const u8, - physical: *const PhysicalMesh, - logical: LogicalMesh, - strategy: Strategy, - ) !Mesh { - const num_devices = physical.countDevices(); - std.debug.assert(num_devices == physical.devices_in_canonical_order.len); - const axis_order = physical.axisOrder().slice(); - if (axis_order.len == 0) return error.InvalidPhysicalMesh; - if (strategy.bindings.len == 0) return error.InvalidStrategy; - - // Create one binding per logical axis - var bindings: @FieldType(Mesh, "bindings") = .empty; - for (logical.axes.slice()) |_| { - bindings.appendAssumeCapacity(.empty); - } - - for (strategy.bindings.slice()) |strat_binding| { - for (logical.axes.slice(), bindings.slice()) |logical_axis, *binding_| { - if (std.mem.eql(u8, std.mem.span(logical_axis), std.mem.span(strat_binding.logical))) { - for (strat_binding.physical.constSlice()) |part| binding_.appendAssumeCapacity(part.axis); - break; - } - } else { - std.debug.panic("{f} expects strategy to only list known logical axes, got unknown {s}", .{ logical, strat_binding.logical }); - } - } - - var mesh: Mesh = .{ - .name = name_, - .logical = logical, - .grid = undefined, - .physical = physical, - .bindings = bindings, - }; - mesh.grid = try DeviceGrid.init(&mesh, &strategy); - std.debug.assert(num_devices == physical.devices_in_canonical_order.len); - std.debug.assert(@mod(mesh.numDevices(), mesh.numPartitions()) == 0); - return mesh; - } - - pub fn initTest(name_: @EnumLiteral(), tagz: anytype) Mesh { - if (!builtin.is_test) @compileError("Cannot use zml.Sharding.Mesh.initTest outside of a test block"); - const axes_tags = Shape.parseTags(tagz); - return .{ - .name = @tagName(name_), - .logical = .{ - .axes = .init(axes_tags), - .intents = .{ .buffer = @splat(.high_bandwidth), .len = axes_tags.len }, - .dimensions = @splat(-1), - }, - .physical = undefined, - .grid = undefined, - .bindings = .empty, - }; - } - - pub fn devicesInCanonicalOrder(mesh: *const Mesh) []const Device { - return mesh.physical.devices_in_canonical_order; - } - - pub fn logicalIndexFromCoords(self: *const Mesh, coords: []const u8) usize { - return self.physical.linearIndexFromCoords(coords); - } - - pub fn numPartitions(self: *const Mesh) u32 { - var partitions: u32 = 1; - for (self.grid.axis_sizes[0..self.grid.rank]) |size| partitions *= size; - return @intCast(partitions); - } - - pub fn numPartitionsForLogicalAxis(self: *const Mesh, logical_axis: anytype) u32 { - const logical_tag = Shape.toTag(logical_axis); - - const logical_ax = self.resolveLogicalAxis(logical_tag) orelse std.debug.panic("mesh {f} has no axis {s}", .{ self, logical_tag }); - return self.grid.axis_sizes[logical_ax]; - } - - pub fn numDevices(self: *const Mesh) u32 { - return @intCast(self.physical.devices_in_canonical_order.len); - } - - pub fn numReplicas(self: *const Mesh) u32 { - return @divExact(self.numDevices(), self.numPartitions()); - } - - pub fn axis(mesh: *const Mesh, ax: @EnumLiteral()) u3 { - return mesh.logical.hasAxis(Shape.toTag(ax)) orelse std.debug.panic("Mesh(name={s}) doesn't have axis {}: {f}", .{ mesh.name, ax, mesh.logical }); - } - - pub fn sdyMeshAttr(self: *const Mesh, ctx: *mlir.Context) *const shardy.MeshAttr { - var axes: stdx.BoundedArray(*const shardy.MeshAxisAttr, MAX_MESH_RANK + 1) = .empty; - var tile_count: usize = 1; - for (self.logical.axes.slice(), 0..) |ax, i| { - const size = self.grid.axis_sizes[i]; - tile_count *= size; - axes.appendAssumeCapacity(.get(ctx, .{ .name = std.mem.span(ax), .size = @intCast(size) })); - } - const replicas = @divExact(self.grid.device_count, tile_count); - if (replicas > 1) { - axes.appendAssumeCapacity(.get(ctx, .{ .name = self.sdyReplicaAxis(), .size = @intCast(replicas) })); - } - var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); - var ids: [Platform.MAX_NUM_DEVICES]i64 = undefined; - for (0..self.grid.device_count) |partition_index| { - const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; - var tile_index: usize = 0; - for (0..self.logical.axes.len) |ax| { - tile_index = tile_index * self.grid.axis_sizes[ax] + coords[ax]; - } - // Device IDs here are executable partition indices, not PJRT device IDs. - ids[tile_index * replicas + replica_indices[tile_index]] = @intCast(partition_index); - replica_indices[tile_index] += 1; - } - return .get(ctx, .{ .axes = axes.constSlice(), .device_ids = ids[0..self.grid.device_count] }); - } - - fn sdyReplicaAxis(self: *const Mesh) []const u8 { - // There are at most MAX_MESH_RANK logical names, so one candidate is free. - inline for (0..MAX_MESH_RANK + 1) |i| { - const name = std.fmt.comptimePrint("__replica_{d}", .{i}); - const taken = for (self.logical.axes.slice()) |ax| { - if (std.mem.eql(u8, std.mem.span(ax), name)) break true; - } else false; - if (!taken) return name; - } - unreachable; - } - - /// Converts `shape`'s declared partitioning into a `#sdy.sharding` attribute - /// referencing this mesh. Entries in `[...]` follow tensor dimension order. - /// Mesh axis indices in the specs resolve to this mesh's logical axis names. - /// The tensor rank and logical mesh rank are independent. - /// - /// Each dimension spec is encoded as follows: - /// - `PartitionSpec.sharded(i)` -> `{"axis_name"}`: split the dimension over - /// logical mesh axis `i`. The dimension is closed to further partitioning. - /// - `.replicated` -> `{}`: keep the dimension whole on each shard. It is - /// closed, so the partitioner cannot assign a mesh axis to it. - /// - `.open` -> `{?}`: let the partitioner infer sharding from the graph. - /// It may acquire available mesh axes or remain replicated. - /// Closed therefore means no additional mesh axes, not necessarily replicated. - /// Every dimension within `shape.rank()` must have a spec other than `.out_of_bound`. - /// - /// Dimension replication (`{}`) applies to one tensor dimension. The optional - /// tensor-wide `replicated={...}` instead names mesh axes: each listed axis - /// must carry identical tensor data across its coordinates and cannot be used - /// to split any dimension, including an open one. - /// - /// For example, take a mesh `@dp_mp` with `data=2`, `model=2`, no extra replicas, - /// and a shape whose dimensions are ordered `batch`, `hidden`: - /// - /// ``` - /// Partitioning: .{ .batch = .data, .hidden = .open } - /// #sdy.sharding<@dp_mp, [{"data"}, {?}]> - /// ``` - /// `batch` is split over `data`. The partitioner may assign `model` to `hidden`. - /// The unused `model` axis stays available because an open dimension exists. - /// - /// ``` - /// Partitioning: .{ .batch = .data, .hidden = .replicated } - /// #sdy.sharding<@dp_mp, [{"data"}, {}], replicated={"model"}> - /// ``` - /// Both dimensions are closed. Unused logical axes are explicitly replicated, - /// so devices differing only in their `model` coordinate hold the same shard. - /// - /// ``` - /// Partitioning: .{ .batch = .open, .hidden = .replicated } - /// #sdy.sharding<@dp_mp, [{?}, {}]> - /// ``` - /// Only `batch` may acquire axes, and both `data` and `model` are available. - /// Adding `replicated={"data", "model"}` would prevent either axis from being - /// inferred for `batch`, despite its `{?}` entry. - pub fn sdyShardingAttrForShape( - data: *const Mesh, - parent_allocator: std.mem.Allocator, - ctx: *mlir.Context, - shape: Shape, - ) error{OutOfMemory}!*const shardy.TensorShardingAttr { - var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); - defer arena.deinit(); - const allocator = arena.allocator(); - const dimensions = try allocator.alloc(*const shardy.DimensionShardingAttr, shape.rank()); - var used: [MAX_MESH_RANK]bool = @splat(false); - const has_open_dimension = shape.containsPartitionSpec(.open); - for (0.., dimensions) |i, *d| { - d.* = switch (shape.partition(i)) { - .out_of_bound => @panic("undefined"), - .replicated => .replicated(ctx), - .open => .open(ctx, &.{}), - else => d: { - const ax = shape.partition(i).meshAxis().?; - used[ax] = true; - break :d .closed(ctx, &.{.named(ctx, std.mem.span(data.logical.axes.get(ax)))}); - }, - }; - } - - var replicated_axes: stdx.BoundedArray(*const shardy.AxisRefAttr, MAX_MESH_RANK + 1) = .empty; - var tile_count: usize = 1; - for (data.logical.axes.slice(), 0..) |ax, i| { - tile_count *= data.grid.axis_sizes[i]; - // Open dimensions can acquire unused logical axes during inference. - // Only reserve those axes for replication when every dimension is closed. - if (!has_open_dimension and !used[i]) replicated_axes.appendAssumeCapacity(.named(ctx, std.mem.span(ax))); - } - if (tile_count < data.grid.device_count) { - replicated_axes.appendAssumeCapacity(.named(ctx, data.sdyReplicaAxis())); - } - - return .get(ctx, .{ .mesh = data.name, .dimensions = dimensions, .replicated_axes = replicated_axes.slice() }); - } - - pub fn gspmdShardingAttrForShape(self: *const Mesh, parent_allocator: std.mem.Allocator, ctx: *mlir.Context, shape: Shape) !*const mlir.Attribute { - var arena = try stdx.arenaWithCapacity(parent_allocator, 1024); - defer arena.deinit(); - const allocator = arena.allocator(); - - var tile_shape: stdx.BoundedArray(i64, Shape.MAX_RANK + 1) = .empty; - var tile_count: usize = 1; - for (0..shape.rank()) |i| { - const size = if (shape.partition(i).meshAxis()) |ax| self.grid.axis_sizes[ax] else 1; - tile_shape.appendAssumeCapacity(size); - tile_count *= size; - } - if (tile_count == 1) return .string(ctx, "{replicated}"); - - // All devices at the same logical tile coordinate are replicas, including - // unused logical axes and physical axes that have no logical binding. - const replicas = @divExact(self.grid.device_count, tile_count); - const has_replication = replicas > 1; - if (has_replication) tile_shape.appendAssumeCapacity(@intCast(replicas)); - - var out: std.Io.Writer.Allocating = .init(allocator); - errdefer out.deinit(); - - const ids = try self.gspmdTileAssignment(allocator, shape); - defer allocator.free(ids); - - try out.writer.writeAll("{devices=["); - for (tile_shape.slice(), 0..) |s, i| { - if (i > 0) try out.writer.writeAll(","); - try out.writer.print("{d}", .{s}); - } - try out.writer.writeAll("]"); - - for (ids, 0..) |id, i| { - if (i == 0) { - try out.writer.print("{d}", .{id}); - } else { - try out.writer.print(",{d}", .{id}); - } - } - - if (has_replication) { - try out.writer.writeAll(" last_tile_dim_replicate"); - } - - try out.writer.writeAll("}"); - - return .string(ctx, try out.toOwnedSlice()); - } - - fn gspmdTileAssignment(self: *const Mesh, allocator: std.mem.Allocator, shape: Shape) ![]u32 { - _ = try Sharding.Placement.init(&shape); - const devices = self.devicesInCanonicalOrder(); - var tile_axes: stdx.BoundedArray(u8, MAX_MESH_RANK) = .empty; - var used_axes: u8 = 0; - var tile_count: usize = 1; - for (0..shape.rank()) |ax| { - const logical_axis = shape.partition(ax).meshAxis() orelse continue; - const bit = @as(u8, 1) << @intCast(logical_axis); - if (used_axes & bit != 0) continue; - used_axes |= bit; - tile_axes.appendAssumeCapacity(@intCast(logical_axis)); - tile_count *= self.grid.axis_sizes[logical_axis]; - } - const replicas_per_tile = @divExact(devices.len, tile_count); - const replica_indices = try allocator.alloc(usize, tile_count); - defer allocator.free(replica_indices); - @memset(replica_indices, 0); - const ids = try allocator.alloc(u32, devices.len); - - for (0..devices.len) |partition_index| { - const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[partition_index]; - var tile_index: usize = 0; - for (tile_axes.constSlice()) |logical_axis| { - tile_index = tile_index * self.grid.axis_sizes[logical_axis] + - coords[logical_axis]; - } - // HLO shardings refer to logical partitions; executable device assignment - // separately maps those partitions to physical PJRT device IDs. - ids[tile_index * replicas_per_tile + replica_indices[tile_index]] = @intCast(partition_index); - replica_indices[tile_index] += 1; - } - return ids; - } - - pub fn deviceAssignment(self: *const Mesh, allocator: std.mem.Allocator) ![]u32 { - const count = self.physical.devices_in_canonical_order.len; - - var ids = try allocator.alloc(u32, count); - @memset(ids, std.math.maxInt(u32)); - - for (self.physical.devices_in_canonical_order) |d| { - const coords = d.coords; - const idx = self.physical.linearIndexFromCoords(coords); - ids[idx] = d.id; - } - - for (ids) |id| { - if (id == std.math.maxInt(u32)) return error.MissingDeviceInTile; - } - - return ids; - } - - pub fn format(self: Mesh, writer: *std.Io.Writer) !void { - try writer.print("Mesh(name={s}, partitions={d}, replicas={d})\n", .{ self.name, self.numPartitions(), self.numReplicas() }); - - try writer.writeAll("┌ Mappings (logical coordinates from physical coordinates):\n"); - const num_axes = self.logical.axes.len; - for (self.logical.axes.slice(), self.logical.intents.slice(), 0..) |tag, intent, logical_axis| { - try writer.print("{s}── {s} ({s}) -> ", .{ if (logical_axis == num_axes - 1) "└" else "├", tag, @tagName(intent) }); - try self.formatAxisMapping(writer, logical_axis); - try writer.writeAll("\n"); - } - try writer.writeAll("\n"); - try writer.print("┌ {f}\n", .{self.physical}); - - try writer.writeAll("┌ Logical coordinates:\n"); - var coords: [MAX_MESH_RANK]usize = @splat(0); - try self.formatLogicalCoordinates(writer, &coords, 0); - } - - /// Reverse engineer the mapping logical -> physical from the device grid. - fn formatAxisMapping(self: Mesh, writer: *std.Io.Writer, logical_axis: usize) !void { - const Part = struct { - tag: PhysicalAxisTag, - physical_size: usize, - physical_stride: usize, - size: usize, - logical_stride: usize, - }; - // A u8 coordinate has at most eight nontrivial mixed-radix digits. - var parts: stdx.BoundedArray(Part, @bitSizeOf(u8)) = .empty; - const logical_size = self.grid.axis_sizes[logical_axis]; - var logical_stride: usize = 1; - while (logical_stride < logical_size) { - const remaining = @divExact(logical_size, logical_stride); - var size = remaining; - const part: Part = found: { - // Prefer the largest matching portion, so whole axes stay readable. - while (size > 1) : (size -= 1) { - if (remaining % size != 0) continue; - for (self.physical.axisOrder().constSlice()) |tag| { - const physical_size: usize = @intCast(self.physical.axis(tag)); - const depth = self.physical.axis_traversal.depth(tag).?; - var physical_stride: usize = 1; - while (physical_stride * size <= physical_size) : (physical_stride += 1) { - if (physical_size % (physical_stride * size) != 0) continue; - for (self.devicesInCanonicalOrder(), 0..) |device, i| { - const coords: [MAX_MESH_RANK]u8 = self.grid.logical_coords[i]; - if ((coords[logical_axis] / logical_stride) % size != - (device.coords[depth] / physical_stride) % size) break; - } else { - break :found .{ - .tag = tag, - .physical_size = physical_size, - .physical_stride = physical_stride, - .size = size, - .logical_stride = logical_stride, - }; - } - } - } - } - try writer.writeAll("device-dependent (see logical coordinates)"); - return; - }; - parts.appendAssumeCapacity(part); - logical_stride *= part.size; - } - if (parts.len == 0) { - try writer.writeAll("0 (replicated)"); - return; - } - var i: usize = parts.len; - while (i > 0) { - i -= 1; - const part = parts.constSlice()[i]; - if (i + 1 < parts.len) try writer.writeAll(" + "); - const partial = part.size != part.physical_size; - if (partial) try writer.writeAll("("); - try writer.writeAll(@tagName(part.tag)); - if (part.physical_stride > 1) try writer.print(" / {d}", .{part.physical_stride}); - if (part.physical_stride * part.size < part.physical_size) try writer.print(" % {d}", .{part.size}); - if (partial) try writer.writeAll(")"); - if (part.logical_stride > 1) try writer.print(" * {d}", .{part.logical_stride}); - } - } - - fn formatLogicalCoordinates(self: Mesh, writer: *std.Io.Writer, coords: *[MAX_MESH_RANK]usize, depth: usize) std.Io.Writer.Error!void { - const axes = self.logical.axes.constSlice(); - if (axes.len == 0) return; - const last = axes.len - 1; - if (depth < last) { - const size: usize = @intCast(self.numPartitionsForLogicalAxis(axes[depth])); - for (0..size) |coord| { - coords[depth] = coord; - try writer.splatByteAll(' ', 4 * depth); - try writer.print("└── {s}={d}\n", .{ axes[depth], coord }); - try self.formatLogicalCoordinates(writer, coords, depth + 1); - } - return; - } - - try writer.splatByteAll(' ', 4 * depth); - try writer.print("└── {s}=", .{axes[last]}); - const size: usize = @intCast(self.numPartitionsForLogicalAxis(axes[last])); - for (0..size) |coord| { - if (coord > 0) try writer.writeAll(","); - try writer.print("{d}", .{coord}); - } - try writer.writeAll(" -> devices="); - for (0..size) |coord| { - coords[last] = coord; - if (coord > 0) try writer.writeAll(","); - var count: usize = 0; - for (0..self.grid.device_count) |device_index| { - if (self.matchesLogicalCoordinates(device_index, coords[0..axes.len])) count += 1; - } - // Keep replicas grouped so each entry corresponds to one last-axis coordinate. - if (count != 1) try writer.writeAll("["); - var written: usize = 0; - for (self.devicesInCanonicalOrder(), 0..) |device, device_index| { - if (!self.matchesLogicalCoordinates(device_index, coords[0..axes.len])) continue; - if (written > 0) try writer.writeAll(","); - try writer.print("{d}", .{device.id}); - written += 1; - } - if (count != 1) try writer.writeAll("]"); - } - try writer.writeAll("\n"); - } - - fn matchesLogicalCoordinates(self: Mesh, device_index: usize, coords: []const usize) bool { - const actual: [MAX_MESH_RANK]u8 = self.grid.logical_coords[device_index]; - for (coords, 0..) |expected, logical_axis| { - if (actual[logical_axis] != expected) return false; - } - return true; - } -}; - -/// Resolved logical coordinates in canonical physical-device order. -/// Unused logical axes have coordinate zero. -pub const DeviceGrid = struct { - pub const Coordinates = [MAX_MESH_RANK]u8; - - comptime { - std.debug.assert(Platform.MAX_NUM_DEVICES <= 256); - } - - rank: u8, - device_count: usize, - axis_sizes: [MAX_MESH_RANK]u32, - logical_coords: [Platform.MAX_NUM_DEVICES]Coordinates align(16), - - fn init(mesh_: *const Mesh, strategy: *const Strategy) !DeviceGrid { - for (strategy.folding.constSlice()) |entry| { - for (entry.sources.constSlice()) |source| { - if (!mesh_.physical.hasAxis(source)) return error.InvalidPhysicalAxis; - } - } - const devices = mesh_.devicesInCanonicalOrder(); - if (devices.len > Platform.MAX_NUM_DEVICES) return error.InvalidPhysicalMesh; - var grid: DeviceGrid = .{ - .rank = @intCast(mesh_.logical.axes.len), - .device_count = devices.len, - .axis_sizes = @splat(1), - .logical_coords = @splat(@splat(0)), - }; - for (mesh_.logical.axes.constSlice(), 0..) |logical_tag, logical_axis| { - var used: std.EnumSet(PhysicalAxisTag) = .empty; - for (strategy.bindings.constSlice()) |binding_| { - if (!std.mem.eql(u8, std.mem.span(binding_.logical), std.mem.span(logical_tag))) continue; - for (binding_.physical.constSlice()) |part| { - if (part.size) |size| { - const info = mesh_.physical.axisInfo(part.axis) orelse return error.InvalidPhysicalAxis; - const physical_size: u32 = @intCast(info.size); - if (size == 0 or part.stride == 0 or size > physical_size or part.stride > physical_size / size) return error.InvalidPartialBinding; - if (physical_size % (part.stride * size) != 0) return error.InvalidPartialBinding; - try grid.appendAxis(mesh_, logical_axis, part.axis, part.stride, size); - } else { - for (strategy.foldSources(part.axis) orelse &.{part.axis}) |source| { - if (used.contains(source)) continue; - used.insert(source); - const info = mesh_.physical.axisInfo(source) orelse return error.InvalidPhysicalAxis; - try grid.appendAxis(mesh_, logical_axis, source, 1, @intCast(info.size)); - } - } - } - } - const dimension = mesh_.logical.dimensions[logical_axis]; - if (dimension != -1) { - if (dimension <= 0) return error.InvalidLogicalDimension; - const size: u32 = @intCast(dimension); - if (grid.axis_sizes[logical_axis] % size != 0) return error.IncompatibleLogicalDimension; - grid.axis_sizes[logical_axis] = size; - // Keep the least-significant portion of the ordered binding; the rest replicates. - for (grid.logical_coords[0..devices.len]) |*row| { - var coords: [MAX_MESH_RANK]u8 = row.*; - coords[logical_axis] = @intCast(coords[logical_axis] % size); - row.* = coords; - } - } - } - // Shared physical axes are valid only when their portions produce a complete - // Cartesian logical grid with the same number of replicas at every coordinate. - var partitions: usize = 1; - for (grid.axis_sizes[0..grid.rank]) |size| { - if (size > devices.len / partitions) return error.OverlappingLogicalAxes; - partitions *= size; - } - if (devices.len % partitions != 0) return error.OverlappingLogicalAxes; - var counts: [Platform.MAX_NUM_DEVICES]usize = @splat(0); - for (grid.logical_coords[0..devices.len]) |row| { - const coords: [MAX_MESH_RANK]u8 = row; - var index: usize = 0; - for (coords[0..grid.rank], grid.axis_sizes[0..grid.rank]) |coord, size| index = index * size + coord; - counts[index] += 1; - } - for (counts[0..partitions]) |count| { - if (count != devices.len / partitions) return error.OverlappingLogicalAxes; - } - return grid; - } - - fn appendAxis(self: *DeviceGrid, mesh_: *const Mesh, logical_axis: usize, tag: PhysicalAxisTag, stride: u32, size: u32) !void { - if (size > Platform.MAX_NUM_DEVICES / self.axis_sizes[logical_axis]) return error.InvalidPartialBinding; - const depth = mesh_.physical.axis_traversal.depth(tag).?; - self.axis_sizes[logical_axis] *= size; - for (mesh_.devicesInCanonicalOrder(), 0..) |device, i| { - var coords: [MAX_MESH_RANK]u8 = self.logical_coords[i]; - coords[logical_axis] = @intCast(@as(u32, coords[logical_axis]) * size + (device.coords[depth] / stride) % size); - self.logical_coords[i] = coords; - } - } -}; - -pub const Strategy = struct { - pub const Fold = struct { - target: PhysicalAxisTag, - sources: PhysicalList, - }; - pub const PhysicalBinding = struct { - axis: PhysicalAxisTag, - stride: u32 = 1, - /// null binds the whole axis (including any explicit fold). - size: ?u32 = null, - }; - pub const Binding = struct { - logical: Shape.Tag, - physical: stdx.BoundedArray(PhysicalBinding, MAX_MESH_RANK), - }; - - pub const Bindings = stdx.BoundedArray(Binding, MAX_MESH_RANK); - pub const Folding = stdx.BoundedArray(Strategy.Fold, MAX_MESH_RANK); - - bindings: Bindings, - folding: Folding, - - // TODO: remove .init, promote .parseBindings to .init - // It's currently an error to call .init and never calling `addBinding` - pub const init: Strategy = .{ .bindings = .empty, .folding = .empty }; - - pub fn parseBindings(bindings: anytype) Strategy { - const err_msg = "Strategy.parseBindings excepts fields to be PhysicalAxisTag or tuple of PhysicalAxisTag, got {}"; - var res: Strategy = .{ .bindings = .empty, .folding = .empty }; - const info = @typeInfo(@TypeOf(bindings)).@"struct"; - if (info.field_names.len == 0) @compileError("Strategy.parseBindings requires at least one binding"); - inline for (info.field_names, info.field_types) |field_name, Field| { - switch (@typeInfo(Field)) { - .enum_literal, .@"enum" => res.addBinding(field_name, @field(bindings, field_name)), - .@"struct" => |struct_info| { - stdx.debug.assertComptime(struct_info.is_tuple, err_msg, .{@TypeOf(bindings)}); - inline for (@field(bindings, field_name)) |axis_tag| { - res.addBinding(field_name, axis_tag); - } - }, - else => stdx.debug.compileError(err_msg, .{@TypeOf(bindings)}), - } - } - return res; - } - - pub fn addBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag) void { - self.appendBinding(Shape.toTag(logical), .{ .axis = physical }); - } - - /// Append the mixed-radix portion `(coordinate / stride) % size` of a physical axis. - /// `stride * size` must divide the physical axis size. Partial bindings do not expand folds. - /// Portions are concatenated in insertion order, most significant first. - pub fn addPartialBinding(self: *Strategy, logical: anytype, physical: PhysicalAxisTag, stride: u32, size: u32) void { - self.appendBinding(Shape.toTag(logical), .{ .axis = physical, .stride = stride, .size = size }); - } - - fn appendBinding(self: *Strategy, logical: Shape.Tag, physical: PhysicalBinding) void { - for (self.bindings.slice()) |*binding_| { - if (std.mem.eql(u8, std.mem.span(binding_.logical), std.mem.span(logical))) { - binding_.physical.appendAssumeCapacity(physical); - return; - } - } - self.bindings.appendAssumeCapacity(.{ .logical = logical, .physical = .init(&.{physical}) }); - } - - fn foldSources(self: *const Strategy, tag: PhysicalAxisTag) ?[]const PhysicalAxisTag { - for (self.folding.constSlice()) |*fold| { - if (fold.target == tag) return fold.sources.constSlice(); - } - return null; - } - - /// Explicitly fold axes: `target` is the kept axis, `sources` define order. - /// If `target` is missing from `sources`, it is prepended. - pub fn addFold(self: *Strategy, target: PhysicalAxisTag, sources: []const PhysicalAxisTag) void { - var list: PhysicalList = .empty; - - var has_target = false; - for (sources) |s| { - if (s == target) { - has_target = true; - break; - } - } - - if (!has_target) list.appendAssumeCapacity(target); - for (sources) |s| list.appendAssumeCapacity(s); - - for (self.folding.slice()) |*f| { - if (f.target == target) { - f.sources = list; - return; - } - } - - self.folding.appendAssumeCapacity(.{ .target = target, .sources = list }); - } - - /// suggest builds a Strategy from logical intents. - pub fn suggest(logical: LogicalMesh, physical: *const PhysicalMesh) Strategy { - var strategy: Strategy = .{ .bindings = .empty, .folding = .empty }; - - const base_order = physical.shardableAxes(); - var available: stdx.BoundedArray(PhysicalAxisTag, MAX_MESH_RANK) = .empty; - for (base_order) |tag| { - if (physical.hasAxis(tag)) available.appendAssumeCapacity(tag); - } - - if (available.len == 0) { - for (physical.axisOrder().slice()) |tag| available.appendAssumeCapacity(tag); - } - - const avail = available.constSlice(); - if (avail.len == 0) return strategy; - var remaining: [MAX_MESH_RANK]u32 = @splat(1); - for (avail, 0..) |tag, i| remaining[i] = @intCast(physical.axis(tag)); - - // Reserve fixed dimensions before automatic axes can consume their capacity. - // Carve fixed portions from the most-significant end, leaving a contiguous - // low-order portion for automatic axes (e.g. model parallelism on one link). - for ([_]bool{ true, false }) |fixed| { - var counters: std.EnumArray(LogicalAxisIntent, u32) = .initFill(0); - inline for (std.meta.tags(LogicalAxisIntent)) |target_intent| { - for (logical.axes.constSlice(), logical.intents.constSlice(), 0..) |tag, intent, logical_axis| { - if (intent != target_intent) continue; - const i = counters.get(target_intent); - counters.set(target_intent, @intCast(i + 1)); - const dimension = logical.dimensions[logical_axis]; - if ((dimension != -1) != fixed) continue; - const preferred = switch (target_intent) { - .high_bandwidth => i % avail.len, - .balanced => (avail.len / 2 + i) % avail.len, - .low_bandwidth => avail.len - 1 - (i % avail.len), - }; - if (fixed) { - if (dimension <= 1) { - strategy.addPartialBinding(tag, avail[preferred], 1, 1); - continue; - } - var needed: u32 = @intCast(dimension); - for (0..avail.len) |offset| { - const axis = if (target_intent == .low_bandwidth) - (preferred + avail.len - offset) % avail.len - else - (preferred + offset) % avail.len; - const size = std.math.gcd(needed, remaining[axis]); - if (size == 1) continue; - remaining[axis] = @divExact(remaining[axis], size); - strategy.addPartialBinding(tag, avail[axis], remaining[axis], size); - needed = @divExact(needed, size); - if (needed == 1) break; - } - // Mesh.init reports an incompatible dimension if capacity was insufficient. - if (needed == @as(u32, @intCast(dimension))) strategy.addPartialBinding(tag, avail[preferred], 1, 1); - } else if (remaining[preferred] > 1) { - strategy.addPartialBinding(tag, avail[preferred], 1, remaining[preferred]); - remaining[preferred] = 1; - } - } - } - } - if (strategy.bindings.len == 0) strategy.addPartialBinding(logical.axes.constSlice()[0], avail[0], 1, 1); - - return strategy; - } -}; - -/// For a given shape, compute the shape of the slice each shard will receive. -pub fn shardedShape(shape: Shape) error{IncompatibleSharding}!Shape { - if (shape._sharding.mesh != null) { - const pl: Placement = try Sharding.Placement.init(&shape); - var local_shape = pl.localShape(shape); - local_shape._sharding = .replicated; - return local_shape; - } else { - // fully replicated - return shape; - } -} - -/// Owns per-device addressing metadata for a shape. -pub const Placement = struct { - pub const Slices = stdx.BoundedArray(Slice1d, Shape.MAX_RANK); - pub const Slice1d = struct { - start: i64, - size: i64, - }; - - local_dims: stdx.BoundedArray(i64, Shape.MAX_RANK), - mesh_ptr: ?*const Mesh, - byte_steps: [MAX_MESH_RANK]u64, - - pub fn init(shape: *const Shape) error{IncompatibleSharding}!Placement { - const byte_strides = shape.computeByteStrides(); - return initWithByteStrides(shape, byte_strides.constSlice()); - } - - /// Reuse the caller's contiguous byte strides without recomputing them. - pub fn initWithByteStrides(shape: *const Shape, byte_strides: []const i64) error{IncompatibleSharding}!Placement { - std.debug.assert(byte_strides.len == shape.rank()); - if (builtin.mode == .debug) std.debug.assert(shape._sharding.partition.hasUniqueAxes()); - var result: Placement = .{ - .local_dims = .{ .buffer = undefined, .len = shape.rank() }, - .mesh_ptr = shape._sharding.mesh, - .byte_steps = @splat(0), - }; - for (shape.dims(), result.local_dims.slice(), 0..) |dim, *result_local_dim, ax| { - result_local_dim.* = dim; - const mesh_ = result.mesh_ptr orelse continue; - const logical_axis = shape.partition(ax).meshAxis() orelse continue; - const count = mesh_.grid.axis_sizes[logical_axis]; - if (@rem(dim, count) != 0) return error.IncompatibleSharding; - const local_dim = @divExact(dim, count); - result_local_dim.* = local_dim; - result.byte_steps[logical_axis] = @intCast(local_dim * byte_strides[ax]); - } - return result; - } - - pub fn mesh(pl: *const Placement) ?*const Mesh { - return pl.mesh_ptr; - } - - /// Reconstruct metadata only for consumers that need a full shard shape. - pub fn localShape(pl: *const Placement, source: Shape) Shape { - std.debug.assert(source.rank() == pl.local_dims.len); - var result = source; - @memcpy(result._dims.slice(), pl.local_dims.constSlice()); - result._sharding.mesh = pl.mesh_ptr; - return result; - } - - pub fn slices(pl: *const Placement, device_index: usize, source: Shape) Slices { - std.debug.assert(source.rank() == pl.local_dims.len); - var res: Slices = .{ .buffer = undefined, .len = pl.local_dims.len }; - if (pl.mesh_ptr == null) { - for (res.slice(), pl.local_dims.constSlice()) |*slice, dim| slice.* = .{ .start = 0, .size = dim }; - return res; - } - const grid = &pl.mesh_ptr.?.grid; - std.debug.assert(device_index < grid.device_count); - const coords: [MAX_MESH_RANK]u8 = grid.logical_coords[device_index]; - var used_axes: u8 = 0; - for (res.slice(), pl.local_dims.constSlice(), 0..) |*r, size, ax| { - var coordinate: i64 = 0; - if (source.partition(ax).meshAxis()) |logical_axis| { - const bit = @as(u8, 1) << @intCast(logical_axis); - if (used_axes & bit == 0) coordinate = coords[logical_axis]; - used_axes |= bit; - } - r.* = .{ .start = coordinate * size, .size = size }; - } - return res; - } - - /// Validate the contiguous source slice once before iterating over devices. - /// The returned pointer includes the slice's byte offset. - pub fn basePtr(pl: *const Placement, slice: Slice) [*]const u8 { - if (builtin.mode == .debug) { - std.debug.assert(pl.local_dims.len == slice.shape.rank()); - } - return slice.constData().ptr; - } - - /// device_index is a canonical ordinal, never a PJRT device ID. - /// base_ptr must reference validated contiguous data for the global shape. - pub fn shardPtr(pl: *const Placement, device_index: usize, base_ptr: [*]const u8) [*]const u8 { - return base_ptr + pl.byteOffset(device_index); - } - - fn byteOffset(pl: *const Placement, device_index: usize) usize { - const mesh_ = pl.mesh() orelse return 0; - std.debug.assert(device_index < mesh_.grid.device_count); - const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[device_index]; - const steps = pl.byte_steps; - // Placement coefficients and grid coordinates describe offsets within the - // validated source slice. Keep arithmetic checks in debug only. - return offset: { - @setRuntimeSafety(builtin.mode == .debug); - var offset: u64 = 0; - inline for (0..MAX_MESH_RANK) |axis| { - offset += @as(u64, coords[axis]) * steps[axis]; - } - break :offset @intCast(offset); - }; - } - - /// Given a Slice, returns the subslice corresponding to a specific device. - /// Depending on the layout, the sub-slice may or may not be contiguous. - pub fn shardSlice(pl: *const Placement, device_index: usize, slice: Slice) Slice { - const shard_ptr = pl.shardPtr(device_index, pl.basePtr(slice)); - const offset_bytes = @intFromPtr(shard_ptr) - @intFromPtr(slice.bytes.ptr); - return .{ - .bytes = slice.bytes, - .mutable = slice.mutable, - .shape = pl.localShape(slice.shape), - .offset_bytes = offset_bytes, - .byte_strides = slice.byte_strides, - }; - } - - pub fn format(pl: *const Placement, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print( - "Placement(local_dims={any}, mesh={*}, byte_steps={any})\n", - .{ pl.local_dims.constSlice(), pl.mesh_ptr, pl.byte_steps }, - ); - } -}; - -const ShardingTest = struct { - pub const ExpectedShard = struct { - // TODO: remove device_id from here, we always have ids from 0 to N - device_id: usize, - slices: []const [2]i64, - }; - - pub const Scenario = struct { - mesh: *const Mesh, - shape: Shape, - - expected_num_partitions: u32, - expected_num_replicas: u32, - expect_error: ?anyerror = null, - expected_sdy: ?[]const u8 = null, - expected_shards: []const ExpectedShard = &.{}, - - pub fn format(scenario: Scenario, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("Scenario: partitions={d}, replicas={d}\n", .{ scenario.expected_num_partitions, scenario.expected_num_replicas }); - if (scenario.expect_error) |err| try writer.print("- error {}\n", .{err}); - if (scenario.expected_sdy) |sdy| try writer.print("- shardy: {s}\n", .{sdy}); - if (scenario.expected_shards.len > 0) try writer.print("- shards:\n", .{}); - for (scenario.expected_shards) |shard| { - try writer.print("\t- device {d}: slices={any}\n", .{ shard.device_id, shard.slices }); - } - } - }; - - allocator: std.mem.Allocator, - - pub fn init(allocator: std.mem.Allocator) ShardingTest { - return .{ .allocator = allocator }; - } - - /// Creates a 1D-3D PhysicalMesh from simple dimensions. - pub fn physical(self: ShardingTest, dims: anytype, geometry: AxisGeometry) !PhysicalMesh { - const info = @typeInfo(@TypeOf(dims)).@"struct"; - const N = info.field_names.len; - var sizes: [N]usize = undefined; - inline for (info.field_names, sizes[0..]) |field_name, *s| { - s.* = @intCast(@field(dims, field_name)); - } - - const tags: [3]PhysicalAxisTag = .{ .link_x, .link_y, .link_z }; - var next_id: u32 = 0; - const root = try self.buildNode(tags[0..N], sizes[0..N], 0, &next_id, geometry); - return try .fromTree(self.allocator, .tpu, root); - } - - fn buildNode(self: ShardingTest, tags: []const PhysicalAxisTag, sizes: []const usize, depth: usize, next_id: *u32, geometry: AxisGeometry) !PhysicalNode { - if (depth == tags.len) { - const id = next_id.*; - next_id.* += 1; - return .{ .leaf = .{ .id = id, .coords = Device.undefined_coords } }; - } - const count = sizes[depth]; - const children = try self.allocator.alloc(PhysicalNode, count); - for (children) |*child| { - child.* = try self.buildNode(tags, sizes, depth + 1, next_id, geometry); - } - return .{ .branch = .{ .tag = tags[depth], .geometry = geometry, .children = children } }; - } - - pub fn run(self: ShardingTest, s: Scenario) !void { - const mesh = s.mesh; - errdefer std.log.warn("Failed mesh {f}", .{s}); - - try std.testing.expectEqual(s.expected_num_partitions, mesh.numPartitions()); - try std.testing.expectEqual(s.expected_num_replicas, mesh.numReplicas()); - try std.testing.expectEqual(mesh.grid.device_count, @as(usize, @intCast(mesh.numDevices()))); - - // Verify MLIR String - if (s.expected_sdy) |expected_attr| { - errdefer std.log.warn("Expected shardy annotation failed", .{}); - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - shardy.registerDialect(registry); - func.registerExtensions(registry); - - var ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - - const actual_attr = try mesh.sdyShardingAttrForShape(self.allocator, ctx, s.shape); - var writer: std.Io.Writer.Allocating = .init(self.allocator); - defer writer.deinit(); - try actual_attr.attribute().format(&writer.writer); - try std.testing.expectEqualStrings(expected_attr, writer.written()); - } - - // Verify Placement logic / Error - const ordered_devices = mesh.devicesInCanonicalOrder(); - if (s.expect_error) |err| { - errdefer std.log.warn("Expected error failed", .{}); - try std.testing.expect(ordered_devices.len > 0); - try std.testing.expectError(err, Sharding.Placement.init(&s.shape)); - return; - } - - // Verify Shard Slices (Math) - if (s.expected_shards.len > 0) { - const pl: Placement = try Sharding.Placement.init(&s.shape); - errdefer std.log.warn("Expected shards failed. Got {f}", .{pl}); - try std.testing.expectEqual(s.expected_shards.len, ordered_devices.len); - for (s.expected_shards, ordered_devices, 0..) |expected, device, device_index| { - { - try std.testing.expectEqual(expected.device_id, device.id); - - // Check pl.slices(device) - const actual_slices = pl.slices(device_index, s.shape); - - // Consider rewriting test cases to use Slice1D. This requires passing the axis - const actual_start_len = try self.allocator.alloc([2]i64, actual_slices.len); - defer self.allocator.free(actual_start_len); - for (actual_start_len, actual_slices.slice()) |*s_l, slice| { - s_l.* = .{ slice.start, slice.size }; - } - - try std.testing.expectEqualSlices([2]i64, expected.slices, actual_start_len); - } - { - // Check pl.shape - const expected_sizes = try self.allocator.alloc(i64, expected.slices.len); - defer self.allocator.free(expected_sizes); - for (expected_sizes, expected.slices) |*expected_size, slice| { - expected_size.* = slice[1]; - } - - try std.testing.expectEqualSlices(i64, expected_sizes, pl.local_dims.constSlice()); - } - } - } - } -}; - -test "mesh: unknown partitioning implies replication" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - - const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth }); - - const strategy: Strategy = .parseBindings(.{ .batch = .link_x }); - const mesh: Sharding.Mesh = try .init("folded_mesh", &physical, logical, strategy); - - try runner.run(.{ - .expected_num_partitions = 2, - .expected_num_replicas = 2, - .mesh = &mesh, - // No partitioning on shape - .shape = Shape.init(.{ .batch = 8 }, .f32), - .expected_sdy = "#sdy.sharding<@folded_mesh, [{}], replicated={\"batch\", \"__replica_0\"}>", - .expected_shards = &.{ - .{ .device_id = 0, .slices = &.{.{ 0, 8 }} }, - .{ .device_id = 1, .slices = &.{.{ 0, 8 }} }, - .{ .device_id = 2, .slices = &.{.{ 0, 8 }} }, - .{ .device_id = 3, .slices = &.{.{ 0, 8 }} }, - }, - }); -} - -test "suggest splits a one-dimensional topology for fixed data and automatic model axes" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - const runner: ShardingTest = .init(allocator); - const logical = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 2, -1 }); - for ([_]usize{ 2, 4, 8 }) |count| { - const physical = try runner.physical(.{count}, .{ .mesh = .torus }); - const mesh_ = try Mesh.init("DP-MP", &physical, logical, .suggest(logical, &physical)); - try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.data)); - try std.testing.expectEqual(@as(u32, @intCast(count / 2)), mesh_.numPartitionsForLogicalAxis(.model)); - try std.testing.expectEqual(1, mesh_.numReplicas()); - for (0..count) |i| { - const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[i]; - try std.testing.expectEqual(@as(u8, @intCast(i / (count / 2))), coords[0]); - try std.testing.expectEqual(@as(u8, @intCast(i % (count / 2))), coords[1]); - } - const shape = Shape.init(.{ .b = 8, .m = 8 }, .f32).withPartitioning(&mesh_, .{ .b = .data, .m = .model }); - try runner.run(.{ .mesh = &mesh_, .shape = shape, .expected_num_partitions = @intCast(count), .expected_num_replicas = 1 }); - } -} - -test "suggest reserves fixed dimensions across physical axes before automatic allocation" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - const physical = try runner.physical(.{ 4, 3 }, .{ .mesh = .torus }); - const logical = try LogicalMesh.init(.{ .model, .data }, &.{ .high_bandwidth, .low_bandwidth }, &.{ -1, 6 }); - const mesh_ = try Mesh.init("split", &physical, logical, .suggest(logical, &physical)); - try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.model)); - try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.data)); - try std.testing.expectEqual(1, mesh_.numReplicas()); - - const fixed = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 3, 4 }); - const fixed_mesh = try Mesh.init("fixed", &physical, fixed, .suggest(fixed, &physical)); - try std.testing.expectEqual(12, fixed_mesh.numPartitions()); - const impossible = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 5, -1 }); - try std.testing.expectError(error.IncompatibleLogicalDimension, Mesh.init("impossible", &physical, impossible, .suggest(impossible, &physical))); - const replica_logical = try LogicalMesh.init(.{ .data, .model }, &.{ .low_bandwidth, .high_bandwidth }, &.{ 1, 1 }); - const replica_mesh = try Mesh.init("replicated", &physical, replica_logical, .suggest(replica_logical, &physical)); - try std.testing.expectEqual(12, replica_mesh.numReplicas()); -} - -test "mesh: suggest strategy realization" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - - const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - - const logical: LogicalMesh = .mesh(.{ - .batch = .low_bandwidth, - .model = .high_bandwidth, - }); - - const strategy: Strategy = .suggest(logical, &physical); - const mesh: Mesh = try .init("suggested_mesh", &physical, logical, strategy); - - try runner.run(.{ - .expected_num_partitions = 4, - .expected_num_replicas = 2, - .mesh = &mesh, - .shape = Shape.init(.{ .batch = 4, .model = 4 }, .f32) - .withPartitioning(&mesh, .{ .batch = .batch, .model = .model }), - .expected_sdy = "#sdy.sharding<@suggested_mesh, [{\"batch\"}, {\"model\"}], replicated={\"__replica_0\"}>", - .expected_shards = &.{ - .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y0, z0 - .{ .device_id = 1, .slices = &.{ .{ 2, 2 }, .{ 0, 2 } } }, // x0, y0, z1 (batch split) - .{ .device_id = 2, .slices = &.{ .{ 0, 2 }, .{ 0, 2 } } }, // x0, y1, z0 (replicated y) - .{ .device_id = 3, .slices = &.{ .{ 2, 2 }, .{ 0, 2 } } }, // x0, y1, z1 - .{ .device_id = 4, .slices = &.{ .{ 0, 2 }, .{ 2, 2 } } }, // x1, y0, z0 (model split) - .{ .device_id = 5, .slices = &.{ .{ 2, 2 }, .{ 2, 2 } } }, - .{ .device_id = 6, .slices = &.{ .{ 0, 2 }, .{ 2, 2 } } }, - .{ .device_id = 7, .slices = &.{ .{ 2, 2 }, .{ 2, 2 } } }, - }, - }); -} - -test "mesh: suggest leaves conflicting logical axes replicated" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - - const physical = try runner.physical(.{4}, .point_to_point); - - const logical: LogicalMesh = .mesh(.{ - .model = .high_bandwidth, - .experts = .high_bandwidth, - }); - - const strategy: Strategy = .suggest(logical, &physical); - - const mesh: Mesh = try .init("fold_mesh", &physical, logical, strategy); - try runner.run(.{ - .expected_num_partitions = 4, - .expected_num_replicas = 1, - .mesh = &mesh, - .shape = Shape.init(.{ .model = 4, .experts = 4 }, .f32) - .withPartitioning(&mesh, .{ .model = .model, .experts = .experts }), - .expected_sdy = "#sdy.sharding<@fold_mesh, [{\"model\"}, {\"experts\"}]>", - .expected_shards = &.{ - .{ .device_id = 0, .slices = &.{ .{ 0, 1 }, .{ 0, 4 } } }, - .{ .device_id = 1, .slices = &.{ .{ 1, 1 }, .{ 0, 4 } } }, - .{ .device_id = 2, .slices = &.{ .{ 2, 1 }, .{ 0, 4 } } }, - .{ .device_id = 3, .slices = &.{ .{ 3, 1 }, .{ 0, 4 } } }, - }, - }); - // Swapping tensor dimensions must not change which logical axis owns the devices. - try runner.run(.{ - .expected_num_partitions = 4, - .expected_num_replicas = 1, - .mesh = &mesh, - .shape = Shape.init(.{ .experts = 4, .model = 4 }, .f32) - .withPartitioning(&mesh, .{ .experts = .experts, .model = .model }), - .expected_shards = &.{ - .{ .device_id = 0, .slices = &.{ .{ 0, 4 }, .{ 0, 1 } } }, - .{ .device_id = 1, .slices = &.{ .{ 0, 4 }, .{ 1, 1 } } }, - .{ .device_id = 2, .slices = &.{ .{ 0, 4 }, .{ 2, 1 } } }, - .{ .device_id = 3, .slices = &.{ .{ 0, 4 }, .{ 3, 1 } } }, - }, - }); -} - -test "mesh: multiple physical axes on one logical dimension (folding)" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - - const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth }); - - { - const strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_y } }); - const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); - try runner.run(.{ - .expected_num_partitions = 4, - .expected_num_replicas = 1, - .mesh = &mesh, - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), - .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"model\"}]>", - .expected_shards = &.{ - .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, - .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, - .{ .device_id = 2, .slices = &.{.{ 8, 4 }} }, - .{ .device_id = 3, .slices = &.{.{ 12, 4 }} }, - }, - }); - } - - { - // Swap link_x and link_y order: - // -> the roles of devices 1 (0,1) and 2 (1,0) are swapped - const strategy: Strategy = .parseBindings(.{ .model = .{ .link_y, .link_x } }); - const mesh: Mesh = try .init("folded_mesh", &physical, logical, strategy); - try runner.run(.{ - .expected_num_partitions = 4, - .expected_num_replicas = 1, - .mesh = &mesh, - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), - .expected_sdy = "#sdy.sharding<@folded_mesh, [{\"model\"}]>", - .expected_shards = &.{ - .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, - .{ .device_id = 1, .slices = &.{.{ 8, 4 }} }, - .{ .device_id = 2, .slices = &.{.{ 4, 4 }} }, - .{ .device_id = 3, .slices = &.{.{ 12, 4 }} }, - }, - }); - } -} - -test "mesh: explicit strategy folding" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - - const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .mesh(.{ .model = .high_bandwidth }); - - var strategy: Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_z }); - - const mesh: Mesh = try .init("strategy_fold", &physical, logical, strategy); - try runner.run(.{ - .expected_num_partitions = 4, - .expected_num_replicas = 2, - .mesh = &mesh, - .shape = Shape.init(.{ .model = 16 }, .f32).withPartitioning(&mesh, .{ .model = .model }), - .expected_sdy = "#sdy.sharding<@strategy_fold, [{\"model\"}], replicated={\"__replica_0\"}>", - .expected_shards = &.{ - .{ .device_id = 0, .slices = &.{.{ 0, 4 }} }, - .{ .device_id = 1, .slices = &.{.{ 4, 4 }} }, - .{ .device_id = 2, .slices = &.{.{ 0, 4 }} }, - .{ .device_id = 3, .slices = &.{.{ 4, 4 }} }, - .{ .device_id = 4, .slices = &.{.{ 8, 4 }} }, - .{ .device_id = 5, .slices = &.{.{ 12, 4 }} }, - .{ .device_id = 6, .slices = &.{.{ 8, 4 }} }, - .{ .device_id = 7, .slices = &.{.{ 12, 4 }} }, - }, - }); -} - -test "mesh: open and replicated dimension mix" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - - const physical: PhysicalMesh = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }); - - const strategy: Strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }); - - const mesh: Mesh = try .init("mix_mesh", &physical, logical, strategy); - try runner.run(.{ - .expected_num_partitions = 4, - .expected_num_replicas = 1, - .mesh = &mesh, - .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh, .{ .batch = .open, .model = .replicated }), - .expected_sdy = "#sdy.sharding<@mix_mesh, [{?}, {}]>", - .expected_shards = &.{ - .{ .device_id = 0, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, - .{ .device_id = 1, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, - .{ .device_id = 2, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, - .{ .device_id = 3, .slices = &.{ .{ 0, 8 }, .{ 0, 8 } } }, - }, - }); -} - -test "mesh: full 3D cluster mesh" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - - const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth, .context = .balanced }); - - const strategy: Strategy = .parseBindings(.{ - .batch = .link_x, - .model = .link_y, - .context = .link_z, - }); - - const mesh: Mesh = try .init("3d_mesh", &physical, logical, strategy); - try runner.run(.{ - .expected_num_partitions = 8, - .expected_num_replicas = 1, - .mesh = &mesh, - // Note: .context and .model are swapped compared to .link_x and .link_z - .shape = Shape.init(.{ .batch = 4, .context = 4, .model = 4 }, .f32) - .withPartitioning(&mesh, .{ .batch = .batch, .model = .model, .context = .context }), - .expected_sdy = "#sdy.sharding<@\"3d_mesh\", [{\"batch\"}, {\"context\"}, {\"model\"}]>", - .expected_shards = &.{ - .{ .device_id = 0, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, - .{ .device_id = 1, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, - .{ .device_id = 2, .slices = &.{ .{ 0, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, - .{ .device_id = 3, .slices = &.{ .{ 0, 2 }, .{ 2, 2 }, .{ 2, 2 } } }, - .{ .device_id = 4, .slices = &.{ .{ 2, 2 }, .{ 0, 2 }, .{ 0, 2 } } }, - .{ .device_id = 5, .slices = &.{ .{ 2, 2 }, .{ 2, 2 }, .{ 0, 2 } } }, - .{ .device_id = 6, .slices = &.{ .{ 2, 2 }, .{ 0, 2 }, .{ 2, 2 } } }, - .{ .device_id = 7, .slices = &.{ .{ 2, 2 }, .{ 2, 2 }, .{ 2, 2 } } }, - }, - }); -} - -test "mesh: num partitions for logical axis" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - - const physical: PhysicalMesh = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const logical: LogicalMesh = .mesh(.{ - .model = .high_bandwidth, - .batch = .low_bandwidth, - }); - - var strategy: Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_z }); - - const mesh: Mesh = try .init("axis_parts_mesh", &physical, logical, strategy); - - try std.testing.expectEqual(4, mesh.numPartitionsForLogicalAxis(.model)); - try std.testing.expectEqual(1, mesh.numPartitionsForLogicalAxis(.batch)); -} - -test "mesh: size-one and unbound logical axes leave all devices replicated" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - const physical = try runner.physical(.{ 1, 6 }, .{ .mesh = .torus }); - const mesh_ = try Mesh.init("replicas", &physical, .mesh(.{ .data = .balanced, .model = .balanced }), .parseBindings(.{ .data = .link_x })); - try runner.run(.{ - .mesh = &mesh_, - .shape = Shape.init(.{ .batch = 12 }, .f32).withPartitioning(&mesh_, .{ .batch = .data }), - .expected_num_partitions = 1, - .expected_num_replicas = 6, - }); -} - -test "mesh: logical partitions and replicas can have different sizes" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - const physical = try runner.physical(.{ 2, 3, 2 }, .{ .mesh = .torus }); - var strategy: Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_z, .link_x }); - const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced }), strategy); - try runner.run(.{ - .mesh = &mesh_, - .shape = Shape.init(.{ .model = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model }), - .expected_num_partitions = 4, - .expected_num_replicas = 3, - }); -} - -test "localized shapes do not retain global sharding" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - const physical = try runner.physical(.{2}, .{ .mesh = .torus }); - const mesh_ = try Mesh.init("local", &physical, .mesh(.{ .batch = .balanced }), .parseBindings(.{ .batch = .link_x })); - try std.testing.expectEqual(mesh_.bindings.constSlice()[0].constSlice().ptr, mesh_.binding(0).ptr); - const global = Shape.init(.{ .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch }); - const local = try shardedShape(global); - try std.testing.expectEqual(4, local.dim(.batch)); - try std.testing.expectEqual(null, local._sharding.mesh); - try std.testing.expectEqual(8, global.dim(.batch)); - try std.testing.expectEqual(&mesh_, global._sharding.mesh); -} - -test "GSPMD tiles use tensor axis order and logical partition indices" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(7 - i); - const mesh_ = try Mesh.init("tiles", &physical, .mesh(.{ .batch = .balanced, .model = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); - const shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }); - const ids = try mesh_.gspmdTileAssignment(arena.allocator(), shape); - try std.testing.expectEqualSlices(u32, &.{ 0, 2, 4, 6, 1, 3, 5, 7 }, ids); -} - -test "DeviceGrid stores aligned logical coordinates in canonical order" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - const physical = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); - for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(3 - i); - const mesh_ = try Mesh.init("grid", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), .parseBindings(.{ .model = .link_y, .data = .link_x })); - const expected = [_][4]u8{ .{ 0, 0, 0, 0 }, .{ 1, 0, 0, 0 }, .{ 0, 1, 0, 0 }, .{ 1, 1, 0, 0 } }; - try std.testing.expectEqual(@as(usize, 0), @intFromPtr(&mesh_.grid.logical_coords) % 16); - for (mesh_.grid.logical_coords[0..mesh_.grid.device_count], expected, 0..) |coords, want, i| { - try std.testing.expectEqual(@intFromPtr(&mesh_.grid.logical_coords) + 4 * i, @intFromPtr(&mesh_.grid.logical_coords[i])); - const actual: [4]u8 = coords; - try std.testing.expectEqualSlices(u8, &want, &actual); - } - try std.testing.expectEqual(@as(usize, 4), @sizeOf(DeviceGrid.Coordinates)); - const shape = Shape.init(.{ .m = 16, .n = 24 }, .f32).withPartitioning(&mesh_, .{ .m = .model, .n = .data }); - const placement = try Sharding.Placement.init(&shape); - const offsets = [_]usize{ 0, 768, 48, 816 }; - var data: [16 * 24 * 4 + 16]u8 = undefined; - var slice = Slice.init(shape, data[16..]); - slice.bytes = &data; - slice.offset_bytes = 16; - const base_ptr = placement.basePtr(slice); - try std.testing.expectEqual(@intFromPtr(&data) + 16, @intFromPtr(base_ptr)); - for (offsets, 0..) |offset, i| { - try std.testing.expectEqual(offset, placement.byteOffset(i)); - try std.testing.expectEqual(@intFromPtr(&data) + 16 + offset, @intFromPtr(placement.shardPtr(i, base_ptr))); - } -} - -test "partial bindings share physical axes and combine portions in order" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - const runner: ShardingTest = .init(allocator); - const physical = try runner.physical(.{ 4, 6 }, .{ .mesh = .torus }); - const logical = try LogicalMesh.init(.{ .model, .data }, &.{ .balanced, .low_bandwidth }, &.{ -1, -1 }); - var strategy: Strategy = .init; - strategy.addPartialBinding(.model, .link_y, 2, 3); - strategy.addPartialBinding(.model, .link_x, 1, 2); - strategy.addPartialBinding(.data, .link_x, 2, 2); - strategy.addPartialBinding(.data, .link_y, 1, 2); - const mesh_ = try Mesh.init("partial", &physical, logical, strategy); - try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.model)); - try std.testing.expectEqual(4, mesh_.numPartitionsForLogicalAxis(.data)); - try std.testing.expectEqual(1, mesh_.numReplicas()); - for (physical.devices_in_canonical_order, 0..) |device, i| { - const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[i]; - try std.testing.expectEqual((device.coords[1] / 2) * 2 + device.coords[0] % 2, coords[0]); - try std.testing.expectEqual((device.coords[0] / 2) * 2 + device.coords[1] % 2, coords[1]); - } - const formatted = try std.fmt.allocPrint(allocator, "{f}", .{mesh_}); - try std.testing.expect(std.mem.indexOf(u8, formatted, "model (balanced) -> (link_y / 2) * 2 + (link_x % 2)") != null); - const shape = Shape.init(.{ .m = 12, .d = 8 }, .f32).withPartitioning(&mesh_, .{ .m = .model, .d = .data }); - try runner.run(.{ .mesh = &mesh_, .shape = shape, .expected_num_partitions = 24, .expected_num_replicas = 1 }); -} - -test "fixed logical dimensions constrain whole partial and folded bindings" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - const physical = try runner.physical(.{ 4, 6 }, .{ .mesh = .torus }); - const logical = try LogicalMesh.init(.{ .model, .data, .unused }, &.{ .balanced, .low_bandwidth, .balanced }, &.{ 2, -1, 1 }); - var strategy: Strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); - const mesh_ = try Mesh.init("constrained", &physical, logical, strategy); - try std.testing.expectEqual(2, mesh_.numPartitionsForLogicalAxis(.model)); - try std.testing.expectEqual(6, mesh_.numPartitionsForLogicalAxis(.data)); - try std.testing.expectEqual(1, mesh_.numPartitionsForLogicalAxis(.unused)); - try std.testing.expectEqual(2, mesh_.numReplicas()); - for (physical.devices_in_canonical_order, 0..) |device, i| { - try std.testing.expectEqual(device.coords[0] % 2, mesh_.grid.logical_coords[i][0]); - } - - const fixed = try LogicalMesh.init(.{.model}, &.{.balanced}, &.{2}); - strategy = .init; - strategy.addPartialBinding(.model, .link_x, 1, 4); - const partial = try Mesh.init("partial", &physical, fixed, strategy); - try std.testing.expectEqual(2, partial.numPartitions()); - try std.testing.expectEqual(12, partial.numReplicas()); - - strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_y, .link_x }); - const folded = try Mesh.init("folded", &physical, try LogicalMesh.init(.{.model}, &.{.balanced}, &.{8}), strategy); - try std.testing.expectEqual(8, folded.numPartitions()); - try std.testing.expectEqual(3, folded.numReplicas()); - for (physical.devices_in_canonical_order, 0..) |device, i| { - try std.testing.expectEqual((device.coords[1] * 4 + device.coords[0]) % 8, folded.grid.logical_coords[i][0]); - } - for ([_]i32{ 5, 32 }) |dimension| { - try std.testing.expectError(error.IncompatibleLogicalDimension, Mesh.init("invalid", &physical, try LogicalMesh.init(.{.model}, &.{.balanced}, &.{dimension}), strategy)); - } -} - -test "partial bindings reject invalid portions and correlated logical axes" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - const physical = try runner.physical(.{8}, .{ .mesh = .torus }); - for ([_][2]u32{ .{ 0, 2 }, .{ 1, 0 }, .{ 3, 2 }, .{ 1, 3 }, .{ 4, 4 }, .{ 1, std.math.maxInt(u32) } }) |portion| { - var strategy: Strategy = .init; - strategy.addPartialBinding(.model, .link_x, portion[0], portion[1]); - try std.testing.expectError(error.InvalidPartialBinding, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); - } - var strategy: Strategy = .init; - strategy.addPartialBinding(.model, .link_y, 1, 2); - try std.testing.expectError(error.InvalidPhysicalAxis, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); - strategy = .init; - strategy.addPartialBinding(.model, .link_x, 1, 2); - strategy.addPartialBinding(.data, .link_x, 1, 2); - // Four claimed partitions fit on eight devices, but only two coordinates occur. - try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), strategy)); -} - -test "DeviceGrid resolves folds once in strategy order" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - const runner: ShardingTest = .init(allocator); - const physical = try runner.physical(.{ 2, 3, 2 }, .{ .mesh = .torus }); - for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(11 - i); - var strategy: Strategy = .parseBindings(.{ .model = .{ .link_x, .link_z } }); - strategy.addFold(.link_x, &.{ .link_z, .link_x }); - const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced }), strategy); - strategy.folding = .empty; - - try std.testing.expectEqual(4, mesh_.numPartitions()); - try std.testing.expectEqual(3, mesh_.numReplicas()); - const expected = [_]u8{ 0, 2, 0, 2, 0, 2, 1, 3, 1, 3, 1, 3 }; - for (expected, 0..) |coord, i| { - try std.testing.expectEqual(coord, mesh_.grid.logical_coords[i][0]); - } - const formatted = try std.fmt.allocPrint(allocator, "{f}", .{mesh_}); - try std.testing.expect(std.mem.indexOf(u8, formatted, "model (balanced) -> link_z * 2 + link_x\n") != null); - try std.testing.expect(std.mem.indexOf(u8, formatted, "model=0,1,2,3 -> devices=[11,9,7],[5,3,1],[10,8,6],[4,2,0]") != null); -} - -test "DeviceGrid validates fold sources and overlapping folded axes" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - const physical = try runner.physical(.{ 2, 2 }, .{ .mesh = .torus }); - var strategy: Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_y, &.{ .link_y, .link_z }); - try std.testing.expectError(error.InvalidPhysicalAxis, Mesh.init("invalid", &physical, .mesh(.{ .model = .balanced }), strategy)); - - strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); - strategy.addFold(.link_x, &.{ .link_x, .link_y }); - try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), strategy)); -} - -test "Placement owns dimensions and byte steps across copies" { - const Helper = struct { - fn create() !Placement { - const shape = Shape.init(.{ .m = 3, .n = 5 }, .f32); - return Placement.init(&shape); - } - }; - var original = try Helper.create(); - original.byte_steps[0] = 60; - const copied = original; - original.local_dims.buffer[0] = 99; - original.byte_steps[0] = 100; - try std.testing.expectEqualSlices(i64, &.{ 3, 5 }, copied.local_dims.constSlice()); - try std.testing.expectEqualSlices(u64, &.{ 60, 0, 0, 0 }, &copied.byte_steps); - const source = Shape.init(.{ .m = 3, .n = 5 }, .f32); - try std.testing.expect(copied.localShape(source).eqlWithTags(source)); - var data: [3 * 5 * 4]u8 = undefined; - try std.testing.expectEqual(@intFromPtr(&data), @intFromPtr(copied.basePtr(.init(source, &data)))); -} - -test "Placement byte steps and dot product exceed 32 bits" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - const physical = try runner.physical(.{Platform.MAX_NUM_DEVICES}, .{ .mesh = .torus }); - const mesh_ = try Mesh.init("wide", &physical, .mesh(.{ .model = .balanced }), .parseBindings(.{ .model = .link_x })); - const shape = Shape.init(.{ .m = 2 * Platform.MAX_NUM_DEVICES, .n = 1 << 30 }, .f32).withPartitioning(&mesh_, .{ .m = .model }); - const placement = try Sharding.Placement.init(&shape); - try std.testing.expectEqual(@as(usize, 1) << 33, placement.byteOffset(1)); - const last_device = Platform.MAX_NUM_DEVICES - 1; - const coords: [MAX_MESH_RANK]u8 = mesh_.grid.logical_coords[last_device]; - try std.testing.expectEqual(@as(u8, @intCast(last_device)), coords[0]); - try std.testing.expectEqual(@as(usize, last_device) << 33, placement.byteOffset(last_device)); -} - -test "DeviceGrid rejects overlapping logical axes" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const runner: ShardingTest = .init(arena.allocator()); - const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - try std.testing.expectError(error.OverlappingLogicalAxes, Mesh.init("overlap", &physical, .mesh(.{ .model = .balanced, .data = .balanced }), .parseBindings(.{ .model = .{ .link_x, .link_y }, .data = .{ .link_y, .link_z } }))); -} - -test "Shardy meshes preserve logical order and unused physical replicas" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - const runner: ShardingTest = .init(allocator); - const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - const mesh_ = try Mesh.init("logical", &physical, .mesh(.{ .batch = .balanced, .model = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); - const registry: *mlir.DialectRegistry = try .init(); - defer registry.deinit(); - shardy.registerDialect(registry); - const ctx: *mlir.Context = try .init(.{ .registry = registry, .threading = false }); - defer ctx.deinit(); - ctx.loadAllAvailableDialects(); - var text: std.Io.Writer.Allocating = .init(allocator); - defer text.deinit(); - try text.writer.print("{f}", .{mesh_.sdyMeshAttr(ctx)}); - try std.testing.expectEqualStrings( - "#sdy.mesh<[\"batch\"=2, \"model\"=2, \"__replica_0\"=2], device_ids=[0, 2, 4, 6, 1, 3, 5, 7]>", - text.written(), - ); -} - -test "Shardy replica axis avoids logical name collisions" { - const mesh_ = Mesh.initTest(.collision, .{ .__replica_0, .__replica_1 }); - try std.testing.expectEqualStrings("__replica_2", mesh_.sdyReplicaAxis()); -} - -test "GSPMD attributes use logical tile sizes and replica groups" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - const ctx: *mlir.Context = try .init(.{ .threading = false }); - defer ctx.deinit(); - const runner: ShardingTest = .init(allocator); - const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - for (physical.devices_in_canonical_order, 0..) |*device, i| device.id = @intCast(7 - i); - const mesh_ = try Mesh.init("tiles", &physical, .mesh(.{ .batch = .balanced, .model = .balanced, .unused = .balanced }), .parseBindings(.{ .batch = .link_z, .model = .link_x })); - - const Case = struct { shape: Shape, expected: []const u8 }; - const cases = [_]Case{ - .{ - .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }), - .expected = "{devices=[2,2,2]0,2,4,6,1,3,5,7 last_tile_dim_replicate}", - }, - .{ - .shape = Shape.init(.{ .model = 8, .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch, .model = .model }), - .expected = "{devices=[2,2,2]0,2,1,3,4,6,5,7 last_tile_dim_replicate}", - }, - .{ - .shape = Shape.init(.{ .batch = 8, .model = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .batch }), - .expected = "{devices=[2,1,4]0,2,4,6,1,3,5,7 last_tile_dim_replicate}", - }, - .{ - .shape = Shape.init(.{ .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .batch = .unused }), - .expected = "{replicated}", - }, - .{ - .shape = Shape.init(.{ .batch = 8 }, .f32), - .expected = "{replicated}", - }, - .{ - .shape = Shape.init(.{}, .f32), - .expected = "{replicated}", - }, - }; - for (cases) |case| { - const actual = try mesh_.gspmdShardingAttrForShape(allocator, ctx, case.shape); - try std.testing.expect(actual.eql(.string(ctx, case.expected))); - } -} - -test "GSPMD attributes preserve folded logical axis order" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - const ctx: *mlir.Context = try .init(.{ .threading = false }); - defer ctx.deinit(); - const runner: ShardingTest = .init(allocator); - const physical = try runner.physical(.{ 2, 2, 2 }, .{ .mesh = .torus }); - var strategy: Strategy = .parseBindings(.{ .model = .link_x, .batch = .link_y }); - strategy.addFold(.link_x, &.{ .link_z, .link_x }); - const mesh_ = try Mesh.init("folded", &physical, .mesh(.{ .model = .balanced, .batch = .balanced }), strategy); - const sharded = Shape.init(.{ .model = 8, .batch = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model, .batch = .batch }); - const actual = try mesh_.gspmdShardingAttrForShape(allocator, ctx, sharded); - try std.testing.expect(actual.eql(.string(ctx, "{devices=[4,2]0,2,4,6,1,3,5,7}"))); - - const partial = Shape.init(.{ .model = 8 }, .f32).withPartitioning(&mesh_, .{ .model = .model }); - const partial_attr = try mesh_.gspmdShardingAttrForShape(allocator, ctx, partial); - try std.testing.expect(partial_attr.eql(.string(ctx, "{devices=[4,2]0,2,4,6,1,3,5,7 last_tile_dim_replicate}"))); -} - -test "GSPMD executes a reduction across a folded logical axis" { - if (comptime !Target.cpu.isEnabled()) return error.SkipZigTest; - - const zml = @import("zml.zig"); - const io = std.testing.io; - const allocator = std.testing.allocator; - - const platform = try Platform.init(allocator, io, .cpu, .{ - .cpu = .{ .device_count = 8 }, - .physical_mesh = .{ .custom = PhysicalMesh.torus3d }, - }); - defer platform.deinit(allocator, io); - var strategy: Strategy = .parseBindings(.{ .model = .link_x, .data = .link_y }); - strategy.addFold(.link_x, &.{ .link_z, .link_x }); - const mesh_ = try platform.registerMeshWithStrategy("logical", .mesh(.{ .data = .balanced, .model = .balanced }), strategy); - - const Functions = struct { - fn forward(input: zml.Tensor) zml.Tensor { - return input.sum(.m).squeeze(.m).withPartitioning(input.shape()._sharding.mesh.?, .{ .n = .data }); - } - }; - const input = zml.Tensor.init(.{ .m = 8, .n = 4 }, .i32).withPartitioning(mesh_, .{ .m = .model, .n = .data }); - const exe = try platform.compileFn(allocator, io, Functions.forward, .{input}, .{ - .meshes = &.{mesh_}, - .partitioner = .gspmd, - }); - defer exe.deinit(); - var values: [8][4]i32 = undefined; - var expected: [4]i32 = @splat(0); - for (&values, 0..) |*row, m| { - for (row, 0..) |*value, n| { - value.* = @intCast(10 * m + n); - expected[n] += value.*; - } - } - - var input_buffer = try zml.Buffer.fromBytes(io, platform, input.shape(), std.mem.asBytes(&values)); - defer input_buffer.deinit(); - var output = try exe.eval(allocator, io, .{input_buffer}); - defer output.deinit(); - try std.testing.expectEqual(expected, try output.getValue([4]i32, io)); -} diff --git a/zml/attention.zig b/zml/attention.zig new file mode 100644 index 00000000..1aa8001b --- /dev/null +++ b/zml/attention.zig @@ -0,0 +1,3 @@ +pub const attention = @import("attention/attention.zig"); +pub const paged_attention = @import("attention/paged_attention.zig"); +pub const flashattn = @import("attention/flashattn.zig"); diff --git a/zml/attention/Options.zig b/zml/attention/Options.zig deleted file mode 100644 index 539f753f..00000000 --- a/zml/attention/Options.zig +++ /dev/null @@ -1,3 +0,0 @@ -/// Query j sees the keys up to token_index + j, as in a decoder. Otherwise every query sees every -/// key, as in an encoder or a diffusion transformer, and token_index is ignored. -causal: bool = true, diff --git a/zml/attention/attention.zig b/zml/attention/attention.zig index edd9b4ca..ad747f04 100644 --- a/zml/attention/attention.zig +++ b/zml/attention/attention.zig @@ -1,172 +1,120 @@ const std = @import("std"); const zml = @import("../zml.zig"); -const flash_attention = @import("../kernels/flash_attention/flash_attention.zig"); +const flashattn = @import("flashattn.zig"); -test { - std.testing.refAllDecls(@This()); -} +const Attention = @This(); -pub const BackendType = enum { - stablehlo, - flash_attention, -}; - -pub const Backend = union(BackendType) { - stablehlo: void, - flash_attention: flash_attention.Backend, +pub const Backend = enum { + vanilla, + cuda_fa2, + cuda_fa3, pub fn auto(platform: *const zml.Platform) Backend { - return if (flash_attention.isAvailable(platform)) - .{ .flash_attention = flash_attention.Backend.auto(platform) } - else - .{ .stablehlo = {} }; - } + return switch (platform.target) { + .cuda => b: { + const first_device = platform.pjrt_client.devices(platform.pjrt_api)[0]; - pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { - return switch (backend) { - .stablehlo => true, - .flash_attention => |b| b.isAvailable(platform), + if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { + break :b if (std.mem.eql(u8, cc, "9.0")) + .cuda_fa3 + else + .cuda_fa2; + } + + break :b .vanilla; + }, + else => .vanilla, }; } }; -pub const Parameters = union(BackendType) { - stablehlo: void, - flash_attention: flash_attention.Parameters, +pub const Parameters = union(Backend) { + vanilla: void, + cuda_fa2: flashattn.fa2.Parameters, + cuda_fa3: flashattn.fa3.Parameters, - pub const InitOptions = union(BackendType) { - stablehlo: void, - flash_attention: flash_attention.Parameters.InitOptions, + pub const InitOptions = union(Backend) { + vanilla: void, + cuda_fa2: flashattn.fa2.Parameters.InitOptions, + cuda_fa3: flashattn.fa3.Parameters.InitOptions, pub fn fromBackend(backend: Backend) InitOptions { return switch (backend) { - .stablehlo => .{ .stablehlo = {} }, - .flash_attention => |b| .{ .flash_attention = .fromBackend(b) }, + .vanilla => .{ .vanilla = {} }, + .cuda_fa2 => .{ .cuda_fa2 = .{} }, + .cuda_fa3 => .{ .cuda_fa3 = .{} }, }; } }; pub fn init(opts: InitOptions) Parameters { return switch (opts) { - .stablehlo => .{ .stablehlo = {} }, - .flash_attention => |v| .{ .flash_attention = .init(v) }, + .vanilla => .{ .vanilla = {} }, + .cuda_fa2 => |v| .{ .cuda_fa2 = .init(v) }, + .cuda_fa3 => |v| .{ .cuda_fa3 = .init(v) }, }; } }; -pub const Metadata = union(BackendType) { - stablehlo: void, - flash_attention: flash_attention.Metadata, +pub const Metadata = union(Backend) { + vanilla: void, + cuda_fa2: flashattn.fa2.Metadata, + cuda_fa3: flashattn.fa3.Metadata, - pub const InitOptions = union(BackendType) { - stablehlo: void, - flash_attention: flash_attention.Metadata.InitOptions, + pub const InitOptions = union(Backend) { + vanilla: void, + cuda_fa2: flashattn.fa2.Metadata.InitOptions, + cuda_fa3: flashattn.fa3.Metadata.InitOptions, - pub fn fromBackend(backend: Backend) InitOptions { + pub fn fromBackend(backend: Backend, seqlen: i64) InitOptions { return switch (backend) { - .stablehlo => .{ .stablehlo = {} }, - .flash_attention => |b| .{ .flash_attention = .fromBackend(b) }, + .vanilla => .{ .vanilla = {} }, + .cuda_fa2 => .{ .cuda_fa2 = .{ .seqlen = seqlen } }, + .cuda_fa3 => .{ .cuda_fa3 = .{ .seqlen = seqlen } }, }; } }; pub fn init(opts: InitOptions) Metadata { return switch (opts) { - .stablehlo => .{ .stablehlo = {} }, - .flash_attention => |v| .{ .flash_attention = .init(v) }, + .vanilla => .{ .vanilla = {} }, + .cuda_fa2 => |o| .{ .cuda_fa2 = flashattn.fa2.Metadata.init(o) }, + .cuda_fa3 => |o| .{ .cuda_fa3 = flashattn.fa3.Metadata.init(o) }, }; } pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { return switch (self) { - .stablehlo => .{ .stablehlo = {} }, - .flash_attention => |v| .{ .flash_attention = try v.initBuffer(io, platform) }, + .vanilla => .{ .vanilla = {} }, + .cuda_fa2 => |v| .{ .cuda_fa2 = try v.initBuffer(io, platform) }, + .cuda_fa3 => |v| .{ .cuda_fa3 = try v.initBuffer(io, platform) }, }; } pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { switch (self.*) { - .stablehlo => {}, - .flash_attention => |*v| flash_attention.Metadata.deinitBuffer(v), + .vanilla => {}, + .cuda_fa2 => |*v| flashattn.fa2.Metadata.deinitBuffer(v), + .cuda_fa3 => |*v| flashattn.fa3.Metadata.deinitBuffer(v), } } }; -pub const Options = @import("Options.zig"); +pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + return switch (parameters) { + .vanilla => b: { + // Generate the attention mask. + const seq_len = k.dim(.k); + var attn_mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, q.dtype(), null); -pub const Partitioning = struct { - const Logical = zml.Sharding.Partitioning.Logical; - - in_specs: struct { - q: Logical, - k: Logical, - v: Logical, - token_index: Logical = .replicated, - /// Omit to replicate backend metadata tensors. - metadata: ?zml.ops.PartitionSpecs(Metadata) = null, - }, - out_specs: Logical, - - /// Shards the `.h` dimension of Q, K, V and the output over `mesh_axis`. - /// Other dimensions, token indices and backend metadata are replicated. - pub fn head(comptime mesh_axis: @EnumLiteral()) Partitioning { - const heads: Logical = .parse(.{ .h = mesh_axis }); - return .{ .in_specs = .{ .q = heads, .k = heads, .v = heads }, .out_specs = heads }; - } -}; - -/// Causal attention as used in transformers. -/// -/// **Shapes**: -/// - q, result: .{ .q, .h, .hd } -/// - k, v: .{ .k, .h, .hd } -/// -/// Where: -/// - .h is the number of head -/// - .q is the number of queries -/// - .k is the number of keys -/// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { - return attentionOpts(q, k, v, token_index, metadata, parameters, .{}, mesh, partitioning); -} - -/// `attention`, causal or not depending on `opts`. -pub fn attentionOpts(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { - const args: Attention = .{ .q = q, .k = k, .v = v, .token_index = token_index, .metadata = metadata, .parameters = parameters, .opts = opts }; - return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, .{ - .in_specs = .{ - .q = partitioning.in_specs.q, - .k = partitioning.in_specs.k, - .v = partitioning.in_specs.v, - .token_index = partitioning.in_specs.token_index, - .metadata = partitioning.in_specs.metadata orelse zml.ops.partitionSpecs(metadata, .replicated), + // Note: in Pytorch it would be very inefficient to generate the full attn_mask, + // then slice into it, but XLA is able to optimize this correctly. + attn_mask = attn_mask.gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, attn_mask.dtype()), token_index.reshape(.{ .coord = 1 }), .{}); + const attn_output = zml.nn.sdpa(q, k, v, .{ .attn_mask = attn_mask, .allow_cudnn = true }); + break :b attn_output; }, - .out_specs = partitioning.out_specs, - }); + .cuda_fa2 => flashattn.fa2.attention(q, k, v, token_index, metadata.cuda_fa2, parameters.cuda_fa2), + .cuda_fa3 => flashattn.fa3.attention(q, k, v, token_index, metadata.cuda_fa3, parameters.cuda_fa3), + }; } - -const Attention = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - token_index: zml.Tensor, - metadata: Metadata, - parameters: Parameters, - opts: Options, - - fn forward(args: @This(), _: zml.Shape) zml.Tensor { - return switch (args.parameters) { - .stablehlo => b: { - const seq_len = args.k.dim(.k); - const mask: ?zml.Tensor = if (args.opts.causal) - zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, args.q.dtype(), null) - .gatherSlices(zml.Shape.init(.{ .q = args.q.dim(.q) }, args.q.dtype()), args.token_index.appendAxes(.{.coord}), .{}) - else - null; - break :b zml.nn.sdpa(args.q, args.k, args.v, .{ .attn_mask = mask }); - }, - .flash_attention => flash_attention.attention(args.q, args.k, args.v, args.token_index, args.metadata.flash_attention, args.parameters.flash_attention, args.opts), - }; - } -}; diff --git a/zml/attention/flashattn.zig b/zml/attention/flashattn.zig new file mode 100644 index 00000000..111b7b1c --- /dev/null +++ b/zml/attention/flashattn.zig @@ -0,0 +1,1438 @@ +const std = @import("std"); + +const flashattn = @import("platforms/cuda/flashattn"); +const platforms = @import("platforms"); +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); +const ffi = zml.pjrt.ffi; + +const AttentionOptions = @import("paged_attention.zig").AttentionOptions; + +pub fn load(allocator: std.mem.Allocator, io: std.Io) !void { + if (comptime platforms.isEnabled(.cuda)) { + try flashattn.load(allocator, io); + } +} + +pub fn register(platform: *const zml.Platform) !void { + if (comptime platforms.isEnabled(.cuda)) { + try fa2.register(platform); + try fa3.register(platform); + try paged_fa2.Decode.register(platform); + try paged_fa2.Prefill.register(platform); + try paged_fa3.Decode.register(platform); + try paged_fa3.Prefill.register(platform); + } +} + +fn flashattnDataTypeFromZmlDataType(dtype: zml.DataType) flashattn.DataType { + return switch (dtype) { + .f32 => .f32, + .f16 => .f16, + .bf16 => .bf16, + .i32 => .i32, + .i8 => .i8, + else => unreachable, + }; +} + +fn tensorFromBuffer(buffer: zml.Buffer, platform: zml.Platform) !flashattn.Tensor { + return .init( + try buffer._shards.get(0).getOpaqueDeviceMemoryDataPointer(platform.pjrt_api), + buffer.shape().dims(), + buffer.shape().withDtype(.u8).computeStrides().constSlice(), + flashattnDataTypeFromZmlDataType(buffer.dtype()), + ); +} + +const Buffer = struct { + shape: zml.Shape, + ptr: *anyopaque, + + fn toFlashattnTensor(buffer: Buffer) flashattn.Tensor { + return .init( + buffer.ptr, + buffer.shape.dims(), + buffer.shape.withDtype(.u8).computeByteStrides().constSlice(), + flashattnDataTypeFromZmlDataType(buffer.shape.dtype()), + ); + } + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print("{f}@{*}", .{ self.shape, self.ptr }); + } +}; + +fn getPlatform(call_frame: *ffi.CallFrame) zml.Platform { + const pjrt_api_ptr = call_frame.attrs.getByName(.scalar, "pjrt_api") orelse unreachable; + std.debug.assert(pjrt_api_ptr.dtype == .u64); + const pjrt_api: ?*zml.pjrt.Api = @ptrFromInt(pjrt_api_ptr.get(usize)); + + const pjrt_client_ptr = call_frame.attrs.getByName(.scalar, "pjrt_client") orelse unreachable; + std.debug.assert(pjrt_client_ptr.dtype == .u64); + const pjrt_client: ?*zml.pjrt.Client = @ptrFromInt(pjrt_client_ptr.get(usize)); + + return .{ .target = .cuda, .pjrt_api = pjrt_api.?, .pjrt_client = pjrt_client.? }; +} + +fn dataTypeFromFfiDataType(ffi_dt: ffi.DataType) zml.DataType { + return switch (ffi_dt) { + .bool => .bool, + .i8 => .i8, + .i16 => .i16, + .i32 => .i32, + .i64 => .i64, + .u8 => .u8, + .u16 => .u16, + .u32 => .u32, + .u64 => .u64, + .f16 => .f16, + .f32 => .f32, + .f64 => .f64, + .bf16 => .bf16, + .c64 => .c64, + .c128 => .c128, + .f8e5m2 => .f8e5m2, + .f8e4m3fn => .f8e4m3fn, + .f8e4m3b11fnuz => .f8e4m3b11fnuz, + .f8e5m2fnuz => .f8e5m2fnuz, + .f8e4m3fnuz => .f8e4m3fnuz, + else => unreachable, + }; +} + +fn shapeFromFfiBuffer(buffer: *const ffi.Buffer) zml.Shape { + return .init(buffer.dims(), dataTypeFromFfiDataType(buffer.dtype)); +} + +fn bufferFromFfiBuffer(ffi_buffer: *const ffi.Buffer) Buffer { + return .{ + .shape = shapeFromFfiBuffer(ffi_buffer), + .ptr = ffi_buffer.data, + }; +} + +fn getScalarAttributeAs(comptime T: type, call_frame: *ffi.CallFrame, attribute_name: []const u8) ?T { + const attribute = call_frame.attrs.getByName(.scalar, attribute_name) orelse return null; + return attribute.get(T); +} + +fn fixupKvCacheBuffer(buffer: Buffer, layer_index: i64) Buffer { + var shape = buffer.shape; + const layer_stride = shape.computeByteStrides().get(0); + shape = shape.remove(0); + const ptr = @as([*]u8, @ptrCast(buffer.ptr)); + return .{ + .shape = shape, + .ptr = ptr + @as(usize, @intCast(layer_stride * layer_index)), + }; +} + +pub fn Wrapper(comptime T: type, run_func: std.meta.DeclEnum(T)) type { + return struct { + pub fn register(platform: *const zml.Platform) !void { + try platform.pjrt_api.ffi().?.register(platform.pjrt_api, T.custom_call_name, "cuda", T.run, .{ .command_buffer_compatible = true }); + } + + pub fn run(call_frame: *ffi.CallFrame) callconv(.c) ?*ffi.Error { + return @field(T, @tagName(run_func))(call_frame) catch b: { + break :b ffi.Error.create(call_frame.api.?, .unknown, "Unknown"); + }; + } + }; +} + +pub const fa2 = struct { + const custom_call_name = "fa2_mha_varlen_fwd"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); + const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &k.toFlashattnTensor(), + &v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + null, + null, + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + + pub const Parameters = struct { + pub const InitOptions = struct {}; + + pub fn init(opts: InitOptions) fa2.Parameters { + _ = opts; // autofix + return .{}; + } + }; + + pub const Metadata = struct { + softmax_lse: zml.Tensor, + softmax_lse_accum: zml.Tensor, + out_accum: zml.Tensor, + + pub const InitOptions = struct { + seqlen: i64, + }; + + pub fn init(opts: InitOptions) Metadata { + return .{ + .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), + .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), + .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + return .{ + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + self.softmax_lse.deinit(); + self.softmax_lse_accum.deinit(); + self.out_accum.deinit(); + } + }; + + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + _ = parameters; // autofix + stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa2.attention support for batch size != 1 is not supported yet.", .{}); + const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); + // TODO(Corendos): replace with cumsum + const cu_seqlens_k = b: { + const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); + break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); + }; + const max_seqlen_q: i32 = @intCast(q_.dim(.q)); + const max_seqlen_k: i32 = @intCast(k_.dim(.k)); + var q = q_.insertAxes(.q, .{.b}).merge(.{ .tot = .{ .b, .q } }); + const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + // TODO(Corendos): replace with cumsum + const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); + + const original_tot = q.dim(.tot); + const num_heads = q_.dim(.h); + const num_heads_k = k_.dim(.h); + const head_size = q_.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = max_seqlen_q == 1 and num_heads > num_heads_k and @mod(head_size, 8) == 0; + if (seqlenq_ngroups_swapped) { + q = q.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .tot, .ngroups, .h, .hd }).merge(.{ .tot = .{ .tot, .ngroups } }); + } + + var o = zml.ops.customCall( + custom_call_name, + .{ + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + metadata.softmax_lse, + metadata.softmax_lse_accum, + metadata.out_accum, + }, + .{q.shape()}, + .{ + //.softmax_scale = @as(f32, 1.0), + .is_causal = true, + .window_size_left = @as(i32, -1), + .window_size_right = @as(i32, -1), + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .num_heads = @as(i32, @intCast(num_heads)), + }, + .{ + .output_operand_aliases = &.{0}, + .has_side_effect = true, + }, + ); + + if (seqlenq_ngroups_swapped) { + o = o.splitAxis(.tot, .{ .tot = original_tot, .ngroups = ngroups }).transpose(.{ .tot, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); + } +}; + +pub const fa3 = struct { + const custom_call_name = "fa3_mha_fwd"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const k = bufferFromFfiBuffer(call_frame.args.buffers()[1]); + const v = bufferFromFfiBuffer(call_frame.args.buffers()[2]); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_q: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_q").?; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &k.toFlashattnTensor(), + &v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + null, + null, + null, + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + + pub const Parameters = struct { + pub const InitOptions = struct {}; + + pub fn init(opts: InitOptions) fa3.Parameters { + _ = opts; // autofix + return .{}; + } + }; + + pub const Metadata = struct { + softmax_lse: zml.Tensor, + softmax_lse_accum: zml.Tensor, + out_accum: zml.Tensor, + scheduler_metadata: zml.Tensor, + + pub const InitOptions = struct { + seqlen: i64, + }; + + pub fn init(opts: InitOptions) Metadata { + return .{ + .softmax_lse = .init(.{1 * 32 * opts.seqlen * 4}, .i8), + .softmax_lse_accum = .init(.{32 * 1 * 32 * 4 * 4}, .i8), + .out_accum = .init(.{32 * 1 * 32 * 4 * 128 * 4}, .i8), + .scheduler_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { + return .{ + .softmax_lse = try zml.Buffer.uninitialized(io, platform, self.softmax_lse.shape(), .{}), + .softmax_lse_accum = try zml.Buffer.uninitialized(io, platform, self.softmax_lse_accum.shape(), .{}), + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .scheduler_metadata = try zml.Buffer.uninitialized(io, platform, self.scheduler_metadata.shape(), .{}), + }; + } + + pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { + self.softmax_lse.deinit(); + self.softmax_lse_accum.deinit(); + self.out_accum.deinit(); + self.scheduler_metadata.deinit(); + } + }; + + pub fn attention(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters) zml.Tensor { + _ = parameters; // autofix + stdx.debug.assert(q_.shape().hasTag(.b) == null or q_.dim(.b) == 1, "fa3.attention support for batch size != 1 is not supported yet.", .{}); + const seqused_k = token_index.addConstant(q_.dim(.q)).reshape(.{1}); + // TODO(Corendos): replace with cumsum + const cu_seqlens_k = b: { + const zero = zml.Tensor.constant(token_index.dtype().zero()).reshape(.{1}); + break :b zml.Tensor.concatenate(&.{ zero, seqused_k }, 0).convert(.i32); + }; + const max_seqlen_q: i32 = @intCast(q_.dim(.q)); + const max_seqlen_k: i32 = @intCast(k_.dim(.k)); + var q = q_.insertAxes(.q, .{.b}).merge(.{ .tot = .{ .b, .q } }); + const k = k_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + const v = v_.insertAxes(.k, .{.b}).merge(.{ .tot = .{ .b, .k } }); + // TODO(Corendos): replace with cumsum + const cu_seqlens_q = zml.Tensor.constantTensor(zml.Shape.init(.{2}, .i32), std.mem.sliceAsBytes(&[2]i32{ 0, max_seqlen_q })); + + var o = zml.ops.customCall( + custom_call_name, + .{ + q, + k, + v, + cu_seqlens_q, + cu_seqlens_k, + metadata.softmax_lse, + metadata.softmax_lse_accum, + metadata.out_accum, + metadata.scheduler_metadata, + }, + .{q.shape()}, + .{ + .is_causal = true, + .window_size_left = @as(i32, -1), + .window_size_right = @as(i32, -1), + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + }, + .{ + .output_operand_aliases = &.{0}, + .has_side_effect = true, + }, + ); + + return o.splitAxis(.tot, .{ .b = 1, .q = q_.dim(.q) }).squeeze(.b); + } +}; + +pub const paged_fa2 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: DecodeParameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: DecodeMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: MixedParameters) usize { + var allocation_size: usize = 0; + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); + + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{8 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: MixedMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + decode_offset: ?zml.Tensor = null, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa2_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa2_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const cu_seqlens_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + const num_heads: i32 = getScalarAttributeAs(i32, call_frame, "num_heads").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA2MhaVarlenFwdParams = .{ + .max_seqlen_q = 1, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .num_splits = 0, + .num_heads = num_heads, + }; + + flashattn.fa2_mha_varlen_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + &cu_seqlens_k.toFlashattnTensor(), + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + &softmax_lse.toFlashattnTensor(), + null, + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k = zml.Tensor.constant(zml.DataType.i32.zero()).broad(decode_parameters.cu_seqlens_q.shape()); + + const original_tot = q.dim(.b); + const num_heads = q.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; + var q2 = q; + if (seqlenq_ngroups_swapped) { + q2 = q2.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q2, + k_cache, + v_cache, + decode_parameters.cu_seqlens_q, + dummy_cu_seqlens_k, + decode_parameters.seqused_k, + decode_parameters.block_table, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + layer_index, + }, + .{q2.shape()}, + .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = num_heads, + .window_size_left = opts.sliding_window, + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o = o.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_prefill.shape()); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_prefill, + dummy_cu_seqlens_k_prefill, + mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + mixed_parameters.metadata.host_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{64 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const dummy_cu_seqlens_k_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(mixed_parameters.cu_seqlens_q_decode.shape()); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const original_tot = q_decode.dim(.b); + const num_heads = q_decode.dim(.h); + const num_heads_k = k_cache.dim(.h); + const head_size = q_decode.dim(.hd); + const ngroups = @divExact(num_heads, num_heads_k); + const seqlenq_ngroups_swapped = num_heads > num_heads_k and @mod(head_size, 8) == 0 and opts.sliding_window < 0; + if (seqlenq_ngroups_swapped) { + q_decode = q_decode.splitAxis(.h, .{ .h = num_heads_k, .ngroups = ngroups }).transpose(.{ .b, .ngroups, .h, .hd }).merge(.{ .b = .{ .b, .ngroups } }); + } + + var o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_decode, + dummy_cu_seqlens_k_decode, + mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + mixed_parameters.metadata.out_accum, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = opts.is_causal, + .max_seqlen_k = context.max_seqlen_k, + .num_heads = q.dim(.h), + .window_size_left = opts.sliding_window, + }, + .{ .has_side_effect = false }, + ); + + if (seqlenq_ngroups_swapped) { + o_decode = o_decode.splitAxis(.b, .{ .b = original_tot, .ngroups = ngroups }).transpose(.{ .b, .h, .ngroups, .hd }).merge(.{ .h = .{ .h, .ngroups } }); + } + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; + +pub const paged_fa3 = struct { + pub const Variant = enum { + decode, + mixed, + }; + + pub const Options = union(Variant) { + decode: DecodeOptions, + mixed: MixedOptions, + + pub fn isPrefill(self: Options) bool { + return self == .mixed; + } + }; + + pub const Parameters = union(Variant) { + decode: DecodeParameters, + mixed: MixedParameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .decode => |decode_options| .{ .decode = DecodeParameters.init(decode_options) }, + .mixed => |mixed_options| .{ .mixed = MixedParameters.init(mixed_options) }, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + .decode => |decode| decode.allocationSize(), + .mixed => |mixed| mixed.allocationSize(), + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .decode => |v| .{ .decode = v.options }, + .mixed => |v| .{ .mixed = v.options }, + }; + } + }; + + pub const DecodeOptions = struct { + batch_size: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const DecodeParameters = struct { + block_table: zml.Tensor, + cu_seqlens_q: zml.Tensor, + seqused_k: zml.Tensor, + + metadata: DecodeMetadata, + options: DecodeOptions, + + pub fn init(options: DecodeOptions) DecodeParameters { + return .{ + .block_table = .init(.{ options.batch_size, options.max_num_pages }, .i32), + .cu_seqlens_q = .init(.{options.batch_size + 1}, .i32), + .seqused_k = .init(.{options.batch_size}, .i32), + .metadata = DecodeMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: DecodeParameters) usize { + var allocation_size: usize = 0; + + allocation_size += self.block_table.byteSize(); + allocation_size += self.cu_seqlens_q.byteSize(); + allocation_size += self.seqused_k.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const DecodeMetadata = struct { + out_accum: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) DecodeMetadata { + return .{ + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), + }; + } + + pub fn initBuffers(self: *const DecodeMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(DecodeMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(DecodeMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: DecodeMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + return allocation_size; + } + }; + + pub const MixedOptions = struct { + batch_size_prefill: usize, + batch_size_decode: usize, + max_num_pages: usize, + max_seqlen_k: usize, + max_token_count: usize, + num_heads: usize, + head_dim: usize, + }; + + pub const MixedParameters = struct { + block_table_prefill: zml.Tensor, + cu_seqlens_q_prefill: zml.Tensor, + seqused_k_prefill: zml.Tensor, + + block_table_decode: zml.Tensor, + cu_seqlens_q_decode: zml.Tensor, + seqused_k_decode: zml.Tensor, + + metadata: MixedMetadata, + options: MixedOptions, + + pub fn init(options: MixedOptions) MixedParameters { + return .{ + .block_table_prefill = .init(.{ options.batch_size_prefill, options.max_num_pages }, .i32), + .cu_seqlens_q_prefill = .init(.{options.batch_size_prefill + 1}, .i32), + .seqused_k_prefill = .init(.{options.batch_size_prefill}, .i32), + .block_table_decode = .init(.{ options.batch_size_decode, options.max_num_pages }, .i32), + .cu_seqlens_q_decode = .init(.{options.batch_size_decode + 1}, .i32), + .seqused_k_decode = .init(.{options.batch_size_decode}, .i32), + .metadata = MixedMetadata.init(options.max_token_count, options.num_heads, options.head_dim), + .options = options, + }; + } + + pub fn allocationSize(self: MixedParameters) usize { + var allocation_size: usize = 0; + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_prefill.byteSize(); + allocation_size += self.seqused_k_prefill.byteSize(); + + allocation_size += self.block_table_decode.byteSize(); + allocation_size += self.cu_seqlens_q_decode.byteSize(); + allocation_size += self.seqused_k_decode.byteSize(); + allocation_size += self.metadata.allocationSize(); + + return allocation_size; + } + }; + + pub const MixedMetadata = struct { + out_accum: zml.Tensor, + host_metadata: zml.Tensor, + + pub fn init(max_token_count: usize, num_heads: usize, head_dim: usize) MixedMetadata { + return .{ + .out_accum = .init(.{32 * max_token_count * num_heads * head_dim * 4}, .i8), + .host_metadata = .init(.{2}, .i32), + }; + } + + pub fn initBuffers(self: *const MixedMetadata, io: std.Io, platform: zml.Platform) !zml.Bufferized(MixedMetadata) { + return .{ + .out_accum = try zml.Buffer.uninitialized(io, platform, self.out_accum.shape(), .{}), + .host_metadata = undefined, + }; + } + + pub fn deinitBuffers(self: *const zml.Bufferized(MixedMetadata)) void { + self.out_accum.deinit(); + } + + pub fn allocationSize(self: MixedMetadata) usize { + var allocation_size: usize = 0; + allocation_size += self.out_accum.byteSize(); + allocation_size += self.host_metadata.byteSize(); + return allocation_size; + } + }; + + pub const Context = struct { + max_seqlen_k: usize, + decode_offset: ?zml.Tensor = null, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + _ = num_heads; // autofix + _ = num_kv_heads; // autofix + _ = head_dim; // autofix + _ = page_size; // autofix + const max_seqlen_k = switch (parameters) { + inline else => |v| v.options.max_seqlen_k, + }; + const decode_offset = switch (parameters) { + .mixed => |mixed_parameters| mixed_parameters.metadata.host_metadata.slice1d(0, .{ .end = 1 }).squeeze(0), + .decode => null, + }; + return .{ .max_seqlen_k = max_seqlen_k, .decode_offset = decode_offset }; + } + }; + + pub const Prefill = struct { + pub const custom_call_name = "paged_fa3_prefill"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[11]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const host_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const max_seqlen_q = @as([*]i32, @ptrCast(@alignCast(host_metadata.ptr)))[1]; + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = max_seqlen_q, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub const Decode = struct { + pub const custom_call_name = "paged_fa3_decode"; + const Wrapped = Wrapper(@This(), .runInner); + + const register = Wrapped.register; + const run = Wrapped.run; + + pub fn runInner(call_frame: *ffi.CallFrame) !?*ffi.Error { + if (call_frame.registeringHook()) return null; + + const layer_index = bufferFromFfiBuffer(call_frame.args.buffers()[10]); + const layer_index_raw = @as([*]i32, @ptrCast(@alignCast(layer_index.ptr)))[0]; + + const q = bufferFromFfiBuffer(call_frame.args.buffers()[0]); + const paged_k = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[1]), layer_index_raw); + const paged_v = fixupKvCacheBuffer(bufferFromFfiBuffer(call_frame.args.buffers()[2]), layer_index_raw); + const cu_seqlens_q = bufferFromFfiBuffer(call_frame.args.buffers()[3]); + const seqused_k = bufferFromFfiBuffer(call_frame.args.buffers()[4]); + const block_table = bufferFromFfiBuffer(call_frame.args.buffers()[5]); + const softmax_lse = bufferFromFfiBuffer(call_frame.args.buffers()[6]); + const softmax_lse_accum = bufferFromFfiBuffer(call_frame.args.buffers()[7]); + const out_accum = bufferFromFfiBuffer(call_frame.args.buffers()[8]); + const scheduler_metadata = bufferFromFfiBuffer(call_frame.args.buffers()[9]); + const o = bufferFromFfiBuffer(call_frame.results.buffers()[0]); + + const softmax_scale: f32 = getScalarAttributeAs(f32, call_frame, "softmax_scale") orelse b: { + const head_dim = q.shape.dim(2); + break :b 1.0 / std.math.sqrt(@as(f32, @floatFromInt(head_dim))); + }; + const is_causal: bool = getScalarAttributeAs(bool, call_frame, "is_causal").?; + const window_size_left: i32 = getScalarAttributeAs(i32, call_frame, "window_size_left") orelse -1; + const window_size_right: i32 = getScalarAttributeAs(i32, call_frame, "window_size_right") orelse -1; + const max_seqlen_k: i32 = getScalarAttributeAs(i32, call_frame, "max_seqlen_k").?; + + const ctx: *ffi.ExecutionContext = @constCast(call_frame.ctx); + const stream = call_frame.api.stream(ctx); + + const params: flashattn.FA3MhaFwdParams = .{ + .max_seqlen_q = 1, + .max_seqlen_k = max_seqlen_k, + .is_causal = is_causal, + .softmax_scale = softmax_scale, + .window_size_left = window_size_left, + .window_size_right = window_size_right, + .softcap = 0.0, + .is_rotary_interleaved = false, + .num_splits = 0, + .sm_margin = 0, + .cp_world_size = 1, + .cp_rank = 0, + }; + + flashattn.fa3_mha_fwd( + &q.toFlashattnTensor(), + &paged_k.toFlashattnTensor(), + &paged_v.toFlashattnTensor(), + &o.toFlashattnTensor(), + &cu_seqlens_q.toFlashattnTensor(), + null, + null, + &seqused_k.toFlashattnTensor(), + &block_table.toFlashattnTensor(), + null, + null, + null, + &softmax_lse.toFlashattnTensor(), + &softmax_lse_accum.toFlashattnTensor(), + &out_accum.toFlashattnTensor(), + &scheduler_metadata.toFlashattnTensor(), + null, + null, + ¶ms, + stream, + ); + + return null; + } + }; + + pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + _ = opts; // autofix + stdx.debug.assert(q.shape().hasTags(.{ .b, .h, .hd }), "Expected q to have tags .b, .h, .hd", .{}); + stdx.debug.assert(k_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_k to have tags .page, .k_chunk, .h, .hd, got {}", .{k_cache.shape()}); + stdx.debug.assert(v_cache.shape().hasTags(.{ .page, .k_chunk, .h, .hd }), "Expected paged_v to have tags .page, .k_chunk, .h, .hd. got {}", .{v_cache.shape()}); + + const o = switch (parameters) { + .decode => |decode_parameters| b: { + const softmax_lse = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * decode_parameters.block_table.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = decode_parameters.block_table.dim(0) + 1 }, .i32)); + + const o = zml.ops.customCall( + Decode.custom_call_name, + .{ + q, + k_cache, + v_cache, + decode_parameters.cu_seqlens_q, + decode_parameters.seqused_k, + decode_parameters.block_table, + softmax_lse, + softmax_lse_accum, + decode_parameters.metadata.out_accum, + scheduler_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + break :b o; + }, + .mixed => |mixed_parameters| b: { + const softmax_lse_prefill = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = q.dim(.b) }, .f32)); + const softmax_lse_accum_prefill = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * q.dim(.b) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_prefill = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_prefill.dim(0) + 1 }, .i32)); + var o = zml.ops.customCall( + Prefill.custom_call_name, + .{ + q, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_prefill, + mixed_parameters.seqused_k_prefill, + mixed_parameters.block_table_prefill, + softmax_lse_prefill, + softmax_lse_accum_prefill, + mixed_parameters.metadata.out_accum, + scheduler_metadata_prefill, + mixed_parameters.metadata.host_metadata, + layer_index, + }, + .{q.shape()}, + .{ + .is_causal = false, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + const batch_size_decode = mixed_parameters.block_table_prefill.dim(0); + const softmax_lse_decode = zml.Tensor.constant(zml.DataType.f32.zero()).broad(.init(.{ .h = q.shape().dim(.h), .q = batch_size_decode }, .f32)); + const softmax_lse_accum_decode = zml.Tensor.constant(zml.DataType.i8.zero()).broad(.init(.{32 * mixed_parameters.block_table_decode.dim(0) * q.dim(.h) * 4}, .i8)); + const scheduler_metadata_decode = zml.Tensor.constant(zml.DataType.i32.zero()).broad(.init(.{ .b = mixed_parameters.block_table_decode.dim(0) + 1 }, .i32)); + var q_decode = q.dynamicSlice1d(0, .{ .start = context.decode_offset.?, .len = batch_size_decode }); + + const o_decode = zml.ops.customCall( + Decode.custom_call_name, + .{ + q_decode, + k_cache, + v_cache, + mixed_parameters.cu_seqlens_q_decode, + mixed_parameters.seqused_k_decode, + mixed_parameters.block_table_decode, + softmax_lse_decode, + softmax_lse_accum_decode, + mixed_parameters.metadata.out_accum, + scheduler_metadata_decode, + layer_index, + }, + .{q_decode.shape()}, + .{ + .is_causal = true, + .max_seqlen_k = context.max_seqlen_k, + }, + .{ .has_side_effect = false }, + ); + + o = o.dynamicUpdateSlice1d(o_decode, 0, context.decode_offset.?); + break :b o; + }, + }; + + return o; + } +}; diff --git a/zml/attention/paged_attention.zig b/zml/attention/paged_attention.zig new file mode 100644 index 00000000..deb63ec7 --- /dev/null +++ b/zml/attention/paged_attention.zig @@ -0,0 +1,145 @@ +const std = @import("std"); +const stdx = @import("stdx"); + +const zml = @import("../zml.zig"); +const flashattn = @import("flashattn.zig"); + +const PagedAttention = @This(); + +pub const Backend = enum { + cuda_fa2, + cuda_fa3, + + pub fn auto(platform: *const zml.Platform) Backend { + return switch (platform.target) { + .cuda => b: { + const first_device = platform.devices[0].pjrt_device; + + if (zml.platform.cuda.tryGetComputeCapabilities(platform, first_device)) |cc| { + if (std.mem.eql(u8, cc, "9.0")) { + break :b .cuda_fa3; + } + } + + break :b .cuda_fa2; + }, + else => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), + }; + } +}; + +pub const Options = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Options, + cuda_fa3: flashattn.paged_fa3.Options, + + pub fn fromBackend(backend: Backend, is_prefill: bool, batch_size: u32, seq_len: u32, page_chunk_size: u32, max_token_count: u32, num_attention_heads: u32, head_dim: u32) Options { + return switch (backend) { + .cuda_fa2 => if (is_prefill) .{ + .cuda_fa2 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa2 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + .cuda_fa3 => if (is_prefill) .{ + .cuda_fa3 = .{ + .mixed = .{ + .batch_size_decode = batch_size, + .batch_size_prefill = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + } else .{ + .cuda_fa3 = .{ + .decode = .{ + .batch_size = batch_size, + .max_num_pages = seq_len / page_chunk_size, + .max_seqlen_k = seq_len, + .max_token_count = max_token_count, + .num_heads = num_attention_heads, + .head_dim = head_dim, + }, + }, + }, + }; + } + + pub fn isPrefill(self: Options) bool { + return switch (self) { + inline else => |v| v.isPrefill(), + }; + } +}; + +pub const Parameters = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Parameters, + cuda_fa3: flashattn.paged_fa3.Parameters, + + pub fn init(options_: Options) Parameters { + return switch (options_) { + .cuda_fa2 => |cuda_fa2_options| .{ .cuda_fa2 = flashattn.paged_fa2.Parameters.init(cuda_fa2_options) }, + .cuda_fa3 => |cuda_fa3_options| .{ .cuda_fa3 = flashattn.paged_fa3.Parameters.init(cuda_fa3_options) }, + }; + } + + pub fn options(self: Parameters) Options { + return switch (self) { + .cuda_fa2 => |v| .{ .cuda_fa2 = v.options() }, + .cuda_fa3 => |v| .{ .cuda_fa3 = v.options() }, + }; + } + + pub fn allocationSize(self: Parameters) usize { + return switch (self) { + inline else => |v| v.allocationSize(), + }; + } +}; + +/// Internal state that can be used inside the model code. It's derived from Parameters and Option. +pub const Context = union(Backend) { + cuda_fa2: flashattn.paged_fa2.Context, + cuda_fa3: flashattn.paged_fa3.Context, + + pub fn init(parameters: Parameters, num_heads: i64, num_kv_heads: i64, head_dim: i64, page_size: i64) Context { + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| .{ .cuda_fa2 = flashattn.paged_fa2.Context.init(cuda_fa2_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + .cuda_fa3 => |cuda_fa3_parameters| .{ .cuda_fa3 = flashattn.paged_fa3.Context.init(cuda_fa3_parameters, num_heads, num_kv_heads, head_dim, page_size) }, + }; + } +}; + +pub const AttentionOptions = struct { + is_causal: bool = true, + sliding_window: i32 = -1, +}; + +pub fn pagedAttention(parameters: Parameters, context: Context, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, layer_index: zml.Tensor, opts: AttentionOptions) zml.Tensor { + _ = k; // autofix + _ = v; // autofix + return switch (parameters) { + .cuda_fa2 => |cuda_fa2_parameters| flashattn.paged_fa2.pagedAttention(cuda_fa2_parameters, context.cuda_fa2, q, k_cache, v_cache, layer_index, opts), + .cuda_fa3 => |cuda_fa3_parameters| flashattn.paged_fa3.pagedAttention(cuda_fa3_parameters, context.cuda_fa3, q, k_cache, v_cache, layer_index, opts), + }; +} diff --git a/zml/buffer.zig b/zml/buffer.zig index 8714d80f..7b0a88ac 100644 --- a/zml/buffer.zig +++ b/zml/buffer.zig @@ -1,113 +1,43 @@ const std = @import("std"); const pjrt = @import("pjrt"); -const platforms = @import("platforms"); const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; -const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; -const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const Sharding = @import("Sharding.zig"); const Slice = @import("slice.zig").Slice; const Target = @import("platform.zig").Target; const testing = @import("testing.zig"); const log = std.log.scoped(.zml); -test { - std.testing.refAllDecls(Buffer); - std.testing.refAllDecls(Buffer.HostAccessible); -} - /// Buffer is a multi-dimension array, whose memory is allocated on an accelerator. /// /// * contains a handle that the ZML runtime can use to convert into a physical address, but there is no guarantee this address is visible from the CPU. /// * loading weights from disk directly to the `device zml.aio.loadBuffers` /// * can be created by calling `HostBuffer.toDevice(platform)`. pub const Buffer = struct { - _platform: *const Platform, _shape: Shape, + platform: *const Platform, _shards: Shards, - /// Whether executions may donate a buffer, writing their outputs into it. - /// `HostAccessible.init` uses it to decide whether devices share allocations. - pub const Donation = enum { - /// Executions only read the buffer. Unlike XLA's undonatable buffers, the - /// host may still rewrite a HostAccessible one between executions. - undonatable, - /// Executions may donate the buffer, replacing its handles with their outputs. - donatable, - }; - - pub const HostAccessible = @import("buffer/HostAccessible.zig"); - - pub const MAX_NUM_SHARDS: u16 = Platform.MAX_NUM_DEVICES; + pub const MAX_NUM_SHARDS: u8 = Platform.MAX_NUM_DEVICES; pub const Shards = stdx.BoundedArray(*pjrt.Buffer, MAX_NUM_SHARDS); - pub const Shard = struct { - _platform: *const Platform, - _pjrt_buffer: *pjrt.Buffer, - - pub fn devicePtr(self: *const Shard) *anyopaque { - return self._pjrt_buffer.opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; - } - }; - - pub const ShardIterator = struct { - _platform: *const Platform, - _shards: []const *pjrt.Buffer, - _index: usize = 0, - - pub fn remaining(self: *ShardIterator) usize { - return self._shards.len -| self._index; - } - - pub fn next(self: *ShardIterator) ?Shard { - defer self._index += 1; - if (self._index >= self._shards.len) return null; - - return .{ - ._pjrt_buffer = self._shards[self._index], - ._platform = self._platform, - }; - } - }; - pub const FromOptions = struct { wait: bool = true, memory: Memory.Kind = .default }; /// Frees the accelerator memory. /// Depending on the platform, the memory is typically not released to the OS /// but just marked as available in the memory pool. pub fn deinit(self: *Buffer) void { + // log.warn("Unloading {f} {d} bytes", .{ self._shape, self._shape.byteSize() }); for (self._shards.constSlice()) |buffer| { - buffer.deinit(self._platform.pjrt_api); + buffer.deinit(self.platform.pjrt_api); } - self._shards = .empty; - } - - /// Given a flat struct (static size, no slices) containing `zml.Buffer`, `deinit` each one of them. - pub fn deinitAll(T: type, buffers: *mem.Bufferized(T)) void { - meta.visitFlatStruct(struct { - fn deinit(_: void, x: *Buffer) void { - x.deinit(); - } - }.deinit, {}, buffers); - } - - /// Given an arbitrary struct `deinit` all `zml.Buffer` containing. - /// If the struct contains slices of `zml.Buffer` the memory of the slices will NOT be freed, - /// This only impacts device memory. - pub fn freeDeviceMemoryButKeepHostMetadataMemory(T: type, buffers: *mem.Bufferized(T)) void { - meta.visit(struct { - fn deinit(_: void, x: *Buffer) void { - x.deinit(); - } - }.deinit, {}, buffers); } /// This Buffer shape. @@ -115,90 +45,68 @@ pub const Buffer = struct { return self._shape; } - pub fn numShards(self: Buffer) u32 { - return @intCast(self._shards.len); - } - - pub fn shards(self: *const Buffer) ShardIterator { - return .{ - ._platform = self._platform, - ._shards = self._shards.constSlice(), - }; - } - pub fn format(self: Buffer, writer: *std.Io.Writer) !void { - const pl = Sharding.Placement.init(&self._shape) catch |err| { - return try writer.print("Buffer({f}) with invalid sharding {} ({})", .{ self._shape, self._shape._sharding, err }); - }; - try writer.print("{f}", .{pl}); + try writer.print("Buffer({f})@{x}", .{ self._shape, @intFromPtr(self.devicePtr()) }); } /// Copies the content of the given buffer from host memory to the accelerator memory. - pub fn from( - io: std.Io, - platform: *const Platform, - shape_: Shape, - data_: []const u8, - opts: FromOptions, - ) !Buffer { - var pjrt_shards: Shards = .empty; - var packed_shape = shape_.packedShape(); - if (packed_shape._sharding.mesh == null) packed_shape._sharding.mesh = platform.replicated_mesh; - const byte_strides = packed_shape.computeByteStrides(); - const placement = Sharding.Placement.initWithByteStrides(&packed_shape, byte_strides.constSlice()) catch |err| { - @branchHint(.cold); - std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ packed_shape, err }); - }; - const ty = pjrtx.bufferTypeFromDtype(packed_shape.dtype()); - const layout = platform.defaultMemoryLayout(placement.local_dims.constSlice(), packed_shape.dtype()); - - errdefer for (pjrt_shards.slice()) |shard| { - shard.deinit(platform.pjrt_api); + pub fn from(io: std.Io, platform: *const Platform, shape_: Shape, data_: []const u8, opts: FromOptions) !Buffer { + var res: Buffer = .{ + .platform = platform, + ._shape = shape_, + ._shards = .{}, }; - stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); - const byte_size: usize = if (packed_shape.rank() == 0) - packed_shape.dtype().sizeOf() - else - @intCast(packed_shape.dim(0) * byte_strides.get(0)); - stdx.debug.assert(data_.len == byte_size, "Expected data length {}, got {}", .{ byte_size, data_.len }); - const base_ptr = data_.ptr; + // We shard only on the first axis so that the chunks are still contiguous. + // TODO: support more advanced sharding specs + stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); + //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); + const n_partitions = platform.sharding().num_partitions; + //const chunk_size = if (sharding_ax) |ax| cs: { + // // This kind of sharding error should be detected earlier on. + // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.from({f}) expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ shape_, ax, n_partitions }); + // break :cs @divExact(shape_.dim(ax), n_partitions); + //} else 0; + //_ = chunk_size; - for (platform.physical_mesh.devices_in_canonical_order, 0..) |device, device_index| { - const memory = platform.devices[device.id].memory(opts.memory).?; - const args: pjrt.Client.BufferFromHostBufferArgs = .{ - // Change for each device - .data = placement.shardPtr(device_index, base_ptr), - .dst = .{ .memory = memory.pjrt_memory }, - // Constant across devices - .layout = layout, - .dims = placement.local_dims.constSlice(), - .buffer_type = ty, - .byte_strides = byte_strides.constSlice(), + const buffer_type = pjrtx.bufferTypeFromDtype(shape_.dtype()); + const byte_strides = shape_.computeByteStrides(); + + const devices = platform.devices; + for (0..n_partitions) |i| { + // If no sharding if found, the given buffer is replicated on all devices. + //const buf = if (sharding_ax) |ax| buf: { + // @panic("sharding not implemented"); + // //const start: i64 = @as(i64, @intCast(i)) * chunk_size; + // //break :buf host_buffer.slice1d(ax, .{ .start = start, .end = start + chunk_size }); + //} else host_buffer; + + const args = pjrt.Client.BufferFromHostBufferArgs{ + .data = data_.ptr, + .buffer_type = buffer_type, + .dims = shape_.dims(), + .byte_strides = byte_strides.slice(), .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .dst = .{ .memory = devices[i].memory(opts.memory).pjrt_memory }, }; const pjrt_buffer, const event = try platform.pjrt_client.bufferFromHostBuffer(platform.pjrt_api, args); - if (event) |ev| ev.deinit(platform.pjrt_api); - pjrt_shards.appendAssumeCapacity(pjrt_buffer); + if (event) |ev| ev.deinit(platform.pjrt_api); + res._shards.appendAssumeCapacity(pjrt_buffer); } if (opts.wait) { - try awaitShards(pjrt_shards.constSlice(), platform, io); + try res.await(io); } - return .{ ._platform = platform, ._shape = shape_, ._shards = pjrt_shards }; + return res; } /// Copies the given Zig bytes to the accelerator memory and /// return a Buffer with the given dimensions. pub fn fromBytes(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8) !Buffer { - return fromBytesOpts(io, platform, sh, data, .{}); - } - - pub fn fromBytesOpts(io: std.Io, platform: *const Platform, sh: Shape, data: []const u8, opts: FromOptions) !Buffer { - return from(io, platform, sh, data, opts); + return from(io, platform, sh, data, .{}); } /// Copies the given zml.Slice to the accelerator memory and @@ -214,72 +122,82 @@ pub const Buffer = struct { /// Creates a Buffer with a single element. pub fn scalar(io: std.Io, platform: *const Platform, val: anytype, dtype_: DataType) !Buffer { const x = dtype_.constant(val); - return fromBytes(io, platform, .scalar(dtype_), x.asBytes()); + return fromBytes(io, platform, Shape.init(.{}, dtype_), x.asBytes()); } pub fn await(self: Buffer, io: std.Io) !void { - try awaitShards(self._shards.constSlice(), self._platform, io); - } - - fn awaitShards(pjrt_shards: []const *pjrt.Buffer, platform: *const Platform, io: std.Io) !void { - for (pjrt_shards) |shard| { - const event = shard.readyEvent(platform.pjrt_api); - defer event.deinit(platform.pjrt_api); - try event.await(platform.pjrt_api, io); + for (self._shards.constSlice()) |buffer| { + const ev = buffer.readyEvent(self.platform.pjrt_api); + defer ev.deinit(self.platform.pjrt_api); + try ev.await(self.platform.pjrt_api, io); } } pub const UnitializedOptions = struct { memory: Memory.Kind = .default }; - pub fn uninitialized( - _: std.Io, - platform: *const Platform, - shape_: Shape, - opts: UnitializedOptions, - ) !Buffer { - std.log.debug("uninitialized {f}", .{shape_}); - var buffer: Buffer = .{ - ._platform = platform, + pub fn uninitialized(io: std.Io, platform: *const Platform, shape_: Shape, opts: UnitializedOptions) !Buffer { + _ = io; // autofix + var res: Buffer = .{ + .platform = platform, ._shape = shape_, - ._shards = .empty, + ._shards = .{}, }; - const metadata: PjrtBufferMetadata = .init(platform, shape_); - errdefer for (buffer._shards.slice()) |shard| { + errdefer for (res._shards.slice()) |shard| { shard.deinit(platform.pjrt_api); }; - stdx.debug.assert(platform.devices[0].memory(opts.memory) != null, "Device doesn't have {} memory", .{opts.memory}); + stdx.debug.assert(platform.sharding().num_replicas == 1, "ZML doesn't support num_replicas > 1 for now, got: {}", .{platform.sharding()}); + //const sharding_ax: ?u3 = std.simd.firstTrue(shape_._sharding_info); + const n_partitions = platform.sharding().num_partitions; - for (platform.physical_mesh.devices_in_canonical_order) |device| { - const memory = platform.devices[device.id].memory(opts.memory).?; - const args: pjrt.Client.CreateUninitializedBufferArgs = .{ - // Change for each device - .dst = .{ .memory = memory.pjrt_memory }, - // Constant across devices - .layout = metadata.layout, - .dims = metadata.placement.local_dims.constSlice(), - .element_type = metadata.ty, - }; + //const shard_shape = if (sharding_ax) |ax| s: { + // // This kind of sharding error should be detected earlier on. + // stdx.debug.assert(@rem(shape_.dim(ax), n_partitions) == 0, "Buffer.uninitialized() expects the sharding axis {} to have a dimension divisble by the number of devices ({}).", .{ ax, n_partitions }); + // const shard_shape = shape_.set(ax, @divExact(shape_.dim(ax), n_partitions)); + // break :s shard_shape; + //} else shape_; - const shard_buffer = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); - buffer._shards.appendAssumeCapacity(shard_buffer); + var args = pjrt.Client.CreateUninitializedBufferArgs{ + .dims = shape_.dims(), + .element_type = pjrtx.bufferTypeFromDtype(shape_.dtype()), + .layout = .{ + .tiled = .{ + .minor_to_major = constants.minorToMajor(shape_.rank()), + .tile_dims = &.{}, + .tile_dims_sizes = &.{}, + }, + }, + // set per device, see below. + .dst = undefined, + }; + + for (platform.devices[0..n_partitions]) |*device| { + args.dst = .{ .memory = device.memory(opts.memory).pjrt_memory }; + const shard = try platform.pjrt_client.createUninitializedBuffer(platform.pjrt_api, args); + res._shards.appendAssumeCapacity(shard); } - return buffer; + return res; } /// Wraps pre-exisiting `pjrt.Buffer` shards into one `zml.Buffer`. - pub fn fromPjrtBuffers(platform: *const Platform, sh: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { + pub fn fromPjrtBuffers(platform: *const Platform, shape_: Shape, pjrt_buffers: []const *pjrt.Buffer) Buffer { stdx.debug.assert(pjrt_buffers.len <= MAX_NUM_SHARDS, "ZML doesn't support having more than {} shards. Received {} shards for one buffer.", .{ MAX_NUM_SHARDS, pjrt_buffers.len }); stdx.debug.assert(pjrt_buffers.len > 0, "fromPjrtBuffers expects at least one buffer, got 0.", .{}); - + var shards: Shards = .{}; + shards.appendSliceAssumeCapacity(pjrt_buffers); return .{ - ._platform = platform, - ._shape = sh, - ._shards = Shards.fromSlice(pjrt_buffers) catch unreachable, + .platform = platform, + ._shape = shape_, + ._shards = shards, }; } + pub fn devicePtr(self: Buffer) *anyopaque { + //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer", .{}); + return self._shards.get(0).opaqueDeviceMemoryDataPointer(self.platform.pjrt_api) catch unreachable; + } + /// Fetches the content of the given buffer into a stack variable of the given type. pub fn getValue(self: Buffer, T: type, io: std.Io) !T { stdx.debug.assert(self._shape.byteSize() == @sizeOf(T), "Buffer {f} has {d} bytes of data, can't load it to a {s} with {d} bytes", .{ self, self._shape.byteSize(), @typeName(T), @sizeOf(T) }); @@ -292,22 +210,10 @@ pub const Buffer = struct { /// Copies the content of the Buffer to the provided slice. pub fn toSlice(self: Buffer, io: std.Io, slice: Slice) !void { - stdx.debug.assert(self._shape.eql(slice.shape), "Buffer shape {f} doesn't match destination slice {f}", .{ self._shape, slice.shape }); - - const placement = placementOrPanic(self._platform, self._shape); - for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { - // TODO: handle replicated information, we shouldn't iterate over all the devices unless needed - const sub_slice = placement.shardSlice(shard_index, slice); - if (!sub_slice.isContiguous()) return error.NonContiguousShardRead; - - const size_bytes = placement.localShape(self._shape).byteSize(); - const destination = sub_slice.data()[0..size_bytes]; - const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, destination); - - if (maybe_event) |event| { - defer event.deinit(self._platform.pjrt_api); - try event.await(self._platform.pjrt_api, io); - } + //stdx.debug.internalAssert(!self.hasShardedAxis(), "TODO: support sharded Buffer -> Host transfer", .{}); + const maybe_event = try self._shards.get(0).toHostBuffer(self.platform.pjrt_api, slice.data()); + if (maybe_event) |event| { + try event.await(self.platform.pjrt_api, io); } } @@ -317,140 +223,8 @@ pub const Buffer = struct { const slice = try Slice.alloc(allocator, self.shape()); errdefer slice.free(allocator); - const placement = placementOrPanic(self._platform, self._shape); - - var shard_slice = try Slice.alloc(allocator, placement.localShape(self._shape)); - defer shard_slice.free(allocator); - - for (0..self._platform.physical_mesh.devices_in_canonical_order.len) |shard_index| { - const sub_slice = placement.shardSlice(shard_index, slice); - const maybe_event = try self._shards.get(shard_index).toHostBuffer(self._platform.pjrt_api, shard_slice.data()); - if (maybe_event) |event| { - defer event.deinit(self._platform.pjrt_api); - try event.await(self._platform.pjrt_api, io); - } - - // TODO: why is this using Slice.copy while `toSlice` errors out ? - // TODO: why is this writing in a copy rather than directly in place. - sub_slice.copy(shard_slice.constData()); - } + try self.toSlice(io, slice); return slice; } - - /// The memory used by this Buffer across all devices - /// ie: `num_devices * shard_byte_size` - /// `shard_byte_size` can be up to `self.shape().byteSize()` when the buffer is fully replicated. - pub fn byteSize(self: Buffer) usize { - const placement = placementOrPanic(self._platform, self._shape); - return placement.localShape(self._shape).byteSize() * self._shards.len; - } - - pub fn opaqueDevicePtr(self: Buffer, device_id: usize) *anyopaque { - return self._shards.get(device_id).opaqueDeviceMemoryDataPointer(self._platform.pjrt_api) catch unreachable; - } - - /// Creates a view of the given buffer - pub fn createView(self: Buffer) !Buffer { - const platform = self._platform; - - var view_shards_buffer: [MAX_NUM_SHARDS]*pjrt.Buffer = undefined; - var view_shard_list: std.ArrayList(*pjrt.Buffer) = .initBuffer(&view_shards_buffer); - errdefer for (view_shard_list.items) |shard| { - shard.deinit(platform.pjrt_api); - }; - - for (self._shards.constSlice()) |shard| { - const view_shard = try platform.pjrt_client.createViewOfDeviceBuffer(platform.pjrt_api, .{ - .device_buffer_ptr = try shard.opaqueDeviceMemoryDataPointer(platform.pjrt_api), - .dims = shard.dimensions(platform.pjrt_api), - .element_type = shard.elementType(platform.pjrt_api), - .layout = try shard.memoryLayout(platform.pjrt_api), - .device = try shard.device(platform.pjrt_api), - }); - view_shard_list.appendAssumeCapacity(view_shard); - } - - return fromPjrtBuffers(platform, self._shape, view_shard_list.items); - } }; - -test "device round-trip" { - const zml = @import("zml.zig"); - const io = std.testing.io; - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - const tp = zml.testing.tpMesh(); - - const x: [8][8]u32 = .{ - .{ 0, 1, 2, 3, 4, 5, 6, 7 }, - .{ 8, 9, 10, 11, 12, 13, 14, 15 }, - .{ 16, 17, 18, 19, 20, 21, 22, 23 }, - .{ 24, 25, 26, 27, 28, 29, 30, 31 }, - .{ 32, 33, 34, 35, 36, 37, 38, 39 }, - .{ 40, 41, 42, 43, 44, 45, 46, 47 }, - .{ 48, 49, 50, 51, 52, 53, 54, 55 }, - .{ 56, 57, 58, 59, 60, 61, 62, 63 }, - }; - - const x_h: zml.Slice = .init(.withPartitioning( - .init(.{ .b = 8, .d = 8 }, .u32), - tp, - .{ .b = .model }, - ), std.mem.asBytes(&x)); - // no free: x_h is stack allocated - const x_d: zml.Buffer = try .fromSlice(io, platform, x_h); - try std.testing.expectEqual(platform.devices.len, x_d.numShards()); - - { - const x_h_reborn: zml.Slice = try x_d.toSliceAlloc(allocator, io); - defer x_h_reborn.free(allocator); - - errdefer std.log.err(" - reference: {d}\n- actual: {d}", .{ x_h, x_h_reborn }); - try zml.testing.expectClose(io, x_h, x_h_reborn, .exact_match); - } - - { - var x_2: @TypeOf(x) = undefined; - const x_h_reborn: zml.Slice = .init(x_h.shape, std.mem.asBytes(&x_2)); - // no free: x_h_reborn is stack allocated - try x_d.toSlice(io, x_h_reborn); - - errdefer std.log.err(" - reference: {d}\n- actual: {d}", .{ x_h, x_h_reborn }); - try zml.testing.expectClose(io, x_h, x_h_reborn, .exact_match); - } -} - -pub const PjrtBufferMetadata = struct { - shape: Shape, - ty: pjrt.BufferType, - placement: Sharding.Placement, - layout: pjrt.MemoryLayout, - - /// Inline because the layout may point to stack memory, as with - /// `Platform.defaultMemoryLayout`. Outside `Buffer` so that - /// `Buffer.HostAccessible` can use it without it being part of the API: - /// zml.zig only exports `Buffer` from this file. - pub inline fn init(platform: *const Platform, shape_: Shape) PjrtBufferMetadata { - - // Shards use the PJRT shape, which packs sub-byte elements. - const packed_shape = shape_.packedShape(); - const placement = placementOrPanic(platform, packed_shape); - - return .{ - .shape = packed_shape, - .ty = pjrtx.bufferTypeFromDtype(packed_shape.dtype()), - .placement = placement, - .layout = platform.defaultMemoryLayout(placement.local_dims.constSlice(), packed_shape.dtype()), - }; - } -}; - -fn placementOrPanic(platform: *const Platform, shape: Shape) Sharding.Placement { - var sh = shape; - if (sh._sharding.mesh == null) sh._sharding.mesh = platform.replicated_mesh; - return Sharding.Placement.init(&sh) catch |err| { - @branchHint(.cold); - std.debug.panic("Buffer shape {f} is incompatible with its mesh: {t}", .{ shape, err }); - }; -} diff --git a/zml/buffer/HostAccessible.zig b/zml/buffer/HostAccessible.zig deleted file mode 100644 index b3e82a9c..00000000 --- a/zml/buffer/HostAccessible.zig +++ /dev/null @@ -1,2202 +0,0 @@ -//! A `Buffer` in `.host_pinned` memory that the host reads and writes in place -//! through views, which a plain `Buffer` placed there doesn't offer. Views -//! address elements as PJRT stores them, with sub-byte elements packed in bytes. -//! The host may only access the buffer between executions. Host pointers are -//! read at creation, and by `await` when an execution replaced the handles. -//! Executions must preserve shape, sharding and layout. -//! -//! Like `Buffer`, it is a value. Pass `buffer` to an execution to read it as -//! an input, or `&buffer` to donate it as an output, then `await` the -//! HostAccessible whose `buffer` the execution replaced. Copies share the -//! mapping, and a copy whose handles the execution did not replace panics on -//! `view` rather than write to an allocation the execution may have released. -const std = @import("std"); -const builtin = @import("builtin"); - -const pjrt = @import("pjrt"); -const stdx = @import("stdx"); - -const Buffer = @import("../buffer.zig").Buffer; -const PjrtBufferMetadata = @import("../buffer.zig").PjrtBufferMetadata; -const Platform = @import("../platform.zig").Platform; -const Shape = @import("../shape.zig").Shape; -const Sharding = @import("../Sharding.zig"); -const testing = @import("../testing.zig"); - -const HostAccessible = @This(); - -/// The device side, with one PJRT buffer per device, for executions to read -/// or donate. -buffer: Buffer, -/// What views read, heap allocated so views can borrow it while the -/// HostAccessible moves. -mapping: *Mapping, - -/// The host side of the buffer. Views and blocks copy nothing from it: the -/// layout and shard placement are prepared once at creation and take several -/// KB, and copying them into every view, slice and block made memcpy most of -/// llmd's decode thread. -const Mapping = struct { - /// The PJRT shape, which packs sub-byte elements in bytes, unlike - /// `buffer._shape`. Views address its elements. - packed_shape: Shape, - shard_shape: Shape, - layout_metadata: LayoutMetadata, - allocations: HostPinnedAllocations, - /// Every shard holds the whole array densely, so element `i` sits at - /// `i * element_size` in each of them, as in replicated buffers on every - /// platform but TPU. `get`, `set` and blocks then skip locating elements - /// through the origins and the layout. - contiguous: bool, - - /// Takes ownership of the allocations, on success only. - fn create( - allocator: std.mem.Allocator, - shape_: Shape, - shard_shape: Shape, - layout: pjrt.MemoryLayout, - allocations: HostPinnedAllocations, - ) !*Mapping { - const mapping = try allocator.create(Mapping); - errdefer allocator.destroy(mapping); - - const layout_metadata: LayoutMetadata = .init(shape_, shard_shape, layout); - - mapping.* = .{ - .packed_shape = shape_, - .shard_shape = shard_shape, - .layout_metadata = layout_metadata, - .allocations = allocations, - .contiguous = layout_metadata.addressing == .row_major and - allocations.regions.len == 1 and - shard_shape.count() == shape_.count(), - }; - - return mapping; - } -}; - -/// The host allocations behind the buffer's shards. Devices that share an -/// allocation map to one entry, and entries holding the same region of the -/// array are its replicas. Entries stay in region order, so a region's -/// replicas are a range of every field. -const HostPinnedAllocations = struct { - entries: std.MultiArrayList(Allocation) = .empty, - /// The regions the entries hold, in order of first appearance. - regions: stdx.BoundedArray(Region, Platform.MAX_NUM_DEVICES) = .empty, - - const empty: HostPinnedAllocations = .{}; - - const Allocation = struct { - /// Global coordinates of the allocation's first element. - origin: Coordinates, - /// Read from `handle`. Blocks borrow their region's range. - ptr: [*]u8, - /// The shard owning the allocation, which must destroy it: an index - /// into `buffer._shards`, whose handles execution may replace. - shard_index: usize, - /// The handle `ptr` was read from. Execution replaces the handles of - /// donated buffers with its outputs, which may not reuse the allocation. - handle: *pjrt.Buffer, - /// Whether `handle` holds an external reference, which keeps the - /// allocation alive for the device views of other devices. - shared: bool = false, - }; - - /// `entries[start..][0..len]` hold the region's replicas, in the order they - /// were added. Reads use the first. - const Region = struct { start: u8, len: u8 }; - - fn deinit(self: *HostPinnedAllocations, allocator: std.mem.Allocator) void { - self.entries.deinit(allocator); - } - - /// Add an allocation to the replicas of its region, within the - /// reserved capacity. - fn add(self: *HostPinnedAllocations, shard_shape: Shape, allocation: Allocation) void { - const region_index = self.findRegion(allocation.origin, shard_shape) orelse { - self.regions.appendAssumeCapacity(.{ .start = @intCast(self.entries.len), .len = 1 }); - self.entries.appendAssumeCapacity(allocation); - return; - }; - const region = &self.regions.slice()[region_index]; - // Keep each region's entries together by shifting the regions after it. - self.entries.insertAssumeCapacity(region.start + region.len, allocation); - region.len += 1; - for (self.regions.slice()[region_index + 1 ..]) |*next| next.start += 1; - } - - /// The index of the region holding the shards that start at `origin`. - fn findRegion(self: *const HostPinnedAllocations, origin: Coordinates, shard_shape: Shape) ?usize { - const origins = self.entries.items(.origin); - for (self.regions.constSlice(), 0..) |region, i| { - if (origin.eqlWithin(origins[region.start], shard_shape)) return i; - } - return null; - } - - /// The host pointers of a region's replicas. - fn replicas(self: *const HostPinnedAllocations, region: Region) []const [*]u8 { - return self.entries.items(.ptr)[region.start..][0..region.len]; - } - - /// Read the host pointers of the allocations from their current handles. - /// Each read takes and releases a hold on the PJRT buffer and allocates an - /// external reference, about 0.9 us: llmd would make thousands per step if - /// views read them, so they are read at creation and by `await` only. - fn refreshPointers(self: *HostPinnedAllocations, api: *const pjrt.Api, handles: []const *pjrt.Buffer) !void { - const shard_indices = self.entries.items(.shard_index); - // Read every pointer before replacing any, so a failure keeps the previous ones. - var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; - for (shard_indices, ptrs[0..shard_indices.len]) |shard_index, *ptr| { - ptr.* = @ptrCast(try handles[shard_index].opaqueDeviceMemoryDataPointer(api)); - } - @memcpy(self.entries.items(.ptr), ptrs[0..shard_indices.len]); - for (self.entries.items(.handle), shard_indices) |*handle, shard_index| handle.* = handles[shard_index]; - } - - /// Whether the host pointers were read from the current handles, which - /// execution replaces for donated buffers. - fn pointersAreCurrent(self: *const HostPinnedAllocations, handles: []const *pjrt.Buffer) bool { - // Host memory in tests has no handles: its pointers never change. - if (handles.len == 0) return true; - for (self.entries.items(.handle), self.entries.items(.shard_index)) |handle, shard_index| { - if (handle != handles[shard_index]) return false; - } - return true; - } - - /// Destroy every handle, the device views before the allocations they borrow. - fn deinitHandles(self: *const HostPinnedAllocations, api: *const pjrt.Api, handles: []const *pjrt.Buffer) void { - const shard_indices = self.entries.items(.shard_index); - for (handles, 0..) |handle, index| { - if (std.mem.indexOfScalar(usize, shard_indices, index) == null) handle.deinit(api); - } - for (shard_indices, self.entries.items(.shared)) |shard_index, shared| { - const handle = handles[shard_index]; - // `shared` is set by the one successful increase, and only undonatable - // buffers share allocations, whose handles execution never replaces. - if (shared) handle.decreaseExternalReferenceCount(api) catch |err| { - std.debug.panic("shared HostAccessible allocation has no reference to release, was it donated? {}", .{err}); - }; - handle.deinit(api); - } - } -}; - -/// Host memory standing in for PJRT buffers, to test layouts and shardings -/// that no available platform produces: every platform but TPU uses dense -/// row-major layouts, and CI has few devices. `origins` holds the row-major -/// index in `shape_` of each allocation's first element, and `ptrs` one pointer -/// per origin. Copies the layout, the origins and the pointers, and borrows the -/// storage. -pub fn initForTests( - allocator: std.mem.Allocator, - shape_: Shape, - shard_shape: Shape, - layout: pjrt.MemoryLayout, - origins: []const usize, - ptrs: []const [*]u8, -) !HostAccessible { - if (!builtin.is_test) @compileError("HostAccessible.initForTests is only available in tests"); - - var allocations: HostPinnedAllocations = .empty; - errdefer allocations.deinit(allocator); - - try allocations.entries.ensureTotalCapacity(allocator, origins.len); - // Without PJRT buffers, there are no handles to read pointers from or destroy. - for (origins, ptrs) |origin, ptr| { - allocations.add( - shard_shape, - .{ - // An empty array has no element to locate the origin of. - .origin = if (shape_.count() == 0) .zero else .unflatten(shape_, origin), - .ptr = ptr, - .shard_index = undefined, - .handle = undefined, - }, - ); - } - - return .{ - .buffer = .{ ._platform = undefined, ._shape = shape_, ._shards = .empty }, - .mapping = try .create(allocator, shape_, shard_shape, layout, allocations), - }; -} - -/// With `.undonatable`, devices holding the same shard share one allocation on -/// CPU, CUDA and ROCm: the first allocates it, and the others read it through -/// device views. Elsewhere each device owns an allocation, and writes reach -/// every one of them. Execution never replaces the handles of an undonatable -/// buffer, so `await` never reads its host pointers again. -/// With `.donatable`, each device owns an allocation: writing a shared one -/// would expose the output of a device to the reads of another. -pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - shape_: Shape, - donation: Buffer.Donation, -) !HostAccessible { - const can_share_allocations = switch (platform.target) { - .cpu, .cuda, .rocm => donation == .undonatable, - else => false, - }; - - var buffer: Buffer = .{ - ._platform = platform, - ._shape = shape_, - ._shards = .empty, - }; - const metadata: PjrtBufferMetadata = .init(platform, shape_); - const devices = platform.physical_mesh.devices_in_canonical_order; - const api = platform.pjrt_api; - - var allocations: HostPinnedAllocations = .empty; - errdefer allocations.deinit(allocator); - - try allocations.entries.ensureTotalCapacity(allocator, devices.len); - errdefer allocations.deinitHandles(api, buffer._shards.constSlice()); - - for (devices, 0..) |device, shard_index| { - const memory = platform.devices[device.id].memory(.host_pinned).?.pjrt_memory; - const origin: Coordinates = .ofShard(&metadata.placement, shard_index, metadata.shape); - const shared_region = if (can_share_allocations) allocations.findRegion(origin, metadata.placement.localShape(metadata.shape)) else null; - - if (shared_region) |region_index| { - // Devices sharing allocations add no replicas: the region has a single entry. - const entry = allocations.regions.get(region_index).start; - const existing = allocations.entries.get(entry); - if (!existing.shared) { - try existing.handle.increaseExternalReferenceCount(api); - allocations.entries.items(.shared)[entry] = true; - } - buffer._shards.appendAssumeCapacity(try platform.pjrt_client.createViewOfDeviceBuffer(api, .{ - .device_buffer_ptr = try existing.handle.opaqueDeviceMemoryDataPointer(api), - .dims = metadata.placement.local_dims.constSlice(), - .element_type = metadata.ty, - .layout = try existing.handle.memoryLayout(api), - // The backing host allocation is accessible across devices - // (portable pinned memory on CUDA/ROCm). This memory space - // associates the view with the current device rather than - // the device that originally allocated the memory. - .memory = memory, - })); - } else { - const handle = try platform.pjrt_client.createUninitializedBuffer(api, .{ - .dims = metadata.placement.local_dims.constSlice(), - .element_type = metadata.ty, - .layout = metadata.layout, - .dst = .{ .memory = memory }, - }); - buffer._shards.appendAssumeCapacity(handle); - // The pointer is read once the creation completes, after `await` below. - allocations.add(metadata.placement.localShape(metadata.shape), .{ - .origin = origin, - .ptr = undefined, - .shard_index = shard_index, - .handle = handle, - }); - } - } - - try buffer.await(io); - try allocations.refreshPointers(api, buffer._shards.constSlice()); - - return .{ - .buffer = buffer, - .mapping = try .create(allocator, metadata.shape, metadata.placement.localShape(metadata.shape), metadata.layout, allocations), - }; -} - -/// Execution must be done with the buffer. Copies must not be used after. -pub fn deinit(self: *const HostAccessible, allocator: std.mem.Allocator) void { - // Host memory in tests has neither handles nor a platform. - if (self.buffer._shards.len != 0) self.mapping.allocations.deinitHandles(self.buffer._platform.pjrt_api, self.buffer._shards.constSlice()); - self.mapping.allocations.deinit(allocator); - allocator.destroy(self.mapping); -} - -/// Wait for execution to finish, and read the host pointers of handles that -/// execution replaced. Take new views to access the output. -pub fn await(self: *HostAccessible, io: std.Io) !void { - try self.buffer.await(io); - const handles = self.buffer._shards.constSlice(); - const allocations = &self.mapping.allocations; - if (!allocations.pointersAreCurrent(handles)) try allocations.refreshPointers(self.buffer._platform.pjrt_api, handles); -} - -/// Borrows the mapping. Executions using the buffer must have completed, and -/// `await` must follow any that replaced the handles: `view` panics before -/// then rather than write to an allocation the execution may have released. -/// The view and its blocks are valid until the next execution starts or the -/// HostAccessible is deinitialized. -pub fn view(self: *const HostAccessible, comptime T: type) View(T) { - const mapping = self.mapping; - std.debug.assert(@sizeOf(T) == mapping.shard_shape.dtype().sizeOf()); - stdx.debug.assert(mapping.allocations.pointersAreCurrent(self.buffer._shards.constSlice()), "await the HostAccessible after execution replaces its handles", .{}); - return .{ .mapping = mapping, .len = mapping.packed_shape.count() }; -} - -/// Prints every element, read through a view: the same rules as `view` apply. -pub fn format(self: *const HostAccessible, writer: *std.Io.Writer) std.Io.Writer.Error!void { - try writer.print("HostAccessible({f}) {{", .{self.mapping.packed_shape}); - switch (self.mapping.shard_shape.dtype()) { - inline else => |dt| { - if (comptime dt.bitSizeOf() < 8) unreachable; - const values = self.view(dt.toZigType()); - var blocks = values.blocks(); - while (blocks.next()) |block| { - for (block.firstReplica()) |value| try writer.print(" {any}", .{value}); - } - }, - } - try writer.writeAll(" }"); -} - -/// A range of the array's elements in row-major order, borrowed from a -/// HostAccessible. Reads use the first replica, and writes reach every replica. -pub fn View(comptime T: type) type { - return struct { - mapping: *const Mapping, - start: usize = 0, - len: usize, - - const Self = @This(); - - /// A view without elements, which never accesses the buffer. - pub const empty: Self = .{ - .mapping = undefined, - .len = 0, - }; - - /// Select `len_` elements starting at `start`, or the rest of the view for null. - pub fn slice(self: Self, start: usize, len_: ?usize) Self { - std.debug.assert(start <= self.len); - const len = len_ orelse self.len - start; - std.debug.assert(len <= self.len - start); - var result = self; - result.start += start; - result.len = len; - return result; - } - - // Single elements are addressed directly, without slicing and iterating. - pub fn get(self: Self, index: usize) T { - const element = self.locateElement(index); - return itemAt(self.mapping.allocations.replicas(element.region)[0], element.byte_offset).*; - } - - pub fn set(self: Self, index: usize, value: T) void { - const element = self.locateElement(index); - for (self.mapping.allocations.replicas(element.region)) |ptr| itemAt(ptr, element.byte_offset).* = value; - } - - fn locateElement(self: *const Self, index: usize) struct { region: HostPinnedAllocations.Region, byte_offset: usize } { - std.debug.assert(index < self.len); - if (self.mapping.contiguous) { - return .{ .region = self.mapping.allocations.regions.get(0), .byte_offset = (self.start + index) * @sizeOf(T) }; - } - const location = self.locate(.unflatten(self.mapping.packed_shape, self.start + index)); - return .{ .region = location.region, .byte_offset = self.mapping.layout_metadata.byteOffset(location.offset, location.local) }; - } - - pub fn fill(self: Self, value: T) void { - var iterator = self.blocks(); - while (iterator.next()) |block| { - for (0..block.replicas.len) |i| @memset(block.replica(i), value); - } - } - - /// Fill with `first + i * step`, where `i` is relative to this view. - /// `T` must be an integer type holding every `i` and value; safe builds check it. - pub fn fillIota(self: Self, first: T, step: T) void { - var iterator = self.blocks(); - while (iterator.next()) |block| { - const items = block.firstReplica(); - // TODO: SIMD - for (items, block.offset..) |*value, i| value.* = first + @as(T, @intCast(i)) * step; - for (1..block.replicas.len) |i| @memcpy(block.replica(i), items); - } - } - - pub fn copyFrom(self: Self, values: []const T) void { - std.debug.assert(values.len == self.len); - var iterator = self.blocks(); - while (iterator.next()) |block| { - for (0..block.replicas.len) |i| @memcpy(block.replica(i), values[block.offset..][0..block.len]); - } - } - - pub fn copyTo(self: Self, values: []T) void { - std.debug.assert(values.len == self.len); - var iterator = self.blocks(); - while (iterator.next()) |block| { - @memcpy(values[block.offset..][0..block.len], block.firstReplica()); - } - } - - /// The index of the first element equal to `value`, searching each - /// block with `std.mem.indexOfScalar`. - pub fn indexOfScalar(self: Self, value: T) ?usize { - var iterator = self.blocks(); - while (iterator.next()) |block| { - if (std.mem.indexOfScalar(T, block.firstReplica(), value)) |index| return block.offset + index; - } - return null; - } - - /// The index of the first element equal to any of `values`, searching - /// each block with `std.mem.indexOfAny`. - pub fn indexOfAny(self: Self, values: []const T) ?usize { - var iterator = self.blocks(); - while (iterator.next()) |block| { - if (std.mem.indexOfAny(T, block.firstReplica(), values)) |index| return block.offset + index; - } - return null; - } - - /// Source and destination must not overlap. - pub fn copyFromView(self: Self, src: Self) void { - std.debug.assert(src.len == self.len); - var iterator = self.blocks(); - while (iterator.next()) |block| { - const items = block.firstReplica(); - src.slice(block.offset, block.len).copyTo(items); - for (1..block.replicas.len) |i| @memcpy(block.replica(i), items); - } - } - - /// The elements as one read-only slice, for reads that index or zip - /// several views. Contiguous buffers, such as replicated ones on every - /// platform but TPU, return their first replica in place; others - /// copy into `scratch`, which must hold `len` elements on every - /// platform. The slice is valid until the next execution, and - /// writes must go through the view. - pub fn constSlice(self: Self, scratch: []T) []const T { - std.debug.assert(scratch.len >= self.len); - if (self.inPlace()) |block| return block.firstReplica(); - self.copyTo(scratch[0..self.len]); - return scratch[0..self.len]; - } - - /// A slice to write the elements into, then `commit`, for writes - /// that compute elements in place. Contiguous buffers hand out their - /// first replica; others hand out `scratch`, which must hold `len` - /// elements on every platform. - pub fn edit(self: Self, scratch: []T) Edit { - std.debug.assert(scratch.len >= self.len); - return .{ - .view = self, - .items = if (self.inPlace()) |block| block.firstReplica() else scratch[0..self.len], - }; - } - - pub const Edit = struct { - view: Self, - /// Holds the previous values or garbage: write every element. - items: []T, - - /// Make the writes reach every replica: copy the first replica to - /// the others, or `items` into the view. - pub fn commit(self: Edit) void { - const block = self.view.inPlace() orelse return self.view.copyFrom(self.items); - for (1..block.replicas.len) |i| @memcpy(block.replica(i), self.items); - } - }; - - /// The view as a single block, when the buffer is contiguous. - fn inPlace(self: Self) ?Block { - if (self.len == 0 or !self.mapping.contiguous) return null; - var iterator = self.blocks(); - return iterator.next().?; - } - - /// Visit runs of consecutive logical elements in global row-major - /// order, without padding, grouping replicas of each logical range. - /// Shards must cover the view with disjoint regions or exact replicas. - pub fn blocks(self: Self) BlockIterator { - return .init(self); - } - - /// Visit the same blocks in reverse order. Elements within each - /// replica slice remain in ascending logical order. - pub fn reverseBlocks(self: Self) ReverseBlockIterator { - return .init(self); - } - - /// A run of elements that are consecutive in the view's row-major order and - /// contiguous in memory, so each replica holds them as a plain slice. - /// Writes must reach every replica; reads use the first. Borrows the buffer's - /// host pointers, like the view it comes from. - pub const Block = struct { - /// Logical position of the first element, relative to this view. - offset: usize, - len: usize, - /// Byte offset of the first element in the allocation of every replica. - byte_offset: usize, - /// Host pointer of each replica's allocation. - replicas: []const [*]u8, - - /// The block's elements in its `i`-th replica. - pub fn replica(self: Block, i: usize) []T { - const ptr: [*]T = @ptrCast(@alignCast(self.replicas[i] + self.byte_offset)); - return ptr[0..self.len]; - } - - /// The elements in the first replica, which reads use. - pub fn firstReplica(self: Block) []T { - return self.replica(0); - } - }; - - pub const BlockIterator = OrderedBlockIterator(false); - pub const ReverseBlockIterator = OrderedBlockIterator(true); - - fn OrderedBlockIterator(comptime reverse: bool) type { - return struct { - view: Self, - /// Next logical offset, or one past it for reverse traversal. - offset: usize, - /// Coordinates of the next block's first (or reverse last) element. - global: Coordinates, - - const Iterator = @This(); - - fn init(v: Self) Iterator { - return .{ - .view = v, - .offset = if (reverse) v.len else 0, - .global = if (v.len == 0 or v.mapping.contiguous) .zero else .unflatten( - v.mapping.packed_shape, - v.start + (if (reverse) v.len - 1 else 0), - ), - }; - } - - pub fn next(self: *Iterator) ?Block { - const v = &self.view; - if (if (reverse) self.offset == 0 else self.offset == v.len) return null; - if (v.mapping.contiguous) { - // The whole view is one run in every replica. - self.offset = if (reverse) 0 else v.len; - return v.blockAt(v.mapping.allocations.regions.get(0), 0, v.len, v.start * @sizeOf(T)); - } - const location = v.locate(self.global); - const bounds = v.blockBounds(location.offset); - const len = if (reverse) - @min(location.offset + 1 - bounds.start, self.offset) - else - @min(bounds.end - location.offset, v.len - self.offset); - // A block is physically contiguous, so its last element's - // address also gives its start without decoding coordinates again. - const byte_offset = v.mapping.layout_metadata.byteOffset(location.offset, location.local) - - (if (reverse) (len - 1) * v.mapping.layout_metadata.element_size else 0); - const offset = if (reverse) self.offset - len else self.offset; - self.offset = if (reverse) offset else offset + len; - if (if (reverse) self.offset != 0 else self.offset != v.len) { - self.global.move(reverse, v.mapping.packed_shape, len); - } - return v.blockAt(location.region, offset, len, byte_offset); - } - }; - } - - const Bounds = struct { start: usize, end: usize }; - - /// The block containing a shard-local element: elements contiguous in both - /// physical memory and global row-major order. Bounds are shard-local - /// row-major indices, not physical memory offsets. - /// - /// Example: an unsharded [2,5] array with row-major 2x2 tiles. - /// - /// 1. Logical indices, with tile boundaries. `_` denotes padding: - /// ```text - /// tile 0 tile 1 tile 2 - /// +-----+ +-----+ +-----+ - /// | 0 1 | | 2 3 | | 4 _ | - /// | 5 6 | | 7 8 | | 9 _ | - /// +-----+ +-----+ +-----+ - /// ``` - /// 2. Tiles are stored consecutively, each in row-major order: - /// ```text - /// [0 1 5 6] [2 3 7 8] [4 _ 9 _] - /// ``` - /// Thus 0 and 1 are adjacent in memory, but 1 and 2 are not. - /// `block_size = 2` is the maximum contiguous logical run. - /// - /// 3. Align runs within each logical row, clipping before padding: - /// ```text - /// last_dim_size = 5 - /// alignment_size = max(5, 2) = 5 - /// - /// alignment_start blocks of logical elements row end - /// 0 [0 1] [2 3] [4] 5 - /// 5 [5 6] [7 8] [9] 10 - /// actual block sizes: 2 2 1 - /// - /// local alignment_start start end returned bounds - /// 4 0 4 5 [4,5) - /// 5 5 5 7 [5,7) - /// 6 5 5 7 [5,7) - /// 9 5 9 10 [9,10) - /// ``` - /// For local=6, start = 6 - (6-5)%2 = 5, end = min(5+2, 5+5) = 7. - /// For local=4, end = min(4+2, 0+5) = 5: the final run is shortened. - /// - /// Without tiles, the same dense row-major array has block_size=10 - /// and alignment_size=10: one block [0,10) spanning both rows. - fn blockBounds(self: *const Self, local: usize) Bounds { - const block_size = self.mapping.layout_metadata.block_size; - // Runs shorter than a row align within that row. Larger runs - // span a whole number of rows, so alignment uses the run itself. - const alignment_size = self.mapping.layout_metadata.alignment_size; - const alignment_start = local - local % alignment_size; - // Round the position relative to alignment_start down to a run boundary. - const start = local - (local - alignment_start) % block_size; - // A row's final run may be shorter; stop at the alignment interval's end. - return .{ .start = start, .end = @min(start + block_size, alignment_start + alignment_size) }; - } - - fn blockAt(self: *const Self, region: HostPinnedAllocations.Region, offset: usize, len: usize, byte_offset: usize) Block { - return .{ - .offset = offset, - .len = len, - .byte_offset = byte_offset, - .replicas = self.mapping.allocations.replicas(region), - }; - } - - fn itemAt(ptr: [*]u8, byte_offset: usize) *T { - return @ptrCast(@alignCast(ptr + byte_offset)); - } - - const Location = struct { - region: HostPinnedAllocations.Region, - /// Coordinates of the element within its shard. - local: Coordinates, - /// Row-major index of the element within its shard. - offset: usize, - }; - - /// Locate the region holding an element, retaining its shard-local - /// coordinates for physical addressing. - fn locate(self: *const Self, global: Coordinates) Location { - const origins = self.mapping.allocations.entries.items(.origin); - for (self.mapping.allocations.regions.constSlice()) |region| { - const origin = origins[region.start]; - var local: Coordinates = .zero; - var offset: usize = 0; - for (self.mapping.shard_shape.dims(), 0..) |dim_, axis| { - const dim: usize = @intCast(dim_); - if (global.axes[axis] < origin.axes[axis] or global.axes[axis] - origin.axes[axis] >= dim) break; - local.axes[axis] = global.axes[axis] - origin.axes[axis]; - offset = offset * dim + local.axes[axis]; - } else { - return .{ .region = region, .local = local, .offset = offset }; - } - } - @panic("The union of all regions should cover the global shape."); - } - }; -} - -/// A position in an array, one coordinate per axis. Axes past the rank stay zero. -const Coordinates = struct { - axes: [Shape.MAX_RANK]usize, - - const zero: Coordinates = .{ .axes = @splat(0) }; - - /// The global coordinates of the first element of a device's shard. - fn ofShard(placement: *const Sharding.Placement, device_index: usize, shape: Shape) Coordinates { - // TODO: derive shard coordinates directly from DeviceGrid instead of reconstructing slices. - var result: Coordinates = .zero; - for (placement.slices(device_index, shape).constSlice(), 0..) |slice, axis| result.axes[axis] = @intCast(slice.start); - return result; - } - - /// The coordinates of the element at `offset` of `shape`, in row-major order. - fn unflatten(shape_: Shape, offset: usize) Coordinates { - var result: Coordinates = .zero; - var remaining = offset; - var axis = shape_.rank(); - while (axis > 0) { - axis -= 1; - const dim: usize = @intCast(shape_.dim(axis)); - result.axes[axis] = remaining % dim; - remaining /= dim; - } - return result; - } - - /// Whether the coordinates match on the axes of `shape`. - fn eqlWithin(self: Coordinates, other: Coordinates, shape_: Shape) bool { - return std.mem.eql(usize, self.axes[0..shape_.rank()], other.axes[0..shape_.rank()]); - } - - /// Advance in row-major order, decoding only axes that carry or borrow. - /// The resulting coordinates must still be within the shape. - fn move(self: *Coordinates, comptime reverse: bool, shape_: Shape, count: usize) void { - const axes = &self.axes; - var carry = count; - var axis = shape_.rank(); - while (carry != 0 and axis > 0) { - axis -= 1; - const dim: usize = @intCast(shape_.dim(axis)); - if (reverse) { - if (carry <= axes[axis]) { - axes[axis] -= carry; - return; - } - carry -= axes[axis] + 1; - axes[axis] = dim - 1 - carry % dim; - } else { - const available = dim - axes[axis]; - if (carry < available) { - axes[axis] += carry; - return; - } - carry -= available; - axes[axis] = carry % dim; - } - carry = carry / dim + 1; - } - std.debug.assert(carry == 0); - } -}; - -/// Physical addressing and logical block boundaries prepared once per HostAccessible. -/// All metadata is stored by value; the source layout is needed only during init. -const LayoutMetadata = struct { - rank: u4, - element_size: usize, - block_size: usize, - alignment_size: usize, - addressing: Addressing, - - const Addressing = union(enum) { - row_major, - strided: ByteStrides, - tiled: Tiled, - - const ByteStrides = [Shape.MAX_RANK]usize; - - const Tiled = struct { - /// Unit and padding dimensions always hold index 0 and are left out. - digits: stdx.BoundedArray(Digit, Shape.MAX_RANK + pjrt.DefaultMemoryLayout.MAX_TILE_DIMS) = .empty, - - /// One physical dimension of a tiled layout, as a digit of one coordinate. - const Digit = struct { - byte_stride: usize, - /// Divides the coordinate, unless `div_shift` replaces it for a power of two. - divisor: usize, - /// A mask when `is_mask` is set; otherwise a general modulus. - modulus: usize, - axis: u3, - div_shift: ?std.math.Log2Int(usize), - is_mask: bool, - - fn init(axis: u3, dim: TiledDims.Dim, byte_stride: usize) Digit { - // Unbounded digits keep every bit of the quotient. - const modulus = if (dim.modulus == 0) std.math.maxInt(usize) else dim.modulus; - const is_mask = dim.modulus == 0 or std.math.isPowerOfTwo(modulus); - return .{ - .byte_stride = byte_stride, - .divisor = dim.divisor, - .modulus = if (is_mask and dim.modulus != 0) modulus - 1 else modulus, - .axis = axis, - .div_shift = if (std.math.isPowerOfTwo(dim.divisor)) @intCast(@ctz(dim.divisor)) else null, - .is_mask = is_mask, - }; - } - - fn value(self: Digit, coordinate: usize) usize { - const quotient = if (self.div_shift) |shift| coordinate >> shift else coordinate / self.divisor; - return if (self.is_mask) quotient & self.modulus else quotient % self.modulus; - } - }; - }; - }; - - fn init(shape: Shape, shard_shape: Shape, layout: pjrt.MemoryLayout) LayoutMetadata { - const last_dim_size: usize = if (shard_shape.rank() == 0) 1 else @intCast(shard_shape.dim(-1)); - const element_size: usize = shard_shape.dtype().sizeOf(); - - // Longest stretch where stepping to the next local index also steps to the next element in memory. - // Suppose a [2,5] shard tiled by (2,2): tiles are stored one after another, each row-major. - // With local indices the memory holds: - // `0 1 5 6 | 2 3 7 8 | 4 _ 9 _` - // 0 and 1 are adjacent but 1 and 2 aren't, so the run is 2. A row-major layout is a single run. - const addressing: Addressing, const physical_run: usize = addr: switch (layout) { - .strides => |strides| { - std.debug.assert(strides.byte_strides.len == shard_shape.rank()); - - const row_major = shard_shape.computeByteStrides(); - if (std.mem.eql(i64, strides.byte_strides, row_major.constSlice())) break :addr .{ .row_major, shard_shape.count() }; - - var byte_strides: Addressing.ByteStrides = @splat(0); - for (strides.byte_strides, 0..) |stride, axis| { - stdx.debug.assert(stride >= 0, "negative destination byte strides are unsupported, got {}", .{stride}); - byte_strides[axis] = @intCast(stride); - } - - const physical_run = - // Rows padded to an alignment keep their elements adjacent: a run is a row. - if (byte_strides[shard_shape.rank() - 1] == element_size) last_dim_size - // Permuted axes, e.g. a column-major [3,4] f32 with strides [4,12], put the - // innermost axis elsewhere in memory, so no two elements of a row are adjacent. - else 1; - break :addr .{ .{ .strided = byte_strides }, physical_run }; - }, - .tiled => |tiled| { - stdx.debug.assert(tiled.minor_to_major.len == shard_shape.rank(), "layout rank {} doesn't match shape rank {}", .{ tiled.minor_to_major.len, shard_shape.rank() }); - - if (tiled.tile_dims.len == 0 and isMinorToMajorRowMajor(tiled.minor_to_major, shard_shape.rank())) break :addr .{ .row_major, shard_shape.count() }; - - // The expanded dimensions are laid out row-major. - const expanded: TiledDims = .init(shard_shape, tiled); - var tiled_addressing: Addressing.Tiled = .{}; - var byte_stride = element_size; - // From the most minor physical dimension, the run grows while each - // dimension is the next digit of the innermost axis. - const last_axis = shard_shape.rank() -| 1; - var physical_run: usize = 1; - var run_grows = true; - var i = expanded.dims.len; - while (i > 0) { - i -= 1; - const dim = expanded.dims.get(i); - if (dim.size > 1) { - if (dim.axis) |axis| tiled_addressing.digits.appendAssumeCapacity(.init(axis, dim, byte_stride)); - if (run_grows) { - const next_digit = if (dim.axis) |axis| axis == last_axis and dim.divisor == physical_run else false; - if (!next_digit) { - run_grows = false; - } else if (dim.modulus == 0) { - // An unbounded digit holds the rest of the axis. - physical_run = last_dim_size; - run_grows = false; - } else { - physical_run *= dim.modulus; - // Padding after the digit's last value ends the run. - run_grows = dim.size == dim.modulus; - } - } - } - byte_stride *= dim.size; - } - physical_run = @min(physical_run, last_dim_size); - - break :addr .{ .{ .tiled = tiled_addressing }, physical_run }; - }, - }; - - // Longest stretch where stepping to the next local index also steps to the next global index. - // Suppose a global [2,6] split into two [2,3] shards, each holding half the columns. - // With global indices the shards hold: - // - shard 1: `0 1 2 | 6 7 8` - // - shard 2: `3 4 5 | 9 10 11` - // 2 and 6 are adjacent in shard 1 but not globally, so the run is 3. - const shard_run = run: { - var run: usize = 1; - var axis = shard_shape.rank(); - while (axis > 0) { - axis -= 1; - run *= @intCast(shard_shape.dim(axis)); - if (shape.dim(axis) != shard_shape.dim(axis)) break; - } - break :run run; - }; - const block_size = if (shard_shape.count() == 0) 0 else @min(physical_run, shard_run); - - return .{ - .rank = shard_shape.rank(), - .element_size = element_size, - .block_size = block_size, - // See `View.blockBounds`. - .alignment_size = @max(last_dim_size, block_size), - .addressing = addressing, - }; - } - - /// The byte offset in its shard of the element at shard-local row-major index - /// `local` and coordinates `coordinates`: row-major layouts use the index, - /// the others the coordinates. - fn byteOffset(self: *const LayoutMetadata, local: usize, coordinates: Coordinates) usize { - switch (self.addressing) { - .row_major => return local * self.element_size, - .strided => |*byte_strides| { - var offset: usize = 0; - for (coordinates.axes[0..self.rank], byte_strides[0..self.rank]) |coord, byte_stride| offset += coord * byte_stride; - return offset; - }, - .tiled => |*tiled| { - var offset: usize = 0; - for (tiled.digits.constSlice()) |digit| offset += digit.value(coordinates.axes[digit.axis]) * digit.byte_stride; - return offset; - }, - } - } -}; - -/// Whether a PJRT minor-to-major order lays the axes out row-major, like ZML shapes. -fn isMinorToMajorRowMajor(minor_to_major: []const i64, rank: usize) bool { - if (minor_to_major.len != rank) return false; - for (minor_to_major, 0..) |axis, i| { - if (axis != @as(i64, @intCast(rank - i - 1))) return false; - } - return true; -} - -/// The physical dimensions of a tiled layout, major to minor. XLA lays a tiled array -/// out row-major over these dimensions; see https://openxla.org/xla/tiled_layout. -const TiledDims = struct { - dims: stdx.BoundedArray(Dim, Shape.MAX_RANK + 2 * pjrt.DefaultMemoryLayout.MAX_TILE_DIMS) = .empty, - - /// Every dimension holds the digit `(coordinate[axis] / divisor) % modulus`. - const Dim = struct { - /// Null for padding, which only holds index 0. - axis: ?u3, - size: usize, - divisor: usize = 1, - /// Zero for a digit without an upper bound, which holds `coordinate / divisor`. - modulus: usize = 0, - - /// The index of the tile along this dimension. - fn tileCount(self: Dim, tile: usize) Dim { - const size = std.math.divCeil(usize, self.size, tile) catch unreachable; - const axis = self.axis orelse return .{ .axis = null, .size = size }; - if (self.modulus == 0) return .{ .axis = axis, .size = size, .divisor = self.divisor * tile }; - if (self.modulus % tile == 0) return .{ .axis = axis, .size = size, .divisor = self.divisor * tile, .modulus = self.modulus / tile }; - // One tile holds every value of the digit. - return .{ .axis = null, .size = size }; - } - - /// The index within the tile along this dimension. - fn tileIndex(self: Dim, tile: usize) Dim { - const axis = self.axis orelse return .{ .axis = null, .size = tile }; - // XLA assumes nested tiles add no padding. - stdx.debug.assert( - self.modulus == 0 or self.modulus % tile == 0 or tile % self.modulus == 0, - "nested tile {} must divide, or be a multiple of, the {} values it tiles", - .{ tile, self.modulus }, - ); - if (self.modulus != 0 and self.modulus < tile) return .{ .axis = axis, .size = tile, .divisor = self.divisor, .modulus = self.modulus }; - return .{ .axis = axis, .size = tile, .divisor = self.divisor, .modulus = tile }; - } - }; - - /// Follows XLA's LayoutUtil::LinearIndexForNestedTiling. The physical dimensions - /// start in minor_to_major order. Each tile then splits the most minor ones, - /// padded with leading unit dimensions if the tile has more, into tile counts - /// followed by indices within the tile. Tile sizes are listed major to minor, so - /// a later tile splits the previous tile's indices and possibly its counts. - /// - /// Two consequences are easy to get wrong: - /// - `tile_dims[j]` of a k-dimensional tile belongs to `minor_to_major[k-1-j]`, - /// not to `minor_to_major[j]` nor to logical axis j. TPU's default - /// `{2,1,0:T(8,128)}` for rank-3 32-bit arrays tiles rows by 8 and columns - /// by 128. - /// - A later tile splits the previous tile's dimensions, not the grid of - /// tiles: `(8,128)(2,1)`, the bf16 packing, interleaves pairs of rows - /// within each tile. TPU tiles bf16 and 8-bit arrays this way. - fn init(shape: Shape, layout: pjrt.MemoryLayout.Tiled) TiledDims { - const rank = shape.rank(); - stdx.debug.assert(layout.minor_to_major.len == rank, "layout rank {} doesn't match shape rank {}", .{ layout.minor_to_major.len, rank }); - var result: TiledDims = .{}; - var i = rank; - while (i > 0) { - i -= 1; - const axis: u3 = @intCast(layout.minor_to_major[i]); - result.dims.appendAssumeCapacity(.{ .axis = axis, .size = @intCast(shape.dim(axis)) }); - } - var cursor: usize = 0; - for (layout.tile_dims_sizes) |tile_rank| { - const tile = layout.tile_dims[cursor..][0..tile_rank]; - cursor += tile_rank; - while (result.dims.len < tile_rank) result.dims.insert(0, .{ .axis = null, .size = 1 }) catch unreachable; - const start = result.dims.len - tile_rank; - for (tile, start..) |size_, d| { - // XLA marks dimensions combined with the next one by a negative size. - stdx.debug.assert(size_ > 0, "unsupported tile dimension {}", .{size_}); - const size: usize = @intCast(size_); - result.dims.appendAssumeCapacity(result.dims.get(d).tileIndex(size)); - result.dims.set(d, result.dims.get(d).tileCount(size)); - } - } - stdx.debug.assert(cursor == layout.tile_dims.len, "layout tile metadata is inconsistent", .{}); - return result; - } -}; - -// -// ============= -// === TESTS === -// ============= -// - -/// The byte offset of an element, computed from the layout directly to check `LayoutMetadata`. -fn referenceByteOffset(shape_: Shape, layout: pjrt.MemoryLayout, index: []const usize) usize { - if (!builtin.is_test) @compileError("referenceByteOffset is only available in tests"); - - return switch (layout) { - .tiled => |tiled| tiledElementOffset(shape_, tiled, index) * shape_.dtype().sizeOf(), - .strides => |strides| b: { - var byte_offset: i64 = 0; - for (index, strides.byte_strides) |coord, stride| { - stdx.debug.assert(stride >= 0, "negative destination byte strides are unsupported, got {}", .{stride}); - byte_offset += @as(i64, @intCast(coord)) * stride; - } - stdx.debug.assert(byte_offset >= 0, "destination byte offset must be non-negative, got {}", .{byte_offset}); - break :b @as(usize, @intCast(byte_offset)); - }, - }; -} - -/// Computes the element offset for one logical index in XLA/PJRT tiled layout order, -/// as XLA's LayoutUtil::LinearIndexForNestedTiling does. A direct port of that -/// function, kept independent of `TiledDims`: a reference sharing the prepared -/// model of tiling once let tests miss offsets that disagreed with XLA. -fn tiledElementOffset(shape_: Shape, layout: pjrt.MemoryLayout.Tiled, index: []const usize) usize { - if (!builtin.is_test) @compileError("tiledElementOffset is only available in tests"); - - const rank = shape_.rank(); - stdx.debug.assert(layout.minor_to_major.len == rank, "layout rank {} doesn't match shape rank {}", .{ layout.minor_to_major.len, rank }); - - // Physical dimensions and indices, major to minor. - const Expanded = stdx.BoundedArray(usize, Shape.MAX_RANK + 2 * pjrt.DefaultMemoryLayout.MAX_TILE_DIMS); - var dims: Expanded = .empty; - var indices: Expanded = .empty; - var i = rank; - while (i > 0) { - i -= 1; - const axis: usize = @intCast(layout.minor_to_major[i]); - dims.appendAssumeCapacity(@intCast(shape_.dim(axis))); - indices.appendAssumeCapacity(index[axis]); - } - - var cursor: usize = 0; - for (layout.tile_dims_sizes) |tile_rank| { - const tile = layout.tile_dims[cursor..][0..tile_rank]; - cursor += tile_rank; - while (dims.len < tile_rank) { - dims.insert(0, 1) catch unreachable; - indices.insert(0, 0) catch unreachable; - } - // Tiled dimensions become tile counts, followed by the indices within the tile. - const start = dims.len - tile_rank; - for (tile, start..) |tile_dim_, d| { - const tile_dim: usize = @intCast(tile_dim_); - dims.appendAssumeCapacity(tile_dim); - indices.appendAssumeCapacity(indices.get(d) % tile_dim); - dims.set(d, std.math.divCeil(usize, dims.get(d), tile_dim) catch unreachable); - indices.set(d, indices.get(d) / tile_dim); - } - } - stdx.debug.assert(cursor == layout.tile_dims.len, "layout tile metadata is inconsistent", .{}); - - var offset: usize = 0; - for (dims.constSlice(), indices.constSlice()) |dim, coord| offset = offset * dim + coord; - return offset; -} - -test "LayoutMetadata byte offsets match physical layouts" { - const unit_tiles: [pjrt.DefaultMemoryLayout.MAX_TILE_DIMS]i64 = @splat(1); - const tile_ranks: [pjrt.DefaultMemoryLayout.MAX_NUM_TILES]usize = @splat(Shape.MAX_RANK); - const Case = struct { shape: Shape, layout: pjrt.MemoryLayout, block_size: usize }; - const cases = [_]Case{ - .{ .shape = .init(.{}, .i32), .layout = .{ .strides = .{ .byte_strides = &.{} } }, .block_size = 1 }, - .{ .shape = .init(.{ 3, 5 }, .i32), .layout = .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, .block_size = 15 }, - .{ .shape = .init(.{ 3, 5 }, .f16), .layout = .{ .strides = .{ .byte_strides = &.{ 16, 2 } } }, .block_size = 5 }, - .{ .shape = .init(.{ 3, 5 }, .i32), .layout = .{ .strides = .{ .byte_strides = &.{ 4, 16 } } }, .block_size = 1 }, - .{ - .shape = .init(.{ 3, 5 }, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, - .block_size = 15, - }, - .{ - .shape = .init(.{ 3, 5 }, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, - .block_size = 1, - }, - .{ - .shape = .init(.{ 3, 5 }, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} } }, - .block_size = 2, - }, - .{ - .shape = .init(.{ 3, 5 }, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} } }, - .block_size = 1, - }, - .{ - .shape = .init(.{ 3, 5 }, .f16), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, - .block_size = 1, - }, - .{ - .shape = .init(.{ 4, 8 }, .f16), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 4, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, - .block_size = 1, - }, - // The nested tile is larger than the values it tiles. - .{ - .shape = .init(.{ 3, 5 }, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2, 4, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, - .block_size = 1, - }, - // The nested tile also splits the tile counts. - .{ - .shape = .init(.{ 4, 8 }, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 1, 4, 2, 1, 1 }, .tile_dims_sizes = &.{ 2, 3 } } }, - .block_size = 1, - }, - .{ - .shape = .init(.{ 2, 3, 5 }, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 4, 2 }, .tile_dims_sizes = &.{2} } }, - .block_size = 2, - }, - .{ - .shape = .init(.{ 2, 3, 5 }, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2 }, .tile_dims_sizes = &.{2} } }, - .block_size = 2, - }, - .{ - .shape = .init(.{ 2, 3, 5 }, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2, 3, 1 }, .tile_dims_sizes = &.{ 2, 2 } } }, - .block_size = 1, - }, - // The nested tile moves pairs of the innermost axis to the most minor dimension. - .{ - .shape = .init(.{ 2, 3, 5 }, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 2, 1 }, .tile_dims = &.{ 2, 4, 2, 2, 1 }, .tile_dims_sizes = &.{ 3, 2 } } }, - .block_size = 2, - }, - // The tile has more dimensions than the shape. - .{ - .shape = .init(.{5}, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} } }, - .block_size = 4, - }, - .{ - .shape = .init(.{ 1, 1, 1, 1, 1, 1, 2, 3 }, .i32), - .layout = .{ .tiled = .{ .minor_to_major = &.{ 7, 6, 5, 4, 3, 2, 1, 0 }, .tile_dims = &unit_tiles, .tile_dims_sizes = &tile_ranks } }, - .block_size = 3, - }, - }; - for (cases) |case| { - const metadata: LayoutMetadata = .init(case.shape, case.shape, case.layout); - try std.testing.expectEqual(case.block_size, metadata.block_size); - for (0..case.shape.count()) |local| { - const index = Coordinates.unflatten(case.shape, local); - const expected = referenceByteOffset(case.shape, case.layout, index.axes[0..case.shape.rank()]); - const actual = metadata.byteOffset(local, index); - try std.testing.expectEqual(expected, actual); - // Elements after the first of a block follow the previous one in memory. - if ((local % metadata.alignment_size) % metadata.block_size != 0) { - try std.testing.expectEqual(metadata.byteOffset(local - 1, Coordinates.unflatten(case.shape, local - 1)) + metadata.element_size, actual); - } - } - } -} - -test "tiled layout offsets follow XLA" { - const Tiled = pjrt.MemoryLayout.Tiled; - // Figure 1 of https://openxla.org/xla/tiled_layout: F32[3,5]{1,0:T(2,2)}. - const tiles: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} }; - try std.testing.expectEqual(17, tiledElementOffset(.init(.{ 3, 5 }, .f32), tiles, &.{ 2, 3 })); - const shape_ = Shape.init(.{ 3, 4 }, .i32); - try std.testing.expectEqual(0, tiledElementOffset(shape_, tiles, &.{ 0, 0 })); - try std.testing.expectEqual(1, tiledElementOffset(shape_, tiles, &.{ 0, 1 })); - try std.testing.expectEqual(2, tiledElementOffset(shape_, tiles, &.{ 1, 0 })); - try std.testing.expectEqual(3, tiledElementOffset(shape_, tiles, &.{ 1, 1 })); - try std.testing.expectEqual(4, tiledElementOffset(shape_, tiles, &.{ 0, 2 })); - try std.testing.expectEqual(8, tiledElementOffset(shape_, tiles, &.{ 2, 0 })); - - // Figure 2: 4x8 tiled by (2,4)(2,1). The nested tile pairs rows within each - // tile, as the (8,128)(2,1) tiling of bf16 on TPU does. - const nested: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 2, 4, 2, 1 }, .tile_dims_sizes = &.{ 2, 2 } }; - const figure2 = [4][8]usize{ - .{ 0, 2, 4, 6, 8, 10, 12, 14 }, - .{ 1, 3, 5, 7, 9, 11, 13, 15 }, - .{ 16, 18, 20, 22, 24, 26, 28, 30 }, - .{ 17, 19, 21, 23, 25, 27, 29, 31 }, - }; - for (figure2, 0..) |offsets, row| { - for (offsets, 0..) |offset, col| try std.testing.expectEqual(offset, tiledElementOffset(.init(.{ 4, 8 }, .bf16), nested, &.{ row, col })); - } - - // A tile lists sizes for the most minor physical dimensions, major to minor. - // Like {2,1,0:T(8,128)} on TPU, this tiles rows by 2 and columns by 4: - // offset = a*32 + (b/2)*16 + (c/4)*8 + (b%2)*4 + c%4. - const rank3: Shape = .init(.{ 2, 3, 5 }, .f32); - const partial: Tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} }; - try std.testing.expectEqual(4, tiledElementOffset(rank3, partial, &.{ 0, 1, 0 })); - try std.testing.expectEqual(8, tiledElementOffset(rank3, partial, &.{ 0, 0, 4 })); - try std.testing.expectEqual(12, tiledElementOffset(rank3, partial, &.{ 0, 1, 4 })); - try std.testing.expectEqual(16, tiledElementOffset(rank3, partial, &.{ 0, 2, 0 })); - try std.testing.expectEqual(51, tiledElementOffset(rank3, partial, &.{ 1, 2, 3 })); - - // Tiles follow the physical order: offset = (c/2)*8 + (r/2)*4 + (c%2)*2 + r%2. - const transposed: Tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{ 2, 2 }, .tile_dims_sizes = &.{2} }; - try std.testing.expectEqual(1, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 1, 0 })); - try std.testing.expectEqual(2, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 0, 1 })); - try std.testing.expectEqual(4, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 2, 0 })); - try std.testing.expectEqual(8, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 0, 2 })); - try std.testing.expectEqual(14, tiledElementOffset(.init(.{ 3, 5 }, .f32), transposed, &.{ 2, 3 })); - - // A tile with more dimensions than the shape pads it with leading unit - // dimensions: offset = (c/4)*8 + c%4. - const wide: Tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{ 2, 4 }, .tile_dims_sizes = &.{2} }; - try std.testing.expectEqual(3, tiledElementOffset(.init(.{5}, .f32), wide, &.{3})); - try std.testing.expectEqual(8, tiledElementOffset(.init(.{5}, .f32), wide, &.{4})); - - // A later tile can also split the previous tile counts: - // offset = r*8 + (c%4)*2 + (c/4)%2. - const cross_tile: Tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{ 1, 4, 2, 1, 1 }, .tile_dims_sizes = &.{ 2, 3 } }; - try std.testing.expectEqual(1, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 0, 4 })); - try std.testing.expectEqual(2, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 0, 1 })); - try std.testing.expectEqual(11, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 1, 5 })); - try std.testing.expectEqual(31, tiledElementOffset(.init(.{ 4, 8 }, .f32), cross_tile, &.{ 3, 7 })); -} - -test "HostAccessible.HostPinnedAllocations keeps each region's replicas together as allocations are added" { - var allocations: HostAccessible.HostPinnedAllocations = .empty; - defer allocations.deinit(std.testing.allocator); - try allocations.entries.ensureTotalCapacity(std.testing.allocator, 5); - var storage: [5]u8 = undefined; - const base: [*]u8 = &storage; - // A vector of 12 split into shards of 4, holding regions A, B, A, C, B: - // replicas arrive interleaved with other regions. - const shard_shape: Shape = .init(.{4}, .u8); - for ([_]usize{ 0, 4, 0, 8, 4 }, 0..) |row, i| { - allocations.add(shard_shape, .{ .origin = .{ .axes = .{row} ++ @as([Shape.MAX_RANK - 1]usize, @splat(0)) }, .ptr = base + i, .shard_index = undefined, .handle = undefined }); - } - - try std.testing.expectEqualSlices([*]u8, &.{ base, base + 2, base + 1, base + 4, base + 3 }, allocations.entries.items(.ptr)); - for (allocations.regions.constSlice(), [_][2]u8{ .{ 0, 2 }, .{ 2, 2 }, .{ 4, 1 } }) |region, expected| { - try std.testing.expectEqual(expected, [2]u8{ region.start, region.len }); - } -} - -test "HostAccessible.format prints the elements in row-major order" { - const shape_ = Shape.init(.{ 2, 2 }, .i32); - const column_major: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; - var storage = [_]i32{ 0, 2, 1, 3 }; - const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, column_major, &.{0}, &.{@ptrCast(&storage)}); - defer pinned.deinit(std.testing.allocator); - var buffer: [128]u8 = undefined; - const printed = try std.fmt.bufPrint(&buffer, "{f}", .{pinned}); - try std.testing.expectStringEndsWith(printed, ") { 0 1 2 3 }"); -} - -test "HostAccessible.View borrows the buffer's pointers through slicing" { - const shape_ = Shape.init(.{4}, .i32); - const layout: pjrt.MemoryLayout = .{ .strides = .{ .byte_strides = &.{4} } }; - var storage = [_]i32{ 0, 1, 2, 3 }; - var other = [_]i32{ 10, 11, 12, 13 }; - const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &.{@ptrCast(&storage)}); - defer pinned.deinit(std.testing.allocator); - const nested = pinned.view(i32).slice(1, null).slice(0, 2); - - // Slices read the pointers in place, which lets `await` refresh them - // without copying them into every view. - try std.testing.expectEqual(@as(*const HostAccessible.Mapping, pinned.mapping), nested.mapping); - pinned.mapping.allocations.entries.items(.ptr)[0] = @ptrCast(&other); - nested.fill(99); - try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3 }, &storage); - try std.testing.expectEqualSlices(i32, &.{ 10, 99, 99, 13 }, &other); -} - -test "HostAccessible.View blocks skip tile padding" { - const shape_ = Shape.init(.{ 3, 5 }, .i32); - const layout: pjrt.MemoryLayout = .{ .tiled = .{ - .minor_to_major = &.{ 1, 0 }, - .tile_dims = &.{ 2, 2 }, - .tile_dims_sizes = &.{2}, - } }; - var storage: [24]i32 = @splat(-1); - const ptrs = [_][*]u8{@ptrCast(&storage)}; - const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &ptrs); - defer pinned.deinit(std.testing.allocator); - const values = pinned.view(i32); - try std.testing.expectEqual(2, pinned.mapping.layout_metadata.block_size); - try std.testing.expectEqual(HostAccessible.View(i32).Bounds{ .start = 0, .end = 2 }, values.blockBounds(1)); - try std.testing.expectEqual(HostAccessible.View(i32).Bounds{ .start = 4, .end = 5 }, values.blockBounds(4)); - var iterator = values.blocks(); - while (iterator.next()) |block| { - for (0..block.replicas.len) |i| { - const items = block.replica(i); - for (items, block.offset..) |*value, index| value.* = @intCast(index + 1); - } - } - try std.testing.expectEqualSlices(i32, &.{ - 1, 2, 6, 7, 3, 4, 8, 9, 5, -1, 10, -1, - 11, 12, -1, -1, 13, 14, -1, -1, 15, -1, -1, -1, - }, &storage); - values.slice(4, 3).fill(90); - try std.testing.expectEqualSlices(i32, &.{ - 1, 2, 90, 90, 3, 4, 8, 9, 90, -1, 10, -1, - 11, 12, -1, -1, 13, 14, -1, -1, 15, -1, -1, -1, - }, &storage); -} - -test "HostAccessible.View nested tiles" { - const shape_ = Shape.init(.{ 3, 5 }, .i32); - const layout: pjrt.MemoryLayout = .{ .tiled = .{ - .minor_to_major = &.{ 1, 0 }, - .tile_dims = &.{ 2, 2, 2, 1 }, - .tile_dims_sizes = &.{ 2, 2 }, - } }; - var storage: [24]i32 = @splat(-1); - const ptrs = [_][*]u8{@ptrCast(&storage)}; - const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &ptrs); - defer pinned.deinit(std.testing.allocator); - const values = pinned.view(i32); - // Pairs of rows are interleaved within each tile, so every block is one element. - try std.testing.expectEqual(1, pinned.mapping.layout_metadata.block_size); - values.copyFrom(&.{ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }); - try std.testing.expectEqualSlices(i32, &.{ - 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, -1, -1, - 11, -1, 12, -1, 13, -1, 14, -1, 15, -1, -1, -1, - }, &storage); -} - -test "HostAccessible.View dense, transposed, strided and partial-rank tiled layouts" { - const shape_ = Shape.init(.{ 2, 3 }, .i32); - const Case = struct { layout: pjrt.MemoryLayout, block_size: usize, expected: []const i32 }; - const cases = [_]Case{ - .{ - .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, - .block_size = 6, - .expected = &.{ 1, 2, 3, 4, 5, 6, -1, -1 }, - }, - .{ - .layout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, - .block_size = 1, - .expected = &.{ 1, 4, 2, 5, 3, 6, -1, -1 }, - }, - .{ - .layout = .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, - .block_size = 6, - .expected = &.{ 1, 2, 3, 4, 5, 6, -1, -1 }, - }, - .{ - .layout = .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, - .block_size = 3, - .expected = &.{ 1, 2, 3, -1, -1, 4, 5, 6 }, - }, - .{ - .layout = .{ .strides = .{ .byte_strides = &.{ 4, 12 } } }, - .block_size = 1, - .expected = &.{ 1, 4, -1, 2, 5, -1, 3, 6 }, - }, - .{ - .layout = .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{2}, .tile_dims_sizes = &.{1} } }, - .block_size = 3, - .expected = &.{ 1, 2, 3, -1, 4, 5, 6, -1 }, - }, - }; - for (cases) |case| { - var storage: [8]i32 = @splat(-1); - const ptrs = [_][*]u8{@ptrCast(&storage)}; - const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, case.layout, &.{0}, &ptrs); - defer pinned.deinit(std.testing.allocator); - const values = pinned.view(i32); - try std.testing.expectEqual(case.block_size, pinned.mapping.layout_metadata.block_size); - values.copyFrom(&.{ 1, 2, 3, 4, 5, 6 }); - try std.testing.expectEqualSlices(i32, case.expected, &storage); - } -} - -test "HostAccessible.View row-major layouts return one block across rows and replicas" { - const shape_: Shape = .init(.{ 3, 5 }, .i32); - const layouts = [_]pjrt.MemoryLayout{ - .{ .strides = .{ .byte_strides = &.{ 20, 4 } } }, - .{ .tiled = .{ .minor_to_major = &.{ 1, 0 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }, - }; - for (layouts) |layout| { - var storage: [2][15]i32 = @splat(@splat(0)); - const ptrs = [_][*]u8{ @ptrCast(&storage[0]), @ptrCast(&storage[1]) }; - const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{ 0, 0 }, &ptrs); - defer pinned.deinit(std.testing.allocator); - const values = pinned.view(i32); - for ([_]HostAccessible.View(i32){ values, values.slice(1, null).slice(1, 10) }) |selected| { - var forward = selected.blocks(); - var reverse = selected.reverseBlocks(); - for ([_]HostAccessible.View(i32).Block{ forward.next().?, reverse.next().? }) |block| { - try std.testing.expectEqual(0, block.offset); - try std.testing.expectEqual(selected.len, block.len); - try std.testing.expectEqual(2, block.replicas.len); - for (0..block.replicas.len, &storage) |i, *replica| { - const items = block.replica(i); - try std.testing.expectEqual(replica[selected.start..].ptr, items.ptr); - try std.testing.expectEqual(selected.len, items.len); - } - } - try std.testing.expectEqual(null, forward.next()); - try std.testing.expectEqual(null, reverse.next()); - } - } -} - -test "HostAccessible.View dense shards stop at gaps in global row-major order" { - const Case = struct { - shape: Shape, - shard_shape: Shape, - block_size: usize, - origins: [2]usize, - expected: [2][]const i32, - }; - const cases = [_]Case{ - .{ - .shape = .init(.{ 2, 6 }, .i32), - .shard_shape = .init(.{ 2, 3 }, .i32), - .block_size = 3, - .origins = .{ 0, 3 }, - .expected = .{ &.{ 0, 1, 2, 6, 7, 8 }, &.{ 3, 4, 5, 9, 10, 11 } }, - }, - .{ - .shape = .init(.{ 2, 6, 2 }, .i32), - .shard_shape = .init(.{ 2, 3, 2 }, .i32), - .block_size = 6, - .origins = .{ 0, 6 }, - .expected = .{ - &.{ 0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 17 }, - &.{ 6, 7, 8, 9, 10, 11, 18, 19, 20, 21, 22, 23 }, - }, - }, - }; - for (cases) |case| { - const strides = case.shard_shape.computeByteStrides(); - var storage: [2][12]i32 = @splat(@splat(-1)); - var ptrs: [2][*]u8 = undefined; - for (&ptrs, &storage) |*ptr, *data| ptr.* = @ptrCast(data); - const pinned = try HostAccessible.initForTests(std.testing.allocator, case.shape, case.shard_shape, .{ .strides = .{ .byte_strides = strides.constSlice() } }, &case.origins, &ptrs); - defer pinned.deinit(std.testing.allocator); - const elements = pinned.view(i32); - try std.testing.expectEqual(case.block_size, pinned.mapping.layout_metadata.block_size); - try std.testing.expectEqual(case.block_size, elements.blockBounds(0).end); - var tail = elements.slice(case.block_size - 1, null).blocks(); - try std.testing.expectEqual(1, tail.next().?.len); - - elements.fillIota(0, 1); - for (storage, case.expected) |data, expected| { - try std.testing.expectEqualSlices(i32, expected, data[0..expected.len]); - } - var actual: [24]i32 = undefined; - elements.copyTo(actual[0..case.shape.count()]); - for (actual[0..case.shape.count()], 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(i)), value); - - // A partial range crosses both a global gap and a physical shard boundary. - const start = case.block_size - 1; - const values = [_]i32{ 90, 91, 92, 93 }; - elements.slice(start, values.len).copyFrom(&values); - for (storage, case.expected) |data, expected| { - for (data[0..expected.len], expected) |value, global| { - const offset: usize = @intCast(global); - try std.testing.expectEqual(if (offset >= start and offset < start + values.len) values[offset - start] else global, value); - } - } - var copied: [4]i32 = undefined; - elements.slice(start, copied.len).copyTo(&copied); - try std.testing.expectEqualSlices(i32, &values, &copied); - } -} - -/// A 4x6 array split into 2x3 shards. Shards 4 to 7 are separate copies of shards 0 to 3. -const TestShards = struct { - storage: [8][8]i32 = @splat(@splat(-1)), - ptrs: [8][*]u8 = undefined, - origins: [8]usize = @splat(0), - pinned: ?HostAccessible = null, - - const shape_ = Shape.init(.{ 4, 6 }, .i32); - const shard_shape = Shape.init(.{ 2, 3 }, .i32); - const tiled: pjrt.MemoryLayout = .{ .tiled = .{ - .minor_to_major = &.{ 1, 0 }, - .tile_dims = &.{ 2, 2 }, - .tile_dims_sizes = &.{2}, - } }; - - fn deinit(self: *TestShards) void { - if (self.pinned) |pinned| pinned.deinit(std.testing.allocator); - } - - /// The view borrows `pinned`, which borrows the storage: the storage must - /// stay in place, and the next call replaces `pinned`. - fn view(self: *TestShards, layout: pjrt.MemoryLayout) !HostAccessible.View(i32) { - for (&self.ptrs, &self.origins, &self.storage, 0..) |*ptr, *origin, *data, i| { - ptr.* = @ptrCast(data); - const row = ((i % 4) / 2) * 2; - const column = (i % 2) * 3; - origin.* = row * 6 + column; - } - self.deinit(); - self.pinned = null; - const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shard_shape, layout, &self.origins, &self.ptrs); - self.pinned = pinned; - return pinned.view(i32); - } - - /// With the tiled layout, every copy of each element holds `expected[global]` - /// and the tile padding is untouched. - fn expectValues(self: *const TestShards, expected: [24]i32) !void { - // Explicit offsets for the two rows of a 2x3 shard tiled by 2x2. - const physical_offsets = [_]usize{ 0, 1, 4, 2, 3, 6 }; - for (self.storage, self.origins) |data, origin| { - for (physical_offsets, 0..) |physical, local| { - const global = origin + (local / 3) * 6 + local % 3; - try std.testing.expectEqual(expected[global], data[physical]); - } - try std.testing.expectEqual(-1, data[5]); - try std.testing.expectEqual(-1, data[7]); - } - } -}; - -test "HostAccessible.View blocks visit partial global ranges in every shard" { - var fixture: TestShards = .{}; - defer fixture.deinit(); - const values = try fixture.view(TestShards.tiled); - var visits: [24]u8 = @splat(0); - // Nested slicing must keep block offsets relative to the selected range. - var iterator = values.slice(1, null).slice(1, 18).blocks(); - while (iterator.next()) |block| { - try std.testing.expect(block.offset + block.len <= 18); - try std.testing.expectEqual(2, block.replicas.len); - for (0..block.replicas.len) |i| { - const items = block.replica(i); - for (items, block.offset + 2..) |*value, global| { - visits[global] += 1; - value.* = @intCast(global); - } - } - } - var expected: [24]i32 = @splat(-1); - for (visits, &expected, 0..) |count, *value, global| { - const selected = global >= 2 and global < 20; - try std.testing.expectEqual(@as(u8, if (selected) 2 else 0), count); - if (selected) value.* = @intCast(global); - } - try fixture.expectValues(expected); - // Starting after the first row partition must skip those shards entirely. - values.set(17, 99); - try std.testing.expectEqual(99, fixture.storage[3][4]); - try std.testing.expectEqual(99, fixture.storage[7][4]); - - const Case = struct { step: i32, expected: [18]i32 }; - const cases = [_]Case{ - .{ .step = 3, .expected = .{ 7, 10, 13, 16, 19, 22, 25, 28, 31, 34, 37, 40, 43, 46, 49, 52, 55, 58 } }, - .{ .step = -2, .expected = .{ 7, 5, 3, 1, -1, -3, -5, -7, -9, -11, -13, -15, -17, -19, -21, -23, -25, -27 } }, - .{ .step = 0, .expected = @splat(7) }, - }; - for (cases) |case| { - values.slice(1, null).slice(1, 18).fillIota(7, case.step); - expected[2..20].* = case.expected; - try fixture.expectValues(expected); - } -} - -test "HostAccessible.View blocks merge shuffled shards and group replicas" { - var fixture: TestShards = .{}; - defer fixture.deinit(); - _ = try fixture.view(TestShards.tiled); - const order = [_]usize{ 3, 4, 1, 6, 2, 0, 7, 5 }; - var ptrs: [8][*]u8 = undefined; - var origins: [8]usize = undefined; - for (order, &ptrs, &origins) |index, *ptr, *origin| { - ptr.* = fixture.ptrs[index]; - origin.* = fixture.origins[index]; - } - const pinned = try HostAccessible.initForTests(std.testing.allocator, TestShards.shape_, TestShards.shard_shape, TestShards.tiled, &origins, &ptrs); - defer pinned.deinit(std.testing.allocator); - const values = pinned.view(i32); - const selected = values.slice(1, null).slice(1, 18); - const offsets = [_]usize{ 0, 1, 3, 4, 6, 7, 9, 10, 12, 13, 15, 16 }; - const lengths = [_]usize{ 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2 }; - const replicas = [_][2]usize{ .{ 4, 0 }, .{ 1, 5 }, .{ 6, 2 }, .{ 3, 7 } }; - const physical_offsets = [_]usize{ 0, 1, 4, 2, 3, 6 }; - var iterator = selected.blocks(); - for (offsets, lengths) |offset, len| { - const block = iterator.next().?; - try std.testing.expectEqual(offset, block.offset); - try std.testing.expectEqual(len, block.len); - try std.testing.expectEqual(2, block.replicas.len); - const block_start = offset + 2; - const region = block_start / 12 * 2 + block_start % 6 / 3; - const physical = physical_offsets[block_start / 6 % 2 * 3 + block_start % 3]; - for (0..block.replicas.len, replicas[region]) |i, storage_index| { - const items = block.replica(i); - try std.testing.expectEqual(fixture.storage[storage_index][physical..].ptr, items.ptr); - } - for (0..block.replicas.len) |i| { - const items = block.replica(i); - for (items, offset + 2..) |*value, global| value.* = @intCast(global); - } - } - try std.testing.expectEqual(null, iterator.next()); - var expected: [24]i32 = @splat(-1); - for (expected[2..20], 2..) |*value, global| value.* = @intCast(global); - try fixture.expectValues(expected); - try expectReverseBlocksMirrorForward(selected); -} - -test "HostAccessible.View blocks group the maximum number of replicas" { - var storage: [Platform.MAX_NUM_DEVICES][4]i32 = @splat(@splat(-1)); - var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; - const origins: [Platform.MAX_NUM_DEVICES]usize = @splat(0); - for (&ptrs, &storage) |*ptr, *items| ptr.* = @ptrCast(items); - const shape_: Shape = .init(.{4}, .i32); - const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{4} } }, &origins, &ptrs); - defer pinned.deinit(std.testing.allocator); - const values = pinned.view(i32); - var iterator = values.blocks(); - const block = iterator.next().?; - try std.testing.expectEqual(0, block.offset); - try std.testing.expectEqual(4, block.len); - try std.testing.expectEqual(Platform.MAX_NUM_DEVICES, block.replicas.len); - try std.testing.expectEqual(null, iterator.next()); - // The descriptors remain valid after advancing the iterator. - for (0..block.replicas.len, &storage) |i, *replica| { - const items = block.replica(i); - try std.testing.expectEqual(replica[0..].ptr, items.ptr); - @memset(items, 42); - try std.testing.expectEqualSlices(i32, &.{ 42, 42, 42, 42 }, replica); - } - try expectReverseBlocksMirrorForward(values); -} - -test "HostAccessible.View visits the maximum number of distinct shuffled regions" { - var storage: [Platform.MAX_NUM_DEVICES]i32 = @splat(-1); - var ptrs: [Platform.MAX_NUM_DEVICES][*]u8 = undefined; - var origins: [Platform.MAX_NUM_DEVICES]usize = undefined; - for (&ptrs, &origins, 0..) |*ptr, *origin, shard| { - const index = storage.len - shard - 1; - ptr.* = @ptrCast(&storage[index]); - origin.* = index; - } - const shape_: Shape = .init(.{Platform.MAX_NUM_DEVICES}, .i32); - const shard_shape: Shape = .init(.{1}, .i32); - const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shard_shape, .{ .strides = .{ .byte_strides = &.{4} } }, &origins, &ptrs); - defer pinned.deinit(std.testing.allocator); - const values = pinned.view(i32); - values.fillIota(0, 1); - var forward = values.blocks(); - var reverse = values.reverseBlocks(); - for (storage, 0..) |value, index| { - try std.testing.expectEqual(@as(i32, @intCast(index)), value); - const block = forward.next().?; - try std.testing.expectEqual(index, block.offset); - try std.testing.expectEqual(1, block.len); - try std.testing.expectEqual(1, block.replicas.len); - try std.testing.expectEqual(storage[index..].ptr, block.firstReplica().ptr); - const reverse_block = reverse.next().?; - const reverse_index = storage.len - index - 1; - try std.testing.expectEqual(reverse_index, reverse_block.offset); - try std.testing.expectEqual(storage[reverse_index..].ptr, reverse_block.firstReplica().ptr); - } - try std.testing.expectEqual(null, forward.next()); - try std.testing.expectEqual(null, reverse.next()); -} - -test "HostAccessible.View multidimensional slices preserve coordinates across carries" { - const shape_: Shape = .init(.{ 2, 3, 5 }, .i32); - const layouts = [_]pjrt.MemoryLayout{ - .{ .strides = .{ .byte_strides = &.{ 60, 20, 4 } } }, - .{ .tiled = .{ .minor_to_major = &.{ 2, 1, 0 }, .tile_dims = &.{ 3, 2 }, .tile_dims_sizes = &.{2} } }, - }; - for (layouts) |layout| { - var storage: [48]i32 = @splat(-1); - const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, layout, &.{0}, &.{@ptrCast(&storage)}); - defer pinned.deinit(std.testing.allocator); - const values = pinned.view(i32); - values.fillIota(0, 1); - var expected: [48]i32 = @splat(-1); - for (0..shape_.count()) |local| { - const index = Coordinates.unflatten(shape_, local); - const physical = referenceByteOffset(shape_, layout, index.axes[0..shape_.rank()]) / @sizeOf(i32); - expected[physical] = @intCast(local); - try std.testing.expectEqual(@as(i32, @intCast(local)), values.get(local)); - } - try std.testing.expectEqualSlices(i32, &expected, &storage); - for (0..shape_.count() + 1) |start| { - for (0..shape_.count() - start + 1) |len| { - try expectReverseBlocksMirrorForward(values.slice(start, len)); - } - } - } -} - -test "HostAccessible.View reverse blocks mirror forward blocks" { - var fixture: TestShards = .{}; - defer fixture.deinit(); - const layouts = [_]pjrt.MemoryLayout{ - TestShards.tiled, - .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, - .{ .strides = .{ .byte_strides = &.{ 4, 8 } } }, - }; - for (layouts) |layout| { - const values = try fixture.view(layout); - for (0..values.len + 1) |start| { - for (0..values.len - start + 1) |len| { - try expectReverseBlocksMirrorForward(values.slice(start, len)); - } - } - } - // Dense blocks can span multiple rows, and scalar blocks have no row axis. - const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, TestShards.shard_shape, TestShards.shard_shape, layouts[1], fixture.origins[0..1], fixture.ptrs[0..1]); - defer dense_pinned.deinit(std.testing.allocator); - const dense = dense_pinned.view(i32); - try expectReverseBlocksMirrorForward(dense.slice(1, 4)); - const scalar_shape: Shape = .init(.{}, .i32); - const scalar_pinned = try HostAccessible.initForTests(std.testing.allocator, scalar_shape, scalar_shape, .{ .strides = .{ .byte_strides = &.{} } }, fixture.origins[0..1], fixture.ptrs[0..1]); - defer scalar_pinned.deinit(std.testing.allocator); - const scalar = scalar_pinned.view(i32); - try expectReverseBlocksMirrorForward(scalar); -} - -fn expectReverseBlocksMirrorForward(values: HostAccessible.View(i32)) !void { - var expected: [48]HostAccessible.View(i32).Block = undefined; - var len: usize = 0; - var offset: usize = 0; - var forward = values.blocks(); - while (forward.next()) |block| : (len += 1) { - try std.testing.expectEqual(offset, block.offset); - expected[len] = block; - offset += block.len; - } - try std.testing.expectEqual(values.len, offset); - var reverse = values.reverseBlocks(); - while (len > 0) { - len -= 1; - const block = reverse.next().?; - try std.testing.expectEqual(expected[len].offset, block.offset); - try std.testing.expectEqual(expected[len].len, block.len); - try std.testing.expectEqual(expected[len].byte_offset, block.byte_offset); - try std.testing.expectEqualSlices([*]u8, expected[len].replicas, block.replicas); - } - try std.testing.expectEqual(null, reverse.next()); -} - -test "HostAccessible.View scalar, empty ranges and rank-one padding" { - var scalar: i32 = 0; - const scalar_ptrs = [_][*]u8{@ptrCast(&scalar)}; - const shape_ = Shape.scalar(.i32); - const scalar_layout: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{}, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; - const scalar_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, scalar_layout, &.{0}, &scalar_ptrs); - defer scalar_pinned.deinit(std.testing.allocator); - const scalar_view = scalar_pinned.view(i32); - scalar_view.copyFrom(&.{42}); - try std.testing.expectEqual(42, scalar); - var past_end = scalar_view.slice(1, null).blocks(); - try std.testing.expectEqual(null, past_end.next()); - - var storage: [8]bool = @splat(false); - const vector_ptrs = [_][*]u8{@ptrCast(&storage)}; - const vector_shape = Shape.init(.{5}, .bool); - const vector_layout: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{0}, .tile_dims = &.{4}, .tile_dims_sizes = &.{1} } }; - const vector_pinned = try HostAccessible.initForTests(std.testing.allocator, vector_shape, vector_shape, vector_layout, &.{0}, &vector_ptrs); - defer vector_pinned.deinit(std.testing.allocator); - const vector = vector_pinned.view(bool); - vector.fill(true); - try std.testing.expectEqualSlices(bool, &.{ true, true, true, true, true, false, false, false }, &storage); - - const empty_shape = Shape.init(.{0}, .bool); - const empty_pinned = try HostAccessible.initForTests(std.testing.allocator, empty_shape, empty_shape, vector_layout, &.{0}, &vector_ptrs); - defer empty_pinned.deinit(std.testing.allocator); - const empty_view = empty_pinned.view(bool); - var empty_blocks = empty_view.blocks(); - try std.testing.expectEqual(null, empty_blocks.next()); - - // The empty view has no storage or metadata to read. - const none: HostAccessible.View(u32) = .empty; - none.fill(0); - none.copyTo(&.{}); - var none_blocks = none.reverseBlocks(); - try std.testing.expectEqual(null, none_blocks.next()); -} - -test "HostAccessible devices holding a shard read one allocation" { - const platform = testing.env(); - const tp = testing.tpMesh(); - - const shape_ = Shape.init(.{ .b = @as(i64, @intCast(platform.devices.len)), .d = 5 }, .i32); - try expectDevicesReadShards(platform, shape_); - try expectDevicesReadShards(platform, shape_.withPartitioning(tp, .{ .b = .model })); -} - -test "HostAccessible devices holding a shard read one allocation in data-parallel groups" { - const allocator = std.testing.allocator; - const io = std.testing.io; - // Two data-parallel groups of two tensor-parallel devices. - const platform = Platform.auto(allocator, io, .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }) catch return error.SkipZigTest; - defer platform.deinit(allocator, io); - const mesh: Sharding.Mesh = try .init( - "data_parallel", - &platform.physical_mesh, - .mesh(.{ .batch = .low_bandwidth, .model = .high_bandwidth }), - .parseBindings(.{ .batch = .link_x, .model = .link_y }), - ); - try expectDevicesReadShards(platform, Shape.init(.{ .b = 4, .d = 5 }, .i32).withPartitioning(&mesh, .{ .b = .batch })); -} - -/// Rows of `.d` elements written once through a view reach every device -/// holding them, and devices holding the same shard share its allocation -/// where the platform allows it. -fn expectDevicesReadShards(platform: *const Platform, shape_: Shape) !void { - if (!builtin.is_test) @compileError("expectDevicesReadShards is only available in tests"); - - const allocator = std.testing.allocator; - const io = std.testing.io; - const api = platform.pjrt_api; - const pinned = try HostAccessible.init(allocator, io, platform, shape_, .undonatable); - defer pinned.deinit(allocator); - const placement: Sharding.Placement = try Sharding.Placement.init(&shape_); - const expected_allocations = switch (platform.target) { - .cpu, .cuda, .rocm => shape_.count() / placement.localShape(shape_).count(), - else => platform.devices.len, - }; - try std.testing.expectEqual(expected_allocations, pinned.mapping.allocations.entries.len); - - pinned.view(i32).fillIota(0, 1); - const row_size: usize = @intCast(shape_.dim(.d)); - const values = try allocator.alloc(i32, placement.localShape(shape_).count()); - defer allocator.free(values); - for (pinned.buffer._shards.constSlice(), 0..) |shard, device_index| { - @memset(values, -1); - if (try shard.toHostBuffer(api, std.mem.sliceAsBytes(values))) |event| { - defer event.deinit(api); - try event.await(api, io); - } - const first_row = Coordinates.ofShard(&placement, device_index, shape_).axes[0]; - for (values, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(first_row * row_size + i)), value); - } -} - -test "HostAccessible executions read shared inputs on every device" { - const zml = @import("../zml.zig"); - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = testing.env(); - const api = platform.pjrt_api; - const shape_ = Shape.init(.{ 3, 5 }, .i32); - const Test = struct { - fn increment(input: zml.Tensor) zml.Tensor { - return input.onMemory(.host_pinned).toMemory(.device).addConstant(1); - } - }; - var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); - defer exe.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - - const pinned = try HostAccessible.init(allocator, io, platform, shape_, .undonatable); - defer pinned.deinit(allocator); - for (0..2) |iteration| { - pinned.view(i32).fillIota(@intCast(iteration * 100), 1); - var output: Buffer = undefined; - runner.run(io, .{pinned.buffer}, .{&output}, .{}); - defer output.deinit(); - try output.await(io); - for (output._shards.constSlice()) |shard| { - var values: [15]i32 = @splat(-1); - if (try shard.toHostBuffer(api, std.mem.asBytes(&values))) |event| { - defer event.deinit(api); - try event.await(api, io); - } - for (values, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(iteration * 100 + i + 1)), value); - } - } -} - -test "HostAccessible.View.constSlice borrows contiguous buffers and copies the others" { - const shape_ = Shape.init(.{ 2, 3 }, .i32); - var scratch: [6]i32 = @splat(-1); - - var dense: [2][6]i32 = .{ .{ 0, 1, 2, 3, 4, 5 }, @splat(-1) }; - const dense_ptrs = [_][*]u8{ @ptrCast(&dense[0]), @ptrCast(&dense[1]) }; - const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &dense_ptrs); - defer dense_pinned.deinit(std.testing.allocator); - const borrowed = dense_pinned.view(i32).slice(1, 4).constSlice(&scratch); - try std.testing.expectEqual(dense[0][1..].ptr, borrowed.ptr); - try std.testing.expectEqualSlices(i32, &.{ 1, 2, 3, 4 }, borrowed); - - var column_major = [_]i32{ 0, 3, 1, 4, 2, 5 }; - const transposed: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; - const transposed_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, transposed, &.{0}, &.{@ptrCast(&column_major)}); - defer transposed_pinned.deinit(std.testing.allocator); - const copied = transposed_pinned.view(i32).slice(1, 4).constSlice(&scratch); - try std.testing.expectEqual(@as([*]const i32, &scratch), copied.ptr); - try std.testing.expectEqualSlices(i32, &.{ 1, 2, 3, 4 }, copied); - - const none: HostAccessible.View(i32) = .empty; - try std.testing.expectEqual(0, none.constSlice(&.{}).len); -} - -test "HostAccessible.View.edit writes contiguous buffers in place and commits every replica" { - const shape_ = Shape.init(.{ 2, 3 }, .i32); - var scratch: [6]i32 = @splat(-1); - - // In place: the first replica takes the writes, and `commit` copies it to the others. - var dense: [2][6]i32 = @splat(@splat(0)); - const dense_ptrs = [_][*]u8{ @ptrCast(&dense[0]), @ptrCast(&dense[1]) }; - const dense_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &dense_ptrs); - defer dense_pinned.deinit(std.testing.allocator); - const dense_edit = dense_pinned.view(i32).slice(1, 4).edit(&scratch); - try std.testing.expectEqual(dense[0][1..].ptr, dense_edit.items.ptr); - @memcpy(dense_edit.items, &[_]i32{ 1, 2, 3, 4 }); - dense_edit.commit(); - for (dense) |replica| try std.testing.expectEqualSlices(i32, &.{ 0, 1, 2, 3, 4, 0 }, &replica); - - // Through the scratch: the buffer only changes on `commit`. - var column_major: [6]i32 = @splat(0); - const transposed: pjrt.MemoryLayout = .{ .tiled = .{ .minor_to_major = &.{ 0, 1 }, .tile_dims = &.{}, .tile_dims_sizes = &.{} } }; - const transposed_pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, transposed, &.{0}, &.{@ptrCast(&column_major)}); - defer transposed_pinned.deinit(std.testing.allocator); - const copied_edit = transposed_pinned.view(i32).slice(1, 4).edit(&scratch); - try std.testing.expectEqual(@as([*]i32, &scratch), copied_edit.items.ptr); - @memcpy(copied_edit.items, &[_]i32{ 1, 2, 3, 4 }); - try std.testing.expectEqualSlices(i32, &.{ 0, 0, 0, 0, 0, 0 }, &column_major); - copied_edit.commit(); - try std.testing.expectEqualSlices(i32, &.{ 0, 3, 1, 4, 2, 0 }, &column_major); - - const none: HostAccessible.View(i32) = .empty; - none.edit(&.{}).commit(); -} - -test "HostAccessible.View.indexOfScalar and indexOfAny search blocks in row-major order" { - var fixture: TestShards = .{}; - defer fixture.deinit(); - const values = try fixture.view(TestShards.tiled); - values.fillIota(0, 1); - values.set(20, 7); - // Tiles split the view into blocks, and shards into regions. - try std.testing.expectEqual(7, values.indexOfScalar(7)); - try std.testing.expectEqual(12, values.slice(8, null).indexOfScalar(7)); - try std.testing.expectEqual(null, values.slice(8, 12).indexOfScalar(7)); - try std.testing.expectEqual(null, values.indexOfScalar(-1)); - try std.testing.expectEqual(null, HostAccessible.View(i32).empty.indexOfScalar(0)); - - try std.testing.expectEqual(5, values.indexOfAny(&.{ 7, 5 })); - try std.testing.expectEqual(12, values.slice(8, null).indexOfAny(&.{ -1, 7 })); - try std.testing.expectEqual(null, values.slice(8, 12).indexOfAny(&.{ 7, -1 })); - try std.testing.expectEqual(null, values.indexOfAny(&.{})); -} - -test "HostAccessible.View indexes replicas of a dense array directly" { - const shape_ = Shape.init(.{ 2, 3 }, .i32); - var storage: [2][6]i32 = @splat(@splat(-1)); - const ptrs = [_][*]u8{ @ptrCast(&storage[0]), @ptrCast(&storage[1]) }; - const pinned = try HostAccessible.initForTests(std.testing.allocator, shape_, shape_, .{ .strides = .{ .byte_strides = &.{ 12, 4 } } }, &.{ 0, 0 }, &ptrs); - defer pinned.deinit(std.testing.allocator); - try std.testing.expect(pinned.mapping.contiguous); - const values = pinned.view(i32); - for (0..6) |i| values.set(i, @intCast(10 + i)); - values.slice(4, null).set(1, 99); - for (storage) |replica| try std.testing.expectEqualSlices(i32, &.{ 10, 11, 12, 13, 14, 99 }, &replica); - try std.testing.expectEqual(99, values.get(5)); - try std.testing.expectEqual(13, values.slice(2, 3).get(1)); - - inline for (.{ false, true }) |reverse| { - const nested = values.slice(1, 4); - var blocks = if (reverse) nested.reverseBlocks() else nested.blocks(); - const block = blocks.next().?; - try std.testing.expectEqual(0, block.offset); - try std.testing.expectEqual(2, block.replicas.len); - for (0..block.replicas.len, &storage) |i, *replica| { - const items = block.replica(i); - try std.testing.expectEqualSlices(i32, &.{ 11, 12, 13, 14 }, items); - try std.testing.expectEqual(replica[1..].ptr, items.ptr); - } - try std.testing.expectEqual(null, blocks.next()); - } -} - -test "HostAccessible.View set and get address single elements in every replica" { - var fixture: TestShards = .{}; - defer fixture.deinit(); - const values = try fixture.view(TestShards.tiled); - var expected: [24]i32 = undefined; - var i: usize = expected.len; - while (i > 0) { - i -= 1; - expected[i] = @intCast(100 + i); - values.set(i, expected[i]); - } - try fixture.expectValues(expected); - for (expected, 0..) |value, index| try std.testing.expectEqual(value, values.get(index)); - - // Indices of a nested view are relative to its start. - const nested = values.slice(7, 10); - nested.set(3, -5); - expected[10] = -5; - try fixture.expectValues(expected); - try std.testing.expectEqual(-5, nested.get(3)); -} - -test "HostAccessible.View copies across layouts, shards and replicas" { - var fixture: TestShards = .{}; - defer fixture.deinit(); - const values = try fixture.view(TestShards.tiled); - values.fill(0); - values.slice(2, 20).fillIota(10, 1); - values.set(17, 99); - const expected = [_]i32{ 0, 0, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 99, 26, 27, 28, 29, 0, 0 }; - var actual: [24]i32 = undefined; - values.copyTo(&actual); - try std.testing.expectEqualSlices(i32, &expected, &actual); - for (expected, 0..) |value, i| try std.testing.expectEqual(value, values.get(i)); - try fixture.expectValues(expected); - - // Blocks expose every replica, respect physical boundaries, and stop at - // the end of a nested view even when the physical run continues. - const nested = values.slice(1, null).slice(2, 4); - var nested_blocks = nested.blocks(); - const first = nested_blocks.next().?; - try std.testing.expectEqual(0, first.offset); - try std.testing.expectEqual(2, first.replicas.len); - for (0..first.replicas.len, [_]usize{ 1, 5 }) |i, storage_index| { - const items = first.replica(i); - try std.testing.expectEqualSlices(i32, expected[3..5], items); - try std.testing.expectEqual(fixture.storage[storage_index][0..].ptr, items.ptr); - } - const middle = nested_blocks.next().?; - try std.testing.expectEqual(2, middle.offset); - for (0..middle.replicas.len) |i| try std.testing.expectEqualSlices(i32, expected[5..6], middle.replica(i)); - const last = nested_blocks.next().?; - try std.testing.expectEqual(3, last.offset); - for (0..last.replicas.len, [_]usize{ 0, 4 }) |i, storage_index| { - const items = last.replica(i); - try std.testing.expectEqualSlices(i32, expected[6..7], items); - try std.testing.expectEqual(fixture.storage[storage_index][2..].ptr, items.ptr); - } - try std.testing.expectEqual(null, nested_blocks.next()); - - // Copy between differently shaped/tiled ranges, including a nonzero source - // and destination start, and verify every destination replica physically. - const dst_shape = Shape.init(.{ 3, 7 }, .i32); - const dst_layout: pjrt.MemoryLayout = .{ .tiled = .{ - .minor_to_major = &.{ 0, 1 }, - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - } }; - var dst_storage: [2][21]i32 = @splat(@splat(-1)); - const dst_ptrs = [_][*]u8{ @ptrCast(&dst_storage[0]), @ptrCast(&dst_storage[1]) }; - const dst_pinned = try HostAccessible.initForTests(std.testing.allocator, dst_shape, dst_shape, dst_layout, &.{ 0, 0 }, &dst_ptrs); - defer dst_pinned.deinit(std.testing.allocator); - const dst = dst_pinned.view(i32); - dst.slice(1, 19).copyFromView(values.slice(1, 22).slice(1, 19)); - for (dst_storage) |data| { - for (0..21) |i| { - const physical = (i % 7) * 3 + i / 7; - try std.testing.expectEqual(if (i > 0 and i < 20) expected[i + 1] else -1, data[physical]); - } - } - var empty: [0]i32 = .{}; - values.slice(values.len, 0).copyTo(&empty); - values.slice(0, 0).copyFromView(dst.slice(0, 0)); - values.slice(values.len, null).fill(-1); - values.slice(values.len, null).fillIota(-1, 1); - values.slice(values.len, null).copyFrom(&empty); - values.copyTo(&actual); - try std.testing.expectEqualSlices(i32, &expected, &actual); -} - -test "HostAccessible await after executable donation permits the next host update" { - const zml = @import("../zml.zig"); - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = testing.env(); - const shape_ = Shape.init(.{ 3, 5 }, .i32); - const Test = struct { - fn increment(input: zml.Tensor) zml.Tensor { - const host_input = input.onMemory(.host_pinned); - return host_input.toMemory(.device).addConstant(1).toMemory(.host_pinned).reuseBuffer(host_input); - } - }; - var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); - defer exe.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - - var pinned = try HostAccessible.init(allocator, io, platform, shape_, .donatable); - defer pinned.deinit(allocator); - try std.testing.expectEqual(pinned.buffer._shards.len, pinned.mapping.allocations.entries.len); - const first_ptr = pinned.mapping.allocations.entries.items(.ptr)[0]; - pinned.view(i32).fillIota(0, 1); - - // Reuse the same runner and output destination as the model execution paths. - // Only await waits for the asynchronous executable before host access. - for (0..3) |iteration| { - runner.run(io, .{pinned.buffer}, .{&pinned.buffer}, .{}); - try pinned.await(io); - - // Reacquire from the output handles, even when donation reuses memory. - const values = pinned.view(i32); - var actual: [15]i32 = undefined; - values.copyTo(&actual); - try std.testing.expectEqual(first_ptr, pinned.mapping.allocations.entries.items(.ptr)[0]); - for (actual, 0..) |value, i| { - const expected: i32 = if (iteration == 0 or i < 3 or i >= 12) - @intCast(i + iteration + 1) - else - @intCast(iteration * 100 + i - 3 + 1); - try std.testing.expectEqual(expected, value); - } - - // Also check the executable's logical output through PJRT, independently - // of the host-visible layout used above. - var device_values: [15]i32 = undefined; - try pinned.buffer.toSlice(io, .init(shape_, std.mem.asBytes(&device_values))); - try std.testing.expectEqualSlices(i32, &actual, &device_values); - - values.slice(3, 9).fillIota(@intCast((iteration + 1) * 100), 1); - } -} - -test "HostAccessible await follows an output that doesn't reuse the pinned allocation" { - const zml = @import("../zml.zig"); - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = testing.env(); - const shape_ = Shape.init(.{ 3, 5 }, .i32); - const Test = struct { - fn increment(input: zml.Tensor) zml.Tensor { - // Without reuseBuffer, execution writes a new host allocation. - return input.onMemory(.host_pinned).toMemory(.device).addConstant(1).toMemory(.host_pinned); - } - }; - var exe = try platform.compileFn(allocator, io, Test.increment, .{zml.Tensor.fromShape(shape_)}, .{}); - defer exe.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - - var pinned = try HostAccessible.init(allocator, io, platform, shape_, .donatable); - defer pinned.deinit(allocator); - pinned.view(i32).fillIota(0, 1); - - for (0..2) |iteration| { - // The input isn't donated, so release it once execution replaced it. - var input = pinned.buffer; - defer input.deinit(); - const previous = pinned.mapping.allocations.entries.items(.ptr)[0]; - runner.run(io, .{input}, .{&pinned.buffer}, .{}); - try pinned.await(io); - const values = pinned.view(i32); - try std.testing.expect(pinned.mapping.allocations.entries.items(.ptr)[0] != previous); - - var actual: [15]i32 = undefined; - values.copyTo(&actual); - for (actual, 0..) |value, i| try std.testing.expectEqual(@as(i32, @intCast(iteration * 100 + i + 1)), value); - var device_values: [15]i32 = undefined; - try pinned.buffer.toSlice(io, .init(shape_, std.mem.asBytes(&device_values))); - try std.testing.expectEqualSlices(i32, &actual, &device_values); - - // Host writes now land in the output, which the next execution reads. - values.fillIota(@intCast((iteration + 1) * 100), 1); - } -} diff --git a/zml/constants.zig b/zml/constants.zig index 39586fa4..ad8ff445 100644 --- a/zml/constants.zig +++ b/zml/constants.zig @@ -13,6 +13,3 @@ pub const MINOR_TO_MAJOR: [MAX_RANK]i64 = b: { pub fn minorToMajor(rank: u8) []const i64 { return MINOR_TO_MAJOR[MINOR_TO_MAJOR.len - rank ..]; } - -pub const ones_i64: [MAX_RANK]i64 = @splat(1); -pub const zeroes_i64: [MAX_RANK]i64 = @splat(0); diff --git a/zml/dtype.zig b/zml/dtype.zig index f5cd9da8..8ec73c3d 100644 --- a/zml/dtype.zig +++ b/zml/dtype.zig @@ -1,7 +1,5 @@ const std = @import("std"); -const stdx = @import("stdx"); - const floats = @import("floats.zig"); const C64 = std.math.Complex(f32); @@ -9,8 +7,8 @@ const C128 = std.math.Complex(f64); pub const DataType = enum(u8) { bool, - f4e2m1, // Note: the support of the float8 is a bit spotty, f8e4m3b11fnuz seems to be the most supported one on Cuda. + f4e2m1, f8e3m4, f8e4m3, f8e4m3b11fnuz, @@ -138,23 +136,9 @@ pub const DataType = enum(u8) { return @FieldType(Value, @tagName(dtype)); } - pub fn toPackedZigType(comptime dtype: DataType) type { - return switch (dtype) { - .bool => bool, - .i2, .i4, .u2, .u4 => |dt| stdx.Packed(@FieldType(Value, @tagName(dt))), - .f4e2m1 => floats.Float4E2M1.Packed, - inline .c64, .c128 => |dt| @FieldType(Value, @tagName(dt)), - else => { - const T = @FieldType(Value, @tagName(dtype)); - if (@bitSizeOf(T) < 8) @compileLog("Forgot type !", T); - return T; - }, - }; - } - pub fn isSignedInt(dtype: DataType) bool { return switch (dtype) { - .i2, .i4, .i8, .i16, .i32, .i64 => true, + .i4, .i8, .i16, .i32, .i64 => true, else => false, }; } @@ -167,9 +151,6 @@ pub const DataType = enum(u8) { pub fn bitSizeOf(self: DataType) u16 { return switch (self) { - .bool => 8, // In PJRT/stablehlo bool always occupy a full byte - .c64 => 64, - .c128 => 128, inline else => |tag| @bitSizeOf(tag.toZigType()), }; } @@ -363,7 +344,7 @@ pub const DataType = enum(u8) { .bool => return self.bool, .float => switch (self) { inline .f16, .f32, .f64 => |v| return @floatCast(v), - inline .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16 => |v| return @floatCast(v.toF32()), + inline .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .bf16 => |v| return @floatCast(v.toF32()), else => {}, }, .int => switch (self) { @@ -374,13 +355,6 @@ pub const DataType = enum(u8) { } std.debug.panic("Unsupported conversion {} -> {s}", .{ self.dtype(), @typeName(T) }); } - - test as { - try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f4e2m1, 1).as(f32)); - try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e3m4, 1).as(f32)); - try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e4m3, 1).as(f32)); - try std.testing.expectEqual(@as(f32, 1.0), Value.init(.f8e8m0, 1).as(f32)); - } }; }; diff --git a/zml/exe.zig b/zml/exe.zig index 32c0ac48..67493887 100644 --- a/zml/exe.zig +++ b/zml/exe.zig @@ -1,18 +1,12 @@ const std = @import("std"); -const builtin = @import("builtin"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); const Buffer = @import("buffer.zig").Buffer; -const Compiler = @import("Compiler.zig"); -const mem = @import("mem.zig"); const meta = @import("meta.zig"); const Platform = @import("platform.zig").Platform; -const tracer = @import("profiling/tracer.zig"); const Shape = @import("shape.zig").Shape; -const Sharding = @import("Sharding.zig"); -const Tensor = @import("tensor.zig").Tensor; pub const Exe = struct { platform: *const Platform, @@ -23,12 +17,7 @@ pub const Exe = struct { input_shapes: []const Shape, output_shapes: []const Shape, - /// Inputs whose buffers are donated to an output (`reuseBuffer`). - /// Runners destroy their PJRT handles after the raw call returns. - donated_input_indices: []const u32, - - num_devices: u32, - num_partitions: u32, + num_devices: u8, arena: std.heap.ArenaAllocator, @@ -36,11 +25,9 @@ pub const Exe = struct { allocator: std.mem.Allocator, platform: *const Platform, exe: *pjrt.LoadedExecutable, - num_devices: u32, - num_partitions: u32, input_shapes: []const Shape, output_shapes: []const Shape, - input_aliasing: []const ?u32, + num_devices: u8, ) !Exe { var arena = std.heap.ArenaAllocator.init(allocator); errdefer arena.deinit(); @@ -48,33 +35,17 @@ pub const Exe = struct { const input_shapes_copy = try arena.allocator().dupe(Shape, input_shapes); const output_shapes_copy = try arena.allocator().dupe(Shape, output_shapes); - var donated_count: usize = 0; - for (input_aliasing) |aliasing| donated_count += @intFromBool(aliasing != null); - - const donated_input_indices = try arena.allocator().alloc(u32, donated_count); - - var donated_index: usize = 0; - for (input_aliasing, 0..) |aliasing, input_index| { - if (aliasing == null) continue; - donated_input_indices[donated_index] = @intCast(input_index); - donated_index += 1; - } - return .{ .platform = platform, .exe = exe, .input_shapes = input_shapes_copy, .output_shapes = output_shapes_copy, - .donated_input_indices = donated_input_indices, .num_devices = num_devices, - .num_partitions = num_partitions, .arena = arena, }; } pub fn deinit(self: *const Exe) void { - if (self.context) |context| context.deinit(self.platform.pjrt_api); - self.exe.deinit(self.platform.pjrt_api); self.arena.deinit(); } @@ -83,7 +54,7 @@ pub const Exe = struct { } pub fn results(self: *const Exe, allocator: std.mem.Allocator) !Results { - return Results.init(allocator, self.output_shapes, self.platform, self.num_devices); + return Results.init(allocator, self.output_shapes, self.num_devices, self.platform); } pub const FlatBuffers = struct { @@ -121,7 +92,7 @@ pub const Exe = struct { expected_shapes: []const Shape, baked_count: usize = 0, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) error{OutOfMemory}!Arguments { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize) !Arguments { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -151,24 +122,9 @@ pub const Exe = struct { var context: LocalContext = .{ .self = self, .current_index = offset + self.baked_count }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *const Buffer) void { - stdx.debug.assert( - context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), - "Expected argument {} to have shape {f}, got {f}", - .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }, - ); - - const expected = context_.self.flat_buffers.num_devices; - const shard_count = buffer._shards.len; - - stdx.debug.assert( - shard_count == expected, - "Argument {} has {d} shards but executable expects {d}", - .{ context_.current_index, shard_count, expected }, - ); - - for (0..expected) |device_index| { - context_.self.flat_buffers.buffers[device_index][context_.current_index] = - buffer._shards.get(device_index); + stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected argument {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + for (0..context_.self.flat_buffers.num_devices) |device_index| { + context_.self.flat_buffers.buffers[device_index][context_.current_index] = buffer._shards.get(device_index); } context_.current_index += 1; @@ -207,7 +163,7 @@ pub const Exe = struct { expected_shapes: []const Shape, - pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, platform: *const Platform, num_devices: usize) !Results { + pub fn init(allocator: std.mem.Allocator, shapes: []const Shape, num_devices: usize, platform: *const Platform) !Results { const flat_buffers = try FlatBuffers.init(allocator, shapes.len, num_devices); errdefer flat_buffers.deinit(allocator); @@ -235,7 +191,7 @@ pub const Exe = struct { var context: LocalContext = .{ .self = self, .current_index = 0 }; meta.visit(struct { fn cb(context_: *LocalContext, buffer: *Buffer) void { - var shards: Buffer.Shards = .empty; + var shards: Buffer.Shards = .{}; for (0..context_.self.flat_buffers.num_devices) |device_index| { shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); } @@ -248,457 +204,62 @@ pub const Exe = struct { pub fn fill(self: *Results, v: anytype) void { const LocalContext = struct { - results: *Results, + self: *Results, current_index: usize = 0, }; - var context: LocalContext = .{ .results = self, .current_index = 0 }; + var context: LocalContext = .{ .self = self, .current_index = 0 }; meta.visit(struct { - fn cb(ctx: *LocalContext, buffer: *Buffer) void { - //stdx.debug.assert(ctx.results.expected_shapes[ctx.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ ctx.current_index, ctx.results.expected_shapes[ctx.current_index], buffer.shape() }); - var shards: Buffer.Shards = .empty; - for (0..ctx.results.flat_buffers.num_devices) |device_index| { - shards.appendAssumeCapacity(ctx.results.flat_buffers.buffers[device_index][ctx.current_index]); + fn cb(context_: *LocalContext, buffer: *Buffer) void { + //stdx.debug.assert(context_.self.expected_shapes[context_.current_index].eql(buffer.shape()), "Expected result {} to have shape {f}, got {f}", .{ context_.current_index, context_.self.expected_shapes[context_.current_index], buffer.shape() }); + var shards: Buffer.Shards = .{}; + for (0..context_.self.flat_buffers.num_devices) |device_index| { + shards.appendAssumeCapacity(context_.self.flat_buffers.buffers[device_index][context_.current_index]); } - buffer.* = Buffer.fromPjrtBuffers(ctx.results.platform, ctx.results.expected_shapes[ctx.current_index], shards.constSlice()); - ctx.current_index += 1; + buffer.* = Buffer.fromPjrtBuffers(context_.self.platform, context_.self.expected_shapes[context_.current_index], shards.constSlice()); + context_.current_index += 1; } }.cb, &context, &v); } }; - /// Executes a single-output executable and waits for its result. - /// The caller owns the returned buffer. Donated inputs are consumed. - pub fn eval(self: *const Exe, allocator: std.mem.Allocator, io: std.Io, input_values: anytype) !Buffer { - stdx.debug.assert(self.output_shapes.len == 1, "eval expects one output, got {d}; use a runner for multiple outputs", .{self.output_shapes.len}); - var result: Buffer = undefined; - var r = try self.runner(allocator); - defer r.deinit(allocator); - r.run(io, input_values, .{&result}, .{ .wait = true }); - return result; - } - - pub fn runner(self: *const Exe, allocator: std.mem.Allocator) !Runner { - return .init(self, allocator); - } - - /// Reusable argument and result storage for a borrowed executable. - /// The executable must outlive the runner, and the runner must not be used concurrently. - pub const Runner = struct { - exe: *const Exe, - args: Arguments, - results: Results, - - pub fn init(exe: *const Exe, allocator: std.mem.Allocator) !Runner { - var arguments = try exe.args(allocator); - errdefer arguments.deinit(allocator); - var results_ = try exe.results(allocator); - errdefer results_.deinit(allocator); - return .{ - .exe = exe, - .args = arguments, - .results = results_, - }; - } - - pub fn deinit(self: *Runner, allocator: std.mem.Allocator) void { - self.results.deinit(allocator); - self.args.deinit(allocator); - } - - /// Destroys donated input handles and fills the output destinations. - /// Use the donated buffer's address as its output destination to replace it safely. - /// Non-donated inputs remain owned by the caller. - pub fn run(self: *Runner, io: std.Io, input_values: anytype, output_refs: anytype, opts: CallOpts) void { - self.args.set(input_values); - self.exe.call(io, self.args, &self.results, opts); - self.exe.deinitDonatedInputs(self.args); - self.results.fill(output_refs); - } - }; - - pub const CallOpts = struct { - /// Waits for execution to complete before returning. - wait: bool = false, - }; - - fn deinitDonatedInputs(self: *const Exe, arguments: Arguments) void { - // Execution deletes donated inputs, but leaves their PJRT handles allocated. - // Arguments retain the old handles even when outputs replace the caller's buffers. - for (self.donated_input_indices) |input_index| { - for (arguments.flat_buffers.buffers[0..self.num_devices]) |device_buffers| { - device_buffers[input_index].deinit(self.platform.pjrt_api); - } - } - } - - /// Executes with optional waiting. The caller owns all input handles, including donated inputs. - /// Donated input contents cannot be used after the call; output buffers have their own handles. - /// Use a runner to destroy donated handles automatically and replace buffers through output destinations. - pub fn call(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { - var span = tracer.span("zml.exe.call", .{ - .wait = opts.wait, - .arg_count = arguments.expected_shapes.len, - .result_count = results_.expected_shapes.len, - }); - defer span.end(); - - var events: [Platform.MAX_NUM_DEVICES]?*pjrt.Event = @splat(null); - - const partition_events = events[0..@intCast(self.num_partitions)]; - const events_slice: ?[]?*pjrt.Event = switch (self.platform.target) { - .neuron => partition_events, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) partition_events else null, - }; + pub fn internalCall(self: *const Exe, io: ?std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + stdx.debug.assert(opts.wait == false or io != null, "io should not be null when waiting for execution completion", .{}); + var events = [_]?*pjrt.Event{null} ** Platform.MAX_NUM_DEVICES; + const sharding = self.platform.sharding(); + const events_slice: ?[]?*pjrt.Event = if (opts.wait) events[0..sharding.num_partitions] else null; self.exe.execute(self.platform.pjrt_api, .{ .arguments = arguments.flat_buffers.buffers, .num_args = arguments.expected_shapes.len, .results = results_.flat_buffers.buffers, .events = events_slice, - // TODO: Set this flag for undonatable buffers. + // this allows to tell a specific buffer shouldn't be donated, + // even if it has been marked as "can be donated" during compilation. + // TODO: expose it ? .non_donatable_input_indices = &.{}, .context = self.context, }) catch |err| { std.debug.panic("PJRT_LoadedExecutable_Execute failed with: {}", .{err}); }; - switch (self.platform.target) { - .neuron => { - for (events_slice.?) |e| { - if (e) |ev| { - if (opts.wait) { - ev.await(self.platform.pjrt_api, io) catch |err| { - std.debug.panic("PJRT execution failed with: {}", .{err}); - }; - } - ev.deinit(self.platform.pjrt_api); - } + if (opts.wait) { + for (events_slice.?) |e| { + if (e) |ev| { + ev.await(self.platform.pjrt_api, io.?) catch unreachable; } - }, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => if (opts.wait) { - for (events_slice.?) |e| { - if (e) |ev| { - ev.await(self.platform.pjrt_api, io) catch |err| { - std.debug.panic("PJRT execution failed with: {}", .{err}); - }; - ev.deinit(self.platform.pjrt_api); - } - } - }, + } } } - pub fn memoryStats(self: *const Exe) pjrt.ApiError!pjrt.CompiledMemoryStats { - const exe = try self.exe.executable(self.platform.pjrt_api); - return try exe.getCompiledMemoryStats(self.platform.pjrt_api); + pub const CallOpts = struct { + wait: bool = false, + }; + + pub fn callOpts(self: *const Exe, io: std.Io, arguments: Arguments, results_: *Results, opts: CallOpts) void { + return self.internalCall(io, arguments, results_, opts); + } + + pub fn call(self: *const Exe, arguments: Arguments, results_: *Results) void { + return self.internalCall(null, arguments, results_, .{}); } }; - -/// A typed executable whose input and output buffer structures are derived from -/// the function used to compile it. -pub fn FnExe(comptime function_: anytype) type { - const function_info = switch (@typeInfo(@TypeOf(function_))) { - .@"fn" => |info| info, - else => @compileError("FnExe expects a function, got " ++ @typeName(@TypeOf(function_))), - }; - if (function_info.attrs.varargs or function_info.param_types.len != 1) { - @compileError("FnExe function must accept exactly one input struct"); - } - - const FunctionInput = typedFunctionInput(function_); - const FunctionOutput = typedFunctionOutput(function_); - - return struct { - raw: Exe, - - pub const Input = mem.Bufferized(FunctionInput); - pub const Output = typedOutputDestinations(mem.Bufferized(FunctionOutput)); - - const Self = @This(); - - pub fn init(raw: Exe) Self { - return .{ .raw = raw }; - } - - pub fn compile( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - opts: Compiler.Options, - args: std.meta.ArgsTuple(@TypeOf(function_)), - ) Compiler.Error!Self { - return .{ .raw = try Compiler.Typed(function_).compile(allocator, io, platform, opts, args) }; - } - - pub fn deinit(self: *const Self) void { - self.raw.deinit(); - } - - pub fn runner(self: *const Self, allocator: std.mem.Allocator) !Runner(.{}) { - return .init(self, allocator, .{}); - } - - /// Reusable argument and result storage for a borrowed executable. - /// The executable must outlive the runner, and the runner must not be used concurrently. - pub fn Runner(comptime baked_fields: anytype) type { - const count = countBackedFields(Input, baked_fields); - - return struct { - exe: *const Exe, - args: Exe.Arguments, - results: Exe.Results, - - const RunnerSelf = @This(); - - pub const BakedInput = structFieldRange(Input, 0, count); - pub const NonBakedInput = structFieldRange(Input, count, @typeInfo(Input).@"struct".field_names.len); - - pub fn init(exe: *const Self, allocator: std.mem.Allocator, baked: BakedInput) !RunnerSelf { - var arguments = try exe.raw.args(allocator); - errdefer arguments.deinit(allocator); - var results = try exe.raw.results(allocator); - errdefer results.deinit(allocator); - arguments.bake(baked); - return .{ .exe = &exe.raw, .args = arguments, .results = results }; - } - - pub fn deinit(self: *RunnerSelf, allocator: std.mem.Allocator) void { - self.results.deinit(allocator); - self.args.deinit(allocator); - } - - pub const Call = struct { - inputs: NonBakedInput, - outputs: Output, - }; - - /// Destroys donated input handles before filling outputs; non-donated inputs remain caller-owned. - pub fn run(self: *RunnerSelf, io: std.Io, call: Call, opts: Exe.CallOpts) void { - self.args.set(call.inputs); - self.exe.call(io, self.args, &self.results, opts); - self.exe.deinitDonatedInputs(self.args); - self.results.fill(call.outputs); - } - }; - } - }; -} - -fn countBackedFields(comptime Input: type, comptime baked_fields: anytype) usize { - const baked_info = switch (@typeInfo(@TypeOf(baked_fields))) { - .@"struct" => |info| info, - else => @compileError("FnExe baked fields must be a tuple of enum literals"), - }; - if (!baked_info.is_tuple) { - @compileError("FnExe baked fields must be a tuple of enum literals"); - } - - const input_info = @typeInfo(Input).@"struct"; - if (baked_info.field_names.len > input_info.field_names.len) { - @compileError("FnExe baked fields must be a prefix of its bufferized input fields"); - } - - inline for (baked_info.field_names, 0..) |tuple_field_name, i| { - const baked_field = @field(baked_fields, tuple_field_name); - if (@TypeOf(baked_field) != @EnumLiteral()) { - @compileError("FnExe baked fields must be enum literals"); - } - if (!std.mem.eql(u8, @tagName(baked_field), input_info.field_names[i])) { - @compileError("FnExe baked fields must be an ordered prefix of its bufferized input fields"); - } - } - return baked_info.field_names.len; -} - -fn structFieldRange(comptime Struct: type, comptime start: usize, comptime end: usize) type { - const info = @typeInfo(Struct).@"struct"; - return @Struct(.auto, null, info.field_names[start..end], info.field_types[start..end], info.field_attrs[start..end]); -} - -fn typedFunctionInput(comptime function_: anytype) type { - const Input = @typeInfo(@TypeOf(function_)).@"fn".param_types[0] orelse - @compileError("FnExe function must have a concrete input type"); - validateFnExeStruct("input", Input); - return Input; -} - -fn typedFunctionOutput(comptime function_: anytype) type { - const Output = @typeInfo(@TypeOf(function_)).@"fn".return_type orelse @compileError("FnExe function must return an output struct"); - validateFnExeStruct("output", Output); - return Output; -} - -fn validateFnExeStruct(comptime role: []const u8, comptime T: type) void { - const info = switch (@typeInfo(T)) { - .@"struct" => |info| info, - else => @compileError("FnExe " ++ role ++ " must be a struct, got " ++ @typeName(T)), - }; - if (info.is_tuple) { - @compileError("FnExe " ++ role ++ " must use named fields"); - } -} - -fn typedOutputDestinations(comptime BufferizedOutput: type) type { - const output_info = @typeInfo(BufferizedOutput).@"struct"; - var field_types: [output_info.field_names.len]type = undefined; - var field_attrs: [output_info.field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; - for (&field_types, &field_attrs, output_info.field_types) |*T, *attrs, Field| { - T.* = *Field; - attrs.* = .{ .@"align" = @alignOf(*Field) }; - } - return @Struct(.auto, null, output_info.field_names, &field_types, &field_attrs); -} - -test "FnExe derives baked and runtime calls from a function" { - const Inputs = struct { - weights: struct { - tensor: Tensor, - scale: f32, - }, - bias: Tensor, - tokens: Tensor, - token_count: usize, - }; - const Outputs = struct { - hidden: Tensor, - cache: struct { - key: Tensor, - length: usize, - }, - metadata: usize, - }; - const Functions = struct { - fn forward(inputs: Inputs) Outputs { - _ = inputs; - return undefined; - } - }; - - const Model = FnExe(Functions.forward); - try std.testing.expect(@hasField(Model.Input, "weights")); - try std.testing.expect(@hasField(Model.Input, "bias")); - try std.testing.expect(@hasField(Model.Input, "tokens")); - try std.testing.expect(!@hasField(Model.Input, "token_count")); - try std.testing.expect(!@hasField(@FieldType(Model.Input, "weights"), "scale")); - - const Baked = Model.Runner(.{ .weights, .bias }).BakedInput; - try std.testing.expect(@hasField(Baked, "weights")); - try std.testing.expect(@hasField(Baked, "bias")); - try std.testing.expect(!@hasField(Baked, "tokens")); - - const Runtime = Model.Runner(.{ .weights, .bias }).NonBakedInput; - try std.testing.expect(!@hasField(Runtime, "weights")); - try std.testing.expect(!@hasField(Runtime, "bias")); - try std.testing.expectEqual(Buffer, @FieldType(Runtime, "tokens")); - - try std.testing.expectEqual(*Buffer, @FieldType(Model.Output, "hidden")); - try std.testing.expectEqual( - *mem.Bufferized(@FieldType(Outputs, "cache")), - @FieldType(Model.Output, "cache"), - ); - try std.testing.expect(!@hasField(Model.Output, "metadata")); -} - -test "Exe raw calls leave donated input handle cleanup to the caller" { - const platform = @import("testing.zig").env(); - const allocator = std.testing.allocator; - const io = std.testing.io; - const Functions = struct { - fn forward(value: Tensor, weight: Tensor) Tensor { - return value.add(weight).reuseBuffer(value); - } - }; - const value: Tensor = .init(.{2}, .i32); - const weight: Tensor = .init(.{2}, .i32); - const exe = try platform.compileFn(allocator, io, Functions.forward, .{ value, weight }, .{}); - defer exe.deinit(); - try std.testing.expectEqualSlices(u32, &.{0}, exe.donated_input_indices); - - var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); - defer weight_buffer.deinit(); - var args = try exe.args(allocator); - defer args.deinit(allocator); - var results = try exe.results(allocator); - defer results.deinit(allocator); - - for ([_]bool{ false, true }) |wait| { - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); - defer value_buffer.deinit(); - args.set(.{ value_buffer, weight_buffer }); - exe.call(io, args, &results, .{ .wait = wait }); - var output = results.get(Buffer); - defer output.deinit(); - - // The donated handle is still valid for cleanup even though its contents are deleted. - for (value_buffer._shards.constSlice()) |shard| { - try std.testing.expect(shard.isDeleted(platform.pjrt_api)); - } - value_buffer.deinit(); - try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); - try std.testing.expectEqual([2]i32{ 10, 20 }, try weight_buffer.getValue([2]i32, io)); - } -} - -test "FnExe runner replaces donated buffers with and without waiting" { - const platform = @import("testing.zig").env(); - const allocator = std.testing.allocator; - const io = std.testing.io; - const Functions = struct { - fn forward(inputs: struct { weight: Tensor, value: Tensor }) struct { value: Tensor } { - return .{ .value = inputs.value.add(inputs.weight).reuseBuffer(inputs.value) }; - } - }; - const Model = FnExe(Functions.forward); - const weight: Tensor = .init(.{2}, .i32); - const value: Tensor = .init(.{2}, .i32); - const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .weight = weight, .value = value }}); - defer exe.deinit(); - - var weight_buffer = try Buffer.fromBytes(io, platform, weight.shape(), std.mem.asBytes(&[2]i32{ 10, 20 })); - defer weight_buffer.deinit(); - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); - defer value_buffer.deinit(); - var runner = try Model.Runner(.{.weight}).init(&exe, allocator, .{ .weight = weight_buffer }); - defer runner.deinit(allocator); - - runner.run(io, .{ - .inputs = .{ .value = value_buffer }, - .outputs = .{ .value = &value_buffer }, - }, .{}); - try std.testing.expectEqual([2]i32{ 11, 22 }, try value_buffer.getValue([2]i32, io)); - - runner.run(io, .{ - .inputs = .{ .value = value_buffer }, - .outputs = .{ .value = &value_buffer }, - }, .{ .wait = true }); - try std.testing.expectEqual([2]i32{ 21, 42 }, try value_buffer.getValue([2]i32, io)); -} - -test "FnExe runner replaces unchanged donated buffers" { - const platform = @import("testing.zig").env(); - const allocator = std.testing.allocator; - const io = std.testing.io; - const Functions = struct { - fn forward(inputs: struct { value: Tensor }) struct { value: Tensor } { - return .{ .value = inputs.value.reuseBuffer(inputs.value) }; - } - }; - const Model = FnExe(Functions.forward); - const value: Tensor = .init(.{2}, .i32); - const exe = try Model.compile(allocator, io, platform, .{}, .{.{ .value = value }}); - defer exe.deinit(); - try std.testing.expectEqualSlices(u32, &.{0}, exe.raw.donated_input_indices); - - var value_buffer = try Buffer.fromBytes(io, platform, value.shape(), std.mem.asBytes(&[2]i32{ 1, 2 })); - defer value_buffer.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - - for ([_]bool{ false, true }) |wait| { - runner.run(io, .{ - .inputs = .{ .value = value_buffer }, - .outputs = .{ .value = &value_buffer }, - }, .{ .wait = wait }); - try std.testing.expectEqual([2]i32{ 1, 2 }, try value_buffer.getValue([2]i32, io)); - } -} diff --git a/zml/floats.zig b/zml/floats.zig index 689c22a8..538722cd 100644 --- a/zml/floats.zig +++ b/zml/floats.zig @@ -8,8 +8,8 @@ fn FloatConversionHelpers(Float: type) type { @compileError(err_msg); } comptime { - for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { - if (!std.mem.eql(u8, field_name, expected_name)) + for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { + if (!std.mem.eql(u8, field.name, expected_name)) @compileError(err_msg); } } @@ -20,7 +20,7 @@ fn FloatConversionHelpers(Float: type) type { const mantissa_bits: u8 = @typeInfo(@FieldType(Float, "mantissa")).int.bits; const exponent_bits: u8 = @typeInfo(@FieldType(Float, "exponent")).int.bits; const f32_mantissa_bits: u8 = @typeInfo(@FieldType(Float32, "mantissa")).int.bits; - const exp_bias: i16 = std.math.maxInt(@Int(.unsigned, exponent_bits - 1)); + const exp_bias: i16 = std.math.maxInt(std.meta.Int(.unsigned, exponent_bits - 1)); const exp_off: u8 = FloatConversionHelpers(Float32).exp_bias - exp_bias; /// Lossy conversion from f32, similar to @floatCast @@ -29,15 +29,16 @@ fn FloatConversionHelpers(Float: type) type { const vf32: Float32 = @bitCast(f); const exponent: i16 = @as(i16, vf32.exponent) - exp_off; - const overflow = std.math.isInf(f) or exponent > std.math.maxInt(@FieldType(Float, "exponent")); + const overflow = exponent > std.math.maxInt(@FieldType(Float, "exponent")); if (overflow) { @branchHint(.unlikely); return if (@hasDecl(Float, "inf")) if (vf32.sign == 0) Float.inf else Float.minus_inf else if (@hasDecl(Float, "nan")) Float.nan - else - std.debug.panic("{} doesn't have infinite representations", .{Float}); + else b: { + break :b if (vf32.sign == 0) Float.max else Float.min; + }; } return if (exponent <= 0) @@ -106,8 +107,8 @@ fn FloatHelpers(Float: type) type { @compileError(err_msg); } comptime { - for (info.@"struct".field_names, &.{ "mantissa", "exponent", "sign" }) |field_name, expected_name| { - if (!std.mem.eql(u8, field_name, expected_name)) + for (info.@"struct".fields, &.{ "mantissa", "exponent", "sign" }) |field, expected_name| { + if (!std.mem.eql(u8, field.name, expected_name)) @compileError(err_msg); } } @@ -270,8 +271,10 @@ test "Float8E4" { }) |Float8T| { try testCustomFloat(Float8T, test_case_e4); try std.testing.expectEqual(0.0, Float8T.fromF32(1.0 / 2048.0).toF32()); - - if (@hasDecl(Float8T, "inf")) @compileError("Float8E4 don't have ±∞ representations"); + if (@hasDecl(Float8T, "inf")) { + try std.testing.expectEqual(Float8T.inf, Float8T.fromF32(128.0)); + try std.testing.expectEqual(Float8T.inf.neg(), Float8T.fromF32(-128.0)); + } } } @@ -502,25 +505,6 @@ pub const Float4E2M1 = packed struct(u4) { } try std.testing.expectEqualSlices(u4, &.{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, @ptrCast(&from_f32_res)); } - - pub const Packed = packed struct(@Int(.unsigned, @bitSizeOf(Float4E2M1) * 2)) { - x: Float4E2M1, - y: Float4E2M1, - - pub fn fromF32(x: f32, y: f32) Packed { - return .{ .x = .fromF32(x), .y = .fromF32(y) }; - } - - pub fn toF32(xy: Packed) [2]f32 { - return .{ xy.x.toF32(), xy.y.toF32() }; - } - - pub fn formatNumber(xy: Packed, w: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { - try xy.x.formatNumber(w, n); - try w.writeByte(','); - try xy.y.formatNumber(w, n); - } - }; }; pub fn floatCast(T: type, x: anytype) T { @@ -544,7 +528,7 @@ pub fn isInf(x: anytype) bool { } if (!@hasDecl(Float, "inf")) return false; - const FBits = @Int(.unsigned, @bitSizeOf(Float)); + const FBits = std.meta.Int(.unsigned, @bitSizeOf(Float)); const remove_sign = ~@as(FBits, 0) >> 1; return @as(FBits, @bitCast(x)) & remove_sign == @as(FBits, @bitCast(Float.inf)); } diff --git a/zml/io.zig b/zml/io.zig index ab8d8e6e..404843d2 100644 --- a/zml/io.zig +++ b/zml/io.zig @@ -2,64 +2,48 @@ const std = @import("std"); const pjrt = @import("pjrt"); const stdx = @import("stdx"); -pub const VFS = @import("vfs"); +pub const VFS = @import("io").VFS; const Buffer = @import("buffer.zig").Buffer; const Bufferized = @import("zml.zig").Bufferized; -const CreateOptions = @import("platform.zig").CreateOptions; -const Exe = @import("exe.zig").Exe; const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); const pjrtx = @import("pjrtx.zig"); const Platform = @import("platform.zig").Platform; -const tracer = @import("profiling/tracer.zig"); const safetensors = @import("safetensors.zig"); const Shape = @import("shape.zig").Shape; -const Sharding = @import("Sharding.zig"); -const Placement = Sharding.Placement; -const Slice = @import("slice.zig").Slice; const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/io"); pub const TensorStore = struct { - pub const Binding = struct { - tensors: []*safetensors.Tensor, - transformed: bool, - memory: Memory.Kind = .default, - }; - registry: *safetensors.TensorRegistry, - meshes: []const *const Sharding.Mesh, - id_to_sources: std.AutoHashMapUnmanaged(Tensor.Id, Binding), + id_map: std.AutoHashMapUnmanaged(usize, *safetensors.Tensor), allocator: std.mem.Allocator, - arena: std.heap.ArenaAllocator, - pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry, meshes: []const *const Sharding.Mesh) TensorStore { - var arena: std.heap.ArenaAllocator = .init(allocator); + pub fn fromRegistry(allocator: std.mem.Allocator, registry: *safetensors.TensorRegistry) TensorStore { return .{ .registry = registry, - .meshes = arena.allocator().dupe(*const Sharding.Mesh, meshes) catch @panic("OOM"), - .id_to_sources = .empty, + .id_map = .empty, .allocator = allocator, - .arena = arena, }; } pub fn deinit(self: *TensorStore) void { - self.id_to_sources.deinit(self.allocator); - self.arena.deinit(); + self.id_map.deinit(self.allocator); } - fn putSourcesNoClobber(self: *TensorStore, id: Tensor.Id, sources: Binding) std.mem.Allocator.Error!void { - const gop = try self.id_to_sources.getOrPut(self.allocator, id); - if (gop.found_existing) { - stdx.debug.panic("Id {} already has associated sources", .{id}); - } - errdefer self.id_to_sources.removeByPtr(gop.key_ptr); + fn bindIdToKey(self: *TensorStore, key: []const u8, id: usize) !void { + const tensor_desc_ptr = self.registry.tensors.getPtr(key).?; - gop.value_ptr.* = sources; + const gop = try self.id_map.getOrPut(self.allocator, id); + if (gop.found_existing) { + stdx.debug.panic("Key {s} already has an associated tensor (id: {})", .{ key, gop.key_ptr.* }); + } + errdefer self.id_map.removeByPtr(gop.key_ptr); + + gop.value_ptr.* = tensor_desc_ptr; } fn getPtrFromKey(self: *const TensorStore, key: []const u8) ?*safetensors.Tensor { @@ -67,19 +51,9 @@ pub const TensorStore = struct { return tensor_desc_ptr; } - fn dupeSource(self: *TensorStore, key: []const u8) ?*safetensors.Tensor { - const entry = self.getPtrFromKey(key) orelse return null; - - const copy = self.arena.allocator().create(safetensors.Tensor) catch @panic("OOM"); - copy.* = entry.*; - - return copy; - } - fn getPtrFromId(self: *const TensorStore, id: usize) ?*safetensors.Tensor { - const sources = self.id_to_sources.get(id) orelse return null; - stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); - return sources.tensors[0]; + const tensor_desc_ptr = self.id_map.get(id) orelse return null; + return tensor_desc_ptr; } pub fn getReader(self: *const TensorStore, key: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { @@ -87,39 +61,15 @@ pub const TensorStore = struct { } pub fn getReaderById(self: *const TensorStore, id: usize, io: std.Io, buffer: []u8) !safetensors.TensorReader { - const sources = self.id_to_sources.get(id) orelse return error.NotFound; - stdx.debug.assert(sources.tensors.len == 1, "Expect tensor with id {} to have only one source, got {}", .{ id, sources.tensors.len }); + const tensor_desc = self.id_map.get(id) orelse return error.NotFound; - return sources.tensors[0].reader(io, buffer, .{}); - } - - pub fn getSourcesById(self: *const TensorStore, id: Tensor.Id) ?Binding { - return self.id_to_sources.get(id); - } - - pub fn getShape(self: *const TensorStore, key: []const u8) ?Shape { - const entry_ptr = self.getPtrFromKey(key) orelse return null; - return entry_ptr.shape; + return safetensors.TensorReader.init(io, tensor_desc.*, buffer); } pub fn view(self: *TensorStore) View { return .{ .store = self }; } - pub fn mesh(store: *const TensorStore, name: @EnumLiteral()) *const Sharding.Mesh { - const name_slice = @tagName(name); - for (store.meshes) |m| { - if (std.mem.eql(u8, name_slice, m.name)) { - return m; - } - } - std.debug.panic( - \\Found no meshes named {s} in TensorStore. - \\Try passing more meshes to `zml.TensorStore.fromRegistry`. - \\Known meshes: {f} - , .{ name_slice, stdx.fmt.slice(store.meshes) }); - } - pub const View = struct { store: *TensorStore, @@ -146,7 +96,7 @@ pub const TensorStore = struct { pub fn withPrefix(self: *const View, prefix_: []const u8) View { var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }); + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{s}.", .{ self.prefix() orelse "", prefix_ }) catch unreachable; return .{ .store = self.store, @@ -157,7 +107,7 @@ pub const TensorStore = struct { pub fn withLayer(self: *const View, index: usize) View { var buffer: [256]u8 = undefined; - const new_prefix = makeKey(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }); + const new_prefix = std.fmt.bufPrint(&buffer, "{s}{d}.", .{ self.prefix() orelse "", index }) catch unreachable; return .{ .store = self.store, @@ -166,115 +116,56 @@ pub const TensorStore = struct { }; } - pub fn prefix(self: *const View) ?[]const u8 { + fn prefix(self: *const View) ?[]const u8 { return if (self.prefix_length == 0) null else self.prefix_buffer[0..self.prefix_length]; } pub fn hasKey(self: *const View, subkey: []const u8) bool { var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; return for (self.store.registry.tensors.keys()) |k| { if (std.mem.startsWith(u8, k, key)) break true; } else false; } - /// Creates a zml.Tensor from a specific entry in the store. - pub fn createTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { - return self.maybeCreateTensor(subkey, tags, mesh_, partitioning) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - pub fn maybeCreateTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) ?Tensor { - const parsed_tags: ?[]const Shape.Tag = comptime parseTags(tags); - if (mesh_ == .replicated) { - return self.maybeCreateTensorInternal(subkey, parsed_tags, .replicated); - } - - const resolved_mesh = self.store.mesh(mesh_); - if (@TypeOf(partitioning) == Sharding.Partitioning) { - return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partitioning }); - } - // Unspecified tensor axes are treated as replicated - const partition: Sharding.Partitioning = if (parsed_tags) |p_tags| - .parse(p_tags, resolved_mesh, .replicated, partitioning) - else - partitioning; - return self.maybeCreateTensorInternal(subkey, parsed_tags, .{ .mesh = resolved_mesh, .partition = partition }); - } - - pub fn createHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), partitioning: anytype) Tensor { - return self.maybeCreateHostPinnedTensor(subkey, tags, mesh_, partitioning) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - pub fn maybeCreateHostPinnedTensor(self: View, subkey: []const u8, tags: anytype, mesh_: @EnumLiteral(), comptime partitioning: anytype) ?Tensor { - const tensor = self.maybeCreateTensor(subkey, comptime parseTags(tags), mesh_, partitioning); - if (tensor) |t| { - const storage = self.store.id_to_sources.getPtr(t.id); - storage.?.memory = .host_pinned; - } - return tensor; - } - - pub fn createReplicatedTensor(self: View, subkey: []const u8, tags: anytype) Tensor { - return self.maybeCreateTensorInternal(subkey, comptime parseTags(tags), .replicated) orelse - stdx.debug.panic("Checkpoint has no tensor named {s}{s}", .{ self.prefix() orelse "", subkey }); - } - - fn maybeCreateTensorInternal(self: View, subkey: []const u8, tags: ?[]const Shape.Tag, sharding: Sharding) ?Tensor { + pub fn maybeCreateTensor(self: View, subkey: []const u8) ?Tensor { var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - const source = self.store.dupeSource(key) orelse return null; + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - const sources = self.store.arena.allocator().alloc(*safetensors.Tensor, 1) catch |e| std.debug.panic("Not handling {} errors", .{e}); - errdefer self.store.arena.allocator().free(sources); - sources[0] = source; + const ptr = self.store.getPtrFromKey(key) orelse return null; - var shape = source.shape; - if (tags) |user_tags| { - stdx.debug.assert(user_tags.len == shape.rank(), "tensor {s} from store has shape {f}, but `createTensor` got only {d} tags: {f}", .{ key, source.shape, user_tags.len, stdx.fmt.stringsZ(user_tags) }); - @memcpy(shape._tags.slice(), user_tags); - } - shape._sharding = sharding; - if (sharding.mesh) |mesh_| shape = shape.withPartitioning(mesh_, sharding.partition); - - const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = sources, .transformed = false, .memory = .default }) catch |e| std.debug.panic("Not handling {} errors", .{e}); + const tensor: Tensor = .fromShape(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; return tensor; } - /// Accepts "null" as tags - fn parseTags(tags: anytype) ?[]const Shape.Tag { - return if (@TypeOf(tags) == @TypeOf(null)) null else Shape.parseTags(tags); + pub fn createTensor(self: View, subkey: []const u8) Tensor { + return self.maybeCreateTensor(subkey).?; } - pub fn maybeCreateBinding(self: View, sources: []const []const u8, shape: Shape) ?Tensor { - const arena = self.store.arena.allocator(); - - var tensor_list = std.ArrayList(*safetensors.Tensor).initCapacity(arena, sources.len) catch |e| std.debug.panic("Not handling {} errors", .{e}); - defer tensor_list.deinit(arena); - + pub fn maybeCreateTensorWithTags(self: View, subkey: []const u8, tagz: anytype) ?Tensor { var buffer: [256]u8 = undefined; - for (sources) |subkey| { - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - const tensor = self.store.dupeSource(key) orelse return null; - tensor_list.appendAssumeCapacity(tensor); - } + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; - const tensors = tensor_list.toOwnedSlice(arena) catch unreachable; - errdefer arena.free(tensors); + const ptr = self.store.getPtrFromKey(key) orelse return null; + ptr.shape = ptr.shape.withTags(tagz); - const tensor: Tensor = .fromShape(shape); - self.store.putSourcesNoClobber(tensor.id, .{ .tensors = tensors, .transformed = true }) catch |e| std.debug.panic("Not handling {} errors", .{e}); + const tensor: Tensor = .fromShape(ptr.shape); + self.store.bindIdToKey(key, tensor.id) catch unreachable; return tensor; } + pub fn createTensorWithTags(self: View, subkey: []const u8, tagz: anytype) Tensor { + return self.maybeCreateTensorWithTags(subkey, tagz).?; + } + pub fn getShape(self: View, subkey: []const u8) ?Shape { var buffer: [256]u8 = undefined; - const key = makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); - return self.store.getShape(key); + const key = std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; } pub fn getShapeOpts(self: View, subkey: []const u8, opts: struct { no_prefix: bool = false }) ?Shape { @@ -282,14 +173,15 @@ pub const TensorStore = struct { const key = if (opts.no_prefix) subkey else b: { - break :b makeKey(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + break :b std.fmt.bufPrint(&buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; }; - return self.store.getShape(key); + const entry_ptr = self.store.getPtrFromKey(key) orelse return null; + return entry_ptr.shape; } pub fn getReader(self: View, subkey: []const u8, io: std.Io, buffer: []u8) !safetensors.TensorReader { var key_buffer: [256]u8 = undefined; - const key = makeKey(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }); + const key = std.fmt.bufPrint(&key_buffer, "{s}{s}", .{ self.prefix() orelse "", subkey }) catch unreachable; return self.store.getReader(key, io, buffer); } @@ -305,300 +197,9 @@ pub const TensorStore = struct { } return count_; } - - fn makeKey(buffer: []u8, comptime fmt: []const u8, args: anytype) []const u8 { - const key = std.fmt.bufPrint(buffer, fmt, args) catch - std.debug.panic("Expected key to be less than {} characters", .{buffer.len}); - return key; - } }; }; -pub const Loader = struct { - allocator: std.mem.Allocator, - platform: *const Platform, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - pinned_buffer_pools: []mem.DynamicBufferPool, - group: stdx.Io.LimitedGroup, - bytes_loaded: std.atomic.Value(usize) = .init(0), - delivered: std.AutoHashMapUnmanaged(Tensor.Id, void) = .empty, - - pub const Opts = struct { - pub const default: Opts = .{ - .parallelism = 1, - .dma_chunks = 2, - .dma_chunk_size = 4096, - }; - parallelism: usize, - dma_chunks: usize, - dma_chunk_size: usize, - }; - - pub fn init(allocator: std.mem.Allocator, platform: *const Platform, opts: Opts) error{OutOfMemory}!Loader { - const pool_count = platform.devices.len; - const dma_allocators = try allocator.alloc(mem.DmaAllocator, pool_count); - errdefer allocator.free(dma_allocators); - for (platform.devices, 0..) |*device, i| { - dma_allocators[i] = .init(allocator, device); - } - - const buffer_pools = try allocator.alloc(mem.DynamicBufferPool, pool_count); - errdefer allocator.free(buffer_pools); - for (buffer_pools) |*pool_| { - pool_.* = .init(opts.dma_chunks, opts.dma_chunk_size); - } - errdefer for (buffer_pools, 0..) |*pool_, i| { - pool_.deinit(dma_allocators[i].allocator()); - }; - - return .{ - .allocator = allocator, - .platform = platform, - .dma_allocators = dma_allocators, - .dma_chunk_size = opts.dma_chunk_size, - .pinned_buffer_pools = buffer_pools, - .group = .init(opts.parallelism), - }; - } - - pub fn deinit(self: *Loader) void { - self.delivered.deinit(self.allocator); - for (self.pinned_buffer_pools, 0..) |*pool, i| pool.deinit(self.dma_allocators[i].allocator()); - self.allocator.free(self.pinned_buffer_pools); - self.allocator.free(self.dma_allocators); - } - - pub fn await(self: *Loader, io: std.Io) std.Io.Cancelable!void { - return self.group.await(io); - } - - pub const LoadOpts = struct { - progress: ?*std.Progress.Node = null, - }; - - pub const LoadError = error{TransformedTensorNotDelivered}; - - pub fn load(self: *Loader, io: std.Io, comptime T: type, model: *const T, buffers: *Bufferized(T), store: *const TensorStore, opts: LoadOpts) LoadError!void { - const tensor_count = meta.count(Tensor, model); - - var arena: std.heap.ArenaAllocator = .init(self.allocator); - defer arena.deinit(); - - const flattened_buffers = arena.allocator().alloc(*Buffer, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); - meta.forEachVisit(buffers, *Buffer, struct { - fn call(i: usize, buffer: *Buffer, flattened_buffers_: []*Buffer) void { - flattened_buffers_[i] = buffer; - } - }.call, .{flattened_buffers}); - - const flattened_tensors = arena.allocator().alloc(*const Tensor, tensor_count) catch @panic("Errors can't be handled in `loadInner`"); - meta.forEachVisit(model, *const Tensor, struct { - fn call(i: usize, tensor: *const Tensor, flattened_tensors_: []*const Tensor) void { - flattened_tensors_[i] = tensor; - } - }.call, .{flattened_tensors}); - - // Refuse before scheduling anything, so a failure leaves no task in flight. - var missing: usize = 0; - for (flattened_tensors) |tensor| { - const sources = store.getSourcesById(tensor.id) orelse continue; - if (!sources.transformed or self.delivered.contains(tensor.id)) { - continue; - } - - missing += 1; - logNotDelivered(arena.allocator(), tensor, sources.tensors); - } - - if (missing != 0) { - return error.TransformedTensorNotDelivered; - } - - for (flattened_tensors, flattened_buffers) |tensor, buffer| { - if (store.getSourcesById(tensor.id)) |sources| { - if (sources.transformed) continue; - } - - self.group.async(io, defaultCallback, .{ self, io, tensor, buffer, store, opts }); - } - } - - fn logNotDelivered(arena: std.mem.Allocator, tensor: *const Tensor, sources: []const *safetensors.Tensor) void { - const max_names = 8; - var names: std.Io.Writer.Allocating = .init(arena); - for (sources[0..@min(sources.len, max_names)], 0..) |source, i| { - names.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break; - } - - if (sources.len > max_names) { - names.writer.print(" and {} more", .{sources.len - max_names}) catch {}; - } - - log.err("Transformed tensor {} {f} was not delivered by loadExecute before load; sources: {s}", .{ tensor.id, tensor.shape(), names.written() }); - } - - pub fn loadBuffer(loader: *Loader, io: std.Io, buffer: *Buffer, tensor: *const Tensor, store: *const TensorStore, opts: LoadOpts) void { - loader.group.async(io, defaultCallback, .{ loader, io, tensor, buffer, store, opts }); - } - - fn defaultCallback(self: *Loader, io: std.Io, tensor: *const Tensor, buffer: *Buffer, store: *const TensorStore, opts: LoadOpts) void { - const sources = store.getSourcesById(tensor.id) orelse { - std.log.debug("Failed to get sources for tensor with id: {}", .{tensor.id}); - return; - }; - stdx.debug.assert(!sources.transformed and sources.tensors.len == 1, "Tensor {} is transformed or has {} sources; `load` only streams single-source tensors", .{ tensor.id, sources.tensors.len }); - - self.loadSingleInner(io, sources.tensors[0], tensor.shape(), buffer, sources.memory, opts) catch |e| { - log.err("Errors are not handled in `defaultCallback`, got {}", .{e}); - unreachable; - }; - } - - fn loadSingle(self: *Loader, io: std.Io, source: *safetensors.Tensor, shape: Shape, buffer: *Buffer, loaded: *bool, opts: LoadOpts) void { - self.loadSingleInner(io, source, shape, buffer, .device, opts) catch |e| { - log.err("Failed to load tensor {s}: {}", .{ source.name, e }); - loaded.* = false; - return; - }; - loaded.* = true; - } - - fn loadSingleInner( - self: *Loader, - io: std.Io, - source: *safetensors.Tensor, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - opts: LoadOpts, - ) !void { - var reader = try source.reader(io, &.{}, .{}); - defer reader.deinit(); - - var writer = try MemoryWriter.init( - self.allocator, - io, - self.platform, - self.pinned_buffer_pools, - self.dma_allocators, - self.dma_chunk_size, - shape, - buffer, - memory, - ); - defer writer.deinit(self.allocator); - - const scale = 1024; - - if (opts.progress) |progress| { - var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); - defer node.end(); - writer.setProgress(&node); - defer writer.setProgress(null); - var progress_writer: ProgressWriter = .init(writer.interface(), &node, .{ .scale = scale }); - const total = try reader.interface.streamRemaining(&progress_writer.interface); - try progress_writer.interface.flush(); - _ = self.bytes_loaded.fetchAdd(total, .monotonic); - } else { - const total = try reader.interface.streamRemaining(writer.interface()); - try writer.interface().flush(); - _ = self.bytes_loaded.fetchAdd(total, .monotonic); - } - } - - pub fn loadExecute( - self: *Loader, - arena: std.mem.Allocator, - io: std.Io, - tensor: Tensor, - buffer: *Buffer, - store: *const TensorStore, - exe: *const Exe, - opts: LoadOpts, - ) error{ NotFound, OutOfMemory, LoadFailed, Canceled }!void { - const sources = (store.getSourcesById(tensor.id) orelse return error.NotFound).tensors; - const buffers = try arena.alloc(Buffer, sources.len); - const loaded = try arena.alloc(bool, sources.len); - @memset(loaded, false); - defer for (buffers, loaded) |*b, l| if (l) b.deinit(); - - var node = if (opts.progress) |progress| p: { - var writer = std.Io.Writer.Allocating.init(arena); - print: { - writer.writer.writeAll("Running executable on ") catch break :print; - for (sources, 0..) |source, i| { - writer.writer.print("{s}{s}", .{ if (i != 0) ", " else "", source.name }) catch break :print; - } - } - break :p progress.start(writer.written(), 1); - } else null; - defer if (node) |*n| n.end(); - - for (sources, 0..) |source, i| { - self.group.async(io, loadSingle, .{ self, io, source, source.shape, &buffers[i], &loaded[i], .{} }); - } - self.group.await(io) catch |err| switch (err) { - error.Canceled => return error.Canceled, - }; - - if (std.mem.findScalar(bool, loaded, false)) |_| { - return error.LoadFailed; - } - - var args = try exe.args(arena); - var results = try exe.results(arena); - - args.set(.{buffers}); - exe.call(io, args, &results, .{ .wait = true }); - - buffer.* = results.get(Buffer); - try self.delivered.put(self.allocator, tensor.id, {}); - } -}; - -test "Loader.loadExecute cleans up donated and non-donated inputs" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = @import("testing.zig").env(); - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try tmp.dir.writeFile(io, .{ .sub_path = "inputs", .data = std.mem.asBytes(&[4]i32{ 1, 2, 10, 20 }) }); - const path = try tmp.dir.realPathFileAlloc(io, "inputs", allocator); - defer allocator.free(path); - - const shape: Shape = .init(.{2}, .i32); - var registry: safetensors.TensorRegistry = .init(allocator); - defer registry.deinit(); - try registry.registerTensor(.{ .name = "value", .file_uri = path, .shape = shape, .offset = 0 }); - try registry.registerTensor(.{ .name = "weight", .file_uri = path, .shape = shape, .offset = 2 * @sizeOf(i32) }); - var store: TensorStore = .fromRegistry(allocator, ®istry, &.{platform.replicated_mesh}); - defer store.deinit(); - var loader = try Loader.init(allocator, platform, .{ .parallelism = 2, .dma_chunks = 2, .dma_chunk_size = 4096 }); - defer loader.deinit(); - - inline for (.{ false, true }) |donate| { - const Functions = struct { - fn forward(value: Tensor, weight: Tensor) Tensor { - const sum = value.add(weight); - return if (donate) sum.reuseBuffer(value) else sum; - } - }; - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - const tensor = store.view().maybeCreateBinding(&.{ "value", "weight" }, shape) orelse unreachable; - const exe = try platform.compileFn(allocator, io, Functions.forward, .{ Tensor.fromShape(shape), Tensor.fromShape(shape) }, .{}); - defer exe.deinit(); - try std.testing.expectEqual(@as(usize, @intFromBool(donate)), exe.donated_input_indices.len); - - var output: Buffer = undefined; - try loader.loadExecute(arena.allocator(), io, tensor, &output, &store, &exe, .{}); - defer output.deinit(); - try std.testing.expectEqual([2]i32{ 11, 22 }, try output.getValue([2]i32, io)); - try std.testing.expect(loader.delivered.contains(tensor.id)); - } -} - pub const ProgressWriter = struct { inner: *std.Io.Writer, progress: *std.Progress.Node, @@ -637,8 +238,7 @@ pub const ProgressWriter = struct { pub fn post(self: *ProgressWriter, len_pre: usize, total: usize) void { self.interface.buffer = self.inner.buffer; self.interface.end = self.inner.end; - const drained_pre = len_pre -| self.interface.end; - self.total += drained_pre + total; + self.total += (len_pre - self.interface.end) + total; self.progress.setCompletedItems(self.total / self.scale); } @@ -679,25 +279,10 @@ pub const MemoryWriter = union(enum) { direct: DirectMemoryWriter, buffered: BufferedMemoryWriter, - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - pools: []mem.DynamicBufferPool, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - ) !MemoryWriter { - return switch (platform.target) { - .cuda, .rocm, .oneapi, .furiosa => .{ - .direct = try .init(allocator, io, platform, pools, dma_allocators, dma_chunk_size, shape, buffer, memory), - }, - .tpu, .neuron, .cpu, .metal => if (memory == .host_pinned) - std.debug.panic("Host pinned memory is not supported on {}", .{platform.target}) - else - .{ .buffered = try .init(allocator, io, platform, shape, buffer) }, + pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer) !MemoryWriter { + return switch (memory.platform.target) { + .cuda => .{ .direct = try .init(allocator, io, memory, pool, shape, buffer) }, + .rocm, .tpu, .neuron, .cpu => .{ .buffered = try .init(allocator, io, memory, shape, buffer) }, }; } @@ -714,27 +299,20 @@ pub const MemoryWriter = union(enum) { .buffered => self.buffered.deinit(allocator), } } - - pub fn setProgress(self: *MemoryWriter, progress: ?*std.Progress.Node) void { - switch (self.*) { - .direct => self.direct.setProgress(progress), - .buffered => {}, - } - } }; pub const BufferedMemoryWriter = struct { io: std.Io, - platform: *const Platform, + memory: *const Memory, shape: Shape, buffer: *Buffer, interface: std.Io.Writer, - pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { + pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, shape: Shape, buffer: *Buffer) !BufferedMemoryWriter { return .{ .io = io, - .platform = platform, .shape = shape, + .memory = memory, .buffer = buffer, .interface = .{ .buffer = try allocator.alloc(u8, shape.byteSize()), @@ -753,22 +331,36 @@ pub const BufferedMemoryWriter = struct { } } - pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + fn flush_(w: *std.Io.Writer) !void { const self: *BufferedMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + const pjrt_api = self.memory.platform.pjrt_api; - self.buffer.* = Buffer.from( - self.io, - self.platform, - self.shape, - @ptrCast(self.interface.buffer), - .{ .wait = true }, - ) catch return std.Io.Writer.Error.WriteFailed; + const args: pjrt.Client.BufferFromHostBufferArgs = .{ + .data = @ptrCast(self.interface.buffer), + .buffer_type = pjrtx.bufferTypeFromDtype(self.shape.dtype()), + .dims = self.shape.dims(), + .byte_strides = self.shape.computeByteStrides().constSlice(), + .host_buffer_semantics = .ImmutableUntilTransferCompletes, + .dst = .{ .memory = self.memory.pjrt_memory }, + }; + const pjrt_buffer, const event = try self.memory.platform.pjrt_client.bufferFromHostBuffer(pjrt_api, args); + self.buffer.* = Buffer.fromPjrtBuffers(self.memory.platform, self.shape, &.{pjrt_buffer}); + if (event) |ev| { + defer ev.deinit(pjrt_api); + try ev.await(pjrt_api, self.io); + } else { + try self.buffer.await(self.io); + } + } + + pub fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { + flush_(w) catch return std.Io.Writer.Error.WriteFailed; } }; -const DirectShardWriter = struct { +pub const DirectMemoryWriter = struct { const EventContext = struct { - self: *DirectShardWriter, + self: *DirectMemoryWriter, err: ?*pjrt.Error = null, pjrt_event: *pjrt.Event, event: std.Io.Event = .unset, @@ -780,7 +372,7 @@ const DirectShardWriter = struct { memory: *const Memory, pool: *mem.DynamicBufferPool, total: usize, - pjrt_buffer: *pjrt.Buffer, + buffer: *Buffer, transfer_manager: *pjrt.AsyncHostToDeviceTransferManager, offset: usize = 0, @@ -789,7 +381,7 @@ const DirectShardWriter = struct { flip_flop: u1 = 0, events_contexts: [2]?EventContext = @splat(null), - pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape) !DirectShardWriter { + pub fn init(allocator: std.mem.Allocator, io: std.Io, memory: *const Memory, pool: *mem.DynamicBufferPool, shape: Shape, buffer: *Buffer) !DirectMemoryWriter { const shape_spec: pjrt.ShapeSpec = .init(shape.dims(), pjrtx.bufferTypeFromDtype(shape.dtype())); const transfer_manager = try memory.platform.pjrt_client.createBuffersForAsyncHostToDevice( memory.platform.pjrt_api, @@ -798,12 +390,9 @@ const DirectShardWriter = struct { .memory = memory.pjrt_memory, }, ); - errdefer transfer_manager.deinit(memory.platform.pjrt_api); - const pjrt_buffer = try transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0); - errdefer pjrt_buffer.deinit(memory.platform.pjrt_api); - - const buf = try pool.get(allocator, io); + const pjrt_buffer = transfer_manager.retrieveBuffer(memory.platform.pjrt_api, 0) catch unreachable; + buffer.* = Buffer.fromPjrtBuffers(memory.platform, shape, &.{pjrt_buffer}); return .{ .allocator = allocator, @@ -811,82 +400,54 @@ const DirectShardWriter = struct { .memory = memory, .pool = pool, .total = shape.byteSize(), - .pjrt_buffer = pjrt_buffer, + .buffer = buffer, .transfer_manager = transfer_manager, .interface = .{ - .buffer = buf[0..@min(buf.len, shape.byteSize())], + .buffer = try pool.get(allocator, io), + .end = 0, .vtable = &.{ .drain = drain, - .flush = flush, - .rebase = rebase, }, }, }; } - pub fn deinit(self: *DirectShardWriter) void { + pub fn deinit(self: *DirectMemoryWriter) void { self.transfer_manager.deinit(self.memory.platform.pjrt_api); } - fn drain(w: *std.Io.Writer, data: []const []const u8, _: usize) std.Io.Writer.Error!usize { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - - const chunk = data[0]; - if (chunk.len > self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; - if (chunk.len > self.total - (self.offset + self.interface.end)) return std.Io.Writer.Error.WriteFailed; - - const needs_fresh_buffer = chunk.len > self.interface.buffer.len - self.interface.end; - if (needs_fresh_buffer) { - try self.interface.flush(); + fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { + _ = data; // autofix + _ = splat; // autofix + const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); + if (self.offset >= self.total) { + return std.Io.Writer.Error.WriteFailed; } - @memcpy(self.interface.buffer[self.interface.end..][0..chunk.len], chunk); - self.interface.end += chunk.len; - - const buffer_full = self.interface.end == self.interface.buffer.len; - if (buffer_full) { - try self.interface.flush(); - } - - return chunk.len; - } - - fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - try self.flushBuffered(); - } - - fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { - const self: *DirectShardWriter = @alignCast(@fieldParentPtr("interface", w)); - if (self.interface.buffer.len - self.interface.end >= capacity) return; - if (preserve != 0) return std.Io.Writer.Error.WriteFailed; - try self.interface.flush(); - } - - fn flushBuffered(self: *DirectShardWriter) std.Io.Writer.Error!void { - if (self.offset >= self.total) return; - const pjrt_api = self.memory.platform.pjrt_api; const current_buffer = self.interface.buffer; const buffered = self.interface.buffered(); + const sliced_buffer = self.interface.buffered()[0..@min(buffered.len, self.total - self.offset)]; + const written = sliced_buffer.len; + const is_last = (self.offset + sliced_buffer.len) >= self.total; - const slice = buffered[0..@min(buffered.len, self.total - self.offset)]; - if (slice.len == 0) return; - const is_last = (self.offset + slice.len) >= self.total; - - const transfer_event = self.transfer_manager.transferData(pjrt_api, 0, slice, @intCast(self.offset), is_last) catch |err| { + const transfer_event = self.transfer_manager.transferData( + pjrt_api, + 0, + sliced_buffer, + @intCast(self.offset), + is_last, + ) catch |err| { log.err("error when transferring data to device: {any}", .{err}); return std.Io.Writer.Error.WriteFailed; }; - const ctx = &self.events_contexts[@intCast(self.flip_flop)]; ctx.* = .{ .self = self, .buffer = current_buffer, .pjrt_event = transfer_event, }; - transfer_event.onReady(pjrt_api, EventContext, struct { fn call(err: ?*pjrt.Error, ctx_: *EventContext) void { ctx_.self.pool.put(ctx_.self.io, ctx_.buffer); @@ -928,816 +489,113 @@ const DirectShardWriter = struct { } } else { self.interface.end = 0; - const buf = self.pool.get(self.allocator, self.io) catch |err| { + self.interface.buffer = self.pool.get(self.allocator, self.io) catch |err| { log.err("unable to get a new buffer from the pool: {any}", .{err}); return std.Io.Writer.Error.WriteFailed; }; - self.interface.buffer = buf[0..@min(buf.len, self.total - (self.offset + slice.len))]; } self.flip_flop ^= 1; - self.offset += slice.len; + self.offset += written; + + return 0; } }; -const ShardProgress = struct { - const scale: usize = 1024; - - node: std.Progress.Node, - label: [32]u8 = undefined, - completed: usize = 0, - - fn set(self: *ShardProgress, completed: usize) void { - self.completed = completed; - self.node.setCompletedItems(std.math.divCeil(usize, self.completed, scale) catch unreachable); - } -}; - -// Dispatch planning bridges two different orders: -// -// 1. Placement traversal emits byte ranges per shard writer, in device order. -// That order is convenient for asking "what bytes belong to this shard?", -// but it does not match how a tensor file is read. -// -// 2. Readers stream tensor bytes in global row-major order. DirectMemoryWriter -// must therefore consume dispatch spans in increasing global byte offset. -// -// We first collect placement spans, sort them by global byte range, then fold -// identical ranges into one primary dispatch span plus mirror writers. Identical -// ranges represent replicated shard data: the reader provides those bytes once, -// the primary writer receives them zero-copy, and mirrors receive a copy. -const DispatchSpans = struct { - const DispatchSpan = struct { - start: usize, - end: usize, - primary_writer: usize, - mirror_writer_start: usize, - mirror_writer_len: usize, - }; - - const PlacementSpan = struct { - writer_index: usize, - start: usize, - len: usize, - order: usize, - }; - - spans: []DispatchSpan, - mirror_writers: []usize, - - fn init(allocator: std.mem.Allocator, shape: Shape) !DispatchSpans { - const placement = try Sharding.Placement.init(&shape); - const ordered_devices = placement.mesh().?.devicesInCanonicalOrder(); - - var placement_span_count: usize = 0; - for (0..ordered_devices.len) |device_index| { - placement_span_count += placementSpanCount(shape, placement.slices(device_index, shape).constSlice()); - } - - var placement_spans: std.ArrayList(PlacementSpan) = try .initCapacity(allocator, placement_span_count); - defer placement_spans.deinit(allocator); - - const byte_strides = shape.computeByteStrides(); - - for (0..ordered_devices.len) |writer_index| { - appendShardPlacementSpans(&placement_spans, shape, placement.slices(writer_index, shape).constSlice(), byte_strides.constSlice(), writer_index); - } - - std.debug.assert(placement_spans.items.len == placement_span_count); - - var spans: std.ArrayList(DispatchSpan) = try .initCapacity(allocator, placement_spans.items.len); - errdefer spans.deinit(allocator); - - var mirror_writers: std.ArrayList(usize) = try .initCapacity(allocator, placement_spans.items.len); - errdefer mirror_writers.deinit(allocator); - - try deduplicateByRange(allocator, placement_spans.items, shape.byteSize(), &spans, &mirror_writers); - - const spans_ = try spans.toOwnedSlice(allocator); - errdefer allocator.free(spans_); - - const mirror_writers_ = try mirror_writers.toOwnedSlice(allocator); - errdefer allocator.free(mirror_writers_); - - return .{ - .spans = spans_, - .mirror_writers = mirror_writers_, - }; - } - - fn deinit(self: DispatchSpans, allocator: std.mem.Allocator) void { - allocator.free(self.spans); - allocator.free(self.mirror_writers); - } - - fn deduplicateByRange( - allocator: std.mem.Allocator, - placement_spans: []PlacementSpan, - total_bytes: usize, - spans: *std.ArrayList(DispatchSpan), - mirror_writers: *std.ArrayList(usize), - ) !void { - const SortContext = struct { - fn lessThan(_: void, lhs: PlacementSpan, rhs: PlacementSpan) bool { - if (lhs.start != rhs.start) return lhs.start < rhs.start; - if (lhs.len != rhs.len) return lhs.len < rhs.len; - return lhs.order < rhs.order; - } - }; - - std.mem.sort(PlacementSpan, placement_spans, {}, SortContext.lessThan); - - var i: usize = 0; - var cursor: usize = 0; - while (i < placement_spans.len) { - const span = placement_spans[i]; - if (span.start != cursor) return error.NonContiguousShardPlacement; - - const mirror_writer_start = mirror_writers.items.len; - var j = i + 1; - while (j < placement_spans.len) : (j += 1) { - const mirror = placement_spans[j]; - if (mirror.start != span.start or mirror.len != span.len) break; - try mirror_writers.append(allocator, mirror.writer_index); - } - - try spans.append(allocator, .{ - .start = span.start, - .end = span.start + span.len, - .primary_writer = span.writer_index, - .mirror_writer_start = mirror_writer_start, - .mirror_writer_len = j - i - 1, - }); - cursor += span.len; - i = j; - } - - if (cursor != total_bytes) return error.NonContiguousShardPlacement; - } - - fn appendPlacementSpan(placement_spans: *std.ArrayList(PlacementSpan), writer_index: usize, start: usize, len: usize) void { - placement_spans.appendAssumeCapacity(.{ - .writer_index = writer_index, - .start = start, - .len = len, - .order = placement_spans.items.len, - }); - } - - fn appendShardPlacementSpans( - placement_spans: *std.ArrayList(PlacementSpan), - shape: Shape, - slices: []const Placement.Slice1d, - byte_strides: []const i64, - writer_index: usize, - ) void { - if (shape.rank() == 0) { - appendPlacementSpan(placement_spans, writer_index, 0, shape.byteSize()); - return; - } - - appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, 0, contiguousSliceAxis(shape, slices), 0); - } - - fn appendShardAxisPlacementSpans( - placement_spans: *std.ArrayList(PlacementSpan), - slices: []const Placement.Slice1d, - byte_strides: []const i64, - writer_index: usize, - axis: usize, - contiguous_axis: usize, - base_start: i64, - ) void { - const slice = slices[axis]; - if (slice.size == 0) return; - - if (axis == contiguous_axis) { - const span_start: usize = @intCast(base_start + slice.start * byte_strides[axis]); - const span_len: usize = @intCast(slice.size * byte_strides[axis]); - appendPlacementSpan(placement_spans, writer_index, span_start, span_len); - return; - } - - var i: i64 = 0; - while (i < slice.size) : (i += 1) { - const child_start = base_start + (slice.start + i) * byte_strides[axis]; - appendShardAxisPlacementSpans(placement_spans, slices, byte_strides, writer_index, axis + 1, contiguous_axis, child_start); - } - } - - fn placementSpanCount(shape: Shape, slices: []const Placement.Slice1d) usize { - if (shape.rank() == 0) return 1; - - const contiguous_axis = contiguousSliceAxis(shape, slices); - var count: usize = 1; - for (slices[0..contiguous_axis]) |slice| { - count *= @intCast(slice.size); - } - return count; - } - - fn contiguousSliceAxis(shape: Shape, slices: []const Placement.Slice1d) usize { - var axis = shape.rank() - 1; - while (axis > 0) { - const slice = slices[axis]; - if (slice.start != 0 or slice.size != shape.dim(axis)) break; - axis -= 1; - } - return axis; - } -}; - -// Direct load writer state machine. -// -// std.Io.Reader -// | -// v -// DirectMemoryWriter.interface.buffer -// | aliases active primary DirectShardWriter DMA buffer -// v -// commitWindow() -// |-- primary: advance DirectShardWriter.interface.end -// |-- mirrors: copy committed bytes with writeAll() -// |-- boundaries: flush full shard buffers / chunk fence -// v -// DirectShardWriter.flushBuffered() -// | -// v -// PJRT AsyncHostToDeviceTransferManager -// -// The public writer never owns a host staging buffer: it exposes a visible -// prefix of a real shard DMA buffer. The visible prefix may extend beyond the -// current dispatch span to coalesce reader requests, but `commitWindow` must -// scatter those bytes before submitting or rotating the active buffer. The -// first active-primary segment is zero-copy; crossed spans and mirrors treat -// the active buffer as scratch and copy into their own shard writers. -// -// `byte_cursor` is the global tensor position. Shard writer `interface.end` is -// local to the current shard DMA buffer. -pub const DirectMemoryWriter = struct { - allocator: std.mem.Allocator, - shard_writers: []DirectShardWriter, - // Global stream-order spans produced from placement; this is the routing table for committed bytes. - dispatch_spans: DispatchSpans, - // Current entry in `dispatch_spans.spans`; advances whenever `byte_cursor` reaches that span end. - span_index: usize = 0, - // Global tensor byte offset already scattered into shard writers. - byte_cursor: usize = 0, - // Shard writer whose DMA buffer is currently exposed as `interface.buffer`. - active_writer_index: usize, - // Start offset of new reader bytes inside the active shard writer buffer. - window_start: usize, - // Logical maximum public alias window before forcing a cross-shard flush fence. +pub const LoadOpts = struct { + parallelism: usize, + store: *const TensorStore, + progress: ?*std.Progress.Node = null, + dma_chunks: usize, dma_chunk_size: usize, - shard_progress: ?[]ShardProgress = null, - interface: std.Io.Writer, - - pub fn init( - allocator: std.mem.Allocator, - io: std.Io, - platform: *const Platform, - pools: []mem.DynamicBufferPool, - dma_allocators: []const mem.DmaAllocator, - dma_chunk_size: usize, - shape: Shape, - buffer: *Buffer, - memory: Memory.Kind, - ) !DirectMemoryWriter { - var placed_shape = shape; - if (placed_shape._sharding.mesh == null) placed_shape._sharding.mesh = platform.replicated_mesh; - const placement = try Sharding.Placement.init(&placed_shape); - const mesh = placement.mesh().?; - const ordered_devices = mesh.devicesInCanonicalOrder(); - var shard_writers = try allocator.alloc(DirectShardWriter, ordered_devices.len); - errdefer allocator.free(shard_writers); - - var initialized: usize = 0; - errdefer for (shard_writers[0..initialized]) |*writer| { - writer.deinit(); - }; - - var pjrt_buffers: Buffer.Shards = .empty; - for (ordered_devices, 0..) |device, i| { - const pool = &pools[device.id]; - const shard_dma_allocator = dma_allocators[device.id].allocator(); - const pjrt_mem = platform.devices[device.id].memory(memory).?; - - shard_writers[i] = try .init(shard_dma_allocator, io, pjrt_mem, pool, placement.localShape(placed_shape)); - initialized += 1; - - pjrt_buffers.appendAssumeCapacity(shard_writers[i].pjrt_buffer); - } - - buffer.* = .fromPjrtBuffers(platform, shape, pjrt_buffers.constSlice()); - - const dispatch_spans: DispatchSpans = try .init(allocator, placed_shape); - errdefer dispatch_spans.deinit(allocator); - - const first_span = dispatch_spans.spans[0]; - const first_writer = &shard_writers[first_span.primary_writer]; - const first_window = @min(dma_chunk_size, first_writer.interface.buffer.len); - - return .{ - .allocator = allocator, - .shard_writers = shard_writers, - .dispatch_spans = dispatch_spans, - .active_writer_index = first_span.primary_writer, - .window_start = first_writer.interface.end, - .dma_chunk_size = dma_chunk_size, - .interface = .{ - .buffer = first_writer.interface.buffer[0..first_window], - .end = 0, - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = rebase, - }, - }, - }; - } - - pub fn deinit(self: *DirectMemoryWriter) void { - self.setProgress(null); - for (self.shard_writers) |*writer| { - writer.deinit(); - } - self.allocator.free(self.shard_writers); - self.dispatch_spans.deinit(self.allocator); - } - - pub fn setProgress(self: *DirectMemoryWriter, progress: ?*std.Progress.Node) void { - const parent = progress orelse { - const states = self.shard_progress orelse return; - for (states) |*s| { - s.node.end(); - } - self.allocator.free(states); - self.shard_progress = null; - return; - }; - - std.debug.assert(self.shard_progress == null); - const states = self.allocator.alloc(ShardProgress, self.shard_writers.len) catch return; - for (states, self.shard_writers, 0..) |*state, writer, i| { - state.completed = 0; - const label = std.fmt.bufPrint(&state.label, "shard[{d}]", .{i}) catch unreachable; - const total_items = std.math.divCeil(usize, writer.total, ShardProgress.scale) catch unreachable; - state.node = parent.start(label, total_items); - state.node.setCompletedItems(0); - } - self.shard_progress = states; - } - - fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - try self.commitWindow(); - if (self.interface.end == self.interface.buffer.len) return std.Io.Writer.Error.WriteFailed; - - const consumed = consumed: { - var n: usize = 0; - for (data[0 .. data.len - 1]) |bytes| { - const copied = self.writeWindow(bytes); - n += copied; - - if (copied < bytes.len) break :consumed n; - } - - const pattern = data[data.len - 1]; - switch (pattern.len) { - 0 => {}, - 1 => { - const dest = self.interface.buffer[self.interface.end..]; - const count = @min(dest.len, splat); - @memset(dest[0..count], pattern[0]); - - self.interface.end += count; - n += count; - }, - else => for (0..splat) |_| { - const copied = self.writeWindow(pattern); - n += copied; - - if (copied < pattern.len) break; - }, - } - - break :consumed n; - }; - - try self.commitWindow(); - return consumed; - } - - fn writeWindow(self: *DirectMemoryWriter, bytes: []const u8) usize { - const dest = self.interface.buffer[self.interface.end..]; - const n = @min(dest.len, bytes.len); - @memcpy(dest[0..n], bytes[0..n]); - self.interface.end += n; - return n; - } - - fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - // Commit the active public window if one remains - if (self.span_index < self.dispatch_spans.spans.len) { - try self.commitWindow(); - } - - for (self.shard_writers, 0..) |*shard_writer, i| { - try shard_writer.interface.flush(); - if (self.shard_progress) |states| { - states[i].set(shard_writer.total); - } - } - - self.interface = .failing; - } - - fn rebase(w: *std.Io.Writer, preserve: usize, capacity: usize) std.Io.Writer.Error!void { - const self: *DirectMemoryWriter = @alignCast(@fieldParentPtr("interface", w)); - - if (self.interface.buffer.len - self.interface.end >= capacity) return; - if (preserve != 0) return std.Io.Writer.Error.WriteFailed; - - try self.commitWindow(); - if (self.interface.buffer.len - self.interface.end < capacity) return std.Io.Writer.Error.WriteFailed; - } - - // `interface.buffer` is a clipped alias of the active primary shard DMA - // buffer. Reader writes advance only `DirectMemoryWriter.interface.end`; - // this function scatters that visible range through dispatch spans. - // - // The leading segment for `active_writer_index` is already in the right - // DMA buffer and is committed by advancing that shard writer. Later spans - // use the same bytes as scratch and are copied before any full buffers are - // submitted or the public alias is rotated. - fn commitWindow(self: *DirectMemoryWriter) std.Io.Writer.Error!void { - // All dispatch spans are consumed; no shard DMA buffer remains valid to - // expose as the public alias until flush marks the writer as failing. - if (self.span_index >= self.dispatch_spans.spans.len) { - self.interface.buffer = &.{}; - self.interface.end = 0; - return; - } - - // Only bytes after `window_start` are new reader writes. Bytes before - // it may already be pending in the active shard writer. - const source = self.interface.buffer[self.window_start..self.interface.end]; - if (source.len == 0) return; - - // The visible alias window may cross many dispatch spans. Walk the - // global cursor and route each subrange to its primary and mirrors. - var consumed: usize = 0; - while (consumed < source.len) { - if (self.span_index >= self.dispatch_spans.spans.len) return std.Io.Writer.Error.WriteFailed; - - const span = self.dispatch_spans.spans[self.span_index]; - if (self.byte_cursor < span.start or self.byte_cursor >= span.end) return std.Io.Writer.Error.WriteFailed; - - const n = @min(source.len - consumed, span.end - self.byte_cursor); - const chunk = source[consumed..][0..n]; - - // The first span in the active alias is already in the primary DMA - // buffer. Later ranges use the active buffer as scratch and must be - // copied into their real destination writer. - if (span.primary_writer == self.active_writer_index and consumed == 0) { - self.shard_writers[span.primary_writer].interface.end += n; - } else { - const primary_writer = &self.shard_writers[span.primary_writer]; - if (span.primary_writer == self.active_writer_index) { - @memmove(primary_writer.interface.buffer[primary_writer.interface.end..][0..n], chunk); - primary_writer.interface.end += n; - } else { - try primary_writer.interface.writeAll(chunk); - } - } - if (self.shard_progress) |states| { - states[span.primary_writer].set(states[span.primary_writer].completed + n); - } - - const mirror_writer_end = span.mirror_writer_start + span.mirror_writer_len; - for (self.dispatch_spans.mirror_writers[span.mirror_writer_start..mirror_writer_end]) |mirror_writer_index| { - const mirror_writer = &self.shard_writers[mirror_writer_index]; - if (mirror_writer_index == self.active_writer_index) { - @memmove(mirror_writer.interface.buffer[mirror_writer.interface.end..][0..n], chunk); - mirror_writer.interface.end += n; - } else { - try mirror_writer.interface.writeAll(chunk); - } - if (self.shard_progress) |states| { - states[mirror_writer_index].set(states[mirror_writer_index].completed + n); - } - } - - self.byte_cursor += n; - consumed += n; - if (self.byte_cursor == span.end) { - self.span_index += 1; - } - } - - // All scratch copies are complete now, so full DMA buffers may be - // submitted without invalidating bytes still needed for scatter. - for (self.shard_writers) |*writer| { - if (writer.interface.end == writer.interface.buffer.len) { - try writer.interface.flush(); - } - } - - if (@mod(self.byte_cursor, self.dma_chunk_size) == 0) { - for (self.shard_writers) |*writer| { - try writer.interface.flush(); - } - } - - // The last committed window may finish the tensor exactly. Leave an - // empty public window so any further writes fail through std.Io. - if (self.span_index >= self.dispatch_spans.spans.len) { - self.interface.buffer = &.{}; - self.interface.end = 0; - return; - } - - // Rotate the public alias to the primary writer for the next stream - // span. `window_start` preserves that writer's already-buffered prefix - // so only newly advanced bytes are scattered on the next commit. - const next_span = self.dispatch_spans.spans[self.span_index]; - const next_writer = &self.shard_writers[next_span.primary_writer]; - self.active_writer_index = next_span.primary_writer; - self.window_start = next_writer.interface.end; - - // Publish the next public alias. Its backing memory is the next primary - // shard buffer, but the visible length is allowed to coalesce across - // upcoming spans until a buffer or tensor boundary. - const total = self.dispatch_spans.spans[self.dispatch_spans.spans.len - 1].end; - const buffer_remaining = next_writer.interface.buffer.len - self.window_start; - const chunk_offset = @mod(self.byte_cursor, self.dma_chunk_size); - const chunk_remaining = if (chunk_offset == 0) self.dma_chunk_size else self.dma_chunk_size - chunk_offset; - const tensor_remaining = total - self.byte_cursor; - const visible_remaining = @min(buffer_remaining, @min(chunk_remaining, tensor_remaining)); - if (visible_remaining == 0) return std.Io.Writer.Error.WriteFailed; - - self.interface.buffer = next_writer.interface.buffer[0 .. self.window_start + visible_remaining]; - self.interface.end = self.window_start; - } + total_bytes: ?*usize = null, }; -const DirectMemoryWriterDeviceTest = struct { - const WriteMode = enum { - stream_remaining, - writable_slice_greedy, - }; - - pub const Scenario = struct { - name: []const u8, - create_options: CreateOptions, - shape: Shape, - logical_mesh: Sharding.LogicalMesh, - strategy: Sharding.Strategy, - write_mode: WriteMode = .stream_remaining, - writable_slice_min_len: usize = 128, - pool_chunks: usize = 4, - pool_chunk_size: usize = 1 << 20, - memory: Memory.Kind = .default, - }; - +pub fn load( + comptime ModelType: type, + model: *const ModelType, allocator: std.mem.Allocator, io: std.Io, + platform: *const Platform, + opts: LoadOpts, +) !Bufferized(ModelType) { + var bufferized = try mem.bufferize(allocator, ModelType, model); - fn run(self: DirectMemoryWriterDeviceTest, scenario: Scenario, partitioning: anytype) !void { - var platform = Platform.auto(self.allocator, self.io, scenario.create_options) catch return error.SkipZigTest; - defer platform.deinit(self.allocator, self.io); + const first_device = platform.devices[0]; // Temporary until sharding is re-exposed + const dma_alloc: mem.DmaAllocator = .init(allocator, &first_device); + var buffer_pool: mem.DynamicBufferPool = .init(opts.dma_chunks, opts.dma_chunk_size); + defer buffer_pool.deinit(dma_alloc.allocator()); - const mesh: Sharding.Mesh = try .init(scenario.name, &platform.physical_mesh, scenario.logical_mesh, scenario.strategy); - std.log.warn("mesh: {f}", .{mesh}); - try self.runDirectMemoryWriter( - platform, - scenario.shape.withPartitioning(&mesh, partitioning), - scenario.write_mode, - scenario.writable_slice_min_len, - scenario.pool_chunks, - scenario.pool_chunk_size, - scenario.memory, - ); - } + const Ctx = struct { + allocator: std.mem.Allocator, + dma_allocator: std.mem.Allocator, + pinned_buffer_pool: *mem.DynamicBufferPool, + io: std.Io, + buffers: []*Buffer, + store: *const TensorStore, + memory: *const Memory, + group: stdx.Io.LimitedGroup, + total: std.atomic.Value(usize) = .init(0), + progress: ?*std.Progress.Node, + }; + var walk_ctx: Ctx = .{ + .buffers = try allocator.alloc(*Buffer, meta.count(Tensor, model)), + .store = opts.store, + .allocator = allocator, + .dma_allocator = dma_alloc.allocator(), + .pinned_buffer_pool = &buffer_pool, + .io = io, + .memory = first_device.memory(.default), + .progress = opts.progress, + .group = .init(opts.parallelism), + }; + defer allocator.free(walk_ctx.buffers); - fn runDirectMemoryWriter( - self: DirectMemoryWriterDeviceTest, - platform: *const Platform, - shape: Shape, - write_mode: WriteMode, - writable_slice_min_len: usize, - pool_chunks: usize, - pool_chunk_size: usize, - memory: Memory.Kind, - ) !void { - const slice = try Slice.alloc(self.allocator, shape); - defer slice.free(self.allocator); + defer if (opts.total_bytes) |total_bytes_ptr| { + total_bytes_ptr.* = walk_ctx.total.load(.monotonic); + }; - for (slice.items(f32), 0..) |*e, i| { - e.* = @as(f32, @floatFromInt(i)); + meta.forEachVisit(&bufferized, *Buffer, struct { + fn call(i: usize, buffer: *Buffer, ctx: *Ctx) void { + ctx.buffers[i] = buffer; } + }.call, .{&walk_ctx}); - const pool_count = platform.devices.len; - const dma_allocators = try self.allocator.alloc(mem.DmaAllocator, pool_count); - defer self.allocator.free(dma_allocators); - for (platform.devices, 0..) |*device, i| { - dma_allocators[i] = .init(self.allocator, device); - } + meta.forEachVisit(model, *const Tensor, struct { + fn call(i: usize, tensor: *const Tensor, ctx: *Ctx) void { + ctx.group.concurrent(ctx.io, struct { + fn call(i_: usize, tensor_: *const Tensor, ctx_: *Ctx) !void { + var reader = ctx_.store.getReaderById(tensor_.id, ctx_.io, &.{}) catch unreachable; + defer reader.deinit(); - const pools = try self.allocator.alloc(mem.DynamicBufferPool, pool_count); - defer self.allocator.free(pools); - for (pools) |*pool| { - pool.* = .init(pool_chunks, pool_chunk_size); - } - defer for (pools, 0..) |*pool, i| { - pool.deinit(dma_allocators[i].allocator()); - }; + var memory_writer = MemoryWriter.init( + ctx_.dma_allocator, + ctx_.io, + ctx_.memory, + ctx_.pinned_buffer_pool, + reader.tensor.shape, + ctx_.buffers[i_], + ) catch unreachable; + defer memory_writer.deinit(ctx_.dma_allocator); - var written_buffer: Buffer = undefined; - var writer: DirectMemoryWriter = try .init( - self.allocator, - self.io, - platform, - pools, - dma_allocators, - pool_chunk_size, - shape, - &written_buffer, - memory, - ); - defer writer.deinit(); - defer written_buffer.deinit(); + const scale = 1024; - switch (write_mode) { - .stream_remaining => { - var reader: std.Io.Reader = .fixed(slice.constData()); - const streamed = try reader.streamRemaining(&writer.interface); - try std.testing.expectEqual(slice.constData().len, streamed); - }, - .writable_slice_greedy => { - var offset: usize = 0; - while (offset < slice.constData().len) { - const min_len = @max(@as(usize, 1), writable_slice_min_len); - const dest = try writer.interface.writableSliceGreedy(min_len); - const to_write = @min(dest.len, slice.constData().len - offset); - if (to_write == 0) return std.Io.Writer.Error.WriteFailed; - @memcpy(dest[0..to_write], slice.constData()[offset..][0..to_write]); - writer.interface.advance(to_write); - offset += to_write; + if (ctx_.progress) |progress| { + var node = progress.start(reader.tensor.name, reader.tensor.shape.byteSize() / scale); + defer node.end(); + var progress_writer: ProgressWriter = .init(memory_writer.interface(), &node, .{ .scale = scale }); + const total = reader.interface.streamRemaining(&progress_writer.interface) catch unreachable; + progress_writer.interface.flush() catch unreachable; + _ = ctx_.total.fetchAdd(total, .monotonic); + } else { + const total = reader.interface.streamRemaining(memory_writer.interface()) catch unreachable; + memory_writer.interface().flush() catch unreachable; + _ = ctx_.total.fetchAdd(total, .monotonic); + } } - }, + }.call, .{ i, tensor, ctx }) catch unreachable; } + }.call, .{&walk_ctx}); + walk_ctx.group.await(io) catch unreachable; - try writer.interface.flush(); - try written_buffer.await(self.io); - - var written_slice = try written_buffer.toSliceAlloc(self.allocator, self.io); - defer written_slice.free(self.allocator); - try std.testing.expectEqualSlices(u8, slice.constData(), written_slice.constData()); - } -}; - -test "DirectMemoryWriter: replicated tensor" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "replicated_auto", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 128 }, .f32), - .logical_mesh = .mesh(.{ .x = .high_bandwidth }), - .strategy = .parseBindings(.{ .x = .link_x }), - }, .{ .rows = .replicated, .cols = .replicated }); -} - -test "DirectMemoryWriter: 1D model split with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_auto", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - }, .{ .rows = .replicated, .cols = .model }); -} - -test "DirectMemoryWriter: 2D batch/model split with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "batch_model_2d_torus", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .batch = 8, .model = 1024 }, .f32), - .logical_mesh = .mesh(.{ - .batch = .low_bandwidth, - .model = .high_bandwidth, - }), - .strategy = .parseBindings(.{ .batch = .link_x, .model = .link_y }), - }, .{ .batch = .batch, .model = .model }); -} - -test "DirectMemoryWriter: folded model mesh with 2x2 physical mesh" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_folded_2d_torus", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = blk: { - var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_y }); - break :blk strategy; - }, - }, .{ .model = .model }); -} - -test "DirectMemoryWriter: writableSliceGreedy with mirrored shards" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "model_auto_writable_slice", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus2d }, - .cpu = .{ .device_count = 4 }, - }, - .shape = Shape.init(.{ .rows = 8, .cols = 1024 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - .write_mode = .writable_slice_greedy, - .writable_slice_min_len = 64, - .pool_chunk_size = 1024, - }, .{ .rows = .replicated, .cols = .model }); -} - -test "DirectMemoryWriter: 3D topology folded model + replicated batch" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "topology_3d_folded_model", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus3d }, - .cpu = .{ .device_count = 8 }, - }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ - .batch = .low_bandwidth, - .model = .high_bandwidth, - }), - .strategy = blk: { - var strategy: Sharding.Strategy = .parseBindings(.{ .model = .link_x }); - strategy.addFold(.link_x, &.{ .link_x, .link_z }); - break :blk strategy; - }, - }, .{ .batch = .replicated, .model = .model }); -} - -test "MemoryWriter can produce a host pinned buffer" { - const case: DirectMemoryWriterDeviceTest = .{ - .allocator = std.testing.allocator, - .io = std.testing.io, - }; - - try case.run(.{ - .name = "host_pinned", - .create_options = .{ - .physical_mesh = .{ .custom = Sharding.PhysicalMesh.torus1d }, - }, - .shape = Shape.init(.{ .batch = 16, .model = 4096 }, .f32), - .logical_mesh = .mesh(.{ .model = .high_bandwidth }), - .strategy = .parseBindings(.{ .model = .link_x }), - .memory = .host_pinned, - }, .{ .batch = .replicated, .model = .model }); + return bufferized; } diff --git a/bin/zml-smi/json/BUILD.bazel b/zml/io/BUILD.bazel similarity index 51% rename from bin/zml-smi/json/BUILD.bazel rename to zml/io/BUILD.bazel index decc8ca1..8407a5c6 100644 --- a/bin/zml-smi/json/BUILD.bazel +++ b/zml/io/BUILD.bazel @@ -1,11 +1,14 @@ load("@rules_zig//zig:defs.bzl", "zig_library") zig_library( - name = "json", - import_name = "zml-smi/json", - main = "json.zig", + name = "io", + srcs = [ + "vfs.zig", + ], + main = "io.zig", visibility = ["//visibility:public"], deps = [ - "//bin/zml-smi/info", + "//stdx", + "//zml/io/vfs", ], ) diff --git a/zml/io/io.zig b/zml/io/io.zig new file mode 100644 index 00000000..78c59e21 --- /dev/null +++ b/zml/io/io.zig @@ -0,0 +1 @@ +pub const VFS = @import("vfs.zig").VFS; diff --git a/zml/io/vfs.zig b/zml/io/vfs.zig new file mode 100644 index 00000000..b9f61550 --- /dev/null +++ b/zml/io/vfs.zig @@ -0,0 +1,333 @@ +const std = @import("std"); + +const vfs = @import("vfs"); + +const log = std.log.scoped(.@"zml/io/vfs"); + +pub const VFS = struct { + pub const File = vfs.File; + pub const HTTP = vfs.HTTP; + pub const HF = vfs.HF; + pub const S3 = vfs.S3; + const VFSBase = vfs.VFSBase; + + const Handles = std.ArrayList(Handle); + const ClosedHandles = std.ArrayList(u32); + + const CWD_HANDLE: u32 = 0; + + const Handle = struct { handle: u32, backend_idx: ?usize }; + + allocator: std.mem.Allocator, + mutex: std.Io.Mutex = .init, + + backends: std.StringArrayHashMapUnmanaged(std.Io) = .empty, + handles: Handles = .{}, + closed_handles: ClosedHandles = .{}, + + base: VFSBase, + + pub fn init(allocator: std.mem.Allocator, base_io: std.Io) !VFS { + const base = VFSBase.init(base_io); + + var handles: Handles = .{}; + try handles.append(allocator, .{ .handle = CWD_HANDLE, .backend_idx = null }); + + return .{ + .allocator = allocator, + .handles = handles, + .base = base, + }; + } + + pub fn deinit(self: *VFS) void { + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + self.backends.deinit(self.allocator); + } + + pub fn register(self: *VFS, scheme: []const u8, backend: std.Io) std.mem.Allocator.Error!void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.backends.put(self.allocator, scheme, backend); + } + + pub fn unregister(self: *VFS, scheme: []const u8) bool { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + return self.backends.remove(scheme); + } + + pub fn io(self: *VFS) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; + } + + fn openHandle(self: *VFS) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (self.closed_handles.pop()) |idx| { + return .{ idx, &self.handles.items[idx] }; + } + return .{ @intCast(self.handles.items.len), try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *VFS, idx: u32) !void { + if (idx == CWD_HANDLE) return; + + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *VFS, file: std.Io.File) struct { *Handle, std.Io } { + self.mutex.lockUncancelable(self.base.inner); + const handle = &self.handles.items[@intCast(file.handle)]; + self.mutex.unlock(self.base.inner); + + return .{ handle, self.getBackend(handle.backend_idx) }; + } + + fn getDirHandle(self: *VFS, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + + if (std.meta.eql(dir, std.Io.Dir.cwd())) return &self.handles.items[CWD_HANDLE]; + return &self.handles.items[@intCast(dir.handle)]; + } + + fn getScheme(self: *VFS, backend_idx: ?usize) ?[]const u8 { + if (backend_idx) |idx| return self.backends.entries.items(.key)[idx] else return null; + } + + fn getBackend(self: *VFS, backend_idx: ?usize) std.Io { + if (backend_idx) |idx| return self.backends.entries.items(.value)[idx] else return self.base.inner; + } + + fn lookupDir(self: *VFS, dir: std.Io.Dir, sub_path: ?[]const u8) !struct { ?usize, std.Io.Dir, std.Io } { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + if (sub_path == null) return .{ null, dir, self.base.inner }; + if (std.fs.path.isAbsolutePosix(sub_path.?)) return .{ null, dir, self.base.inner }; + + const uri = std.Uri.parse(sub_path.?) catch null; + if (uri) |u| { + const backend_idx: ?usize = for (self.backends.entries.items(.key), 0..) |s, idx| { + if (std.mem.eql(u8, u.scheme, s)) break idx; + } else null; + if (backend_idx == null) return error.VFSNotRegistered; + return .{ backend_idx, std.Io.Dir.cwd(), self.getBackend(backend_idx) }; + } else { + return .{ null, std.Io.Dir.cwd(), self.base.inner }; + } + } else { + const handle = self.getDirHandle(dir); + if (handle.backend_idx) |backend_idx| { + return .{ backend_idx, .{ .handle = @intCast(handle.handle) }, self.getBackend(backend_idx) }; + } else { + return .{ null, .{ .handle = @intCast(handle.handle) }, self.base.inner }; + } + } + } + + fn stripScheme(path: []const u8) []const u8 { + const uri = std.Uri.parse(path) catch return path; + return path[uri.scheme.len + 3 ..]; + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening dir '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.OpenError.Unexpected; + }; + const fs_dir = try backend.vtable.dirOpenDir(backend.userdata, dir_, stripScheme(sub_path), options); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(fs_dir.handle), + .backend_idx = backend_idx, + }; + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for dir stat : {any}", .{err}); + return std.Io.Dir.StatError.Unexpected; + }; + return backend.vtable.dirStat(backend.userdata, dir_); + } + + pub fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir stat file '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.StatFileError.Unexpected; + }; + return backend.vtable.dirStatFile(backend.userdata, dir_, stripScheme(sub_path), options); + } + + pub fn dirAccess(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for dir access '{s}' : {any}", .{ sub_path, err }); + return std.Io.Dir.AccessError.Unexpected; + }; + return backend.vtable.dirAccess(backend.userdata, dir_, stripScheme(sub_path), options); + } + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, sub_path) catch |err| { + log.err("Failed to lookup backend for opening file '{s}' : {any}", .{ sub_path, err }); + return std.Io.File.OpenError.Unexpected; + }; + const file = try backend.vtable.dirOpenFile(backend.userdata, dir_, stripScheme(sub_path), flags); + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = .{ + .handle = @intCast(file.handle), + .backend_idx = backend_idx, + }; + return .{ .handle = @intCast(idx) }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| { + _, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for closing dir : {any}", .{err}); + continue; + }; + backend.vtable.dirClose(backend.userdata, &.{dir_}); + self.closeHandle(@intCast(dir.handle)) catch unreachable; + } + } + + fn dirRead(userdata: ?*anyopaque, reader: *std.Io.Dir.Reader, entries: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + _, const dir_, const backend = self.lookupDir(reader.dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path : {any}", .{err}); + return std.Io.Dir.RealPathError.Unexpected; + }; + + const original_dir = reader.dir; + reader.dir = dir_; + defer reader.dir = original_dir; + + return backend.vtable.dirRead(backend.userdata, reader, entries); + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, null) catch |err| { + log.err("Failed to lookup backend for dir real path : {any}", .{err}); + return std.Io.Dir.RealPathError.Unexpected; + }; + + if (self.getScheme(backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.dirRealPath(backend.userdata, dir_, out_buffer); + } + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const backend_idx, const dir_, const backend = self.lookupDir(dir, path_name) catch |err| { + log.err("Failed to lookup backend for dir real path file '{s}' : {any}", .{ path_name, err }); + return std.Io.Dir.RealPathFileError.Unexpected; + }; + + if (self.getScheme(backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.dirRealPathFile(backend.userdata, dir_, path_name, out_buffer); + } + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileStat(backend.userdata, .{ .handle = @intCast(handle.handle) }); + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileLength(backend.userdata, .{ .handle = @intCast(handle.handle) }); + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| { + const handle, const backend = self.getFileHandle(file); + backend.vtable.fileClose(backend.userdata, &.{.{ .handle = @intCast(handle.handle) }}); + self.closeHandle(@intCast(file.handle)) catch unreachable; + } + } + + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileReadStreaming(backend.userdata, .{ .handle = @intCast(handle.handle) }, data); + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileReadPositional(backend.userdata, .{ .handle = @intCast(handle.handle) }, data, offset); + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileSeekBy(backend.userdata, .{ .handle = @intCast(handle.handle) }, relative_offset); + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + return backend.vtable.fileSeekTo(backend.userdata, .{ .handle = @intCast(handle.handle) }, absolute_offset); + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *VFS = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle, const backend = self.getFileHandle(file); + + if (self.getScheme(handle.backend_idx)) |s| { + const prefix = try std.fmt.bufPrint(out_buffer, "{s}://", .{s}); + const path_len = try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle) }, out_buffer[prefix.len..]); + return prefix.len + path_len; + } else { + return try backend.vtable.fileRealPath(backend.userdata, .{ .handle = @intCast(handle.handle) }, out_buffer); + } + } +}; diff --git a/vfs/BUILD.bazel b/zml/io/vfs/BUILD.bazel similarity index 52% rename from vfs/BUILD.bazel rename to zml/io/vfs/BUILD.bazel index c8de1e0c..9f79dd24 100644 --- a/vfs/BUILD.bazel +++ b/zml/io/vfs/BUILD.bazel @@ -1,25 +1,17 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") +load("@rules_zig//zig:defs.bzl", "zig_library", "zig_binary") zig_library( name = "vfs", srcs = [ "base.zig", "file.zig", - "gcs.zig", "hf.zig", "http.zig", - "parallel_read.zig", "s3.zig", ], - main = "vfs.zig", - tags = ["manual"], - visibility = ["//visibility:public"], deps = [ "//stdx", ], -) - -zig_test( - name = "test", - deps = [":vfs"], + main = "index.zig", + visibility = ["//visibility:public"], ) diff --git a/vfs/base.zig b/zml/io/vfs/base.zig similarity index 75% rename from vfs/base.zig rename to zml/io/vfs/base.zig index 1e66ca89..e586d10b 100644 --- a/vfs/base.zig +++ b/zml/io/vfs/base.zig @@ -11,7 +11,6 @@ pub const VFSBase = struct { pub fn vtable(overrides: anytype) std.Io.VTable { var new_vtable: std.Io.VTable = .{ - .crashHandler = crashHandler, .async = async, .concurrent = concurrent, .await = await, @@ -23,13 +22,10 @@ pub const VFSBase = struct { .recancel = recancel, .swapCancelProtection = swapCancelProtection, .checkCancel = checkCancel, + .select = select, .futexWait = futexWait, .futexWaitUncancelable = futexWaitUncancelable, .futexWake = futexWake, - .operate = operate, - .batchAwaitAsync = batchAwaitAsync, - .batchAwaitConcurrent = batchAwaitConcurrent, - .batchCancel = batchCancel, .dirCreateDir = dirCreateDir, .dirCreateDirPath = dirCreateDirPath, .dirCreateDirPathOpen = dirCreateDirPathOpen, @@ -38,7 +34,6 @@ pub const VFSBase = struct { .dirStatFile = dirStatFile, .dirAccess = dirAccess, .dirCreateFile = dirCreateFile, - .dirCreateFileAtomic = dirCreateFileAtomic, .dirOpenFile = dirOpenFile, .dirClose = dirClose, .dirRead = dirRead, @@ -47,7 +42,6 @@ pub const VFSBase = struct { .dirDeleteFile = dirDeleteFile, .dirDeleteDir = dirDeleteDir, .dirRename = dirRename, - .dirRenamePreserve = dirRenamePreserve, .dirSymLink = dirSymLink, .dirReadLink = dirReadLink, .dirSetOwner = dirSetOwner, @@ -59,9 +53,11 @@ pub const VFSBase = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, + .fileWriteStreaming = fileWriteStreaming, .fileWritePositional = fileWritePositional, .fileWriteFileStreaming = fileWriteFileStreaming, .fileWriteFilePositional = fileWriteFilePositional, + .fileReadStreaming = fileReadStreaming, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -78,39 +74,24 @@ pub const VFSBase = struct { .fileUnlock = fileUnlock, .fileDowngradeLock = fileDowngradeLock, .fileRealPath = fileRealPath, - .fileHardLink = fileHardLink, - .fileMemoryMapCreate = fileMemoryMapCreate, - .fileMemoryMapDestroy = fileMemoryMapDestroy, - .fileMemoryMapSetLength = fileMemoryMapSetLength, - .fileMemoryMapRead = fileMemoryMapRead, - .fileMemoryMapWrite = fileMemoryMapWrite, .processExecutableOpen = processExecutableOpen, .processExecutablePath = processExecutablePath, .lockStderr = lockStderr, .tryLockStderr = tryLockStderr, .unlockStderr = unlockStderr, - .processCurrentPath = processCurrentPath, .processSetCurrentDir = processSetCurrentDir, - .processSetCurrentPath = processSetCurrentPath, - .processReplace = processReplace, - .processSpawn = processSpawn, - .childWait = childWait, - .childKill = childKill, - .progressParentFile = progressParentFile, - .inheritParentDir = inheritParentDir, - .inheritParentFile = inheritParentFile, .now = now, - .clockResolution = clockResolution, .sleep = sleep, - .random = random, - .randomSecure = randomSecure, .netListenIp = netListenIp, .netAccept = netAccept, .netBindIp = netBindIp, .netConnectIp = netConnectIp, .netListenUnix = netListenUnix, .netConnectUnix = netConnectUnix, - .netSocketCreatePair = netSocketCreatePair, + .netSend = netSend, + .netReceive = netReceive, + .netRead = netRead, + .netWrite = netWrite, .netWriteFile = netWriteFile, .netClose = netClose, .netShutdown = netShutdown, @@ -128,11 +109,6 @@ pub const VFSBase = struct { return @ptrCast(@alignCast(userdata.?)); } - pub fn crashHandler(userdata: ?*anyopaque) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - self.inner.vtable.crashHandler(self.inner.userdata); - } - pub fn async(userdata: ?*anyopaque, result: []u8, result_alignment: std.mem.Alignment, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque, result: *anyopaque) void) ?*std.Io.AnyFuture { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.async(self.inner.userdata, result, result_alignment, context, context_alignment, start); @@ -153,12 +129,12 @@ pub const VFSBase = struct { return self.inner.vtable.cancel(self.inner.userdata, any_future, result, result_alignment); } - pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) void) void { + pub fn groupAsync(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) std.Io.Cancelable!void) void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.groupAsync(self.inner.userdata, group, context, context_alignment, start); } - pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) void) std.Io.ConcurrentError!void { + pub fn groupConcurrent(userdata: ?*anyopaque, group: *std.Io.Group, context: []const u8, context_alignment: std.mem.Alignment, start: *const fn (context: *const anyopaque) std.Io.Cancelable!void) std.Io.ConcurrentError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.groupConcurrent(self.inner.userdata, group, context, context_alignment, start); } @@ -188,6 +164,11 @@ pub const VFSBase = struct { return self.inner.vtable.checkCancel(self.inner.userdata); } + pub fn select(userdata: ?*anyopaque, futures: []const *std.Io.AnyFuture) std.Io.Cancelable!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.select(self.inner.userdata, futures); + } + pub fn futexWait(userdata: ?*anyopaque, ptr: *const u32, expected: u32, timeout: std.Io.Timeout) std.Io.Cancelable!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.futexWait(self.inner.userdata, ptr, expected, timeout); @@ -203,26 +184,6 @@ pub const VFSBase = struct { return self.inner.vtable.futexWake(self.inner.userdata, ptr, max_waiters); } - pub fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.operate(self.inner.userdata, operation); - } - - pub fn batchAwaitAsync(userdata: ?*anyopaque, batch: *std.Io.Batch) std.Io.Cancelable!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.batchAwaitAsync(self.inner.userdata, batch); - } - - pub fn batchAwaitConcurrent(userdata: ?*anyopaque, batch: *std.Io.Batch, timeout: std.Io.Timeout) std.Io.Batch.AwaitConcurrentError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.batchAwaitConcurrent(self.inner.userdata, batch, timeout); - } - - pub fn batchCancel(userdata: ?*anyopaque, batch: *std.Io.Batch) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - self.inner.vtable.batchCancel(self.inner.userdata, batch); - } - pub fn dirCreateDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, permissions: std.Io.Dir.Permissions) std.Io.Dir.CreateDirError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirCreateDir(self.inner.userdata, dir, sub_path, permissions); @@ -258,17 +219,12 @@ pub const VFSBase = struct { return self.inner.vtable.dirAccess(self.inner.userdata, dir, sub_path, options); } - pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { + pub fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.CreateFlags) std.Io.File.OpenError!std.Io.File { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirCreateFile(self.inner.userdata, dir, sub_path, flags); } - pub fn dirCreateFileAtomic(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.CreateFileAtomicOptions) std.Io.Dir.CreateFileAtomicError!std.Io.File.Atomic { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.dirCreateFileAtomic(self.inner.userdata, dir, sub_path, options); - } - - pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + pub fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirOpenFile(self.inner.userdata, dir, sub_path, flags); } @@ -308,11 +264,6 @@ pub const VFSBase = struct { return self.inner.vtable.dirRename(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path); } - pub fn dirRenamePreserve(userdata: ?*anyopaque, old_dir: std.Io.Dir, old_sub_path: []const u8, new_dir: std.Io.Dir, new_sub_path: []const u8) std.Io.Dir.RenamePreserveError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.dirRenamePreserve(self.inner.userdata, old_dir, old_sub_path, new_dir, new_sub_path); - } - pub fn dirSymLink(userdata: ?*anyopaque, dir: std.Io.Dir, target_path: []const u8, sym_link_path: []const u8, flags: std.Io.Dir.SymLinkFlags) std.Io.Dir.SymLinkError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.dirSymLink(self.inner.userdata, dir, target_path, sym_link_path, flags); @@ -368,6 +319,11 @@ pub const VFSBase = struct { self.inner.vtable.fileClose(self.inner.userdata, files); } + pub fn fileWriteStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize) std.Io.File.Writer.Error!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileWriteStreaming(self.inner.userdata, file, header, data, splat); + } + pub fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileWritePositional(self.inner.userdata, file, header, data, splat, offset); @@ -383,6 +339,11 @@ pub const VFSBase = struct { return self.inner.vtable.fileWriteFilePositional(self.inner.userdata, file, header, reader, limit, offset); } + pub fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.fileReadStreaming(self.inner.userdata, file, data); + } + pub fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.fileReadPositional(self.inner.userdata, file, data, offset); @@ -463,37 +424,7 @@ pub const VFSBase = struct { return self.inner.vtable.fileRealPath(self.inner.userdata, file, out_buffer); } - pub fn fileHardLink(userdata: ?*anyopaque, file: std.Io.File, dir: std.Io.Dir, path: []const u8, options: std.Io.File.HardLinkOptions) std.Io.File.HardLinkError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.fileHardLink(self.inner.userdata, file, dir, path, options); - } - - pub fn fileMemoryMapCreate(userdata: ?*anyopaque, file: std.Io.File, options: std.Io.File.MemoryMap.CreateOptions) std.Io.File.MemoryMap.CreateError!std.Io.File.MemoryMap { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.fileMemoryMapCreate(self.inner.userdata, file, options); - } - - pub fn fileMemoryMapDestroy(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - self.inner.vtable.fileMemoryMapDestroy(self.inner.userdata, memory_map); - } - - pub fn fileMemoryMapSetLength(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap, length: usize) std.Io.File.MemoryMap.SetLengthError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.fileMemoryMapSetLength(self.inner.userdata, memory_map, length); - } - - pub fn fileMemoryMapRead(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap) std.Io.File.ReadPositionalError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.fileMemoryMapRead(self.inner.userdata, memory_map); - } - - pub fn fileMemoryMapWrite(userdata: ?*anyopaque, memory_map: *std.Io.File.MemoryMap) std.Io.File.WritePositionalError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.fileMemoryMapWrite(self.inner.userdata, memory_map); - } - - pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.Dir.OpenFileOptions) std.process.OpenExecutableError!std.Io.File { + pub fn processExecutableOpen(userdata: ?*anyopaque, flags: std.Io.File.OpenFlags) std.process.OpenExecutableError!std.Io.File { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processExecutableOpen(self.inner.userdata, flags); } @@ -518,89 +449,29 @@ pub const VFSBase = struct { return self.inner.vtable.unlockStderr(self.inner.userdata); } - pub fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) std.process.CurrentPathError!usize { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.processCurrentPath(self.inner.userdata, buffer); - } - pub fn processSetCurrentDir(userdata: ?*anyopaque, dir: std.Io.Dir) std.process.SetCurrentDirError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.processSetCurrentDir(self.inner.userdata, dir); } - pub fn processSetCurrentPath(userdata: ?*anyopaque, path: []const u8) std.process.SetCurrentPathError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.processSetCurrentPath(self.inner.userdata, path); - } - - pub fn processReplace(userdata: ?*anyopaque, options: std.process.ReplaceOptions) std.process.ReplaceError { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.processReplace(self.inner.userdata, options); - } - - pub fn processSpawn(userdata: ?*anyopaque, options: std.process.SpawnOptions) std.process.SpawnError!std.process.Child { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.processSpawn(self.inner.userdata, options); - } - - pub fn childWait(userdata: ?*anyopaque, child: *std.process.Child) std.process.Child.WaitError!std.process.Child.Term { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.childWait(self.inner.userdata, child); - } - - pub fn childKill(userdata: ?*anyopaque, child: *std.process.Child) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - self.inner.vtable.childKill(self.inner.userdata, child); - } - - pub fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!std.Io.File { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.progressParentFile(self.inner.userdata); - } - - pub fn inheritParentDir(userdata: ?*anyopaque, handle: std.Io.Dir.Handle) std.Io.InheritParentHandleError!std.Io.Dir { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.inheritParentDir(self.inner.userdata, handle); - } - - pub fn inheritParentFile(userdata: ?*anyopaque, handle: std.Io.File.Handle, flags: std.Io.File.Flags) std.Io.InheritParentHandleError!std.Io.File { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.inheritParentFile(self.inner.userdata, handle, flags); - } - - pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Timestamp { + pub fn now(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.Error!std.Io.Timestamp { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.now(self.inner.userdata, clock); } - pub fn clockResolution(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Clock.ResolutionError!std.Io.Duration { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.clockResolution(self.inner.userdata, clock); - } - - pub fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.Cancelable!void { + pub fn sleep(userdata: ?*anyopaque, timeout: std.Io.Timeout) std.Io.SleepError!void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.sleep(self.inner.userdata, timeout); } - pub fn random(userdata: ?*anyopaque, buffer: []u8) void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - self.inner.vtable.random(self.inner.userdata, buffer); - } - - pub fn randomSecure(userdata: ?*anyopaque, buffer: []u8) std.Io.RandomSecureError!void { - const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.randomSecure(self.inner.userdata, buffer); - } - - pub fn netListenIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Socket { + pub fn netListenIp(userdata: ?*anyopaque, address: std.Io.net.IpAddress, options: std.Io.net.IpAddress.ListenOptions) std.Io.net.IpAddress.ListenError!std.Io.net.Server { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netListenIp(self.inner.userdata, address, options); } - pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle, options: std.Io.net.Server.AcceptOptions) std.Io.net.Server.AcceptError!std.Io.net.Socket { + pub fn netAccept(userdata: ?*anyopaque, server: std.Io.net.Socket.Handle) std.Io.net.Server.AcceptError!std.Io.net.Stream { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netAccept(self.inner.userdata, server, options); + return self.inner.vtable.netAccept(self.inner.userdata, server); } pub fn netBindIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.BindOptions) std.Io.net.IpAddress.BindError!std.Io.net.Socket { @@ -608,7 +479,7 @@ pub const VFSBase = struct { return self.inner.vtable.netBindIp(self.inner.userdata, address, options); } - pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Socket { + pub fn netConnectIp(userdata: ?*anyopaque, address: *const std.Io.net.IpAddress, options: std.Io.net.IpAddress.ConnectOptions) std.Io.net.IpAddress.ConnectError!std.Io.net.Stream { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); return self.inner.vtable.netConnectIp(self.inner.userdata, address, options); } @@ -623,9 +494,24 @@ pub const VFSBase = struct { return self.inner.vtable.netConnectUnix(self.inner.userdata, address); } - pub fn netSocketCreatePair(userdata: ?*anyopaque, options: std.Io.net.Socket.CreatePairOptions) std.Io.net.Socket.CreatePairError![2]std.Io.net.Socket { + pub fn netSend(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, msgs: []std.Io.net.OutgoingMessage, flags: std.Io.net.SendFlags) struct { ?std.Io.net.Socket.SendError, usize } { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - return self.inner.vtable.netSocketCreatePair(self.inner.userdata, options); + return self.inner.vtable.netSend(self.inner.userdata, handle, msgs, flags); + } + + pub fn netReceive(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, message_buffer: []std.Io.net.IncomingMessage, data_buffer: []u8, flags: std.Io.net.ReceiveFlags, timeout: std.Io.Timeout) struct { ?std.Io.net.Socket.ReceiveTimeoutError, usize } { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netReceive(self.inner.userdata, handle, message_buffer, data_buffer, flags, timeout); + } + + pub fn netRead(userdata: ?*anyopaque, src: std.Io.net.Socket.Handle, data: [][]u8) std.Io.net.Stream.Reader.Error!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netRead(self.inner.userdata, src, data); + } + + pub fn netWrite(userdata: ?*anyopaque, dest: std.Io.net.Socket.Handle, header: []const u8, data: []const []const u8, splat: usize) std.Io.net.Stream.Writer.Error!usize { + const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); + return self.inner.vtable.netWrite(self.inner.userdata, dest, header, data, splat); } pub fn netWriteFile(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.net.Stream.Writer.WriteFileError!usize { @@ -633,9 +519,9 @@ pub const VFSBase = struct { return self.inner.vtable.netWriteFile(self.inner.userdata, handle, header, reader, limit); } - pub fn netClose(userdata: ?*anyopaque, sockets: []const std.Io.net.Socket) void { + pub fn netClose(userdata: ?*anyopaque, handles: []const std.Io.net.Socket.Handle) void { const self: *VFSBase = @ptrCast(@alignCast(userdata.?)); - self.inner.vtable.netClose(self.inner.userdata, sockets); + self.inner.vtable.netClose(self.inner.userdata, handles); } pub fn netShutdown(userdata: ?*anyopaque, handle: std.Io.net.Socket.Handle, how: std.Io.net.ShutdownHow) std.Io.net.ShutdownError!void { diff --git a/vfs/file.zig b/zml/io/vfs/file.zig similarity index 60% rename from vfs/file.zig rename to zml/io/vfs/file.zig index 1b077c33..0c52eada 100644 --- a/vfs/file.zig +++ b/zml/io/vfs/file.zig @@ -5,14 +5,14 @@ const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/vfs/file"); +const log = std.log.scoped(.@"zml/io/vfs/file"); pub const DirectIoError = error{ UnsupportedPlatform, MisalignedBuffer, BufferTooSmall, UnexpectedFcntlResult, -}; +} || std.posix.FcntlError; fn canUseDirectIO() bool { if (builtin.target.os.tag == .linux) { @@ -21,30 +21,9 @@ fn canUseDirectIO() bool { return false; } -fn getStatusFlags(file: std.Io.File) DirectIoError!c_int { - while (true) { - const result: c_int = std.posix.system.fcntl(file.handle, std.posix.F.GETFL, @as(c_int, 0)); - switch (std.posix.errno(result)) { - .SUCCESS => return result, - .INTR => continue, - else => return DirectIoError.UnexpectedFcntlResult, - } - } -} - -fn setStatusFlags(file: std.Io.File, flags: c_int) DirectIoError!void { - while (true) { - switch (std.posix.errno(std.posix.system.fcntl(file.handle, std.posix.F.SETFL, flags))) { - .SUCCESS => return, - .INTR => continue, - else => return DirectIoError.UnexpectedFcntlResult, - } - } -} - fn useDirectIO(file: std.Io.File) DirectIoError!bool { if (comptime canUseDirectIO()) { - const flags = try getStatusFlags(file); + const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); return (flags & direct_flag) != 0; } else { @@ -54,11 +33,12 @@ fn useDirectIO(file: std.Io.File) DirectIoError!bool { fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { if (comptime canUseDirectIO()) { - const flags = try getStatusFlags(file); + const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); if ((flags & direct_flag) == 0) return; - try setStatusFlags(file, flags & ~direct_flag); + const result = try std.posix.fcntl(file.handle, std.posix.F.SETFL, flags & ~@as(c_uint, @bitCast(@as(u32, @intCast(direct_flag))))); + if (result != 0) return DirectIoError.UnexpectedFcntlResult; } else { return DirectIoError.UnsupportedPlatform; } @@ -66,9 +46,11 @@ fn switchToBufferedIO(file: std.fs.File) DirectIoError!void { fn switchToDirectIO(file: std.Io.File) DirectIoError!void { if (builtin.os.tag == .linux and canUseDirectIO()) { - const flags = try getStatusFlags(file); + const flags = try std.posix.fcntl(file.handle, std.posix.F.GETFL, 0); const direct_flag: c_int = @bitCast(std.posix.O{ .DIRECT = true }); - try setStatusFlags(file, flags | direct_flag); + + const result = try std.posix.fcntl(file.handle, std.posix.F.SETFL, flags | direct_flag); + if (result != 0) return DirectIoError.UnexpectedFcntlResult; } else { return DirectIoError.UnsupportedPlatform; } @@ -80,30 +62,12 @@ pub const File = struct { direct_io_alignment: std.mem.Alignment = .fromByteUnits(4 * 1024), }; - const Handle = struct { - inner_handle: std.Io.File.Handle, - inner_flags: std.Io.File.Flags, - direct_io_handle: ?std.Io.File.Handle, - - pub fn innerFile(self: *Handle) std.Io.File { - return .{ - .handle = self.inner_handle, - .flags = self.inner_flags, - }; - } - - pub fn directIoFile(self: *Handle) std.Io.File { - return .{ - .handle = self.direct_io_handle.?, - .flags = self.inner_flags, - }; - } - }; + const Handle = struct { inner_handle: std.Io.File.Handle, direct_io_handle: ?std.Io.File.Handle }; allocator: std.mem.Allocator, mutex: std.Io.Mutex = .init, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .empty, + closed_handles: std.ArrayList(u32) = .{}, config: Config, base: VFSBase, @@ -124,12 +88,10 @@ pub const File = struct { return .{ .userdata = &self.base, .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, .dirOpenDir = dirOpenDir, .dirStat = dirStat, .dirStatFile = dirStatFile, .dirAccess = dirAccess, - .dirCreateFile = dirCreateFile, .dirOpenFile = dirOpenFile, .dirClose = dirClose, .dirRead = dirRead, @@ -138,9 +100,7 @@ pub const File = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, - .fileWritePositional = fileWritePositional, - .fileWriteFileStreaming = fileWriteFileStreaming, - .fileWriteFilePositional = fileWriteFilePositional, + .fileReadStreaming = fileReadStreaming, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -173,7 +133,7 @@ pub const File = struct { return self.handles.at(@intCast(file.handle)); } - fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!bool { + fn canOpenWithDirectIO(self: *File, file: std.Io.File, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!bool { const file_stat = self.base.inner.vtable.fileStat(self.base.inner.userdata, file) catch |err| switch (err) { else => return std.Io.File.OpenError.Unexpected, }; @@ -183,61 +143,27 @@ pub const File = struct { fn innerFile(self: *File, file: std.Io.File, data: []const []u8, position: u64) std.Io.File { const handle = self.getFileHandle(file); - - if (handle.direct_io_handle == null) { - return handle.innerFile(); - } + const direct_io_handle = handle.direct_io_handle orelse return .{ .handle = handle.inner_handle }; const alignment_bytes: usize = self.config.direct_io_alignment.toByteUnits(); if (!std.mem.isAligned(@as(usize, position), alignment_bytes)) { - return handle.innerFile(); + return .{ .handle = handle.inner_handle }; } var total_size: usize = 0; for (data) |buf| { if (!std.mem.isAligned(@intFromPtr(buf.ptr), alignment_bytes)) { - return handle.innerFile(); + return .{ .handle = handle.inner_handle }; } total_size += buf.len; } if (!std.mem.isAligned(total_size, alignment_bytes)) { - return handle.innerFile(); + return .{ .handle = handle.inner_handle }; } - return handle.directIoFile(); - } - - fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - switch (operation) { - .file_read_streaming => |o| { - const handle = self.getFileHandle(o.file); - const position: u64 = @intCast(std.posix.system.lseek(handle.inner_handle, 0, std.posix.SEEK.CUR)); - const inner_file = self.innerFile(o.file, o.data, position); - return self.base.inner.vtable.operate(self.base.inner.userdata, .{ - .file_read_streaming = .{ - .file = inner_file, - .data = o.data, - }, - }); - }, - .file_write_streaming => |o| { - const handle = self.getFileHandle(o.file); - return self.base.inner.vtable.operate(self.base.inner.userdata, .{ - .file_write_streaming = .{ - .file = handle.innerFile(), - .header = o.header, - .data = o.data, - .splat = o.splat, - }, - }); - }, - .device_io_control, .net_receive, .net_read, .net_send, .net_write => { - return self.base.inner.vtable.operate(self.base.inner.userdata, operation); - }, - } + return .{ .handle = direct_io_handle }; } fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, options: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { @@ -260,21 +186,7 @@ pub const File = struct { return self.base.inner.vtable.dirAccess(self.base.inner.userdata, dir, sub_path, options); } - fn dirCreateFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.CreateFileOptions) std.Io.File.OpenError!std.Io.File { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const inner_file = try self.base.inner.vtable.dirCreateFile(self.base.inner.userdata, dir, sub_path, flags); - errdefer inner_file.close(self.base.inner); - - const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; - handle.* = .{ - .inner_handle = inner_file.handle, - .inner_flags = inner_file.flags, - .direct_io_handle = null, - }; - return .{ .handle = @intCast(idx), .flags = inner_file.flags }; - } - - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, flags: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const inner_file = try self.base.inner.vtable.dirOpenFile(self.base.inner.userdata, dir, sub_path, flags); errdefer inner_file.close(self.base.inner); @@ -304,11 +216,10 @@ pub const File = struct { const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = .{ .inner_handle = inner_file.handle, - .inner_flags = inner_file.flags, .direct_io_handle = direct_io_handle, }; - return .{ .handle = @intCast(idx), .flags = inner_file.flags }; + return .{ .handle = @intCast(idx) }; } fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { @@ -334,13 +245,15 @@ pub const File = struct { fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileStat(self.base.inner.userdata, handle.innerFile()); + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + return self.base.inner.vtable.fileStat(self.base.inner.userdata, inner_file); } fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileLength(self.base.inner.userdata, handle.innerFile()); + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + return self.base.inner.vtable.fileLength(self.base.inner.userdata, inner_file); } fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { @@ -348,60 +261,24 @@ pub const File = struct { for (files) |file| { const handle = self.getFileHandle(file); - if (handle.direct_io_handle) |_| { - self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.directIoFile()}); + if (handle.direct_io_handle) |dio_handle| { + const direct_io_file: std.Io.File = .{ .handle = dio_handle }; + self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{direct_io_file}); } - self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{handle.innerFile()}); + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + self.base.inner.vtable.fileClose(self.base.inner.userdata, &.{inner_file}); self.closeHandle(@intCast(file.handle)) catch unreachable; } } - fn fileWritePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, data: []const []const u8, splat: usize, offset: u64) std.Io.File.WritePositionalError!usize { + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileWritePositional(self.base.inner.userdata, handle.innerFile(), header, data, splat, offset); - } - - fn fileWriteFileStreaming(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit) std.Io.File.Writer.WriteFileError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const destination = self.getFileHandle(file); - const source = self.getFileHandle(reader.file); - - const original_source_io = reader.io; - const original_source_file = reader.file; - reader.io = self.base.inner; - reader.file = source.innerFile(); - defer { - reader.io = original_source_io; - reader.file = original_source_file; - } - - return self.base.inner.vtable.fileWriteFileStreaming(self.base.inner.userdata, destination.innerFile(), header, reader, limit); - } - - fn fileWriteFilePositional(userdata: ?*anyopaque, file: std.Io.File, header: []const u8, reader: *std.Io.File.Reader, limit: std.Io.Limit, offset: u64) std.Io.File.WriteFilePositionalError!usize { - const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); - const own_io = self.io(); - if (reader.io.userdata != own_io.userdata or reader.io.vtable != own_io.vtable) return error.Unimplemented; - - const destination = self.getFileHandle(file); - const source = self.getFileHandle(reader.file); - - const original_source_io = reader.io; - const original_source_file = reader.file; - reader.io = self.base.inner; - reader.file = source.innerFile(); - defer { - reader.io = original_source_io; - reader.file = original_source_file; - } - - return self.base.inner.vtable.fileWriteFilePositional(self.base.inner.userdata, destination.innerFile(), header, reader, limit, offset); + const position: u64 = @intCast(std.posix.system.lseek(handle.inner_handle, 0, std.posix.SEEK.CUR)); + const inner_file = self.innerFile(file, data, position); + return self.base.inner.vtable.fileReadStreaming(self.base.inner.userdata, inner_file, data); } fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { @@ -413,18 +290,21 @@ pub const File = struct { fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileSeekBy(self.base.inner.userdata, handle.innerFile(), relative_offset); + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + return self.base.inner.vtable.fileSeekBy(self.base.inner.userdata, inner_file, relative_offset); } fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileSeekTo(self.base.inner.userdata, handle.innerFile(), absolute_offset); + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + return self.base.inner.vtable.fileSeekTo(self.base.inner.userdata, inner_file, absolute_offset); } fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { const self: *File = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); - return self.base.inner.vtable.fileRealPath(self.base.inner.userdata, handle.innerFile(), out_buffer); + const inner_file: std.Io.File = .{ .handle = handle.inner_handle }; + return self.base.inner.vtable.fileRealPath(self.base.inner.userdata, inner_file, out_buffer); } }; diff --git a/vfs/hf.zig b/zml/io/vfs/hf.zig similarity index 69% rename from vfs/hf.zig rename to zml/io/vfs/hf.zig index 8770fe18..50e2811b 100644 --- a/vfs/hf.zig +++ b/zml/io/vfs/hf.zig @@ -2,159 +2,9 @@ const std = @import("std"); const stdx = @import("stdx"); -const parallel_read = @import("parallel_read.zig"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/vfs/hf"); - -const ParallelRead = struct { - const Pool = parallel_read.Pool(Job); - - const Batch = struct { - state: parallel_read.BatchState, - uri: std.Uri, - url: []const u8, - authorization: std.http.Client.Request.Headers.Value, - }; - - pub const Job = struct { - data: []const []u8, - file_offset: u64, - chunk_offset: usize, - chunk_len: usize, - batch: *Batch, - - pub fn perform(job: Job, client: *std.http.Client) anyerror!parallel_read.Status { - var range_buf: [64]u8 = undefined; - const range_header = std.fmt.bufPrint( - &range_buf, - "bytes={d}-{d}", - .{ job.file_offset, job.file_offset + @as(u64, @intCast(job.chunk_len - 1)) }, - ) catch unreachable; - - var req = client.request(.GET, job.batch.uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = job.batch.authorization, - }, - .extra_headers = &.{.{ .name = "Range", .value = range_header }}, - }) catch |err| { - switch (err) { - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to connect: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to connect: {}", .{err}); - return err; - }, - } - }; - defer req.deinit(); - - req.sendBodiless() catch |err| switch (err) { - error.WriteFailed => { - log.warn("Failed to send headers: {}", .{err}); - return .retry(); - }, - }; - - var redirect_buffer: [8 * 1024]u8 = undefined; - var res = req.receiveHead(&redirect_buffer) catch |err| { - switch (err) { - // stale keep-alive / peer closed while waiting for response head - error.HttpConnectionClosing, - error.HttpRequestTruncated, - - // transport read/write failure; retry on a fresh connection - error.ReadFailed, - error.WriteFailed, - - // transient connect failures - error.ConnectionRefused, - error.ConnectionResetByPeer, - error.HostUnreachable, - error.NetworkUnreachable, - error.NetworkDown, - error.Timeout, - error.NameServerFailure, - => { - log.warn("Failed to receive headers: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to receive headers: {}", .{err}); - return err; - }, - } - }; - - if (res.head.status != .partial_content and res.head.status != .ok) { - const status: parallel_read.Status = switch (res.head.status) { - .request_timeout => .retry(), - .too_many_requests => b: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "RateLimit")) { - var parts = std.mem.splitScalar(u8, header.value, ';'); - while (parts.next()) |part| { - const p = std.mem.trim(u8, part, " \t"); - if (std.mem.startsWith(u8, p, "t=")) { - const seconds = std.fmt.parseInt(u32, p[2..], 10) catch continue; - break :b .{ .retry_after = .fromSeconds(seconds) }; - } - } - } - } - break :b .retry(); - }, - else => if (res.head.status.class() == .server_error) .retry() else { - log.err("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); - return error.RequestFailed; - }, - }; - log.warn("Failed to perform read for {s}\n{s}", .{ job.batch.url, res.head.bytes }); - return status; - } - - const content_range = blk: { - var it = res.head.iterateHeaders(); - while (it.next()) |header| { - if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { - break :blk parallel_read.parseContentRange(header.value); - } - } - break :blk null; - }; - - const reader = res.reader(&.{}); - parallel_read.readChunk(reader, content_range, job.file_offset, job.data, job.chunk_offset, job.chunk_len) catch |err| { - switch (err) { - error.EndOfStream, - error.ReadFailed, - => { - log.warn("Failed to read from response: {}", .{err}); - return .retry(); - }, - else => { - log.err("Failed to read from response: {}", .{err}); - return err; - }, - } - }; - - return .success; - } - }; -}; +const log = std.log.scoped(.@"zml/io/vfs/hf"); pub const API = struct { const TREE_URL_TEMPLATE = "https://huggingface.co/api/models/{[repo]s}/{[model]s}/tree/{[rev]s}/{[path]s}?expand=false&recursive=true&limit=1000"; @@ -200,18 +50,6 @@ const RepoKey = struct { }; pub const HF = struct { - pub const InitOpts = struct { - read_pool: parallel_read.InitOpts = .{ - // Chunk size needs to be big enough to avoid hitting the rate limits of HF. - .chunk_size = 32 << 20, - .num_workers = 32, - .queue_capacity = 128, - .max_retries = 5, - .retry_initial_delay = .fromMilliseconds(500), - .retry_max_delay = .fromSeconds(30), - }, - }; - pub const Repo = struct { repo: []const u8, model: []const u8, @@ -264,21 +102,14 @@ pub const HF = struct { mutex: std.Io.Mutex = .init, client: *std.http.Client, authorization: std.http.Client.Request.Headers.Value, - read_pool: *ParallelRead.Pool, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .empty, + closed_handles: std.ArrayList(u32) = .{}, base: VFSBase, trees: std.StringHashMapUnmanaged(std.ArrayList(TreeNode)) = .{}, dir_read_states: std.AutoHashMapUnmanaged(*std.Io.Dir.Reader, ReadState) = .{}, - pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8, opts: InitOpts) !HF { - const read_pool = try allocator.create(ParallelRead.Pool); - errdefer allocator.destroy(read_pool); - - try read_pool.init(allocator, inner, http_client, opts.read_pool); - errdefer read_pool.deinit(allocator, inner); - - var self: HF = .{ + pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, hf_token: ?[]const u8) !HF { + return .{ .allocator = allocator, .base = .init(inner), .client = http_client, @@ -289,21 +120,15 @@ pub const HF = struct { } else blk: { break :blk .default; }, - .read_pool = read_pool, }; - errdefer switch (self.authorization) { - .default, .omit => {}, - .override => |t| self.allocator.free(t), - }; - - return self; } - pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, environ_map: *std.process.Environ.Map) !HF { - const hf_token = if (environ_map.get("HF_TOKEN")) |token| blk: { - break :blk try allocator.dupe(u8, token); - } else blk: { - const home_path = environ_map.get("HOME") orelse break :blk null; + pub fn auto(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client) !HF { + const hf_token = if (std.process.getEnvVarOwned(allocator, "HF_TOKEN")) |token| blk: { + break :blk token; + } else |_| blk: { + const home_path = std.process.getEnvVarOwned(allocator, "HOME") catch break :blk null; + defer allocator.free(home_path); var path_buf: [256]u8 = undefined; const token_path = std.fmt.bufPrint(&path_buf, "{s}/.cache/huggingface/token", .{home_path}) catch break :blk null; @@ -317,17 +142,14 @@ pub const HF = struct { break :blk token; }; - defer if (hf_token) |tk| allocator.free(tk); + defer if (hf_token) |token| allocator.free(token); if (hf_token == null) log.warn("No Hugging Face authentication token found in environment or home config; proceeding without authentication.", .{}); - return init(allocator, inner, http_client, hf_token, .{}); + return init(allocator, inner, http_client, hf_token); } pub fn deinit(self: *HF) void { - self.read_pool.deinit(self.allocator, self.base.inner); - self.allocator.destroy(self.read_pool); - var idx: usize = 0; while (idx < self.handles.len) : (idx += 1) { const is_closed = for (self.closed_handles.items) |closed_idx| { @@ -362,7 +184,6 @@ pub const HF = struct { return .{ .userdata = &self.base, .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, .dirOpenDir = dirOpenDir, .dirStat = dirStat, .dirStatFile = dirStatFile, @@ -375,6 +196,7 @@ pub const HF = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, + .fileReadStreaming = fileReadStreaming, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -469,13 +291,9 @@ pub const HF = struct { .path = "", }); const uri = try std.Uri.parse(url); - log.info("Fetching from HF: {s}", .{url}); var req = try self.client.request(.GET, uri, .{ - .headers = .{ - .accept_encoding = .{ .override = "identity" }, - .authorization = self.authorization, - }, + .headers = .{ .authorization = self.authorization }, }); defer req.deinit(); @@ -499,7 +317,7 @@ pub const HF = struct { ); defer parsed.deinit(); - var tree_root: std.ArrayList(TreeNode) = .empty; + var tree_root: std.ArrayList(TreeNode) = .{}; for (parsed.value) |item| { try insertTreeNode(self.allocator, &tree_root, item); } @@ -587,7 +405,7 @@ pub const HF = struct { var parts = std.mem.tokenizeScalar(u8, item.path, '/'); var current_list = root; - var parents: std.ArrayList(*TreeNode) = .empty; + var parents: std.ArrayList(*TreeNode) = .{}; defer parents.deinit(allocator); while (true) { @@ -606,7 +424,7 @@ pub const HF = struct { if (!is_last) { try parents.append(allocator, f); if (f.children == null) { - f.children = .empty; + f.children = .{}; } current_list = &f.children.?; } else if (std.mem.eql(u8, item.type, "file")) { @@ -618,7 +436,7 @@ pub const HF = struct { .name = try allocator.dupe(u8, part), .kind = if (is_file) .file else .directory, .size = if (is_last) item.size else 0, - .children = if (is_file) null else .empty, + .children = if (is_file) null else .{}, }; try current_list.append(allocator, new_node); @@ -635,29 +453,6 @@ pub const HF = struct { } } - fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); - switch (operation) { - .file_read_streaming => |o| { - const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; - }; - - if (total == 0) { - return .{ .file_read_streaming = error.EndOfStream }; - } - - handle.pos += @intCast(total); - return .{ .file_read_streaming = total }; - }, - .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { - return self.base.inner.vtable.operate(self.base.inner.userdata, operation); - }, - } - } - fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); @@ -687,7 +482,6 @@ pub const HF = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, - .block_size = 1, }; } @@ -713,7 +507,6 @@ pub const HF = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, - .block_size = 1, }; } @@ -732,7 +525,7 @@ pub const HF = struct { if (node == null) return std.Io.Dir.AccessError.FileNotFound; } - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); var path_buffer: [8 * 1024]u8 = undefined; @@ -745,7 +538,7 @@ pub const HF = struct { const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, full_path, size) catch return std.Io.File.OpenError.Unexpected; - return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + return .{ .handle = @intCast(idx) }; } fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { @@ -826,7 +619,6 @@ pub const HF = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, - .block_size = 1, }; } @@ -842,12 +634,23 @@ pub const HF = struct { } } + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const total = self.performRead(handle, data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + handle.pos += @intCast(total); + return total; + } + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HF = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); return self.performRead(handle, data, offset) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; + return std.Io.File.Reader.Error.Unexpected; }; } @@ -877,44 +680,76 @@ pub const HF = struct { fn performRead(self: *HF, handle: *Handle, data: []const []u8, offset: u64) !usize { if (offset >= handle.size) return 0; - var read_size: usize = 0; - for (data) |buf| read_size += buf.len; - read_size = @intCast(@min(handle.size - offset, read_size)); - if (read_size == 0) return 0; - - const job_count = std.math.divCeil(usize, read_size, self.read_pool.chunk_size) catch unreachable; - - const jobs = try self.allocator.alloc(ParallelRead.Job, job_count); - defer self.allocator.free(jobs); - const pending: u32 = @intCast(job_count); + var range_buf: [64]u8 = undefined; + const range_header = blk: { + var total_bytes: u64 = 0; + for (data) |buf| { + total_bytes += @as(u64, buf.len); + } + const remaining = handle.size - offset; + const take = @min(remaining, total_bytes); + const end = offset + take - 1; + break :blk std.fmt.bufPrint(&range_buf, "bytes={d}-{d}", .{ offset, end }) catch unreachable; + }; const repo: Repo = try .parse(handle.uri); var url_buffer: [8 * 1024]u8 = undefined; - const url = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); + const url: []const u8 = try std.fmt.bufPrint(&url_buffer, API.LFS_FILE_URL_TEMPLATE, repo); + const uri: std.Uri = try .parse(url); + var req = try self.client.request(.GET, uri, .{ + .headers = .{ .authorization = self.authorization }, + .extra_headers = &.{.{ .name = "Range", .value = range_header }}, + }); + defer req.deinit(); - var batch: ParallelRead.Batch = .{ - .state = .{ .pending = .init(pending) }, - .uri = uri, - .url = url, - .authorization = self.authorization, - }; + try req.sendBodiless(); - for (jobs, 0..) |*job, i| { - const chunk_offset = i * self.read_pool.chunk_size; - job.* = .{ - .data = data, - .file_offset = offset + @as(u64, @intCast(chunk_offset)), - .chunk_offset = chunk_offset, - .chunk_len = @min(self.read_pool.chunk_size, read_size - chunk_offset), - .batch = &batch, - }; + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .partial_content) { + log.err("Failed to perform read for {s}", .{url}); + log.err("{s}", .{res.head.bytes}); + return error.RequestFailed; } - try self.read_pool.job_queue.putAll(self.base.inner, jobs); - batch.state.waitUncancelable(self.base.inner); - if (batch.state.anyError()) |err| return err; + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parseContentRange(header.value); + } + } + break :blk null; + }; - return read_size; + const reader = res.reader(&.{}); + + if (content_range) |cr| { + if (cr.start < offset) { + try reader.discardAll(offset - cr.start); + } + } + + return try reader.readSliceShort(data[0]); + } + + const ContentRange = struct { + start: u64, + end: u64, + total: u64, + }; + + fn parseContentRange(value: []const u8) ?ContentRange { + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; } }; diff --git a/vfs/http.zig b/zml/io/vfs/http.zig similarity index 91% rename from vfs/http.zig rename to zml/io/vfs/http.zig index 9054983e..6106ff08 100644 --- a/vfs/http.zig +++ b/zml/io/vfs/http.zig @@ -4,7 +4,7 @@ const stdx = @import("stdx"); const VFSBase = @import("base.zig").VFSBase; -const log = std.log.scoped(.@"zml/vfs/http"); +const log = std.log.scoped(.@"zml/io/vfs/http"); pub const HTTP = struct { const Handle = struct { @@ -42,7 +42,7 @@ pub const HTTP = struct { client: *std.http.Client, protocol: Protocol, handles: stdx.SegmentedList(Handle, 0) = .{}, - closed_handles: std.ArrayList(u32) = .empty, + closed_handles: std.ArrayList(u32) = .{}, base: VFSBase, pub fn init(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client, protocol: Protocol) !HTTP { @@ -73,7 +73,6 @@ pub const HTTP = struct { return .{ .userdata = &self.base, .vtable = &comptime VFSBase.vtable(.{ - .operate = operate, .dirOpenDir = dirOpenDir, .dirStat = dirStat, .dirStatFile = dirStatFile, @@ -86,6 +85,7 @@ pub const HTTP = struct { .fileStat = fileStat, .fileLength = fileLength, .fileClose = fileClose, + .fileReadStreaming = fileReadStreaming, .fileReadPositional = fileReadPositional, .fileSeekBy = fileSeekBy, .fileSeekTo = fileSeekTo, @@ -139,29 +139,6 @@ pub const HTTP = struct { return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); } - fn operate(userdata: ?*anyopaque, operation: std.Io.Operation) std.Io.Cancelable!std.Io.Operation.Result { - const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); - switch (operation) { - .file_read_streaming => |o| { - const handle = self.getFileHandle(o.file); - const total = self.performRead(handle, o.data, handle.pos) catch |err| { - log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); - return .{ .file_read_streaming = error.EndOfStream }; - }; - - if (total == 0) { - return .{ .file_read_streaming = error.EndOfStream }; - } - - handle.pos += @intCast(total); - return .{ .file_read_streaming = total }; - }, - .file_write_streaming, .device_io_control, .net_receive, .net_read, .net_send, .net_write => { - return self.base.inner.vtable.operate(self.base.inner.userdata, operation); - }, - } - } - fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); @@ -187,7 +164,6 @@ pub const HTTP = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, - .block_size = 0, }; } @@ -204,13 +180,12 @@ pub const HTTP = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, - .block_size = 1, }; } fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} - fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenFileOptions) std.Io.File.OpenError!std.Io.File { + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; @@ -221,7 +196,7 @@ pub const HTTP = struct { const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; - return .{ .handle = @intCast(idx), .flags = .{ .nonblocking = false } }; + return .{ .handle = @intCast(idx) }; } fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { @@ -263,7 +238,6 @@ pub const HTTP = struct { .atime = null, .mtime = std.Io.Timestamp.zero, .ctime = std.Io.Timestamp.zero, - .block_size = 1, }; } @@ -279,12 +253,23 @@ pub const HTTP = struct { } } + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const total = self.performRead(handle, data, handle.pos) catch |err| { + log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, handle.pos, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + handle.pos += @intCast(total); + return total; + } + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { const self: *HTTP = @fieldParentPtr("base", VFSBase.as(userdata)); const handle = self.getFileHandle(file); return self.performRead(handle, data, offset) catch |err| { log.err("Failed to perform read for file {s} at pos {d}: {any}", .{ handle.uri, offset, err }); - return std.Io.File.ReadPositionalError.Unexpected; + return std.Io.File.Reader.Error.Unexpected; }; } diff --git a/zml/io/vfs/index.zig b/zml/io/vfs/index.zig new file mode 100644 index 00000000..605fc8c6 --- /dev/null +++ b/zml/io/vfs/index.zig @@ -0,0 +1,9 @@ +const log = std.log.scoped(.@"zml/io/vfs/vfs"); + +const std = @import("std"); + +pub const File = @import("file.zig").File; +pub const HF = @import("hf.zig").HF; +pub const HTTP = @import("http.zig").HTTP; +pub const S3 = @import("s3.zig").S3; +pub const VFSBase = @import("base.zig").VFSBase; diff --git a/zml/io/vfs/s3.zig b/zml/io/vfs/s3.zig new file mode 100644 index 00000000..288ab054 --- /dev/null +++ b/zml/io/vfs/s3.zig @@ -0,0 +1,684 @@ +const std = @import("std"); + +const stdx = @import("stdx"); + +const VFSBase = @import("base.zig").VFSBase; + +const log = std.log.scoped(.@"zml/io/vfs/s3"); + +pub const AwsSigV4 = struct { + access_key: []const u8, + secret_key: []const u8, + region: []const u8, + service: []const u8 = "s3", + + const ALGORITHM = "AWS4-HMAC-SHA256"; + pub const UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; + + pub fn generateAuthHeader( + self: AwsSigV4, + allocator: std.mem.Allocator, + method: std.http.Method, + endpoint: []const u8, + uri: std.Uri, + timestamp: []const u8, + extra_headers: []const std.http.Header, + ) ![]const u8 { + const date = timestamp[0..8]; + + var buf: [256]u8 = undefined; + const host = getHostForEndpoint(endpoint, &buf); + + // Build sorted headers list + var headers: std.ArrayList(std.http.Header) = .{}; + defer headers.deinit(allocator); + + try headers.append(allocator, .{ .name = "host", .value = host }); + try headers.append(allocator, .{ .name = "x-amz-content-sha256", .value = UNSIGNED_PAYLOAD }); + try headers.append(allocator, .{ .name = "x-amz-date", .value = timestamp }); + + for (extra_headers) |h| { + try headers.append(allocator, .{ .name = h.name, .value = h.value }); + } + + // Important: sort by header name + std.mem.sort( + std.http.Header, + headers.items, + {}, + struct { + fn lessThan(_: void, a: std.http.Header, b: std.http.Header) bool { + return std.mem.order(u8, a.name, b.name) == .lt; + } + }.lessThan, + ); + + // Build canonical headers and signed headers + var canonical_headers_list: std.ArrayList(u8) = .{}; + defer canonical_headers_list.deinit(allocator); + + var signed_headers_list: std.ArrayList(u8) = .{}; + defer signed_headers_list.deinit(allocator); + + for (headers.items, 0..) |header, i| { + // Add to canonical headers: "name:value\n" + try canonical_headers_list.appendSlice(allocator, header.name); + try canonical_headers_list.append(allocator, ':'); + try canonical_headers_list.appendSlice(allocator, header.value); + try canonical_headers_list.append(allocator, '\n'); + + // Add to signed headers list: "name1;name2;name3" + if (i > 0) try signed_headers_list.append(allocator, ';'); + try signed_headers_list.appendSlice(allocator, header.name); + } + + const canonical_headers = canonical_headers_list.items; + const signed_headers = signed_headers_list.items; + + // Build canonical request + var canonical_request_list: std.ArrayList(u8) = .{}; + defer canonical_request_list.deinit(allocator); + + const canonical_uri = if (uri.path.percent_encoded.len > 0) uri.path.percent_encoded else "/"; + const canonical_query = ""; // todo + + // METHOD\nURI\nQUERY\nHEADERS\nSIGNED_HEADERS\nPAYLOAD_HASH + try canonical_request_list.appendSlice(allocator, @tagName(method)); + try canonical_request_list.append(allocator, '\n'); + try canonical_request_list.appendSlice(allocator, canonical_uri); + try canonical_request_list.append(allocator, '\n'); + try canonical_request_list.appendSlice(allocator, canonical_query); + try canonical_request_list.append(allocator, '\n'); + try canonical_request_list.appendSlice(allocator, canonical_headers); + try canonical_request_list.append(allocator, '\n'); + try canonical_request_list.appendSlice(allocator, signed_headers); + try canonical_request_list.append(allocator, '\n'); + try canonical_request_list.appendSlice(allocator, UNSIGNED_PAYLOAD); + + // Hash canonical request + const cr_hash = sha256(canonical_request_list.items); + var cr_hash_hex: [64]u8 = undefined; + _ = std.fmt.bufPrint(&cr_hash_hex, "{x}", .{&cr_hash}) catch unreachable; + + // Build credential scope: date/region/service/aws4_request + const credential_scope = try std.fmt.allocPrint(allocator, "{s}/{s}/{s}/aws4_request", .{ date, self.region, self.service }); + defer allocator.free(credential_scope); + + // Build string to sign + var string_to_sign_list: std.ArrayList(u8) = .{}; + defer string_to_sign_list.deinit(allocator); + + try string_to_sign_list.appendSlice(allocator, ALGORITHM); + try string_to_sign_list.append(allocator, '\n'); + try string_to_sign_list.appendSlice(allocator, timestamp); + try string_to_sign_list.append(allocator, '\n'); + try string_to_sign_list.appendSlice(allocator, credential_scope); + try string_to_sign_list.append(allocator, '\n'); + try string_to_sign_list.appendSlice(allocator, &cr_hash_hex); + + // Calculate signing key + const k_date = hmacSha256WithPrefix("AWS4", self.secret_key, date); + const k_region = hmacSha256(&k_date, self.region); + const k_service = hmacSha256(&k_region, self.service); + const k_signing = hmacSha256(&k_service, "aws4_request"); + + // Calculate signature + const signature = hmacSha256(&k_signing, string_to_sign_list.items); + var signature_hex: [64]u8 = undefined; + _ = std.fmt.bufPrint(&signature_hex, "{x}", .{&signature}) catch unreachable; + + // Build final Authorization header + return try std.fmt.allocPrint(allocator, "{s} Credential={s}/{s}, SignedHeaders={s}, Signature={s}", .{ + ALGORITHM, + self.access_key, + credential_scope, + signed_headers, + signature_hex[0..64], + }); + } + + fn getHostForEndpoint(endpoint: []const u8, buf: []u8) []const u8 { + const uri = std.Uri.parse(endpoint) catch return endpoint; + const host_raw = uri.host orelse return endpoint; + const host = switch (host_raw) { + .raw => |r| r, + .percent_encoded => |p| p, + }; + if (uri.port) |port| { + const len = std.fmt.bufPrint(buf, "{s}:{d}", .{ host, port }) catch return endpoint; + return buf[0..len.len]; + } + return host; + } + + fn hmacSha256WithPrefix(prefix: []const u8, secret: []const u8, data: []const u8) [32]u8 { + var key_buf: [128]u8 = undefined; + @memcpy(key_buf[0..prefix.len], prefix); + @memcpy(key_buf[prefix.len..][0..secret.len], secret); + const key = key_buf[0 .. prefix.len + secret.len]; + + var out: [32]u8 = undefined; + std.crypto.auth.hmac.sha2.HmacSha256.create(&out, data, key); + return out; + } + + fn sha256(data: []const u8) [32]u8 { + var hash: [32]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(data, &hash, .{}); + return hash; + } + + fn hmacSha256(key: []const u8, data: []const u8) [32]u8 { + var out: [32]u8 = undefined; + std.crypto.auth.hmac.sha2.HmacSha256.create(&out, data, key); + return out; + } +}; + +pub const S3 = struct { + const Handle = struct { + pub const Type = enum { file, directory }; + + type: Type, + uri: []const u8, + pos: u64, + size: u64, + + pub fn init(allocator: std.mem.Allocator, handle_type: Type, path: []const u8, size: u64) !Handle { + return .{ .type = handle_type, .uri = try allocator.dupe(u8, path), .pos = 0, .size = size }; + } + + pub fn deinit(self: *Handle, allocator: std.mem.Allocator) void { + allocator.free(self.uri); + } + }; + + pub const Credentials = struct { + access_key: []const u8, + secret_key: []const u8, + endpoint_url: []const u8, + region: []const u8 = "us-east-1", // todo: no default? + + pub fn fromEnv(allocator: std.mem.Allocator) !Credentials { + const access_key = std.process.getEnvVarOwned(allocator, "AWS_ACCESS_KEY_ID") catch |err| { + log.err("AWS_ACCESS_KEY_ID environment variable not set", .{}); + return err; + }; + errdefer allocator.free(access_key); + + const secret_key = std.process.getEnvVarOwned(allocator, "AWS_SECRET_ACCESS_KEY") catch |err| { + log.err("AWS_SECRET_ACCESS_KEY environment variable not set", .{}); + return err; + }; + errdefer allocator.free(secret_key); + + const endpoint_url = std.process.getEnvVarOwned(allocator, "AWS_ENDPOINT_URL") catch + std.process.getEnvVarOwned(allocator, "AWS_ENDPOINT_URL_S3") catch |err| { + log.err("Neither AWS_ENDPOINT_URL nor AWS_ENDPOINT_URL_S3 environment variable is set", .{}); + return err; + }; + errdefer allocator.free(endpoint_url); + + const region = std.process.getEnvVarOwned(allocator, "AWS_REGION") catch + std.process.getEnvVarOwned(allocator, "AWS_DEFAULT_REGION") catch + try allocator.dupe(u8, "us-east-1"); + + return .{ .access_key = access_key, .secret_key = secret_key, .endpoint_url = endpoint_url, .region = region }; + } + + pub fn deinit(self: *Credentials, allocator: std.mem.Allocator) void { + allocator.free(self.access_key); + allocator.free(self.secret_key); + allocator.free(self.endpoint_url); + allocator.free(self.region); + } + }; + + allocator: std.mem.Allocator, + mutex: std.Io.Mutex = .init, + client: *std.http.Client, + access_key: []const u8, + secret_key: []const u8, + endpoint_url: []const u8, + region: []const u8, + handles: stdx.SegmentedList(Handle, 0) = .{}, + closed_handles: std.ArrayList(u32) = .{}, + base: VFSBase, + + pub fn init( + allocator: std.mem.Allocator, + inner: std.Io, + http_client: *std.http.Client, + credentials: Credentials, + ) !S3 { + return .{ + .allocator = allocator, + .base = .init(inner), + .client = http_client, + .access_key = try allocator.dupe(u8, credentials.access_key), + .secret_key = try allocator.dupe(u8, credentials.secret_key), + .endpoint_url = try allocator.dupe(u8, credentials.endpoint_url), + .region = try allocator.dupe(u8, credentials.region), + }; + } + + pub fn fromEnv(allocator: std.mem.Allocator, inner: std.Io, http_client: *std.http.Client) !S3 { + var credentials = try Credentials.fromEnv(allocator); + defer credentials.deinit(allocator); + + return init(allocator, inner, http_client, credentials); + } + + pub fn deinit(self: *S3) void { + var idx: usize = 0; + while (idx < self.handles.len) : (idx += 1) { + const is_closed = for (self.closed_handles.items) |closed_idx| { + if (closed_idx == idx) break true; + } else false; + if (!is_closed) self.handles.at(idx).deinit(self.allocator); + } + self.handles.deinit(self.allocator); + self.closed_handles.deinit(self.allocator); + self.allocator.free(self.access_key); + self.allocator.free(self.secret_key); + self.allocator.free(self.endpoint_url); + self.allocator.free(self.region); + } + + pub fn io(self: *S3) std.Io { + return .{ + .userdata = &self.base, + .vtable = &comptime VFSBase.vtable(.{ + .dirOpenDir = dirOpenDir, + .dirStat = dirStat, + .dirStatFile = dirStatFile, + .dirAccess = dirAccess, + .dirOpenFile = dirOpenFile, + .dirClose = dirClose, + .dirRead = dirRead, + .dirRealPath = dirRealPath, + .dirRealPathFile = dirRealPathFile, + .fileStat = fileStat, + .fileLength = fileLength, + .fileClose = fileClose, + .fileReadStreaming = fileReadStreaming, + .fileReadPositional = fileReadPositional, + .fileSeekBy = fileSeekBy, + .fileSeekTo = fileSeekTo, + .fileRealPath = fileRealPath, + }), + }; + } + + fn openHandle(self: *S3) !struct { u32, *Handle } { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + if (self.closed_handles.pop()) |idx| return .{ idx, self.handles.at(idx) }; + return .{ @intCast(self.handles.len), try self.handles.addOne(self.allocator) }; + } + + fn closeHandle(self: *S3, idx: u32) !void { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + self.handles.at(idx).deinit(self.allocator); + try self.closed_handles.append(self.allocator, idx); + } + + fn getFileHandle(self: *S3, file: std.Io.File) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(file.handle)); + } + + fn getDirHandle(self: *S3, dir: std.Io.Dir) *Handle { + self.mutex.lockUncancelable(self.base.inner); + defer self.mutex.unlock(self.base.inner); + return self.handles.at(@intCast(dir.handle)); + } + + fn resolvePath(self: *S3, dir: std.Io.Dir, sub_path: []const u8, out_buffer: []u8) ![]u8 { + if (std.meta.eql(dir, std.Io.Dir.cwd())) { + return try std.fmt.bufPrint(out_buffer, "{s}", .{sub_path}); + } + const handle = self.getDirHandle(dir); + const trimmed_uri = std.mem.trimEnd(u8, handle.uri, "/"); + const trimmed_sub_path = std.mem.trimStart(u8, sub_path, "/"); + + if (trimmed_uri.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_sub_path}); + if (trimmed_sub_path.len == 0) return try std.fmt.bufPrint(out_buffer, "{s}", .{trimmed_uri}); + return try std.fmt.bufPrint(out_buffer, "{s}/{s}", .{ trimmed_uri, trimmed_sub_path }); + } + + // todo + fn parseBucketKey(path: []const u8) struct { bucket: []const u8, key: []const u8 } { + const trimmed = std.mem.trim(u8, path, "/"); + if (std.mem.indexOfScalar(u8, trimmed, '/')) |idx| { + return .{ .bucket = trimmed[0..idx], .key = if (idx + 1 < trimmed.len) trimmed[idx + 1 ..] else "" }; + } + return .{ .bucket = trimmed, .key = "" }; + } + + // todo + const UrlStyle = enum { + virtual_hosted, // AWS + path_style, // MinIO + }; + + // todo + fn buildS3Url(self: *S3, allocator: std.mem.Allocator, bucket: []const u8, key: []const u8) ![]const u8 { + const trimmed_endpoint = std.mem.trimEnd(u8, self.endpoint_url, "/"); + + if (key.len > 0) { + return try std.fmt.allocPrint(allocator, "{s}/{s}/{s}", .{ trimmed_endpoint, bucket, key }); + } + return try std.fmt.allocPrint(allocator, "{s}/{s}/", .{ trimmed_endpoint, bucket }); + } + + // todo + fn getTimestamp(allocator: std.mem.Allocator, io_: std.Io) ![]const u8 { + const now_ts = try std.Io.Clock.now(.real, io_); + + const now: u64 = @intCast(@divFloor(now_ts.nanoseconds, std.time.ns_per_s)); + const epoch_secs: std.time.epoch.EpochSeconds = .{ .secs = now }; + const year_day = epoch_secs.getEpochDay().calculateYearDay(); + const month_day = year_day.calculateMonthDay(); + const day_seconds = epoch_secs.getDaySeconds(); + + return try std.fmt.allocPrint(allocator, "{d:0>4}{d:0>2}{d:0>2}T{d:0>2}{d:0>2}{d:0>2}Z", .{ + year_day.year, + @intFromEnum(month_day.month), + month_day.day_index + 1, + day_seconds.getHoursIntoDay(), + day_seconds.getMinutesIntoHour(), + day_seconds.getSecondsIntoMinute(), + }); + } + + fn fetchSize(self: *S3, dir: std.Io.Dir, sub_path: []const u8) !u64 { + var path_buffer: [8 * 1024]u8 = undefined; + const path = try self.resolvePath(dir, sub_path, &path_buffer); + const parsed = parseBucketKey(path); + + const url = try self.buildS3Url(self.allocator, parsed.bucket, parsed.key); + defer self.allocator.free(url); + + const uri = std.Uri.parse(url) catch return error.BadPathName; + + const timestamp = try getTimestamp(self.allocator, self.base.inner); + defer self.allocator.free(timestamp); + + const signer: AwsSigV4 = .{ + .access_key = self.access_key, + .secret_key = self.secret_key, + .region = self.region, + }; + + const auth_header = try signer.generateAuthHeader(self.allocator, .HEAD, self.endpoint_url, uri, timestamp, &.{}); + defer self.allocator.free(auth_header); + + var req = try self.client.request(.HEAD, uri, .{ + .redirect_behavior = .not_allowed, + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = &.{ + .{ .name = "Authorization", .value = auth_header }, + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + const res = try req.receiveHead(&redirect_buffer); + + switch (res.head.status.class()) { + .success => return res.head.content_length.?, + else => { + log.err("Failed to fetch size for {s}: {s}", .{ url, res.head.bytes }); + return error.ServerError; + }, + } + } + + fn performRead(self: *S3, handle: *Handle, data: []const []u8, offset: u64) !usize { + if (offset >= handle.size) return 0; + + const parsed = parseBucketKey(handle.uri); + + const url = try self.buildS3Url(self.allocator, parsed.bucket, parsed.key); + defer self.allocator.free(url); + + const uri = std.Uri.parse(url) catch return error.BadPathName; + + // Calculate range + var total_bytes: u64 = 0; + for (data) |buf| total_bytes += buf.len; + const remaining = handle.size - offset; + const take = @min(remaining, total_bytes); + const end = offset + take - 1; + + const range_header = try std.fmt.allocPrint(self.allocator, "bytes={d}-{d}", .{ offset, end }); + defer self.allocator.free(range_header); + + const timestamp = try getTimestamp(self.allocator, self.base.inner); + defer self.allocator.free(timestamp); + + const signer: AwsSigV4 = .{ + .access_key = self.access_key, + .secret_key = self.secret_key, + .region = self.region, + }; + + const auth_header = try signer.generateAuthHeader(self.allocator, .GET, self.endpoint_url, uri, timestamp, &.{}); + defer self.allocator.free(auth_header); + + var req = try self.client.request(.GET, uri, .{ + .headers = .{ .accept_encoding = .{ .override = "identity" } }, + .extra_headers = &.{ + .{ .name = "Authorization", .value = auth_header }, + .{ .name = "x-amz-date", .value = timestamp }, + .{ .name = "x-amz-content-sha256", .value = AwsSigV4.UNSIGNED_PAYLOAD }, + .{ .name = "Range", .value = range_header }, + }, + }); + defer req.deinit(); + + try req.sendBodiless(); + + var redirect_buffer: [8 * 1024]u8 = undefined; + var res = try req.receiveHead(&redirect_buffer); + + if (res.head.status != .partial_content and res.head.status != .ok) { + log.err("Failed to read {s}: {s}", .{ handle.uri, res.head.bytes }); + return error.RequestFailed; + } + + const content_range = blk: { + var it = res.head.iterateHeaders(); + while (it.next()) |header| { + if (std.ascii.eqlIgnoreCase(header.name, "Content-Range")) { + break :blk parseContentRange(header.value); + } + } + break :blk null; + }; + + const reader = res.reader(&.{}); + + if (content_range) |cr| { + if (cr.start < offset) { + try reader.discardAll(offset - cr.start); + } + } + + return try reader.readSliceShort(data[0]); + } + + const ContentRange = struct { + start: u64, + end: u64, + total: u64, + }; + + fn parseContentRange(value: []const u8) ?ContentRange { + // Format: "bytes start-end/total" + const space = std.mem.indexOfScalar(u8, value, ' ') orelse return null; + const dash = std.mem.indexOfScalar(u8, value, '-') orelse return null; + const slash = std.mem.indexOfScalar(u8, value, '/') orelse return null; + + return .{ + .start = std.fmt.parseInt(u64, value[space + 1 .. dash], 10) catch return null, + .end = std.fmt.parseInt(u64, value[dash + 1 .. slash], 10) catch return null, + .total = std.fmt.parseInt(u64, value[slash + 1 ..], 10) catch return null, + }; + } + + fn dirOpenDir(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.OpenOptions) std.Io.Dir.OpenError!std.Io.Dir { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.Dir.OpenError.SystemResources; + const idx, const handle = self.openHandle() catch return std.Io.Dir.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .directory, path, 0) catch return std.Io.Dir.OpenError.Unexpected; + return .{ .handle = @intCast(idx) }; + } + + fn dirStat(userdata: ?*anyopaque, dir: std.Io.Dir) std.Io.Dir.StatError!std.Io.Dir.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + return .{ + .inode = 0, + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .directory, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + }; + } + + fn dirStatFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.Dir.StatFileOptions) std.Io.Dir.StatFileError!std.Io.File.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch return std.Io.Dir.StatFileError.Unexpected; + return .{ + .inode = 0, + .nlink = 0, + .size = size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + }; + } + + fn dirAccess(_: ?*anyopaque, _: std.Io.Dir, _: []const u8, _: std.Io.Dir.AccessOptions) std.Io.Dir.AccessError!void {} + + fn dirOpenFile(userdata: ?*anyopaque, dir: std.Io.Dir, sub_path: []const u8, _: std.Io.File.OpenFlags) std.Io.File.OpenError!std.Io.File { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const size = self.fetchSize(dir, sub_path) catch return std.Io.File.OpenError.Unexpected; + var path_buffer: [8 * 1024]u8 = undefined; + const path = self.resolvePath(dir, sub_path, &path_buffer) catch return std.Io.File.OpenError.SystemResources; + const idx, const handle = self.openHandle() catch return std.Io.File.OpenError.Unexpected; + handle.* = Handle.init(self.allocator, .file, path, size) catch return std.Io.File.OpenError.Unexpected; + return .{ .handle = @intCast(idx) }; + } + + fn dirClose(userdata: ?*anyopaque, dirs: []const std.Io.Dir) void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + for (dirs) |dir| self.closeHandle(@intCast(dir.handle)) catch unreachable; + } + + fn dirRead(_: ?*anyopaque, _: *std.Io.Dir.Reader, _: []std.Io.Dir.Entry) std.Io.Dir.Reader.Error!usize { + log.err("dirRead unsupported for S3 VFS", .{}); + return std.Io.Dir.Reader.Error.Unexpected; + } + + fn dirRealPath(userdata: ?*anyopaque, dir: std.Io.Dir, out_buffer: []u8) std.Io.Dir.RealPathError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getDirHandle(dir); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.Dir.RealPathError.SystemResources; + return path.len; + } + + fn dirRealPathFile(userdata: ?*anyopaque, dir: std.Io.Dir, path_name: []const u8, out_buffer: []u8) std.Io.Dir.RealPathFileError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const real_path = self.resolvePath(dir, path_name, out_buffer) catch return std.Io.Dir.RealPathFileError.NameTooLong; + return real_path.len; + } + + fn fileStat(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.StatError!std.Io.File.Stat { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return .{ + .inode = @intCast(file.handle), + .nlink = 0, + .size = handle.size, + .permissions = .fromMode(0o444), + .kind = .file, + .atime = null, + .mtime = std.Io.Timestamp.zero, + .ctime = std.Io.Timestamp.zero, + }; + } + + fn fileLength(userdata: ?*anyopaque, file: std.Io.File) std.Io.File.LengthError!u64 { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + return self.getFileHandle(file).size; + } + + fn fileClose(userdata: ?*anyopaque, files: []const std.Io.File) void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + for (files) |file| self.closeHandle(@intCast(file.handle)) catch unreachable; + } + + fn fileReadStreaming(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8) std.Io.File.Reader.Error!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const total = self.performRead(handle, data, handle.pos) catch |err| { + log.err("Failed to read {s} at {d}: {any}", .{ handle.uri, handle.pos, err }); + return std.Io.File.Reader.Error.Unexpected; + }; + handle.pos += @intCast(total); + return total; + } + + fn fileReadPositional(userdata: ?*anyopaque, file: std.Io.File, data: []const []u8, offset: u64) std.Io.File.ReadPositionalError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + return self.performRead(handle, data, offset) catch |err| { + log.err("Failed to read {s} at {d}: {any}", .{ handle.uri, offset, err }); + return std.Io.File.ReadPositionalError.Unexpected; + }; + } + + fn fileSeekBy(userdata: ?*anyopaque, file: std.Io.File, relative_offset: i64) std.Io.File.SeekError!void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + + if (relative_offset >= 0) { + handle.pos = handle.pos +| @as(u64, @intCast(relative_offset)); + } else { + const abs_offset: u64 = @intCast(-relative_offset); + if (abs_offset > handle.pos) { + handle.pos = 0; + } else { + handle.pos -= abs_offset; + } + } + } + + fn fileSeekTo(userdata: ?*anyopaque, file: std.Io.File, absolute_offset: u64) std.Io.File.SeekError!void { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + handle.pos = absolute_offset; + } + + fn fileRealPath(userdata: ?*anyopaque, file: std.Io.File, out_buffer: []u8) std.Io.File.RealPathError!usize { + const self: *S3 = @fieldParentPtr("base", VFSBase.as(userdata)); + const handle = self.getFileHandle(file); + const path = std.fmt.bufPrint(out_buffer, "{s}", .{handle.uri}) catch return std.Io.File.RealPathError.SystemResources; + return path.len; + } +}; diff --git a/zml/kernel/BUILD.bazel b/zml/kernel/BUILD.bazel deleted file mode 100644 index 61ab44f8..00000000 --- a/zml/kernel/BUILD.bazel +++ /dev/null @@ -1,15 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "kernel", - srcs = [ - "kernel.zig", - "//zml/kernel/cuda_tile", - "//zml/kernel/cute", - "//zml/kernel/fly", - "//zml/kernel/metal", - "//zml/kernel/mosaic_tpu", - "//zml/kernel/tcl", - "//zml/kernel/triton", - ], -) diff --git a/zml/kernel/cuda_tile/BUILD.bazel b/zml/kernel/cuda_tile/BUILD.bazel deleted file mode 100644 index 89b7977e..00000000 --- a/zml/kernel/cuda_tile/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "cuda_tile", - srcs = [ - "cuda_tile.zig", - "test.zig", - ], -) diff --git a/zml/kernel/cuda_tile/cuda_tile.zig b/zml/kernel/cuda_tile/cuda_tile.zig deleted file mode 100644 index 3f6d8734..00000000 --- a/zml/kernel/cuda_tile/cuda_tile.zig +++ /dev/null @@ -1,163 +0,0 @@ -const std = @import("std"); - -const cuda_tile_builder = @import("dsl/cuda_tile/builder"); -pub const Builder = cuda_tile_builder.Builder; -pub const Value = cuda_tile_builder.Value; -pub const DType = cuda_tile_builder.DType; -pub const FinishError = cuda_tile_builder.FinishError; -pub const EntryHint = cuda_tile_builder.EntryHint; -pub const Arch = cuda_tile_builder.Arch; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cuda_tile_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i4, .u4 => .i4, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - .f4e2m1 => .f4e2m1fn, - else => std.debug.panic("zml.kernel.cuda_tile.from: dtype {s} has no CUDA Tile IR equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// No `num_warps`/`num_stages`: warp and CTA tuning is - /// `optimization_hints` inside the IR (`Opts.hints`). - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Outputs XLA zeroes before the launch, by position in - /// `spec.outputs`, ascending. - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cuda_tile_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(&.{}); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cuda_tile.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cudaTile(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/cuda_tile/test.zig b/zml/kernel/cuda_tile/test.zig deleted file mode 100644 index c929b0b9..00000000 --- a/zml/kernel/cuda_tile/test.zig +++ /dev/null @@ -1,28 +0,0 @@ -const std = @import("std"); - -const cuda_tile = @import("cuda_tile.zig"); - -test "cuda_tile kernel emits a module XLA can parse" { - const Cfg = struct { n: i64 }; - const AddOne = cuda_tile.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cuda_tile.Builder, cfg: Cfg) cuda_tile.FinishError!void { - const a = try b.declareArgs(.{ .x = .{ .ptr = .f32 }, .out = .{ .ptr = .f32 } }); - const vx = b.partitionView(b.tensorView(a.x, &.{cfg.n}, &.{1}), &.{128}, .{}); - const vo = b.partitionView(b.tensorView(a.out, &.{cfg.n}, &.{1}), &.{128}, .{}); - const bid = b.tileBlockId(); - _ = b.store(b.load(vx, &.{bid.x}).add(1.0), vo, &.{bid.x}); - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1024 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda_tile.module @add_one") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "entry @add_one(") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "load_view_tko") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "store_view_tko") != null); -} diff --git a/zml/kernel/cute/BUILD.bazel b/zml/kernel/cute/BUILD.bazel deleted file mode 100644 index 7a613072..00000000 --- a/zml/kernel/cute/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "cute", - srcs = [ - "cute.zig", - "test.zig", - ], -) diff --git a/zml/kernel/cute/cute.zig b/zml/kernel/cute/cute.zig deleted file mode 100644 index 2e8c919f..00000000 --- a/zml/kernel/cute/cute.zig +++ /dev/null @@ -1,264 +0,0 @@ -const std = @import("std"); - -const cute_builder = @import("dsl/cute/builder"); -pub const AlgebraExpr = cute_builder.AlgebraExpr; -pub const Builder = cute_builder.Builder; -pub const ArgSpec = cute_builder.ArgSpec; -pub const Value = cute_builder.Value; -pub const View = cute_builder.View; -pub const Atom = cute_builder.Atom; -pub const DType = cute_builder.DType; -pub const AlgebraToken = cute_builder.AlgebraToken; -pub const LayoutSpec = cute_builder.LayoutSpec; -pub const ConstrainedDynamic = cute_builder.ConstrainedDynamic; -pub const FinishError = cute_builder.FinishError; -pub const TmaConfig = cute_builder.TmaConfig; -pub const TmaFormat = cute_builder.TmaFormat; -pub const BlockScaledMmaConfig = cute_builder.BlockScaledMmaConfig; -pub const CtaGroup = cute_builder.CtaGroup; -pub const ctaCount = cute_builder.ctaCount; -pub const MBarrierScope = cute_builder.MBarrierScope; -pub const OperandMajorMode = cute_builder.OperandMajorMode; -pub const Tcgen05SmemSwizzle = cute_builder.Tcgen05SmemSwizzle; -pub const Tcgen05SmemDesc = cute_builder.Tcgen05SmemDesc; -pub const Tcgen05InstrDesc = cute_builder.Tcgen05InstrDesc; -pub const CopyBulkTensorTileG2SOp = cute_builder.CopyBulkTensorTileG2SOp; -pub const CopyBulkTensor2DGather4G2SOp = cute_builder.CopyBulkTensor2DGather4G2SOp; -pub const CopyBulkTensorTileS2GOp = cute_builder.CopyBulkTensorTileS2GOp; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(cute_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4e2m1fn, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - .f8e8m0 => .f8e8m0fnu, - else => std.debug.panic("zml.kernel.cute.from: dtype {s} has no CuTe equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - /// `kernel.launch(grid=, block=)` in the Python DSL; dynamic shared - /// memory is what the kernel declares. - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - block: [3]i32, - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, block: [3]i32) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - // XLA passes one pointer per input then per output; it does - // not check the count against the kernel. - const expected = spec.inputs.len + spec.outputs.len; - if (b.args.len != expected) { - std.debug.panic("zml.kernel.cute.Kernel({s}): declareArgs declared {d} arguments, inputs + outputs are {d}", .{ name, b.args.len, expected }); - } - - return b.finish(block); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg, opts.block) catch |err| - std.debug.panic("zml.kernel.cute.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .block = opts.block, - .zeroed_outputs = opts.zeroed_outputs, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} - -fn ProgramSpec(comptime Config: type) type { - return struct { - /// Public host entry selected by XLA. The callback may give the - /// nested CUDA kernel a different (private) symbol. - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -/// A complete CuTe program: one or more `cuda.kernel` operations inside -/// a `gpu.module`, followed by a public `func.func` which constructs the -/// launch-time CuTe objects and calls `cuda.launch_ex`. -/// -/// This is the source-level equivalent of invoking a Python `@cute.jit` -/// object whose `__call__` builds TMA descriptors and launches an -/// `@cute.kernel`. XLA executes the generated host function on its CUDA -/// stream, so the custom call deliberately carries no external grid or -/// block dimensions. -pub fn Program( - comptime ConfigT: type, - comptime spec: ProgramSpec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - zeroed_outputs: []const i32 = &.{}, - scalars: []const i64 = &.{}, - symmetric_workspace: ?ops.CuteOps.SymmetricWorkspace = null, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try cute_builder.Builder.openProgram(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - return b.finishProgram(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.cute.Program({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - const tensor_results = ops.cute(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .zeroed_outputs = opts.zeroed_outputs, - .scalars = opts.scalars, - .symmetric_workspace = opts.symmetric_workspace, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/cute/test.zig b/zml/kernel/cute/test.zig deleted file mode 100644 index f4d457a3..00000000 --- a/zml/kernel/cute/test.zig +++ /dev/null @@ -1,34 +0,0 @@ -const std = @import("std"); - -const cute = @import("cute.zig"); - -test "cute kernel emits the module the CuTe compiler takes" { - const Cfg = struct { n: i64 }; - const AddOne = cute.Kernel(Cfg, .{ - .name = "add_one", - .inputs = &.{"x"}, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *cute.Builder, cfg: Cfg) cute.FinishError!void { - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - .out = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.n} } }, - }); - const i = b.blockIdx().x.mul(b.blockDim().x).add(b.threadIdx().x); - var guard = b.openIf(i.lt(cfg.n)); - { - a.out.set(.{i}, a.x.get(.{i}).add(1.0)); - guard.close(); - } - } - }.run, - }); - - const ir = try AddOne.emit(std.testing.allocator, .{ .n = 1000 }, .{ 128, 1, 1 }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "\"func.func\"() <{function_type = (") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "sym_name = \"add_one\"") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "nvvm.reqntid = array") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cute.memref.load") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.module") == null); -} diff --git a/zml/kernel/fly/BUILD.bazel b/zml/kernel/fly/BUILD.bazel deleted file mode 100644 index 57203521..00000000 --- a/zml/kernel/fly/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "fly", - srcs = [ - "fly.zig", - "test.zig", - ], -) diff --git a/zml/kernel/fly/fly.zig b/zml/kernel/fly/fly.zig deleted file mode 100644 index b7d6c943..00000000 --- a/zml/kernel/fly/fly.zig +++ /dev/null @@ -1,248 +0,0 @@ -const std = @import("std"); - -const fly_builder = @import("dsl/fly/builder"); -pub const Builder = fly_builder.Builder; -pub const Value = fly_builder.Value; -pub const DType = fly_builder.DType; -pub const FinishError = fly_builder.FinishError; -pub const TiledCopy = fly_builder.TiledCopy; -pub const TiledMma = fly_builder.TiledMma; -pub const Arch = fly_builder.Arch; -pub const MmaFlavor = fly_builder.MmaFlavor; -pub const fly_rocdl = fly_builder.fly_rocdl; -pub const layout = fly_builder.layout; -pub const Layout = fly_builder.Layout; -pub const Tile = fly_builder.Tile; -pub const IntTuple = fly_builder.IntTuple; -pub const L = fly_builder.L; -pub const rowMajor = fly_builder.rowMajor; -pub const colMajor = fly_builder.colMajor; -pub const ordered = fly_builder.ordered; -pub const tile = fly_builder.tile; -pub const it = fly_builder.it; -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -/// Threads per wavefront: 64 on CDNA, 32 on RDNA. -pub fn waveSize(p: *const platform_.Platform) i32 { - const cc = platform_.rocm.computeCapability(p) orelse return 64; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => 64, - .rdna2, .rdna3, .rdna3_5, .rdna4 => 32, - }; -} - -/// The plugin sizes a block in wavefronts, so a thread count has to divide -/// evenly: a kernel's layouts are built for an exact number of threads and -/// spare ones would partition out of range. -fn warpsFor(p: *const platform_.Platform, threads: i32) i32 { - const wave = waveSize(p); - if (@rem(threads, wave) != 0) { - std.debug.panic("zml.kernel.fly: {d} threads is not a multiple of the {d}-wide wavefront", .{ threads, wave }); - } - return @divExact(threads, wave); -} - -/// The matrix atom this device provides, or null when it has none: WMMA -/// arrived with RDNA3, so gfx1030 has no matrix instruction at all. -pub fn mmaFlavor(p: *const platform_.Platform) ?MmaFlavor { - const cc = platform_.rocm.computeCapability(p) orelse return .cdna3_mfma; - return switch (cc.architecture()) { - .cdna1, .cdna2, .cdna3, .cdna4 => .cdna3_mfma, - .rdna3, .rdna3_5 => .gfx11_wmma, - .rdna4 => .gfx120x_wmma, - .rdna2 => null, - }; -} -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(fly_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - .f8e5m2fnuz => .f8e5m2fnuz, - else => std.debug.panic("zml.kernel.fly.from: dtype {s} has no Fly equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const arg_names = spec.inputs ++ spec.outputs; - /// By name, as `!fly.memref` views. - pub const Args = StructOf(arg_names, Value); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - /// Threads per block. Converted to the wavefronts the plugin - /// wants using the device's wave size, which is 64 on CDNA and - /// 32 on RDNA, so a kernel states the thread count its layouts - /// were built for and runs on both. - threads: i32, - /// > 0 sets `amdgpu-waves-per-eu = "N, N"`: a hard occupancy clamp. - waves_per_eu: i32 = 0, - /// Required when the body uses `fly.get_dyn_shared`. - shared_mem_bytes: i64 = 0, - /// Inputs first, then outputs; see `zeroedOutput`. A no-op - /// inside command buffers. - zeroed_args: []const i32 = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - }; - - /// The arguments `run` was declared with. - pub fn args(b: *Builder) Args { - var out: Args = undefined; - inline for (arg_names, 0..) |fname, i| @field(out, fname) = b.arg(i); - return out; - } - - /// `zeroed_args` index of an output. - pub fn zeroedOutput(comptime f: std.meta.FieldEnum(Outputs)) i32 { - return @intCast(spec.inputs.len + @backingInt(f)); - } - - /// `shapes` is inputs, then outputs. - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT, shapes: []const Shape) ![:0]const u8 { - std.debug.assert(shapes.len == arg_names.len); - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try fly_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - var specs: [arg_names.len]Builder.ArgSpec = undefined; - inline for (arg_names, 0..) |fname, i| { - const shape = shapes[i]; - specs[i] = .{ - .name = fname, - .dtype = from(shape.dtype()), - .dims = if (shape.rank() > 0) shape.dims() else null, - }; - } - try b.declareArgsLow(&specs); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var shapes: [arg_names.len]Shape = undefined; - inline for (0..spec.inputs.len) |i| shapes[i] = inputs_arr[i].shape(); - inline for (0..spec.outputs.len) |i| shapes[spec.inputs.len + i] = outputs_arr[i]; - - const ir = emit(cur.allocator, opts.cfg, &shapes) catch |err| - std.debug.panic("zml.kernel.fly.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const tensor_results = ops.fly(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .num_warps = warpsFor(cur.platform, opts.threads), - .waves_per_eu = opts.waves_per_eu, - .shared_mem_bytes = opts.shared_mem_bytes, - .zeroed_args = opts.zeroed_args, - .output_operand_aliases = aliases.constSlice(), - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/fly/test.zig b/zml/kernel/fly/test.zig deleted file mode 100644 index ae788d96..00000000 --- a/zml/kernel/fly/test.zig +++ /dev/null @@ -1,149 +0,0 @@ -const std = @import("std"); - -const fly_builder = @import("dsl/fly/builder"); - -const fly = @import("fly.zig"); -const zml = @import("../../zml.zig"); - -/// C = A + B over (M, N) f32, 128-bit vectorized and predicated on the -/// ragged edge. -const FlyVectorAdd = struct { - const Cfg = struct { - thr: fly.Layout = fly.ordered(.{ 8, 16 }, .{ 1, 0 }), - val: fly.Layout = fly.ordered(.{ 1, 4 }, .{ 0, 1 }), - }; - - const K = fly.Kernel(Cfg, .{ - .name = "vector_add", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitVectorAdd(b, cfg.thr, cfg.val, a.a, a.b, a.c); - } -}; - -/// C = A @ B^T with one MFMA-tiled block. -const FlyTiledMma = struct { - const Cfg = struct { flavor: fly.MmaFlavor }; - const K = fly.Kernel(Cfg, .{ - .name = "tiled_mma", - .inputs = &.{ "a", "b" }, - .outputs = &.{"c"}, - .run = run, - }); - - fn run(b: *fly.Builder, cfg: Cfg) fly.FinishError!void { - const a = K.args(b); - fly_builder.emitTiledMma(b, cfg.flavor, a.a, a.b, a.c); - } -}; - -test "fly kernel emits a module XLA can parse" { - const shapes = [_]zml.Shape{ .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32), .init(.{ 100, 1000 }, .f32) }; - const ir = try FlyVectorAdd.K.emit(std.testing.allocator, .{}, &shapes); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "gpu.func @vector_add(%arg0: !fly.ptr, %arg1: !fly.ptr, %arg2: !fly.ptr) kernel") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "fly.static : !fly.tile<[8|64]>") != null); -} - -test "fly kernels run on rocm" { - const platform = zml.testing.env(); - if (platform.target != .rocm) return error.SkipZigTest; - - const allocator = std.testing.allocator; - - // Compile `forward`, upload both inputs, run, and bring the result back. - const call = struct { - fn f(comptime forward: anytype, p: *const zml.Platform, ta: zml.Tensor, ha: anytype, tb: zml.Tensor, hb: anytype) !zml.Slice { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ ta, tb }, p, .{}); - defer exe.deinit(); - var ba: zml.Buffer = try .fromBytes(std.testing.io, p, ta.shape(), std.mem.sliceAsBytes(ha)); - defer ba.deinit(); - var bb: zml.Buffer = try .fromBytes(std.testing.io, p, tb.shape(), std.mem.sliceAsBytes(hb)); - defer bb.deinit(); - var out = try exe.eval(std.testing.allocator, std.testing.io, .{ ba, bb }); - defer out.deinit(); - return out.toSliceAlloc(std.testing.allocator, std.testing.io); - } - }.f; - - // vector add: elementwise over a ragged (100, 1000), predicated at the edge. - { - const M, const N = .{ 100, 1000 }; - const Mod = struct { - pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return FlyVectorAdd.K.call(.{ .a = a, .b = b }, .{ .c = a.shape() }, .{ - .cfg = .{}, - .grid = .{ (M + 8 - 1) / 8, (N + 64 - 1) / 64, 1 }, - .threads = 8 * 16, - }).c; - } - }; - - const ha = try allocator.alloc(f32, M * N); - defer allocator.free(ha); - const hb = try allocator.alloc(f32, M * N); - defer allocator.free(hb); - for (ha, hb, 0..) |*va, *vb, i| { - va.* = @floatFromInt(i % 977); - vb.* = @floatFromInt(1000 + (i % 313)); - } - - var host = try call(Mod.forward, platform, .init(.{ M, N }, .f32), ha, .init(.{ M, N }, .f32), hb); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(ha[i] + hb[i], v); - } - - // tiled mma: C = A @ B^T in one block of 256 threads, over whatever matrix - // atom this device has. CDNA multiplies f32 through MFMA; RDNA3+ has WMMA, - // whose verifier rejects f32 operands, so the operand type and K come from - // the flavour rather than being written into the test. - if (fly.mmaFlavor(platform)) |flavor| switch (flavor) { - inline else => |f| { - const Elem = switch (comptime f.operandDType()) { - .f32 => f32, - .f16 => f16, - else => @compileError("unhandled MMA operand type"), - }; - const M, const N = .{ 64, 64 }; - const Kd = comptime f.blockK(); - const operand_dtype: zml.DataType = switch (comptime f.operandDType()) { - .f32 => .f32, - .f16 => .f16, - else => unreachable, - }; - - const Mod = struct { - pub fn forward(a: zml.Tensor, b: zml.Tensor) zml.Tensor { - return FlyTiledMma.K.call(.{ .a = a, .b = b }, .{ .c = zml.Shape.init(.{ M, N }, .f32) }, .{ - .cfg = .{ .flavor = f }, - .grid = .{ 1, 1, 1 }, - .threads = comptime f.blockThreads(), - }).c; - } - }; - - var ha: [M * Kd]Elem = undefined; - var hb: [N * Kd]Elem = undefined; - for (&ha, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 7) % 13)) - 6; - for (&hb, 0..) |*v, i| v.* = @as(Elem, @floatFromInt((i * 5) % 11)) - 5; - - var host = try call(Mod.forward, platform, .init(.{ M, Kd }, operand_dtype), &ha, .init(.{ N, Kd }, operand_dtype), &hb); - defer host.free(allocator); - - // f16 operands accumulate in f32, so the error is set by the input - // rounding rather than the sum. - const tol: f32 = if (Elem == f16) 1e-2 else 1e-3; - const out = host.items(f32); - for (0..M) |m| for (0..N) |n| { - var acc: f32 = 0; - for (0..Kd) |k| acc += @as(f32, ha[m * Kd + k]) * @as(f32, hb[n * Kd + k]); - try std.testing.expectApproxEqAbs(acc, out[m * N + n], tol); - }; - }, - }; -} diff --git a/zml/kernel/kernel.zig b/zml/kernel/kernel.zig deleted file mode 100644 index 204c6204..00000000 --- a/zml/kernel/kernel.zig +++ /dev/null @@ -1,12 +0,0 @@ -pub const cuda_tile = @import("cuda_tile/cuda_tile.zig"); -pub const cute = @import("cute/cute.zig"); -pub const fly = @import("fly/fly.zig"); -pub const metal = @import("metal/metal.zig"); -pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const tcl = @import("tcl/tcl.zig"); -pub const triton = @import("triton/triton.zig"); - -test { - const std = @import("std"); - std.testing.refAllDecls(@This()); -} diff --git a/zml/kernel/metal/BUILD.bazel b/zml/kernel/metal/BUILD.bazel deleted file mode 100644 index 4007beb4..00000000 --- a/zml/kernel/metal/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "metal", - srcs = [ - "metal.zig", - "test.zig", - ], -) diff --git a/zml/kernel/metal/metal.zig b/zml/kernel/metal/metal.zig deleted file mode 100644 index 9626a1de..00000000 --- a/zml/kernel/metal/metal.zig +++ /dev/null @@ -1,237 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -pub const Builder = metal_builder.Builder; -pub const Value = metal_builder.Value; -pub const Type = metal_builder.Type; -pub const Struct = metal_builder.Struct; -pub const ArgSpec = metal_builder.ArgSpec; -pub const LoopOpts = metal_builder.LoopOpts; -pub const FinishError = metal_builder.FinishError; -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(metal_builder.registerDialects); -} - -pub fn from(dt: DataType) Type { - return switch (dt) { - .bool => .bool, - .i8 => .char, - .i16 => .short, - .i32 => .int, - .i64 => .long, - .u8 => .uchar, - .u16 => .ushort, - .u32 => .uint, - .u64 => .ulong, - .f16 => .half, - .bf16 => .bfloat, - .f32 => .float, - else => std.debug.panic("zml.kernel.metal.from: dtype {s} has no MSL equivalent", .{@tagName(dt)}), - }; -} - -/// The bytes of a `constant T &` argument: `.of(@as(i32, n))`, or a host -/// `extern struct` laid out like a `constant S &` argument. -pub const Constant = struct { - buffer: [256]u8 = undefined, - len: usize = 0, - - pub fn of(value: anytype) Constant { - const raw = std.mem.asBytes(&value); - var c: Constant = .{ .len = raw.len }; - @memcpy(c.buffer[0..raw.len], raw); - return c; - } - - pub fn bytes(self: *const Constant) []const u8 { - return self.buffer[0..self.len]; - } -}; - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - /// The arguments bound to input tensors. - inputs: []const [:0]const u8, - /// The arguments bound to output tensors; one named like an - /// input updates it in place. - outputs: []const [:0]const u8, - /// The `constant` arguments given by value at the call. - constants: []const [:0]const u8 = &.{}, - run: *const fn (*Builder, Config) FinishError!void, - fast_math: bool = true, - }; -} - -/// A Metal kernel: `run` declares its arguments with `declareArgs`, by -/// name, and builds its body. -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - pub const Constants = StructOf(spec.constants, Constant); - - pub const CallOpts = struct { - cfg: ConfigT, - /// Threadgroups per grid. - grid: [3]u32, - /// Threads per threadgroup. - threadgroup: [3]u32, - /// Bytes of `[[threadgroup(0)]]` memory. - threadgroup_memory: u64 = 0, - constants: Constants = std.mem.zeroes(Constants), - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - return build(allocator, cfg, &arguments); - } - - fn build(allocator: std.mem.Allocator, cfg: ConfigT, arguments: *std.ArrayList(metal_builder.Argument)) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - var b = Builder.open(allocator, ctx, name, .{ .fast_math = spec.fast_math }); - defer b.deinit(); - try spec.run(&b, cfg); - const ir = try b.finish(); - errdefer allocator.free(ir); - try arguments.appendSlice(allocator, b.arguments.items); - return ir; - } - - fn indexOf(names: []const [:0]const u8, arg: []const u8) ?usize { - for (names, 0..) |n, i| if (std.mem.eql(u8, n, arg)) return i; - return null; - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - const allocator = cur.allocator; - - var arguments: std.ArrayList(metal_builder.Argument) = .empty; - defer arguments.deinit(allocator); - const ir = build(allocator, opts.cfg, &arguments) catch |err| - std.debug.panic("zml.kernel.metal.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer allocator.free(ir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |field, i| inputs_arr[i] = @field(inputs, field); - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |field, i| outputs_arr[i] = @field(outputs, field); - var constants: [spec.constants.len][]const u8 = undefined; - inline for (spec.constants, 0..) |field, i| constants[i] = @field(opts.constants, field).bytes(); - - var bound: [spec.outputs.len]bool = @splat(false); - var bound_inputs: [spec.inputs.len]bool = @splat(false); - var bound_constants: [spec.constants.len]bool = @splat(false); - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - var bindings: std.ArrayList(i32) = .empty; - defer bindings.deinit(allocator); - for (arguments.items) |arg| { - if (arg.kind == .threadgroup) { - // `threadgroup_memory` sizes `[[threadgroup(0)]]` alone. - if (arg.index != 0) std.debug.panic("zml.kernel.metal.Kernel({s}): threadgroup argument {s} is [[threadgroup({d})]], and only [[threadgroup(0)]] gets memory", .{ name, arg.name, arg.index }); - continue; - } - std.debug.assert(arg.index == bindings.items.len); - const input = indexOf(spec.inputs, arg.name); - const output = indexOf(spec.outputs, arg.name); - if (input) |i| bound_inputs[i] = true; - if (output) |o| bound[o] = true; - const binding: i32 = if (input) |i| blk: { - if (output) |o| aliases.appendAssumeCapacity(.{ .output_index = @intCast(o), .operand_index = @intCast(i) }); - break :blk @intCast(i); - } else if (output) |o| - @intCast(spec.inputs.len + o) - else if (indexOf(spec.constants, arg.name)) |k| blk: { - if (constants[k].len != arg.size) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} has {d} bytes, the argument {d}", .{ name, arg.name, constants[k].len, arg.size }); - bound_constants[k] = true; - break :blk -1 - @as(i32, @intCast(k)); - } else std.debug.panic("zml.kernel.metal.Kernel({s}): argument {s} is no input, output or constant", .{ name, arg.name }); - bindings.append(allocator, binding) catch @panic("OOM"); - } - for (bound, spec.outputs) |b, output| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): output {s} is no argument", .{ name, output }); - for (bound_inputs, spec.inputs) |b, input| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): input {s} is no argument", .{ name, input }); - for (bound_constants, spec.constants) |b, constant| if (!b) std.debug.panic("zml.kernel.metal.Kernel({s}): constant {s} is no argument", .{ name, constant }); - - const tensor_results = ops.metal(inputs_arr, outputs_arr, .{ - .name = name, - .ir = ir, - .grid = opts.grid, - .threadgroup = opts.threadgroup, - .threadgroup_memory = opts.threadgroup_memory, - .arguments = bindings.items, - .constants = &constants, - .output_operand_aliases = aliases.constSlice(), - }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |field, i| @field(results, field) = tensor_results[i]; - return results; - } - }; -} diff --git a/zml/kernel/metal/test.zig b/zml/kernel/metal/test.zig deleted file mode 100644 index 0a342af0..00000000 --- a/zml/kernel/metal/test.zig +++ /dev/null @@ -1,71 +0,0 @@ -const std = @import("std"); - -const metal = @import("metal.zig"); -const zml = @import("../../zml.zig"); - -/// `out[i] = a[i] * scale + b[i]` over `n` floats. -const MetalAxpy = struct { - const Cfg = struct {}; - const K = metal.Kernel(Cfg, .{ - .name = "axpy", - .inputs = &.{ "a", "b" }, - .outputs = &.{"out"}, - .constants = &.{ "scale", "n" }, - .run = run, - }); - - fn run(b: *metal.Builder, _: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .a = .{ .buffer = .{ .type = .float, .const_ = true } }, - .b = .{ .buffer = .{ .type = .float, .const_ = true } }, - .out = .{ .buffer = .{ .type = .float } }, - .scale = .{ .constant = .float }, - .n = .{ .constant = .uint }, - .tid = .{ .thread_position_in_grid = .uint }, - }); - b.returnIf(a.tid.ge(a.n)); - a.out.store(a.tid, b.fma(a.a.load(a.tid), a.scale, a.b.load(a.tid))); - } -}; - -test "metal kernels run on metal" { - const allocator = std.testing.allocator; - - const ir = try MetalAxpy.K.emit(allocator, .{}); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "air.fma") != null); - - const platform = zml.testing.env(); - if (platform.target != .metal) return error.SkipZigTest; - - const n = 1000; - const Mod = struct { - pub fn forward(x: zml.Tensor, y: zml.Tensor) zml.Tensor { - return MetalAxpy.K.call(.{ .a = x, .b = y }, .{ .out = x.shape() }, .{ - .cfg = .{}, - .grid = .{ (n + 255) / 256, 1, 1 }, - .threadgroup = .{ 256, 1, 1 }, - .constants = .{ .scale = .of(@as(f32, 0.5)), .n = .of(@as(u32, n)) }, - }).out; - } - }; - const x: zml.Tensor = .init(.{n}, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, zml.Tensor.init(.{n}, .f32) }, platform, .{}); - defer exe.deinit(); - - var hx: [n]f32 = undefined; - var hy: [n]f32 = undefined; - for (&hx, &hy, 0..) |*a, *b, i| { - a.* = @floatFromInt(i); - b.* = 1; - } - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var by: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&hy)); - defer by.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{ bx, by }); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), hx, hy) |out, a, b| try std.testing.expectEqual(a * 0.5 + b, out); -} diff --git a/zml/kernel/mosaic_tpu/BUILD.bazel b/zml/kernel/mosaic_tpu/BUILD.bazel deleted file mode 100644 index ba3b6b66..00000000 --- a/zml/kernel/mosaic_tpu/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "mosaic_tpu", - srcs = ["mosaic_tpu.zig"], -) diff --git a/zml/kernel/mosaic_tpu/mosaic_tpu.zig b/zml/kernel/mosaic_tpu/mosaic_tpu.zig deleted file mode 100644 index a09e8368..00000000 --- a/zml/kernel/mosaic_tpu/mosaic_tpu.zig +++ /dev/null @@ -1,284 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const mosaic_tpu_builder = @import("dsl/mosaic_tpu/builder"); -const Builder = mosaic_tpu_builder.Builder; -const DType = mosaic_tpu_builder.DType; -const FinishError = mosaic_tpu_builder.FinishError; -const mosaic_tpu_dialect = @import("mlir/dialects/mosaic_tpu"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(mosaic_tpu_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.mosaic_tpu.from: dtype {s} has no Mosaic-TPU equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallExtras = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, - additional_attributes: []const mlir.NamedAttribute = &.{}, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - dynamic_grid_bounds: []const Tensor = &.{}, - }; - - pub const CallOpts = struct { - cfg: ConfigT, - extras: CallExtras = .{}, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try mosaic_tpu_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finishOpts(&.{}, .{ .canonicalize = true }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - const backend_config = buildBackendConfig(cur.allocator, ir, .{ - .vmem_limit_bytes = opts.extras.vmem_limit_bytes, - .disable_bounds_checks = opts.extras.disable_bounds_checks, - .disable_semaphore_checks = opts.extras.disable_semaphore_checks, - .has_communication = opts.extras.has_communication, - }) catch |err| - std.debug.panic("zml.kernel.mosaic_tpu.Kernel({s}).call: backend_config build failed: {}", .{ name, err }); - defer cur.allocator.free(backend_config); - - var values: [spec.inputs.len]*const mlir.Value = undefined; - inline for (spec.inputs, 0..) |fname, i| values[i] = @field(inputs, fname).value(); - - var res_types: [spec.outputs.len]*const mlir.Type = undefined; - inline for (&res_types, spec.outputs) |*r, fname| { - r.* = mlirx.Type.rankedTensor(cur.mlir_ctx, @field(outputs, fname)); - } - - const aliases = resolveOutputOperandAliases(opts.extras.output_operand_aliases, @intCast(opts.extras.dynamic_grid_bounds.len)); - - const op = callTpuCustomCall(.{ - .inputs = &values, - .result_types = &res_types, - .backend_config = backend_config, - .aliases = aliases.constSlice(), - .dynamic_grid_bounds = opts.extras.dynamic_grid_bounds, - .additional_attributes = opts.extras.additional_attributes, - }); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - const out = @field(outputs, fname); - @field(results, fname) = Tensor._result(out, op.result(i)); - } - return results; - } - }; -} - -const CustomCallConfig = struct { - body: []const u8, - has_communication: ?bool = null, - serialization_format: i64 = 1, - needs_layout_passes: bool = true, - shape_invariant_numerics: bool = false, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, -}; - -const ScopedMemoryConfig = struct { - memory_space: i64 = 1, - offset: i64 = 0, - size: i64, -}; - -const BackendConfig = struct { - custom_call_config: CustomCallConfig, - scoped_memory_configs: ?[]const ScopedMemoryConfig = null, -}; - -const BackendConfigOpts = struct { - vmem_limit_bytes: ?i64 = null, - disable_bounds_checks: ?bool = null, - disable_semaphore_checks: ?bool = null, - has_communication: ?bool = null, -}; - -fn buildBackendConfig( - allocator: std.mem.Allocator, - ir: [:0]const u8, - opts: BackendConfigOpts, -) ![]u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - const module = try mlir.Module.parse(ctx, ir); - defer module.deinit(); - - mosaic_tpu_dialect.registerMosaicSerdePass(); - const prev_allow_unregistered = ctx.allowUnregisteredDialects(); - ctx.setAllowUnregisteredDialects(true); - defer ctx.setAllowUnregisteredDialects(prev_allow_unregistered); - - const pm = mlir.PassManager.init(ctx); - defer pm.deinit(); - try pm.asOpPassManager().addPipeline("mosaic-serde{serialize=true}"); - try pm.runOnOp(module.operation()); - - const Encoder = std.base64.standard.Encoder; - const b64 = b64: { - var bytecode: std.Io.Writer.Allocating = .init(allocator); - defer bytecode.deinit(); - try module.operation().writeBytecode(.{ .desired_emit_version = 0 }, &bytecode.writer); - const buf = try allocator.alloc(u8, Encoder.calcSize(bytecode.written().len)); - break :b64 Encoder.encode(buf, bytecode.written()); - }; - defer allocator.free(b64); - - const config = BackendConfig{ - .custom_call_config = .{ - .body = b64, - .has_communication = opts.has_communication, - .disable_bounds_checks = opts.disable_bounds_checks, - .disable_semaphore_checks = opts.disable_semaphore_checks, - }, - .scoped_memory_configs = if (opts.vmem_limit_bytes) |vmem| - &.{.{ .size = vmem }} - else - null, - }; - - var json: std.Io.Writer.Allocating = .init(allocator); - errdefer json.deinit(); - - try json.writer.print("{f}", .{std.json.fmt(config, .{ .emit_null_optional_fields = false })}); - return try json.toOwnedSlice(); -} - -const TpuCustomCallArgs = struct { - inputs: []const *const mlir.Value, - result_types: []const *const mlir.Type, - backend_config: []const u8, - aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias, - dynamic_grid_bounds: []const Tensor, - additional_attributes: []const mlir.NamedAttribute, -}; - -fn callTpuCustomCall(args: TpuCustomCallArgs) *mlir.Operation { - const cur = Compiler.current(); - - var all_inputs: stdx.BoundedArray(*const mlir.Value, stablehlo.CustomCallOpts.MAX_OPERANDS) = .empty; - for (args.dynamic_grid_bounds) |t| all_inputs.appendAssumeCapacity(t.value()); - all_inputs.appendSliceAssumeCapacity(args.inputs); - - return stablehlo.custom_call( - cur.mlir_ctx, - all_inputs.constSlice(), - args.result_types, - .{ - .call_target_name = "tpu_custom_call", - .backend_config = .{ .original = args.backend_config }, - .has_side_effect = false, - .api_version = .original, - .additional_attributes = args.additional_attributes, - .output_operand_aliases = args.aliases, - }, - cur.location, - ).appendTo(cur.currentScope().block); -} diff --git a/zml/kernel/tcl/BUILD.bazel b/zml/kernel/tcl/BUILD.bazel deleted file mode 100644 index bc7a0aa2..00000000 --- a/zml/kernel/tcl/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "tcl", - srcs = [ - "tcl.zig", - "test.zig", - ], -) diff --git a/zml/kernel/tcl/tcl.zig b/zml/kernel/tcl/tcl.zig deleted file mode 100644 index 023e862f..00000000 --- a/zml/kernel/tcl/tcl.zig +++ /dev/null @@ -1,206 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const metal_builder = @import("dsl/metal/builder"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const tcl_builder = @import("dsl/tcl/builder"); -pub const Builder = tcl_builder.Builder; -pub const TensorOperation = tcl_builder.TensorOperation; -pub const Value = tcl_builder.Value; -pub const Axis = tcl_builder.Axis; -pub const Expr = tcl_builder.Expr; -pub const Dram = tcl_builder.Dram; -pub const Context = tcl_builder.Context; -pub const DType = tcl_builder.DType; -pub const VeOp = tcl_builder.VeOp; -pub const ReduceMode = tcl_builder.ReduceMode; -pub const Predicate = tcl_builder.Predicate; -pub const Tactic = tcl_builder.Tactic; -pub const FinishError = tcl_builder.FinishError; - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const mlirx = @import("../../mlirx.zig"); -const ops = @import("../../ops.zig"); -const platform_ = @import("../../platform.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -test { - _ = @import("test.zig"); -} - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(tcl_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .bool, - .i4 => .i4, - .i8 => .i8, - .i16 => .i16, - .i32 => .i32, - .i64 => .i64, - .u8 => .u8, - .u16 => .u16, - .u32 => .u32, - .u64 => .u64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f4e2m1 => .f4_e2, - .f8e4m3fn => .f8_e4, - .f8e5m2 => .f8_e5, - else => std.debug.panic("zml.kernel.tcl.from: dtype {s} has no TCL equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -/// A TCL function compiled by the Furiosa plugin as one native unit: -/// `declareArgs` declares the inputs in order, `ret` the outputs. -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - return build(allocator, cfg, null, null, null); - } - - /// `updated`, when given, receives the input each output updates - /// in place (`Builder.update`), or null. - fn build(allocator: std.mem.Allocator, cfg: ConfigT, inputs: ?[]const Shape, outputs: ?[]const Shape, updated: ?*[spec.outputs.len]?usize) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try Builder.open(allocator, ctx, name); - defer b.deinit(); - try spec.run(&b, cfg); - - const results = b.results orelse std.debug.panic("zml.kernel.tcl.Kernel({s}): run did not call ret", .{name}); - if (b.args.len != spec.inputs.len or results.len != spec.outputs.len) { - std.debug.panic("zml.kernel.tcl.Kernel({s}): {d} arguments and {d} results for {d} inputs and {d} outputs", .{ name, b.args.len, results.len, spec.inputs.len, spec.outputs.len }); - } - - if (inputs) |shapes| for (b.args, shapes, spec.inputs) |t, s, n| check(t, s, n); - if (outputs) |shapes| for (results, shapes, spec.outputs) |t, s, n| check(t, s, n); - if (updated) |u| for (u, results) |*dst, r| { - dst.* = b.updatedArgument(r); - }; - - return b.finish(); - } - - fn check(t: tcl_builder.Tensor, s: Shape, field: []const u8) void { - var same = from(s.dtype()) == t.dtype; - if (t.shards > 1) { - // A partition holds one chip's share of a chip-split DRAM tensor. - var n: i64 = 1; - for (t.axes) |a| n *= a.size; - same = same and n == @as(i64, @intCast(s.count())) * t.shards; - } else { - same = same and s.rank() == t.axes.len; - if (same) for (t.axes, 0..) |a, i| { - same = same and a.size == s.dim(i); - }; - } - - if (!same) std.debug.panic("zml.kernel.tcl.Kernel({s}): {s} is {f}, the TCL function has {s} over {d} axes", .{ name, field, s, @tagName(t.dtype), t.axes.len }); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - var input_shapes: [spec.inputs.len]Shape = undefined; - inline for (spec.inputs, 0..) |fname, i| { - inputs_arr[i] = @field(inputs, fname); - input_shapes[i] = inputs_arr[i].shape(); - } - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - var updated: [spec.outputs.len]?usize = undefined; - const ir = build(cur.allocator, opts.cfg, &input_shapes, &outputs_arr, &updated) catch |err| - std.debug.panic("zml.kernel.tcl.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ir); - - var aliases: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, spec.outputs.len) = .empty; - for (updated, 0..) |input, output| if (input) |i| aliases.appendAssumeCapacity(.{ - .output_index = @intCast(output), - .operand_index = @intCast(i), - }); - const tensor_results = ops.tcl(inputs_arr, outputs_arr, .{ .ir = ir, .output_operand_aliases = aliases.constSlice() }); - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| { - @field(results, fname) = tensor_results[i]; - } - - return results; - } - }; -} diff --git a/zml/kernel/tcl/test.zig b/zml/kernel/tcl/test.zig deleted file mode 100644 index b9ae1875..00000000 --- a/zml/kernel/tcl/test.zig +++ /dev/null @@ -1,123 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const tcl = @import("tcl.zig"); - -/// `furiosa.tcl.examples.softmax` over f32. -const TclSoftmax = struct { - const Cfg = struct { b: i64, s: i64, d: i64 }; - const K = tcl.Kernel(Cfg, .{ - .name = "softmax_kernel", - .inputs = &.{"x"}, - .outputs = &.{"y"}, - .run = run, - }); - - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - const B = b.axis("B", cfg.b); - const S = b.axis("S", cfg.s); - const D = b.axis("D", cfg.d); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ B, S, D } } }); - - const op = b.tensorOperation(.{}); - const x = op.fetch(t.x, .{}); - const e = x.subf(x.reduce(&.{D}, .maxf)).exp(); - b.ret(&.{try op.commit(e.divf(e.reduce(&.{D}, .addf)), .{})}); - } -}; - -test "tcl kernels run on furiosa" { - const allocator = std.testing.allocator; - const cfg: TclSoftmax.Cfg = .{ .b = 2, .s = 16, .d = 256 }; - - const ir = try TclSoftmax.K.emit(allocator, cfg); - defer allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "func.func @softmax_kernel(") != null); - - const platform = zml.testing.env(); - if (platform.target != .furiosa) return error.SkipZigTest; - - const Mod = struct { - pub fn forward(x: zml.Tensor) zml.Tensor { - return TclSoftmax.K.call(.{ .x = x }, .{ .y = x.shape() }, .{ .cfg = cfg }).y; - } - }; - const x: zml.Tensor = .init(.{ cfg.b, cfg.s, cfg.d }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{x}, platform, .{}); - defer exe.deinit(); - - var input: [cfg.b * cfg.s * cfg.d]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 37)) * 0.25 - 4; - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&input)); - defer x_buffer.deinit(); - var result = try exe.eval(allocator, std.testing.io, .{x_buffer}); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - - const out = host.items(f32); - var row: usize = 0; - while (row < cfg.b * cfg.s) : (row += 1) { - const in = input[row * cfg.d ..][0..cfg.d]; - var max: f32 = in[0]; - for (in) |v| max = @max(max, v); - var sum: f32 = 0; - for (in) |v| sum += @exp(v - max); - for (in, out[row * cfg.d ..][0..cfg.d]) |v, y| try std.testing.expectApproxEqAbs(@exp(v - max) / sum, y, 1e-4); - } -} - -/// exp over a tensor split across two chips in one launch, returning the -/// split result and its all_gather. -const TclSplitExp = struct { - const Cfg = struct {}; - const K = tcl.Kernel(Cfg, .{ .name = "split_exp", .inputs = &.{"x"}, .outputs = &.{ "y", "all" }, .run = run }); - fn run(b: *tcl.Builder, _: Cfg) tcl.FinishError!void { - const C = b.axis("C", 2); - const A = b.axis("A", 1024); - const t = try b.declareArgs(.{ .x = .{ .dtype = .f32, .axes = &.{ C, A }, .dram = tcl.Dram{ .chip = &.{.{ .axis = C }} } } }); - const op = b.tensorOperation(.{ .context = .{ .layout = .{ .chip = .{ .axis = C } } } }); - const local = try op.commit(op.fetch(t.x, .{}).exp(), .{}); - const all = try b.allGather(local, C, .{}); - b.ret(&.{ try b.asDram(local, .{ .chip = &.{.{ .axis = C }} }), try b.asDram(all, .{}) }); - } - fn body(ctx: struct { x: zml.Tensor }, shapes: [2]zml.Shape) [2]zml.Tensor { - const r = K.call(.{ .x = ctx.x }, .{ .y = shapes[0], .all = shapes[1] }, .{ .cfg = .{} }); - return .{ r.y, r.all }; - } -}; - -// Needs FURIOSA_VISIBLE_DEVICES=0,1. -test "tcl multi-chip kernels run on two furiosa chips" { - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - if (platform.target != .furiosa or platform.devices.len != 2) return error.SkipZigTest; - const tp = zml.testing.tpMesh(); - const x_shape = zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32).withPartitioning(tp, .{ .c = .model }); - const Mod = struct { - pub fn forward(x: zml.Tensor, mesh: *const zml.Sharding.Mesh) [2]zml.Tensor { - const r = zml.ops.manualComputation(TclSplitExp.body, .{ .x = x }, .{ x.shape(), zml.Shape.init(.{ .c = 2, .a = 1024 }, .f32) }, mesh, .{ - .in_specs = .{ .x = .parse(.{ .c = .model }) }, - .out_specs = .{ .parse(.{ .c = .model }), .replicated }, - }); - return .{ r[0], r[1] }; - } - }; - const x: zml.Tensor = .fromShape(x_shape); - var exe = try zml.module.compile(allocator, std.testing.io, Mod.forward, .{ x, tp }, platform, .{ .meshes = &.{tp} }); - defer exe.deinit(); - var hx: [2 * 1024]f32 = undefined; - for (&hx, 0..) |*v, i| v.* = @as(f32, @floatFromInt(i % 13)) * 0.125 - 0.5; - var bx: zml.Buffer = try .fromBytes(std.testing.io, platform, x_shape, std.mem.sliceAsBytes(&hx)); - defer bx.deinit(); - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - var res: [2]zml.Buffer = undefined; - runner.run(std.testing.io, .{bx}, .{ &res[0], &res[1] }, .{ .wait = true }); - defer for (&res) |*r| r.deinit(); - for (res) |r| { - var host = try r.toSliceAlloc(allocator, std.testing.io); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectApproxEqRel(@exp(hx[i]), v, 1e-4); - } -} diff --git a/zml/kernel/triton/BUILD.bazel b/zml/kernel/triton/BUILD.bazel deleted file mode 100644 index 1e992352..00000000 --- a/zml/kernel/triton/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//zml/kernel:__pkg__"]) - -filegroup( - name = "triton", - srcs = ["triton.zig"], -) diff --git a/zml/kernel/triton/triton.zig b/zml/kernel/triton/triton.zig deleted file mode 100644 index 6a91a03f..00000000 --- a/zml/kernel/triton/triton.zig +++ /dev/null @@ -1,165 +0,0 @@ -const std = @import("std"); - -const func = @import("mlir/dialects/func"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); -const triton_builder = @import("dsl/triton/builder"); -pub const Builder = triton_builder.Builder; -pub const Value = triton_builder.Value; -pub const DType = triton_builder.DType; -pub const FinishError = triton_builder.FinishError; - -const Compiler = @import("../../Compiler.zig"); -const DataType = @import("../../dtype.zig").DataType; -const ops = @import("../../ops.zig"); -const Shape = @import("../../shape.zig").Shape; -const Tensor = @import("../../tensor.zig").Tensor; - -/// Build a struct type with one field per name, all of type `FieldT`. -fn StructOf(comptime names: []const [:0]const u8, comptime FieldT: type) type { - var name_slices: [names.len][]const u8 = undefined; - for (names, 0..) |n, i| name_slices[i] = n; - return @Struct(.auto, null, &name_slices, &@splat(FieldT), &@splat(.{})); -} - -fn makeKernelContext(comptime registerDialects: fn (*mlir.DialectRegistry) void) std.mem.Allocator.Error!*mlir.Context { - mlir.registerPasses("Transforms"); - - const registry = try mlir.DialectRegistry.init(); - defer registry.deinit(); - - registerDialects(registry); - - func.registerExtensions(registry); - - const ctx = try mlir.Context.init(.{ .registry = registry, .threading = false }); - ctx.loadAllAvailableDialects(); - - return ctx; -} - -fn resolveOutputOperandAliases( - aliases_opt: anytype, - operand_offset: i64, -) stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) { - var out: stdx.BoundedArray(stablehlo.CustomCallOpts.OutputOperandAlias, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - - if (aliases_opt) |a| { - inline for (@typeInfo(@TypeOf(a)).@"struct".field_names, 0..) |field_name, output_index| { - if (@field(a, field_name)) |operand| out.appendAssumeCapacity(.{ - .output_index = @intCast(output_index), - .operand_index = @as(i64, @intCast(@backingInt(operand))) + operand_offset, - }); - } - } - - return out; -} - -pub fn newContext() std.mem.Allocator.Error!*mlir.Context { - return makeKernelContext(triton_builder.registerDialects); -} - -pub fn from(dt: DataType) DType { - return switch (dt) { - .bool => .i1, - .i8, .u8 => .i8, - .i16, .u16 => .i16, - .i32, .u32 => .i32, - .i64, .u64 => .i64, - .f16 => .f16, - .bf16 => .bf16, - .f32 => .f32, - .f64 => .f64, - .f8e4m3fn => .f8e4m3fn, - .f8e4m3fnuz => .f8e4m3fnuz, - .f8e5m2 => .f8e5m2, - else => std.debug.panic("zml.kernel.triton.from: dtype {s} has no Triton equivalent", .{@tagName(dt)}), - }; -} - -fn Spec(comptime Config: type) type { - return struct { - name: [:0]const u8, - inputs: []const [:0]const u8, - outputs: []const [:0]const u8, - run: *const fn (*Builder, Config) FinishError!void, - }; -} - -pub fn Kernel( - comptime ConfigT: type, - comptime spec: Spec(ConfigT), -) type { - return struct { - pub const name: [:0]const u8 = spec.name; - pub const Config = ConfigT; - pub const Inputs = StructOf(spec.inputs, Tensor); - pub const Outputs = StructOf(spec.outputs, Shape); - pub const Results = StructOf(spec.outputs, Tensor); - - pub const CallOpts = struct { - cfg: ConfigT, - grid: [3]i32, - num_stages: i32, - num_warps: i32, - output_operand_aliases: ?ops.CustomCallOutputOperandAliases(Inputs, Outputs) = null, - debug: bool = false, - /// Bytes for Triton's implicit global scratch argument, across all CTAs. - global_scratch_memory_size: i32 = 0, - }; - - pub fn emit(allocator: std.mem.Allocator, cfg: ConfigT) ![:0]const u8 { - const ctx = try newContext(); - defer ctx.deinit(); - - var b = try triton_builder.Builder.open(allocator, ctx, name); - defer b.deinit(); - - try spec.run(&b, cfg); - - return b.finish(); - } - - pub fn call(inputs: Inputs, outputs: Outputs, opts: CallOpts) Results { - const cur = Compiler.current(); - - const ttir = emit(cur.allocator, opts.cfg) catch |err| - std.debug.panic("zml.kernel.triton.Kernel({s}).call: emit failed: {}", .{ name, err }); - defer cur.allocator.free(ttir); - - var inputs_arr: [spec.inputs.len]Tensor = undefined; - inline for (spec.inputs, 0..) |fname, i| inputs_arr[i] = @field(inputs, fname); - - var outputs_arr: [spec.outputs.len]Shape = undefined; - inline for (spec.outputs, 0..) |fname, i| outputs_arr[i] = @field(outputs, fname); - - const aliases = resolveOutputOperandAliases(opts.output_operand_aliases, 0); - - const call_opts: ops.TritonOps = .{ - .debug = opts.debug, - .name = name, - .ir = ttir, - .grid = opts.grid, - .num_stages = opts.num_stages, - .num_warps = opts.num_warps, - .output_operand_aliases = aliases.constSlice(), - .is_tma_allowed = opts.global_scratch_memory_size > 0, - .global_scratch_memory_size = opts.global_scratch_memory_size, - }; - - const triton_results = if (opts.global_scratch_memory_size > 0) blk: { - // XLA maps the final custom-call result to Triton's implicit - // scratch pointer; it must not appear in the TTIR signature. - const extended = outputs_arr ++ [_]Shape{.init(.{opts.global_scratch_memory_size}, .u8)}; - const res = ops.triton(inputs_arr, extended, call_opts); - break :blk res[0..spec.outputs.len].*; - } else ops.triton(inputs_arr, outputs_arr, call_opts); - - var results: Results = undefined; - inline for (spec.outputs, 0..) |fname, i| @field(results, fname) = triton_results[i]; - return results; - } - }; -} diff --git a/zml/kernels/flash_attention/README.md b/zml/kernels/flash_attention/README.md deleted file mode 100644 index 81a35cf4..00000000 --- a/zml/kernels/flash_attention/README.md +++ /dev/null @@ -1,287 +0,0 @@ -# Flash attention semantics - -This directory exposes causal scaled dot-product attention through -[`zml.flash_attention.attention`](flash_attention.zig). This document describes -the attention operation, the FlashAttention algorithm, and ZML's current semantic -contract. Shapes below are logical tensor shapes. Sharding describes ownership -of those tensors across devices; platform-specific execution layouts are outside -the scope of this document. - -## The attention operation - -For one sequence and one head, let `Q` contain `Lq` query vectors, and let `K` and -`V` contain `Lk` key/value vectors. Each query is compared with each permitted key; -the resulting probabilities weight the corresponding values: - -```text -S = scale * Q @ transpose(K) + M -P = softmax(S, axis=keys) -O = P @ V -``` - -The usual scale is `1 / sqrt(D)`, where `D` is the query/key head dimension. -An additive mask `M` is zero for allowed pairs and negative infinity for forbidden -pairs. Softmax normalizes each query row independently over keys, so a masked key -has zero probability. Heads and batch elements are independent. ZML's -[`sdpa`](../../nn.zig) implements this reference computation. - -Attention in general permits a value dimension `Dv` different from the query/key -dimension `D`; the output then has dimension `Dv`. This module uses the same -`.hd = D` for queries, keys, values, and output. - -FlashAttention can evaluate either unrestricted attention or masked attention. -Causality is a choice of mask, rather than an inherent property of the algorithm. -Other attention APIs may offer local windows, additive score biases, a custom -scale, and dropout on the probabilities during training. These options change -the operation and must be matched when comparing results. The -[upstream FlashAttention API](https://github.com/Dao-AILab/flash-attention#how-to-use-flashattention) -documents such options; ZML's interface here exposes a narrower operation. - -## What makes attention "flash" - -FlashAttention computes the same mathematical attention result while avoiding -storage of the complete `Lq × Lk` score and probability matrices. It processes -query/key/value tiles and combines partial results using an online softmax. -This reduces intermediate memory and memory traffic. Dense attention still -requires `O(Lq * Lk * D)` arithmetic per head; for self-attention with `Lq = Lk`, -that remains quadratic in sequence length. "Exact" means the attention formula -is preserved, with ordinary floating-point rounding differences. See the -[FlashAttention paper](https://arxiv.org/abs/2205.14135). - -For each query row, online softmax maintains a maximum `m`, an exponential sum -`l`, and an unnormalized weighted-value vector `u`. Given scores `s_j` for a new -tile of permitted keys, update: - -```text -m_new = max(m, max_j(s_j)) -a = exp(m - m_new) -p_j = exp(s_j - m_new) -l_new = a * l + sum_j(p_j) -u_new = a * u + sum_j(p_j * V_j) -O = u / l # after all tiles -``` - -Initialize with `m = -infinity`, `l = 0`, and `u = 0`; skip tiles with no permitted -keys. Rescaling the previous accumulators preserves a single normalization across -all tiles. Independently normalizing each tile and adding its output would give -a different result. The [FlashAttention-2 paper, sections 2–3](https://arxiv.org/html/2307.08691v1) -describes this recurrence and how training can recompute probability tiles during -the backward pass instead of retaining the full matrix. This module exposes only -the attention output, with no backward entry point or returned probabilities. - -## Heads and head dimensions - -`D` counts features in **one head**. It is distinct from the number of heads and -the sequence length. In a conventional multi-head projection, -`D = hidden_size / Hq`; models with an explicit head dimension need not follow -that relation. The query projection has width `Hq * D`, while each key/value -projection has width `Hkv * D`. - -Typical dimensions include: - -| Head dimension | Context | -| --- | --- | -| `64` | A common transformer head size; also the size used by this module's shared tests. | -| `128` | Another common size; Qwen2.5-7B uses `3584 / 28 = 128`. | -| `256` | Used by models such as Gemma 2 2B and 9B. | -| `512` | Used by Gemma 4 global attention layers, for example in Gemma 4 31B. | - -The [FlashAttention-2 paper](https://arxiv.org/html/2307.08691v1) describes typical -head sizes around 64–128. The model examples follow the -[Qwen2.5-7B configuration](https://huggingface.co/Qwen/Qwen2.5-7B-Instruct/raw/main/config.json) -and [Gemma configurations](https://github.com/google/gemma_pytorch/blob/main/gemma/config.py). -Gemma can also use different head dimensions within one model: the -[Gemma 4 31B configuration](https://huggingface.co/google/gemma-4-31B/blob/main/config.json) -specifies `head_dim = 256` for local attention and `global_head_dim = 512` for -global attention. Use the dimension configured for the specific layer. -These are model dimensions, not a list of supported kernel sizes or a mathematical -requirement that `D` be a power of two. - -The number of query heads `Hq` may differ from the number of key/value heads -`Hkv`. ZML uses contiguous groups of query heads: - -```text -G = Hq / Hkv -kv_head(h) = floor(h / G) -``` - -`Hq` must be divisible by `Hkv`, and keys and values must have the same head count. -This covers multi-head attention (MHA: `Hq = Hkv`), grouped-query attention -(GQA: `1 < Hkv < Hq`), and multi-query attention (MQA: `Hkv = 1`). For example, -with `Hq = 64` and `Hkv = 8`, query heads 0–7 use KV head 0, heads 8–15 use KV -head 1, and so on. Sharing keys and values does not merge query heads: each query -head still has its own scores, normalization, and output. - -## ZML inputs and output - -The entry point is: - -```zig -attention(q, k, v, token_index, metadata, parameters) -``` - -The reference contract uses these tagged shapes, with an optional `.b` batch axis: - -| Argument | Unbatched shape | Batched shape | Meaning | -| --- | --- | --- | --- | -| `q` | `{ .q = Lq, .h = Hq, .hd = D }` | `{ .b = B, .q = Lq, .h = Hq, .hd = D }` | Queries to evaluate. | -| `k`, `v` | `{ .k = Lk, .h = Hkv, .hd = D }` | `{ .b = B, .k = Lk, .h = Hkv, .hd = D }` | Keys and values in sequence order. | -| `token_index` | Scalar | `{ .b = B }` or scalar | Zero-based position of the first query in the key sequence; a scalar applies to the whole batch. | -| Result | Same as `q` | Same as `q` | Weighted values for every query head. | - -Use matching floating-point dtypes for `q`, `k`, and `v`; the output retains the -query dtype. The shared tests use `bf16` tensors and `u32` positions. `metadata` -and `parameters` provide execution configuration rather than additional masks, -score biases, or model projections. - -### Causality and query positions - -Let `t = token_index` for a sequence. Query row `i` represents position `t + i`, -and key row `j` represents position `j`. The inclusive causal rule is: - -```text -allowed(i, j) = (j <= t + i) -M[i, j] = 0 if allowed(i, j), otherwise -infinity -``` - -The reference path constructs a square causal mask over `Lk` positions, gathers -`Lq` consecutive rows starting at `t`, and passes that mask to `sdpa`. Valid inputs -require `Lq > 0`, `t >= 0`, and `t + Lq <= Lk`. The test helper checks the upper -bound; callers must satisfy it rather than rely on out-of-range gather behavior. -Each valid query can attend to at least its own key, so fully masked rows are -outside this contract. - -| Use case | Query positions | Keys visible to query row `i` | -| --- | --- | --- | -| Initial prefill | `t = 0`, usually `Lq = Lk` | `0` through `i`. | -| Chunked prefill | `Lq > 1`, starting at `t` | `0` through `t + i`. | -| Single-token decode | `Lq = 1`, current token at `t` | `0` through `t`, including the current token. | - -For `Lq = 2`, `Lk = 5`, and `t = 2`, the mask is: - -```text - key position - 0 1 2 3 4 -query at 2 1 1 1 0 0 -query at 3 1 1 1 1 0 -``` - -Here `1` means permitted. `Lk` may be the capacity of a preallocated KV buffer; -it does not determine the current query position. All permitted KV entries must -already be populated. Positions after the last query are masked, and the call -does not append to or update a KV cache. - -For unequal query and key lengths, causal alignment must be specified explicitly. -The [upstream FlashAttention API](https://github.com/Dao-AILab/flash-attention#21-change-behavior-of-causal-flag) -uses bottom-right alignment: `j <= i + Lk - Lq`. ZML's reference rule agrees with -that convention when `t = Lk - Lq`. An explicit `token_index` also describes -queries earlier in a larger buffer, as in the example above. - -### Sharding and local attention - -Sharding changes where attention is evaluated, not its mathematical result. -The global output still has the query's tagged shape and dtype. Each local call -must receive the matching queries, keys, values, and positions for the batch -elements and heads it owns. - -| Axis or input | Sharding semantics | -| --- | --- | -| Batch `.b` | Sequences are independent. Q, K, V, and the output must assign the same batch elements to each shard. | -| Heads `.h` | Each shard must have the KV heads needed by its query heads, preserving `kv_head(h) = floor(h / G)`. K and V must have matching head ownership. | -| Query sequence `.q` | Query rows are independent once the required KV entries are available. Their causal positions remain positions in the original sequence. | -| Key sequence `.k` | Softmax normalizes over all permitted keys for a query, including keys on other shards if the sequence is distributed. Separate local softmax outputs cannot simply be added or averaged. | -| Features `.hd` | Every score uses the complete head dimension and the global scale `1 / sqrt(D)`. Splitting features requires combining partial dot products before softmax. | -| `token_index` | A scalar applies to every shard. Per-sequence positions follow the batch partition and are replicated across head shards of the same sequence. Head or batch sharding does not change their values. | - -For head sharding without KV replication, a partition count `P` must divide -`Hkv`, and therefore also `Hq`. Each shard owns complete, contiguous GQA groups: -`Hq_local = Hq / P`, `Hkv_local = Hkv / P`, and the local ratio remains `G`. -For example, with `Hq = 32`, `Hkv = 8`, and two shards, the first shard owns query -heads 0–15 and KV heads 0–3; the second owns query heads 16–31 and KV heads 4–7. -Each query still uses exactly the same KV head as in the unsharded computation. -MQA has only one KV head, so distributing its query heads over multiple devices -requires replicating that KV head rather than dividing it into independent heads. - -The local-kernel execution contract keeps `.q`, `.k`, and `.hd` complete on each -shard, with parallelism across batches and/or complete head groups. Thus each -local call has the full sequence coordinates and feature vectors needed by the -causal mask, dot products, and normalization. The mathematical possibilities for -sequence and feature sharding above do not imply that these local kernels -implement distributed sequence or feature reductions. - -An **open** partition means placement is unresolved; it does not mean the tensor -must be replicated. Operations such as cache slicing and RoPE can reopen -partition annotations even when their inputs were sharded. Before entering a -local kernel, resolve compatible partitions on a common mesh and align Q, K, V, -and positions. The KV layout anchors this alignment: an explicit KV head -partition is retained, while an open head partition can use a compatible mesh -axis whose partition count divides `Hkv`. Preserve explicit batch ownership and -do not assign the same device partition independently to both batch and heads. -An explicit replicated KV head axis remains replicated; open heads can also -remain replicated when no compatible head partition is available. Automatic -partition selection is an execution choice, not an additional attention argument. - -The output follows the **resolved** query partition. Its partition annotations -need not match the incoming query's unresolved annotations, and aligning operands -may require communication. In particular, unnecessarily resolving open KV heads -to replicated can gather the full cache on every attention call even though the -head groups could be evaluated independently. This changes cost, not attention -semantics, and does not modify the caller's KV cache. - -Splitting KV work into internal tiles or partitions is distinct from sharding the -input tensors across devices. Either way, combining partial attention states -must preserve the single softmax normalization. For disjoint key subsets with -states `(m_r, l_r, u_r)` as defined above: - -```text -m = max_r(m_r) -l = sum_r(exp(m_r - m) * l_r) -u = sum_r(exp(m_r - m) * u_r) -O = u / l -``` - -An empty subset contributes zero (`m_r = -infinity`, `l_r = 0`, `u_r = 0`). -Every valid query has at least one permitted key overall, so its merged -normalization is nonempty. - -### Scope and current conformance - -The public call always requests causal attention with scale `1 / sqrt(D)` and no -dropout. It has no arguments for an arbitrary mask, padding lengths, packed -sequence boundaries, a local window, ALiBi, score soft-capping, a softmax sink, or -a custom scale. The lower-level `sdpa` and `causalAttnMask` helpers expose some -additional options, but this wrapper does not pass them through. Separate batch -elements can have different `token_index` values; concatenating independent -sequences along `.k` does not isolate them from each other. - -Q/K/V projections, positional transformations such as RoPE, and any Q/K -normalization must happen before this call. Head concatenation and the output -projection happen afterward. A model's use of a familiar head dimension alone -does not establish compatibility with this operation. - -The reference contract above is broader than current kernel coverage. In -particular, nonzero-offset multi-query prefill and the optional batch dimension -are not uniformly supported by the current kernels; an existing prefill path -requires equal query/key lengths and assumes positions begin at zero. Treat -these as current conformance gaps, not alternative definitions of causality. - -The tests in [tests.zig](tests.zig) compare results -against `sdpa` with the gathered causal mask. The original shared cases cover -`D = 64`, MHA and GQA, -single-token decode at full and partial context lengths, batched decode with -different positions, and an eight-query chunk beginning at position 56 in a -64-position KV buffer. Additional conformance cases exercise head dimensions -through `512`, MQA, `bf16`/`f16`/`f32`, initial and offset prefill, partial tiles, -transposed tagged layouts, scalar batch positions, head/batch sharding, open -query/KV partitions, and split decode with empty partitions and cache tails. These -cases are enabled for kernels that support them; their presence does not imply -uniform coverage across every kernel. FP32 results are also checked against an -independent host calculation using FP64 arithmetic. - -Numerical agreement is tolerance-based. The reference evaluates softmax in -`f32`, then converts probabilities to the query dtype before multiplying by -values. Different accumulation orders and rounding points can produce small -differences even when the mathematical operation is identical. The shared tests -use absolute tolerance `5e-3`, relative tolerance `1e-2`, relative epsilon `1e-3`, -and a minimum close fraction of `0.99`; these are test criteria rather than a -universal error bound. diff --git a/zml/kernels/flash_attention/cute/BUILD.bazel b/zml/kernels/flash_attention/cute/BUILD.bazel deleted file mode 100644 index b83a642e..00000000 --- a/zml/kernels/flash_attention/cute/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "cute", - srcs = [ - "attention.zig", - "cute.zig", - ], -) diff --git a/zml/kernels/flash_attention/cute/README.md b/zml/kernels/flash_attention/cute/README.md deleted file mode 100644 index 8c55d2c8..00000000 --- a/zml/kernels/flash_attention/cute/README.md +++ /dev/null @@ -1,145 +0,0 @@ -# CuTe flash attention scheduling - -The [parent README](../README.md) defines attention and sharding semantics. -This document describes decode scheduling in this implementation and the -FlashInfer heuristics that inform it. Scheduling changes parallelism and -intermediate storage, not the attention formula or public arguments. - -## What FlashInfer does - -These references are pinned to FlashInfer commit -`2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a`; FlashInfer has multiple execution paths, -so these rules describe the cited CUDA decode paths rather than every backend. - -[Single-sequence decode](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/decode.cuh#L705-L744) -uses one attention launch when the live sequence has at most 256 tokens or no -scratch buffer is supplied. Otherwise it queries the compiled kernel's resident -blocks per SM and the device's SM count, then computes: - -```text -grid_budget = resident_blocks_per_SM * SM_count -split_budget = floor(grid_budget / KV_heads) -chunk_tokens = max(ceil(live_tokens / split_budget), 256) -splits = ceil(live_tokens / chunk_tokens) -``` - -It launches partial attention over these chunks followed by `MergeStates`. -The merge preserves a single softmax normalization across the chunks. - -[Batched paged decode](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/scheduler.cuh#L75-L98) -uses the per-request lengths to search for a chunk size whose total block count -fits the occupancy budget. Its -[work estimator](https://github.com/flashinfer-ai/flashinfer/blob/2a2cceab07c7bfca0525065fbfdc5fb2cff0c33a/include/flashinfer/attention/scheduler.cuh#L174-L205) -disables splitting if `batch_size * KV_heads` already fills that budget. The -minimum chunk is `max(floor(128 / page_size), 1)` pages. Without CUDA graphs it -also avoids splitting when partitioning adds no requests. Graph execution can -retain a padded split schedule to keep launch and allocation sizes stable. - -## Adaptation to this kernel - -[`decodeSplits`](attention.zig) runs after sharding has established each device's -local dimensions. A block owns at most four query heads belonging to one KV -head, so our unsplit decode block count is: - -```text -G = local_query_heads / local_KV_heads -base_blocks = local_batch * local_KV_heads * ceil(G / 4) -``` - -Counting query-head groups matters: a large GQA group already launches multiple -blocks per KV head in this kernel. Using global batch or head counts would -underestimate the need for splitting after partitioning across devices. - -XLA compiles and owns the CUDA function after this planner runs. We cannot call -CUDA's occupancy calculator on that function here. Instead, the planner uses -PJRT's `core_count` and `shared_memory_per_block_optin` attributes and a -static limit of twelve 128-thread blocks per SM: - -```text -padded_D = next_power_of_two(max(D, 32)) -shared_bytes_per_block = 2 * 16 * padded_D * sizeof(dtype) # K and V tiles -blocks_per_SM = max(1, min(12, floor(shared_budget / shared_bytes_per_block))) -grid_budget = 4 * SM_count * blocks_per_SM -``` - -The per-block shared-memory limit serves as a conservative shared-memory budget; -it is not a measurement of SM capacity or compiled-kernel occupancy. The -twelve-block limit is a scheduling estimate, not a measured occupancy or a -hardware constant. It provides enough resident work to hide latency in the -128-thread kernel; register usage and other hardware limits can reduce actual -occupancy. Four scheduling waves provide additional work for the serial -KV-tile loop; this is a measured adaptation, whereas the cited FlashInfer CUDA -planner targets one resident grid. Larger heads and FP32 reduce the budget through their shared-memory requirements. For -multiple visible devices, the smallest reported SM count and shared-memory -budget are used. Missing or invalid attributes disable splitting. - -The selection rules are: - -1. Prefill (`Lq > 1`) remains unsplit. -2. A cache capacity of at most 64 tokens remains unsplit. -3. If `base_blocks >= grid_budget`, keep one partition per block's head group. -4. Otherwise set `split_budget = ceil(grid_budget / base_blocks)`, - `minimum_chunk = 16` for capacities through 256, otherwise `32`, - `chunk = max(minimum_chunk, ceil(cache_capacity / split_budget))`, and - `splits = ceil(cache_capacity / chunk)`. -5. If the result is one split, launch attention directly into the output. Other - cases allocate FP32 partial states and launch attention followed by a merge. - -The chunk floors are one or two of this kernel's 16-key tiles. They adapt -FlashInfer's minimum-work-per-partition heuristic to our serial per-tile score -loop. Applying its literal 128/256-token floors and a smaller block budget -regressed several measured cases. The workload-based budget and split/merge -structure follow FlashInfer, but these constants are specific to this kernel. -The split budget rounds up a partial wave: rounding 32 partitions down to 31, -for example, can add a whole tile to every block. The grid budget is therefore -a target, and the launched grid may exceed it by less than one split's worth -of `base_blocks`. - -## Capacity and live lengths - -Our graph receives `token_index` on device, potentially with a different value -for each batch element. Planning uses static cache capacity to bound the grid -and scratch allocation without reading positions back to the host. At runtime, -each sequence distributes its **live** prefix of `token_index + 1` keys evenly -across the planned splits. Therefore the actual chunk can be smaller than the -planning floor. Empty partitions contribute zero to the merge. - -This is not FlashInfer's host-side ragged-length search. A short live sequence -in a large cache can still use the split path, and the merge launch remains in -the graph even when only one partition has keys. The schedule does not change -from token to token, so it can be reused during generation and graph capture. - -Partial states contain the weighted-value accumulator, maximum, and exponential -sum in FP32. Their storage is -`4 * local_batch * splits * local_query_heads * (D + 2)` bytes. Reducing splits -as the batch grows reduces both scratch traffic and merge work. Scores are -scaled by `log2(e) / sqrt(D)` so attention and merging use base-2 exponentials. - -## Validation and timing - -The existing attention harness covers GQA/MQA, different positions within a -batch, partial tiles, empty partitions, sharding, three dtypes, and head -dimensions through 512. Planner tests cover the short-cache boundary, decreasing -split counts as batch size grows, resource limits, and extra query-head groups. -Runtime checks include batches through 2048 at the 64/65-token scheduling -boundary; they establish correctness, not throughput at that batch size. - -An opt-in benchmark in [cute.zig](cute.zig) compares the previous 32-split policy -with the planner at batches 1, 4, 16, and 64, and full contexts of 128, 2048, and -8192 tokens. It uses BF16, 16 query heads, four KV heads, and `D = 128` per device. -Sixteen dependent attention calls amortize host overhead; reported microseconds -per attention call are the median of 16 samples after three warmups. This is a -repeated-cache kernel benchmark, not end-to-end model throughput. - -```sh -bazel test //zml:test --//platforms:cuda=true --//platforms:cpu=false \ - --@rules_zig//zig/settings:zigopt=--test-filter \ - --@rules_zig//zig/settings:zigopt=zml.test.zml \ - --@rules_zig//zig/settings:zigopt=--test-filter \ - '--@rules_zig//zig/settings:zigopt=CuTe decode' \ - --test_env=CUDA_VISIBLE_DEVICES=0 \ - --test_env=ZML_CUTE_ATTENTION_BENCHMARK=1 --test_output=all -``` - -`CUTE_DECODE_BENCH` rows contain batch, context length, policy, split count, and -microseconds per call. Timing is not a correctness assertion. diff --git a/zml/kernels/flash_attention/cute/attention.zig b/zml/kernels/flash_attention/cute/attention.zig deleted file mode 100644 index ec0fce16..00000000 --- a/zml/kernels/flash_attention/cute/attention.zig +++ /dev/null @@ -1,818 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); -const llvm = @import("mlir/dialects/llvm"); -const vector = @import("mlir/dialects/vector"); - -const zml = @import("../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const Value = cute.Value; - -pub const Config = struct { - batch: i64, - query_length: i64, - key_length: i64, - query_heads: i64, - kv_heads: i64, - head_dim: i64, - dtype: cute.DType, - splits: i64 = 1, - /// Query j sees the keys up to positions + j. Otherwise every query sees every key and positions is ignored. - causal: bool = true, -}; - -/// Planning uses local (post-sharding) dimensions. XLA owns the compiled CUDA -/// function, so this is an estimated resource budget, not an occupancy query. -pub const DecodeResources = struct { - multiprocessors: i64, - shared_bytes: i64, -}; - -pub fn decodeSplits(q: zml.Shape, k: zml.Shape, resources: DecodeResources) i64 { - const query_length = q.dim(.q); - const key_length = k.dim(.k); - const num_heads = q.dim(.h); - const num_kv_heads = k.dim(.h); - const head_dim = q.dim(.hd); - const batch = if (q.hasTag(.b) != null) q.dim(.b) else 1; - const dtype = q.dtype(); - - if (query_length != 1 or key_length <= 64) return 1; - if (resources.multiprocessors <= 0 or resources.shared_bytes <= 0) return 1; - const group = @divExact(num_heads, num_kv_heads); - const blocks_per_sequence = num_kv_heads * @divTrunc(group + 3, 4); - const base_blocks = batch * blocks_per_sequence; - var padded_dim: i64 = 32; - while (padded_dim < head_dim) padded_dim *= 2; - const bytes_per_element: i64 = if (dtype == .f32) 4 else 2; - const shared_per_block = 2 * tile_keys * padded_dim * bytes_per_element; - const blocks_per_sm = @max(1, @min(12, @divTrunc(resources.shared_bytes, shared_per_block))); - // Multiple scheduling waves keep the serial tile loop supplied with work. - // The measured tradeoff and distinction from CUDA occupancy are in README. - const grid_budget = 4 * resources.multiprocessors * blocks_per_sm; - if (base_blocks >= grid_budget) return 1; - // Round up a partial wave: flooring can leave SMs idle and add a whole - // extra KV tile to every block (e.g. 31 partitions versus 32). - const split_budget = @divTrunc(grid_budget + base_blocks - 1, base_blocks); - // One tile for short caches, two otherwise. FlashInfer's larger chunk - // floor is tuned for a different per-block KV processing rate. - // With static dense shapes, capacity bounds the scratch space and grid; - // each invocation balances its live prefix over these partitions on device. - const min_chunk: i64 = if (key_length <= 256) tile_keys else 2 * tile_keys; - const chunk = @max(min_chunk, @divTrunc(key_length + split_budget - 1, split_budget)); - return @divTrunc(key_length + chunk - 1, chunk); -} - -test "CuTe decode planner respects workload and resource limits" { - const resources: DecodeResources = .{ .multiprocessors = 170, .shared_bytes = 101376 }; - var q = zml.Shape.init(.{ .b = 1, .q = 1, .h = 16, .hd = 128 }, .bf16); - var k = zml.Shape.init(.{ .b = 1, .k = 2048, .h = 4, .hd = 128 }, .bf16); - try std.testing.expectEqual(64, decodeSplits(q, k, resources)); - k = k.setDim(.k, 64); - try std.testing.expectEqual(1, decodeSplits(q, k, resources)); - k = k.setDim(.k, 65); - try std.testing.expectEqual(5, decodeSplits(q, k, resources)); - k = k.setDim(.k, 2048); - var previous: i64 = 64; - for ([_]i64{ 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048 }) |batch| { - q = q.setDim(.b, batch); - k = k.setDim(.b, batch); - const splits = decodeSplits(q, k, resources); - try std.testing.expect(splits <= previous); - try std.testing.expect(splits >= 1); - try std.testing.expect(splits == 1 or (splits - 1) * batch * 4 < 8160); - previous = splits; - } - try std.testing.expectEqual(1, previous); - q = q.setDim(.b, 16); - k = k.setDim(.b, 16); - q = q.setDim(.hd, 512); - k = k.setDim(.hd, 512); - q = q.withDtype(.f32); - k = k.withDtype(.f32); - try std.testing.expectEqual(11, decodeSplits(q, k, resources)); - q = q.setDim(.q, 7); - try std.testing.expectEqual(1, decodeSplits(q, k, resources)); - q = q.setDim(.q, 1); - try std.testing.expectEqual(1, decodeSplits(q, k, .{ .multiprocessors = 0, .shared_bytes = 0 })); - // More query-head groups already create more independent CTAs. - q = q.setDim(.b, 1); - k = k.setDim(.b, 1); - q = q.setDim(.hd, 128); - k = k.setDim(.hd, 128); - q = q.withDtype(.bf16); - k = k.withDtype(.bf16); - q = q.setDim(.h, 1024); - try std.testing.expectEqual(32, decodeSplits(q, k, resources)); -} - -pub const block: [3]i32 = .{ 128, 1, 1 }; -const tile_keys = 16; - -pub const Kernel = cute.Kernel(Config, .{ - .name = "flash_attention_cute", - .inputs = &.{ "q", "k", "v", "positions" }, - .outputs = &.{"out"}, - .run = run, -}); - -fn run(b: *B, cfg: Config) cute.FinishError!void { - std.debug.assert(cfg.head_dim > 0 and cfg.head_dim <= 512); - inline for (.{ 1, 2, 4, 8, 16 }) |fragments| { - if (cfg.head_dim <= fragments * 32) return runSized(b, cfg, fragments); - } - unreachable; -} - -/// Four warps own four (query position, query head) rows sharing one KV head. -/// Only a KV tile lives in shared memory; maxima, sums, and outputs stay in -/// registers. All lanes participate in shuffles, including padded rows/features. -fn runSized(b: *B, cfg: Config, comptime fragments: usize) cute.FinishError!void { - const padded_dim = fragments * 32; - const split = cfg.splits > 1; - const out_shape = if (split) &[_]i64{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 } else &[_]i64{ cfg.batch, cfg.query_length, cfg.query_heads, cfg.head_dim }; - const a = try b.declareArgs(.{ - .q = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.query_length, cfg.query_heads, cfg.head_dim } } }, - .k = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.key_length, cfg.kv_heads, cfg.head_dim } } }, - .v = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.key_length, cfg.kv_heads, cfg.head_dim } } }, - .positions = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.batch} } }, - .out = .{ .tensor = .{ .dtype = if (split) .f32 else cfg.dtype, .shape = out_shape } }, - }); - const tid = b.threadIdx().x; - const lane = tid.rem(32); - const group = @divExact(cfg.query_heads, cfg.kv_heads); - const batch = b.blockIdx().y.div(cfg.kv_heads); - const kv_head = b.blockIdx().y.rem(cfg.kv_heads); - const first_row = b.blockIdx().x.mul(4); - const row = first_row.add(tid.div(32)); - // Padded warps compute a valid dummy row and suppress their final stores. - const query = row.div(group).minimum(cfg.query_length - 1); - const head = kv_head.mul(group).add(row.rem(group)); - const start = a.positions.get(.{batch}); - const position = if (cfg.causal) start.add(query) else b.constant(.i32, cfg.key_length - 1); - const live_end = if (cfg.causal) - start.add(first_row.add(3).div(group).minimum(cfg.query_length - 1)).add(1) - else - b.constant(.i32, cfg.key_length); - // Partition the live cache, rather than its capacity, so decode uses all - // blocks even when only a prefix of the cache is populated. - const chunk = live_end.add(cfg.splits - 1).div(cfg.splits); - const begin = if (split) b.blockIdx().z.mul(chunk) else b.constant(.i32, 0); - const end = if (split) begin.add(chunk).minimum(live_end) else live_end; - const zero = b.constant(.f32, 0); - const neg_inf = b.constant(.f32, -std.math.inf(f32)); - - var queries: [fragments]Value = undefined; - inline for (0..fragments) |f| { - const d = lane.add(f * 32); - var valid = b.openIfElse(d.lt(cfg.head_dim), .{.f32}); - { - valid.yieldThen(.{a.q.get(.{ batch, query, head, d }).to(.f32)}); - } - { - valid.yieldElse(.{zero}); - } - queries[f] = valid.results[0]; - } - - const layout = b.makeLayout(&.{ tile_keys, padded_dim }, .{ .stride = &.{ padded_dim, 1 } }); - const keys = b.makeTensor(b.allocSmem(cfg.dtype, tile_keys * padded_dim, 16), layout); - const values = b.makeTensor(b.allocSmem(cfg.dtype, tile_keys * padded_dim, 16), layout); - var initial: [fragments + 2]Value = @splat(zero); - initial[0] = neg_inf; - var tiles = b.openFor(begin, end, tile_keys, initial); - { - - // Zero-fill partial tiles and feature padding before the CTA-wide barrier. - var copy = b.openFor(tid, tile_keys * padded_dim, 128, .{}); - { - const tile_key = copy.iv.div(padded_dim); - const d = copy.iv.rem(padded_dim); - const key = tiles.iv.add(tile_key); - var valid_load = b.openIfElse(key.lt(end).bitAnd(d.lt(cfg.head_dim)), .{ cfg.dtype, cfg.dtype }); - { - valid_load.yieldThen(.{ a.k.get(.{ batch, key, kv_head, d }), a.v.get(.{ batch, key, kv_head, d }) }); - } - { - valid_load.yieldElse(.{ zero.to(cfg.dtype), zero.to(cfg.dtype) }); - } - keys.set(.{ tile_key, d }, valid_load.results[0]); - values.set(.{ tile_key, d }, valid_load.results[1]); - copy.close(); - } - b.syncThreads(); - - // Each lane keeps one score; dot products reduce across the feature lanes. - var scores = b.openFor(0, tile_keys, 1, .{neg_inf}); - { - var dot = zero; - inline for (0..fragments) |f| { - dot = dot.add(queries[f].mul(keys.get(.{ scores.iv, lane.add(f * 32) }).to(.f32))); - } - inline for (.{ 16, 8, 4, 2, 1 }) |offset| dot = dot.add(b.shuffleXor(dot, offset)); - const scale = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(cfg.head_dim))); - const score = b.select(tiles.iv.add(scores.iv).le(position).bitAnd(tiles.iv.add(scores.iv).lt(end)), dot.mul(scale), neg_inf); - scores.yield(.{b.select(lane.eq(scores.iv), score, scores.carried[0])}); - } - var maximum = scores.results[0]; - inline for (.{ 16, 8, 4, 2, 1 }) |offset| maximum = maximum.maximum(b.shuffleXor(maximum, offset)); - maximum = maximum.maximum(tiles.carried[0]); - const rescale = b.exp2Approx(tiles.carried[0].sub(maximum)); - const probability = b.exp2Approx(scores.results[0].sub(maximum)); - var sum = probability; - inline for (.{ 16, 8, 4, 2, 1 }) |offset| sum = sum.add(b.shuffleXor(sum, offset)); - sum = sum.add(tiles.carried[1].mul(rescale)); - - var accumulators: [fragments]Value = undefined; - inline for (0..fragments) |f| accumulators[f] = tiles.carried[f + 2].mul(rescale); - var weighted = b.openFor(0, tile_keys, 1, accumulators); - { - const weight = b.shuffleIdx(probability, weighted.iv); - inline for (0..fragments) |f| { - accumulators[f] = weighted.carried[f].add(weight.mul(values.get(.{ weighted.iv, lane.add(f * 32) }).to(.f32))); - } - weighted.yield(accumulators); - } - // No warp may overwrite the tile before every warp has consumed it. - b.syncThreads(); - var next: [fragments + 2]Value = undefined; - next[0] = maximum; - next[1] = sum; - inline for (0..fragments) |f| next[f + 2] = weighted.results[f]; - tiles.yield(next); - } - - inline for (0..fragments) |f| { - const col = lane.add(f * 32); - var valid_store = b.openIf(row.lt(cfg.query_length * group).bitAnd(col.lt(cfg.head_dim))); - { - if (split) { - a.out.set(.{ batch, b.blockIdx().z, query, head, col }, tiles.results[f + 2]); - } else { - a.out.set(.{ batch, query, head, col }, tiles.results[f + 2].div(tiles.results[1])); - } - valid_store.close(); - } - } - if (split) { - var stats = b.openIf(row.lt(cfg.query_length * group).bitAnd(lane.eq(0))); - { - a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim }, tiles.results[0]); - a.out.set(.{ batch, b.blockIdx().z, query, head, cfg.head_dim + 1 }, tiles.results[1]); - stats.close(); - } - } -} - -/// Merge unnormalized partial attention states in FP32. Empty partitions have -/// m=-inf, l=0, u=0 and contribute zero, including at the first decode token. -pub const Merge = cute.Kernel(Config, .{ - .name = "flash_attention_cute_merge", - .inputs = &.{"partials"}, - .outputs = &.{"out"}, - .run = merge, -}); - -fn merge(b: *B, cfg: Config) cute.FinishError!void { - const a = try b.declareArgs(.{ - .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ cfg.batch, cfg.splits, cfg.query_heads, cfg.head_dim + 2 } } }, - .out = .{ .tensor = .{ .dtype = cfg.dtype, .shape = &.{ cfg.batch, cfg.query_heads, cfg.head_dim } } }, - }); - const batch = b.blockIdx().y; - const head = b.blockIdx().x; - const lane = b.threadIdx().x.rem(32); - const zero = b.constant(.f32, 0); - const neg_inf = b.constant(.f32, -std.math.inf(f32)); - // Every warp reduces the partition statistics, then independently handles - // a contiguous range of output features. - var maxima = b.openFor(lane, cfg.splits, 32, .{neg_inf}); - { - maxima.yield(.{maxima.carried[0].maximum(a.partials.get(.{ batch, maxima.iv, head, cfg.head_dim }))}); - } - var maximum = maxima.results[0]; - inline for (.{ 16, 8, 4, 2, 1 }) |offset| maximum = maximum.maximum(b.shuffleXor(maximum, offset)); - const weights = b.makeTensor(b.allocSmem(.f32, cfg.splits, 16), b.makeLayout(&.{cfg.splits}, .{})); - var stats = b.openFor(b.threadIdx().x, cfg.splits, 128, .{}); - { - weights.set(.{stats.iv}, b.exp2Approx(a.partials.get(.{ batch, stats.iv, head, cfg.head_dim }).sub(maximum))); - stats.close(); - } - b.syncThreads(); - var features = b.openFor(b.threadIdx().x, cfg.head_dim, 128, .{}); - { - var sum = b.openFor(0, cfg.splits, 1, .{ zero, zero }); - { - const weight = weights.get(.{sum.iv}); - sum.yield(.{ - sum.carried[0].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, features.iv }))), - sum.carried[1].add(weight.mul(a.partials.get(.{ batch, sum.iv, head, cfg.head_dim + 1 }))), - }); - } - a.out.set(.{ batch, head, features.iv }, sum.results[0].div(sum.results[1])); - features.close(); - } -} - -// ==================== Prefill ==================== -// -// FlashAttention-2 forward for `query_length > 1`, with the tiling FA2's sm80 kernel uses on sm_120 -// (`Flash_fwd_kernel_traits<128, 128, 64, 4>`): a 128-thread block owns `prefillRows` queries of one -// head and every warp owns whole rows (all the keys of a tile, the whole head dimension). Scores and -// output stay in `mma.sync` accumulators, so P is reused from registers as the A operand of P @ V, -// without cross-warp reductions. Q, K and V tiles go through swizzled shared memory: V_j is copied -// while S_j = Q K_j^T is computed, K_{j+1} while O += P_j V_j is. -// -// It is a program, not a bare kernel: its host function launches the kernel with the dynamic shared -// memory the kernel allocates (up to 96 KiB). A bare kernel's shared memory is static, which ptxas -// caps at 48 KiB before sm_120. - -pub const Prefill = cute.Program(Config, .{ - .name = "flash_attention_cute_prefill", - .inputs = &.{ "q", "k", "v", "positions" }, - .outputs = &.{"out"}, - .run = prefillProgram, -}); - -const prefill_kernel_name = "flash_attention_cute_prefill_kernel"; - -/// Keys per tile (FA2's kBlockN). -const prefill_keys = 64; - -/// Whether `Prefill` handles `cfg`; the other configurations use `Kernel`. -/// Same as FA2 we only support 16-bit inputs -pub fn prefillSupported(cfg: Config) bool { - const dtype_ok = cfg.dtype == .bf16 or cfg.dtype == .f16; - const head_dim_ok = cfg.head_dim == 32 or cfg.head_dim == 64 or cfg.head_dim == 128 or cfg.head_dim == 256; - return cfg.query_length > 1 and cfg.splits == 1 and dtype_ok and head_dim_ok; -} - -/// m16 row tiles per warp: two up to head dimension 128, one above so the output accumulators fit in registers. -fn prefillRowTiles(head_dim: i64) i64 { - return if (head_dim <= 128) 2 else 1; -} - -/// Queries per block (FA2's kBlockM). -fn prefillRows(cfg: Config) i64 { - return 4 * 16 * prefillRowTiles(cfg.head_dim); -} - -/// Query tiles on x, so the blocks resident at the same time share the K/V of one head in L2. -fn prefillGrid(cfg: Config) [3]i32 { - return .{ @intCast(@divTrunc(cfg.query_length + prefillRows(cfg) - 1, prefillRows(cfg))), @intCast(cfg.query_heads), @intCast(cfg.batch) }; -} - -fn prefillProgram(b: *B, cfg: Config) cute.FinishError!void { - std.debug.assert(prefillSupported(cfg)); - switch (cfg.head_dim) { - inline 32, 64, 128, 256 => |d| switch (cfg.dtype) { - inline .bf16, .f16 => |dt| { - b.beginFunction(prefill_kernel_name, .cuda_kernel); - try prefillSized(b, cfg, d, dt); - b.endFunction(block); - - b.beginFunction(Prefill.name, .host); - const host = try b.declareArgs(.{ - .q = .{ .ptr = dt }, - .k = .{ .ptr = dt }, - .v = .{ .ptr = dt }, - .positions = .{ .ptr = cute.DType.i32 }, - .out = .{ .ptr = dt }, - }); - const grid = prefillGrid(cfg); - const config = b.makeLaunchConfig(.{ - .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, - .block = .{ b.constant(.i32, block[0]), b.constant(.i32, block[1]), b.constant(.i32, block[2]) }, - .dynamic_smem = b.kernelSmemSize(prefill_kernel_name), - .stream = b.cudaStream(), - }); - const launch = b.launchEx(prefill_kernel_name, config, .{ host.q, host.k, host.v, host.positions, host.out }); - b.returnHostStatus(b.cudaResultStatus(launch)); - b.endFunction(null); - }, - else => unreachable, - }, - else => unreachable, - } -} - -/// PTX `max.f32` (FMNMX). `Value.maximum` and `Value.maxnum` both lower to NaN-handling compare and -/// select sequences (three instructions); prefill scores are never NaN, masked ones are -inf. -fn fmax(b: *B, x: Value, y: Value) Value { - return b.callIntrinsic("llvm.nvvm.fmax.f", &.{ x, y }, .float(b.ctx, .f32)).?; -} - -fn unpackStruct(b: *B, comptime n: usize, value: Value, dtype: cute.DType) [n]Value { - var result: [n]Value = undefined; - for (&result, 0..) |*r, i| { - r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); - } - return result; -} - -/// `ldmatrix.x4`: lanes 8i..8i+7 address the rows of matrix i, the result holds the four 8x8 fragments. -fn matrixLoad(b: *B, comptime transpose: bool, address: Value) [4]Value { - const assembly = "ldmatrix.sync.aligned.m8n8.x4" ++ (if (transpose) ".trans" else "") ++ ".shared.b16 {$0, $1, $2, $3}, [$4];"; - const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ - .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.int(b.ctx, .i32)))), - .constraints = "=r,=r,=r,=r,r", - .has_side_effects = true, - }, b.loc())); - return unpackStruct(b, 4, result, .i32); -} - -/// `mma.sync.m16n8k16` with f32 accumulators: acc += a (16x16, row) @ b (16x8, col). -fn mma(b: *B, comptime dtype: cute.DType, a: [4]Value, rhs: [2]Value, acc: [4]Value) [4]Value { - const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ - "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; - const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, rhs[0].inner, rhs[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ - .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), - .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f", - }, b.loc())); - return unpackStruct(b, 4, result, .f32); -} - -/// The four f32 accumulators of one mma.sync, carried across loop iterations as one value (the loop -/// carries at most 64 values). -fn packAcc(b: *B, values: [4]Value) Value { - return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); -} - -fn unpackAcc(b: *B, value: Value) [4]Value { - var result: [4]Value = undefined; - for (&result, 0..) |*r, i| { - r.* = b.vectorExtract(value, @intCast(i), .f32); - } - return result; -} - -/// Comptime layout of one prefill block, shared by the setup and the per-tile steps. -/// -/// Tiles are row major with the 16-byte vector index of every row XORed with row % swizzle_rows, so -/// 16-byte cp.async stores and 8-row ldmatrix reads are free of bank conflicts. Shared memory addresses -/// are folded the way CuTe's static layouts let nvcc fold them: the XORed row bits of every address a -/// thread uses are a per-thread constant, and the k-step part of a column offset is bit-disjoint from -/// the per-thread part, so each ldmatrix / cp.async address is a per-thread base plus a constant -/// (XORed into the 16-byte vector index). -fn PrefillTile(comptime d: i64, comptime dt: cute.DType) type { - return struct { - const row_tiles = prefillRowTiles(d); - const rows = 4 * 16 * row_tiles; - const n = prefill_keys; - const score_tiles = n / 8; - const out_tiles = d / 8; - const row_bytes = d * 2; - /// The XORed vector index must stay in its row: rows of d=32 only have 4 vectors. - const swizzle_rows = @min(8, d / 8); - const q_bytes = rows * row_bytes; - const kv_bytes = n * row_bytes; - /// Rows covered by one 16-byte cp.async per thread of the block. - const copy_rows = @divExact(128, d / 8); - /// Running max and per-thread partial sum of the two rows of every row tile. - const stats = 2 * row_tiles; - /// Carried by the key loop: maxima, sums, then the packed output accumulators of every - /// (row tile, 8 output features). - const carried = 2 * stats + row_tiles * out_tiles; - const scale: f32 = std.math.log2e / std.math.sqrt(@as(f32, @floatFromInt(d))); - - comptime { - // copy_rows rows starting at a multiple of copy_rows never carry into the XORed bits. - std.debug.assert(std.math.isPowerOfTwo(copy_rows)); - std.debug.assert(std.math.isPowerOfTwo(swizzle_rows)); - } - - b: *B, - cfg: Config, - lane: Value, - warp: Value, - first_query: Value, - start: Value, - q_smem: Value, - k_smem: Value, - v_smem: Value, - k_ptr: Value, - v_ptr: Value, - kv_base: Value, - tiles: Value, - /// cp.async: this thread copies the 16-byte vector `copy_col` of rows `copy_row + i * copy_rows`. - copy_row: Value, - copy_col: Value, - /// Byte offset in a tile of this thread's first cp.async destination, without the XORed part. - copy_row_offset: Value, - copy_vector_offset: Value, - /// ldmatrix rows of this thread, as byte offsets, and their XORed 16-byte vector offsets. - q_frag_row: Value, - k_frag_row: Value, - v_frag_row: Value, - qv_frag_vector: Value, - k_frag_vector: Value, - - const Self = @This(); - - fn init(b: *B, cfg: Config, tid: Value, first_query: Value, start: Value, shared: Value, k_ptr: Value, v_ptr: Value, kv_base: Value, tiles: Value) Self { - const lane = tid.rem(32); - const warp = tid.div(32); - const copy_row = tid.div(d / 8); - const copy_col = tid.rem(d / 8); - const lane8 = lane.rem(8); - // The fragment rows a thread reads are lane % 8 plus multiples of 8. - const lane_xor = lane.rem(swizzle_rows); - return .{ - .b = b, - .cfg = cfg, - .lane = lane, - .warp = warp, - .first_query = first_query, - .start = start, - .q_smem = shared, - .k_smem = shared.add(q_bytes), - .v_smem = shared.add(q_bytes + kv_bytes), - .k_ptr = k_ptr, - .v_ptr = v_ptr, - .kv_base = kv_base, - .tiles = tiles, - .copy_row = copy_row, - .copy_col = copy_col, - .copy_row_offset = copy_row.mul(row_bytes), - .copy_vector_offset = copy_col.bitXor(copy_row.rem(swizzle_rows)).mul(16), - // Q (A operand): rows lane % 16, vectors lane / 16 (+ 2 per k-step). - .q_frag_row = shared.add(warp.mul(16 * row_tiles).add(lane.rem(16)).mul(row_bytes)), - // K (B operand): keys lane % 8 + 8 * (lane / 16), vectors (lane / 8) % 2 (+ 2 per k-step). - .k_frag_row = shared.add(q_bytes).add(lane8.add(lane.div(16).mul(8)).mul(row_bytes)), - // V (B operand, transposed): keys lane % 8 + 8 * ((lane / 8) % 2), vectors lane / 16 (+ 2 per 16 features). - .v_frag_row = shared.add(q_bytes + kv_bytes).add(lane8.add(lane.div(8).rem(2).mul(8)).mul(row_bytes)), - .qv_frag_vector = lane.div(16).bitXor(lane_xor).mul(16), - .k_frag_vector = lane.div(8).rem(2).bitXor(lane_xor).mul(16), - }; - } - - /// Destination of the i-th cp.async of this thread in a tile starting at `base`. - fn copyDst(t: Self, base: Value, comptime i: usize) Value { - const xor = ((copy_rows * i) % swizzle_rows) * 16; - const vector_offset = if (xor == 0) t.copy_vector_offset else t.copy_vector_offset.bitXor(xor); - return base.add(t.copy_row_offset).add(vector_offset).add(copy_rows * i * row_bytes); - } - - /// Copy keys `key_base..key_base + n` of `src` into `dst`; keys past the end are zero-filled. - fn loadTile(t: Self, src: Value, dst: Value, key_base: Value) void { - const b = t.b; - const stride = t.cfg.kv_heads * d; - const first_key = key_base.add(t.copy_row); - const safe = src.add(t.kv_base.mul(2)); - const first = safe.add(first_key.to(.i64).mul(stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); - inline for (0..@divExact(n, copy_rows)) |i| { - const valid = first_key.add(copy_rows * i).lt(t.cfg.key_length); - const source = b.select(valid, first.add(copy_rows * i * stride * 2), safe); - b.cpAsync16(t.copyDst(dst, i), source, valid); - } - } - - fn queryOf(t: Self, mi: usize, h: usize) Value { - return t.first_query.add(t.warp.mul(16 * row_tiles)).add(mi * 16).add(t.lane.div(4)).add(h * 8); - } - - fn accIndex(comptime mi: usize, comptime di: usize) usize { - return @intCast(2 * stats + @as(i64, mi) * out_tiles + @as(i64, di)); - } - - /// One key tile. `masked` tiles apply the causal and key length masks; the other ones are - /// fully visible to every query of the block. - fn step(t: Self, comptime masked: bool, tile: Value, carried_in: [carried]Value) [carried]Value { - const b = t.b; - const lane = t.lane; - const zero = b.constant(.f32, 0); - const neg_inf = b.constant(.f32, -std.math.inf(f32)); - const key_base = tile.mul(n); - b.cpAsyncWaitAll(); - b.syncThreads(); - t.loadTile(t.v_ptr, t.v_smem, key_base); - - // S = Q K^T, k-steps of 16 features. - var s: [row_tiles][score_tiles][4]Value = @splat(@splat(@splat(zero))); - inline for (0..d / 16) |ks| { - var qa: [row_tiles][4]Value = undefined; - inline for (0..row_tiles) |mi| { - qa[mi] = matrixLoad(b, false, t.q_frag_row.add(mi * 16 * row_bytes).add(t.qv_frag_vector.bitXor(ks * 32))); - } - const k_vector = t.k_frag_vector.bitXor(ks * 32); - inline for (0..score_tiles / 2) |np| { - const kb = matrixLoad(b, false, t.k_frag_row.add(np * 16 * row_bytes).add(k_vector)); - inline for (0..row_tiles) |mi| { - s[mi][2 * np] = mma(b, dt, qa[mi], .{ kb[0], kb[1] }, s[mi][2 * np]); - s[mi][2 * np + 1] = mma(b, dt, qa[mi], .{ kb[2], kb[3] }, s[mi][2 * np + 1]); - } - } - } - - // Online softmax in base 2 on the raw scores, the scale is folded into the exponent: - // p = exp2(s * scale - m * scale). Thread rows: lane / 4 (accumulators 0, 1) and lane / 4 + 8 (2, 3). - var next: [carried]Value = undefined; - var alpha: [row_tiles][2]Value = undefined; - inline for (0..row_tiles) |mi| { - if (masked) { - inline for (0..score_tiles) |ni| { - inline for (0..4) |j| { - const key = key_base.add(ni * 8).add(lane.rem(4).mul(2)).add(j % 2); - const in_range = key.lt(t.cfg.key_length); - const visible = if (t.cfg.causal) in_range.bitAnd(key.le(t.start.add(t.queryOf(mi, j / 2)))) else in_range; - s[mi][ni][j] = b.select(visible, s[mi][ni][j], neg_inf); - } - } - } - inline for (0..2) |h| { - var row_max = s[mi][0][h * 2]; - inline for (0..score_tiles) |ni| { - inline for (0..2) |j| { - row_max = fmax(b, row_max, s[mi][ni][h * 2 + j]); - } - } - inline for (.{ 2, 1 }) |offset| { - row_max = fmax(b, row_max, b.shuffleXor(row_max, offset)); - } - const old_max = carried_in[mi * 2 + h]; - const new_max = fmax(b, old_max, row_max); - // A row without any visible key so far keeps m = -inf: exponentiate against a finite value. - const neg_scaled_max = fmax(b, new_max, b.constant(.f32, -std.math.floatMax(f32))).mul(-scale); - alpha[mi][h] = b.exp2Approx(old_max.mul(scale).add(neg_scaled_max)); - next[mi * 2 + h] = new_max; - var sum = carried_in[stats + mi * 2 + h].mul(alpha[mi][h]); - inline for (0..score_tiles) |ni| { - inline for (0..2) |j| { - s[mi][ni][h * 2 + j] = b.exp2Approx(s[mi][ni][h * 2 + j].mul(scale).add(neg_scaled_max)); - sum = sum.add(s[mi][ni][h * 2 + j]); - } - } - next[stats + mi * 2 + h] = sum; - } - } - - // All warps are done with K_j once V_j has landed: start copying K_{j+1}. - b.cpAsyncWaitAll(); - b.syncThreads(); - var prefetch = b.openIf(tile.add(1).lt(t.tiles)); - { - t.loadTile(t.k_ptr, t.k_smem, key_base.add(n)); - prefetch.close(); - } - - // O = O * alpha + P V. The accumulators of two adjacent score tiles are the A operand of one k-step. - var o: [row_tiles][out_tiles][4]Value = undefined; - inline for (0..row_tiles) |mi| { - inline for (0..out_tiles) |di| { - const prev = unpackAcc(b, carried_in[accIndex(mi, di)]); - inline for (0..4) |j| { - o[mi][di][j] = prev[j].mul(alpha[mi][j / 2]); - } - } - } - inline for (0..n / 16) |ks| { - var pa: [row_tiles][4]Value = undefined; - inline for (0..row_tiles) |mi| { - pa[mi] = .{ - b.packHalf2(s[mi][2 * ks][0], s[mi][2 * ks][1], dt), - b.packHalf2(s[mi][2 * ks][2], s[mi][2 * ks][3], dt), - b.packHalf2(s[mi][2 * ks + 1][0], s[mi][2 * ks + 1][1], dt), - b.packHalf2(s[mi][2 * ks + 1][2], s[mi][2 * ks + 1][3], dt), - }; - } - const v_row = t.v_frag_row.add(ks * 16 * row_bytes); - inline for (0..out_tiles / 2) |dp| { - const vb = matrixLoad(b, true, v_row.add(t.qv_frag_vector.bitXor(dp * 32))); - inline for (0..row_tiles) |mi| { - o[mi][2 * dp] = mma(b, dt, pa[mi], .{ vb[0], vb[1] }, o[mi][2 * dp]); - o[mi][2 * dp + 1] = mma(b, dt, pa[mi], .{ vb[2], vb[3] }, o[mi][2 * dp + 1]); - } - } - } - inline for (0..row_tiles) |mi| { - inline for (0..out_tiles) |di| { - next[accIndex(mi, di)] = packAcc(b, o[mi][di]); - } - } - return next; - } - }; -} - -fn prefillSized(b: *B, cfg: Config, comptime d: i64, comptime dt: cute.DType) cute.FinishError!void { - const T = PrefillTile(d, dt); - const a = try b.declareArgs(.{ - .q = .{ .ptr = dt }, - .k = .{ .ptr = dt }, - .v = .{ .ptr = dt }, - .positions = .{ .ptr = cute.DType.i32 }, - .out = .{ .ptr = dt }, - }); - b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); - - const tid = b.threadIdx().x; - const first_query = b.blockIdx().x.mul(T.rows); - const head = b.blockIdx().y; - const batch = b.blockIdx().z; - const kv_head = head.div(@divExact(cfg.query_heads, cfg.kv_heads)); - // One shared storage object (like a Python CuTe `SharedStorage`): the host compiler sizes the - // launch's dynamic shared memory from it, which `kernelSmemSize` returns. `allocSmem` and - // `allocSmemView` both become static shared memory instead. - b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 1))); - const fields = [_][]const u8{ - std.fmt.comptimePrint("sQ:{d}:0", .{T.q_bytes}), - std.fmt.comptimePrint("sK:{d}:{d}", .{ T.kv_bytes, T.q_bytes }), - std.fmt.comptimePrint("sV:{d}:{d}", .{ T.kv_bytes, T.q_bytes + T.kv_bytes }), - }; - const smem = b.allocSmemStorage(.i8, 128, b.layoutType(b.layoutSpec(.{T.q_bytes + 2 * T.kv_bytes}, .{1})), 0, &fields); - const shared = b.ptrToInt(b.getIter(smem, b.ptrTy(.i8, .smem, 128) catch @panic("bad shared memory pointer")).value(), .i32); - const q_ptr = b.ptrToInt(a.q, .i64); - const out_ptr = b.ptrToInt(a.out, .i64); - const start = b.loadWords(1, b.ptrToInt(a.positions, .i64).add(batch.to(.i64).mul(4)), .global, .{})[0]; - - // Element offsets of this (batch, head) in q/out and (batch, kv head) in k/v, and the q/out row stride. - const q_stride = cfg.query_heads * d; - const q_base = batch.to(.i64).mul(cfg.query_length * q_stride).add(head.to(.i64).mul(d)); - const kv_base = batch.to(.i64).mul(cfg.key_length * cfg.kv_heads * d).add(kv_head.to(.i64).mul(d)); - - // Causal: the last query of the block sees keys up to start + last_query, and every query of the - // block sees the keys up to start + first_query. Otherwise only the last tile, past the end of the - // keys, is masked. - const last_query = first_query.add(T.rows).minimum(cfg.query_length).sub(1); - const tiles = if (cfg.causal) - start.add(last_query).add(1).minimum(cfg.key_length).add(T.n - 1).div(T.n) - else - b.constant(.i32, @divTrunc(cfg.key_length + T.n - 1, T.n)); - const full_tiles = if (cfg.causal) - start.add(first_query).add(1).minimum(cfg.key_length).div(T.n).minimum(tiles) - else - b.constant(.i32, @divTrunc(cfg.key_length, T.n)); - - const t: T = .init(b, cfg, tid, first_query, start, shared, b.ptrToInt(a.k, .i64), b.ptrToInt(a.v, .i64), kv_base, tiles); - - // Out of range rows are zero-filled and never read from global memory. - { - const first_row = first_query.add(t.copy_row); - const safe = q_ptr.add(q_base.mul(2)); - const first = safe.add(first_row.to(.i64).mul(q_stride).add(t.copy_col.mul(8).to(.i64)).mul(2)); - inline for (0..@divExact(T.rows, T.copy_rows)) |i| { - const valid = first_row.add(T.copy_rows * i).lt(cfg.query_length); - b.cpAsync16(t.copyDst(t.q_smem, i), b.select(valid, first.add(T.copy_rows * i * q_stride * 2), safe), valid); - } - } - t.loadTile(t.k_ptr, t.k_smem, b.constant(.i32, 0)); - - var initial: [T.carried]Value = undefined; - const zero = b.constant(.f32, 0); - for (initial[0..T.stats]) |*v| { - v.* = b.constant(.f32, -std.math.inf(f32)); - } - for (initial[T.stats .. 2 * T.stats]) |*v| { - v.* = zero; - } - for (initial[2 * T.stats ..]) |*v| { - v.* = packAcc(b, .{ zero, zero, zero, zero }); - } - - var full = b.openFor(0, full_tiles, 1, initial); - { - full.yield(t.step(false, full.iv, full.carried[0..T.carried].*)); - } - var diagonal = b.openFor(full_tiles, tiles, 1, full.results[0..T.carried].*); - { - diagonal.yield(t.step(true, diagonal.iv, diagonal.carried[0..T.carried].*)); - } - const results = diagonal.results[0..T.carried].*; - - // Normalize by the row sums (reduced over the quad) and store pairs of features as one 32-bit word. - inline for (0..T.row_tiles) |mi| { - inline for (0..2) |h| { - var sum = results[T.stats + mi * 2 + h]; - inline for (.{ 2, 1 }) |offset| { - sum = sum.add(b.shuffleXor(sum, offset)); - } - const norm = b.select(sum.gt(0), b.rcpApprox(sum), zero); - const query = t.queryOf(mi, h); - var valid = b.openIf(query.lt(cfg.query_length)); - { - const row_offset = q_base.add(query.to(.i64).mul(q_stride)).add(t.lane.rem(4).mul(2).to(.i64)); - inline for (0..T.out_tiles) |di| { - const acc = unpackAcc(b, results[T.accIndex(mi, di)]); - const word = b.packHalf2(acc[h * 2].mul(norm), acc[h * 2 + 1].mul(norm), dt); - b.storeWords(1, out_ptr.add(row_offset.add(di * 8).mul(2)), .global, .{word}); - } - valid.close(); - } - } - } -} - -test "CuTe attention emits tiled online softmax through head dimension 512" { - for ([_]i64{ 64, 80, 128, 256, 512 }) |head_dim| { - const ir = try Kernel.emit(std.testing.allocator, .{ - .batch = 2, - .query_length = 7, - .key_length = 37, - .query_heads = 6, - .kv_heads = 2, - .head_dim = head_dim, - .dtype = .bf16, - }, block); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "scf.for") != null); - } -} diff --git a/zml/kernels/flash_attention/cute/cute.zig b/zml/kernels/flash_attention/cute/cute.zig deleted file mode 100644 index 36e0953b..00000000 --- a/zml/kernels/flash_attention/cute/cute.zig +++ /dev/null @@ -1,96 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const kernel = @import("attention.zig"); - -test { - _ = kernel; -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - const cc = zml.platform.cuda.computeCapability(platform) orelse return false; - return cc.major >= 8; -} - -/// `causal`: query j sees the keys up to token_index + j. Otherwise every key, and token_index is ignored. -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, causal: bool) zml.Tensor { - stdx.debug.assert(q.shape().hasTags(.{ .q, .h, .hd }) and k.shape().hasTags(.{ .k, .h, .hd }) and v.shape().hasTags(.{ .k, .h, .hd }), "CuTe attention requires q/h/hd and k/h/hd tags", .{}); - const batched = q.shape().hasTag(.b) != null; - const rank: usize = if (batched) 4 else 3; - stdx.debug.assert(q.rank() == rank and k.rank() == rank and v.rank() == rank, "CuTe attention expects rank-3 tensors or rank-4 tensors with a batch axis", .{}); - stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype(), "CuTe attention requires matching Q/K/V dtypes", .{}); - stdx.debug.assert(q.dtype() == .bf16 or q.dtype() == .f16 or q.dtype() == .f32, "CuTe attention supports bf16, f16, and f32", .{}); - stdx.debug.assert(q.dim(.hd) > 0 and q.dim(.hd) <= 512 and q.dim(.hd) == k.dim(.hd) and k.dim(.hd) == v.dim(.hd), "CuTe attention requires matching head dimensions in [1, 512]", .{}); - stdx.debug.assert(k.dim(.h) > 0 and q.dim(.h) > 0 and @rem(q.dim(.h), k.dim(.h)) == 0 and k.dim(.h) == v.dim(.h), "CuTe attention requires Hq divisible by Hkv and matching K/V heads", .{}); - stdx.debug.assert(q.dim(.q) > 0 and (!causal or q.dim(.q) <= k.dim(.k)) and k.dim(.k) == v.dim(.k), "CuTe attention requires 0 < Lq, Lq <= Lk when causal, and matching K/V lengths", .{}); - stdx.debug.assert(token_index.dtype().isInteger(), "CuTe attention requires integer token positions", .{}); - if (batched) { - stdx.debug.assert(k.shape().hasTag(.b) != null and v.shape().hasTag(.b) != null and q.dim(.b) == k.dim(.b) and q.dim(.b) == v.dim(.b), "CuTe attention requires matching batch dimensions", .{}); - stdx.debug.assert(token_index.rank() == 0 or (token_index.rank() == 1 and token_index.shape().hasTag(.b) != null and token_index.dim(.b) == q.dim(.b)), "CuTe attention requires a scalar position or one position per batch element", .{}); - } else { - stdx.debug.assert(token_index.rank() == 0, "Unbatched CuTe attention requires a scalar token position", .{}); - } - - const positions = if (batched) - token_index.convert(.i32).broad(zml.Shape.init(.{ .b = q.dim(.b) }, .i32)) - else - token_index.convert(.i32); - const q_ = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }); - const k_ = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }); - const v_ = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }); - - const splits = kernel.decodeSplits(q_.shape(), k_.shape(), decodeResources(zml.Compiler.current().platform)); - - const cfg: kernel.Config = .{ - .batch = if (batched) q_.dim(.b) else 1, - .query_length = q_.dim(.q), - .key_length = k_.dim(.k), - .query_heads = q_.dim(.h), - .kv_heads = k_.dim(.h), - .head_dim = q_.dim(.hd), - .dtype = zml.kernel.cute.from(q_.dtype()), - .splits = splits, - .causal = causal, - }; - - const out = if (kernel.prefillSupported(cfg)) - kernel.Prefill.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ .cfg = cfg }).out - else if (cfg.splits > 1) b: { - const partials = kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ - .out = zml.Shape.init(.{ cfg.batch, cfg.splits, cfg.query_length, cfg.query_heads, cfg.head_dim + 2 }, .f32), - }, .{ - .cfg = cfg, - .grid = .{ @intCast(@divTrunc(@divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), @intCast(cfg.splits) }, - .block = kernel.block, - }).out; - break :b kernel.Merge.call(.{ .partials = partials }, .{ .out = q_.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.query_heads), @intCast(cfg.batch), 1 }, - .block = kernel.block, - }).out; - } else kernel.Kernel.call(.{ .q = q_, .k = k_, .v = v_, .positions = positions }, .{ .out = q_.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(@divTrunc(cfg.query_length * @divExact(cfg.query_heads, cfg.kv_heads) + 3, 4)), @intCast(cfg.kv_heads * cfg.batch), 1 }, - .block = kernel.block, - }).out; - - return out.transpose(q.shape()); -} - -/// Returns the minimum number of multiprocessors and shared memory per block across all devices in the platform. -/// If there are no devices, returns 0 for both values. -/// If any device does not have the required attributes, returns 0 for both values. -fn decodeResources(platform: *const zml.Platform) kernel.DecodeResources { - var result: kernel.DecodeResources = .{ .multiprocessors = std.math.maxInt(i64), .shared_bytes = std.math.maxInt(i64) }; - if (platform.devices.len == 0) return .{ .multiprocessors = 0, .shared_bytes = 0 }; - for (platform.devices) |device| { - const sms = device.pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return .{ .multiprocessors = 0, .shared_bytes = 0 }; - const shared = device.pjrt_desc.attribute(platform.pjrt_api, "shared_memory_per_block_optin") orelse return .{ .multiprocessors = 0, .shared_bytes = 0 }; - result.multiprocessors = @min(result.multiprocessors, sms.int64); - result.shared_bytes = @min(result.shared_bytes, shared.int64); - } - return result; -} diff --git a/zml/kernels/flash_attention/flash_attention.zig b/zml/kernels/flash_attention/flash_attention.zig deleted file mode 100644 index 3fb898db..00000000 --- a/zml/kernels/flash_attention/flash_attention.zig +++ /dev/null @@ -1,139 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -pub const cute = @import("cute/cute.zig"); -pub const metal = @import("metal/metal.zig"); -pub const nki = @import("nki/nki.zig"); -const zml = @import("../../zml.zig"); -const Options = @import("../../attention/Options.zig"); - -test { - std.testing.refAllDecls(@This()); - _ = @import("tests.zig"); -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - return switch (platform.target) { - .neuron => true, - .metal => true, - .cuda => if (cute.isAvailable(platform)) true else false, - .cpu, .rocm, .tpu, .oneapi, .furiosa => false, - }; -} - -pub const Backend = enum { - cute, - nki, - metal, - - pub fn auto(platform: *const zml.Platform) Backend { - return switch (platform.target) { - .neuron => .nki, - .metal => .metal, - .cuda => .cute, - .cpu, .rocm, .tpu, .oneapi, .furiosa => stdx.debug.panic("Flash attention is not available on the {} platform", .{platform.target}), - }; - } - - pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { - return switch (backend) { - .cute => cute.isAvailable(platform), - .nki => platform.target == .neuron, - .metal => platform.target == .metal, - }; - } -}; - -pub const Parameters = union(Backend) { - cute: void, - nki: nki.Parameters, - metal: void, - - pub const InitOptions = union(Backend) { - cute: void, - nki: nki.Parameters, - metal: void, - - pub fn fromBackend(backend: Backend) InitOptions { - return switch (backend) { - .cute => .{ .cute = {} }, - .nki => .{ .nki = .init() }, - .metal => .{ .metal = {} }, - }; - } - }; - - pub fn init(opts: InitOptions) Parameters { - return switch (opts) { - .cute => .{ .cute = {} }, - .nki => |v| .{ .nki = v }, - .metal => .{ .metal = {} }, - }; - } -}; - -pub const Metadata = union(Backend) { - cute: void, - nki: void, - metal: metal.Metadata, - - pub const InitOptions = union(Backend) { - cute: void, - nki: void, - metal: void, - - pub fn fromBackend(backend: Backend) InitOptions { - return switch (backend) { - .cute => .{ .cute = {} }, - .nki => .{ .nki = {} }, - .metal => .{ .metal = {} }, - }; - } - }; - - pub fn init(opts: InitOptions) Metadata { - return switch (opts) { - .cute => .{ .cute = {} }, - .nki => .{ .nki = {} }, - .metal => .{ .metal = .init() }, - }; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - return switch (self) { - .cute => .{ .cute = {} }, - .nki => .{ .nki = {} }, - inline else => |v, tag| @unionInit(zml.Bufferized(Metadata), @tagName(tag), try v.initBuffer(io, platform)), - }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { - switch (self.*) { - .cute => {}, - .nki => {}, - .metal => |*v| metal.Metadata.deinitBuffer(v), - } - } -}; - -/// Causal attention as used in transformers. -/// -/// **Shapes**: -/// - q, result: .{ .q, .h, .hd } -/// - k, v: .{ .k, .h, .hd } -/// -/// Where: -/// - .h is the number of head -/// - .q is the number of queries -/// - .k is the number of keys -/// - .hd is the head dimension -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata, parameters: Parameters, opts: Options) zml.Tensor { - zml.stdx.debug.assert(zml.Compiler.current().manual_computation_depth > 0, "Attention kernels require an explicit manualComputation boundary", .{}); - return switch (parameters) { - .cute => cute.attention(q, k, v, token_index, opts.causal), - // The NKI and Metal kernels are causal only. - .nki => if (opts.causal) nki.attention(q, k, v, token_index, parameters.nki) else zml.nn.sdpa(q, k, v, .{}), - .metal => if (opts.causal) metal.attention(q, k, v, token_index, metadata.metal) else zml.nn.sdpa(q, k, v, .{}), - }; -} diff --git a/zml/kernels/flash_attention/metal/BUILD.bazel b/zml/kernels/flash_attention/metal/BUILD.bazel deleted file mode 100644 index f82ade45..00000000 --- a/zml/kernels/flash_attention/metal/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "metal", - srcs = [ - "flash_attn.zig", - "metal.zig", - ], -) diff --git a/zml/kernels/flash_attention/metal/flash_attn.zig b/zml/kernels/flash_attention/metal/flash_attn.zig deleted file mode 100644 index de4b68da..00000000 --- a/zml/kernels/flash_attention/metal/flash_attn.zig +++ /dev/null @@ -1,732 +0,0 @@ -//! Metal flash attention over contiguous `[batch, keys, kv_heads, head_dim]` -//! K/V, causal from `token_index`: fa_vec and fa_vec_reduce for decode, -//! fa_tiled otherwise. - -const std = @import("std"); - -const metal = @import("../../../kernel/kernel.zig").metal; - -const ArgSpec = metal.ArgSpec; -const Builder = metal.Builder; -const Type = metal.Type; -const Value = metal.Value; - -const flt_max = std.math.floatMax(f32); -/// A masked score: below any real one yet finite, since fast math assumes -/// no infinities. Halved so subtracting a running maximum cannot overflow. -const masked_score = -flt_max / 2.0; -const max_half = 65504.0; -const simd_width = 32; - -/// fa_vec: bf16 decode, one query per batch row. Simdgroup s takes query head -/// s % heads of the threadgroup's KV head group and position split s / heads. -pub const FaVec = struct { - pub const Cfg = struct { - head_dim: i32, - heads: i32, - splits: i32, - }; - - pub const Args: metal.Struct = .{ .name = "fa_vec_args", .fields = &.{ - .{ .name = "num_heads", .type = .int }, - .{ .name = "n_kv", .type = .int }, - .{ .name = "k_len", .type = .int }, - .{ .name = "scale", .type = .float }, - } }; - - pub const HostArgs = extern struct { - num_heads: i32, - n_kv: i32, - k_len: i32, - scale: f32, - }; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "fa_vec", - .inputs = &.{ "q", "k", "v", "token_index" }, - .outputs = &.{"dst"}, - .constants = &.{"args"}, - .run = run, - }); - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - switch (cfg.head_dim) { - 32, 64, 128, 256, 512 => {}, - else => std.debug.panic("fa_vec: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), - } - const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; - const a = try b.declareArgs(.{ - .args = .{ .constant_struct = Args }, - .q = char_in, - .k = char_in, - .v = char_in, - .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, - .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, - .tgpg = .{ .threadgroups_per_grid = .uint3 }, - .tiisg = .{ .thread_index_in_simdgroup = .ushort }, - .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, - .nsg = .{ .simdgroups_per_threadgroup = .ushort }, - }); - impl(b, cfg, a); - } - - fn impl(b: *Builder, cfg: Cfg, a: anytype) void { - const D = cfg.head_dim; - const D4 = @divExact(D, 4); - // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. - const NL = @min(D4, simd_width); - const R = @divExact(simd_width, NL); - const NI = @divExact(D4, NL); - const C = simd_width; - const HG = cfg.heads; - const NWG = cfg.splits; - const full: metal.LoopOpts = .{ .unroll = .full }; - - const num_heads = a.args.field("num_heads"); - const n_kv = a.args.field("n_kv"); - const group = num_heads.div(n_kv); - const groups = group.div(HG); - const r = a.tgpig.at(2).to(.int); - const tg_head = a.tgpig.at(1).to(.int); - const ikv = tg_head.div(groups); - const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); - const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); - const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); - const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); - - const kv_len = a.token_index.load(r).add(1); - const live = if (NWG > 1) liveSplits(b, kv_len, NWG) else b.lift(@as(i32, 1)); - - if (NWG > 1) { - var idle = b.openIf(iwg.ge(live)); - { - b.return_(); - idle.close(); - } - } - - const position_bytes = n_kv.to(.ulong).mul(D * 2); - const first = r.to(.ulong).mul(a.args.field("k_len").to(.ulong)).mul(position_bytes).add(ikv.to(.ulong).mul(D * 2)); - const k0 = a.k.add(first); - const v0 = a.v.add(first); - - const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); - const sub = a.tiisg.div(NL).to(.int); - const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); - const qr = b.array(.float4, @intCast(NI)); - const o = b.array(.float4, @intCast(NI)); - var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = init.iv; - qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); - o.store(ii, b.constant(.float4, 0)); - init.close(); - } - - const S = b.local(.float, 0.0); - const M = b.local(.float, -flt_max / 2.0); - - const ic0 = b.local(.int, iwg); - var blocks = b.openLoop(.{}); - { - const ic = ic0.mul(C); - var past = b.openIf(ic.ge(kv_len)); - { - blocks.breakLoop(); - past.close(); - } - - const s = b.local(.float, 0.0); - var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); - { - const cc = qk.iv; - const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); - const pk4 = k0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); - const d = b.local(.float, 0.0); - var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = dots.iv; - d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); - dots.close(); - } - if (R == 1) { - s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); - } else { - var x = d.rvalue(); - var mask: i32 = 1; - while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); - const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); - s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); - } - qk.close(); - } - - const pos = ic.add(a.tiisg.to(.int)); - const score = b.ternary(pos.lt(kv_len), s.mul(a.args.field("scale")), b.constant(.float, -max_half)); - - const m = M.rvalue(); - M.assign(b.simdMax(b.max(M.rvalue(), score))); - const ms = b.exp(m.sub(M)); - const vs = b.exp(score.sub(M)); - S.assign(S.mul(ms).add(b.simdSum(vs))); - - var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - o.store(rescale.iv, o.load(rescale.iv).mul(ms)); - rescale.close(); - } - - var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); - { - const cc = pv.iv; - const row = cc.to(.int).mul(R).add(sub); - const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); - const pv4 = v0.add(p.to(.ulong).mul(position_bytes)).to(.bfloat4).add(lane); - const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); - var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = dims.iv; - o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); - dims.close(); - } - pv.close(); - } - - blocks.next(); - ic0.assign(ic0.add(live)); - blocks.close(); - } - - var mask: i32 = NL; - while (mask < simd_width) : (mask *= 2) { - var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); - sum.close(); - } - } - var first_row = if (R > 1) b.openIf(sub.eq(0)) else null; - if (NWG == 1) { - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); - out.close(); - } - } else { - // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). - const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); - dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); - out.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); - dst1.store(b.lift(2).mul(iwg), S); - dst1.store(b.lift(2).mul(iwg).add(1), M); - lane0.close(); - } - } - if (first_row) |*scope| scope.close(); - } -}; - -fn liveSplits(b: *Builder, kv_len: Value, splits: i32) Value { - return b.min(splits, kv_len.add(simd_width - 1).div(simd_width)); -} - -/// fa_vec_reduce: merges the fa_vec split partials. -pub const FaVecReduce = struct { - pub const Cfg = struct { - splits: i32, - head_dim: i32, - }; - - pub const simdgroups = 8; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "fa_vec_reduce", - .inputs = &.{ "htmp", "token_index" }, - .outputs = &.{"dst"}, - .constants = &.{"args"}, - .run = run, - }); - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - const a = try b.declareArgs(.{ - .args = .{ .constant_struct = FaVec.Args }, - .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, - .token_index = .{ .buffer = .{ .type = .int, .const_ = true } }, - .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, - .tgpg = .{ .threadgroups_per_grid = .uint2 }, - .tiisg = .{ .thread_index_in_simdgroup = .ushort }, - .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, - }); - const NWG = cfg.splits; - if (NWG > 4 * simd_width) std.debug.panic("fa_vec_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); - const NP = @divFloor(NWG + simd_width - 1, simd_width); - const DV = cfg.head_dim; - const DV4 = @divExact(DV, 4); - const full: metal.LoopOpts = .{ .unroll = .full }; - - const rid = a.tgpig.at(0).to(.ulong); - const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); - const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); - const live = liveSplits(b, a.token_index.load(r).add(1), NWG); - - const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); - const Mj = b.array(.float, @intCast(NP)); - const Sj = b.array(.float, @intCast(NP)); - const mx = b.local(.float, -flt_max / 2.0); - var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = load.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - const ok = idx.lt(live); - Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); - Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); - mx.assign(b.max(mx.rvalue(), Mj.load(j))); - load.close(); - } - const m = b.simdMax(mx.rvalue()); - const sum = b.local(.float, 0.0); - var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = weigh.iv; - Mj.store(j, b.exp(Mj.load(j).sub(m))); - sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); - weigh.close(); - } - const S = b.simdSum(sum.rvalue()); - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - - const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); - const acc = b.local(.float4, b.constant(.float4, 0)); - var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = parts.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - var ok = b.openIf(idx.lt(live)); - { - acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); - ok.close(); - } - parts.close(); - } - const v = b.simdSum(acc.rvalue()); - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst4.store(i, v.mul(inv_sum).to(.bfloat4)); - lane0.close(); - } - } -}; - -/// fa_tiled: causal FlashAttention-2 over simdgroup matrices. Queries past -/// `live` output zeros. -pub const FaTiled = struct { - pub const Cfg = struct { - dtype: Type, - head_size: i32, - bq: i32 = 32, - tile_kv: i32 = 32, - num_threads: i32 = 128, - }; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "fa_tiled", - .inputs = &.{ "q", "k", "v", "token_index", "live" }, - .outputs = &.{"out"}, - .constants = &.{ "num_kv_heads", "scale", "q_len", "k_len" }, - .run = run, - }); - - fn supported(cfg: Cfg) bool { - const tile = tileConfig(cfg.head_size) orelse return false; - return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and - cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; - } - - pub fn tileConfig(head_size: i64) ?[3]i32 { - return switch (head_size) { - 32, 64, 96, 128 => .{ 32, 32, 128 }, - 256 => .{ 16, 16, 64 }, - 512 => .{ 8, 8, 32 }, - else => null, - }; - } - - pub fn threadgroupMemory(cfg: Cfg) u64 { - const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); - return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); - } - - /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. - fn fragCoord(b: *Builder, lane_id: Value) Value { - const qid = lane_id.to(.short).div(4).to(.short); - const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); - const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); - return b.vec(.short2, .{ fn_, fm }); - } - - fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { - var x = v.rvalue(); - inline for (.{ 1, 8 }) |mask| { - const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); - x = switch (op) { - .max => b.max(x, y), - .sum => x.add(y), - }; - } - return x; - } - - fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { - p.store(i, p.load(i).insert(lane, x)); - } - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - if (!supported(cfg)) std.debug.panic("fa_tiled: {s} at head size {d} and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); - const T = cfg.dtype; - const HEAD_SIZE = cfg.head_size; - const BQ = cfg.bq; - const TILE_KV = cfg.tile_kv; - const NUM_THREADS = cfg.num_threads; - const int_arg: ArgSpec = .{ .constant = .int }; - const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; - const a = try b.declareArgs(.{ - .out = .{ .buffer = .{ .type = T } }, - .q = .{ .buffer = .{ .type = T, .const_ = true } }, - .k = .{ .buffer = .{ .type = T, .const_ = true } }, - .v = .{ .buffer = .{ .type = T, .const_ = true } }, - .token_index = int_in, - .live = int_in, - .num_kv_heads = int_arg, - .scale = .{ .constant = .float }, - .q_len = int_arg, - .k_len = int_arg, - .shared_mem = .{ .threadgroup = .char }, - .tgp = .{ .threadgroup_position_in_grid = .uint3 }, - .tgpg = .{ .threadgroups_per_grid = .uint3 }, - .tpt = .{ .thread_position_in_threadgroup = .uint3 }, - .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, - .lane = .{ .thread_index_in_simdgroup = .uint }, - }); - - const NUM_SIMD_LANES = 32; - const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); - const TD = @divExact(HEAD_SIZE, 8); - const TK = @divExact(TILE_KV, 8); - if (@divExact(BQ, NUM_SG) != 8) @panic("fa_tiled: ROWS_PER_SG must be 8"); - const unroll: metal.LoopOpts = .{ .unroll = .enable }; - - const thread_idx = a.tpt.x().to(.int); - const head_idx = a.tgp.x().to(.int); - const q_pos_start = a.tgp.y().to(.int).mul(BQ); - const batch = a.tgp.z().to(.int); - const num_heads = a.tgpg.x().to(.int); - const kv_head_idx = head_idx.div(num_heads.div(a.num_kv_heads)); - const q_stride = num_heads.mul(HEAD_SIZE); - const kv_token_stride = a.num_kv_heads.mul(HEAD_SIZE); - - const block_rows = b.min(BQ, a.q_len.sub(q_pos_start)); - const live = a.live.load(0); - const valid_q = b.min(block_rows, b.max(0, live.sub(q_pos_start))); - const out_base = a.out.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); - - var dead = b.openIf(valid_q.le(0)); - { - var zero = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); - { - const i = zero.iv; - out_base.store(i.div(HEAD_SIZE).mul(q_stride).add(i.rem(HEAD_SIZE)), b.constant(T, 0)); - zero.close(); - } - b.return_(); - dead.close(); - } - - // Query i of the batch row is at position context_len + i. - const context_len = a.token_index.load(batch); - const seq_len = context_len.add(live); - const kv_base = batch.to(.long).mul(a.k_len).mul(kv_token_stride).add(kv_head_idx.mul(HEAD_SIZE)); - - const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); - const LD = HEAD_SIZE + SMEM_PAD; - const Q_ELEMS = BQ * LD; - const KV_ELEMS = TILE_KV * LD; - - const Q_smem = a.shared_mem.to(T); - const K_smem = Q_smem.add(Q_ELEMS); - const V_smem = K_smem.add(KV_ELEMS); - - const log2e: f32 = std.math.log2e; - const scale_log2 = a.scale.mul(log2e); - - const q_base = a.q.add(batch.mul(a.q_len).add(q_pos_start).to(.long).mul(q_stride).add(head_idx.mul(HEAD_SIZE))); - var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); - { - const i = load_q.iv; - const r = i.div(HEAD_SIZE); - const d = i.rem(HEAD_SIZE); - var value = b.openIfElse(r.lt(valid_q), .{T}); - { - value.yieldThen(.{q_base.load(r.mul(q_stride).add(d))}); - } - { - value.yieldElse(.{b.constant(T, 0)}); - } - Q_smem.store(r.mul(LD).add(d), value.results[0]); - load_q.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const T2: Type = .vec(T.scalar, 2); - const q_reg = b.array(T2, @intCast(TD)); - var load_q_reg = b.openForOpts(0, TD, 1, unroll); - { - const d = load_q_reg.iv; - const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - q_reg.store(d, tmp.threadElements()); - load_q_reg.close(); - } - - const fc = fragCoord(b, a.lane.to(.ushort)); - const fm = fc.y(); - const fn_ = fc.x(); - - const max_score = b.local(.float, masked_score); - const sum_score = b.local(.float, 0.0); - - const o_reg = b.array(.float2, @intCast(TD)); - var zero_o = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(zero_o.iv, b.constant(.float2, 0)); - zero_o.close(); - } - - const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); - const last_q_pos = context_len.add(q_pos_start).add(valid_q).sub(1); - - var tiles = b.openFor(0, num_kv_tiles, 1); - { - const tile_idx = tiles.iv; - const tile_start = tile_idx.mul(TILE_KV); - - var past = b.openIf(tile_start.gt(last_q_pos)); - { - tiles.breakLoop(); - past.close(); - } - - const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; - const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); - const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; - std.debug.assert(@rem(LD, VEC) == 0); - var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); - { - const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); - const kv_pos = tile_start.add(t); - var present = b.openIfElse(kv_pos.lt(seq_len), .{}); - { - const off = kv_base.add(kv_pos.to(.long).mul(kv_token_stride)); - const k_ptr = a.k.add(off); - const v_ptr = a.v.add(off); - var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); - { - const d = loop.iv; - K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k_ptr.add(d).to(LoadUnit).load(0)); - V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v_ptr.add(d).to(LoadUnit).load(0)); - loop.close(); - } - present.elseBranch(); - } - { - const zero = b.constant(LoadUnit, 0); - var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); - { - const d = loop.iv; - K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); - V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); - loop.close(); - } - present.close(); - } - rows.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const s_reg = b.array(.float2, @intCast(TK)); - var zero_s = b.openForOpts(0, TK, 1, unroll); - { - s_reg.store(zero_s.iv, b.constant(.float2, 0)); - zero_s.close(); - } - - var qk_dims = b.openForOpts(0, TD, 1, unroll); - { - const d = qk_dims.iv; - const q_frag = b.matrix(T.scalar, q_reg.load(d)); - var qk_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = qk_keys.iv; - const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); - const s_frag = b.matrix(.float, s_reg.load(k)); - s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); - qk_keys.close(); - } - qk_dims.close(); - } - - const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); - const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); - const tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(tile_start.add(TILE_KV).sub(1).lt(context_len.add(q_pos_start))); - - var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); - { - var scale = b.openForOpts(0, TK, 1, unroll); - { - const k = scale.iv; - setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); - setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); - scale.close(); - } - masking.elseBranch(); - } - { - var keys = b.openForOpts(0, TK, 1, unroll); - { - const k = keys.iv; - var pair = b.openForOpts(0, 2, 1, unroll); - { - const jj = pair.iv; - const s = s_reg.load(k).at(jj).mul(scale_log2); - const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); - const masked = row_masked.bitOr(kv_pos.ge(seq_len)).bitOr(kv_pos.gt(q_abs_pos)); - setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); - pair.close(); - } - keys.close(); - } - masking.close(); - } - - const local_max = b.local(.float, masked_score); - var max_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = max_keys.iv; - local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); - max_keys.close(); - } - const row_max = fragRowReduce(b, .max, local_max); - - const prev_max = max_score.rvalue(); - const grown_max = b.max(prev_max, row_max); - const grew = grown_max.gt(prev_max); - const was_empty = prev_max.eq(masked_score); - const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); - const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); - // A tile masking a whole row leaves the maximum masked: zero would - // pin it, and later scores far below zero would underflow. - max_score.assign(grown_max); - - const local_sum = b.local(.float, 0.0); - var p_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = p_keys.iv; - var pair = b.openForOpts(0, 2, 1, unroll); - { - const jj = pair.iv; - const s = s_reg.load(k).at(jj); - const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); - setLane(s_reg, k, jj, p); - local_sum.assign(local_sum.add(p)); - pair.close(); - } - p_keys.close(); - } - const row_sum = fragRowReduce(b, .sum, local_sum); - - sum_score.assign(sum_score.mul(factor).add(row_sum)); - - var rescale_o = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); - rescale_o.close(); - } - - var pv_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = pv_keys.iv; - const p_frag = b.matrix(.float, s_reg.load(k)); - var pv_dims = b.openForOpts(0, TD, 1, unroll); - { - const d = pv_dims.iv; - const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - const o_frag = b.matrix(.float, o_reg.load(d)); - o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); - pv_dims.close(); - } - pv_keys.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - tiles.close(); - } - - const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); - var normalize = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); - normalize.close(); - } - - const O_smem = Q_smem.to(.float); - var store_o = b.openForOpts(0, TD, 1, unroll); - { - const d = store_o.iv; - const o_frag = b.matrix(.float, o_reg.load(d)); - b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - store_o.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - var write_out = b.openFor(thread_idx, block_rows.mul(HEAD_SIZE), NUM_THREADS); - { - const i = write_out.iv; - const r = i.div(HEAD_SIZE); - const d = i.rem(HEAD_SIZE); - const value = b.ternary(r.lt(valid_q), O_smem.load(r.mul(LD).add(d)).to(T), b.constant(T, 0)); - out_base.store(r.mul(q_stride).add(d), value); - write_out.close(); - } - } -}; - -test "the flash attention kernels emit verified modules" { - const allocator = std.testing.allocator; - for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { - for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { - allocator.free(try FaVec.Kernel.emit(allocator, .{ .head_dim = head_dim, .heads = launch[0], .splits = launch[1] })); - } - for ([_]i32{ 8, 128 }) |splits| { - allocator.free(try FaVecReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); - } - } - for ([_]Type{ .half, .bfloat }) |dtype| { - for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { - const tile = FaTiled.tileConfig(head_size).?; - allocator.free(try FaTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2] })); - } - } -} diff --git a/zml/kernels/flash_attention/metal/metal.zig b/zml/kernels/flash_attention/metal/metal.zig deleted file mode 100644 index bdfb00c0..00000000 --- a/zml/kernels/flash_attention/metal/metal.zig +++ /dev/null @@ -1,143 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../../zml.zig"); -const flash_attn = @import("flash_attn.zig"); - -pub const Metadata = struct { - /// Live queries of a single-sequence prefill; 0 means all. - num_tokens: zml.Tensor, - - pub fn init() Metadata { - return .{ .num_tokens = .fromShape(.scalar(.u32)) }; - } - - pub fn initBuffer(self: Metadata, io: std.Io, platform: *const zml.Platform) !zml.Bufferized(Metadata) { - _ = self; - return .{ .num_tokens = try zml.Buffer.scalar(io, platform, 0, .u32) }; - } - - pub fn deinitBuffer(self: *zml.Bufferized(Metadata)) void { - self.num_tokens.deinit(); - } -}; - -/// Causal attention: fa_vec for bf16 decode, fa_tiled otherwise, sdpa for -/// unsupported dtypes and head sizes. -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, metadata: Metadata) zml.Tensor { - const dtype = q.dtype(); - const head_dim = q.dim(.hd); - if (!((dtype == .bf16 or dtype == .f16) and k.dtype() == dtype and v.dtype() == dtype and flash_attn.FaTiled.tileConfig(head_dim) != null)) { - const seq_len = k.dim(.k); - const mask = zml.nn.causalAttnMask(.{ .q = seq_len, .k = seq_len }, dtype, null) - .gatherSlices(zml.Shape.init(.{ .q = q.dim(.q) }, dtype), token_index.appendAxes(.{.coord}), .{}); - return zml.nn.sdpa(q, k, v, .{ .attn_mask = mask }); - } - - const batched = q.shape().hasTag(.b) != null; - const qb = if (batched) q.transpose(.{ .b, .q, .h, .hd }) else q.transpose(.{ .q, .h, .hd }).insertAxes(0, .{.b}); - const kb = if (batched) k.transpose(.{ .b, .k, .h, .hd }) else k.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); - const vb = if (batched) v.transpose(.{ .b, .k, .h, .hd }) else v.transpose(.{ .k, .h, .hd }).insertAxes(0, .{.b}); - const positions = token_index.convert(.i32).broad(.init(.{ .b = qb.dim(.b) }, .i32)); - const scale = 1.0 / @sqrt(@as(f32, @floatFromInt(head_dim))); - - const vec_head_dim = head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512; - const out = if (qb.dim(.q) == 1 and dtype == .bf16 and vec_head_dim) - decode(qb, kb, vb, positions, scale) - else - prefill(qb, kb, vb, positions, liveQueries(qb, metadata), scale); - return (if (batched) out else out.squeeze(.b)).transpose(q.shape()); -} - -fn liveQueries(q: zml.Tensor, metadata: Metadata) zml.Tensor { - const all: zml.Tensor = .scalar(q.dim(.q), .i32); - if (q.dim(.b) != 1 or q.dim(.q) == 1) return all; - const num_tokens = metadata.num_tokens.convert(.i32); - return num_tokens.cmp(.GE, .scalar(1, .i32)).select(num_tokens.minimum(all), all); -} - -/// `q` is `[b, 1, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. -fn decode(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, scale: f32) zml.Tensor { - const batch: i32 = @intCast(q.dim(.b)); - const num_heads: i32 = @intCast(q.dim(.h)); - const head_dim: i32 = @intCast(q.dim(.hd)); - const num_kv_heads: i32 = @intCast(k.dim(.h)); - const keys: i32 = @intCast(k.dim(.k)); - const nrows = batch * num_heads; - const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), keys, coreCount()); - const args: flash_attn.FaVec.HostArgs = .{ .num_heads = num_heads, .n_kv = num_kv_heads, .k_len = keys, .scale = scale }; - - const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); - const out = flash_attn.FaVec.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions }, .{ .dst = dst }, .{ - .cfg = .{ .head_dim = head_dim, .heads = launch.heads, .splits = launch.splits }, - .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(batch) }, - .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, - .constants = .{ .args = .of(args) }, - }).dst; - if (launch.splits == 1) return out; - - return flash_attn.FaVecReduce.Kernel.call(.{ .htmp = out, .token_index = positions }, .{ .dst = q.shape() }, .{ - .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, - .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * flash_attn.FaVecReduce.simdgroups)), 1 }, - .threadgroup = .{ 32, flash_attn.FaVecReduce.simdgroups, 1 }, - .constants = .{ .args = .of(args) }, - }).dst; -} - -/// `q` is `[b, q, h, hd]`, `k` and `v` `[b, k, kv_h, hd]`. -fn prefill(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, positions: zml.Tensor, live: zml.Tensor, scale: f32) zml.Tensor { - const queries: i32 = @intCast(q.dim(.q)); - const tile = flash_attn.FaTiled.tileConfig(q.dim(.hd)).?; - const cfg: flash_attn.FaTiled.Cfg = .{ - .dtype = zml.kernel.metal.from(q.dtype()), - .head_size = @intCast(q.dim(.hd)), - .bq = tile[0], - .tile_kv = tile[1], - .num_threads = tile[2], - }; - return flash_attn.FaTiled.Kernel.call(.{ .q = q, .k = k, .v = v, .token_index = positions, .live = live }, .{ .out = q.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(q.dim(.h)), @intCast(@divFloor(queries + tile[0] - 1, tile[0])), @intCast(q.dim(.b)) }, - .threadgroup = .{ @intCast(tile[2]), 1, 1 }, - .threadgroup_memory = flash_attn.FaTiled.threadgroupMemory(cfg), - .constants = .{ - .num_kv_heads = .of(@as(i32, @intCast(k.dim(.h)))), - .scale = .of(scale), - .q_len = .of(queries), - .k_len = .of(@as(i32, @intCast(k.dim(.k)))), - }, - }).out; -} - -const VecLaunch = struct { - heads: i32, - splits: i32, - splits_per_threadgroup: i32, -}; - -/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a -/// row's length exit early, so `splits` follows the core and row counts. -fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { - var heads: i32 = @min(group, 4); - while (@rem(group, heads) != 0) heads -= 1; - const want: i32 = if (rows <= 2 * cores) 128 else 64; - const chunks = @divFloor(span + 31, 32); - var splits: i32 = 1; - while (splits * 2 <= @min(want, chunks)) splits *= 2; - return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; -} - -fn coreCount() i32 { - const platform = zml.Compiler.current().platform; - const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse - std.debug.panic("Metal device has no core_count attribute", .{}); - return @intCast(attribute.int64); -} - -test vecLaunch { - try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); -} - -test { - std.testing.refAllDecls(flash_attn); -} diff --git a/zml/kernels/flash_attention/nki/BUILD.bazel b/zml/kernels/flash_attention/nki/BUILD.bazel deleted file mode 100644 index f8e4f470..00000000 --- a/zml/kernels/flash_attention/nki/BUILD.bazel +++ /dev/null @@ -1,30 +0,0 @@ -load("@rules_python//python:py_binary.bzl", "py_binary") - -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "attention_kernel", - srcs = ["attention.py"], -) - -filegroup( - name = "nki", - srcs = ["nki.zig"], -) - -py_binary( - name = "attention_harness", - srcs = ["attention_harness.py"], - data = [ - ":attention.py", - "@libzml_neuron//:sandbox", - ], - visibility = ["//visibility:public"], - deps = [ - "@neuron_py_deps//ml_dtypes:pkg", - "@neuron_py_deps//neuronx_cc:pkg", - "@neuron_py_deps//nki:pkg", - "@neuron_py_deps//numpy:pkg", - "@rules_python//python/runfiles", - ], -) diff --git a/zml/kernels/flash_attention/nki/attention.py b/zml/kernels/flash_attention/nki/attention.py deleted file mode 100644 index cf76d9ae..00000000 --- a/zml/kernels/flash_attention/nki/attention.py +++ /dev/null @@ -1,473 +0,0 @@ -import math - -import nki.isa as nisa -import nki.language as nl - -NEG_INF = -3.3895313892515355e38 - - -def decode(q, k, v, token_index): - q_len = q.shape[0] - num_q_heads = q.shape[1] - d_head = q.shape[2] - seq_len = k.shape[0] - num_kv_heads = k.shape[1] - heads_per_kv = num_q_heads // num_kv_heads - scale = 1.0 / (d_head**0.5) - - assert q_len == 1 - assert k.shape == v.shape - assert q.shape[2] == k.shape[2] - assert num_q_heads % num_kv_heads == 0 - - VALUE_TILE_TOKENS = nl.tile_size.pmax - assert seq_len % VALUE_TILE_TOKENS == 0, ( - "seq_len " - f"({seq_len}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" - ) - # Largest tensor-engine K tile that exactly partitions this static sequence. - BLOCK_K = math.gcd(seq_len, nl.tile_size.gemm_moving_fmax) - assert seq_len % BLOCK_K == 0, ( - f"seq_len ({seq_len}) must be a multiple of BLOCK_K ({BLOCK_K})" - ) - assert BLOCK_K % VALUE_TILE_TOKENS == 0, ( - "BLOCK_K " - f"({BLOCK_K}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" - ) - num_kv_blocks = seq_len // BLOCK_K - value_tiles_per_k_tile = BLOCK_K // VALUE_TILE_TOKENS - - out = nl.ndarray((q_len, num_q_heads, d_head), dtype=q.dtype, buffer=nl.shared_hbm) - token = nl.ndarray((1, 1), dtype=token_index.dtype, buffer=nl.sbuf) - nisa.dma_copy(dst=token, src=token_index[0:1, 0:1]) - - positions = nl.ndarray((q_len, BLOCK_K), dtype=token_index.dtype, buffer=nl.sbuf) - qk = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.psum) - qk_sbuf = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) - masked_qk = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) - neg_tile = nl.full( - (heads_per_kv, BLOCK_K), NEG_INF, dtype=nl.float32, buffer=nl.sbuf - ) - tile_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) - new_row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) - old_max_delta = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) - old_max_scale = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) - row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) - neg_row_max = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) - exp_tile = nl.ndarray((heads_per_kv, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf) - tile_sum = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) - sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) - scaled_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) - new_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) - inverse_sum_row = nl.ndarray((heads_per_kv, 1), dtype=nl.float32, buffer=nl.sbuf) - running_out = nl.ndarray((heads_per_kv, d_head), dtype=nl.float32, buffer=nl.sbuf) - scaled_running_out = nl.ndarray( - (heads_per_kv, d_head), dtype=nl.float32, buffer=nl.sbuf - ) - attn_out_psum = nl.ndarray((heads_per_kv, d_head), dtype=nl.float32, buffer=nl.psum) - attn_out_sbuf = nl.ndarray((heads_per_kv, d_head), dtype=q.dtype, buffer=nl.sbuf) - - prob_t_psum = nl.ndarray( - (VALUE_TILE_TOKENS, heads_per_kv), dtype=nl.float32, buffer=nl.psum - ) - prob_t = nl.ndarray( - (VALUE_TILE_TOKENS, heads_per_kv), dtype=v.dtype, buffer=nl.sbuf - ) - v_sbuf = nl.ndarray((VALUE_TILE_TOKENS, d_head), dtype=v.dtype, buffer=nl.sbuf) - - for kv_head in nl.affine_range(num_kv_heads): - q_head = kv_head * heads_per_kv - - q_sbuf = nl.ndarray((d_head, heads_per_kv), dtype=q.dtype, buffer=nl.sbuf) - nisa.dma_transpose( - dst=q_sbuf, - src=q[0, q_head : q_head + heads_per_kv, :], - ) - nisa.activation(dst=q_sbuf, op=nl.copy, data=q_sbuf, scale=scale) - - nisa.iota(positions, [[1, BLOCK_K]]) - - k_sbuf = nl.ndarray((d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf) - nisa.dma_transpose(dst=k_sbuf, src=k[nl.ds(0, BLOCK_K), kv_head, :]) - nisa.memset(dst=qk, value=0.0) - nisa.nc_matmul(dst=qk, stationary=q_sbuf, moving=k_sbuf) - nisa.tensor_copy(dst=qk_sbuf, src=qk) - - valid = nl.less_equal(positions, nl.broadcast_to(token, positions.shape)) - masked_qk = nl.where( - nl.broadcast_to(valid, masked_qk.shape), - qk_sbuf, - neg_tile, - dtype=nl.float32, - ) - nisa.tensor_reduce(dst=row_max, op=nl.maximum, data=masked_qk, axis=(1,)) - nisa.tensor_scalar( - dst=neg_row_max, - data=row_max, - op0=nl.multiply, - operand0=-1.0, - ) - nisa.activation( - dst=exp_tile, - op=nl.exp, - data=masked_qk, - bias=neg_row_max, - reduce_op=nl.add, - reduce_res=sum_row, - reduce_cmd=nisa.reduce_cmd.reset_reduce, - ) - - nisa.memset(dst=attn_out_psum, value=0.0) - for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): - value_start = value_tile_idx * VALUE_TILE_TOKENS - nisa.nc_transpose( - dst=prob_t_psum, - data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], - ) - nisa.tensor_copy(dst=prob_t, src=prob_t_psum) - - nisa.dma_copy( - dst=v_sbuf, - src=v[nl.ds(value_start, VALUE_TILE_TOKENS), kv_head, :], - ) - nisa.nc_matmul(dst=attn_out_psum, stationary=prob_t, moving=v_sbuf) - nisa.tensor_copy(dst=running_out, src=attn_out_psum) - - for k_tail_idx in nl.affine_range(num_kv_blocks - 1): - k_block_idx = k_tail_idx + 1 - k_start = k_block_idx * BLOCK_K - nisa.iota(positions, [[1, BLOCK_K]], offset=k_start) - - k_sbuf = nl.ndarray((d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf) - nisa.dma_transpose(dst=k_sbuf, src=k[nl.ds(k_start, BLOCK_K), kv_head, :]) - nisa.memset(dst=qk, value=0.0) - nisa.nc_matmul(dst=qk, stationary=q_sbuf, moving=k_sbuf) - nisa.tensor_copy(dst=qk_sbuf, src=qk) - - valid = nl.less_equal(positions, nl.broadcast_to(token, positions.shape)) - masked_qk = nl.where( - nl.broadcast_to(valid, masked_qk.shape), - qk_sbuf, - neg_tile, - dtype=nl.float32, - ) - nisa.tensor_reduce(dst=tile_max, op=nl.maximum, data=masked_qk, axis=(1,)) - nisa.tensor_tensor( - dst=new_row_max, - data1=row_max, - data2=tile_max, - op=nl.maximum, - ) - nisa.tensor_scalar( - dst=old_max_delta, - data=row_max, - op0=nl.subtract, - operand0=new_row_max, - ) - nisa.activation(dst=old_max_scale, op=nl.exp, data=old_max_delta) - nisa.tensor_scalar( - dst=neg_row_max, - data=new_row_max, - op0=nl.multiply, - operand0=-1.0, - ) - nisa.activation( - dst=exp_tile, - op=nl.exp, - data=masked_qk, - bias=neg_row_max, - reduce_op=nl.add, - reduce_res=tile_sum, - reduce_cmd=nisa.reduce_cmd.reset_reduce, - ) - nisa.tensor_scalar( - dst=scaled_sum_row, - data=sum_row, - op0=nl.multiply, - operand0=old_max_scale, - ) - nisa.tensor_tensor( - dst=new_sum_row, - data1=scaled_sum_row, - data2=tile_sum, - op=nl.add, - ) - nisa.tensor_scalar( - dst=scaled_running_out, - data=running_out, - op0=nl.multiply, - operand0=old_max_scale, - ) - - nisa.memset(dst=attn_out_psum, value=0.0) - for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): - value_start = value_tile_idx * VALUE_TILE_TOKENS - nisa.nc_transpose( - dst=prob_t_psum, - data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], - ) - nisa.tensor_copy(dst=prob_t, src=prob_t_psum) - - nisa.dma_copy( - dst=v_sbuf, - src=v[nl.ds(k_start + value_start, VALUE_TILE_TOKENS), kv_head, :], - ) - nisa.nc_matmul(dst=attn_out_psum, stationary=prob_t, moving=v_sbuf) - - nisa.tensor_tensor( - dst=running_out, - data1=scaled_running_out, - data2=attn_out_psum, - op=nl.add, - ) - nisa.tensor_copy(dst=row_max, src=new_row_max) - nisa.tensor_copy(dst=sum_row, src=new_sum_row) - - nisa.reciprocal(dst=inverse_sum_row, data=sum_row) - nisa.tensor_scalar( - dst=running_out, - data=running_out, - op0=nl.multiply, - operand0=inverse_sum_row, - ) - nisa.tensor_copy(dst=attn_out_sbuf, src=running_out) - - nisa.dma_copy( - dst=out[0, q_head : q_head + heads_per_kv, :], - src=attn_out_sbuf, - ) - - return out - - -def prefill(q, k, v): - q_len = q.shape[0] - num_q_heads = q.shape[1] - d_head = q.shape[2] - seq_len = k.shape[0] - num_kv_heads = k.shape[1] - heads_per_kv = num_q_heads // num_kv_heads - scale = 1.0 / (d_head**0.5) - - assert q.shape[2] == k.shape[2] - assert k.shape == v.shape - assert num_q_heads % num_kv_heads == 0 - assert q_len == seq_len - - VALUE_TILE_TOKENS = nl.tile_size.pmax - BLOCK_Q = nl.tile_size.pmax - # Largest tensor-engine K tile that exactly partitions this static sequence. - BLOCK_K = math.gcd(seq_len, nl.tile_size.gemm_moving_fmax) - - assert q_len % BLOCK_Q == 0, ( - f"q_len ({q_len}) must be a multiple of BLOCK_Q ({BLOCK_Q})" - ) - assert seq_len % BLOCK_K == 0, ( - f"seq_len ({seq_len}) must be a multiple of BLOCK_K ({BLOCK_K})" - ) - assert BLOCK_K % BLOCK_Q == 0, ( - f"BLOCK_K ({BLOCK_K}) must be a multiple of BLOCK_Q ({BLOCK_Q})" - ) - assert BLOCK_K % VALUE_TILE_TOKENS == 0, ( - "BLOCK_K " - f"({BLOCK_K}) must be a multiple of VALUE_TILE_TOKENS ({VALUE_TILE_TOKENS})" - ) - - num_q_blocks = q_len // BLOCK_Q - q_blocks_per_k_tile = BLOCK_K // BLOCK_Q - value_tiles_per_k_tile = BLOCK_K // VALUE_TILE_TOKENS - - out = nl.ndarray((q_len, num_q_heads, d_head), dtype=q.dtype, buffer=nl.shared_hbm) - - for kv_head in nl.affine_range(num_kv_heads): - for q_idx in nl.affine_range(heads_per_kv): - q_head = kv_head * heads_per_kv + q_idx - - for q_block_idx in nl.affine_range(num_q_blocks): - q_start = q_block_idx * BLOCK_Q - - q_sbuf = nl.ndarray((d_head, BLOCK_Q), dtype=q.dtype, buffer=nl.sbuf) - nisa.dma_transpose( - dst=q_sbuf, - src=q[nl.ds(q_start, BLOCK_Q), q_head, :], - ) - nisa.activation(dst=q_sbuf, op=nl.copy, data=q_sbuf, scale=scale) - - row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) - sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) - running_out = nl.ndarray( - (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf - ) - - masked_qk = nl.ndarray( - (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf - ) - qk_sbuf = nl.ndarray( - (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf - ) - tile_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) - new_row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) - old_max_delta = nl.ndarray( - (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf - ) - old_max_scale = nl.ndarray( - (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf - ) - neg_row_max = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) - exp_tile = nl.ndarray( - (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.sbuf - ) - tile_sum = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) - scaled_sum_row = nl.ndarray( - (BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf - ) - new_sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) - scaled_running_out = nl.ndarray( - (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf - ) - attn_out_psum = nl.ndarray( - (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.psum - ) - prob_t_psum = nl.ndarray( - (VALUE_TILE_TOKENS, BLOCK_Q), dtype=nl.float32, buffer=nl.psum - ) - prob_t = nl.ndarray( - (VALUE_TILE_TOKENS, BLOCK_Q), dtype=v.dtype, buffer=nl.sbuf - ) - qk_psum = nl.ndarray( - (BLOCK_Q, BLOCK_K), dtype=nl.float32, buffer=nl.psum - ) - v_sbuf = nl.ndarray( - (VALUE_TILE_TOKENS, d_head), dtype=v.dtype, buffer=nl.sbuf - ) - - nisa.memset(dst=row_max, value=NEG_INF) - nisa.memset(dst=sum_row, value=0.0) - nisa.memset(dst=running_out, value=0.0) - active_k_tiles = q_block_idx // q_blocks_per_k_tile + 1 - - for k_block_idx in nl.affine_range(active_k_tiles): - k_start = k_block_idx * BLOCK_K - k_sbuf = nl.ndarray( - (d_head, BLOCK_K), dtype=k.dtype, buffer=nl.sbuf - ) - nisa.dma_transpose( - dst=k_sbuf, - src=k[nl.ds(k_start, BLOCK_K), kv_head, :], - ) - nisa.memset(dst=qk_psum, value=0.0) - nisa.nc_matmul(dst=qk_psum, stationary=q_sbuf, moving=k_sbuf) - nisa.tensor_copy(dst=qk_sbuf, src=qk_psum) - nisa.affine_select( - dst=masked_qk, - pattern=[[-1, BLOCK_K]], - offset=q_start - k_start, - channel_multiplier=1, - on_true_tile=qk_sbuf, - on_false_value=NEG_INF, - cmp_op=nl.greater_equal, - ) - nisa.tensor_reduce( - dst=tile_max, - op=nl.maximum, - data=masked_qk, - axis=(1,), - ) - nisa.tensor_tensor( - dst=new_row_max, - data1=row_max, - data2=tile_max, - op=nl.maximum, - ) - nisa.tensor_scalar( - dst=old_max_delta, - data=row_max, - op0=nl.subtract, - operand0=new_row_max, - ) - nisa.activation(dst=old_max_scale, op=nl.exp, data=old_max_delta) - nisa.tensor_scalar( - dst=neg_row_max, - data=new_row_max, - op0=nl.multiply, - operand0=-1.0, - ) - nisa.activation( - dst=exp_tile, - op=nl.exp, - data=masked_qk, - bias=neg_row_max, - reduce_op=nl.add, - reduce_res=tile_sum, - reduce_cmd=nisa.reduce_cmd.reset_reduce, - ) - nisa.tensor_scalar( - dst=scaled_sum_row, - data=sum_row, - op0=nl.multiply, - operand0=old_max_scale, - ) - nisa.tensor_tensor( - dst=new_sum_row, - data1=scaled_sum_row, - data2=tile_sum, - op=nl.add, - ) - nisa.tensor_scalar( - dst=scaled_running_out, - data=running_out, - op0=nl.multiply, - operand0=old_max_scale, - ) - - nisa.memset(dst=attn_out_psum, value=0.0) - for value_tile_idx in nl.affine_range(value_tiles_per_k_tile): - value_start = value_tile_idx * VALUE_TILE_TOKENS - nisa.nc_transpose( - dst=prob_t_psum, - data=exp_tile[:, nl.ds(value_start, VALUE_TILE_TOKENS)], - ) - nisa.tensor_copy(dst=prob_t, src=prob_t_psum) - nisa.dma_copy( - dst=v_sbuf, - src=v[ - nl.ds(k_start + value_start, VALUE_TILE_TOKENS), - kv_head, - :, - ], - ) - nisa.nc_matmul( - dst=attn_out_psum, - stationary=prob_t, - moving=v_sbuf, - ) - - nisa.tensor_tensor( - dst=running_out, - data1=scaled_running_out, - data2=attn_out_psum, - op=nl.add, - ) - nisa.tensor_copy(dst=row_max, src=new_row_max) - nisa.tensor_copy(dst=sum_row, src=new_sum_row) - - inv_sum_row = nl.ndarray((BLOCK_Q, 1), dtype=nl.float32, buffer=nl.sbuf) - normalized_out = nl.ndarray( - (BLOCK_Q, d_head), dtype=nl.float32, buffer=nl.sbuf - ) - attn_out = nl.ndarray((BLOCK_Q, d_head), dtype=q.dtype, buffer=nl.sbuf) - nisa.reciprocal(dst=inv_sum_row, data=sum_row) - nisa.tensor_scalar( - dst=normalized_out, - data=running_out, - op0=nl.multiply, - operand0=inv_sum_row, - ) - nisa.tensor_copy(dst=attn_out, src=normalized_out) - - nisa.dma_copy( - dst=out[nl.ds(q_start, BLOCK_Q), q_head, :], - src=attn_out, - ) - - return out diff --git a/zml/kernels/flash_attention/nki/attention_harness.py b/zml/kernels/flash_attention/nki/attention_harness.py deleted file mode 100644 index 34ce5717..00000000 --- a/zml/kernels/flash_attention/nki/attention_harness.py +++ /dev/null @@ -1,331 +0,0 @@ -import argparse -import dataclasses -import importlib.util -import os -import sys -import time -from pathlib import Path - -import ml_dtypes -import nki -import numpy as np -from python.runfiles import Runfiles - - -DEFAULT_SEQ_LEN = 128 -ATOL = 1e-2 -RTOL = 1e-2 -MIN_CLOSE_FRACTION = 0.99 -INPUT_CASES = ("baseline", "signed") - - -@dataclasses.dataclass(frozen=True) -class ModelSpec: - name: str - num_heads: int - num_kv_heads: int - head_dim: int - - @property - def heads_per_kv(self): - return self.num_heads // self.num_kv_heads - - -MODEL_PRESETS = { - "llama-3.1-8b": ModelSpec( - name="llama-3.1-8b", - num_heads=32, - num_kv_heads=8, - head_dim=128, - ), -} - - -def parse_args(): - parser = argparse.ArgumentParser( - description="Run the classic Neuron NKI attention correctness contract." - ) - parser.add_argument( - "--model", - choices=(*MODEL_PRESETS.keys(), "custom"), - default="llama-3.1-8b", - ) - parser.add_argument("--seq-len", type=int, default=DEFAULT_SEQ_LEN) - parser.add_argument("--dtype", choices=("bf16", "f16", "f32"), default="bf16") - parser.add_argument("--run", choices=("both", "prefill", "decode"), default="both") - parser.add_argument("--mode", choices=("verify", "benchmark"), default="verify") - parser.add_argument("--warmups", type=int, default=1) - parser.add_argument("--iterations", type=int, default=5) - parser.add_argument("--num-heads", type=int, default=None) - parser.add_argument("--num-kv-heads", type=int, default=None) - parser.add_argument("--head-dim", type=int, default=None) - return parser.parse_args() - - -def model_spec(args): - custom_values = (args.num_heads, args.num_kv_heads, args.head_dim) - if args.model != "custom": - if any(value is not None for value in custom_values): - raise ValueError("shape flags are only accepted with --model=custom") - return MODEL_PRESETS[args.model] - - if any(value is None for value in custom_values): - raise ValueError("--model=custom requires --num-heads, --num-kv-heads, and --head-dim") - spec = ModelSpec( - name="custom", - num_heads=args.num_heads, - num_kv_heads=args.num_kv_heads, - head_dim=args.head_dim, - ) - validate_spec(spec) - return spec - - -def validate_spec(spec): - if spec.num_heads <= 0 or spec.num_kv_heads <= 0 or spec.head_dim <= 0: - raise ValueError("model dimensions must be positive") - if spec.num_heads % spec.num_kv_heads != 0: - raise ValueError("num_heads must be divisible by num_kv_heads") - - -def validate_seq_len(seq_len): - if seq_len <= 0 or seq_len % 128 != 0: - raise ValueError("--seq-len must be a positive multiple of 128") - - -def validate_execution_args(args): - if args.mode == "verify": - return - if args.warmups < 0: - raise ValueError("--warmups must be non-negative") - if args.iterations <= 0: - raise ValueError("--iterations must be positive") - - -def decode_positions(seq_len): - return tuple(dict.fromkeys((0, 1, seq_len // 2, seq_len - 1))) - - -def configure_runtime_environment(mode): - if mode == "benchmark": - os.environ.pop("NEURON_RT_LOG_LEVEL", None) - - -def load_nki_attention_module(): - path = Path(__file__).with_name("attention.py") - spec = importlib.util.spec_from_file_location("zml_nki_attention", path) - if spec is None or spec.loader is None: - raise RuntimeError(f"could not load {path}") - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -def add_neuron_tools_to_path(): - runfiles = Runfiles.Create() - python_neuronx_cc = Path( - runfiles.Rlocation("+neuron_packages+libzml_neuron/sandbox/bin/python-shims/neuronx-cc") - ) - neuronx_cc = Path( - runfiles.Rlocation("+neuron_packages+libzml_neuron/sandbox/bin/neuronx-cc") - ) - os.environ["PATH"] = os.pathsep.join( - (str(python_neuronx_cc.parent), str(neuronx_cc.parent), os.environ["PATH"]) - ) - - -def numpy_dtype(name): - if name == "bf16": - return ml_dtypes.bfloat16 - if name == "f16": - return np.float16 - if name == "f32": - return np.float32 - raise ValueError(f"unsupported dtype: {name}") - - -def fill_input(shape, offset, input_case, dtype): - values = np.linspace(0.0, 1.0, num=int(np.prod(shape)), endpoint=False, dtype=np.float32) - if input_case == "signed": - data = offset + (values - 0.5) * 0.25 - else: - data = offset + values * 0.03 - return data.reshape(shape).astype(dtype) - - -def qkv_inputs(spec, seq_len, input_case, decode, dtype): - q_len = 1 if decode else seq_len - q = fill_input((q_len, spec.num_heads, spec.head_dim), 0.01, input_case, dtype) - k = fill_input((seq_len, spec.num_kv_heads, spec.head_dim), 0.02, input_case, dtype) - v = fill_input((seq_len, spec.num_kv_heads, spec.head_dim), 0.03, input_case, dtype) - return q, k, v - - -def reference_attention(q, k, v, token_index=None): - q_len, num_heads, head_dim = q.shape - seq_len, num_kv_heads, _ = k.shape - heads_per_kv = num_heads // num_kv_heads - scale = 1.0 / np.sqrt(head_dim) - q_f32 = q.astype(np.float32) - k_f32 = k.astype(np.float32) - v_f32 = v.astype(np.float32) - out = np.empty((q_len, num_heads, head_dim), dtype=np.float32) - - if token_index is None: - valid = np.arange(seq_len)[None, :] <= np.arange(q_len)[:, None] - else: - if q_len != 1: - raise ValueError("decode reference expects q_len=1") - valid = (np.arange(seq_len) <= token_index)[None, :] - - for kv_head in range(num_kv_heads): - keys = k_f32[:, kv_head, :] - values = v_f32[:, kv_head, :] - for q_idx in range(heads_per_kv): - q_head = kv_head * heads_per_kv + q_idx - scores = (q_f32[:, q_head, :] @ keys.T) * scale - scores = np.where(valid, scores, -np.inf) - scores -= np.max(scores, axis=1, keepdims=True) - probs = np.exp(scores) - probs /= np.sum(probs, axis=1, keepdims=True) - out[:, q_head, :] = probs @ values - - return out.astype(q.dtype) - - -def assert_attention_close(label, expected, actual): - expected_f32 = expected.astype(np.float32) - actual_f32 = actual.astype(np.float32) - finite = np.isfinite(actual_f32) - close = np.isclose(expected_f32, actual_f32, atol=ATOL, rtol=RTOL) - close_fraction = float(close.mean()) - max_abs = float(np.nanmax(np.abs(expected_f32 - actual_f32))) - print( - f"{label} finite={int(finite.sum())}/{actual_f32.size} " - f"close_fraction={close_fraction:.6f} max_abs={max_abs:.6g}" - ) - if not finite.all(): - raise AssertionError(f"{label}: output contains NaN or Inf") - if close_fraction < MIN_CLOSE_FRACTION: - raise AssertionError(f"{label}: close_fraction={close_fraction:.6f}") - - -def timed_run(kernel, inputs, mode, warmups, iterations): - if mode == "verify": - start = time.perf_counter() - output = kernel(*inputs) - elapsed = time.perf_counter() - start - return output, elapsed, 1 - - output = kernel(*inputs) - for _ in range(warmups): - output = kernel(*inputs) - - total = 0.0 - for _ in range(iterations): - start = time.perf_counter() - output = kernel(*inputs) - total += time.perf_counter() - start - return output, total, iterations - - -def print_timing(label, elapsed, iterations, output): - if iterations == 1: - print(f"{label} elapsed={elapsed * 1000.0:.3f}ms output={output.shape}") - return - - average = elapsed / iterations - print( - f"{label} total={elapsed * 1000.0:.3f}ms " - f"avg={average * 1000.0:.3f}ms iterations={iterations} output={output.shape}" - ) - - -def run_prefill(prefill_kernel, spec, seq_len, input_case, dtype, mode, warmups, iterations): - q, k, v = qkv_inputs(spec, seq_len, input_case, decode=False, dtype=dtype) - output, elapsed, timed_iterations = timed_run( - prefill_kernel, - (q, k, v), - mode, - warmups, - iterations, - ) - label = f"prefill input_case={input_case}" - if mode == "verify": - expected = reference_attention(q, k, v) - assert_attention_close(label, expected, output) - print_timing(label, elapsed, timed_iterations, output) - - -def run_decode(decode_kernel, spec, seq_len, input_case, token_index, dtype, mode, warmups, iterations): - q, k, v = qkv_inputs(spec, seq_len, input_case, decode=True, dtype=dtype) - token = np.array([[token_index]], dtype=np.uint32) - output, elapsed, timed_iterations = timed_run( - decode_kernel, - (q, k, v, token), - mode, - warmups, - iterations, - ) - label = f"decode input_case={input_case} token_index={token_index}" - if mode == "verify": - expected = reference_attention(q, k, v, token_index) - assert_attention_close(label, expected, output) - print_timing(label, elapsed, timed_iterations, output) - - -def main(): - args = parse_args() - spec = model_spec(args) - validate_spec(spec) - validate_seq_len(args.seq_len) - validate_execution_args(args) - positions = decode_positions(args.seq_len) - dtype = numpy_dtype(args.dtype) - - configure_runtime_environment(args.mode) - add_neuron_tools_to_path() - module = load_nki_attention_module() - prefill_kernel = nki.jit(module.prefill) - decode_kernel = nki.jit(module.decode) - - header = ( - "classic_attention_contract " - f"model={spec.name} seq_len={args.seq_len} num_heads={spec.num_heads} " - f"num_kv_heads={spec.num_kv_heads} heads_per_kv={spec.heads_per_kv} " - f"head_dim={spec.head_dim} dtype={args.dtype} run={args.run} " - f"mode={args.mode}" - ) - if args.mode == "benchmark": - header += f" warmups={args.warmups} iterations={args.iterations}" - print(f"{header} decode_positions={positions}") - for input_case in INPUT_CASES: - if args.run in ("both", "prefill"): - run_prefill( - prefill_kernel, - spec, - args.seq_len, - input_case, - dtype, - args.mode, - args.warmups, - args.iterations, - ) - if args.run in ("both", "decode"): - for token_index in positions: - run_decode( - decode_kernel, - spec, - args.seq_len, - input_case, - token_index, - dtype, - args.mode, - args.warmups, - args.iterations, - ) - - -if __name__ == "__main__": - main() diff --git a/zml/kernels/flash_attention/nki/nki.zig b/zml/kernels/flash_attention/nki/nki.zig deleted file mode 100644 index f98a545d..00000000 --- a/zml/kernels/flash_attention/nki/nki.zig +++ /dev/null @@ -1,59 +0,0 @@ -const platforms = @import("platforms"); - -const zml = @import("../../../zml.zig"); -const stdx = zml.stdx; - -const KernelSpec = struct { - name: []const u8, - entrypoint: []const u8, - source_path: []const u8, -}; - -pub const Parameters = struct { - compiler_target: []const u8, - decode_kernel: KernelSpec, - prefill_kernel: KernelSpec, - - pub fn init() Parameters { - if (comptime platforms.isEnabled(.neuron)) { - const nki_kernel = @import("platforms/neuron/nki_kernel"); - const compiler_target = nki_kernel.compilerTargetFromInstance(); - return .{ - .compiler_target = @tagName(compiler_target), - .decode_kernel = switch (compiler_target) { - .trn1, .inf2, .trn1n, .trn2, .trn2n, .trn3 => .{ - .name = "nki_attention_trn1_decode", - .entrypoint = "decode", - .source_path = "zml/zml/attention/nki/attention.py", - }, - }, - .prefill_kernel = switch (compiler_target) { - .trn1, .inf2, .trn1n, .trn2, .trn2n, .trn3 => .{ - .name = "nki_attention_trn1_prefill", - .entrypoint = "prefill", - .source_path = "zml/zml/attention/nki/attention.py", - }, - }, - }; - } - - stdx.debug.panic("NKI attention requires the Neuron platform", .{}); - } -}; - -/// Attention for Neuron. Both prefill and decode lower to NKI custom calls. -pub fn attention(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor, parameters: Parameters) zml.Tensor { - const decode = q.dim(.q) == 1; - const kernel = if (decode) parameters.decode_kernel else parameters.prefill_kernel; - const options: zml.ops.NeuronNkiOps = .{ - .name = kernel.name, - .entrypoint = kernel.entrypoint, - .source_path = kernel.source_path, - .compiler_target = parameters.compiler_target, - }; - const output = if (decode) - zml.ops.neuronNki(.{ q, k, v, token_index.broad(zml.Shape.init(.{ .row = 1, .col = 1 }, token_index.dtype())) }, .{q.shape()}, options)[0] - else - zml.ops.neuronNki(.{ q, k, v }, .{q.shape()}, options)[0]; - return output.convert(q.dtype()); -} diff --git a/zml/kernels/flash_attention/tests.zig b/zml/kernels/flash_attention/tests.zig deleted file mode 100644 index c3c3dca2..00000000 --- a/zml/kernels/flash_attention/tests.zig +++ /dev/null @@ -1,346 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const flash_attention = @import("flash_attention.zig"); - -test "attention: q=1,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); - - try testAttention( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{36}, - ); -} - -test "attention: b=4,q=1,qh=64,kh=8" { - // Full attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 63, 63, 63, 63 }, - ); - - // Partial attention - try testAttention( - .init(.{ .b = 4, .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .b = 4, .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{ 61, 57, 23, 63 }, - ); -} - -test "attention: q=1,qh=8,kh=8" { - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{62}, - ); - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{63}, - ); -} - -test "attention: q=1,k=4096,qh=32,kh=8,hd=128" { - try testAttention( - .init(.{ .q = 1, .h = 32, .hd = 128 }, .bf16), - .init(.{ .k = 4096, .h = 8, .hd = 128 }, .bf16), - &.{3000}, - ); -} - -test "attention: q=1,k=1000,qh=8,kh=2,hd=32" { - try testAttention( - .init(.{ .q = 1, .h = 8, .hd = 32 }, .bf16), - .init(.{ .k = 1000, .h = 2, .hd = 32 }, .bf16), - &.{900}, - ); -} - -test "attention: q=8,qh=64,kh=8" { - try testAttention( - .init(.{ .q = 8, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 64, .h = 8, .hd = 64 }, .bf16), - &.{56}, - ); -} - -test "attention: prefill, q=k=300,qh=8,kh=4" { - // Neither length is a multiple of the query (128) or key (64) tiles. - try testAttention( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries start at 800: every query tile sees full key tiles before the diagonal. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{800}, - ); -} - -test "attention: prefill, q=k=200,qh=4,kh=2,hd=32" { - // Rows of 32 features only hold 4 16-byte vectors: the shared memory swizzle must stay inside them. - try testAttention( - .init(.{ .q = 200, .h = 4, .hd = 32 }, .bf16), - .init(.{ .k = 200, .h = 2, .hd = 32 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, q=k=130,qh=4,kh=2,hd=256" { - try testAttention( - .init(.{ .q = 130, .h = 4, .hd = 256 }, .bf16), - .init(.{ .k = 130, .h = 2, .hd = 256 }, .bf16), - &.{0}, - ); -} - -test "attention: prefill, b=2,q=150,k=300,qh=4,kh=2" { - try testAttention( - .init(.{ .b = 2, .q = 150, .h = 4, .hd = 128 }, .bf16), - .init(.{ .b = 2, .k = 300, .h = 2, .hd = 128 }, .bf16), - &.{ 150, 100 }, - ); -} - -test "attention: non-causal prefill, q=k=300,qh=8,kh=4" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 8, .hd = 128 }, .bf16), - .init(.{ .k = 300, .h = 4, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, b=2,q=200,k=1000,qh=4,kh=4,hd=64" { - // Queries are a suffix of the keys, as for the image tokens of a joint text/image attention. - try testAttentionOpts( - .init(.{ .b = 2, .q = 200, .h = 4, .hd = 64 }, .bf16), - .init(.{ .b = 2, .k = 1000, .h = 4, .hd = 64 }, .bf16), - &.{ 0, 0 }, - .{ .causal = false }, - ); -} - -test "attention: non-causal prefill, more queries than keys, q=300,k=100,qh=4,kh=2" { - try testAttentionOpts( - .init(.{ .q = 300, .h = 4, .hd = 128 }, .bf16), - .init(.{ .k = 100, .h = 2, .hd = 128 }, .bf16), - &.{0}, - .{ .causal = false }, - ); -} - -test "attention: non-causal decode ignores token_index, q=1,k=1000,qh=64,kh=8" { - try testAttentionOpts( - .init(.{ .q = 1, .h = 64, .hd = 64 }, .bf16), - .init(.{ .k = 1000, .h = 8, .hd = 64 }, .bf16), - &.{5}, - .{ .causal = false }, - ); -} - -pub fn testAttention(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32) !void { - return testAttentionOpts(q_shape, k_shape, token_index_h, .{}); -} - -fn testAttentionOpts(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options) !void { - return testAttentionBackends(q_shape, k_shape, token_index_h, opts, std.enums.values(flash_attention.Backend)); -} - -fn testAttentionBackends(q_shape: zml.Shape, k_shape: zml.Shape, token_index_h: []const u32, opts: zml.attention.Options, backends: []const flash_attention.Backend) !void { - const io = std.testing.io; - const allocator = std.testing.allocator; - - const platform = zml.testing.env(); - - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const token_index_shape: zml.Shape = if (token_index_h.len > 1) b: { - std.debug.assert(token_index_h.len == q_shape.dim(.b)); - break :b .init(.{ .b = q_shape.dim(.b) }, .u32); - } else .init(.{}, .u32); - - const max_k_idx: i64 = k_shape.dim(.k) - 1; - const max_q_idx: i64 = q_shape.dim(.q) - 1; - if (opts.causal) { - // Check for out of bound reads, token_index is ignored by non-causal attention. - for (token_index_h) |index| { - std.debug.assert(index + max_q_idx <= max_k_idx); - } - } - - const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, token_index: zml.Tensor } = .{ - .q = .fromShape(q_shape), - .k = .fromShape(k_shape), - .v = .fromShape(k_shape), - .token_index = .fromShape(token_index_shape), - }; - - var q = try attentionTestInput(q_shape, 17); - defer q.deinit(); - var k = try attentionTestInput(k_shape, 29); - defer k.deinit(); - var v = try attentionTestInput(k_shape, 43); - defer v.deinit(); - var token_index = try zml.Buffer.fromBytes(io, platform, token_index_shape, @ptrCast(token_index_h)); - defer token_index.deinit(); - - const meshes = platform.meshes.values(); - const mesh = zml.testing.tpMesh(); - const partitioning: zml.attention.Partitioning = if (@rem(k_shape.dim(.h), mesh.numPartitionsForLogicalAxis(.model)) == 0) .head(.model) else .{ - .in_specs = .{ .q = .replicated, .k = .replicated, .v = .replicated }, - .out_specs = .replicated, - }; - const AttentionCall = struct { - fn forward(q_: zml.Tensor, k_: zml.Tensor, v_: zml.Tensor, position: zml.Tensor, metadata: zml.attention.Metadata, parameters: zml.attention.Parameters, opts_: zml.attention.Options, mesh_: *const zml.Mesh, partitioning_: zml.attention.Partitioning) zml.Tensor { - return zml.attention.attentionOpts(q_, k_, v_, position, metadata, parameters, opts_, mesh_, partitioning_); - } - }; - const reference_metadata: zml.attention.Metadata = .{ .stablehlo = {} }; - const reference_parameters: zml.attention.Parameters = .{ .stablehlo = {} }; - const reference_exe = try platform.compileFn(allocator, io, AttentionCall.forward, .{ tensors.q, tensors.k, tensors.v, tensors.token_index, reference_metadata, reference_parameters, opts, mesh, partitioning }, .{ - .program_name = "attention_stablehlo", - .meshes = meshes, - }); - defer reference_exe.deinit(); - - var reference_d = try reference_exe.eval(allocator, io, .{ q, k, v, token_index, reference_metadata }); - defer reference_d.deinit(); - try reference_d.await(io); - try std.testing.expectEqualDeep(q_shape.withPartitioning(mesh, partitioning.out_specs)._sharding, reference_d.shape()._sharding); - const reference_h: zml.Slice = try reference_d.toSliceAlloc(allocator, io); - defer reference_h.free(allocator); - - for (backends) |backend| { - if (!backend.isAvailable(platform)) continue; - - const metadata: flash_attention.Metadata = .init(.fromBackend(backend)); - const parameters: flash_attention.Parameters = .init(.fromBackend(backend)); - const exe = try platform.compileFn( - allocator, - io, - AttentionCall.forward, - .{ tensors.q, tensors.k, tensors.v, tensors.token_index, .{ .flash_attention = metadata }, .{ .flash_attention = parameters }, opts, mesh, partitioning }, - .{ - .program_name = try std.fmt.allocPrint(arena, "attention_{t}", .{backend}), - .meshes = meshes, - }, - ); - defer exe.deinit(); - - var metadata_d = try metadata.initBuffer(io, platform); - defer flash_attention.Metadata.deinitBuffer(&metadata_d); - - var output_d = try exe.eval(allocator, io, .{ q, k, v, token_index, metadata_d }); - defer output_d.deinit(); - try output_d.await(io); - const output_h = try output_d.toSliceAlloc(allocator, io); - defer output_h.free(allocator); - - errdefer std.log.err( - \\ Attention test failed, {0t} output doesn't match reference. - \\ - reference: {1d} - \\ - {0t}: {2d} - , .{ backend, reference_h, output_h }); - try zml.testing.expectClose(io, reference_h, output_h, .{ - .absolute_tolerance = 5e-3, - .relative_tolerance = 1e-2, - .epsilon_relative = 1e-3, - .minimum_close_fraction = 0.99, - }); - } -} - -fn attentionTestInput(shape: zml.Shape, seed: u64) !zml.Buffer { - const host = try zml.Slice.alloc(std.testing.allocator, shape); - defer host.free(std.testing.allocator); - var prng: std.Random.DefaultPrng = .init(seed); - const random = prng.random(); - switch (shape.dtype()) { - inline .bf16, .f16, .f32 => |dtype| { - for (host.items(dtype.toZigType())) |*value| { - const x = random.float(f32) * 4 - 2; - value.* = if (dtype == .bf16) .fromF32(x) else @floatCast(x); - } - }, - else => unreachable, - } - return zml.Buffer.fromSlice(std.testing.io, zml.testing.env(), host); -} - -/// Independent f64 oracle: the SDPA reference uses fast dot precision, so its -/// f32 result alone cannot establish the accuracy of the kernel's f32 arithmetic. -fn checkAttentionF32(q: zml.Slice, k: zml.Slice, v: zml.Slice, positions: []const u32, output: zml.Slice) !void { - const Layout = struct { - b: usize, - s: usize, - h: usize, - d: usize, - - fn stride(shape: zml.Shape, tag: anytype) usize { - const axis = shape.hasTag(tag) orelse return 0; - var result: usize = 1; - for (shape.dims()[axis + 1 ..]) |dim| result *= @intCast(dim); - return result; - } - - fn init(shape: zml.Shape) @This() { - return .{ .b = stride(shape, .b), .s = if (shape.hasTag(.q) != null) stride(shape, .q) else stride(shape, .k), .h = stride(shape, .h), .d = stride(shape, .hd) }; - } - - fn offset(self: @This(), b: usize, s: usize, h: usize, d: usize) usize { - return b * self.b + s * self.s + h * self.h + d * self.d; - } - }; - const ql = Layout.init(q.shape); - const kl = Layout.init(k.shape); - const vl = Layout.init(v.shape); - const batch: usize = if (q.shape.hasTag(.b) != null) @intCast(q.shape.dim(.b)) else 1; - const queries: usize = @intCast(q.shape.dim(.q)); - const heads: usize = @intCast(q.shape.dim(.h)); - const dim: usize = @intCast(q.shape.dim(.hd)); - const group: usize = @intCast(@divExact(q.shape.dim(.h), k.shape.dim(.h))); - const scores = try std.testing.allocator.alloc(f64, @intCast(k.shape.dim(.k))); - defer std.testing.allocator.free(scores); - const scale = 1.0 / @sqrt(@as(f64, @floatFromInt(dim))); - for (0..batch) |b| { - const start: usize = positions[if (positions.len == 1) 0 else b]; - for (0..queries) |i| { - for (0..heads) |h| { - var maximum: f64 = -std.math.inf(f64); - const active = scores[0 .. start + i + 1]; - for (active, 0..) |*score, j| { - var dot: f64 = 0; - for (0..dim) |d| dot += @as(f64, q.items(f32)[ql.offset(b, i, h, d)]) * @as(f64, k.items(f32)[kl.offset(b, j, h / group, d)]); - score.* = dot * scale; - maximum = @max(maximum, score.*); - } - var sum: f64 = 0; - for (active) |*score| { - score.* = @exp(score.* - maximum); - sum += score.*; - } - for (0..dim) |d| { - var expected: f64 = 0; - for (active, 0..) |weight, j| expected += weight * @as(f64, v.items(f32)[vl.offset(b, j, h / group, d)]); - expected /= sum; - const actual = output.items(f32)[ql.offset(b, i, h, d)]; - try std.testing.expectApproxEqAbs(expected, @as(f64, actual), 2e-5 + 2e-5 * @abs(expected)); - } - } - } - } -} diff --git a/zml/kernels/moe/api.zig b/zml/kernels/moe/api.zig deleted file mode 100644 index b66593c1..00000000 --- a/zml/kernels/moe/api.zig +++ /dev/null @@ -1,164 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const Tensor = @import("../../tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/moe"); - -pub const Options = struct { - activation: Activation, - /// Quantize activations for Triton FP8 GEMMs; false keeps BF16 activations. - quantize_input: bool, - /// Where routing weights are applied; FlashInfer, Mosaic, Metal and TCL require after_down. - routing_weight_placement: RoutingWeightPlacement, -}; - -pub const ActivationKind = enum { - /// Gelu activation function - gelu, - /// ReLU activation function - relu, - /// SiLU activation function - silu, - /// SwiGLU activation function (including clamped/clipped/scaled/biased variants) - swiglu, - /// SwiGLU activation function with step function - swiglu_step, - /// GeGlu activation function - geglu, - /// GeGlu activation function with tanh - geglu_tanh, -}; - -pub const Activation = union(ActivationKind) { - gelu: void, - relu: void, - silu: void, - swiglu: struct { - limit: ?f32 = null, - scale: ?f32 = null, - bias: ?f32 = null, - }, - swiglu_step: struct { - limit: ?f32 = null, - }, - geglu: void, - geglu_tanh: void, - - pub fn applyActivation(self: Activation, input: Tensor, layout: ProjectionLayout) Tensor { - const x = input.convert(.f32); - return switch (self) { - .gelu => x.gelu(), - .relu => x.relu(), - .silu => x.silu(), - .swiglu, .swiglu_step, .geglu, .geglu_tanh => b: { - const mid = @divFloor(x.dim(.out), 2); - var gate, var up = switch (layout) { - .concatenated => .{ x.slice(.out, .{ .end = mid }), x.slice(.out, .{ .start = mid }) }, - .row_interleaved_1 => .{ x.slice(.out, .{ .start = 0, .step = 2 }), x.slice(.out, .{ .start = 1, .step = 2 }) }, - .row_interleaved_8 => blk: { - const grouped = x.splitAxis(.out, .{ .group = @divExact(mid, 8), .pair = 2, .row = 8 }); - break :blk .{ - grouped.slice(.pair, .{ .end = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), - grouped.slice(.pair, .{ .start = 1 }).merge(.{ .out = .{ .group, .pair, .row } }), - }; - }, - }; - - break :b switch (self) { - .swiglu => |parameters| { - stdx.debug.assert(parameters.scale == null, "triton and fly moe backend don't support swiglu scale", .{}); - - const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - - // Apply limit on gate and clamp up - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - - // Apply bias - up = if (parameters.bias) |bias| up.addConstant(bias) else up; - - break :b gate.silu().mul(up); - }, - .swiglu_step => |parameters| { - gate = gate.silu(); - - const limit: ?Tensor = if (parameters.limit) |limit| .scalar(limit, x.dtype()) else null; - gate = if (limit) |l| gate.minimum(l) else gate; - up = if (limit) |l| up.clamp(l.negate(), l) else up; - break :b gate.mul(up); - }, - .geglu_tanh => gate.gelu().mul(up), - .geglu => { - log.warn("The geglu activation function was requested but we only support the tanh approximation", .{}); - break :b gate.gelu().mul(up); - }, - else => unreachable, // already treated by the top-level switch - }; - }, - }; - } -}; - -pub const RoutingWeightPlacement = enum { before_down, after_down }; - -pub const ProjectionLayout = enum { - concatenated, // G0 G1 … Gn U0 U1 … Un - row_interleaved_1, // G0 U0 G1 U1 … - row_interleaved_8, // G0…G7 U0…U7 G8…G15 U8…U15 … -}; - -/// How a backend expects the expert weights to be stored -/// The packer writes this layout and forwardMoe reads it -pub const ExpertsLayout = struct { - /// Gate/up ordering before applying the backend-specific packing. - gate_up: ProjectionLayout, - /// Backend specific transformation of the weights and scales. - packing: Packing, - - pub const Packing = enum { - /// Experts stacked as stored in the checkpoint. - plain, - /// Block scales swizzled to the 128x4 tensor-core layout (rows tiled by 4x32, columns by 4). - swizzled_scales, - /// Weights, block scales and global scales in the FlashInfer CUTLASS NVFP4 layout. - flashinfer_nvfp4, - /// MXFP4 gate/up rows interleaved by 8. E8M0 scales use 512-byte - /// 128x4 atoms in MN-major order `[expert, k/128, rows/128, 512]`, - /// represented as `[expert, k/128, rows]` i32 words of four scales. - mxfp4_interleaved8, - }; -}; - -/// Routing for a local token batch and a contiguous range of experts. -/// IDs retain the model's global expert numbering, while the projection weights -/// contain only the local experts. Kernels compute those experts contributions and -/// `forwardMoe` sums them along the expert axis. -pub const LocalRouting = struct { - /// Global expert IDs for local tokens, with shape {.t, .topk} - ids: Tensor, - weights: Tensor, - global_num_experts: u32, - local_num_experts: u32, - /// Scalar i32 containing the first local expert's global ID. - /// Null when all experts are local and global IDs already index the weights. - expert_start: ?Tensor = null, - - /// Maps routing IDs to local weight indices, using -1 for nonlocal experts. - /// For a shard owning experts 4–7, routing IDs [1, 6] become [-1, 2]. - pub fn localIds(self: LocalRouting) Tensor { - const start = self.expert_start orelse return self.ids; - const local = self.ids.cmp(.GE, start).logical(.AND, self.ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(self.ids.sub(start), .scalar(-1, .i32)); - } - - /// Returns a lookup table from every global expert ID to its local weight - /// index or -1 if nonlocal. Returns null when every expert is local. - pub fn expertMap(self: LocalRouting) ?Tensor { - const start = self.expert_start orelse return null; - const ids = Tensor.arange(.{ .end = self.global_num_experts }, .i32).withTags(.{.expert}); - const local = ids.cmp(.GE, start).logical(.AND, ids.cmp(.LT, start.addConstant(self.local_num_experts))); - return local.select(ids.sub(start), .scalar(-1, .i32)); - } -}; diff --git a/zml/kernels/moe/cute/BUILD.bazel b/zml/kernels/moe/cute/BUILD.bazel deleted file mode 100644 index ca87f37a..00000000 --- a/zml/kernels/moe/cute/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "cute", - srcs = [ - "cute.zig", - "//zml/kernels/moe/cute/sm10x", - ], -) diff --git a/zml/kernels/moe/cute/cute.zig b/zml/kernels/moe/cute/cute.zig deleted file mode 100644 index 8bc03c9c..00000000 --- a/zml/kernels/moe/cute/cute.zig +++ /dev/null @@ -1,139 +0,0 @@ -//! Blackwell MXFP4 and NVFP4 MoE backend with persistent CuTe GEMMs. - -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const Options = @import("../api.zig").Options; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const LocalRouting = @import("../api.zig").LocalRouting; -pub const kernels = @import("sm10x/moe.zig"); - -// `moe.zig`'s `refAllDecls` reaches this file but not its imports, so the -// kernel emit tests are only collected if `kernels` is referenced here. -test { - _ = kernels; -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - const cc = zml.platform.cuda.computeCapability(platform) orelse return false; - return cc.major == 10; -} - -/// The NVFP4 kernels are only validated on SM103. -pub fn isNvfp4Supported(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - const cc = zml.platform.cuda.computeCapability(platform) orelse return false; - return cc.major == 10 and cc.minor == 3; -} - -pub fn validateOptions(opts: Options) void { - stdx.debug.assert(opts.activation == .swiglu, "cute backend only accepts swiglu activation, got {}", .{opts.activation}); - stdx.debug.assert(opts.activation.swiglu.bias == null, "cute backend requires swiglu bias to be null", .{}); - stdx.debug.assert(opts.activation.swiglu.scale == null, "cute backend requires swiglu scale to be null", .{}); -} - -/// MXFP4 weights are `u8` (or `i8`) packed FP4 with E8M0 block scales. NVFP4 -/// weights are packed `u8` with E4M3 block scales; their weight global scale -/// is the per-expert multiplier dequantizing each GEMM output (it folds in -/// the inverse of the input's global scale), and their input scale is the -/// multiplier applied to the GEMM's input before NVFP4 quantization. Both -/// expect block scales in the 128x4 tensor-core layout. MXFP4 gate/up rows -/// interleave by 8 and its scale atoms are MN-major packed i32 words. -pub fn fusedExperts( - input: zml.Tensor, - routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - options: Options, -) zml.Tensor { - validateOptions(options); - stdx.debug.assert(input.dtype() == .bf16, "cute backend only supports bf16 inputs, got {}", .{input.dtype()}); - stdx.debug.assert(gate_up.bias == null and down.bias == null, "cute backend expects gate_up bias and down bias to be null", .{}); - const gq = gate_up.quantization orelse @panic("cute backend requires gate_up quantization to be set"); - const dq = down.quantization orelse @panic("cute backend requires down quantization to be set"); - stdx.debug.assert(gq.scheme == dq.scheme, "cute expects gate_up and down quantization scheme to match, got {} and {}", .{ gq.scheme, dq.scheme }); - - const format: Format = switch (gq.scheme) { - .mxfp4 => blk: { - // Weight storage for mxfp4 in HF is expressed as u8 or i8 - stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "cute expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "cute expects down weight dtype to be u8 or i8, got {}", .{down.weight.dtype()}); - // Check the boundary quantizer and CuTe GEMM tile constraints. - kernels.validateMxfp4Shapes(down.weight.dim(1), down.weight.dim(2) * 2); - // The SwiGLU clamp is a kernel parameter, but the routing weight is - // multiplied in the up epilogue, before the down projection: applying it - // after would have to move into the down epilogue or the reduction. - stdx.debug.assert(options.routing_weight_placement == .before_down, "cute MXFP4 backend only supports routing_weight_placement = .before_down, got {}", .{options.routing_weight_placement}); - break :blk .mxfp4; - }, - .nvfp4 => blk: { - stdx.debug.assert(!options.quantize_input, "cute NVFP4 backend always quantizes its inputs to NVFP4", .{}); - stdx.debug.assert(gq.swizzled_scales and dq.swizzled_scales, "cute NVFP4 backend expects swizzled block scales", .{}); - stdx.debug.assert(gate_up.weight.dtype() == .u8 and down.weight.dtype() == .u8, "cute NVFP4 backend expects packed u8 weights, got {} and {}", .{ gate_up.weight.dtype(), down.weight.dtype() }); - const scale = struct { - fn f(global: ?zml.Quantization.GlobalScale) zml.Tensor { - return (global orelse @panic("cute NVFP4 backend requires global and input scales")).asMultiplier(); - } - }.f; - break :blk .{ .nvfp4 = .{ - .alpha1 = scale(gq.global_scale), - .alpha2 = scale(dq.global_scale), - .act1 = scale(gq.input_scale), - .act2 = scale(dq.input_scale), - } }; - }, - else => stdx.debug.panic("cute backend only supports mxfp4 and nvfp4 quantization, got {}", .{gq.scheme}), - }; - - const hidden = down.weight.dim(1); - const intermediate = down.weight.dim(2) * 2; - const topk = routing.ids.dim(.topk); - const experts = gate_up.weight.dim(.expert); - const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); - const tokens = ids.dim(.token); - const routing_weights = routing.weights.convert(.f32).reshape(.{ tokens, topk }); - - return switch (format) { - .mxfp4 => kernels.forwardMxfp4(tokens, hidden, intermediate, experts, topk, options.activation.swiglu.limit, .{ - .x = input.reshape(.{ tokens, hidden }), - .routing_weights = routing_weights, - .w1 = gate_up.weight, - .s1 = gq.scales, - .w2 = down.weight, - .s2 = dq.scales, - .ids = ids, - }, null), - .nvfp4 => |f| kernels.forwardNvfp4(tokens, hidden, intermediate, experts, topk, .{ - .x = input.reshape(.{ tokens, hidden }), - .routing_weights = routing_weights, - .ids = ids, - .w1 = gate_up.weight, - .s1 = gq.scales, - .w2 = down.weight, - .s2 = dq.scales, - // `alpha` is per expert; `act` is one scale shared by every - // expert, which the kernels read per expert. - .alpha1 = f.alpha1.reshape(.{experts}), - .alpha2 = f.alpha2.reshape(.{experts}), - .act1 = f.act1.reshape(.{}).broad(.init(.{experts}, .f32)), - .act2 = f.act2.reshape(.{}).broad(.init(.{experts}, .f32)), - }, .{ - .swiglu_limit = options.activation.swiglu.limit, - .routing_weight_placement = options.routing_weight_placement, - }), - }.reshape(input.shape().dims()).withTags(input.shape()); -} - -/// The format-specific tensors: NVFP4's per-expert global scales. They stay -/// as given until the sharded body, so they split across ranks like the -/// weights. -const Format = union(enum) { - mxfp4, - nvfp4: struct { - alpha1: zml.Tensor, - alpha2: zml.Tensor, - act1: zml.Tensor, - act2: zml.Tensor, - }, -}; diff --git a/zml/kernels/moe/cute/sm10x/BUILD.bazel b/zml/kernels/moe/cute/sm10x/BUILD.bazel deleted file mode 100644 index 2b23bfbb..00000000 --- a/zml/kernels/moe/cute/sm10x/BUILD.bazel +++ /dev/null @@ -1,14 +0,0 @@ -package(default_visibility = ["//zml/kernels/moe/cute:__pkg__"]) - -filegroup( - name = "sm10x", - srcs = [ - "common.zig", - "moe.zig", - "mxfp4.zig", - "nvfp4.zig", - "persistent_mxfp4.zig", - "persistent_nvfp4.zig", - "reduce.zig", - ], -) diff --git a/zml/kernels/moe/cute/sm10x/common.zig b/zml/kernels/moe/cute/sm10x/common.zig deleted file mode 100644 index da639c75..00000000 --- a/zml/kernels/moe/cute/sm10x/common.zig +++ /dev/null @@ -1,227 +0,0 @@ -//! Machinery shared by the MXFP4 and NVFP4 CuTe MoE kernels: pipeline -//! barriers and state, the static persistent tile scheduler, direct routing, -//! shared-memory pointers, boundary kernel launches and small IR helpers. -const std = @import("std"); -const mlir = @import("mlir"); - -const zml = @import("../../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const V = cute.Value; - -pub fn typed(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { - return .{ .mlir_type = .{ .type_ = type_ } }; -} - -pub fn gridTyped(type_: *const mlir.Type) struct { mlir_type: cute.ArgSpec.MlirTypeSpec } { - return .{ .mlir_type = .{ .type_ = type_, .grid_constant = true } }; -} - -pub fn barrierAt(b: *B, base: V, stage: V) V { - return b.addOffset( - base, - b.makeIntTuple(.{stage}), - b.ptrTy(.i64, .smem, 8) catch @panic("bad pipeline barrier pointer"), - ).value(); -} - -pub const PipelinePosition = struct { - index: V, - phase: V, -}; - -/// Scalar form of CUTLASS's StaticPersistentTileScheduler state for the -/// one-CTA DeepSeek specialization. Python carries the decoded M/N/L -/// coordinate, validity predicate, linear work index, and executed-tile -/// count through every role's `scf.while`. Keeping the same state shape is -/// important to the CuTe software-pipeline passes, even though N is always -/// zero for one tile per group. -pub const PersistentWork = struct { - m: V, - n: V, - group: V, - valid: V, - linear: V, - executed: V, -}; - -/// Direct routing (one token) runs one single-row group per route. Routes -/// owned by other expert-parallel ranks carry expert -1: the persistent tiles -/// cover the valid routes only, in route order, so they do not occupy a wave. -pub const DirectRoutes = struct { - expert_ids: cute.View, - routes: usize, - - pub fn count(self: DirectRoutes, b: *B) V { - var valid = b.constant(.i32, 0); - for (0..self.routes) |r| valid = valid.add(self.expertAt(b, r).ge(0).to(.i32)); - return valid; - } - - /// Route of the `k`-th valid route. - fn nth(self: DirectRoutes, b: *B, k: V) V { - var seen = b.constant(.i32, 0); - var route = b.constant(.i32, 0); - for (0..self.routes) |r| { - const valid = self.expertAt(b, r).ge(0); - route = b.select(valid.bitAnd(seen.eq(k)), b.constant(.i32, r), route); - seen = seen.add(valid.to(.i32)); - } - return route; - } - - fn expertAt(self: DirectRoutes, b: *B, r: usize) V { - return self.expert_ids.get(.{b.constant(.i32, r)}, .i32); - } -}; - -pub fn persistentWorkAt( - b: *B, - linear: V, - executed: V, - active: V, - tiles_m: V, - div_m: V, - div_n: V, - direct: ?DirectRoutes, -) PersistentWork { - // StaticPersistentTileScheduler uses two first-class FastDivmod objects - // to decode the column-major (M, N=1, L) problem layout. Preserve those - // operations even though ordinary rem/div is mathematically equivalent; - // CuTe's scheduler reconstruction and loop canonicalization key off these - // values and carry them through every persistent loop. - const group_and_m = b.fastDivmod(linear, div_m); - const group_and_n = b.fastDivmod(group_and_m[0], div_n); - const group = if (direct) |routes| routes.nth(b, group_and_n[0]) else group_and_n[0]; - return .{ - .m = b.makeWarpUniform(group_and_m[1]), - .n = b.makeWarpUniform(group_and_n[1]), - .group = b.makeWarpUniform(group), - .valid = b.makeWarpUniform(linear.lt(active.mul(tiles_m))), - .linear = linear, - .executed = executed, - }; -} - -pub fn advancePersistentWork( - b: *B, - linear: V, - executed: V, - persistent_clusters: V, - active: V, - tiles_m: V, - div_m: V, - div_n: V, - direct: ?DirectRoutes, -) PersistentWork { - return persistentWorkAt(b, linear.add(persistent_clusters), executed.add(1), active, tiles_m, div_m, div_n, direct); -} - -/// Equivalent to CUTLASS PipelineState.advance(). Keep the index and phase -/// as explicit loop-carried SSA values instead of recovering them with -/// division/remainder; the generated CuTe program then has the same circular -/// state transition as the Python DSL. -pub fn advancePipeline(b: *B, index: V, phase: V, comptime stages: usize) PipelinePosition { - const advanced = index.add(1); - const wraps = advanced.eq(stages); - const next_index = b.select(wraps, 0, advanced); - var next_phase = b.openIfElse(wraps, .{phase}); - { - next_phase.yieldThen(.{phase.bitXor(1)}); - } - { - next_phase.yieldElse(.{phase}); - } - return .{ .index = next_index, .phase = next_phase.results[0] }; -} - -pub fn peekBarrierIfRemaining(b: *B, base: V, count: V, limit: usize, state: PipelinePosition) V { - var peek = b.openIfElse(count.lt(limit), .{.i1}); - { - peek.yieldThen(.{b.mbarrierTryWait(barrierAt(b, base, state.index), state.phase, .{ .scope = .cta })}); - } - { - peek.yieldElse(.{b.constant(.i1, true)}); - } - return peek.results[0]; -} - -/// `cute.add_offset` infers the alignment a pointer keeps as the gcd of the -/// base alignment and the offset, and `cute.recast_iter` may not change it, so -/// the field's own alignment has to be what the offset leaves. -pub fn sharedPointerAt( - b: *B, - base: V, - comptime byte_offset: i64, - comptime dtype: cute.DType, - comptime alignment: u64, -) V { - const moved = comptime movedAlignment(byte_offset); - comptime std.debug.assert(moved >= alignment); - const byte_ptr = b.addOffset( - base, - b.makeIntTuple(.{byte_offset}), - b.ptrTy(.i8, .smem, moved) catch @panic("bad shared byte pointer"), - ); - return b.recastPointer(byte_ptr, dtype, .smem, moved); -} - -/// Alignment of `shared_base + byte_offset`, which the shared storage is -/// allocated with. -fn movedAlignment(comptime byte_offset: i64) u64 { - return offsetAlignment(1024, byte_offset, 1); -} - -/// `cute.add_offset` keeps the gcd of the operand alignment and the offset, -/// and the op's result type has to say so. -pub fn offsetAlignment(base_bytes: u64, offset_elements: i64, element_bytes: u64) u64 { - if (offset_elements == 0) return base_bytes; - const moved = @as(u64, @intCast(@abs(offset_elements))) * element_bytes; - return @max(std.math.gcd(base_bytes, moved), element_bytes); -} - -pub fn Tuple(comptime T: type, comptime len: usize) type { - return @Tuple(&@as([len]type, @splat(T))); -} - -pub fn arity(comptime T: type) usize { - return @typeInfo(T).@"struct".field_names.len; -} - -/// The MLIR types of a tuple of values, as `openWhile` expects them. -pub fn typesOf(values: anytype) Tuple(*const mlir.Type, arity(@TypeOf(values))) { - const len = comptime arity(@TypeOf(values)); - var out: Tuple(*const mlir.Type, len) = undefined; - inline for (0..len) |i| out[i] = values[i].type_(); - return out; -} - -/// Forward a whole loop-carried array, as `yieldBefore` expects a tuple. -pub fn tupleOf(array: anytype) Tuple(@TypeOf(array[0]), array.len) { - var out: Tuple(@TypeOf(array[0]), array.len) = undefined; - inline for (0..array.len) |i| out[i] = array[i]; - return out; -} - -pub fn fmt(b: *B, comptime format: []const u8, args: anytype) []const u8 { - return std.fmt.allocPrint(b.arena.allocator(), format, args) catch @panic("OOM"); -} - -pub fn view(b: *B, value: V) cute.View { - return .{ .inner = value.inner, .kernel = b }; -} - -/// Host launch of a boundary kernel: forward the buffer pointers, with PDL. -pub fn launchBoundary(b: *B, kernel_name: [:0]const u8, grid: [3]i32, args: anytype) cute.FinishError!void { - const one = b.constant(.i32, 1); - const config = b.makeLaunchConfig(.{ - .grid = .{ b.constant(.i32, grid[0]), b.constant(.i32, grid[1]), b.constant(.i32, grid[2]) }, - .block = .{ b.constant(.i32, 128), one, one }, - .dynamic_smem = b.kernelSmemSize(kernel_name), - .stream = b.cudaStream(), - .cluster = .{ one, one, one }, - .use_pdl = true, - }); - const launch = b.launchEx(kernel_name, config, args); - b.returnHostStatus(b.cudaResultStatus(launch)); -} diff --git a/zml/kernels/moe/cute/sm10x/moe.zig b/zml/kernels/moe/cute/sm10x/moe.zig deleted file mode 100644 index ff6edec2..00000000 --- a/zml/kernels/moe/cute/sm10x/moe.zig +++ /dev/null @@ -1,348 +0,0 @@ -//! SM100 CuTe MoE, MXFP8 x MXFP4 (the Zig port of the CuTe DSL `NativeMoE`) -//! and NVFP4 x NVFP4: routing, routed input quantization, persistent up GEMM -//! with a fused SwiGLU + quantization epilogue, persistent down GEMM storing -//! rows in route order, and the top-k reduction. MXFP4 applies routing weights -//! before the down projection, with the reference model's BF16 rounding. -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../../../../zml.zig"); -const RoutingWeightPlacement = @import("../../api.zig").RoutingWeightPlacement; -const triton_kernels = @import("../../triton/moe.zig"); -pub const mxfp4 = @import("mxfp4.zig"); -pub const mxfp4_persistent = @import("persistent_mxfp4.zig"); -pub const nvfp4 = @import("nvfp4.zig"); -pub const nvfp4_persistent = @import("persistent_nvfp4.zig"); -const reduce = @import("reduce.zig"); - -pub const Mxfp4Inputs = struct { - /// `[tokens, hidden]` BF16. - x: zml.Tensor, - /// `[tokens, topk]` FP32. - routing_weights: zml.Tensor, - /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved by 8. - w1: zml.Tensor, - /// `[experts, hidden / 128, 2 * intermediate]` i32 words of four E8M0 scales, - /// in MN-major 128x4 atoms, with rows interleaved like `w1`. - s1: zml.Tensor, - /// `[experts, hidden, intermediate / 2]` packed FP4. - w2: zml.Tensor, - /// `[experts, intermediate / 128, hidden]` i32 words in MN-major 128x4 atoms. - s2: zml.Tensor, - /// `[tokens, topk]` local expert ids. - ids: zml.Tensor, -}; - -/// The boundary quantizer handles four 128-column tiles per CTA. The GEMMs -/// support 128-column tiles and a fixed set of reduction tile counts. -pub fn validateMxfp4Shapes(hidden: i64, intermediate: i64) void { - stdx.debug.assert(hidden > 0 and @rem(hidden, 4 * 128) == 0, "cute backend requires hidden dimension to be a positive multiple of 512, got {}", .{hidden}); - stdx.debug.assert(intermediate > 0 and @rem(intermediate, 128) == 0, "cute backend requires intermediate dimension to be a positive multiple of 128, got {}", .{intermediate}); - stdx.debug.assert(supportedReductionTiles(@divTrunc(hidden, 128)), "cute backend does not support hidden reduction tile count {}, got hidden dimension {}", .{ @divTrunc(hidden, 128), hidden }); - stdx.debug.assert(supportedReductionTiles(@divTrunc(intermediate, 128)), "cute backend does not support intermediate reduction tile count {}, got intermediate dimension {}", .{ @divTrunc(intermediate, 128), intermediate }); -} - -fn supportedReductionTiles(tiles: i64) bool { - return switch (tiles) { - 16, 18, 32, 40 => true, - else => false, - }; -} - -/// Route each token/top-k pair to its own one-row group instead of grouping -/// routes by expert. A token's top-k experts are distinct, so for a single -/// token grouping cannot share any weight tile and only adds the routing -/// kernels. Python `grouped=False`. -fn isDirect(tokens: i64) bool { - return tokens == 1; -} - -/// GEMM tile N, which is also the routed expert-group size. Wider tiles -/// reuse each weight tile across more routes but pad more rows per expert, -/// so they only pay off once experts receive tens of routes. -fn tileN(tokens: i64, experts: i64, topk: i64) i64 { - // Direct routing keeps one row per group: the narrowest MMA wastes the - // least of each B tile and leaves room for a 12-stage A/B ring. - if (isDirect(tokens)) return 8; - if (tokens < 512) return 16; - // Prefill: each group re-reads the whole expert weight matrix, so widen - // the tile once experts hold enough routes to fill it. Half the routes - // are remote under expert parallelism, hence the conservative estimate. - // The thresholds are measured on the DeepSeek V4.1 expert geometry. - const routes_per_expert = @divTrunc(tokens * topk, 2 * experts); - if (routes_per_expert >= 128) return 128; - if (routes_per_expert >= 32) return 64; - return 32; -} - -/// One persistent CTA per SM. Each CTA needs the full 227 KiB of shared -/// memory, so launching more CTAs than SMs (e.g. 160 on a 152-SM GB300) -/// runs the surplus as a second wave and nearly doubles the GEMM time. -fn persistentCtas() i64 { - const platform = zml.Compiler.current().platform; - if (platform.devices.len == 0) return 148; - const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 148; - return if (attribute.int64 > 0) attribute.int64 else 148; -} - -// TODO(Corentin): settle on groups/capacity. - -/// Upper bound on the number of `group`-row expert groups for all routes. -fn computeCapacity(tokens: i64, topk: i64, experts: i64, group: i64) i64 { - return @divTrunc(tokens * topk + group - 1, group) + experts - 1; -} - -/// Where each route lives in the grouped GEMM operands. -const Routing = struct { - /// Expert groups, i.e. the GEMMs' L extent. - groups: i64, - /// Route -> physical routed row. Null for direct routing, where route - /// `r` is row `r`. - route_map: ?zml.Tensor, - /// Physical routed row -> route. Direct routing does not read it. - route_inverse: zml.Tensor, - /// `[group experts | group sizes | active group count]` for grouped - /// routing, `[route experts]` for direct routing. - schedule: zml.Tensor, -}; - -const atomic_route_threshold = 6 * 2048; - -fn route(tokens: i64, experts: i64, topk: i64, n: i64, direct: bool, ids: zml.Tensor) Routing { - const routes = tokens * topk; - const flat_ids = ids.convert(.i32).reshape(.{routes}); - if (direct) return .{ .groups = routes, .route_map = null, .route_inverse = flat_ids, .schedule = flat_ids }; - const capacity = computeCapacity(tokens, topk, experts, n); - - std.debug.assert(capacity > 0); - // `route` rescans every route once per expert: cheapest for decode, but - // quadratic. Past a few thousand tokens, count with atomics instead. - const r: struct { counts: zml.Tensor, pos: zml.Tensor } = if (routes >= atomic_route_threshold) blk: { - const counted = triton_kernels.RouteAtomic.Kernel.call( - .{ .ids = ids, .zeros = zml.Tensor.zeroes(.init(.{experts}, .i32)) }, - .{ .counts = .init(.{experts}, .i32), .pos = .init(.{routes}, .i32) }, - .{ - .cfg = .{ - .tokens = tokens, - .topk = topk, - .expert_offset = 0, - .experts = experts, - }, - .grid = .{ @intCast(@divTrunc(routes + triton_kernels.RouteAtomic.route_block - 1, triton_kernels.RouteAtomic.route_block)), 1, 1 }, - .num_warps = 4, - .num_stages = 1, - .output_operand_aliases = .{ .counts = .zeros }, - }, - ); - break :blk .{ .counts = counted.counts, .pos = counted.pos }; - } else blk: { - const scanned = triton_kernels.Route.Kernel.call( - .{ .ids = ids }, - .{ .counts = .init(.{experts}, .i32), .pos = .init(.{routes}, .i32) }, - .{ - .cfg = .{ .tokens = tokens, .topk = topk, .expert_offset = 0 }, - .grid = .{ @intCast(experts), 1, 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - break :blk .{ .counts = scanned.counts, .pos = scanned.pos }; - }; - - const s = triton_kernels.Schedule.Kernel.call( - .{ .ids = ids, .counts = r.counts, .pos = r.pos }, - .{ - .map = .init(.{capacity * n}, .i32), - .perm = .init(.{tokens * topk}, .i32), - .sched = .init(.{2 * capacity + 1}, .i32), - }, - .{ - .cfg = .{ - .tokens = tokens, - .topk = topk, - .expert_offset = 0, - .experts = experts, - .capacity = capacity, - .group = n, - }, - .grid = .{ @intCast(@divTrunc(tokens * topk + 127, 128)), 1, 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - - return .{ .groups = capacity, .route_map = s.perm, .route_inverse = s.map, .schedule = s.sched }; -} - -/// `[tokens, hidden]` BF16 output of the MXFP4 MoE layer. `tile_n` is the -/// GEMM tile N, for tuning; null selects it from the batch size. -pub fn forwardMxfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, swiglu_limit: ?f32, a: Mxfp4Inputs, tile_n: ?i64) zml.Tensor { - const n = tile_n orelse tileN(tokens, experts, topk); - const direct = isDirect(tokens); - const routing = route(tokens, experts, topk, n, direct, a.ids); - const routes = tokens * topk; - - const up_cfg: mxfp4_persistent.Config = .{ - .experts = experts, - .m = 2 * intermediate, - .n = n, - .k = hidden, - .groups = routing.groups, - .persistent_ctas = persistentCtas(), - .epilogue = .swiglu_mxfp8, - .routes = routes, - .direct = direct, - .swiglu_limit = swiglu_limit, - }; - var down_cfg = up_cfg; - down_cfg.m = hidden; - down_cfg.k = intermediate; - down_cfg.epilogue = .route_rows; - - const q1 = mxfp4.groupedQuantize(a.x, routing.route_map orelse routing.schedule, .{ - .tokens = tokens, - .hidden = hidden, - .topk = topk, - .group_size = up_cfg.groupRows(), - .capacity = routing.groups, - .direct = direct, - }); - const q2 = mxfp4_persistent.upQuantized(up_cfg, .{ - .weight = a.w1, - .schedule = routing.schedule, - .input_quant = q1.q, - .weight_scale = a.s1, - .input_scale = q1.s, - }, routing.route_inverse, a.routing_weights.reshape(.{routes})); - const down = mxfp4_persistent.downRows(down_cfg, .{ - .weight = a.w2, - .schedule = routing.schedule, - .input_quant = q2.q, - .weight_scale = a.s2, - .input_scale = q2.s, - }, routing.route_inverse); - return reduce.topk(down, routing.route_map orelse routing.schedule, .{ - .tokens = tokens, - .hidden = hidden, - .topk = topk, - .columns_per_cta = reduce.columnsPerCta(tokens, hidden), - }); -} - -pub const Nvfp4Inputs = struct { - /// `[tokens, hidden]` BF16. - x: zml.Tensor, - /// `[tokens, topk]` FP32. - routing_weights: zml.Tensor, - /// `[tokens, topk]` local expert ids, -1 for routes of other ranks. - ids: zml.Tensor, - /// `[experts, 2 * intermediate, hidden / 2]` packed FP4, gate/up rows interleaved. - w1: zml.Tensor, - /// `w1` E4M3 scales `[experts, 2 * intermediate, hidden / 16]` in the 128x4 layout. - s1: zml.Tensor, - /// `[experts, hidden, intermediate / 2]` packed FP4. - w2: zml.Tensor, - /// `w2` E4M3 scales `[experts, hidden, intermediate / 16]` in the 128x4 layout. - s2: zml.Tensor, - /// `[experts]` FP32 dequantization scale of each GEMM output, which folds - /// in the inverse of the GEMM input's global scale. - alpha1: zml.Tensor, - alpha2: zml.Tensor, - /// `[experts]` FP32 global scale each GEMM's input is multiplied by - /// before NVFP4 quantization. - act1: zml.Tensor, - act2: zml.Tensor, -}; - -pub const Nvfp4Options = struct { - swiglu_limit: ?f32 = null, - routing_weight_placement: RoutingWeightPlacement = .after_down, - /// GEMM tile N; null selects it from the batch size. - tile_n: ?i64 = null, -}; - -pub fn validateNvfp4Shapes(hidden: i64, intermediate: i64) void { - const k_tile = nvfp4_persistent.k_tile; - stdx.debug.assert(@rem(hidden, k_tile) == 0, "cute NVFP4 backend expects hidden to be a multiple of {}, got {}", .{ k_tile, hidden }); - stdx.debug.assert(@rem(intermediate, k_tile) == 0, "cute NVFP4 backend expects intermediate to be a multiple of {}, got {}", .{ k_tile, intermediate }); -} - -/// NVFP4 GEMM tile N: the expected routes of one expert, so that each -/// expert's weight tiles are read once per group instead of once per few rows. -fn nvfp4TileN(tokens: i64, experts: i64, topk: i64) i64 { - if (isDirect(tokens)) return 8; - const per_expert = std.math.divCeil(i64, tokens * topk, experts) catch unreachable; - return @intCast(std.math.clamp(std.math.ceilPowerOfTwoAssert(u64, @intCast(per_expert)), 16, 128)); -} - -/// `[tokens, hidden]` BF16 output of the NVFP4 MoE layer. -pub fn forwardNvfp4(tokens: i64, hidden: i64, intermediate: i64, experts: i64, topk: i64, a: Nvfp4Inputs, opts: Nvfp4Options) zml.Tensor { - validateNvfp4Shapes(hidden, intermediate); - const n = opts.tile_n orelse nvfp4TileN(tokens, experts, topk); - const direct = isDirect(tokens); - const routes = tokens * topk; - const flat_ids = a.ids.convert(.i32).reshape(.{routes}); - const routing = route(tokens, experts, topk, n, direct, a.ids); - const route_map = routing.route_map orelse flat_ids; - const weights = a.routing_weights.convert(.f32).reshape(.{routes}); - - const up_cfg: nvfp4_persistent.Config = .{ - .experts = experts, - .m = 2 * intermediate, - .n = n, - .k = hidden, - .groups = routing.groups, - .persistent_ctas = persistentCtas(), - .epilogue = .swiglu_nvfp4, - .routes = routes, - .direct = direct, - .swiglu_limit = opts.swiglu_limit, - .routing_weights = opts.routing_weight_placement == .before_down, - }; - var down_cfg = up_cfg; - down_cfg.m = hidden; - down_cfg.k = intermediate; - down_cfg.epilogue = .route_rows; - down_cfg.routing_weights = opts.routing_weight_placement == .after_down; - - const q1 = nvfp4.groupedQuantize(a.x, route_map, flat_ids, a.act1, .{ - .tokens = tokens, - .hidden = hidden, - .topk = topk, - .experts = experts, - .group_size = up_cfg.groupRows(), - .capacity = routing.groups, - .direct = direct, - }); - const q2 = nvfp4_persistent.upQuantized(up_cfg, .{ - .weight = a.w1, - .schedule = routing.schedule, - .input_quant = q1.q, - .weight_scale = a.s1.bitCast(.u8), - .input_scale = q1.s, - .alpha = a.alpha1, - }, routing.route_inverse, weights, a.act2); - const down = nvfp4_persistent.downRows(down_cfg, .{ - .weight = a.w2, - .schedule = routing.schedule, - .input_quant = q2.q, - .weight_scale = a.s2.bitCast(.u8), - .input_scale = q2.s, - .alpha = a.alpha2, - }, routing.route_inverse, weights); - return reduce.topk(down, route_map, .{ - .tokens = tokens, - .hidden = hidden, - .topk = topk, - .columns_per_cta = reduce.columnsPerCta(tokens, hidden), - }); -} - -test { - _ = mxfp4; - _ = mxfp4_persistent; - _ = nvfp4; - _ = nvfp4_persistent; - _ = reduce; -} diff --git a/zml/kernels/moe/cute/sm10x/mxfp4.zig b/zml/kernels/moe/cute/sm10x/mxfp4.zig deleted file mode 100644 index 29e68977..00000000 --- a/zml/kernels/moe/cute/sm10x/mxfp4.zig +++ /dev/null @@ -1,139 +0,0 @@ -//! CuTe input quantization kernel in front of the persistent MXFP4 GEMMs of -//! `persistent_mxfp4.zig`: routed MXFP8 input quantization. -const std = @import("std"); - -const zml = @import("../../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const launchBoundary = @import("common.zig").launchBoundary; - -pub const GroupedQuantConfig = struct { - tokens: i64, - hidden: i64, - topk: i64, - /// Rows stored per expert group. - group_size: i64, - capacity: i64, - /// Route `r` goes to row `r * group_size`; `route_map` is not read. - direct: bool = false, -}; - -// A program rather than a bare kernel so its launch carries the -// programmatic-dependent-launch attribute: it then starts while the previous -// kernel drains, and waits on `griddepcontrol` before its first read. A bare -// kernel is launched by XLA without that attribute. -const GroupedQuantize = cute.Program(GroupedQuantConfig, .{ - .name = "mxfp4_cute_grouped_quantize", - .inputs = &.{ "x", "route_map" }, - .outputs = &.{ "q", "s" }, - .run = groupedQuantizeProgram, -}); - -const quantize_kernel_name = "mxfp4_cute_grouped_quantize_kernel"; - -/// Quantize each token once, then scatter it to the physical rows assigned by -/// the routing. Scale bytes use the 128x4 layout consumed by the SM100 -/// block-scaled MMA scale-factor tensor map. -pub fn groupedQuantize(x: zml.Tensor, route_map: zml.Tensor, cfg: GroupedQuantConfig) struct { q: zml.Tensor, s: zml.Tensor } { - std.debug.assert(cfg.group_size == 8 or cfg.group_size == 16 or cfg.group_size == 32 or cfg.group_size == 64 or cfg.group_size == 128); - const result = GroupedQuantize.call( - .{ .x = x, .route_map = route_map }, - .{ - .q = .init(.{ cfg.capacity * cfg.group_size, cfg.hidden }, .f8e4m3fn), - .s = .init(.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 }, .u8), - }, - .{ .cfg = cfg }, - ); - return .{ .q = result.q, .s = result.s }; -} - -/// Python `prepare_input_vector`: each thread quantizes four consecutive -/// values and stores them as one packed word, so eight lanes cover one -/// 32-value MX block. -fn groupedQuantizeProgram(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { - b.beginFunction(quantize_kernel_name, .cuda_kernel); - try groupedQuantizeKernel(b, cfg); - b.endFunction(.{ 128, 1, 1 }); - - b.beginFunction(GroupedQuantize.name, .host); - // Host parameters carry the custom call's element types; the kernel reads - // four FP8 values per 32-bit word. - const host = try b.declareArgs(.{ - .x = .{ .ptr = cute.DType.bf16 }, - .route_map = .{ .ptr = cute.DType.i32 }, - .q = .{ .ptr = cute.DType.f8e4m3fn }, - .s = .{ .ptr = cute.DType.i8 }, - }); - try launchBoundary(b, quantize_kernel_name, .{ @intCast(@divExact(cfg.hidden, 4 * 128)), @intCast(cfg.tokens), 1 }, .{ - host.x, host.route_map, b.recastPointer(host.q, .i32, .gmem, 16), host.s, - }); - b.endFunction(null); -} - -fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { - std.debug.assert(@rem(cfg.hidden, 4 * 128) == 0); - const a = try b.declareArgs(.{ - .x = .{ .tensor = .{ .dtype = .bf16, .shape = &.{ cfg.tokens, cfg.hidden } } }, - .route_map = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, - // Four FP8 values per word. - .q = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.capacity * cfg.group_size, @divExact(cfg.hidden, 4) } } }, - .s = .{ .tensor = .{ .dtype = .i8, .shape = &.{ cfg.capacity, @divExact(cfg.hidden, 32) * 128 } } }, - }); - - const tid = b.threadIdx().x; - // This grid may have started before its producer finished: the routing and - // the previous layer write what it reads below. - b.waitForDependency(); - const token = b.blockIdx().y; - const word = b.blockIdx().x.mul(128).add(tid); - var x: [4]cute.Value = undefined; - for (&x, 0..) |*value, i| value.* = a.x.get(.{ token, word.mul(4).add(@as(i32, @intCast(i))) }).to(.f32); - - var maximum = x[0].abs().maximum(x[1].abs()).maximum(x[2].abs().maximum(x[3].abs())); - inline for (.{ 1, 2, 4 }) |offset| { - maximum = maximum.maximum(b.shuffleXor(maximum, offset)); - } - const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); - const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); - const inverse_scale = b.constant(.i32, 254).sub(exponent).shl(23).bitCast(.f32); - const packed_word = b.packFp8x4(x[0].mul(inverse_scale), x[1].mul(inverse_scale), x[2].mul(inverse_scale), x[3].mul(inverse_scale)); - - const scale_col = word.div(8); - for (0..@intCast(cfg.topk)) |slot| { - const route = token.mul(cfg.topk).add(slot); - const row = if (cfg.direct) route.mul(cfg.group_size) else a.route_map.get(.{route}); - // Routes of other expert-parallel ranks are not scheduled. - var scheduled = if (cfg.direct) null else b.openIf(row.ge(0)); - a.q.set(.{ row, word }, packed_word); - - var leader = b.openIf(tid.rem(8).eq(0)); - const scale_row = row.div(cfg.group_size); - const row_in_group = row.rem(cfg.group_size); - const swizzled_col = scale_col.div(4).mul(512) - .add(row_in_group.rem(32).mul(16)) - .add(row_in_group.div(32).mul(4)) - .add(scale_col.rem(4)); - a.s.set(.{ scale_row, swizzled_col }, exponent.to(.i8)); - leader.close(); - if (scheduled) |*s| s.close(); - } - // Only now are the rows this CTA owns complete: release the persistent - // GEMM, which waits on this signal before its first TMA read. - b.launchDependents(); -} - -test "MXFP4 CuTe input quantization emits for decode and prefill batches" { - inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { - const quant_ir = try GroupedQuantize.emit(std.testing.allocator, .{ - .tokens = tokens, - .hidden = 5120, - .topk = 6, - .group_size = 16, - .capacity = @divTrunc(tokens * 6 + 15, 16) + 191, - }); - defer std.testing.allocator.free(quant_ir); - try std.testing.expect(std.mem.indexOf(u8, quant_ir, "griddepcontrol.launch_dependents;") != null); - try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cuda.launch_ex") != null); - try std.testing.expect(std.mem.indexOf(u8, quant_ir, "cvt.rn.satfinite.e4m3x2.f32") != null); - } -} diff --git a/zml/kernels/moe/cute/sm10x/nvfp4.zig b/zml/kernels/moe/cute/sm10x/nvfp4.zig deleted file mode 100644 index 5afee090..00000000 --- a/zml/kernels/moe/cute/sm10x/nvfp4.zig +++ /dev/null @@ -1,176 +0,0 @@ -//! CuTe input quantization kernel in front of the persistent NVFP4 GEMMs of -//! `persistent_nvfp4.zig`: routed NVFP4 input quantization. -const std = @import("std"); - -const zml = @import("../../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const V = cute.Value; - -pub const GroupedQuantConfig = struct { - tokens: i64, - hidden: i64, - topk: i64, - experts: i64, - /// Rows stored per expert group. - group_size: i64, - capacity: i64, - /// Route `r` goes to row `r * group_size`; `route_map` is not read. - direct: bool = false, -}; - -const GroupedQuantize = cute.Program(GroupedQuantConfig, .{ - .name = "nvfp4_cute_grouped_quantize", - .inputs = &.{ "x", "route_map", "ids", "global_scale" }, - .outputs = &.{ "q", "s" }, - .run = groupedQuantizeProgram, -}); - -const quantize_kernel_name = "nvfp4_cute_grouped_quantize_kernel"; -/// Packed FP4 words (eight values) a CTA quantizes per token. -const words_per_cta = 128; - -/// NVFP4 E4M3 block scale of a block maximum: `scale = e4m3(global * -/// maximum / 6)` and the quantization multiplier `global / scale`, zero for an -/// all-zero block. -pub fn blockScale(b: *B, maximum: V, global: V) struct { bits: V, multiplier: V } { - const raw = global.mul(maximum.mul(1.0 / 6.0)).minimum(448.0); - const zero = b.constant(.f32, 0); - const bits = b.packFp8x4(raw, zero, zero, zero).bitAnd(0xff); - const scale = b.e4m3ToF32(bits); - const inverse = b.unaryF32("rcp.approx.ftz.f32", scale.mul(b.unaryF32("rcp.approx.ftz.f32", global))); - return .{ .bits = bits, .multiplier = b.select(scale.ne(0.0), inverse, zero) }; -} - -/// Offset of the scale of row `row` (within its 128-row group) and 16-wide -/// block `block` in the 128x4 tensor-core layout. -pub fn swizzledScaleOffset(row: V, block: V) V { - return block.div(4).mul(512) - .add(row.rem(32).mul(16)) - .add(row.div(32).mul(4)) - .add(block.rem(4)); -} - -/// Quantize each token once per routed expert and scatter it to the physical -/// rows assigned by the routing: packed FP4 `[capacity * group, hidden / 2]` -/// and E4M3 scales `[capacity, hidden / 16 * 128]` in the 128x4 layout. -/// `ids` are the `[tokens * topk]` local experts and `global_scale` the -/// `[experts]` FP32 activation global scales. -pub fn groupedQuantize(x: zml.Tensor, route_map: zml.Tensor, ids: zml.Tensor, global_scale: zml.Tensor, cfg: GroupedQuantConfig) struct { q: zml.Tensor, s: zml.Tensor } { - std.debug.assert(@rem(cfg.hidden, 64) == 0); - const result = GroupedQuantize.call( - .{ .x = x, .route_map = route_map, .ids = ids, .global_scale = global_scale }, - .{ - .q = .init(.{ cfg.capacity * cfg.group_size, @divExact(cfg.hidden, 2) }, .u8), - .s = .init(.{ cfg.capacity, @divExact(cfg.hidden, 16) * 128 }, .u8), - }, - .{ .cfg = cfg }, - ); - return .{ .q = result.q, .s = result.s }; -} - -fn groupedQuantizeProgram(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { - b.beginFunction(quantize_kernel_name, .cuda_kernel); - try groupedQuantizeKernel(b, cfg); - b.endFunction(.{ words_per_cta, 1, 1 }); - - b.beginFunction(GroupedQuantize.name, .host); - const host = try b.declareArgs(.{ - .x = .{ .ptr = cute.DType.bf16 }, - .route_map = .{ .ptr = cute.DType.i32 }, - .ids = .{ .ptr = cute.DType.i32 }, - .global_scale = .{ .ptr = cute.DType.f32 }, - .q = .{ .ptr = cute.DType.i8 }, - .s = .{ .ptr = cute.DType.i8 }, - }); - const one = b.constant(.i32, 1); - const words = @divExact(cfg.hidden, 8); - const config = b.makeLaunchConfig(.{ - .grid = .{ b.constant(.i32, @as(i32, @intCast(@divTrunc(words + words_per_cta - 1, words_per_cta)))), b.constant(.i32, @as(i32, @intCast(cfg.tokens))), one }, - .block = .{ b.constant(.i32, words_per_cta), one, one }, - .dynamic_smem = b.kernelSmemSize(quantize_kernel_name), - .stream = b.cudaStream(), - .cluster = .{ one, one, one }, - .use_pdl = true, - }); - const launch = b.launchEx(quantize_kernel_name, config, .{ - b.recastPointer(host.x, .i32, .gmem, 16), host.route_map, host.ids, host.global_scale, b.recastPointer(host.q, .i32, .gmem, 16), host.s, - }); - b.returnHostStatus(b.cudaResultStatus(launch)); - b.endFunction(null); -} - -/// Each thread quantizes eight consecutive values into one packed word, so -/// two lanes cover one 16-value NVFP4 block. -fn groupedQuantizeKernel(b: *B, cfg: GroupedQuantConfig) cute.FinishError!void { - const words = @divExact(cfg.hidden, 8); - const routes = cfg.tokens * cfg.topk; - const a = try b.declareArgs(.{ - // Two BF16 values per word. - .x = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens, @divExact(cfg.hidden, 2) } } }, - .route_map = .{ .tensor = .{ .dtype = .i32, .shape = &.{routes} } }, - .ids = .{ .tensor = .{ .dtype = .i32, .shape = &.{routes} } }, - .global_scale = .{ .tensor = .{ .dtype = .f32, .shape = &.{cfg.experts} } }, - // Eight FP4 values per word. - .q = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.capacity * cfg.group_size, words } } }, - .s = .{ .tensor = .{ .dtype = .i8, .shape = &.{ cfg.capacity, @divExact(cfg.hidden, 16) * 128 } } }, - }); - - const tid = b.threadIdx().x; - // The routing and the previous layer write what this grid reads. - b.waitForDependency(); - const token = b.blockIdx().y; - const word = b.blockIdx().x.mul(words_per_cta).add(tid); - // Out-of-range lanes still take part in the pair reduction. - const in_range = word.lt(@as(i32, @intCast(words))); - const safe_word = b.select(in_range, word, b.constant(.i32, 0)); - var x: [8]V = undefined; - for (0..4) |i| { - const pair = a.x.get(.{ token, safe_word.mul(4).add(@as(i32, @intCast(i))) }); - x[2 * i] = pair.shl(16).bitCast(.f32); - x[2 * i + 1] = pair.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32); - } - var maximum = x[0].abs(); - for (x[1..]) |value| maximum = maximum.maximum(value.abs()); - maximum = maximum.maximum(b.shuffleXor(maximum, 1)); - - const block = safe_word.div(2); - var in_range_scope = b.openIf(in_range); - for (0..@intCast(cfg.topk)) |slot| { - const route = token.mul(cfg.topk).add(slot); - const row = if (cfg.direct) route.mul(cfg.group_size) else a.route_map.get(.{route}); - // Routes of other expert-parallel ranks are not scheduled. - const expert = a.ids.get(.{route}); - var scheduled = b.openIf(row.ge(0).bitAnd(expert.ge(0))); - const scale = blockScale(b, maximum, a.global_scale.get(.{expert})); - var quantized: [8]V = undefined; - for (&quantized, x) |*q, value| q.* = value.mul(scale.multiplier); - a.q.set(.{ row, safe_word }, b.packFp4x8(quantized)); - - var leader = b.openIf(tid.rem(2).eq(0)); - a.s.set(.{ row.div(cfg.group_size), swizzledScaleOffset(row.rem(cfg.group_size), block) }, scale.bits.to(.i8)); - leader.close(); - scheduled.close(); - } - in_range_scope.close(); - // The rows this CTA owns are complete: release the persistent GEMM. - b.launchDependents(); -} - -test "NVFP4 CuTe quantization emits for decode and prefill batches" { - inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { - const ir = try GroupedQuantize.emit(std.testing.allocator, .{ - .tokens = tokens, - .hidden = 5120, - .topk = 6, - .experts = 192, - .group_size = 16, - .capacity = @divTrunc(tokens * 6 + 15, 16) + 191, - .direct = tokens == 1, - }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "griddepcontrol.launch_dependents;") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cvt.rn.satfinite.e2m1x2.f32") != null); - } -} diff --git a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig deleted file mode 100644 index 30062a3a..00000000 --- a/zml/kernels/moe/cute/sm10x/persistent_mxfp4.zig +++ /dev/null @@ -1,1923 +0,0 @@ -//! Source-level SM100 MXFP4 grouped GEMM. -//! -//! This file is the Zig CuTe-DSL counterpart of the CuTe DSL -//! `Sm100BlockScaledPersistentDenseGemmKernel` with its fused MoE epilogues. -//! It emits a complete CuTe program: the nested `cuda.kernel` consumes the -//! objects constructed by the public host `func.func`, and the host function -//! creates the block-scaled MMA objects, four TMA tensor maps, persistent -//! scheduler parameters, and the PDL-enabled launch. -//! -//! One CTA computes a 128xN output tile of one expert group: warp 5 drives -//! the TMA ring, warp 4 issues the S2T scale copies and UMMA, and warps 0-3 -//! drain the accumulator from tensor memory into one of the two epilogues. -const std = @import("std"); -const mlir = @import("mlir"); -const nvvm = @import("mlir/dialects/nvvm"); - -const zml = @import("../../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const V = cute.Value; -const common = @import("common.zig"); -const typed = common.typed; -const gridTyped = common.gridTyped; -const barrierAt = common.barrierAt; -const PipelinePosition = common.PipelinePosition; -const PersistentWork = common.PersistentWork; -const DirectRoutes = common.DirectRoutes; -const persistentWorkAt = common.persistentWorkAt; -const advancePersistentWork = common.advancePersistentWork; -const advancePipeline = common.advancePipeline; -const peekBarrierIfRemaining = common.peekBarrierIfRemaining; -const sharedPointerAt = common.sharedPointerAt; -const offsetAlignment = common.offsetAlignment; -const typesOf = common.typesOf; -const tupleOf = common.tupleOf; -const fmt = common.fmt; -const view = common.view; - -const accumulator_stages = 2; - -/// Shared-memory plan of the Python kernel for one tile width, copied from -/// its generated `SharedStorage` (`smem.struct_fields`). The A/B ring depth -/// is whatever fits next to the two C epilogue stages; the fused epilogues -/// only use the start of `sC` as scratch. -pub const Tile = struct { - n: i64, - stages: usize, - smem_bytes: i64, - tmem_columns: u16, - empty_barriers: i64, - accumulator_full: i64, - accumulator_empty: i64, - tmem_dealloc: i64, - tmem_holding: i64, - c: i64, - a: i64, - b: i64, - sfa: i64, - sfb: i64, - /// Bytes reserved at `c`. Only the epilogue scratch is used; the wide - /// prefill tiles keep just that instead of two full C stages. - c_bytes: ?i64 = null, - - pub fn of(comptime n: i64) Tile { - return switch (n) { - 8 => .{ - .n = 8, - .stages = 12, - .smem_bytes = 230400, - .tmem_columns = 64, - .empty_barriers = 96, - .accumulator_full = 192, - .accumulator_empty = 208, - .tmem_dealloc = 224, - .tmem_holding = 232, - .c = 1024, - .a = 9216, - .b = 205824, - .sfa = 218112, - .sfb = 224256, - }, - 16 => .{ - .n = 16, - .stages = 11, - .smem_bytes = 232448, - .tmem_columns = 64, - .empty_barriers = 88, - .accumulator_full = 176, - .accumulator_empty = 192, - .tmem_dealloc = 208, - .tmem_holding = 216, - .c = 1024, - .a = 17408, - .b = 197632, - .sfa = 220160, - .sfb = 226304, - }, - 32 => .{ - .n = 32, - .stages = 9, - .smem_bytes = 228352, - .tmem_columns = 128, - .empty_barriers = 72, - .accumulator_full = 144, - .accumulator_empty = 160, - .tmem_dealloc = 176, - .tmem_holding = 184, - .c = 1024, - .a = 33792, - .b = 181248, - .sfa = 218112, - .sfb = 223232, - }, - // Prefill tiles. One expert weight tile feeds 64 or 128 routed - // rows instead of 32, which is what keeps large batches from - // re-reading the expert weights once per group. They hold only - // the epilogue scratch at `c`, so the A/B ring stays deep. - 64 => .{ - .n = 64, - .stages = 8, - .smem_bytes = 206848, - .tmem_columns = 256, - .empty_barriers = 64, - .accumulator_full = 128, - .accumulator_empty = 144, - .tmem_dealloc = 160, - .tmem_holding = 168, - .c = 1024, - .c_bytes = 512, - .a = 2048, - .sfa = 133120, - .sfb = 137216, - .b = 141312, - }, - 128 => .{ - .n = 128, - .stages = 6, - .smem_bytes = 204800, - .tmem_columns = 512, - .empty_barriers = 48, - .accumulator_full = 96, - .accumulator_empty = 112, - .tmem_dealloc = 128, - .tmem_holding = 136, - .c = 1024, - .c_bytes = 512, - .a = 2048, - .b = 100352, - .sfa = 198656, - .sfb = 201728, - }, - else => @compileError("unsupported MXFP4 tile N"), - }; - } - - /// Routed rows an epilogue pass handles: one per warp lane. - pub fn chunkRows(self: Tile) i64 { - return @min(self.n, 32); - } - - /// Epilogue passes over the tile width. - pub fn chunks(self: Tile) usize { - return @intCast(@divExact(self.n, self.chunkRows())); - } - - const a_stage_bytes = 128 * 128; - const sf_stage_bytes = 512; - - fn bStageBytes(self: Tile) i64 { - return self.n * 128; - } - - fn cStageElements(self: Tile) i64 { - return 128 * self.n; - } - - /// TMA bytes landing in one A/B stage: unpacked FP4 A is counted as - /// 128x128 nibbles, then the FP8 B tile and both 512-byte scale atoms. - fn transactionBytes(self: Tile) u32 { - return @intCast(8192 + self.bStageBytes() + 2 * sf_stage_bytes); - } - - fn fields(self: Tile, b: *B) []const []const u8 { - const arena = b.arena.allocator(); - const s: i64 = @intCast(self.stages); - const out = arena.alloc([]const u8, 11) catch @panic("OOM"); - const Field = struct { []const u8, i64, i64 }; - const list = [_]Field{ - .{ "ab_full_mbar_ptr", 8 * s, 0 }, - .{ "ab_empty_mbar_ptr", 8 * s, self.empty_barriers }, - .{ "acc_full_mbar_ptr", 16, self.accumulator_full }, - .{ "acc_empty_mbar_ptr", 16, self.accumulator_empty }, - .{ "tmem_dealloc_mbar", 8, self.tmem_dealloc }, - .{ "tmem_holding_buf", 4, self.tmem_holding }, - .{ "sC", self.c_bytes orelse accumulator_stages * self.cStageElements() * 4, self.c }, - .{ "sA", s * a_stage_bytes, self.a }, - .{ "sB", s * self.bStageBytes(), self.b }, - .{ "sSFA", s * sf_stage_bytes, self.sfa }, - .{ "sSFB", s * sf_stage_bytes, self.sfb }, - }; - for (list, out) |field, *text| { - text.* = std.fmt.allocPrint(arena, "{s}:{d}:{d}", .{ field[0], field[1], field[2] }) catch @panic("OOM"); - } - return out; - } -}; - -const StageStorage = struct { - a: V, - b: V, - sfa: V, - sfb: V, - c: V, - load_barriers: V, - mma_barriers: V, - accumulator_full_barriers: V, - accumulator_empty_barriers: V, -}; - -const StageBuffers = struct { - a_destination: cute.View, - b_destination: cute.View, - sfa_destination: cute.View, - sfb_destination: cute.View, - load_barrier: V, - mma_barrier: V, -}; - -const StagedOperands = struct { - a: cute.View, - b: cute.View, - sfa: cute.View, - sfb: cute.View, -}; - -/// What the epilogue warps do with each finished 128xN accumulator tile. -pub const Epilogue = enum { - /// Up projection: clamped SwiGLU of the interleaved gate/up rows, BF16 - /// rounding and the routing weight as in the reference model, then MXFP8 - /// quantization straight into the down GEMM's input and 128x4 scale - /// layouts. Python `fused=1`. - swiglu_mxfp8, - /// Down projection: BF16 rows stored in route order `[routes, m]`, - /// skipping padding rows. Python `fused=2`. BF16 halves the traffic of - /// this store and of the top-k reduction that reads it back; the routed - /// partial sums are rounded exactly like the reference finalize. - route_rows, -}; - -pub const Config = struct { - experts: i64, - /// GEMM output rows (4608 for gate/up, 5120 for down). - m: i64, - /// Tile N: rows in one routed expert group. - n: i64 = 16, - /// Reduction dimension (5120 for gate/up, 2304 for down). - k: i64, - groups: i64, - /// Resident CTAs of the static persistent tile scheduler: one per SM. - persistent_ctas: i64 = 148, - epilogue: Epilogue, - /// Token/top-k routes; sizes the route-indexed epilogue operands. - routes: i64, - /// Clamp applied to both halves of the fused SwiGLU. Emitted as an IR - /// constant, so a threaded value costs exactly what a literal did. - swiglu_limit: ?f32 = 10.0, - /// Ungrouped routing: group `g` is route `g` and holds one live row. - /// `schedule` is then just the `[routes]` expert ids; its size and - /// active-count sections and `route_inverse` are not read. - direct: bool = false, - - /// Rows stored per group in the routed operands: one whole N tile, of - /// which direct routing fills only the first. A tensor map whose row mode - /// had extent one would lose that mode altogether — the TMA atom infers a - /// zero stride for it — and every group would then read the same row. - pub fn groupRows(self: Config) i64 { - return self.n; - } - - pub fn routedRows(self: Config) i64 { - return self.groups * self.groupRows(); - } -}; - -/// `schedule` is the routing schedule `[group experts | group sizes | active -/// group count]`; the kernel reads the first and last sections in place. -const inputs = [_][:0]const u8{ - "weight", - "schedule", - "input_quant", - "weight_scale", - "input_scale", -}; - -/// `route_inverse` maps each physical routed row to its token/top-k route; -/// entries of padding rows are never read. -const UpQuantized = cute.Program(Config, .{ - .name = "zml_mxfp4_up_swiglu_zig", - .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights" }), - .outputs = &.{ "q", "s" }, - .run = buildProgram, -}); - -const DownRows = cute.Program(Config, .{ - .name = "zml_mxfp4_down_rows_zig", - .inputs = &(inputs ++ [_][:0]const u8{"route_inverse"}), - .outputs = &.{"output"}, - .run = buildProgram, -}); - -pub const Inputs = struct { - weight: zml.Tensor, - schedule: zml.Tensor, - input_quant: zml.Tensor, - weight_scale: zml.Tensor, - input_scale: zml.Tensor, -}; - -/// Up projection with the SwiGLU + MXFP8 epilogue: the routed FP8 rows -/// `[groups * n, m / 2]` and their 128x4 scales `[groups, m / 64 * 128]`, -/// ready for the down GEMM. `routing_weights` is `[routes]` FP32. -pub fn upQuantized(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor) struct { q: zml.Tensor, s: zml.Tensor } { - std.debug.assert(cfg.epilogue == .swiglu_mxfp8); - const activations = @divExact(cfg.m, 2); - const result = UpQuantized.call(.{ - .weight = a.weight, - .schedule = a.schedule, - .input_quant = a.input_quant, - .weight_scale = a.weight_scale, - .input_scale = a.input_scale, - .route_inverse = route_inverse, - .routing_weights = routing_weights, - }, .{ - .q = .init(.{ cfg.routedRows(), activations }, .f8e4m3fn), - .s = .init(.{ cfg.groups, @divExact(activations, 32) * 128 }, .u8), - }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }); - return .{ .q = result.q, .s = result.s }; -} - -/// Down projection storing FP32 `[routes, m]` in token/top-k order. -pub fn downRows(cfg: Config, a: Inputs, route_inverse: zml.Tensor) zml.Tensor { - std.debug.assert(cfg.epilogue == .route_rows); - return DownRows.call(.{ - .weight = a.weight, - .schedule = a.schedule, - .input_quant = a.input_quant, - .weight_scale = a.weight_scale, - .input_scale = a.input_scale, - .route_inverse = route_inverse, - }, .{ .output = .init(.{ cfg.routes, cfg.m }, .bf16) }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }).output; -} - -/// The two projections share this kernel but get distinct symbols, so a -/// profile shows them as separate rows instead of one merged entry. -fn deviceName(comptime epilogue: Epilogue) [:0]const u8 { - return switch (epilogue) { - .swiglu_mxfp8 => "mxfp4_sm100_persistent_grouped_gemm_up_swiglu", - .route_rows => "mxfp4_sm100_persistent_grouped_gemm_down_rows", - }; -} - -fn buildProgram(b: *B, cfg: Config) cute.FinishError!void { - return switch (cfg.n) { - inline 8, 16, 32, 64, 128 => |n| switch (cfg.epilogue) { - inline else => |epilogue| buildTiledProgram(b, cfg, comptime Tile.of(n), epilogue), - }, - else => std.debug.panic("unsupported MXFP4 tile N {d}", .{cfg.n}), - }; -} - -/// Types and layouts shared by the device ABI and the host launch. -fn Layouts(comptime tile: Tile) type { - return struct { - const n = tile.n; - const stages = tile.stages; - const ab_swizzle: B.Swizzle = .{ .bits = 3, .base = 4, .shift = 3 }; - - fn aCoord(b: *B, cfg: Config) cute.LayoutSpec { - return b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); - } - fn bCoord(b: *B, cfg: Config) cute.LayoutSpec { - return b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); - } - fn sfCoord(b: *B) cute.LayoutSpec { - return b.layoutSpec( - .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, - .{ - .{ .{ b.basis(8, 1, .{0}), b.basis(2, 1, .{0}) }, b.basis(1, 1, .{1}) }, - .{ .{ 0, b.basis(1, 2, .{0}) }, b.basis(1, 1, .{2}) }, - .{ 0, b.basis(1, 1, .{3}) }, - }, - ); - } - fn aBasis(b: *B) cute.LayoutSpec { - return b.layoutSpec(.{ 128, 128, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); - } - fn bBasis(b: *B) cute.LayoutSpec { - return b.layoutSpec(.{ 128, n, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); - } - fn sfBasis(b: *B) cute.LayoutSpec { - return b.layoutSpec( - .{ .{ 2, 4, 32 }, 1, 1, 1 }, - .{ .{ b.basis(1, 1, .{ 1, 0, 1 }), b.basis(1, 1, .{ 0, 0, 1 }), b.basis(1, 1, .{ 0, 0, 0 }) }, b.basis(1, 1, .{ 0, 1 }), b.basis(1, 1, .{ 1, 1 }), b.basis(1, 1, .{ 2, 1 }) }, - ); - } - fn tmaA(b: *B, cfg: Config) cute.TmaConfig { - return .{ .dtype = .f4e2m1fn, .copy_bits = 65536, .global_basis = aBasis(b), .coordinate_layout = aCoord(b, cfg), .coordinate_rank = 3, .format = .u4_unpack_u8 }; - } - fn tmaB(b: *B, cfg: Config) cute.TmaConfig { - return .{ .dtype = .f8e4m3fn, .copy_bits = @intCast(tile.bStageBytes() * 8), .global_basis = bBasis(b), .coordinate_layout = bCoord(b, cfg), .coordinate_rank = 3 }; - } - fn tmaSf(b: *B) cute.TmaConfig { - return .{ .dtype = .f8e8m0fnu, .copy_bits = 4096, .global_basis = sfBasis(b), .coordinate_layout = sfCoord(b), .coordinate_rank = 4, .format = .u16 }; - } - // Shared-memory layouts, in the same element units as Python. - fn aSmem(b: *B, comptime staged: bool) cute.LayoutSpec { - return if (staged) - b.layoutSpec(.{ .{ 128, 32 }, 1, 4, stages }, .{ .{ 128, 1 }, 0, 32, Tile.a_stage_bytes }) - else - b.layoutSpec(.{ .{ 128, 32 }, 1, 4 }, .{ .{ 128, 1 }, 0, 32 }); - } - fn bSmem(b: *B, comptime staged: bool) cute.LayoutSpec { - return if (staged) - b.layoutSpec(.{ .{ n, 32 }, 1, 4, stages }, .{ .{ 128, 1 }, 0, 32, tile.bStageBytes() }) - else - b.layoutSpec(.{ .{ n, 32 }, 1, 4 }, .{ .{ 128, 1 }, 0, 32 }); - } - fn sfSmem(b: *B, comptime staged: bool) cute.LayoutSpec { - return if (staged) - b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 32, 1 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1, Tile.sf_stage_bytes }) - else - b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 32, 1 } }, 1, 4 }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1 }); - } - }; -} - -/// Device memrefs of the route-indexed epilogue operands. -const EpilogueTypes = struct { - sizes: *const mlir.Type, - inverse: *const mlir.Type, - weights: *const mlir.Type, - q: *const mlir.Type, - s: *const mlir.Type, - rows: *const mlir.Type, - - fn of(b: *B, cfg: Config) EpilogueTypes { - const activations = @divExact(cfg.m, 2); - return .{ - // Group sizes follow the group experts inside `schedule`. - .sizes = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(cfg.groups, 1))), - .inverse = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.routedRows(), 1))), - .weights = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(@max(cfg.routes, 1), 1))), - .q = b.memrefType(.f8e4m3fn, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.routedRows(), activations }, .{ activations, 1 }))), - .s = b.memrefType(.i8, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.groups, @divExact(activations, 32) * 128 }, .{ @divExact(activations, 32) * 128, 1 }))), - .rows = b.memrefType(.bf16, .gmem, 16, b.layoutType(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), - }; - } -}; - -fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { - @setEvalBranchQuota(20_000); - std.debug.assert(@rem(cfg.m, 128) == 0); - std.debug.assert(@rem(cfg.k, 128) == 0); - const L = Layouts(tile); - const public_name = b.name; - - const ids_memref = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.groups, 1))); - // The active group count is the last schedule entry, 4-byte aligned. - const active_memref = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(1, 1))); - const et = EpilogueTypes.of(b, cfg); - const i32_type = cute.DType.i32.toMlir(b.ctx); - const mma_type = b.blockScaledMmaType(.{ .n = @intCast(tile.n) }); - const mma_sfb_type = b.blockScaledMmaType(.{ .n = 128 }); - - // Device ABI. The host function below constructs every non-buffer value. - b.beginFunction(deviceName(epilogue), .cuda_kernel); - const d = try b.declareArgs(switch (epilogue) { - .swiglu_mxfp8 => .{ - .expert_ids = typed(ids_memref), - .active_count = typed(active_memref), - .mma = typed(mma_type), - .mma_sfb = typed(mma_sfb_type), - .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), - .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), - .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), - .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), - .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), - .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), - .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), - .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), - .group_sizes = typed(et.sizes), - .route_inverse = typed(et.inverse), - .routing_weights = typed(et.weights), - .q = typed(et.q), - .s = typed(et.s), - .tiles_m = typed(i32_type), - }, - .route_rows => .{ - .expert_ids = typed(ids_memref), - .active_count = typed(active_memref), - .mma = typed(mma_type), - .mma_sfb = typed(mma_sfb_type), - .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), - .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), - .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), - .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), - .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), - .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), - .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), - .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), - .group_sizes = typed(et.sizes), - .route_inverse = typed(et.inverse), - .rows = typed(et.rows), - .tiles_m = typed(i32_type), - }, - }); - b.setFunctionAttribute(nvvm.minctasm(b.ctx, 1)); - b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 2))); - b.setFunctionAttribute(.named(b.ctx, "smem.hint_smem_base_uniform", .unit(b.ctx))); - b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{ - .named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin()), - .named(b.ctx, "non_portable_cluster_size_allowed", .int(b.ctx, .i32, 1)), - }))); - - // Prefetch every tensor-map descriptor before the specialized warps begin - // the persistent pipeline. This is the work PDL buys us: it runs while the - // producing grid is still writing the operands. - const warp = b.makeWarpUniform(b.makeWarpUniform(b.threadIdx().x.div(32))); - var tma_warp = b.openIf(warp.eq(5)); - { - inline for (.{ d.tma_a, d.tma_b, d.tma_sfa, d.tma_sfb }) |value| { - const descriptor: cute.Atom = .{ .inner = value.inner, .kernel = b }; - b.prefetchTmaDesc(descriptor); - } - tma_warp.close(); - } - // Everything below reads the schedule and the quantized activations the - // preceding grid produces, so the whole block waits here. - b.waitForDependency(); - buildUmmaDevice(b, cfg, tile, epilogue, d); - // The epilogue has written every row this CTA owns. - b.launchDependents(); - b.endFunction(.{ 192, 1, 1 }); - - // Public host ABI: the XLA operands, the outputs, then the problem shape. - // XLA owns packed storage as signless bytes. Recast only inside the host - // program when constructing typed tensor maps. - b.beginFunction(public_name, .host); - const operands = .{ - .weight = .{ .ptr = cute.DType.i8 }, - .schedule = .{ .ptr = cute.DType.i32 }, - .input_quant = .{ .ptr = cute.DType.f8e4m3fn }, - .weight_scale = .{ .ptr = cute.DType.i32 }, - .input_scale = .{ .ptr = cute.DType.i8 }, - }; - const shape = .{ - .problem_m = typed(i32_type), - .problem_n = typed(i32_type), - .problem_k = typed(i32_type), - .problem_l = typed(i32_type), - .expert_count = typed(i32_type), - }; - _ = try b.declareArgs(switch (epilogue) { - .swiglu_mxfp8 => .{ - .weight = operands.weight, - .schedule = operands.schedule, - .input_quant = operands.input_quant, - .weight_scale = operands.weight_scale, - .input_scale = operands.input_scale, - .route_inverse = .{ .ptr = cute.DType.i32 }, - .routing_weights = .{ .ptr = cute.DType.f32 }, - .q = .{ .ptr = cute.DType.f8e4m3fn }, - .s = .{ .ptr = cute.DType.i8 }, - .problem_m = shape.problem_m, - .problem_n = shape.problem_n, - .problem_k = shape.problem_k, - .problem_l = shape.problem_l, - .expert_count = shape.expert_count, - }, - .route_rows => .{ - .weight = operands.weight, - .schedule = operands.schedule, - .input_quant = operands.input_quant, - .weight_scale = operands.weight_scale, - .input_scale = operands.input_scale, - .route_inverse = .{ .ptr = cute.DType.i32 }, - .output = .{ .ptr = cute.DType.bf16 }, - .problem_m = shape.problem_m, - .problem_n = shape.problem_n, - .problem_k = shape.problem_k, - .problem_l = shape.problem_l, - .expert_count = shape.expert_count, - }, - }); - try buildHostLaunch(b, cfg, tile, epilogue); - b.endFunction(null); -} - -/// Host argument positions: the five mainloop operands, the epilogue's -/// extra inputs and outputs, then the five problem-shape scalars. -fn HostArgs(comptime epilogue: Epilogue) type { - return struct { - const weight = 0; - const schedule = 1; - const input_quant = 2; - const weight_scale = 3; - const input_scale = 4; - const route_inverse = 5; - const routing_weights = 6; - const output = switch (epilogue) { - .swiglu_mxfp8 => 7, - .route_rows => 6, - }; - const scales = 8; - const problem_m = switch (epilogue) { - .swiglu_mxfp8 => 9, - .route_rows => 7, - }; - const problem_n = problem_m + 1; - const problem_k = problem_m + 2; - const problem_l = problem_m + 3; - const expert_count = problem_m + 4; - }; -} - -/// Complete staged shared-memory tensors, as Python's `PipelineTmaUmma` -/// consumers see them. Per-stage views are slices of these. -fn stagedOperands(b: *B, comptime tile: Tile, storage: StageStorage) StagedOperands { - const L = Layouts(tile); - const a_full = b.makeView( - b.recastSwizzledPointer(storage.a, .i8, .smem, 1024, L.ab_swizzle), - b.staticLayout(L.aSmem(b, true)), - b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad A shared pointer"), b.layoutType(L.aSmem(b, true))), - ); - const b_full = b.makeView( - b.recastSwizzledPointer(storage.b, .f8e4m3fn, .smem, 1024, L.ab_swizzle), - b.staticLayout(L.bSmem(b, true)), - b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad B shared pointer"), b.layoutType(L.bSmem(b, true))), - ); - const sfa_full = b.makeView(storage.sfa, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); - const sfb_full = b.makeView(storage.sfb, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); - return .{ .a = a_full, .b = b_full, .sfa = sfa_full, .sfb = sfb_full }; -} - -fn stageBuffersAt(b: *B, comptime tile: Tile, storage: StageStorage, stage_index: V) StageBuffers { - const L = Layouts(tile); - // Keep the shared operands as staged tensors and select the current stage - // with CuTe slicing. This is the representation used by the Python - // PipelineTmaUmma implementation; rebuilding a memref from a runtime - // pointer loses the staged descriptor semantics at ring wrap. - const full = stagedOperands(b, tile, storage); - const all = cute.AlgebraToken.all; - const a = b.slice(full.a, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad A stage pointer"), b.layoutType(L.aSmem(b, false)))); - const rhs = b.slice(full.b, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad B stage pointer"), b.layoutType(L.bSmem(b, false)))); - // One scale stage is 512 bytes, so slicing a stage out of the ring keeps - // that alignment, not the array's. - const sf_stage_align = Tile.sf_stage_bytes; - const sfa = b.slice(full.sfa, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); - const sfb = b.slice(full.sfb, .{ all, all, all, stage_index }, b.memrefType(.f8e8m0fnu, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); - - // `tma_partition` preserves a singleton CTA mode. Keep it in the copy - // views: it is part of the executable TMA atom's expected algebra even - // for a one-CTA cluster. - const a_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.a_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); - const b_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ tile.bStageBytes(), 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); - const sf_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.sf_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); - - return .{ - .a_destination = b.withLayout(a, a_partition_layout), - .b_destination = b.withLayout(rhs, b_partition_layout), - .sfa_destination = b.withLayout(sfa, sf_partition_layout), - .sfb_destination = b.withLayout(sfb, sf_partition_layout), - .load_barrier = barrierAt(b, storage.load_barriers, stage_index), - .mma_barrier = barrierAt(b, storage.mma_barriers, stage_index), - }; -} - -/// Issue the four transactions that fill one mainloop stage. The caller is -/// already restricted to the TMA warp; one elected lane owns the mbarrier -/// arrival. -fn issueTmaStage( - b: *B, - comptime tile: Tile, - comptime reduction_tiles: usize, - partitioned_a_target: cute.View, - partitioned_b_target: cute.View, - stage: StageBuffers, - k_tile: V, - tile_m: V, - group: V, - expert: V, - exec_a: cute.Atom, - exec_b: cute.Atom, - exec_sfa: cute.Atom, - exec_sfb: cute.Atom, -) void { - const n = tile.n; - const all = cute.AlgebraToken.all; - const dynamic = cute.AlgebraToken.dynamic; - const div_128: cute.ConstrainedDynamic = .{ .divisible_by = 128 }; - const div_n: cute.ConstrainedDynamic = .{ .divisible_by = @intCast(n) }; - const a_copy_layout = b.staticLayout(b.layoutSpec( - .{ .{ .{ 128, 128 }, 1 }, .{1} }, - .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, - )); - const b_copy_layout = b.staticLayout(b.layoutSpec( - .{ .{ .{ 128, n }, 1 }, .{1} }, - .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, - )); - const sf_copy_spec = b.layoutSpec( - .{ .{ 512, 1 }, .{1} }, - .{ .{ b.basis(1, 2, .{0}), 0 }, .{0} }, - ); - // Follow the same coordinate algebra as the Python kernel. TMA - // partitioning produces a tensor over (tile, M, K, expert/group); first - // select the output tile and expert/group, then select the K stage. The - // remaining iterator carries the tensor-map coordinates needed by copy. - const a_tile_k = b.slice( - partitioned_a_target, - .{ all, tile_m, all, expert }, - b.coordTensorTypeWithTuple(.{ 0, div_128, dynamic }, b.layoutSpec( - .{ .{ .{ 128, 128 }, 1 }, reduction_tiles }, - .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{0}) }, - )), - ); - const a_tile = b.slice( - a_tile_k, - .{ all, k_tile }, - b.coordTensorTypeWithTuple(.{ div_128, div_128, dynamic }, b.layoutSpec( - .{.{ .{ 128, 128 }, 1 }}, - .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, - )), - ); - const a_source = b.rebuildCoordTensorIterator(3, a_tile, .{ div_128, div_128, dynamic }, .{ 128, 128, 1 }, a_copy_layout); - // B's tile spans all of a group's rows, so its N-tile index is zero. Pass - // it as a value rather than a literal: a statically zero leaf collapses out - // of the coordinate tuple, and the TMA lowering then shifts the remaining - // tensor-map coordinates, silently reading group zero for every group. - const b_tile_n = b.constant(.i32, 0); - const b_tile_k = b.slice( - partitioned_b_target, - .{ all, b_tile_n, all, group }, - b.coordTensorTypeWithTuple(.{ 0, div_n, dynamic }, b.layoutSpec( - .{ .{ .{ 128, n }, 1 }, reduction_tiles }, - .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{0}) }, - )), - ); - const b_tile = b.slice( - b_tile_k, - .{ all, k_tile }, - b.coordTensorTypeWithTuple(.{ div_128, div_n, dynamic }, b.layoutSpec( - .{.{ .{ 128, n }, 1 }}, - .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, - )), - ); - const b_source = b.rebuildCoordTensorIterator(3, b_tile, .{ div_128, div_n, dynamic }, .{ 128, n, 1 }, b_copy_layout); - const sfa_source = b.makeCoordTensor(.{ 0, tile_m, k_tile, expert }, b.staticLayout(sf_copy_spec)); - const sfb_source = b.makeCoordTensor(.{ 0, b.constant(.i32, 0), k_tile, group }, b.staticLayout(sf_copy_spec)); - - var elected = b.openIf(b.electSync()); - { - b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); - elected.close(); - } - b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); - b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); - b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); - b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); -} - -fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { - // The stage offset is a whole tile width, which `cute.add_offset` needs to - // know: it derives the result alignment from what it knows of the offset. - return b.addOffset( - tmem, - b.makeIntTupleDivBy(stage.mul(tile.n), @intCast(tile.n)), - b.ptrTy(.f32, .tmem, 16) catch @panic("bad accumulator TMEM pointer"), - ).value(); -} - -/// Warp 5 runs independently over every persistent output tile. Keep the -/// producer index and phase as loop-carried values across tiles, exactly like -/// CUTLASS PipelineState. Only the per-tile count is reset to zero. -fn runTmaProducer( - b: *B, - comptime tile: Tile, - comptime k_tiles: usize, - storage: StageStorage, - active: V, - tiles_m: V, - expert_ids: cute.View, - direct: ?DirectRoutes, - partitioned_a_target: cute.View, - partitioned_b_target: cute.View, - exec_a: cute.Atom, - exec_b: cute.Atom, - exec_sfa: cute.Atom, - exec_sfb: cute.Atom, -) void { - const initial_index = b.constant(.i32, 0); - const initial_phase = b.constant(.i32, 1); - const initial_count = b.constant(.i32, 0); - const persistent_clusters = b.gridDim().z; - const cta_m = b.blockIdx().x; - const cta_n = b.blockIdx().y; - const cta_l = b.constant(.i32, 0); - const div_m = b.fastDivmodCreate(tiles_m); - const div_n = b.fastDivmodCreate(1); - const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); - const carried = .{ - initial_work.m, initial_work.n, initial_work.group, initial_work.valid, - initial_count, initial_index, initial_phase, persistent_clusters, - initial_work.linear, cta_m, cta_n, cta_l, - initial_work.executed, tiles_m, active, div_m, - div_n, - }; - var tiles = b.openWhile(carried, carried); - { - tiles.yieldBefore(tiles.before_carried[3], tiles.before_carried); - } - { - const tile_m = tiles.after_carried[0]; - const group = tiles.after_carried[2]; - const expert = expert_ids.get(.{group}, .i32); - const first_state: PipelinePosition = .{ - .index = tiles.after_carried[5], - .phase = tiles.after_carried[6], - }; - const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); - var stages = b.openFor( - 0, - k_tiles, - 1, - .{ first_ready, b.constant(.i32, 0), first_state.index, first_state.phase }, - ); - { - const k = stages.carried[1]; - const state: PipelinePosition = .{ - .index = stages.carried[2], - .phase = stages.carried[3], - }; - const buffers = stageBuffersAt(b, tile, storage, state.index); - var pending = b.openIf(stages.carried[0].eq(false)); - { - b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); - pending.close(); - } - issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); - const next_count = stages.carried[1].add(1); - const next_state = advancePipeline(b, state.index, state.phase, tile.stages); - const next_ready = peekBarrierIfRemaining(b, storage.mma_barriers, next_count, k_tiles, next_state); - stages.yield(.{ next_ready, next_count, next_state.index, next_state.phase }); - } - const next_work = advancePersistentWork( - b, - tiles.after_carried[8], - tiles.after_carried[12], - tiles.after_carried[7], - tiles.after_carried[14], - tiles.after_carried[13], - tiles.after_carried[15], - tiles.after_carried[16], - direct, - ); - tiles.yieldAfter(.{ - next_work.m, - next_work.n, - next_work.group, - next_work.valid, - stages.results[1], - stages.results[2], - stages.results[3], - tiles.after_carried[7], - next_work.linear, - tiles.after_carried[9], - tiles.after_carried[10], - tiles.after_carried[11], - next_work.executed, - tiles.after_carried[13], - tiles.after_carried[14], - tiles.after_carried[15], - tiles.after_carried[16], - }); - } - - // PipelineTmaUmma.producer_tail: drain every empty barrier before the TMA - // warp exits so no outstanding tcgen05 arrival outlives the CTA. - var tail_state: PipelinePosition = .{ - .index = tiles.results[5], - .phase = tiles.results[6], - }; - inline for (0..tile.stages) |stage| { - b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); - if (stage + 1 < tile.stages) { - tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); - } - } -} - -/// SM100 MXFP4 mainloop objects of one pipeline stage, produced by -/// `StagedMxfp4Mainloop.atStage`. The dependent CuTe result types are -/// centralized here; callers only provide tensors and the allocated -/// tensor-memory pointer. -const Mxfp4Mainloop = struct { - a: V, - b: V, - accumulator: V, - sfa: V, - sfb: V, - sfa_copy: cute.Atom, - sfb_copy: cute.Atom, - sfa_source: V, - sfb_source: V, - sfa_target: V, - sfb_target: V, - - pub fn copyScales(self: Mxfp4Mainloop, builder: *B) void { - builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); - builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); - } -}; - -/// Pipeline-wide SM100 MXFP4 mainloop objects. CUTLASS derives the UMMA and -/// S2T descriptors from the complete staged tensors once, then slices those -/// descriptors at the current PipelineState index. Keeping the stage mode in -/// the descriptor is required when the circular buffer wraps back to stage 0. -const StagedMxfp4Mainloop = struct { - a: cute.View, - b: cute.View, - sfa_source: cute.View, - sfb_source: cute.View, - sfa: V, - sfb: V, - sfa_copy: cute.Atom, - sfb_copy: cute.Atom, - sfa_target: V, - sfb_target: V, - /// MMA tile N: accumulator columns per stage. - n: i64, - - pub fn atStage(self: StagedMxfp4Mainloop, b: *B, index: V, accumulator_tmem: V) Mxfp4Mainloop { - const all = cute.AlgebraToken.all; - const a = b.slice( - self.a, - .{ all, all, all, index }, - b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 })), - ); - const rhs = b.slice( - self.b, - .{ all, all, all, index }, - b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 })), - ); - const sfa_stage = b.slice( - self.sfa_source, - .{ all, all, all, all, index }, - b.smemDescViewType(b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1 }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0 })), - ); - const sfb_stage = b.slice( - self.sfb_source, - .{ all, all, all, all, index }, - b.smemDescViewType(b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1 }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0 })), - ); - const source_grouped_spec = b.layoutSpec(.{ .{ .{ 32, 1, 1, 4 }, 1 }, .{ 1, 1, 1 } }, .{ .{ .{ 1, 1, 1, 0 }, 0 }, .{ 0, 0, 0 } }); - const source_grouped_layout = b.staticLayout(source_grouped_spec); - const source_grouped_ty = b.smemDescViewType(source_grouped_spec); - const sfa_source = b.makeView( - b.getIter(sfa_stage, b.smemDescType()), - source_grouped_layout, - source_grouped_ty, - ); - const sfb_source = b.makeView( - b.getIter(sfb_stage, b.smemDescType()), - source_grouped_layout, - source_grouped_ty, - ); - const acc_spec = b.layoutSpec(.{ .{ 128, self.n }, 1, 1 }, .{ .{ 65536, 1 }, 0, 0 }); - const acc_layout = b.staticLayout(acc_spec); - const accumulator = b.makeView( - accumulator_tmem, - acc_layout, - b.memrefType(.f32, .tmem, 16, b.layoutType(acc_spec)), - ); - return .{ - .a = a.value(), - .b = rhs.value(), - .accumulator = accumulator.value(), - .sfa = self.sfa, - .sfb = self.sfb, - .sfa_copy = self.sfa_copy, - .sfb_copy = self.sfb_copy, - .sfa_source = sfa_source.value(), - .sfb_source = sfb_source.value(), - .sfa_target = self.sfa_target, - .sfb_target = self.sfb_target, - }; - } -}; - -/// Build the same pipeline-wide descriptors as Python's -/// `Sm100BlockScaledPersistentDenseGemmKernel`: `make_fragment_A/B` and the -/// S2T partitions see the complete staged tensors. `atStage` performs the -/// per-iteration slicing afterwards. -fn makeMxfp4StagedMainloop( - b: *B, - tiled_mma: cute.Atom, - comptime n: i64, - comptime stages: usize, - smem_a: V, - smem_b: V, - smem_sfa: V, - smem_sfb: V, - scale_tmem: V, -) StagedMxfp4Mainloop { - // Descriptor strides are in 16-byte units: a 128x128 unpacked-FP4 A - // stage is 16 KiB and an Nx128 FP8 B stage is N*128 bytes. - const frag_a = b.mmaMakeFragment( - tiled_mma, - smem_a, - 0, - b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, 1024 })), - ); - const frag_b = b.mmaMakeFragment( - tiled_mma, - smem_b, - 1, - b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, n * 8 })), - ); - - // TMEM columns: two N-wide accumulator stages, then 16 SFA and 16 SFB - // columns, matching num_accumulator_tmem_cols/num_sfa_tmem_cols in Python. - const sf_spec = b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 32, 1 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 0 } }, 0, 1 }); - const sf_layout = b.staticLayout(sf_spec); - const tmem_f32_ptr = b.ptrTy(.f32, .tmem, 16) catch @panic("bad scale-factor TMEM pointer"); - const tmem_sf_ptr = b.ptrTy(.f8e8m0fnu, .tmem, 16) catch @panic("bad scale-factor TMEM pointer"); - const sfa_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n), tmem_f32_ptr); - const sfb_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n + 16), tmem_f32_ptr); - const sfa_ptr = b.recastIter(sfa_col, tmem_sf_ptr); - const sfb_ptr = b.recastIter(sfb_col, tmem_sf_ptr); - const sf_memref = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(sf_spec)); - const sfa = b.makeView(sfa_ptr, sf_layout, sf_memref); - const sfb = b.makeView(sfb_ptr, sf_layout, sf_memref); - - const smem_filtered_ty = b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(b.layoutSpec( - .{ .{ .{ .{ 32, 4 }, 1 }, .{ 1, 1 } }, 1, 4, stages }, - .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 0 } }, 0, 1, 512 }, - ))); - const tmem_filtered_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(b.layoutSpec( - .{ .{ .{ .{ 32, 4 }, 4 }, .{ 1, 1 } }, 1, 4 }, - .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 0 } }, 0, 1 }, - ))); - const sfa_smem_filtered = b.filterZeros(smem_sfa, null, smem_filtered_ty); - const sfb_smem_filtered = b.filterZeros(smem_sfb, null, smem_filtered_ty); - const sfa_tmem_filtered = b.filterZeros(sfa, null, tmem_filtered_ty); - const sfb_tmem_filtered = b.filterZeros(sfb, null, tmem_filtered_ty); - - const copy_atom_ty = b.s2tCopyAtomType(.f8e8m0fnu, .{ .num_dp = 32, .num_bit = 128, .broadcast = .x4 }); - const copy_atom = b.makeAtom(copy_atom_ty, .{}); - const copy_ty = b.tiledCopyType( - copy_atom_ty, - b.layoutSpec(.{ 1, .{ 32, .{ 4, 4 }, 4 } }, .{ 0, .{ 1, .{ 512, 32 }, 128 } }), - &.{ b.layoutSpec(512, 1), b.layoutSpec(1, 0), b.layoutSpec(4, 1) }, - ); - const sfa_copy = b.makeS2tCopy(copy_atom, sfa_tmem_filtered, copy_ty); - const sfb_copy = b.makeS2tCopy(copy_atom, sfb_tmem_filtered, copy_ty); - - const source_partition_ty = b.memrefType(.f8e8m0fnu, .smem, 1024, b.layoutType(b.layoutSpec( - .{ .{ .{ 32, 4, 4, 4 }, 1 }, 1, 1, 1, stages }, - .{ .{ .{ 16, 1, 4, 0 }, 0 }, 0, 0, 0, 512 }, - ))); - const source_desc_ty = b.smemDescViewType(b.layoutSpec( - .{ .{ .{ 32, 1, 1, 4 }, 1 }, 1, 1, 1, stages }, - .{ .{ .{ 1, 1, 1, 0 }, 0 }, 0, 0, 0, 32 }, - )); - const target_partition_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(b.layoutSpec( - .{ .{ .{ 32, 4, .{ 4, 4 } }, 1 }, 1, 1, 1 }, - .{ .{ .{ 262144, 1, .{ 4, 8388608 } }, 0 }, 0, 0, 0 }, - ))); - const target_grouped_spec = b.layoutSpec( - .{ .{ .{ 32, 4, .{ 4, 4 } }, 1 }, .{ 1, 1, 1 } }, - .{ .{ .{ 262144, 1, .{ 4, 8388608 } }, 0 }, .{ 0, 0, 0 } }, - ); - const target_grouped_ty = b.memrefType(.f8e8m0fnu, .tmem, 16, b.layoutType(target_grouped_spec)); - const sfa_src_partition = b.tiledCopyPartition(sfa_copy, sfa_smem_filtered, .{0}, false, source_partition_ty); - const sfb_src_partition = b.tiledCopyPartition(sfb_copy, sfb_smem_filtered, .{0}, false, source_partition_ty); - const sfa_desc = b.s2tSmemDescriptor(sfa_copy, sfa_src_partition, source_desc_ty); - const sfb_desc = b.s2tSmemDescriptor(sfb_copy, sfb_src_partition, source_desc_ty); - const sfa_dst_partition = b.tiledCopyPartition(sfa_copy, sfa_tmem_filtered, .{0}, true, target_partition_ty); - const sfb_dst_partition = b.tiledCopyPartition(sfb_copy, sfb_tmem_filtered, .{0}, true, target_partition_ty); - const target_grouped_layout = b.staticLayout(target_grouped_spec); - const sfa_dst = b.makeView(b.getIter(sfa_dst_partition, tmem_sf_ptr), target_grouped_layout, target_grouped_ty); - const sfb_dst = b.makeView(b.getIter(sfb_dst_partition, tmem_sf_ptr), target_grouped_layout, target_grouped_ty); - - return .{ - .a = frag_a, - .b = frag_b, - .sfa_source = sfa_desc, - .sfb_source = sfb_desc, - .sfa = sfa.value(), - .sfb = sfb.value(), - .sfa_copy = sfa_copy, - .sfb_copy = sfb_copy, - .sfa_target = sfa_dst.value(), - .sfb_target = sfb_dst.value(), - .n = n, - }; -} - -/// Warp 4 drains the A/B ring into one of two independent accumulator stages. -fn runMmaConsumer( - b: *B, - comptime tile: Tile, - comptime k_tiles: usize, - storage: StageStorage, - tiled_mma: cute.Atom, - tmem_holding: V, - active: V, - tiles_m: V, -) void { - const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); - // Match Python's construction order: derive A/B UMMA fragments and S2T - // scale descriptors from the complete staged tensors once. The K loop - // slices these compiler-derived descriptors by PipelineState.index. - const staged = stagedOperands(b, tile, storage); - const staged_mainloop = makeMxfp4StagedMainloop( - b, - tiled_mma, - tile.n, - tile.stages, - staged.a.value(), - staged.b.value(), - staged.sfa.value(), - staged.sfb.value(), - tmem, - ); - const persistent_clusters = b.gridDim().z; - const div_m = b.fastDivmodCreate(tiles_m); - const div_n = b.fastDivmodCreate(1); - // The MMA warp never reads the group, only the tile count. - const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, null); - const carried = .{ - initial_work.m, initial_work.n, initial_work.group, initial_work.valid, - b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), tiled_mma.value(), - b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 1), persistent_clusters, - initial_work.linear, b.blockIdx().x, b.blockIdx().y, b.constant(.i32, 0), - initial_work.executed, tiles_m, active, div_m, - div_n, - }; - var tiles = b.openWhile(carried, carried); - { - tiles.yieldBefore(tiles.before_carried[3], tiles.before_carried); - } - { - // Python peeks the first AB-full token before it blocks on an available - // accumulator stage. Preserve that ordering so the two independent - // pipelines enter the K loop with the same outstanding operations. - const first_state: PipelinePosition = .{ - .index = tiles.after_carried[5], - .phase = tiles.after_carried[6], - }; - const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); - const accumulator_state: PipelinePosition = .{ - .index = tiles.after_carried[9], - .phase = tiles.after_carried[10], - }; - const accumulator_stage = accumulator_state.index; - const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); - b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); - - const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; - const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); - var stages = b.openFor( - 0, - k_tiles, - 1, - // Match PipelineTmaUmma's Python loop-carried ABI exactly. The - // software-pipeline pass recognizes the leading try-wait token and - // keeps the MMA atom immediately after it. - .{ first_ready, initial_mma.value(), b.constant(.i32, 0), first_state.index, first_state.phase }, - ); - { - const k = stages.carried[2]; - const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; - var pending = b.openIf(stages.carried[0].eq(false)); - { - b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); - pending.close(); - } - const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); - const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); - mainloop.copyScales(b); - const loop_mma: cute.Atom = .{ .inner = stages.carried[1].inner, .kernel = b }; - const mma = b.setMmaAccumulate(loop_mma, k.ne(0)); - b.gemm( - mma, - mainloop.accumulator, - .{ mainloop.a, mainloop.sfa }, - .{ mainloop.b, mainloop.sfb }, - mainloop.accumulator, - ); - var commit_lane = b.openIf(b.electSync()); - { - b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); - commit_lane.close(); - } - const next_count = stages.carried[2].add(1); - const next_state = advancePipeline(b, state.index, state.phase, tile.stages); - const next_ready = peekBarrierIfRemaining(b, storage.load_barriers, next_count, k_tiles, next_state); - stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); - } - - var full_lane = b.openIf(b.electSync()); - { - b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); - full_lane.close(); - } - const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); - const next_acc_count = tiles.after_carried[8].add(1); - const next_work = advancePersistentWork( - b, - tiles.after_carried[12], - tiles.after_carried[16], - tiles.after_carried[11], - tiles.after_carried[18], - tiles.after_carried[17], - tiles.after_carried[19], - tiles.after_carried[20], - null, - ); - tiles.yieldAfter(.{ - next_work.m, - next_work.n, - next_work.group, - next_work.valid, - stages.results[2], - stages.results[3], - stages.results[4], - stages.results[1], - next_acc_count, - next_accumulator_state.index, - next_accumulator_state.phase, - tiles.after_carried[11], - next_work.linear, - tiles.after_carried[13], - tiles.after_carried[14], - tiles.after_carried[15], - next_work.executed, - tiles.after_carried[17], - tiles.after_carried[18], - tiles.after_carried[19], - tiles.after_carried[20], - }); - } - - // Match PipelineUmmaAsync.producer_tail. The state points at the next - // accumulator slot; advance to the last slot used by this producer and - // wait until the epilogue has released it before the MMA warp exits. - var accumulator_tail: PipelinePosition = .{ - .index = tiles.results[9], - .phase = tiles.results[10], - }; - inline for (0..accumulator_stages - 1) |_| { - accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); - } - b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); -} - -/// Epilogue operands; which fields are set depends on the `Epilogue`. -const EpilogueOperands = struct { - group_sizes: cute.View, - route_inverse: cute.View, - routing_weights: ?cute.View = null, - q: ?cute.View = null, - s: ?cute.View = null, - rows: ?cute.View = null, - direct: bool, - /// `Config.groupRows`. - group_rows: i64 = 0, - /// `Config.swiglu_limit`. - swiglu_limit: ?f32 = 10.0, -}; - -/// Warps 0-3 drain each TMEM accumulator stage into registers, release it to -/// the MMA warp, then write the tile out according to `epilogue`. -fn runEpilogue( - b: *B, - comptime tile: Tile, - comptime epilogue: Epilogue, - storage: StageStorage, - tmem_holding: V, - active: V, - tiles_m: V, - direct: ?DirectRoutes, - operands: EpilogueOperands, -) void { - const n = tile.n; - const tid = b.threadIdx().x; - const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); - const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); - - const div_m = b.fastDivmodCreate(tiles_m); - const div_n = b.fastDivmodCreate(1); - const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); - const carried = .{ - initial_work.m, initial_work.n, initial_work.group, initial_work.valid, - b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), initial_work.linear, - initial_work.executed, - }; - var tiles = b.openWhile(carried, .{ - initial_work.m, - initial_work.n, - initial_work.group, - .i32, - .i32, - .i32, - initial_work.linear, - initial_work.executed, - }); - { - tiles.yieldBefore(tiles.before_carried[3], .{ - tiles.before_carried[0], - tiles.before_carried[1], - tiles.before_carried[2], - tiles.before_carried[4], - tiles.before_carried[5], - tiles.before_carried[6], - tiles.before_carried[7], - tiles.before_carried[8], - }); - } - { - const tile_m = tiles.after_carried[0]; - const group = tiles.after_carried[2]; - const accumulator_state: PipelinePosition = .{ - .index = tiles.after_carried[4], - .phase = tiles.after_carried[5], - }; - const stage = accumulator_state.index; - b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); - - const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); - const warp_tmem = b.addOffset( - accumulator_tmem, - b.makeIntTupleDivBy(warp.mul(32 * 65536), 4), - b.ptrTy(.f32, .tmem, 16) catch @panic("bad epilogue TMEM pointer"), - ).value(); - // A warp lane holds one routed row, so tiles wider than a warp are drained - // 32 rows at a time. The accumulator stage is released once its last chunk - // is in registers. - const chunk_rows: usize = comptime @intCast(tile.chunkRows()); - const tile_chunks = comptime tile.chunks(); - inline for (0..tile_chunks) |chunk| { - const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffset( - warp_tmem, - b.makeIntTuple(.{@as(i32, @intCast(chunk * chunk_rows))}), - b.ptrTy(.f32, .tmem, offsetAlignment(16, @intCast(chunk * chunk_rows), 4)) catch @panic("bad epilogue TMEM pointer"), - ).value(); - // Route metadata does not depend on the accumulator. - const routed = routedLanes(b, tile, operands, group, chunk); - const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); - b.fenceTmemLoad(); - if (chunk + 1 == tile_chunks) { - // Registers now own the accumulator values, so all 128 epilogue - // threads release this TMEM stage before the stores. - _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); - } - var columns: [@intCast(@min(n, 32))]V = undefined; - inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32); - switch (epilogue) { - .swiglu_mxfp8 => storeTileSwiglu(b, tile, storage, operands, routed, tile_m, group, &columns, chunk), - .route_rows => storeTileRows(b, tile, operands, routed, tile_m, &columns, chunk), - } - } - const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); - const next_work = advancePersistentWork(b, tiles.after_carried[6], tiles.after_carried[7], b.gridDim().z, active, tiles_m, div_m, div_n, direct); - tiles.yieldAfter(.{ - next_work.m, - next_work.n, - next_work.group, - next_work.valid, - tiles.after_carried[3].add(1), - next_accumulator_state.index, - next_accumulator_state.phase, - next_work.linear, - next_work.executed, - }); - } - - var release_warp = b.openIf(warp.eq(0)); - { - // Permit release and deallocation are warp collective as well. - b.relinquishTmemAllocPermit(.{}); - release_warp.close(); - } - b.barrierCtaSyncAligned(1, 128); - var dealloc_warp = b.openIf(warp.eq(0)); - { - b.deallocTmem(tmem, tile.tmem_columns, .{}); - dealloc_warp.close(); - } -} - -/// E8M0 exponent of the power-of-two MX scale for a block maximum, rounded -/// up so that `maximum / scale <= 448`, and the matching inverse scale. -fn mxScale(b: *B, maximum: V) struct { exponent: V, inverse: V } { - const raw_bits = maximum.maximum(1e-4).mul(1.0 / 448.0).bitCast(.i32); - const exponent = raw_bits.shrLogical(23).add(raw_bits.bitAnd(0x7fffff).ne(0).to(.i32)); - return .{ .exponent = exponent, .inverse = b.constant(.i32, 254).sub(exponent).shl(23).bitCast(.f32) }; -} - -/// Per-row routing data of one expert group, spread across a warp: lane `l` -/// holds routed row `l` of the group. Columns index it with `shuffleIdx`. -const RoutedLanes = struct { - count: V, - /// Token/top-k route of the lane's row; 0 for padding rows. - route: V, - /// Routing weight of the lane's row (up projection only). - weight: ?V, -}; - -fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, comptime chunk: usize) RoutedLanes { - // Only the up projection applies routing weights. - const weights = operands.routing_weights; - if (operands.direct) return .{ - .count = b.constant(.i32, 1), - .route = group, - .weight = if (weights) |w| w.get(.{group}, .f32) else null, - }; - const lane = b.threadIdx().x.rem(32); - const count = operands.group_sizes.get(.{group}, .i32); - // Lanes past the tile width alias row 0, which is always live. Wide tiles - // run one 32-row chunk at a time, so lane `l` holds row `32 * chunk + l`. - const chunk_rows = comptime tile.chunkRows(); - const in_chunk = b.select(lane.lt(chunk_rows), lane, b.constant(.i32, 0)); - const row_in_group = if (chunk == 0) in_chunk else in_chunk.add(@as(i64, @intCast(chunk)) * chunk_rows); - const live = row_in_group.lt(count); - const inverse = operands.route_inverse.get(.{group.mul(tile.n).add(row_in_group)}, .i32); - const route = b.select(live, inverse, b.constant(.i32, 0)); - const weight = if (weights) |w| w.get(.{route}, .f32) else null; - return .{ .count = count, .route = route, .weight = weight }; -} - -/// Thread `t` owns output row `128 * tile_m + t` of the interleave-8 -/// projection. Gate and up lanes are paired by XOR 8. Each gate thread -/// computes one SwiGLU activation per routed row; 32 consecutive activations -/// span two warps, whose partial maxima meet in shared memory. -/// -/// At decode sizes most rows of a group are padding, whose outputs are never -/// read. `count` is uniform across the CTA, so whole 8-row chunks of padding -/// are skipped; per-row branches would serialize the shuffle chains of -/// independent rows. Do not duplicate this body under a full-group branch: -/// ptxas then loses track of convergence at the tile loop's warp-uniform -/// shuffles, and lanes still outside the gate-lane stores read garbage. -fn storeTileSwiglu(b: *B, comptime tile: Tile, storage: StageStorage, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, columns: []const V, comptime chunk: usize) void { - // Direct routing has exactly one live row per group. - if (operands.direct) - swigluRows(b, tile, storage, operands, routed, tile_m, group, columns, 1, false, chunk) - else - swigluRows(b, tile, storage, operands, routed, tile_m, group, columns, comptime @intCast(tile.chunkRows()), true, chunk); -} - -const swiglu_chunk = 8; - -/// The first `live` rows of `storeTileSwiglu`; with `skip_padding`, chunks -/// of rows past `count` are skipped. -fn swigluRows(b: *B, comptime tile: Tile, storage: StageStorage, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, columns: []const V, comptime live: usize, comptime skip_padding: bool, comptime chunk: usize) void { - const n = comptime tile.chunkRows(); - // First routed row of this epilogue pass. - const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); - const tid = b.threadIdx().x; - const lane = tid.rem(32); - const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); - const is_gate = tid.rem(16).lt(8); - const first_row = group.mul(operands.group_rows); - const f32_type = cute.DType.f32.toMlir(b.ctx); - // The TMA C staging buffer is unused by this epilogue. - const scratch_layout = b.layoutSpec(.{ 4, n }, .{ n, 1 }); - const scratch = b.makeView(storage.c, b.staticLayout(scratch_layout), b.memrefType(.f32, .smem, 1024, b.layoutType(scratch_layout))); - const row_chunks = std.math.divCeil(usize, live, swiglu_chunk) catch unreachable; - - var activations: [@intCast(n)]V = undefined; - // Lane `c` collects the warp maximum of column `c`. - var lane_maximum = b.constant(.f32, 0); - for (0..row_chunks) |row_chunk| { - const first = row_chunk * swiglu_chunk; - const last = @min(first + swiglu_chunk, live); - // Row 0 of an active group is always live. - const guarded = skip_padding and (row_chunk > 0 or chunk > 0); - const types: [swiglu_chunk + 1]*const mlir.Type = @splat(f32_type); - // `@TypeOf` does not evaluate its operand. - var guard: ?@TypeOf(b.openIfElse(routed.count, types)) = if (guarded) b.openIfElse(routed.count.gt(chunk_first + @as(i32, @intCast(first))), types) else null; - var chunk_maximum = lane_maximum; - for (columns[first..last], activations[first..last], first..) |value, *activation, column| { - const c: i32 = @intCast(column); - // The reference model rounds the projection to BF16. - var gate = value.to(.bf16).to(.f32); - if (operands.swiglu_limit) |limit| { - gate = gate.minimum(limit); - } - var up_value = b.shuffleXor(value, 8).to(.bf16).to(.f32); - if (operands.swiglu_limit) |limit| { - up_value = up_value.maximum(-limit).minimum(limit); - } - // up * SiLU(gate), with the hardware approximations Python's - // fastmath selects: exp(-g) = 2^(-g * log2 e). - const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); - var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); - result = result.mul(b.shuffleIdx(routed.weight.?, c)).to(.bf16).to(.f32); - // Up lanes hold no activation and must not raise the maxima. - activation.* = b.select(is_gate, result, b.constant(.f32, 0)); - const maximum = b.warpMaxNonNegative(activation.*.abs()); - chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); - } - if (guard) |*g| { - var then_values: [swiglu_chunk + 1]V = @splat(chunk_maximum); - for (activations[first..last], then_values[0 .. last - first]) |activation, *t| t.* = activation; - g.yieldThen(then_values); - // Padding chunk: zero activations, maxima unchanged. The zero is - // emitted in the else block, where it is used. - var else_values: [swiglu_chunk + 1]V = @splat(b.constant(.f32, 0)); - else_values[swiglu_chunk] = lane_maximum; - g.yieldElse(else_values); - for (activations[first..last], g.results[0 .. last - first]) |*activation, result| activation.* = result; - lane_maximum = g.results[swiglu_chunk]; - } else { - lane_maximum = chunk_maximum; - } - } - var owner = b.openIf(lane.lt(n)); - { - scratch.set(.{ warp, lane }, lane_maximum); - owner.close(); - } - b.barrierCtaSyncAligned(1, 128); - - // Lane `c` turns the two half-block maxima of column `c` into its scale. - const pair = warp.bitAnd(~@as(i32, 1)); - const lane_column = b.select(lane.lt(n), lane, b.constant(.i32, 0)); - const scale = mxScale(b, scratch.get(.{ pair, lane_column }, .f32).maximum(scratch.get(.{ pair.add(1), lane_column }, .f32))); - // The next tile reuses the scratch maxima. - b.barrierCtaSyncAligned(1, 128); - - const activation_row = tid.div(16).mul(8).add(tid.rem(8)); - const activation_index = tile_m.mul(64).add(activation_row); - for (0..row_chunks) |row_chunk| { - const first = row_chunk * swiglu_chunk; - const last = @min(first + swiglu_chunk, live); - var guard: ?@TypeOf(b.openIf(routed.count)) = if (skip_padding and (chunk > 0 or first > 0)) b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(first)))) else null; - var inverses: [swiglu_chunk]V = undefined; - for (inverses[0 .. last - first], first..) |*inverse, column| inverse.* = b.shuffleIdx(scale.inverse, @as(i32, @intCast(column))); - var gate_lane = b.openIf(is_gate); - { - for (activations[first..last], inverses[0 .. last - first], first..) |activation, inverse, column| { - const quantized = activation.mul(inverse).maximum(-448.0).minimum(448.0).to(.f8e4m3fn); - operands.q.?.set(.{ first_row.add(chunk_first + @as(i32, @intCast(column))), activation_index }, quantized); - } - gate_lane.close(); - } - if (guard) |*g| g.close(); - } - - // Warps 0 and 2 each own one 32-activation block of this tile. - var scale_lane = b.openIf(warp.rem(2).eq(0).bitAnd(lane.add(chunk_first).lt(routed.count))); - { - const block = tile_m.mul(2).add(warp.div(2)); - const swizzled = block.div(4).mul(512).add(lane.mul(16)).add(chunk * 4).add(block.rem(4)); - operands.s.?.set(.{ group, swizzled }, scale.exponent.to(.i8)); - scale_lane.close(); - } -} - -/// Python `down_epilogue`: store each live routed row at its token/top-k -/// route, so the final reduction reads rows in route order. -fn storeTileRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, columns: []const V, comptime chunk: usize) void { - const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); - const output_column = tile_m.mul(128).add(b.threadIdx().x); - if (operands.direct) { - operands.rows.?.set(.{ routed.route, output_column }, columns[0].to(.bf16)); - return; - } - var routes: [32]V = undefined; - for (routes[0..columns.len], 0..) |*route, column| route.* = b.shuffleIdx(routed.route, @as(i32, @intCast(column))); - for (columns, routes[0..columns.len], 0..) |value, route, column| { - var live: ?@TypeOf(b.openIf(routed.count)) = if (column == 0 and chunk == 0) null else b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(column)))); - operands.rows.?.set(.{ route, output_column }, value.to(.bf16)); - if (live) |*l| l.close(); - } -} - -fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue, d: anytype) void { - const L = Layouts(tile); - const n = tile.n; - const stages = tile.stages; - // Allocate the complete Python SharedStorage object as one compiler-visible - // memref. `cute.kernel_smem_size` only accounts for memref allocations; - // treating every field as an architecture pointer left the launch at zero - // dynamic shared bytes and made the first TMA instruction fault. - const shared = b.allocSmemStorage( - .i8, - 1024, - b.layoutType(b.layoutSpec(.{tile.smem_bytes}, .{1})), - 1, - tile.fields(b), - ); - const shared_base = b.getIter( - shared, - b.ptrTy(.i8, .smem, 1024) catch @panic("bad shared base pointer"), - ).value(); - - const tmem_holding = sharedPointerAt(b, shared_base, tile.tmem_holding, .i32, 8); - const storage: StageStorage = .{ - .c = sharedPointerAt(b, shared_base, tile.c, .f32, 1024), - .a = sharedPointerAt(b, shared_base, tile.a, .i8, 1024), - .b = sharedPointerAt(b, shared_base, tile.b, .f8e4m3fn, 1024), - .sfa = sharedPointerAt(b, shared_base, tile.sfa, .f8e8m0fnu, 1024), - .sfb = sharedPointerAt(b, shared_base, tile.sfb, .f8e8m0fnu, 1024), - .load_barriers = sharedPointerAt(b, shared_base, 0, .i64, 1024), - .mma_barriers = sharedPointerAt(b, shared_base, tile.empty_barriers, .i64, 8), - .accumulator_full_barriers = sharedPointerAt(b, shared_base, tile.accumulator_full, .i64, 16), - .accumulator_empty_barriers = sharedPointerAt(b, shared_base, tile.accumulator_empty, .i64, 16), - }; - - const tid = b.threadIdx().x; - const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); - const tiled_mma: cute.Atom = .{ .inner = d.mma.inner, .kernel = b }; - - // Reproduce the Python CuTe path for the two matrix operands: - // coordinate tensor -> local_tile -> tiled_mma.partition -> group modes - // -> tma_partition. All dependent result types come from Zig layout - // objects; this kernel does not embed textual MLIR. - const m_tiles = @divExact(cfg.m, 128); - const k_tiles = @divExact(cfg.k, 128); - const a_local_spec = b.layoutSpec( - .{ 128, 128, m_tiles, k_tiles, cfg.experts }, - .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, - ); - // The group tile covers all of B's rows, so the N-tile mode has extent one - // and its stride is inferred as zero. Direct routing is no different: the - // tensor map still spans a whole tile of rows, zero-filling all but one. - const b_local_spec = b.layoutSpec( - .{ n, 128, 1, k_tiles, cfg.groups }, - .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), 0, b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, - ); - const all_tiles = .{ cute.AlgebraToken.all, cute.AlgebraToken.all, cute.AlgebraToken.all }; - const tiled_a = b.localTile(d.gA, b.makeTile(&.{ 128, 128 }), all_tiles, b.coordTensorType(3, a_local_spec), null); - const tiled_b = b.localTile(d.gB, b.makeTile(&.{ n, 128 }), all_tiles, b.coordTensorType(3, b_local_spec), null); - - const a_mma_spec = b.layoutSpec( - .{ .{ 128, 32 }, 1, 4, m_tiles, k_tiles, cfg.experts }, - .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}), b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, - ); - const b_mma_spec = b.layoutSpec( - .{ .{ n, 32 }, 1, 4, 1, k_tiles, cfg.groups }, - .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}), 0, b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, - ); - const mma_a = b.tiledMmaPartition(tiled_mma, tiled_a, .{0}, 0, b.coordTensorType(3, a_mma_spec)); - const mma_b = b.tiledMmaPartition(tiled_mma, tiled_b, .{0}, 1, b.coordTensorType(3, b_mma_spec)); - - const a_grouped_layout = b.staticLayout(b.layoutSpec( - .{ .{ .{ 128, 32 }, 1, 4 }, m_tiles, k_tiles, cfg.experts }, - .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}) }, b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, - )); - const b_grouped_layout = b.staticLayout(b.layoutSpec( - .{ .{ .{ n, 32 }, 1, 4 }, 1, k_tiles, cfg.groups }, - .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(32, 1, .{0}) }, b.basis(n, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, - )); - const staged = stagedOperands(b, tile, storage); - const smem_a_grouped = b.withLayout(staged.a, b.staticLayout(b.layoutSpec( - .{ .{ .{ 128, 32 }, 1, 4 }, stages }, - .{ .{ .{ 128, 1 }, 0, 32 }, Tile.a_stage_bytes }, - ))); - const smem_b_grouped = b.withLayout(staged.b, b.staticLayout(b.layoutSpec( - .{ .{ .{ n, 32 }, 1, 4 }, stages }, - .{ .{ .{ 128, 1 }, 0, 32 }, tile.bStageBytes() }, - ))); - const one_cta = b.staticLayout(b.layoutSpec(.{1}, .{0})); - const a_partition_layout = b.layoutSpec(.{ .{ Tile.a_stage_bytes, 1 }, stages }, .{ .{ 1, 0 }, Tile.a_stage_bytes }); - const b_partition_layout = b.layoutSpec(.{ .{ tile.bStageBytes(), 1 }, stages }, .{ .{ 1, 0 }, tile.bStageBytes() }); - const a_target_layout = b.layoutSpec( - .{ .{ .{ 128, 128 }, 1 }, m_tiles, k_tiles, cfg.experts }, - .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(128, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, - ); - const b_target_layout = b.layoutSpec( - .{ .{ .{ 128, n }, 1 }, 1, k_tiles, cfg.groups }, - .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(n, 1, .{1}), b.basis(128, 1, .{0}), b.basis(1, 1, .{2}) }, - ); - const partitioned_a = b.tmaPartition( - .{ .inner = d.tma_a.inner, .kernel = b }, - .{0}, - one_cta, - smem_a_grouped, - .{b.withLayout(mma_a, a_grouped_layout)}, - b.memrefTypeFromPointer(b.swizzledPtrTy(.i8, .smem, 1024, L.ab_swizzle) catch @panic("bad smem pointer"), b.layoutType(a_partition_layout)), - &.{b.coordTensorType(3, a_target_layout)}, - ); - const partitioned_b = b.tmaPartition( - .{ .inner = d.tma_b.inner, .kernel = b }, - .{0}, - one_cta, - smem_b_grouped, - .{b.withLayout(mma_b, b_grouped_layout)}, - b.memrefTypeFromPointer(b.swizzledPtrTy(.f8e4m3fn, .smem, 1024, L.ab_swizzle) catch @panic("bad smem pointer"), b.layoutType(b_partition_layout)), - &.{b.coordTensorType(3, b_target_layout)}, - ); - - const exec_a = b.makeExecTma(.{ .inner = d.tma_a.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaA(b, cfg))); - const exec_b = b.makeExecTma(.{ .inner = d.tma_b.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaB(b, cfg))); - const exec_sfa = b.makeExecTma(.{ .inner = d.tma_sfa.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); - const exec_sfb = b.makeExecTma(.{ .inner = d.tma_sfb.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); - - const expert_ids: cute.View = .{ .inner = d.expert_ids.inner, .kernel = b }; - const active_count: cute.View = .{ .inner = d.active_count.inner, .kernel = b }; - const direct: ?DirectRoutes = if (cfg.direct) .{ .expert_ids = expert_ids, .routes = @intCast(cfg.groups) } else null; - const active = if (direct) |routes| routes.count(b) else active_count.get(.{0}, .i32); - - // Match CUTLASS PipelineTmaUmma/PipelineUmmaAsync initialization. Warp 0 - // owns initialization, and one elected lane initializes each complete - // barrier array. mbarrier.init is not warp collective: issuing it from - // every lane races on the same shared state and breaks the first phase - // transition when the A/B ring wraps. - var init_warp = b.openIf(warp.eq(0)); - { - const barrier_arrays = .{ - .{ storage.load_barriers, stages, 1 }, - .{ storage.mma_barriers, stages, 1 }, - .{ storage.accumulator_full_barriers, accumulator_stages, 1 }, - .{ storage.accumulator_empty_barriers, accumulator_stages, 128 }, - }; - inline for (barrier_arrays) |array| { - var lane = b.openIf(b.electSync()); - { - inline for (0..array[1]) |stage| { - b.mbarrierInit(barrierAt(b, array[0], b.constant(.i32, stage)), array[2]); - } - lane.close(); - } - } - b.fenceMbarrierInit(); - init_warp.close(); - } - b.syncThreads(); - - // Each role walks the same persistent tile sequence independently. The - // AB and accumulator mbarriers carry all cross-warp dependencies. - var tma_warp = b.openIf(warp.eq(5)); - { - switch (k_tiles) { - inline 16, 18, 32, 40 => |kt| runTmaProducer(b, tile, kt, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb), - else => std.debug.panic("unsupported MXFP4 reduction tile count {d}", .{k_tiles}), - } - tma_warp.close(); - } - - var mma_warp = b.openIf(warp.eq(4)); - { - // Python's MMA role reaches its own TmemAllocator.wait_for_alloc site. - b.barrierCtaSyncAligned(2, 160); - switch (k_tiles) { - inline 16, 18, 32, 40 => |kt| runMmaConsumer(b, tile, kt, storage, tiled_mma, tmem_holding, active, d.tiles_m), - else => unreachable, - } - mma_warp.close(); - } - - var epilogue_warps = b.openIf(warp.lt(4)); - { - // TmemAllocator.allocate is issued collectively by warp 0. The four - // epilogue warps then meet the MMA warp at named barrier 2. - var alloc_warp = b.openIf(warp.eq(0)); - { - b.allocTmem(tile.tmem_columns, tmem_holding, .{}); - alloc_warp.close(); - } - b.barrierCtaSyncAligned(2, 160); - switch (epilogue) { - .swiglu_mxfp8 => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ - .group_sizes = view(b, d.group_sizes), - .route_inverse = view(b, d.route_inverse), - .routing_weights = view(b, d.routing_weights), - .q = view(b, d.q), - .s = view(b, d.s), - .direct = cfg.direct, - .group_rows = cfg.groupRows(), - .swiglu_limit = cfg.swiglu_limit, - }), - .route_rows => runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, .{ - .group_sizes = view(b, d.group_sizes), - .route_inverse = view(b, d.route_inverse), - .rows = view(b, d.rows), - .direct = cfg.direct, - }), - } - epilogue_warps.close(); - } -} - -fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { - const L = Layouts(tile); - const host_arg = HostArgs(epilogue); - const n = tile.n; - const a_layout = b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ cfg.k, 1, cfg.m * cfg.k }); - const b_layout = b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ cfg.k, 1, cfg.k * cfg.groupRows() }); - // `cute.tile_to_shape` derives that the dynamic strides step whole - // 512-byte scale atoms, and the result type has to carry it. - const sf_stride: cute.ConstrainedDynamic = .{ .divisible_by = Tile.sf_stage_bytes }; - const dynamic_sf_layout = b.layoutSpec( - .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, - .{ .{ .{ 16, 4 }, sf_stride }, .{ .{ 0, 1 }, 512 }, .{ 0, sf_stride } }, - ); - - const gA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight), .f4e2m1fn, .gmem, 16), b.staticLayout(a_layout)); - // `schedule` is `[group experts | group sizes | active count]`. - const schedule = b.arg(host_arg.schedule); - const ids = b.makeTensorView(schedule, b.staticLayout(b.layoutSpec(cfg.groups, 1))); - const active_ptr = b.addOffset( - schedule, - b.makeIntTuple(.{2 * cfg.groups}), - b.ptrTy(.i32, .gmem, offsetAlignment(16, 2 * cfg.groups, 4)) catch @panic("bad active count pointer"), - ); - const active = b.makeTensorView(active_ptr, b.staticLayout(b.layoutSpec(1, 1))); - const gB = b.makeTensorView(b.arg(host_arg.input_quant), b.staticLayout(b_layout)); - // Match Python's host construction instead of materializing these two - // layouts with `cute.static`. In particular, SFB's M extent is one; CuTe - // must still retain that mode while encoding the tensor map. - const sf_atom_layout = b.staticLayout(b.layoutSpec( - .{ .{ 32, 4 }, .{ 32, 4 } }, - .{ .{ 16, 4 }, .{ 0, 1 } }, - )); - // The Zig dialect binding exposes the zero-based form of Python CuTe's - // `(2, 1, 3)` tile-to-shape order. - const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); - const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); - const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); - // Weight scales step M atoms first; activation scales retain K-major order. - const sfa_order = b.makeIntTuple(.{ 0, 1, 2 }); - const sfa_layout = b.layoutSpec( - .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, - .{ .{ .{ 16, 4 }, 512 }, .{ .{ 0, 1 }, sf_stride }, .{ 0, sf_stride } }, - ); - const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sfa_order, b.layoutType(sfa_layout)); - const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); - const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfa_layout); - const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e8m0fnu, .gmem, 16), dynamic_sfb_layout); - - const zero_tmem = b.intToPtr(0, .f8e8m0fnu, .tmem, 1); - const mma = b.makeBlockScaledMma(.{ .n = @intCast(n) }, zero_tmem); - const mma_sfb = b.makeBlockScaledMma(.{ .n = 128 }, zero_tmem); - - const a_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.aSmem(b, false)); - const b_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.bSmem(b, false)); - const sf_smem = b.staticLayout(L.sfSmem(b, false)); - const a_map = b.staticLayout(b.layoutSpec(.{ .{ 128, 32 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(32, 1, .{1}) })); - const b_map = b.staticLayout(b.layoutSpec(.{ .{ n, 32 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(32, 1, .{1}) })); - const sf_map = b.staticLayout(b.layoutSpec( - .{ .{ .{ 32, 4 }, 32 }, 1, 4 }, - .{ .{ .{ b.basis(1, 1, .{ 0, 0, 0 }), b.basis(1, 1, .{ 0, 0, 1 }) }, b.basis(1, 1, .{ 1, 0, 0 }) }, 0, b.basis(1, 1, .{ 1, 0, 1 }) }, - )); - - const tma_a = b.makeTiledTmaLoadAtom(gA.value(), a_smem.value(), a_map.value(), L.tmaA(b, cfg)); - const tma_b = b.makeTiledTmaLoadAtom(gB.value(), b_smem.value(), b_map.value(), L.tmaB(b, cfg)); - const tma_sfa = b.makeTiledTmaLoadAtom(gSFA.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); - const tma_sfb = b.makeTiledTmaLoadAtom(gSFB.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); - - const one = b.constant(.i32, 1); - const tiles_m = b.constant(.i32, @divExact(cfg.m, 128)); - const grid_z = b.constant(.i32, @min(@divExact(cfg.m, 128) * cfg.groups, cfg.persistent_ctas)); - const config = b.makeLaunchConfig(.{ - .grid = .{ one, one, grid_z }, - .block = .{ b.constant(.i32, 192), one, one }, - .dynamic_smem = b.kernelSmemSize(deviceName(epilogue)), - .stream = b.cudaStream(), - .cluster = .{ one, one, one }, - .use_pdl = true, - }); - const mainloop = .{ - ids, active, mma.tiled, mma_sfb.tiled, tma_a.atom, tma_a.tensor, tma_b.atom, tma_b.tensor, - tma_sfa.atom, tma_sfa.tensor, tma_sfb.atom, tma_sfb.tensor, - }; - const activations = @divExact(cfg.m, 2); - const group_sizes = b.makeTensorView( - b.addOffset(schedule, b.makeIntTuple(.{cfg.groups}), b.ptrTy(.i32, .gmem, offsetAlignment(16, cfg.groups, 4)) catch @panic("bad group size pointer")), - b.staticLayout(b.layoutSpec(cfg.groups, 1)), - ); - const launch = switch (epilogue) { - .swiglu_mxfp8 => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ - group_sizes, - b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), - b.makeTensorView(b.arg(host_arg.routing_weights), b.staticLayout(b.layoutSpec(@max(cfg.routes, 1), 1))), - b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ cfg.routedRows(), activations }, .{ activations, 1 }))), - b.makeTensorView(b.arg(host_arg.scales), b.staticLayout(b.layoutSpec(.{ cfg.groups, @divExact(activations, 32) * 128 }, .{ @divExact(activations, 32) * 128, 1 }))), - tiles_m, - }), - .route_rows => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ - group_sizes, - b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), - b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), - tiles_m, - }), - }; - b.returnHostStatus(b.cudaResultStatus(launch)); -} - -test "MXFP4 GEMM programs emit their host launch for every tile width" { - inline for (.{ 8, 16, 32 }) |n| { - inline for (.{ .{ UpQuantized, Epilogue.swiglu_mxfp8, 4608, 5120 }, .{ DownRows, Epilogue.route_rows, 5120, 2304 } }) |program| { - const cfg: Config = .{ - .experts = 192, - .m = program[2], - .n = n, - .k = program[3], - .groups = 227, - .epilogue = program[1], - .routes = 576, - .direct = n == 8, - }; - const ir = try program[0].emit(std.testing.allocator, cfg); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, deviceName(program[1])) != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "make_non_exec_tiled_tma_load") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, std.fmt.comptimePrint("sm100.mma_bs<128x{d}x32", .{n})) != null); - } - } -} diff --git a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig b/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig deleted file mode 100644 index 0de45007..00000000 --- a/zml/kernels/moe/cute/sm10x/persistent_nvfp4.zig +++ /dev/null @@ -1,1679 +0,0 @@ -//! Source-level SM100 NVFP4 grouped GEMM. -//! -//! The NVFP4 counterpart of `persistent_mxfp4.zig`, following the CuTe DSL -//! `Sm100BlockScaledPersistentDenseGemmKernel` with FP4 x FP4 operands, E4M3 -//! scale factors every 16 elements and the `kind::mxf4nvf4.block16` MMA. Both -//! operands are packed FP4 in global and shared memory (TMA format `U4`); one -//! pipeline stage holds a 256-deep K slice, consumed by four K = 64 MMAs. -//! -//! One CTA computes a 128xN output tile of one expert group: warp 5 drives -//! the TMA ring, warp 4 issues the S2T scale copies and UMMA, and warps 0-3 -//! drain the accumulator from tensor memory into one of the two epilogues. -//! The pipeline, scheduler and warp roles are shared with the MXFP4 kernel. -const std = @import("std"); -const mlir = @import("mlir"); -const nvvm = @import("mlir/dialects/nvvm"); - -const zml = @import("../../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const V = cute.Value; -const common = @import("common.zig"); -const nvfp4 = @import("nvfp4.zig"); - -const accumulator_stages = 2; -/// K elements of one pipeline stage: four K = 64 FP4 MMAs. -pub const k_tile = 256; -/// Largest dynamic shared-memory allocation of an SM100 CTA. -const smem_capacity = 232448; -const mma_kind: cute.BlockScaledMmaConfig.Kind = .nvfp4_nvfp4; - -/// Shared-memory plan for one tile width, laid out like the Python -/// `SharedStorage`: barriers, epilogue scratch, then the A, B, SFA and SFB -/// rings. The ring is as deep as the capacity allows. -pub const Tile = struct { - n: i64, - stages: usize, - smem_bytes: i64, - tmem_columns: u16, - empty_barriers: i64, - accumulator_full: i64, - accumulator_empty: i64, - tmem_dealloc: i64, - tmem_holding: i64, - c: i64, - a: i64, - b: i64, - sfa: i64, - sfb: i64, - - /// 128x256 packed FP4. - const a_stage_bytes = 128 * k_tile / 2; - /// One 128x16 E4M3 scale tile: four 512-byte 128x4 atoms. - const sf_stage_bytes = 128 * k_tile / 16; - const c_bytes = 512; - const header_bytes = 1024; - - pub fn of(comptime n: i64) Tile { - comptime { - std.debug.assert(n >= 8 and n <= 128 and @rem(n, 8) == 0); - const per_stage = a_stage_bytes + @divExact(n * k_tile, 2) + 2 * sf_stage_bytes; - const stages: usize = @intCast(@divTrunc(smem_capacity - header_bytes - c_bytes, per_stage)); - const s: i64 = @intCast(stages); - const a = header_bytes + 1024; - const b = a + s * a_stage_bytes; - const sfa = b + @divExact(s * n * k_tile, 2); - const sfb = sfa + s * sf_stage_bytes; - // Accumulators, then 16 SFA and 16 SFB columns. - const columns = std.math.ceilPowerOfTwo(u16, @max(32, accumulator_stages * n + 32)) catch unreachable; - return .{ - .n = n, - .stages = stages, - .smem_bytes = sfb + s * sf_stage_bytes, - .tmem_columns = columns, - .empty_barriers = 8 * s, - .accumulator_full = 16 * s, - .accumulator_empty = 16 * s + 16, - .tmem_dealloc = 16 * s + 32, - .tmem_holding = 16 * s + 40, - .c = header_bytes, - .a = a, - .b = b, - .sfa = sfa, - .sfb = sfb, - }; - } - } - - /// Routed rows an epilogue pass handles: one per warp lane. - pub fn chunkRows(self: Tile) i64 { - return @min(self.n, 32); - } - - /// Epilogue passes over the tile width. - pub fn chunks(self: Tile) usize { - return @intCast(@divExact(self.n, self.chunkRows())); - } - - fn bStageBytes(self: Tile) i64 { - return @divExact(self.n * k_tile, 2); - } - - /// TMA bytes landing in one stage: packed A and B, then both scale tiles. - fn transactionBytes(self: Tile) u32 { - return @intCast(a_stage_bytes + self.bStageBytes() + 2 * sf_stage_bytes); - } - - fn fields(self: Tile, b: *B) []const []const u8 { - const arena = b.arena.allocator(); - const s: i64 = @intCast(self.stages); - const out = arena.alloc([]const u8, 11) catch @panic("OOM"); - const Field = struct { []const u8, i64, i64 }; - const list = [_]Field{ - .{ "ab_full_mbar_ptr", 8 * s, 0 }, - .{ "ab_empty_mbar_ptr", 8 * s, self.empty_barriers }, - .{ "acc_full_mbar_ptr", 16, self.accumulator_full }, - .{ "acc_empty_mbar_ptr", 16, self.accumulator_empty }, - .{ "tmem_dealloc_mbar", 8, self.tmem_dealloc }, - .{ "tmem_holding_buf", 4, self.tmem_holding }, - .{ "sC", c_bytes, self.c }, - .{ "sA", s * a_stage_bytes, self.a }, - .{ "sB", s * self.bStageBytes(), self.b }, - .{ "sSFA", s * sf_stage_bytes, self.sfa }, - .{ "sSFB", s * sf_stage_bytes, self.sfb }, - }; - for (list, out) |field, *text| { - text.* = std.fmt.allocPrint(arena, "{s}:{d}:{d}", .{ field[0], field[1], field[2] }) catch @panic("OOM"); - } - return out; - } -}; - -const StageStorage = struct { - a: V, - b: V, - sfa: V, - sfb: V, - c: V, - load_barriers: V, - mma_barriers: V, - accumulator_full_barriers: V, - accumulator_empty_barriers: V, -}; - -const StageBuffers = struct { - a_destination: cute.View, - b_destination: cute.View, - sfa_destination: cute.View, - sfb_destination: cute.View, - load_barrier: V, - mma_barrier: V, -}; - -const StagedOperands = struct { - a: cute.View, - b: cute.View, - sfa: cute.View, - sfb: cute.View, -}; - -/// What the epilogue warps do with each finished 128xN accumulator tile. -pub const Epilogue = enum { - /// Up projection: per-expert dequantization scale, BF16 rounding, - /// SwiGLU of the interleaved gate/up rows (optionally clamped), the - /// routing weight when it precedes the down projection, BF16 rounding, - /// then NVFP4 quantization (E4M3 scale per 16 activations, scaled by the - /// per-expert activation global scale) straight into the down GEMM's - /// packed input and 128x4 scale layouts. - swiglu_nvfp4, - /// Down projection: per-expert dequantization scale and, when it follows - /// the down projection, the routing weight, then BF16 rows stored in - /// route order `[routes, m]`, skipping padding rows. - route_rows, -}; - -pub const Config = struct { - experts: i64, - /// GEMM output rows (2 x intermediate for gate/up, hidden for down). - m: i64, - /// Tile N: rows in one routed expert group. - n: i64 = 16, - /// Reduction dimension, a multiple of `k_tile`. - k: i64, - groups: i64, - /// Resident CTAs of the static persistent tile scheduler: one per SM. - persistent_ctas: i64 = 148, - epilogue: Epilogue, - /// Token/top-k routes; sizes the route-indexed epilogue operands. - routes: i64, - /// Clamp of both halves of the fused SwiGLU, if any. - swiglu_limit: ?f32 = null, - /// Multiply rows by their routing weight in this projection's epilogue. - routing_weights: bool = false, - /// Ungrouped routing: group `g` is route `g` and holds one live row. - direct: bool = false, - - /// Rows stored per group in the routed operands: one whole N tile. - pub fn groupRows(self: Config) i64 { - return self.n; - } - - pub fn routedRows(self: Config) i64 { - return self.groups * self.groupRows(); - } -}; - -/// `schedule` is the routing schedule `[group experts | group sizes | active -/// group count]`; `alpha` the per-expert FP32 dequantization scale of the -/// GEMM output. -const inputs = [_][:0]const u8{ - "weight", - "schedule", - "input_quant", - "weight_scale", - "input_scale", - "alpha", -}; - -/// `route_inverse` maps each physical routed row to its token/top-k route; -/// `act_scale` is the per-expert global scale of the down GEMM's NVFP4 input. -const UpQuantized = cute.Program(Config, .{ - .name = "zml_nvfp4_up_swiglu_zig", - .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights", "act_scale" }), - .outputs = &.{ "q", "s" }, - .run = buildProgram, -}); - -const DownRows = cute.Program(Config, .{ - .name = "zml_nvfp4_down_rows_zig", - .inputs = &(inputs ++ [_][:0]const u8{ "route_inverse", "routing_weights" }), - .outputs = &.{"output"}, - .run = buildProgram, -}); - -pub const Inputs = struct { - /// `[experts, m, k / 2]` packed FP4. - weight: zml.Tensor, - schedule: zml.Tensor, - /// `[groups * n, k / 2]` packed FP4. - input_quant: zml.Tensor, - /// `[experts, m, k / 16]` E4M3 bytes in the 128x4 layout. - weight_scale: zml.Tensor, - /// `[groups, k / 16 * 128]` E4M3 bytes in the 128x4 layout. - input_scale: zml.Tensor, - /// `[experts]` FP32. - alpha: zml.Tensor, -}; - -/// Up projection with the SwiGLU + NVFP4 epilogue: the routed packed FP4 -/// rows `[groups * n, m / 4]` and their 128x4 E4M3 scales -/// `[groups, m / 32 * 128]`, ready for the down GEMM. `routing_weights` is -/// `[routes]` FP32 and `act_scale` `[experts]` FP32. -pub fn upQuantized(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor, act_scale: zml.Tensor) struct { q: zml.Tensor, s: zml.Tensor } { - std.debug.assert(cfg.epilogue == .swiglu_nvfp4); - const activations = @divExact(cfg.m, 2); - const result = UpQuantized.call(.{ - .weight = a.weight, - .schedule = a.schedule, - .input_quant = a.input_quant, - .weight_scale = a.weight_scale, - .input_scale = a.input_scale, - .alpha = a.alpha, - .route_inverse = route_inverse, - .routing_weights = routing_weights, - .act_scale = act_scale, - }, .{ - .q = .init(.{ cfg.routedRows(), @divExact(activations, 2) }, .u8), - .s = .init(.{ cfg.groups, @divExact(activations, 16) * 128 }, .u8), - }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }); - return .{ .q = result.q, .s = result.s }; -} - -/// Down projection storing BF16 `[routes, m]` in token/top-k order. -pub fn downRows(cfg: Config, a: Inputs, route_inverse: zml.Tensor, routing_weights: zml.Tensor) zml.Tensor { - std.debug.assert(cfg.epilogue == .route_rows); - return DownRows.call(.{ - .weight = a.weight, - .schedule = a.schedule, - .input_quant = a.input_quant, - .weight_scale = a.weight_scale, - .input_scale = a.input_scale, - .alpha = a.alpha, - .route_inverse = route_inverse, - .routing_weights = routing_weights, - }, .{ .output = .init(.{ @max(cfg.routes, 1), cfg.m }, .bf16) }, .{ .cfg = cfg, .scalars = &.{ cfg.m, cfg.n, cfg.k, cfg.groups, cfg.experts } }).output; -} - -const typed = common.typed; -const gridTyped = common.gridTyped; -const fmt = common.fmt; -const view = common.view; -const barrierAt = common.barrierAt; -const PipelinePosition = common.PipelinePosition; -const DirectRoutes = common.DirectRoutes; -const persistentWorkAt = common.persistentWorkAt; -const advancePersistentWork = common.advancePersistentWork; -const advancePipeline = common.advancePipeline; -const peekBarrierIfRemaining = common.peekBarrierIfRemaining; -const sharedPointerAt = common.sharedPointerAt; -const offsetAlignment = common.offsetAlignment; -const typesOf = common.typesOf; -const tupleOf = common.tupleOf; - -fn deviceName(comptime epilogue: Epilogue) [:0]const u8 { - return switch (epilogue) { - .swiglu_nvfp4 => "nvfp4_sm100_persistent_grouped_gemm_up_swiglu", - .route_rows => "nvfp4_sm100_persistent_grouped_gemm_down_rows", - }; -} - -pub const tile_widths = [_]i64{ 8, 16, 32, 64, 128 }; - -fn buildProgram(b: *B, cfg: Config) cute.FinishError!void { - return switch (cfg.n) { - inline 8, 16, 32, 64, 128 => |n| switch (cfg.epilogue) { - inline else => |epilogue| buildTiledProgram(b, cfg, comptime Tile.of(n), epilogue), - }, - else => std.debug.panic("unsupported NVFP4 tile N {d}", .{cfg.n}), - }; -} - -/// Types and layouts shared by the device ABI and the host launch, as the -/// CuTe DSL derives them for FP4 x FP4 with 16-element E4M3 scale vectors. -fn Layouts(comptime tile: Tile) type { - return struct { - const n = tile.n; - const stages = tile.stages; - const ab_swizzle: B.Swizzle = .{ .bits = 3, .base = 4, .shift = 3 }; - - fn aCoord(b: *B, cfg: Config) cute.LayoutSpec { - return b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); - } - fn bCoord(b: *B, cfg: Config) cute.LayoutSpec { - return b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); - } - /// TMA coordinates of the 128x4 scale layout: byte in the atom, K - /// atom, M block, L. - fn sfCoord(b: *B) cute.LayoutSpec { - return b.layoutSpec( - .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 16, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, - .{ - .{ .{ b.basis(8, 1, .{0}), b.basis(2, 1, .{0}) }, b.basis(1, 1, .{2}) }, - .{ .{ 0, b.basis(1, 2, .{0}) }, b.basis(1, 1, .{1}) }, - .{ 0, b.basis(1, 1, .{3}) }, - }, - ); - } - fn aBasis(b: *B) cute.LayoutSpec { - return b.layoutSpec(.{ k_tile, 128, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); - } - fn bBasis(b: *B) cute.LayoutSpec { - return b.layoutSpec(.{ k_tile, n, 1 }, .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), b.basis(1, 1, .{2}) }); - } - fn sfBasis(b: *B) cute.LayoutSpec { - return b.layoutSpec( - .{ .{ 2, 4, 32 }, 4, 1, 1 }, - .{ .{ b.basis(1, 1, .{ 1, 0, 1 }), b.basis(1, 1, .{ 0, 0, 1 }), b.basis(1, 1, .{ 0, 0, 0 }) }, b.basis(1, 1, .{ 1, 1 }), b.basis(1, 1, .{ 0, 1 }), b.basis(1, 1, .{ 2, 1 }) }, - ); - } - fn tmaA(b: *B, cfg: Config) cute.TmaConfig { - return .{ .dtype = .f4e2m1fn, .copy_bits = 128 * k_tile * 4, .global_basis = aBasis(b), .coordinate_layout = aCoord(b, cfg), .coordinate_rank = 3, .format = .u4 }; - } - fn tmaB(b: *B, cfg: Config) cute.TmaConfig { - return .{ .dtype = .f4e2m1fn, .copy_bits = @intCast(n * k_tile * 4), .global_basis = bBasis(b), .coordinate_layout = bCoord(b, cfg), .coordinate_rank = 3, .format = .u4 }; - } - fn tmaSf(b: *B) cute.TmaConfig { - return .{ .dtype = .f8e4m3fn, .copy_bits = Tile.sf_stage_bytes * 8, .global_basis = sfBasis(b), .coordinate_layout = sfCoord(b), .coordinate_rank = 4, .format = .u16 }; - } - // Shared-memory layouts in FP4 elements for A/B, bytes for scales. - fn aSmem(b: *B, comptime staged: bool) cute.LayoutSpec { - return if (staged) - b.layoutSpec(.{ .{ 128, 64 }, 1, 4, stages }, .{ .{ k_tile, 1 }, 0, 64, 2 * Tile.a_stage_bytes }) - else - b.layoutSpec(.{ .{ 128, 64 }, 1, 4 }, .{ .{ k_tile, 1 }, 0, 64 }); - } - fn bSmem(b: *B, comptime staged: bool) cute.LayoutSpec { - return if (staged) - b.layoutSpec(.{ .{ n, 64 }, 1, 4, stages }, .{ .{ k_tile, 1 }, 0, 64, 2 * tile.bStageBytes() }) - else - b.layoutSpec(.{ .{ n, 64 }, 1, 4 }, .{ .{ k_tile, 1 }, 0, 64 }); - } - fn sfSmem(b: *B, comptime staged: bool) cute.LayoutSpec { - return if (staged) - b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 16, 4 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512, Tile.sf_stage_bytes }) - else - b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 16, 4 } }, 1, 4 }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512 }); - } - fn fp4SmemPtrTy(b: *B) *const mlir.Type { - return b.swizzledPtrTy(.f4e2m1fn, .smem, 1024, ab_swizzle) catch @panic("bad FP4 shared pointer"); - } - }; -} - -/// Device memrefs of the epilogue operands. -const EpilogueTypes = struct { - sizes: *const mlir.Type, - inverse: *const mlir.Type, - weights: *const mlir.Type, - per_expert: *const mlir.Type, - q: *const mlir.Type, - s: *const mlir.Type, - rows: *const mlir.Type, - - fn of(b: *B, cfg: Config) EpilogueTypes { - const activations = @divExact(cfg.m, 2); - return .{ - .sizes = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(cfg.groups, 1))), - .inverse = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.routedRows(), 1))), - .weights = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(@max(cfg.routes, 1), 1))), - .per_expert = b.memrefType(.f32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.experts, 1))), - // Eight packed FP4 activations per word. - .q = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.routedRows(), @divExact(activations, 8) }, .{ @divExact(activations, 8), 1 }))), - .s = b.memrefType(.i8, .gmem, 16, b.layoutType(b.layoutSpec(.{ cfg.groups, @divExact(activations, 16) * 128 }, .{ @divExact(activations, 16) * 128, 1 }))), - .rows = b.memrefType(.bf16, .gmem, 16, b.layoutType(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), - }; - } -}; - -fn buildTiledProgram(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { - @setEvalBranchQuota(20_000); - std.debug.assert(@rem(cfg.m, 128) == 0); - std.debug.assert(@rem(cfg.k, k_tile) == 0); - const L = Layouts(tile); - const public_name = b.name; - - const ids_memref = b.memrefType(.i32, .gmem, 16, b.layoutType(b.layoutSpec(cfg.groups, 1))); - const active_memref = b.memrefType(.i32, .gmem, 4, b.layoutType(b.layoutSpec(1, 1))); - const et = EpilogueTypes.of(b, cfg); - const i32_type = cute.DType.i32.toMlir(b.ctx); - const mma_type = b.blockScaledMmaType(.{ .kind = mma_kind, .n = @intCast(tile.n) }); - const mma_sfb_type = b.blockScaledMmaType(.{ .kind = mma_kind, .n = 128 }); - - b.beginFunction(deviceName(epilogue), .cuda_kernel); - const d = try b.declareArgs(switch (epilogue) { - .swiglu_nvfp4 => .{ - .expert_ids = typed(ids_memref), - .active_count = typed(active_memref), - .mma = typed(mma_type), - .mma_sfb = typed(mma_sfb_type), - .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), - .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), - .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), - .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), - .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), - .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), - .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), - .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), - .alpha = typed(et.per_expert), - .group_sizes = typed(et.sizes), - .route_inverse = typed(et.inverse), - .routing_weights = typed(et.weights), - .act_scale = typed(et.per_expert), - .q = typed(et.q), - .s = typed(et.s), - .tiles_m = typed(i32_type), - }, - .route_rows => .{ - .expert_ids = typed(ids_memref), - .active_count = typed(active_memref), - .mma = typed(mma_type), - .mma_sfb = typed(mma_sfb_type), - .tma_a = gridTyped(b.tmaLoadAtomType(L.tmaA(b, cfg))), - .gA = typed(b.coordTensorType(3, L.aCoord(b, cfg))), - .tma_b = gridTyped(b.tmaLoadAtomType(L.tmaB(b, cfg))), - .gB = typed(b.coordTensorType(3, L.bCoord(b, cfg))), - .tma_sfa = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), - .gSFA = typed(b.coordTensorType(4, L.sfCoord(b))), - .tma_sfb = gridTyped(b.tmaLoadAtomType(L.tmaSf(b))), - .gSFB = typed(b.coordTensorType(4, L.sfCoord(b))), - .alpha = typed(et.per_expert), - .group_sizes = typed(et.sizes), - .route_inverse = typed(et.inverse), - .routing_weights = typed(et.weights), - .rows = typed(et.rows), - .tiles_m = typed(i32_type), - }, - }); - b.setFunctionAttribute(nvvm.minctasm(b.ctx, 1)); - b.setFunctionAttribute(.named(b.ctx, "smem.partition_num", .int(b.ctx, .i32, 2))); - b.setFunctionAttribute(.named(b.ctx, "smem.hint_smem_base_uniform", .unit(b.ctx))); - b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{ - .named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin()), - .named(b.ctx, "non_portable_cluster_size_allowed", .int(b.ctx, .i32, 1)), - }))); - - // Prefetch every tensor-map descriptor while the producing grid drains. - const warp = b.makeWarpUniform(b.makeWarpUniform(b.threadIdx().x.div(32))); - var tma_warp = b.openIf(warp.eq(5)); - inline for (.{ d.tma_a, d.tma_b, d.tma_sfa, d.tma_sfb }) |value| { - const descriptor: cute.Atom = .{ .inner = value.inner, .kernel = b }; - b.prefetchTmaDesc(descriptor); - } - tma_warp.close(); - b.waitForDependency(); - buildUmmaDevice(b, cfg, tile, epilogue, d); - b.launchDependents(); - b.endFunction(.{ 192, 1, 1 }); - - // Public host ABI: the XLA operands, the outputs, then the problem shape. - b.beginFunction(public_name, .host); - const operands = .{ - .weight = .{ .ptr = cute.DType.i8 }, - .schedule = .{ .ptr = cute.DType.i32 }, - .input_quant = .{ .ptr = cute.DType.i8 }, - .weight_scale = .{ .ptr = cute.DType.i8 }, - .input_scale = .{ .ptr = cute.DType.i8 }, - .alpha = .{ .ptr = cute.DType.f32 }, - .route_inverse = .{ .ptr = cute.DType.i32 }, - .routing_weights = .{ .ptr = cute.DType.f32 }, - }; - const shape = .{ - .problem_m = typed(i32_type), - .problem_n = typed(i32_type), - .problem_k = typed(i32_type), - .problem_l = typed(i32_type), - .expert_count = typed(i32_type), - }; - _ = try b.declareArgs(switch (epilogue) { - .swiglu_nvfp4 => .{ - .weight = operands.weight, - .schedule = operands.schedule, - .input_quant = operands.input_quant, - .weight_scale = operands.weight_scale, - .input_scale = operands.input_scale, - .alpha = operands.alpha, - .route_inverse = operands.route_inverse, - .routing_weights = operands.routing_weights, - .act_scale = .{ .ptr = cute.DType.f32 }, - .q = .{ .ptr = cute.DType.i8 }, - .s = .{ .ptr = cute.DType.i8 }, - .problem_m = shape.problem_m, - .problem_n = shape.problem_n, - .problem_k = shape.problem_k, - .problem_l = shape.problem_l, - .expert_count = shape.expert_count, - }, - .route_rows => .{ - .weight = operands.weight, - .schedule = operands.schedule, - .input_quant = operands.input_quant, - .weight_scale = operands.weight_scale, - .input_scale = operands.input_scale, - .alpha = operands.alpha, - .route_inverse = operands.route_inverse, - .routing_weights = operands.routing_weights, - .output = .{ .ptr = cute.DType.bf16 }, - .problem_m = shape.problem_m, - .problem_n = shape.problem_n, - .problem_k = shape.problem_k, - .problem_l = shape.problem_l, - .expert_count = shape.expert_count, - }, - }); - try buildHostLaunch(b, cfg, tile, epilogue); - b.endFunction(null); -} - -/// Host argument positions: the six mainloop operands, the route operands, -/// the epilogue's extra inputs and outputs, then the five shape scalars. -fn HostArgs(comptime epilogue: Epilogue) type { - return struct { - const weight = 0; - const schedule = 1; - const input_quant = 2; - const weight_scale = 3; - const input_scale = 4; - const alpha = 5; - const route_inverse = 6; - const routing_weights = 7; - const act_scale = 8; - const output = switch (epilogue) { - .swiglu_nvfp4 => 9, - .route_rows => 8, - }; - const scales = 10; - const problem_m = switch (epilogue) { - .swiglu_nvfp4 => 11, - .route_rows => 9, - }; - const problem_n = problem_m + 1; - const problem_k = problem_m + 2; - const problem_l = problem_m + 3; - const expert_count = problem_m + 4; - }; -} - -/// Complete staged shared-memory tensors. Per-stage views are slices. -fn stagedOperands(b: *B, comptime tile: Tile, storage: StageStorage) StagedOperands { - const L = Layouts(tile); - const a_full = b.makeView( - b.recastSwizzledPointer(storage.a, .f4e2m1fn, .smem, 1024, L.ab_swizzle), - b.staticLayout(L.aSmem(b, true)), - b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.aSmem(b, true))), - ); - const b_full = b.makeView( - b.recastSwizzledPointer(storage.b, .f4e2m1fn, .smem, 1024, L.ab_swizzle), - b.staticLayout(L.bSmem(b, true)), - b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.bSmem(b, true))), - ); - const sfa_full = b.makeView(storage.sfa, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); - const sfb_full = b.makeView(storage.sfb, b.staticLayout(L.sfSmem(b, true)), b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(L.sfSmem(b, true)))); - return .{ .a = a_full, .b = b_full, .sfa = sfa_full, .sfb = sfb_full }; -} - -fn stageBuffersAt(b: *B, comptime tile: Tile, storage: StageStorage, stage_index: V) StageBuffers { - const L = Layouts(tile); - const full = stagedOperands(b, tile, storage); - const all = cute.AlgebraToken.all; - const a = b.slice(full.a, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.aSmem(b, false)))); - const rhs = b.slice(full.b, .{ all, all, all, stage_index }, b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(L.bSmem(b, false)))); - const sf_stage_align = 1024; - const sfa = b.slice(full.sfa, .{ all, all, all, stage_index }, b.memrefType(.f8e4m3fn, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); - const sfb = b.slice(full.sfb, .{ all, all, all, stage_index }, b.memrefType(.f8e4m3fn, .smem, sf_stage_align, b.layoutType(L.sfSmem(b, false)))); - - // `tma_partition` keeps a singleton CTA mode; the copy views keep it too. - const a_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ 2 * Tile.a_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); - const b_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ 2 * tile.bStageBytes(), 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); - const sf_partition_layout = b.staticLayout(b.layoutSpec(.{ .{ Tile.sf_stage_bytes, 1 }, .{1} }, .{ .{ 1, 0 }, .{0} })); - - return .{ - .a_destination = b.withLayout(a, a_partition_layout), - .b_destination = b.withLayout(rhs, b_partition_layout), - .sfa_destination = b.withLayout(sfa, sf_partition_layout), - .sfb_destination = b.withLayout(sfb, sf_partition_layout), - .load_barrier = barrierAt(b, storage.load_barriers, stage_index), - .mma_barrier = barrierAt(b, storage.mma_barriers, stage_index), - }; -} - -/// The TMA coordinates of one stage: slice the TMA-partitioned operand at -/// its output tile and L index, then at the K stage, and rebuild the -/// iterator as the Python front end does. -fn tmaSource(b: *B, partitioned: cute.View, tile_coord: anytype, k: V, comptime rows: i64, reduction_tiles: usize, copy_layout: cute.View) cute.View { - const all = cute.AlgebraToken.all; - const tile_k = b.slice( - partitioned, - tile_coord, - b.coordTensorTypeWithTuple(.{ 0, cute.ConstrainedDynamic{ .divisible_by = rows }, cute.AlgebraToken.dynamic }, b.layoutSpec( - .{ .{ .{ k_tile, rows }, 1 }, reduction_tiles }, - .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(k_tile, 1, .{0}) }, - )), - ); - const div_k = cute.ConstrainedDynamic{ .divisible_by = k_tile }; - const div_rows = cute.ConstrainedDynamic{ .divisible_by = rows }; - const dynamic = cute.AlgebraToken.dynamic; - const stage = b.slice( - tile_k, - .{ all, k }, - b.coordTensorTypeWithTuple(.{ div_k, div_rows, dynamic }, b.layoutSpec( - .{.{ .{ k_tile, rows }, 1 }}, - .{.{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }}, - )), - ); - return b.rebuildCoordTensorIterator(3, stage, .{ div_k, div_rows, dynamic }, .{ k_tile, rows, 1 }, copy_layout); -} - -/// Issue the four transactions that fill one mainloop stage. -fn issueTmaStage( - b: *B, - comptime tile: Tile, - reduction_tiles: usize, - partitioned_a_target: cute.View, - partitioned_b_target: cute.View, - stage: StageBuffers, - k: V, - tile_m: V, - group: V, - expert: V, - exec_a: cute.Atom, - exec_b: cute.Atom, - exec_sfa: cute.Atom, - exec_sfb: cute.Atom, -) void { - const n = tile.n; - const all = cute.AlgebraToken.all; - const a_copy_layout = b.staticLayout(b.layoutSpec( - .{ .{ .{ k_tile, 128 }, 1 }, .{1} }, - .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, - )); - const b_copy_layout = b.staticLayout(b.layoutSpec( - .{ .{ .{ k_tile, n }, 1 }, .{1} }, - .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, - )); - // One stage moves four 512-byte atoms, one per K = 64 slice. - const sf_copy_spec = b.layoutSpec( - .{ .{ .{ 512, 4 }, 1 }, .{1} }, - .{ .{ .{ b.basis(1, 2, .{0}), b.basis(1, 1, .{1}) }, 0 }, .{0} }, - ); - // B's tile spans all of a group's rows, so its N-tile index is zero. Pass - // it as a value: a static zero leaf collapses out of the coordinate tuple - // and shifts the remaining tensor-map coordinates. - const b_tile_n = b.constant(.i32, 0); - const a_source = tmaSource(b, partitioned_a_target, .{ all, tile_m, all, expert }, k, 128, reduction_tiles, a_copy_layout); - const b_source = tmaSource(b, partitioned_b_target, .{ all, b_tile_n, all, group }, k, n, reduction_tiles, b_copy_layout); - // Scale coordinates: (byte, K atom, M block, L); a stage covers 4 K atoms. - const k_atom = k.mul(4); - const sfa_source = b.makeCoordTensor(.{ 0, k_atom, tile_m, expert }, b.staticLayout(sf_copy_spec)); - const sfb_source = b.makeCoordTensor(.{ 0, k_atom, b.constant(.i32, 0), group }, b.staticLayout(sf_copy_spec)); - - var elected = b.openIf(b.electSync()); - b.mbarrierArriveExpectTx(stage.load_barrier, tile.transactionBytes()); - elected.close(); - b.copy(b.setTmaBarrier(exec_a, stage.load_barrier), .{a_source}, .{stage.a_destination}, .{}); - b.copy(b.setTmaBarrier(exec_b, stage.load_barrier), .{b_source}, .{stage.b_destination}, .{}); - b.copy(b.setTmaBarrier(exec_sfa, stage.load_barrier), .{sfa_source}, .{stage.sfa_destination}, .{}); - b.copy(b.setTmaBarrier(exec_sfb, stage.load_barrier), .{sfb_source}, .{stage.sfb_destination}, .{}); -} - -fn accumulatorTmemAt(b: *B, comptime tile: Tile, tmem: V, stage: V) V { - return b.addOffset( - tmem, - b.makeIntTupleDivBy(stage.mul(tile.n), @intCast(tile.n)), - b.ptrTy(.f32, .tmem, 16) catch @panic("bad accumulator TMEM pointer"), - ).value(); -} - -/// Warp 5 runs independently over every persistent output tile. -fn runTmaProducer( - b: *B, - comptime tile: Tile, - k_tiles: usize, - storage: StageStorage, - active: V, - tiles_m: V, - expert_ids: cute.View, - direct: ?DirectRoutes, - partitioned_a_target: cute.View, - partitioned_b_target: cute.View, - exec_a: cute.Atom, - exec_b: cute.Atom, - exec_sfa: cute.Atom, - exec_sfb: cute.Atom, -) void { - const initial_index = b.constant(.i32, 0); - const initial_phase = b.constant(.i32, 1); - const initial_count = b.constant(.i32, 0); - const persistent_clusters = b.gridDim().z; - const cta_m = b.blockIdx().x; - const cta_n = b.blockIdx().y; - const cta_l = b.constant(.i32, 0); - const div_m = b.fastDivmodCreate(tiles_m); - const div_n = b.fastDivmodCreate(1); - const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); - const carried = .{ - initial_work.m, initial_work.n, initial_work.group, initial_work.valid, - initial_count, initial_index, initial_phase, persistent_clusters, - initial_work.linear, cta_m, cta_n, cta_l, - initial_work.executed, tiles_m, active, div_m, - div_n, - }; - var tiles = b.openWhile(carried, typesOf(carried)); - tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); - const tile_m = tiles.after_carried[0]; - const group = tiles.after_carried[2]; - const expert = expert_ids.get(.{group}, .i32); - const first_state: PipelinePosition = .{ - .index = tiles.after_carried[5], - .phase = tiles.after_carried[6], - }; - const first_ready = b.mbarrierTryWait(barrierAt(b, storage.mma_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); - var stages = b.openFor( - 0, - @as(i32, @intCast(k_tiles)), - 1, - .{ first_ready, b.constant(.i32, 0), first_state.index, first_state.phase }, - ); - const k = stages.carried[1]; - const state: PipelinePosition = .{ - .index = stages.carried[2], - .phase = stages.carried[3], - }; - const buffers = stageBuffersAt(b, tile, storage, state.index); - var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierWait(buffers.mma_barrier, state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); - pending.close(); - issueTmaStage(b, tile, k_tiles, partitioned_a_target, partitioned_b_target, buffers, k, tile_m, group, expert, exec_a, exec_b, exec_sfa, exec_sfb); - const next_count = stages.carried[1].add(1); - const next_state = advancePipeline(b, state.index, state.phase, tile.stages); - const next_ready = peekBarrierIfRemaining(b, storage.mma_barriers, next_count, k_tiles, next_state); - stages.yield(.{ next_ready, next_count, next_state.index, next_state.phase }); - const next_work = advancePersistentWork( - b, - tiles.after_carried[8], - tiles.after_carried[12], - tiles.after_carried[7], - tiles.after_carried[14], - tiles.after_carried[13], - tiles.after_carried[15], - tiles.after_carried[16], - direct, - ); - tiles.yieldAfter(.{ - next_work.m, - next_work.n, - next_work.group, - next_work.valid, - stages.results[1], - stages.results[2], - stages.results[3], - tiles.after_carried[7], - next_work.linear, - tiles.after_carried[9], - tiles.after_carried[10], - tiles.after_carried[11], - next_work.executed, - tiles.after_carried[13], - tiles.after_carried[14], - tiles.after_carried[15], - tiles.after_carried[16], - }); - - // PipelineTmaUmma.producer_tail: drain every empty barrier before exit. - var tail_state: PipelinePosition = .{ - .index = tiles.results[5], - .phase = tiles.results[6], - }; - inline for (0..tile.stages) |stage| { - b.mbarrierWait(barrierAt(b, storage.mma_barriers, tail_state.index), tail_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); - if (stage + 1 < tile.stages) { - tail_state = advancePipeline(b, tail_state.index, tail_state.phase, tile.stages); - } - } -} - -/// NVFP4 mainloop objects of one pipeline stage. -const Mainloop = struct { - a: V, - b: V, - accumulator: V, - sfa: V, - sfb: V, - sfa_copy: cute.Atom, - sfb_copy: cute.Atom, - sfa_source: V, - sfb_source: V, - sfa_target: V, - sfb_target: V, - - pub fn copyScales(self: Mainloop, builder: *B) void { - builder.copy(self.sfa_copy, .{self.sfa_source}, .{self.sfa_target}, .{}); - builder.copy(self.sfb_copy, .{self.sfb_source}, .{self.sfb_target}, .{}); - } -}; - -/// Pipeline-wide UMMA and S2T descriptors derived from the complete staged -/// tensors once; `atStage` slices them at the current stage index. -const StagedMainloop = struct { - a: cute.View, - b: cute.View, - sfa_source: cute.View, - sfb_source: cute.View, - sfa: V, - sfb: V, - sfa_copy: cute.Atom, - sfb_copy: cute.Atom, - sfa_target: V, - sfb_target: V, - n: i64, - - pub fn atStage(self: StagedMainloop, b: *B, index: V, accumulator_tmem: V) Mainloop { - const all = cute.AlgebraToken.all; - const a = b.slice(self.a, .{ all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 }))); - const rhs = b.slice(self.b, .{ all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4 }, .{ 0, 0, 2 }))); - const sfa_stage = b.slice(self.sfa_source, .{ all, all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32 }))); - const sfb_stage = b.slice(self.sfb_source, .{ all, all, all, all, index }, b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32 }))); - const source_grouped_spec = b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, .{ 1, 1, 4 } }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, .{ 0, 0, 32 } }); - const source_grouped_layout = b.staticLayout(source_grouped_spec); - const source_grouped_ty = b.smemDescViewType(source_grouped_spec); - const sfa_source = b.makeView(b.getIter(sfa_stage, b.smemDescType()), source_grouped_layout, source_grouped_ty); - const sfb_source = b.makeView(b.getIter(sfb_stage, b.smemDescType()), source_grouped_layout, source_grouped_ty); - const acc_spec = b.layoutSpec(.{ .{ 128, self.n }, 1, 1 }, .{ .{ 65536, 1 }, 0, 0 }); - const acc_layout = b.staticLayout(acc_spec); - const accumulator = b.makeView( - accumulator_tmem, - acc_layout, - b.memrefType(.f32, .tmem, 16, b.layoutType(acc_spec)), - ); - return .{ - .a = a.value(), - .b = rhs.value(), - .accumulator = accumulator.value(), - .sfa = self.sfa, - .sfb = self.sfb, - .sfa_copy = self.sfa_copy, - .sfb_copy = self.sfb_copy, - .sfa_source = sfa_source.value(), - .sfb_source = sfb_source.value(), - .sfa_target = self.sfa_target, - .sfb_target = self.sfb_target, - }; - } -}; - -fn makeStagedMainloop( - b: *B, - tiled_mma: cute.Atom, - comptime n: i64, - comptime stages: usize, - smem_a: V, - smem_b: V, - smem_sfa: V, - smem_sfb: V, - scale_tmem: V, -) StagedMainloop { - // Descriptor strides are in 16-byte units: one 64-deep K slice of packed - // FP4 is 32 bytes, a 128x256 A stage 16 KiB and an Nx256 B stage N*128 bytes. - const frag_a = b.mmaMakeFragment( - tiled_mma, - smem_a, - 0, - b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, Tile.a_stage_bytes / 16 })), - ); - const frag_b = b.mmaMakeFragment( - tiled_mma, - smem_b, - 1, - b.smemDescViewType(b.layoutSpec(.{ 1, 1, 4, stages }, .{ 0, 0, 2, n * 8 })), - ); - - // TMEM columns: two N-wide accumulator stages, then 16 SFA and 16 SFB - // columns: four K = 64 slices of four E4M3 scales per lane each. - const sf_spec = b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 16, 4 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 1 } }, 0, 16 }); - const sf_layout = b.staticLayout(sf_spec); - const sfa_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n), b.ptrTy(.f32, .tmem, 16) catch @panic("bad TMEM pointer")); - const sfb_col = b.addOffset(scale_tmem, b.staticIntTuple(2 * n + 16), b.ptrTy(.f32, .tmem, 16) catch @panic("bad TMEM pointer")); - const sfa_ptr = b.recastIter(sfa_col, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")); - const sfb_ptr = b.recastIter(sfb_col, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")); - const sf_memref = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(sf_spec)); - const sfa = b.makeView(sfa_ptr, sf_layout, sf_memref); - const sfb = b.makeView(sfb_ptr, sf_layout, sf_memref); - - const smem_filtered_ty = b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 1 }, .{ 1, 4 } }, 1, 4, stages }, .{ .{ .{ .{ 16, 4 }, 0 }, .{ 0, 1 } }, 0, 512, 2048 }))); - const tmem_filtered_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4 }, 4 }, .{ 1, 4 } }, 1, 4 }, .{ .{ .{ .{ 262144, 4 }, 8388608 }, .{ 0, 1 } }, 0, 16 }))); - const sfa_smem_filtered = b.filterZeros(smem_sfa, null, smem_filtered_ty); - const sfb_smem_filtered = b.filterZeros(smem_sfb, null, smem_filtered_ty); - const sfa_tmem_filtered = b.filterZeros(sfa, null, tmem_filtered_ty); - const sfb_tmem_filtered = b.filterZeros(sfb, null, tmem_filtered_ty); - - const copy_atom_ty = b.s2tCopyAtomType(.f8e4m3fn, .{ .num_dp = 32, .num_bit = 128, .broadcast = .x4 }); - const copy_atom = b.makeAtom(copy_atom_ty, .{}); - const copy_ty = b.tiledCopyType(copy_atom_ty, b.layoutSpec(.{ 1, .{ 32, .{ 4, 4 }, 4 } }, .{ 0, .{ 1, .{ 32, 128 }, 512 } }), &.{ b.layoutSpec(.{ 32, 4, 16 }, .{ 1, 512, 32 }), b.layoutSpec(1, 0), b.layoutSpec(1, 0) }); - const sfa_copy = b.makeS2tCopy(copy_atom, sfa_tmem_filtered, copy_ty); - const sfb_copy = b.makeS2tCopy(copy_atom, sfb_tmem_filtered, copy_ty); - - const source_partition_ty = b.memrefType(.f8e4m3fn, .smem, 1024, b.layoutType(b.layoutSpec(.{ .{ .{ .{ 32, 4, 4 }, 4 }, 1 }, 1, 1, 4, stages }, .{ .{ .{ .{ 16, 4, 4 }, 0 }, 0 }, 0, 0, 512, 2048 }))); - const source_desc_ty = b.smemDescViewType(b.layoutSpec(.{ .{ .{ .{ 32, 1, 1 }, 4 }, 1 }, 1, 1, 4, stages }, .{ .{ .{ .{ 1, 1, 1 }, 0 }, 0 }, 0, 0, 32, 128 })); - const target_partition_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(b.layoutSpec(.{ .{ .{ 32, 16, 4 }, 1 }, 1, 1, 4 }, .{ .{ .{ 262144, 1, 8388608 }, 0 }, 0, 0, 16 }))); - const target_grouped_spec = b.layoutSpec(.{ .{ .{ 32, 16, 4 }, 1 }, .{ 1, 1, 4 } }, .{ .{ .{ 262144, 1, 8388608 }, 0 }, .{ 0, 0, 16 } }); - const target_grouped_ty = b.memrefType(.f8e4m3fn, .tmem, 16, b.layoutType(target_grouped_spec)); - const sfa_src_partition = b.tiledCopyPartition(sfa_copy, sfa_smem_filtered, .{0}, false, source_partition_ty); - const sfb_src_partition = b.tiledCopyPartition(sfb_copy, sfb_smem_filtered, .{0}, false, source_partition_ty); - const sfa_desc = b.s2tSmemDescriptor(sfa_copy, sfa_src_partition, source_desc_ty); - const sfb_desc = b.s2tSmemDescriptor(sfb_copy, sfb_src_partition, source_desc_ty); - const sfa_dst_partition = b.tiledCopyPartition(sfa_copy, sfa_tmem_filtered, .{0}, true, target_partition_ty); - const sfb_dst_partition = b.tiledCopyPartition(sfb_copy, sfb_tmem_filtered, .{0}, true, target_partition_ty); - const target_grouped_layout = b.staticLayout(target_grouped_spec); - const sfa_dst = b.makeView(b.getIter(sfa_dst_partition, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")), target_grouped_layout, target_grouped_ty); - const sfb_dst = b.makeView(b.getIter(sfb_dst_partition, b.ptrTy(.f8e4m3fn, .tmem, 16) catch @panic("bad scale-factor TMEM pointer")), target_grouped_layout, target_grouped_ty); - - return .{ - .a = frag_a, - .b = frag_b, - .sfa_source = sfa_desc, - .sfb_source = sfb_desc, - .sfa = sfa.value(), - .sfb = sfb.value(), - .sfa_copy = sfa_copy, - .sfb_copy = sfb_copy, - .sfa_target = sfa_dst.value(), - .sfb_target = sfb_dst.value(), - .n = n, - }; -} - -/// Warp 4 drains the A/B ring into one of two independent accumulator stages. -fn runMmaConsumer( - b: *B, - comptime tile: Tile, - k_tiles: usize, - storage: StageStorage, - tiled_mma: cute.Atom, - tmem_holding: V, - active: V, - tiles_m: V, -) void { - const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); - const staged = stagedOperands(b, tile, storage); - const staged_mainloop = makeStagedMainloop( - b, - tiled_mma, - tile.n, - tile.stages, - staged.a.value(), - staged.b.value(), - staged.sfa.value(), - staged.sfb.value(), - tmem, - ); - const persistent_clusters = b.gridDim().z; - const div_m = b.fastDivmodCreate(tiles_m); - const div_n = b.fastDivmodCreate(1); - const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, null); - const carried = .{ - initial_work.m, initial_work.n, initial_work.group, initial_work.valid, - b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), tiled_mma.value(), - b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 1), persistent_clusters, - initial_work.linear, b.blockIdx().x, b.blockIdx().y, b.constant(.i32, 0), - initial_work.executed, tiles_m, active, div_m, - div_n, - }; - var tiles = b.openWhile(carried, typesOf(carried)); - tiles.yieldBefore(tiles.before_carried[3], tupleOf(tiles.before_carried)); - const first_state: PipelinePosition = .{ - .index = tiles.after_carried[5], - .phase = tiles.after_carried[6], - }; - const first_ready = b.mbarrierTryWait(barrierAt(b, storage.load_barriers, first_state.index), first_state.phase, .{ .scope = .cta }); - const accumulator_state: PipelinePosition = .{ - .index = tiles.after_carried[9], - .phase = tiles.after_carried[10], - }; - const accumulator_stage = accumulator_state.index; - const accumulator_empty = barrierAt(b, storage.accumulator_empty_barriers, accumulator_stage); - b.mbarrierWait(accumulator_empty, accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); - - const persistent_mma: cute.Atom = .{ .inner = tiles.after_carried[7].inner, .kernel = b }; - const initial_mma = b.setMmaAccumulate(persistent_mma, b.constant(.i1, false)); - var stages = b.openFor( - 0, - @as(i32, @intCast(k_tiles)), - 1, - .{ first_ready, initial_mma.value(), b.constant(.i32, 0), first_state.index, first_state.phase }, - ); - const k = stages.carried[2]; - const state: PipelinePosition = .{ .index = stages.carried[3], .phase = stages.carried[4] }; - var pending = b.openIf(stages.carried[0].eq(false)); - b.mbarrierWait(barrierAt(b, storage.load_barriers, state.index), state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); - pending.close(); - const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, accumulator_stage); - const mainloop = staged_mainloop.atStage(b, state.index, accumulator_tmem); - mainloop.copyScales(b); - const loop_mma: cute.Atom = .{ .inner = stages.carried[1].inner, .kernel = b }; - const mma = b.setMmaAccumulate(loop_mma, k.ne(0)); - b.gemm( - mma, - mainloop.accumulator, - .{ mainloop.a, mainloop.sfa }, - .{ mainloop.b, mainloop.sfb }, - mainloop.accumulator, - ); - var commit_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.mma_barriers, state.index), .{}); - commit_lane.close(); - const next_count = stages.carried[2].add(1); - const next_state = advancePipeline(b, state.index, state.phase, tile.stages); - const next_ready = peekBarrierIfRemaining(b, storage.load_barriers, next_count, k_tiles, next_state); - stages.yield(.{ next_ready, mma.value(), next_count, next_state.index, next_state.phase }); - - var full_lane = b.openIf(b.electSync()); - b.tcgen05Commit(barrierAt(b, storage.accumulator_full_barriers, accumulator_stage), .{}); - full_lane.close(); - const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); - const next_acc_count = tiles.after_carried[8].add(1); - const next_work = advancePersistentWork( - b, - tiles.after_carried[12], - tiles.after_carried[16], - tiles.after_carried[11], - tiles.after_carried[18], - tiles.after_carried[17], - tiles.after_carried[19], - tiles.after_carried[20], - null, - ); - tiles.yieldAfter(.{ - next_work.m, - next_work.n, - next_work.group, - next_work.valid, - stages.results[2], - stages.results[3], - stages.results[4], - stages.results[1], - next_acc_count, - next_accumulator_state.index, - next_accumulator_state.phase, - tiles.after_carried[11], - next_work.linear, - tiles.after_carried[13], - tiles.after_carried[14], - tiles.after_carried[15], - next_work.executed, - tiles.after_carried[17], - tiles.after_carried[18], - tiles.after_carried[19], - tiles.after_carried[20], - }); - - // PipelineUmmaAsync.producer_tail. - var accumulator_tail: PipelinePosition = .{ - .index = tiles.results[9], - .phase = tiles.results[10], - }; - inline for (0..accumulator_stages - 1) |_| { - accumulator_tail = advancePipeline(b, accumulator_tail.index, accumulator_tail.phase, accumulator_stages); - } - b.mbarrierWait(barrierAt(b, storage.accumulator_empty_barriers, accumulator_tail.index), accumulator_tail.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); -} - -/// Epilogue operands; which fields are set depends on the `Epilogue`. -const EpilogueOperands = struct { - expert_ids: cute.View, - alpha: cute.View, - group_sizes: cute.View, - route_inverse: cute.View, - routing_weights: ?cute.View = null, - act_scale: ?cute.View = null, - q: ?cute.View = null, - s: ?cute.View = null, - rows: ?cute.View = null, - direct: bool, - group_rows: i64 = 0, - swiglu_limit: ?f32 = null, -}; - -/// Warps 0-3 drain each TMEM accumulator stage into registers, release it to -/// the MMA warp, then write the tile out according to `epilogue`. -fn runEpilogue( - b: *B, - comptime tile: Tile, - comptime epilogue: Epilogue, - storage: StageStorage, - tmem_holding: V, - active: V, - tiles_m: V, - direct: ?DirectRoutes, - operands: EpilogueOperands, -) void { - const n = tile.n; - const tid = b.threadIdx().x; - const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); - const tmem = b.retrieveTmemPtr(tmem_holding, .f32, 16); - - const div_m = b.fastDivmodCreate(tiles_m); - const div_n = b.fastDivmodCreate(1); - const initial_work = persistentWorkAt(b, b.blockIdx().z, b.constant(.i32, 0), active, tiles_m, div_m, div_n, direct); - const carried = .{ - initial_work.m, initial_work.n, initial_work.group, initial_work.valid, - b.constant(.i32, 0), b.constant(.i32, 0), b.constant(.i32, 0), initial_work.linear, - initial_work.executed, - }; - var tiles = b.openWhile(carried, .{ - initial_work.m.type_(), - initial_work.n.type_(), - initial_work.group.type_(), - cute.DType.i32.toMlir(b.ctx), - cute.DType.i32.toMlir(b.ctx), - cute.DType.i32.toMlir(b.ctx), - initial_work.linear.type_(), - initial_work.executed.type_(), - }); - tiles.yieldBefore(tiles.before_carried[3], .{ - tiles.before_carried[0], - tiles.before_carried[1], - tiles.before_carried[2], - tiles.before_carried[4], - tiles.before_carried[5], - tiles.before_carried[6], - tiles.before_carried[7], - tiles.before_carried[8], - }); - const tile_m = tiles.after_carried[0]; - const group = tiles.after_carried[2]; - const accumulator_state: PipelinePosition = .{ - .index = tiles.after_carried[4], - .phase = tiles.after_carried[5], - }; - const stage = accumulator_state.index; - // Per-expert scalars do not depend on the accumulator. - const expert = operands.expert_ids.get(.{group}, .i32); - const alpha = operands.alpha.get(.{expert}, .f32); - const act_scale = if (operands.act_scale) |scales| scales.get(.{expert}, .f32) else null; - b.mbarrierWait(barrierAt(b, storage.accumulator_full_barriers, stage), accumulator_state.phase, .{ .scope = .cta, .time_limit = 10_000_000, .test_first = false }); - - const accumulator_tmem = accumulatorTmemAt(b, tile, tmem, stage); - const warp_tmem = b.addOffset( - accumulator_tmem, - b.makeIntTupleDivBy(warp.mul(32 * 65536), 4), - b.ptrTy(.f32, .tmem, 16) catch @panic("bad epilogue TMEM pointer"), - ).value(); - const chunk_rows: usize = comptime @intCast(tile.chunkRows()); - const tile_chunks = comptime tile.chunks(); - inline for (0..tile_chunks) |chunk| { - const chunk_tmem = if (chunk == 0) warp_tmem else b.addOffset( - warp_tmem, - b.makeIntTuple(.{@as(i32, @intCast(chunk * chunk_rows))}), - b.ptrTy(.f32, .tmem, offsetAlignment(16, @intCast(chunk * chunk_rows), 4)) catch @panic("bad epilogue TMEM pointer"), - ).value(); - const routed = routedLanes(b, tile, operands, group, chunk); - const values = b.tmemLoadVector(chunk_tmem, @intCast(chunk_rows), .{ .num_dp = 32, .num_b = 32, .num_rep = @intCast(chunk_rows) }); - b.fenceTmemLoad(); - if (chunk + 1 == tile_chunks) { - _ = b.mbarrierArrive(barrierAt(b, storage.accumulator_empty_barriers, stage), .{}); - } - var columns: [@intCast(@min(n, 32))]V = undefined; - inline for (&columns, 0..) |*column, i| column.* = b.vectorExtract(values, i, .i32).bitCast(.f32).mul(alpha); - switch (epilogue) { - .swiglu_nvfp4 => storeTileSwiglu(b, tile, operands, routed, tile_m, group, act_scale.?, &columns, chunk), - .route_rows => storeTileRows(b, tile, operands, routed, tile_m, &columns, chunk), - } - } - const next_accumulator_state = advancePipeline(b, accumulator_state.index, accumulator_state.phase, accumulator_stages); - const next_work = advancePersistentWork(b, tiles.after_carried[6], tiles.after_carried[7], b.gridDim().z, active, tiles_m, div_m, div_n, direct); - tiles.yieldAfter(.{ - next_work.m, - next_work.n, - next_work.group, - next_work.valid, - tiles.after_carried[3].add(1), - next_accumulator_state.index, - next_accumulator_state.phase, - next_work.linear, - next_work.executed, - }); - - var release_warp = b.openIf(warp.eq(0)); - b.relinquishTmemAllocPermit(.{}); - release_warp.close(); - b.barrierCtaSyncAligned(1, 128); - var dealloc_warp = b.openIf(warp.eq(0)); - b.deallocTmem(tmem, tile.tmem_columns, .{}); - dealloc_warp.close(); -} - -/// Per-row routing data of one expert group, spread across a warp: lane `l` -/// holds routed row `l` of the current chunk. -const RoutedLanes = struct { - count: V, - /// Token/top-k route of the lane's row; 0 for padding rows. - route: V, - /// Routing weight of the lane's row, when this projection applies it. - weight: ?V, -}; - -fn routedLanes(b: *B, comptime tile: Tile, operands: EpilogueOperands, group: V, comptime chunk: usize) RoutedLanes { - const weights = operands.routing_weights; - if (operands.direct) return .{ - .count = b.constant(.i32, 1), - .route = group, - .weight = if (weights) |w| w.get(.{group}, .f32) else null, - }; - const lane = b.threadIdx().x.rem(32); - const count = operands.group_sizes.get(.{group}, .i32); - // Lanes past the tile width alias row 0, which is always live. - const chunk_rows = comptime tile.chunkRows(); - const in_chunk = b.select(lane.lt(@as(i32, @intCast(chunk_rows))), lane, b.constant(.i32, 0)); - const row_in_group = if (chunk == 0) in_chunk else in_chunk.add(@as(i32, @intCast(@as(i64, @intCast(chunk)) * chunk_rows))); - const live = row_in_group.lt(count); - const inverse = operands.route_inverse.get(.{group.mul(tile.n).add(row_in_group)}, .i32); - const route = b.select(live, inverse, b.constant(.i32, 0)); - const weight = if (weights) |w| w.get(.{route}, .f32) else null; - return .{ .count = count, .route = route, .weight = weight }; -} - -const swiglu_chunk = 8; - -/// Thread `t` owns output row `128 * tile_m + t` of the interleaved -/// projection: even threads hold a gate and their odd neighbour the matching -/// up value, so each even thread computes one SwiGLU activation per routed -/// row. A warp's 16 activations are exactly one NVFP4 block, whose maximum is -/// a single warp reduction. Activation `a` of row `r` lands in nibble `a` of -/// row `r` of `q`, and its scale at the 128x4 position of `(r, a / 16)`. -fn storeTileSwiglu(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, act_scale: V, columns: []const V, comptime chunk: usize) void { - if (operands.direct) - swigluRows(b, tile, operands, routed, tile_m, group, act_scale, columns, 1, false, chunk) - else - swigluRows(b, tile, operands, routed, tile_m, group, act_scale, columns, comptime @intCast(tile.chunkRows()), true, chunk); -} - -fn swigluRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, group: V, act_scale: V, columns: []const V, comptime live: usize, comptime skip_padding: bool, comptime chunk: usize) void { - const n = comptime tile.chunkRows(); - const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); - const tid = b.threadIdx().x; - const lane = tid.rem(32); - const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); - const is_gate = tid.rem(2).eq(0); - const first_row = group.mul(operands.group_rows); - const f32_type = cute.DType.f32.toMlir(b.ctx); - const row_chunks = std.math.divCeil(usize, live, swiglu_chunk) catch unreachable; - - var activations: [@intCast(n)]V = undefined; - // Lane `c` collects the block maximum of column `c`. - var lane_maximum = b.constant(.f32, 0); - for (0..row_chunks) |row_chunk| { - const first = row_chunk * swiglu_chunk; - const last = @min(first + swiglu_chunk, live); - const guarded = skip_padding and (row_chunk > 0 or chunk > 0); - const types: [swiglu_chunk + 1]*const mlir.Type = @splat(f32_type); - var guard: ?@TypeOf(b.openIfElse(routed.count, tupleOf(types))) = if (guarded) b.openIfElse(routed.count.gt(chunk_first + @as(i32, @intCast(first))), tupleOf(types)) else null; - var chunk_maximum = lane_maximum; - for (columns[first..last], activations[first..last], first..) |value, *activation, column| { - const c: i32 = @intCast(column); - // The reference rounds the projection to BF16. - var gate = value.to(.bf16).to(.f32); - var up_value = b.shuffleXor(value, 1).to(.bf16).to(.f32); - if (operands.swiglu_limit) |limit| { - gate = gate.minimum(limit); - up_value = up_value.maximum(-limit).minimum(limit); - } - // up * SiLU(gate), exp(-g) = 2^(-g * log2 e). - const exp_neg = b.unaryF32("ex2.approx.ftz.f32", gate.mul(-std.math.log2e)); - var result = up_value.mul(gate).mul(b.unaryF32("rcp.approx.ftz.f32", exp_neg.add(1.0))); - if (routed.weight) |weight| result = result.mul(b.shuffleIdx(weight, c)); - // The NVFP4 quantizer reads BF16 activations. - result = result.to(.bf16).to(.f32); - // Odd threads hold no activation and must not raise the maxima. - activation.* = b.select(is_gate, result, b.constant(.f32, 0)); - const maximum = b.warpMaxNonNegative(activation.*.abs()); - chunk_maximum = b.select(lane.eq(c), maximum, chunk_maximum); - } - if (guard) |*g| { - var then_values: [swiglu_chunk + 1]V = @splat(chunk_maximum); - for (activations[first..last], then_values[0 .. last - first]) |activation, *t| t.* = activation; - g.yieldThen(tupleOf(then_values)); - var else_values: [swiglu_chunk + 1]V = @splat(b.constant(.f32, 0)); - else_values[swiglu_chunk] = lane_maximum; - g.yieldElse(tupleOf(else_values)); - for (activations[first..last], g.results[0 .. last - first]) |*activation, result| activation.* = result; - lane_maximum = g.results[swiglu_chunk]; - } else { - lane_maximum = chunk_maximum; - } - } - const scale = nvfp4.blockScale(b, lane_maximum, act_scale); - - // Lane `16 h` writes the word of activations `16 * warp + 8 h ..`. - const word = tile_m.mul(8).add(warp.mul(2)).add(lane.div(16)); - const word_lane = lane.rem(16).eq(0); - for (0..row_chunks) |row_chunk| { - const first = row_chunk * swiglu_chunk; - const last = @min(first + swiglu_chunk, live); - var guard: ?@TypeOf(b.openIf(routed.count)) = if (skip_padding and (chunk > 0 or first > 0)) b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(first)))) else null; - for (activations[first..last], first..) |activation, column| { - const c: i32 = @intCast(column); - const quantized = activation.mul(b.shuffleIdx(scale.multiplier, c)); - // Gate lane `4i` pairs with gate lane `4i + 2`, then pairs of - // bytes and pairs of halves meet at lanes `8i` and `16i`. - const pair = b.packFp4x2(quantized, b.shuffleXor(quantized, 2)); - const half = pair.bitOr(b.shuffleXor(pair, 4).shl(8)); - const packed_word = half.bitOr(b.shuffleXor(half, 8).shl(16)); - var writer = b.openIf(word_lane); - operands.q.?.set(.{ first_row.add(chunk_first + c), word }, packed_word); - writer.close(); - } - if (guard) |*g| g.close(); - } - - // Lane `c` stores the scale of row `c`, block `4 * tile_m + warp`. - var scale_lane = b.openIf(lane.lt(@as(i32, @intCast(n))).bitAnd(lane.add(chunk_first).lt(routed.count))); - const swizzled = tile_m.mul(512).add(lane.mul(16)).add(chunk * 4).add(warp); - operands.s.?.set(.{ group, swizzled }, scale.bits.to(.i8)); - scale_lane.close(); -} - -/// Store each live routed row at its token/top-k route, so the final -/// reduction reads rows in route order. -fn storeTileRows(b: *B, comptime tile: Tile, operands: EpilogueOperands, routed: RoutedLanes, tile_m: V, columns: []const V, comptime chunk: usize) void { - const chunk_first: i32 = comptime @intCast(chunk * @as(usize, @intCast(tile.chunkRows()))); - const output_column = tile_m.mul(128).add(b.threadIdx().x); - if (operands.direct) { - const value = if (routed.weight) |weight| columns[0].mul(weight) else columns[0]; - operands.rows.?.set(.{ routed.route, output_column }, value.to(.bf16)); - return; - } - var routes: [32]V = undefined; - var weights: [32]?V = @splat(null); - for (routes[0..columns.len], weights[0..columns.len], 0..) |*route, *weight, column| { - route.* = b.shuffleIdx(routed.route, @as(i32, @intCast(column))); - if (routed.weight) |w| weight.* = b.shuffleIdx(w, @as(i32, @intCast(column))); - } - for (columns, routes[0..columns.len], weights[0..columns.len], 0..) |value, route, weight, column| { - var live: ?@TypeOf(b.openIf(routed.count)) = if (column == 0 and chunk == 0) null else b.openIf(routed.count.gt(chunk_first + @as(i32, @intCast(column)))); - const scaled = if (weight) |w| value.mul(w) else value; - operands.rows.?.set(.{ route, output_column }, scaled.to(.bf16)); - if (live) |*l| l.close(); - } -} - -fn buildUmmaDevice(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue, d: anytype) void { - const L = Layouts(tile); - const n = tile.n; - const stages = tile.stages; - const shared = b.allocSmemStorage( - .i8, - 1024, - b.layoutType(b.layoutSpec(.{tile.smem_bytes}, .{1})), - 1, - tile.fields(b), - ); - const shared_base = b.getIter( - shared, - b.ptrTy(.i8, .smem, 1024) catch @panic("bad shared base pointer"), - ).value(); - - const tmem_holding = sharedPointerAt(b, shared_base, tile.tmem_holding, .i32, 8); - const storage: StageStorage = .{ - .c = sharedPointerAt(b, shared_base, tile.c, .f32, 1024), - .a = sharedPointerAt(b, shared_base, tile.a, .i8, 1024), - .b = sharedPointerAt(b, shared_base, tile.b, .i8, 1024), - .sfa = sharedPointerAt(b, shared_base, tile.sfa, .f8e4m3fn, 1024), - .sfb = sharedPointerAt(b, shared_base, tile.sfb, .f8e4m3fn, 1024), - .load_barriers = sharedPointerAt(b, shared_base, 0, .i64, 1024), - .mma_barriers = sharedPointerAt(b, shared_base, tile.empty_barriers, .i64, 8), - .accumulator_full_barriers = sharedPointerAt(b, shared_base, tile.accumulator_full, .i64, 16), - .accumulator_empty_barriers = sharedPointerAt(b, shared_base, tile.accumulator_empty, .i64, 16), - }; - - const tid = b.threadIdx().x; - const warp = b.makeWarpUniform(b.makeWarpUniform(tid.div(32))); - const tiled_mma: cute.Atom = .{ .inner = d.mma.inner, .kernel = b }; - - // coordinate tensor -> local_tile -> tiled_mma.partition -> group modes - // -> tma_partition, with the result types of the CuTe DSL. - const m_tiles = @divExact(cfg.m, 128); - const k_tiles = @divExact(cfg.k, k_tile); - const m_step = b.basis(128, 1, .{1}); - const k_step = b.basis(k_tile, 1, .{0}); - const a_local_spec = b.layoutSpec( - .{ 128, k_tile, m_tiles, k_tiles, cfg.experts }, - .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), m_step, k_step, b.basis(1, 1, .{2}) }, - ); - const b_local_spec = b.layoutSpec( - .{ n, k_tile, 1, k_tiles, cfg.groups }, - .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}), 0, k_step, b.basis(1, 1, .{2}) }, - ); - const all_tiles = .{ cute.AlgebraToken.all, cute.AlgebraToken.all, cute.AlgebraToken.all }; - const tiled_a = b.localTile(d.gA, b.makeTile(&.{ 128, k_tile }), all_tiles, b.coordTensorType(3, a_local_spec), null); - const tiled_b = b.localTile(d.gB, b.makeTile(&.{ n, k_tile }), all_tiles, b.coordTensorType(3, b_local_spec), null); - - const a_mma_spec = b.layoutSpec( - .{ .{ 128, 64 }, 1, 4, m_tiles, k_tiles, cfg.experts }, - .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}), m_step, k_step, b.basis(1, 1, .{2}) }, - ); - const b_mma_spec = b.layoutSpec( - .{ .{ n, 64 }, 1, 4, 1, k_tiles, cfg.groups }, - .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}), 0, k_step, b.basis(1, 1, .{2}) }, - ); - const mma_a = b.tiledMmaPartition(tiled_mma, tiled_a, .{0}, 0, b.coordTensorType(3, a_mma_spec)); - const mma_b = b.tiledMmaPartition(tiled_mma, tiled_b, .{0}, 1, b.coordTensorType(3, b_mma_spec)); - - const a_grouped_layout = b.staticLayout(b.layoutSpec( - .{ .{ .{ 128, 64 }, 1, 4 }, m_tiles, k_tiles, cfg.experts }, - .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}) }, m_step, k_step, b.basis(1, 1, .{2}) }, - )); - const b_grouped_layout = b.staticLayout(b.layoutSpec( - .{ .{ .{ n, 64 }, 1, 4 }, 1, k_tiles, cfg.groups }, - .{ .{ .{ b.basis(1, 1, .{1}), b.basis(1, 1, .{0}) }, 0, b.basis(64, 1, .{0}) }, b.basis(n, 1, .{1}), k_step, b.basis(1, 1, .{2}) }, - )); - const staged = stagedOperands(b, tile, storage); - const smem_a_grouped = b.withLayout(staged.a, b.staticLayout(b.layoutSpec( - .{ .{ .{ 128, 64 }, 1, 4 }, stages }, - .{ .{ .{ k_tile, 1 }, 0, 64 }, 2 * Tile.a_stage_bytes }, - ))); - const smem_b_grouped = b.withLayout(staged.b, b.staticLayout(b.layoutSpec( - .{ .{ .{ n, 64 }, 1, 4 }, stages }, - .{ .{ .{ k_tile, 1 }, 0, 64 }, 2 * tile.bStageBytes() }, - ))); - const one_cta = b.staticLayout(b.layoutSpec(.{1}, .{0})); - const a_partition_layout = b.layoutSpec(.{ .{ 2 * Tile.a_stage_bytes, 1 }, stages }, .{ .{ 1, 0 }, 2 * Tile.a_stage_bytes }); - const b_partition_layout = b.layoutSpec(.{ .{ 2 * tile.bStageBytes(), 1 }, stages }, .{ .{ 1, 0 }, 2 * tile.bStageBytes() }); - const a_target_layout = b.layoutSpec( - .{ .{ .{ k_tile, 128 }, 1 }, m_tiles, k_tiles, cfg.experts }, - .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, m_step, k_step, b.basis(1, 1, .{2}) }, - ); - const b_target_layout = b.layoutSpec( - .{ .{ .{ k_tile, n }, 1 }, 1, k_tiles, cfg.groups }, - .{ .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0 }, b.basis(n, 1, .{1}), k_step, b.basis(1, 1, .{2}) }, - ); - const partitioned_a = b.tmaPartition( - .{ .inner = d.tma_a.inner, .kernel = b }, - .{0}, - one_cta, - smem_a_grouped, - .{b.withLayout(mma_a, a_grouped_layout)}, - b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(a_partition_layout)), - &.{b.coordTensorType(3, a_target_layout)}, - ); - const partitioned_b = b.tmaPartition( - .{ .inner = d.tma_b.inner, .kernel = b }, - .{0}, - one_cta, - smem_b_grouped, - .{b.withLayout(mma_b, b_grouped_layout)}, - b.memrefTypeFromPointer(L.fp4SmemPtrTy(b), b.layoutType(b_partition_layout)), - &.{b.coordTensorType(3, b_target_layout)}, - ); - - const exec_a = b.makeExecTma(.{ .inner = d.tma_a.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaA(b, cfg))); - const exec_b = b.makeExecTma(.{ .inner = d.tma_b.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaB(b, cfg))); - const exec_sfa = b.makeExecTma(.{ .inner = d.tma_sfa.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); - const exec_sfb = b.makeExecTma(.{ .inner = d.tma_sfb.inner, .kernel = b }, b.tmaLoadExecAtomType(L.tmaSf(b))); - - const expert_ids: cute.View = .{ .inner = d.expert_ids.inner, .kernel = b }; - const active_count: cute.View = .{ .inner = d.active_count.inner, .kernel = b }; - const direct: ?DirectRoutes = if (cfg.direct) .{ .expert_ids = expert_ids, .routes = @intCast(cfg.groups) } else null; - const active = if (direct) |routes| routes.count(b) else active_count.get(.{0}, .i32); - - // Warp 0 initializes the barriers; one elected lane per barrier array. - var init_warp = b.openIf(warp.eq(0)); - const barrier_arrays = .{ - .{ storage.load_barriers, stages, 1 }, - .{ storage.mma_barriers, stages, 1 }, - .{ storage.accumulator_full_barriers, accumulator_stages, 1 }, - .{ storage.accumulator_empty_barriers, accumulator_stages, 128 }, - }; - inline for (barrier_arrays) |array| { - var lane = b.openIf(b.electSync()); - inline for (0..array[1]) |stage| { - b.mbarrierInit(barrierAt(b, array[0], b.constant(.i32, stage)), array[2]); - } - lane.close(); - } - b.fenceMbarrierInit(); - init_warp.close(); - b.syncThreads(); - - const reduction_tiles: usize = @intCast(k_tiles); - var tma_warp = b.openIf(warp.eq(5)); - runTmaProducer(b, tile, reduction_tiles, storage, active, d.tiles_m, expert_ids, direct, partitioned_a.targets[0], partitioned_b.targets[0], exec_a, exec_b, exec_sfa, exec_sfb); - tma_warp.close(); - - var mma_warp = b.openIf(warp.eq(4)); - b.barrierCtaSyncAligned(2, 160); - runMmaConsumer(b, tile, reduction_tiles, storage, tiled_mma, tmem_holding, active, d.tiles_m); - mma_warp.close(); - - var epilogue_warps = b.openIf(warp.lt(4)); - var alloc_warp = b.openIf(warp.eq(0)); - b.allocTmem(tile.tmem_columns, tmem_holding, .{}); - alloc_warp.close(); - b.barrierCtaSyncAligned(2, 160); - const shared_operands: EpilogueOperands = .{ - .expert_ids = expert_ids, - .alpha = view(b, d.alpha), - .group_sizes = view(b, d.group_sizes), - .route_inverse = view(b, d.route_inverse), - .routing_weights = if (cfg.routing_weights) view(b, d.routing_weights) else null, - .direct = cfg.direct, - .group_rows = cfg.groupRows(), - .swiglu_limit = cfg.swiglu_limit, - }; - var operands = shared_operands; - switch (epilogue) { - .swiglu_nvfp4 => { - operands.act_scale = view(b, d.act_scale); - operands.q = view(b, d.q); - operands.s = view(b, d.s); - }, - .route_rows => operands.rows = view(b, d.rows), - } - runEpilogue(b, tile, epilogue, storage, tmem_holding, active, d.tiles_m, direct, operands); - epilogue_warps.close(); -} - -fn buildHostLaunch(b: *B, cfg: Config, comptime tile: Tile, comptime epilogue: Epilogue) cute.FinishError!void { - const L = Layouts(tile); - const host_arg = HostArgs(epilogue); - const n = tile.n; - const a_layout = b.layoutSpec(.{ cfg.m, cfg.k, cfg.experts }, .{ cfg.k, 1, cfg.m * cfg.k }); - const b_layout = b.layoutSpec(.{ cfg.groupRows(), cfg.k, cfg.groups }, .{ cfg.k, 1, cfg.k * cfg.groupRows() }); - // `cute.tile_to_shape` derives that the dynamic strides step whole - // 512-byte scale atoms, and the result type has to carry it. - const sf_stride: cute.ConstrainedDynamic = .{ .divisible_by = 512 }; - const dynamic_sf_layout = b.layoutSpec( - .{ .{ .{ 32, 4 }, cute.AlgebraToken.dynamic }, .{ .{ 16, 4 }, cute.AlgebraToken.dynamic }, .{ 1, cute.AlgebraToken.dynamic } }, - .{ .{ .{ 16, 4 }, sf_stride }, .{ .{ 0, 1 }, 512 }, .{ 0, sf_stride } }, - ); - - // XLA owns packed storage as signless bytes; recast to FP4 here. - const gA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight), .f4e2m1fn, .gmem, 16), b.staticLayout(a_layout)); - const schedule = b.arg(host_arg.schedule); - const ids = b.makeTensorView(schedule, b.staticLayout(b.layoutSpec(cfg.groups, 1))); - const active_ptr = b.addOffset( - schedule, - b.makeIntTuple(.{2 * cfg.groups}), - b.ptrTy(.i32, .gmem, offsetAlignment(16, 2 * cfg.groups, 4)) catch @panic("bad active count pointer"), - ); - const active = b.makeTensorView(active_ptr, b.staticLayout(b.layoutSpec(1, 1))); - const gB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_quant), .f4e2m1fn, .gmem, 16), b.staticLayout(b_layout)); - const sf_atom_layout = b.staticLayout(b.layoutSpec( - .{ .{ 32, 4 }, .{ 16, 4 } }, - .{ .{ 16, 4 }, .{ 0, 1 } }, - )); - // Zero-based form of Python CuTe's `(2, 1, 3)` tile-to-shape order. - const sf_order = b.makeIntTuple(.{ 1, 0, 2 }); - const sfa_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_m), b.arg(host_arg.problem_k), b.arg(host_arg.expert_count) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); - const sfb_shape = b.makeShapeValues(.{ b.arg(host_arg.problem_n), b.arg(host_arg.problem_k), b.arg(host_arg.problem_l) }, b.shapeType(.{ cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic, cute.AlgebraToken.dynamic })); - const dynamic_sfa_layout = b.tileToShape(sf_atom_layout, sfa_shape, sf_order, b.layoutType(dynamic_sf_layout)); - const dynamic_sfb_layout = b.tileToShape(sf_atom_layout, sfb_shape, sf_order, b.layoutType(dynamic_sf_layout)); - const gSFA = b.makeTensorView(b.recastPointer(b.arg(host_arg.weight_scale), .f8e4m3fn, .gmem, 16), dynamic_sfa_layout); - const gSFB = b.makeTensorView(b.recastPointer(b.arg(host_arg.input_scale), .f8e4m3fn, .gmem, 16), dynamic_sfb_layout); - - const zero_tmem = b.intToPtr(0, .f8e4m3fn, .tmem, 1); - const mma = b.makeBlockScaledMma(.{ .kind = mma_kind, .n = @intCast(n) }, zero_tmem); - const mma_sfb = b.makeBlockScaledMma(.{ .kind = mma_kind, .n = 128 }, zero_tmem); - - const a_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.aSmem(b, false)); - const b_smem = b.makeComposedLayout(L.ab_swizzle, 0, L.bSmem(b, false)); - const sf_smem = b.staticLayout(L.sfSmem(b, false)); - const a_map = b.staticLayout(b.layoutSpec(.{ .{ 128, 64 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(64, 1, .{1}) })); - const b_map = b.staticLayout(b.layoutSpec(.{ .{ n, 64 }, 1, 4 }, .{ .{ b.basis(1, 1, .{0}), b.basis(1, 1, .{1}) }, 0, b.basis(64, 1, .{1}) })); - const sf_map = b.staticLayout(b.layoutSpec( - .{ .{ .{ 32, 4 }, .{ 16, 4 } }, 1, 4 }, - .{ .{ .{ b.basis(1, 1, .{ 0, 0, 0 }), b.basis(1, 1, .{ 0, 0, 1 }) }, .{ b.basis(1, 1, .{ 1, 0, 0 }), b.basis(1, 1, .{ 1, 0, 1 }) } }, 0, b.basis(1, 1, .{ 1, 1 }) }, - )); - - const tma_a = b.makeTiledTmaLoadAtom(gA.value(), a_smem.value(), a_map.value(), L.tmaA(b, cfg)); - const tma_b = b.makeTiledTmaLoadAtom(gB.value(), b_smem.value(), b_map.value(), L.tmaB(b, cfg)); - const tma_sfa = b.makeTiledTmaLoadAtom(gSFA.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); - const tma_sfb = b.makeTiledTmaLoadAtom(gSFB.value(), sf_smem.value(), sf_map.value(), L.tmaSf(b)); - - const one = b.constant(.i32, 1); - const tiles_m = b.constant(.i32, @divExact(cfg.m, 128)); - const grid_z = b.constant(.i32, @min(@divExact(cfg.m, 128) * cfg.groups, cfg.persistent_ctas)); - const config = b.makeLaunchConfig(.{ - .grid = .{ one, one, grid_z }, - .block = .{ b.constant(.i32, 192), one, one }, - .dynamic_smem = b.kernelSmemSize(deviceName(epilogue)), - .stream = b.cudaStream(), - .cluster = .{ one, one, one }, - .use_pdl = true, - }); - const group_sizes = b.makeTensorView( - b.addOffset(schedule, b.makeIntTuple(.{cfg.groups}), b.ptrTy(.i32, .gmem, offsetAlignment(16, cfg.groups, 4)) catch @panic("bad group size pointer")), - b.staticLayout(b.layoutSpec(cfg.groups, 1)), - ); - const per_expert = b.staticLayout(b.layoutSpec(cfg.experts, 1)); - const mainloop = .{ - ids, active, mma.tiled, mma_sfb.tiled, tma_a.atom, tma_a.tensor, tma_b.atom, tma_b.tensor, - tma_sfa.atom, tma_sfa.tensor, tma_sfb.atom, tma_sfb.tensor, b.makeTensorView(b.arg(host_arg.alpha), per_expert), group_sizes, b.makeTensorView(b.arg(host_arg.route_inverse), b.staticLayout(b.layoutSpec(cfg.routedRows(), 1))), b.makeTensorView(b.arg(host_arg.routing_weights), b.staticLayout(b.layoutSpec(@max(cfg.routes, 1), 1))), - }; - const activations = @divExact(cfg.m, 2); - const launch = switch (epilogue) { - .swiglu_nvfp4 => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ - b.makeTensorView(b.arg(host_arg.act_scale), per_expert), - b.makeTensorView(b.recastPointer(b.arg(host_arg.output), .i32, .gmem, 16), b.staticLayout(b.layoutSpec(.{ cfg.routedRows(), @divExact(activations, 8) }, .{ @divExact(activations, 8), 1 }))), - b.makeTensorView(b.arg(host_arg.scales), b.staticLayout(b.layoutSpec(.{ cfg.groups, @divExact(activations, 16) * 128 }, .{ @divExact(activations, 16) * 128, 1 }))), - tiles_m, - }), - .route_rows => b.launchEx(deviceName(epilogue), config, mainloop ++ .{ - b.makeTensorView(b.arg(host_arg.output), b.staticLayout(b.layoutSpec(.{ @max(cfg.routes, 1), cfg.m }, .{ cfg.m, 1 }))), - tiles_m, - }), - }; - b.returnHostStatus(b.cudaResultStatus(launch)); -} - -test "NVFP4 GEMM programs emit their host launch for every tile width" { - inline for (tile_widths) |n| { - inline for (.{ .{ UpQuantized, Epilogue.swiglu_nvfp4, 4608, 5120 }, .{ DownRows, Epilogue.route_rows, 5120, 2304 } }) |program| { - const cfg: Config = .{ - .experts = 192, - .m = program[2], - .n = n, - .k = program[3], - .groups = 227, - .epilogue = program[1], - .routes = 576, - .direct = n == 8, - .routing_weights = program[1] == .route_rows, - }; - const ir = try program[0].emit(std.testing.allocator, cfg); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, deviceName(program[1])) != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "tma_format = U4") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, std.fmt.comptimePrint("sm100.mma_bs<128x{d}x64", .{n})) != null); - } - } -} diff --git a/zml/kernels/moe/cute/sm10x/reduce.zig b/zml/kernels/moe/cute/sm10x/reduce.zig deleted file mode 100644 index 63244ac4..00000000 --- a/zml/kernels/moe/cute/sm10x/reduce.zig +++ /dev/null @@ -1,124 +0,0 @@ -//! CuTe top-k reduction behind the persistent MoE GEMMs: sums each token's -//! BF16 route rows. -const std = @import("std"); - -const zml = @import("../../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const launchBoundary = @import("common.zig").launchBoundary; - -pub const Config = struct { - tokens: i64, - hidden: i64, - topk: i64, - columns_per_cta: i64, -}; - -// A program rather than a bare kernel so its launch carries the -// programmatic-dependent-launch attribute (see `mxfp4.zig`). -const Topk = cute.Program(Config, .{ - .name = "moe_cute_reduce_topk", - .inputs = &.{ "d", "routes" }, - .outputs = &.{"y"}, - .run = topkProgram, -}); - -const topk_kernel_name = "moe_cute_reduce_topk_kernel"; - -/// Amortize launch overhead for large prefills while retaining enough CTAs -/// to saturate the GPU for decode and small batches. -pub fn columnsPerCta(tokens: i64, hidden: i64) i64 { - const elements = tokens * hidden; - if (tokens > 1 and elements >= 512 * 2048) return 2048; - if (tokens > 1 and elements >= 512 * 1024) return 1024; - // Two columns per thread is the smallest tile: 128 threads, one 32-bit - // word each. - return 256; -} - -/// Sum the `[tokens * topk, hidden]` BF16 route rows of each token into BF16. -/// Routing weights were already applied before the down projection. -/// `routes` is negative for routes this rank does not compute (expert -/// parallelism): their rows were never written. -pub fn topk(d: zml.Tensor, routes: zml.Tensor, cfg: Config) zml.Tensor { - return Topk.call( - .{ .d = d, .routes = routes }, - .{ .y = .init(.{ cfg.tokens, cfg.hidden }, .bf16) }, - .{ .cfg = cfg }, - ).y; -} - -/// Each thread owns two adjacent BF16 columns, read and written as one -/// 32-bit word: a warp then moves a full 128-byte line per access. -fn topkProgram(b: *B, cfg: Config) cute.FinishError!void { - b.beginFunction(topk_kernel_name, .cuda_kernel); - try topkKernel(b, cfg); - b.endFunction(.{ 128, 1, 1 }); - - b.beginFunction(Topk.name, .host); - // The kernel reads and writes two BF16 columns per 32-bit word. - const host = try b.declareArgs(.{ - .d = .{ .ptr = cute.DType.bf16 }, - .routes = .{ .ptr = cute.DType.i32 }, - .y = .{ .ptr = cute.DType.bf16 }, - }); - try launchBoundary(b, topk_kernel_name, .{ - @intCast(@divTrunc(cfg.hidden + cfg.columns_per_cta - 1, cfg.columns_per_cta)), - @intCast(cfg.tokens), - 1, - }, .{ b.recastPointer(host.d, .i32, .gmem, 16), host.routes, b.recastPointer(host.y, .i32, .gmem, 16) }); - b.endFunction(null); -} - -fn topkKernel(b: *B, cfg: Config) cute.FinishError!void { - const pairs = @divExact(cfg.hidden, 2); - const a = try b.declareArgs(.{ - .d = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens * cfg.topk, pairs } } }, - .routes = .{ .tensor = .{ .dtype = .i32, .shape = &.{cfg.tokens * cfg.topk} } }, - .y = .{ .tensor = .{ .dtype = .i32, .shape = &.{ cfg.tokens, pairs } } }, - }); - - // The down projection's rows arrive from a grid this one may overlap. - b.waitForDependency(); - const token = b.blockIdx().y; - const pairs_per_cta = @divExact(cfg.columns_per_cta, 2); - const first_pair = b.blockIdx().x.mul(pairs_per_cta).add(b.threadIdx().x); - for (0..@intCast(@divExact(pairs_per_cta, 128))) |chunk| { - const pair = first_pair.add(chunk * 128); - var in_bounds = b.openIf(pair.lt(pairs)); - var low = b.constant(.f32, 0); - var high = b.constant(.f32, 0); - for (0..@intCast(cfg.topk)) |slot| { - const route = token.mul(cfg.topk).add(slot); - // Routes of other expert-parallel ranks were never written. - var computed = b.openIfElse(a.routes.get(.{route}).ge(0), .{ cute.DType.f32.toMlir(b.ctx), cute.DType.f32.toMlir(b.ctx) }); - const word = a.d.get(.{ route, pair }); - computed.yieldThen(.{ - low.add(word.shl(16).bitCast(.f32)), - high.add(word.bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))).bitCast(.f32)), - }); - computed.yieldElse(.{ low, high }); - low = computed.results[0]; - high = computed.results[1]; - } - const low_bits = low.to(.bf16).to(.f32).bitCast(.i32).shrLogical(16); - const high_bits = high.to(.bf16).to(.f32).bitCast(.i32).bitAnd(@as(i32, @bitCast(@as(u32, 0xffff0000)))); - a.y.set(.{ token, pair }, low_bits.bitOr(high_bits)); - in_bounds.close(); - } - b.launchDependents(); -} - -test "CuTe top-k reduction emits for decode and prefill batches" { - inline for (.{ @as(i64, 1), 6, 96, 16_384 }) |tokens| { - const ir = try Topk.emit(std.testing.allocator, .{ - .tokens = tokens, - .hidden = 5120, - .topk = 6, - .columns_per_cta = columnsPerCta(tokens, 5120), - }); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "cuda.launch_ex") != null); - try std.testing.expect(std.mem.indexOf(u8, ir, "scf.if") != null); - } -} diff --git a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel b/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel deleted file mode 100644 index c9c5d9a6..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "flashinfer_cutlass", - srcs = [ - "contract.zig", - "flashinfer_cutlass.zig", - "runtime.zig", - ], -) diff --git a/zml/kernels/moe/flashinfer_cutlass/contract.zig b/zml/kernels/moe/flashinfer_cutlass/contract.zig deleted file mode 100644 index b4b77eaa..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/contract.zig +++ /dev/null @@ -1,40 +0,0 @@ -pub fn Inputs(comptime T: type) type { - return struct { - hidden_states: T, - fc1_weights: T, - fc2_weights: T, - topk_weights: T, - topk_ids: T, - fc1_act_global: T, - fc1_weight_block: T, - fc1_global: T, - fc2_act_global: T, - fc2_weight_block: T, - fc2_global: T, - }; -} - -pub fn Outputs(comptime T: type) type { - return struct { - output: T, - workspace: T, - }; -} - -pub const Attributes = struct { - runners: u64 = 0, - num_tokens: i64, - hidden_size: i64, - intermediate_size: i64, - num_experts: i32, - top_k: i32, - activation: i32, - enable_pdl: bool, - fc1_act_per_expert: bool, - fc2_act_per_expert: bool, - gemm1_tactic: i32, - gemm2_tactic: i32, -}; - -pub const compute_on_host: bool = false; -pub const name = "flashinfer_cutlass_nvfp4_routed_moe"; diff --git a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig b/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig deleted file mode 100644 index 92317c7c..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/flashinfer_cutlass.zig +++ /dev/null @@ -1,355 +0,0 @@ -const std = @import("std"); - -const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); -const platforms = @import("platforms"); -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const GlobalOptions = @import("../api.zig").Options; -const Activation = @import("../api.zig").Activation; -const LocalRouting = @import("../api.zig").LocalRouting; -const unpackedWeight = @import("../weights.zig").unpackedWeight; -const contract = @import("contract.zig"); -const Attributes = contract.Attributes; -const runtime = @import("runtime.zig"); - -const log = std.log.scoped(.@"zml/moe/flashinfer_cutlass"); - -pub const auto_tactic: i32 = -1; - -pub const Options = struct { - /// Device used at graph-construction time to select the architecture - /// library and size the XLA-owned scratch buffer. - workspace_query_device: i32, - activation: Activation, - enable_pdl: bool, - gemm1_tactic: i32, - /// GEMM2 uses FlashInfer's absolute tactic index. Query tacticCounts() to - /// obtain the first valid GEMM2 index. - gemm2_tactic: i32, - - pub fn fromGlobalOptions(opts: GlobalOptions) Options { - return .{ - .workspace_query_device = 0, - .activation = opts.activation, - .enable_pdl = false, - .gemm1_tactic = auto_tactic, - .gemm2_tactic = auto_tactic, - }; - } -}; - -pub fn validateOptions(opts: GlobalOptions) void { - stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); - stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); - stdx.debug.assert(opts.activation == .swiglu, "flashinfer_cutlass backend only accepts swiglu activation, got {}", .{opts.activation}); - stdx.debug.assert(opts.activation.swiglu.limit == null, "Activation thresholds require the Triton MoE backend", .{}); - stdx.debug.assert(opts.activation.swiglu.bias == null, "flashinfer_cutlass backend requires swiglu bias to be null", .{}); - stdx.debug.assert(opts.activation.swiglu.scale == null, "flashinfer_cutlass backend requires swiglu scale to be null", .{}); -} - -const Inputs = contract.Inputs(zml.Tensor); - -const Outputs = contract.Outputs(zml.Shape); - -fn currentRunners() !*runtime.Runners { - const platform = zml.Compiler.current().platform; - const runners = platform.state.cuda.flashinfer_cutlass_moe_runners; - return runners orelse error.RunnersNotLoaded; -} - -fn cutlassActivation(activation: Activation) i32 { - return switch (activation) { - .gelu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GELU, - .relu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_RELU, - .silu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SILU, - .swiglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU, - .swiglu_step => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_SWIGLU_STEP, - .geglu => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU, - .geglu_tanh => fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_ACTIVATION_GEGLU_TANH, - }; -} - -pub fn call(input_tensors: Inputs, output_shapes: Outputs, attributes: Attributes) zml.ops.ShapeToTensor(Outputs) { - const opts: zml.ops.CustomCallOptions = .{ - .has_side_effect = false, - .output_operand_aliases = comptime zml.ops.customCallOutputOperandAliases(Inputs, Outputs, null), - .compute_on_host = contract.compute_on_host, - }; - - return zml.ops.typedCustomCall( - contract.name, - opts, - input_tensors, - output_shapes, - attributes, - ); -} - -fn computeCapability(platform: *const zml.Platform) !u16 { - const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnsupportedPlatform; - return switch (cc.sm()) { - 100, 103, 120 => |sm| sm, - else => error.UnsupportedArchitecture, - }; -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; - _ = computeCapability(platform) catch return false; - return true; -} - -pub fn isNvfp4Supported(platform: *const zml.Platform) bool { - if (platform.state.cuda.flashinfer_cutlass_moe_runners == null) return false; - const compute_capability = computeCapability(platform) catch return false; - return switch (compute_capability) { - 100, 103, 120 => true, - else => false, - }; -} - -fn roundUp(value: i64, alignment: i64) i64 { - return @divTrunc(value + alignment - 1, alignment) * alignment; -} - -pub fn fc1BlockScaleShape( - num_experts: i64, - hidden_size: i64, - intermediate_size: i64, - activation: Activation, -) zml.Shape { - const rows = switch (activation) { - .silu, .gelu, .relu => intermediate_size, - .swiglu, .geglu, .geglu_tanh, .swiglu_step => 2 * intermediate_size, - }; - return .init( - .{ num_experts, roundUp(rows, 128), roundUp(@divExact(hidden_size, 16), 4) }, - .f8e4m3fn, - ); -} - -pub fn fc2BlockScaleShape( - num_experts: i64, - hidden_size: i64, - intermediate_size: i64, -) zml.Shape { - return .init( - .{ num_experts, roundUp(hidden_size, 128), roundUp(@divExact(intermediate_size, 16), 4) }, - .f8e4m3fn, - ); -} - -fn isGlobalOrPerExpertScale(tensor: zml.Tensor, num_experts: i64) bool { - return tensor.dtype() == .f32 and - (tensor.rank() == 0 or (tensor.rank() == 1 and tensor.dim(0) == num_experts)); -} - -fn validateInputs( - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, - options: Options, -) !Attributes { - if (hidden_states.dtype() != .bf16 or - fc1_weights.dtype() != .f4e2m1 or - fc2_weights.dtype() != .f4e2m1 or - topk_weights.dtype() != .f32 or - topk_ids.dtype() != .i32 or - fc1_weight_block.dtype() != .f8e4m3fn or - fc2_weight_block.dtype() != .f8e4m3fn or - fc1_global.dtype() != .f32 or - fc2_global.dtype() != .f32) - { - return error.UnsupportedType; - } - if (hidden_states.rank() != 2 or - fc1_weights.rank() != 3 or - fc2_weights.rank() != 3 or - topk_weights.rank() != 2 or - topk_ids.rank() != 2) - { - return error.InvalidShape; - } - - const tokens = hidden_states.dim(.t); - const hidden_size = hidden_states.dim(.d); - const num_experts = fc1_weights.dim(0); - const fc1_rows = fc1_weights.dim(1); - const intermediate_size = switch (options.activation) { - .silu, .gelu, .relu => fc1_rows, - .swiglu, .geglu, .geglu_tanh, .swiglu_step => @divExact(fc1_rows, 2), - }; - const top_k = topk_ids.dim(.topk); - - if (tokens <= 0 or hidden_size <= 0 or - num_experts <= 0 or intermediate_size <= 0 or - @mod(hidden_size, 16) != 0 or @mod(intermediate_size, 16) != 0 or - top_k <= 0 or top_k > num_experts) - { - return error.InvalidShape; - } - if (fc1_weights.dim(2) != hidden_size or - fc2_weights.dim(0) != num_experts or - fc2_weights.dim(1) != hidden_size or - fc2_weights.dim(2) != intermediate_size or - topk_weights.dim(.t) != tokens or - topk_weights.dim(.topk) != top_k or - topk_ids.dim(.t) != tokens or - !fc1_weight_block.shape().eql(fc1BlockScaleShape( - num_experts, - hidden_size, - intermediate_size, - options.activation, - )) or - !fc2_weight_block.shape().eql(fc2BlockScaleShape( - num_experts, - hidden_size, - intermediate_size, - )) or - !fc1_global.shape().eql(.init(.{num_experts}, .f32)) or - !fc2_global.shape().eql(.init(.{num_experts}, .f32)) or - !isGlobalOrPerExpertScale(fc1_act_global, num_experts) or - !isGlobalOrPerExpertScale(fc2_act_global, num_experts)) - { - return error.InvalidShape; - } - - return .{ - .num_tokens = tokens, - .hidden_size = hidden_size, - .intermediate_size = intermediate_size, - .num_experts = @intCast(num_experts), - .top_k = @intCast(top_k), - .activation = cutlassActivation(options.activation), - .enable_pdl = options.enable_pdl, - .fc1_act_per_expert = fc1_act_global.rank() == 1, - .fc2_act_per_expert = fc2_act_global.rank() == 1, - .gemm1_tactic = options.gemm1_tactic, - .gemm2_tactic = options.gemm2_tactic, - }; -} - -/// Routed NVFP4 CUTLASS MoE. Weights are logical E2M1 tensors in [expert, output, input] order. -/// Block scales must use FlashInfer nvfp4_block_scale_interleave layout -/// BF16 activations are dynamically quantized to NVFP4 by the fused runner before each expert GEMM. -pub fn fusedExpertsNvfp4( - hidden_states: zml.Tensor, - fc1_weights: zml.Tensor, - fc2_weights: zml.Tensor, - topk_weights: zml.Tensor, - topk_ids: zml.Tensor, - fc1_act_global: zml.Tensor, - fc1_weight_block: zml.Tensor, - fc1_global: zml.Tensor, - fc2_act_global: zml.Tensor, - fc2_weight_block: zml.Tensor, - fc2_global: zml.Tensor, - options: Options, -) zml.Tensor { - const runners = currentRunners() catch |e| stdx.debug.panic("Failed to get runners: {}", .{e}); - var attributes = validateInputs( - hidden_states, - fc1_weights, - fc2_weights, - topk_weights, - topk_ids, - fc1_act_global, - fc1_weight_block, - fc1_global, - fc2_act_global, - fc2_weight_block, - fc2_global, - options, - ) catch |e| stdx.debug.panic("Invalid inputs: {}", .{e}); - - attributes.runners = @intFromPtr(runners); - const deviceRunner = runners.ensureRunner(options.workspace_query_device) catch |e| stdx.debug.panic("Failed to ensure runner: {}", .{e}); - const context = runtime.makeContext(attributes); - var requirements = std.mem.zeroes(fi_cutlass_moe.WorkspaceRequirements); - requirements.struct_size = @sizeOf(fi_cutlass_moe.WorkspaceRequirements); - runtime.checkStatus( - deviceRunner.api, - deviceRunner.api.getWorkspaceRequirements( - deviceRunner.runner, - &context, - &requirements, - ), - ) catch |e| stdx.debug.panic("Failed to get workspace requirements: {}", .{e}); - - const result = call( - .{ - .hidden_states = hidden_states, - .fc1_weights = fc1_weights, - .fc2_weights = fc2_weights, - .topk_weights = topk_weights, - .topk_ids = topk_ids, - .fc1_act_global = fc1_act_global, - .fc1_weight_block = fc1_weight_block, - .fc1_global = fc1_global, - .fc2_act_global = fc2_act_global, - .fc2_weight_block = fc2_weight_block, - .fc2_global = fc2_global, - }, - .{ - .output = hidden_states.shape(), - .workspace = .init(.{@as(i64, @intCast(requirements.total_bytes))}, .u8), - }, - attributes, - ); - return result.output; -} - -pub fn fusedExperts( - input: zml.Tensor, - routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - opts: GlobalOptions, -) zml.Tensor { - if (comptime !platforms.isEnabled(.cuda)) { - @panic("FlashInfer CUTLASS MoE is only supported on CUDA platforms"); - } - - if (gate_up.bias != null or down.bias != null) { - @panic("FlashInfer CUTLASS MoE does not support bias in gate_up or down linear layers"); - } - - const topk_ids = routing.localIds(); - // The runner ignores routes whose local expert id is -1. - const topk_weights = topk_ids.cmp(.GE, .scalar(0, .i32)).select(routing.weights, .scalar(-1, routing.weights.dtype())); - - const runner_options: Options = .fromGlobalOptions(opts); - - const quant_scheme: ?zml.Quantization.Scheme = if (gate_up.quantization) |q| q.scheme else null; - stdx.debug.assert(quant_scheme == .nvfp4, "FlashInfer CUTLASS MoE only supports NVFP4, got {?}", .{quant_scheme}); - - const gate_up_weight_unpacked = unpackedWeight(gate_up); - const down_weight_unpacked = unpackedWeight(down); - - // TODO(Corentin): Do error checking on nvfp4 - // Also, maybe pass `zml.nn.Linear` directly - return fusedExpertsNvfp4( - input, - gate_up_weight_unpacked, - down_weight_unpacked, - topk_weights, - topk_ids, - gate_up.quantization.?.input_scale.?.asMultiplier(), - gate_up.quantization.?.scales, - gate_up.quantization.?.global_scale.?.asMultiplier(), - down.quantization.?.input_scale.?.asMultiplier(), - down.quantization.?.scales, - down.quantization.?.global_scale.?.asMultiplier(), - runner_options, - ); -} diff --git a/zml/kernels/moe/flashinfer_cutlass/runtime.zig b/zml/kernels/moe/flashinfer_cutlass/runtime.zig deleted file mode 100644 index 028ef892..00000000 --- a/zml/kernels/moe/flashinfer_cutlass/runtime.zig +++ /dev/null @@ -1,223 +0,0 @@ -const std = @import("std"); - -const fi_cutlass_moe = @import("platforms/cuda/flashinfer_cutlass_moe"); -const pjrt = @import("pjrt"); - -const DataType = @import("../../../dtype.zig").DataType; -const contract = @import("contract.zig"); - -const log = std.log.scoped(.@"zml/kernels/moe/flashinfer_cutlass/runtime"); - -// TODO(Corentin): This was coming from Platform but we are now downstream. -// Find a way to solve that. -// Maybe the max_num_devices could be its own module because it only depends on Target. -const max_num_devices = 64; - -// TODO(Corentin): replace that with Shape when its dependencies are untangled and -// it doesn't depend on the platform anymore. -fn count(buffer: *const pjrt.ffi.Buffer) usize { - if (buffer.rank == 0) return @intCast(buffer.dims()[0]); - - var c: usize = 1; - for (buffer.dims()) |dim| c = c * @as(usize, @intCast(dim)); - - return c; -} - -fn byteSize(buffer: *const pjrt.ffi.Buffer) usize { - const dt: DataType = switch (buffer.dtype) { - .invalid => @panic("Found an invalid pjrt buffer"), - .token => @panic("Found an token pjrt buffer"), - inline else => |tag| @field(DataType, @tagName(tag)), - }; - return std.math.divCeil(usize, count(buffer) * dt.bitSizeOf(), 8) catch unreachable; -} - -pub fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - // TODO(Corentin): Find a place to abstract that - var input_buffers: contract.Inputs(*const pjrt.ffi.Buffer) = undefined; - inline for (@typeInfo(contract.Inputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { - const buf = call_frame.args.buffers()[i]; - @field(input_buffers, field_name) = buf; - } - - var output_buffers: contract.Outputs(*const pjrt.ffi.Buffer) = undefined; - inline for (@typeInfo(contract.Outputs(*const pjrt.ffi.Buffer)).@"struct".field_names, 0..) |field_name, i| { - const buf = call_frame.results.buffers()[i]; - @field(output_buffers, field_name) = buf; - } - - const attributes: contract.Attributes = switch (@typeInfo(contract.Attributes)) { - .void => {}, - .@"struct" => blk: { - var attrs: contract.Attributes = undefined; - inline for (@typeInfo(contract.Attributes).@"struct".field_names, @typeInfo(contract.Attributes).@"struct".field_types) |field_name, Field| { - // TODO: Use getByIndex instead? - const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse - @panic("Attribute not found: " ++ field_name); - @field(attrs, field_name) = attribute.get(Field); - } - break :blk attrs; - }, - else => @compileError("Unsupported attributes type: " ++ @typeName(contract.Attributes)), - }; - - return handlerInner(call_frame, input_buffers, output_buffers, attributes) catch |err| - pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); -} - -fn handlerInner( - call_frame: *pjrt.ffi.CallFrame, - inputs: contract.Inputs(*const pjrt.ffi.Buffer), - outputs: contract.Outputs(*const pjrt.ffi.Buffer), - attributes: contract.Attributes, -) !?*pjrt.ffi.Error { - const device = try call_frame.ctx.getDeviceOrdinal(call_frame.api); - const runners = try runnersFromAttributes(attributes); - const deviceRunner = try runners.ensureRunner(device); - const context = makeContext(attributes); - - var io = std.mem.zeroes(fi_cutlass_moe.Io); - io.struct_size = @sizeOf(fi_cutlass_moe.Io); - io.input = inputs.hidden_states.data; - io.token_selected_experts = @ptrCast(@alignCast(inputs.topk_ids.data)); - io.token_final_scales = @ptrCast(@alignCast(inputs.topk_weights.data)); - io.fc1_expert_weights = inputs.fc1_weights.data; - io.fc2_expert_weights = inputs.fc2_weights.data; - io.quant_scales[0] = inputs.fc1_act_global.data; - io.quant_scales[1] = inputs.fc1_weight_block.data; - io.quant_scales[2] = inputs.fc1_global.data; - io.quant_scales[3] = inputs.fc2_act_global.data; - io.quant_scales[4] = inputs.fc2_weight_block.data; - io.quant_scales[5] = inputs.fc2_global.data; - io.quant_scale_count = 6; - io.quant_scale_per_expert_mask = - (@as(u32, @intFromBool(attributes.fc1_act_per_expert)) << 0) | - (@as(u32, @intFromBool(attributes.fc2_act_per_expert)) << 3); - io.output = outputs.output.data; - - var workspace = std.mem.zeroes(fi_cutlass_moe.Workspace); - workspace.struct_size = @sizeOf(fi_cutlass_moe.Workspace); - workspace.data = outputs.workspace.data; - workspace.data_bytes = byteSize(outputs.workspace); - - try checkStatus( - deviceRunner.api, - deviceRunner.api.run( - deviceRunner.runner, - &context, - &io, - &workspace, - @ptrCast(call_frame.api.stream(call_frame.ctx)), - attributes.gemm1_tactic, - attributes.gemm2_tactic, - ), - ); - return null; -} - -pub fn makeContext(attributes: contract.Attributes) fi_cutlass_moe.Context { - var context = std.mem.zeroes(fi_cutlass_moe.Context); - context.struct_size = @sizeOf(fi_cutlass_moe.Context); - context.num_tokens = attributes.num_tokens; - context.hidden_size = attributes.hidden_size; - context.intermediate_size = attributes.intermediate_size; - context.num_experts = attributes.num_experts; - context.num_experts_on_rank = attributes.num_experts; - context.top_k = attributes.top_k; - context.tp_size = 1; - context.ep_size = 1; - context.activation = @intCast(attributes.activation); - context.enable_pdl = @intFromBool(attributes.enable_pdl); - context.swizzled_input_sf = 1; - return context; -} - -pub fn checkStatus(api: *const fi_cutlass_moe.Api, status: fi_cutlass_moe.Status) !void { - if (status == fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_SUCCESS) return; - if (api.lastError()) |message| { - log.err("FlashInfer CUTLASS MoE failed (status {d}): {s}", .{ - status, - std.mem.span(message), - }); - } else { - log.err("FlashInfer CUTLASS MoE failed with status {d}", .{status}); - } - return switch (status) { - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_INVALID_ARGUMENT => error.InvalidArgument, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_UNSUPPORTED => error.UnsupportedArchitecture, - fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_STATUS_CUDA_ERROR => error.Cuda, - else => error.FlashinferCutlassMoe, - }; -} - -fn runnersFromAttributes(attributes: contract.Attributes) !*Runners { - if (attributes.runners == 0) return error.RunnersNotLoaded; - return @ptrFromInt(attributes.runners); -} - -pub const Runners = struct { - runners: [max_num_devices]?DeviceRunner = @splat(null), - - pub fn deinit(self: *Runners) void { - for (self.runners) |device_runner| { - if (device_runner) |runner| { - _ = runner.api.runnerDestroy(runner.runner); - } - } - } - - pub fn ensureRunner(self: *Runners, device: i32) !DeviceRunner { - if (device < 0 or device >= max_num_devices) return error.UnsupportedDevice; - - const index: usize = @intCast(device); - if (self.runners[index]) |runner| return runner; - - const api = try fi_cutlass_moe.apiForDevice(device); - const options = runnerOptions(device); - var runner: ?*fi_cutlass_moe.Runner = null; - try checkStatus(api, api.runnerCreate(&options, &runner)); - const result: DeviceRunner = .{ - .api = api, - .runner = runner orelse return error.RunnerInitializationFailed, - }; - self.runners[index] = result; - return result; - } -}; - -const DeviceRunner = struct { - api: *fi_cutlass_moe.Api, - runner: *fi_cutlass_moe.Runner, -}; - -fn runnerOptions(device: i32) fi_cutlass_moe.RunnerOptions { - var options = std.mem.zeroes(fi_cutlass_moe.RunnerOptions); - options.struct_size = @sizeOf(fi_cutlass_moe.RunnerOptions); - options.activation_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.weight_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_PACKED_FP4; - options.output_dtype = fi_cutlass_moe.c.ZML_FI_CUTLASS_MOE_DTYPE_BF16; - options.device = device; - options.use_fused_finalize = 1; - return options; -} - -pub fn tacticCounts( - runners: *const Runners, - device: i32, -) !struct { gemm1: i32, gemm2: i32 } { - const deviceRunner = try runners.ensureRunner(device); - var gemm1: i32 = 0; - var gemm2: i32 = 0; - try checkStatus( - deviceRunner.api, - deviceRunner.api.getTacticCounts(deviceRunner.runner, &gemm1, &gemm2), - ); - return .{ .gemm1 = gemm1, .gemm2 = gemm2 }; -} - -pub fn load(allocator: std.mem.Allocator, io: std.Io, sm: u16) !void { - try fi_cutlass_moe.load(allocator, io, sm); -} diff --git a/zml/kernels/moe/fly/BUILD.bazel b/zml/kernels/moe/fly/BUILD.bazel deleted file mode 100644 index cb3b2ff4..00000000 --- a/zml/kernels/moe/fly/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "fly", - srcs = [ - "fly.zig", - "moe.zig", - ], -) diff --git a/zml/kernels/moe/fly/fly.zig b/zml/kernels/moe/fly/fly.zig deleted file mode 100644 index ba3d3c00..00000000 --- a/zml/kernels/moe/fly/fly.zig +++ /dev/null @@ -1,294 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const Tensor = zml.Tensor; -const Shape = zml.Shape; -const DataType = zml.DataType; -const ExpertsLayout = @import("../api.zig").ExpertsLayout; -const ProjectionLayout = @import("../api.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const Activation = @import("../api.zig").Activation; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; -const fly = @import("moe.zig"); - -const log = std.log.scoped(.@"zml/moe/fused_experts"); - -test { - std.testing.refAllDecls(@This()); -} - -pub fn fusedExperts( - hidden_states: Tensor, - routing_: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - opts: Options, -) Tensor { - const expert_map_ = routing_.expertMap(); - const layout: ProjectionLayout = if (gate_up_.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; - const num_tokens = hidden_states.dim(.t); - var launch_config = launchConfigForTokens(num_tokens); - - const gate_up_scheme = gate_up_.quantizationScheme(); - const down_scheme = down_.quantizationScheme(); - if (gate_up_scheme) |scheme| switch (scheme) { - .nvfp4 => stdx.debug.panic("fly backend doesn't support gate_up nvfp4 quantization scheme", .{}), - .fp8_block128 => launch_config.block_size_k = 128, - .fp8_block32 => launch_config.block_size_k = 32, - .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, - }; - - var down_launch_config = launchConfigForTokens(num_tokens); - if (down_scheme) |scheme| switch (scheme) { - .nvfp4 => stdx.debug.panic("fly backend doesn't support down nvfp4 quantization scheme", .{}), - .fp8_block128 => down_launch_config.block_size_k = 128, - .fp8_block32 => down_launch_config.block_size_k = 32, - .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, - }; - - const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); - const down = down_.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); - - stdx.debug.assert(hidden.dtype() == .bf16, "fly backend expected BF16 hidden states, got {}", .{hidden.dtype()}); - if (gate_up_scheme == .mxfp4) { - stdx.debug.assert(gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1, "fly backend expected u8, i8 or f4e2m1 gate_up data type when quantization scheme is mxfp4, got {}", .{gate_up.dtype()}); - } else { - stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "fly backend expected gate_up data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{gate_up.dtype()}); - } - if (down_scheme == .mxfp4) { - stdx.debug.assert(down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1, "fly backend expected u8, i8 or f4e2m1 down data type when quantization scheme is mxfp4, got {}", .{down.dtype()}); - } else { - stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "fly backend expected down data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{down.dtype()}); - } - - stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "fly backend expected f32 or bf16 routing weights, got {}", .{routing_weights.dtype()}); - stdx.debug.assert(ids.dtype() == .i32, "fly backend expected i32 expert ids, got {}", .{ids.dtype()}); - const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); - const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); - stdx.debug.assert(hidden.dim(.in) == gate_up_k, "fly backend expected hidden width to match gate/up input width, got {} and {}", .{ hidden.dim(.in), gate_up_k }); - const activation_reduction: i64 = switch (opts.activation) { - .gelu, .relu, .silu => 1, - .swiglu, .swiglu_step, .geglu, .geglu_tanh => 2, - }; - stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "fly backend expected gate/up output width to be divisible by {}, got {}", .{ activation_reduction, gate_up.dim(.out) }); - stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "fly backend expected down input width to match activated width, got {} and {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); - stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "fly backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); - stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "fly backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); - stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "fly backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (expert_map_) |expert_map| - stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "fly backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - - const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); - const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - - const expert_ids = if (expert_map_) |expert_map| - expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) - else - routing.expert_ids; - - const hidden_quant, const input_scale = prepareInput(hidden, gate_up_scheme, gate_up.dtype(), opts.quantize_input); - - const gate_up_out = gemm(.{ - .input = hidden_quant, - .weight = gate_up, - .bias = gate_up_.bias, - .input_scale = input_scale, - .weight_scale = gate_up_.quantizationScales(), - .routing = routing, - .expert_ids = expert_ids, - .launch_config = launch_config, - .quant_scheme = gate_up_scheme, - .top_k = @intCast(ids.dim(.topk)), - .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), - }); - - var activated = opts.activation.applyActivation(gate_up_out, layout); - if (opts.routing_weight_placement == .before_down) { - const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); - activated = activated.mul(weights.broad(activated.shape())); - } - const activated_quant, const down_input_scale = prepareInput(activated, down_scheme, down.dtype(), opts.quantize_input); - - const down_out = gemm(.{ - .input = activated_quant, - .weight = down, - .bias = down_.bias, - .input_scale = down_input_scale, - .weight_scale = down_.quantizationScales(), - .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, - .routing = routing, - .expert_ids = expert_ids, - .launch_config = down_launch_config, - .quant_scheme = down_scheme, - .top_k = 1, - .output_shape = Shape.init(.{ .token = num_tokens, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), - }); - - const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); - - return output.reshape(.{ .t = num_tokens, .d = down.dim(.out) }); -} - -// TODO(Corentin): unused for now -pub fn supports(hidden_states: Tensor, routing: LocalRouting, gate_up_: zml.nn.Linear, down_: zml.nn.Linear, opts: Options) bool { - if (hidden_states.dtype() != .bf16 or gate_up_.quantizationScheme() != .mxfp4 or - down_.quantizationScheme() != .mxfp4 or gate_up_.bias != null or down_.bias != null or - opts.routing_weight_placement != .before_down) return false; - const gate_up = gate_up_.weight; - const down = down_.weight; - if (gate_up.rank() != 3 or down.rank() != 3) return false; - const gate_up_scale = gate_up_.quantizationScales() orelse return false; - const down_scale = down_.quantizationScales() orelse return false; - const experts = routing.global_num_experts; - if (experts < 1 or experts > 512) return false; - var max_width: i64 = hidden_states.dim(.d); - inline for (.{ .{ gate_up, gate_up_scale }, .{ down, down_scale } }) |projection| { - const weight, const scale = projection; - const packing_factor: i64 = switch (weight.dtype()) { - .u8, .i8 => 2, - .f4e2m1 => 1, - else => return false, - }; - const n = weight.dim(1); - const k = weight.dim(2) * packing_factor; - if (!fly.Mfma.supportsShape(n, k)) return false; - max_width = @max(max_width, n); - if (weight.dim(0) < 1 or weight.dim(0) > 512 or - scale.rank() != 3 or scale.dim(0) != weight.dim(0) or scale.dim(1) != n or scale.dim(2) != @divExact(k, 32)) return false; - switch (scale.dtype()) { - .u8, .i8, .f8e8m0 => {}, - else => return false, - } - } - const routes = hidden_states.dim(.t) * routing.ids.dim(.topk); - return routes > 0 and routes <= @divTrunc(std.math.maxInt(i32), max_width); -} - -pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, _: zml.DataType, _: bool) struct { Tensor, ?Tensor } { - std.debug.assert(scheme == .mxfp4); - return .{ x.convert(.bf16), null }; -} - -pub const GemmOptions = struct { - input: Tensor, - weight: Tensor, - bias: ?Tensor = null, - input_scale: ?Tensor = null, - weight_scale: ?Tensor = null, - routing_weights: ?Tensor = null, - routing: Routing, - expert_ids: Tensor, - launch_config: LaunchConfig, - quant_scheme: ?zml.Quantization.Scheme = null, - top_k: usize, - output_shape: Shape, -}; - -pub fn gemm(opts: GemmOptions) zml.Tensor { - zml.stdx.debug.assert(opts.quant_scheme == .mxfp4 and opts.bias == null and opts.routing_weights == null and opts.input_scale == null, "Fly MoE requires MXFP4 weights, BF16 activations and routing weights before the down projection", .{}); - const weight = if (opts.weight.dtype() == .f4e2m1) - opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) - else - opts.weight; - return fly.Mfma.call(opts.input, weight, opts.weight_scale.?.bitCast(.u8), opts.routing.sorted_token_ids, opts.expert_ids, opts.routing.num_tokens_post_padded, opts.output_shape, opts.top_k, opts.routing.naive_block_assignment, opts.launch_config.block_size_m); -} - -pub const Routing = struct { - sorted_token_ids: Tensor, - expert_ids: Tensor, - num_tokens_post_padded: Tensor, - max_num_tokens_padded: i64, - num_assignments: i64, - naive_block_assignment: bool, -}; - -pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing { - std.debug.assert(num_experts > 0 and num_experts <= 512); - std.debug.assert(block_size_m > 0 and block_size_m <= 128 and @mod(block_size_m, 16) == 0); - const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); - const num_assignments = ids.dim(.token) * ids.dim(.topk); - // GEMV handles up to six routes; MFMA shares weights across sorted rows. - const naive_block_assignment = num_assignments <= 6; - const max_num_tokens_padded = if (naive_block_assignment or num_assignments < num_experts) - num_assignments * block_size_m - else - (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; - const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { - break :blk .{ - Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), - ids.reshape(.{ .g = num_assignments }), - Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), - }; - } else blk: { - const flat_experts = ids.reshape(.{ .g = num_assignments }); - const aligned = fly.Align.call( - flat_experts, - Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)), - Tensor.zeroes(Shape.init(.{ .g = @divExact(max_num_tokens_padded, block_size_m) }, .i32)), - Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), - Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)), - block_size_m, - ); - const sorted = fly.Sort.call(flat_experts, aligned.sorted_token_ids, aligned.cumsum); - break :blk .{ sorted.sorted_token_ids, aligned.expert_ids, aligned.num_tokens_post_pad }; - }; - return .{ - .sorted_token_ids = sorted_token_ids, - .expert_ids = expert_ids, - .num_tokens_post_padded = num_tokens_post_padded, - .max_num_tokens_padded = max_num_tokens_padded, - .num_assignments = num_assignments, - .naive_block_assignment = naive_block_assignment, - }; -} - -// ============================================================================= -// Config / validation helpers -// ============================================================================= - -const LaunchConfig = struct { - block_size_m: usize, - block_size_n: usize, - block_size_k: usize, - group_size_m: usize, - num_warps: i32, - num_stages: i32, -}; - -const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ - .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, - .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, - .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, - .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, - .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, -}; - -fn launchConfigForTokens(num_tokens: i64) LaunchConfig { - var best = launch_configs[0]; - for (launch_configs[1..]) |candidate| { - // The table is ascending, so ties retain the smaller token bucket. - if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { - best = candidate; - } - } - return best.config; -} diff --git a/zml/kernels/moe/fly/moe.zig b/zml/kernels/moe/fly/moe.zig deleted file mode 100644 index 3cb82b96..00000000 --- a/zml/kernels/moe/fly/moe.zig +++ /dev/null @@ -1,747 +0,0 @@ -const std = @import("std"); -const zml = @import("../../../zml.zig"); -const Tensor = zml.Tensor; -const Shape = zml.Shape; -const fly = zml.kernel.fly; -const Builder = fly.Builder; -const Value = fly.Value; - -pub const Align = struct { - const Result = struct { - sorted_token_ids: Tensor, - expert_ids: Tensor, - num_tokens_post_pad: Tensor, - cumsum: Tensor, - }; - - fn supports(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) bool { - for ([_]Tensor{ ids, sorted, experts, total, cumsum }) |tensor| { - if (tensor.rank() != 1 or tensor.dtype() != .i32) return false; - } - const num_experts = cumsum.dim(0) - 1; - if (num_experts < 1 or num_experts > 512 or total.dim(0) != 1 or ids.dim(0) > std.math.maxInt(i32) or - block_size <= 0 or block_size > 128 or @mod(block_size, 16) != 0) return false; - const capacity_bound = if (ids.dim(0) < num_experts) - ids.dim(0) * block_size - else - @divTrunc(ids.dim(0) + num_experts * (block_size - 1) + block_size - 1, block_size) * block_size; - return capacity_bound <= std.math.maxInt(i32) and sorted.dim(0) >= capacity_bound and - sorted.dim(0) <= std.math.maxInt(i32) and @mod(sorted.dim(0), block_size) == 0 and - experts.dim(0) == @divTrunc(sorted.dim(0), block_size); - } - - /// All four outputs alias their corresponding input buffers. Invalid expert IDs - /// are ignored, and all unused expert blocks are marked -1. Prefix ends in LDS - /// let each thread find its output block's expert without scanning every expert. - pub fn call(ids: Tensor, sorted: Tensor, experts: Tensor, total: Tensor, cumsum: Tensor, block_size: i64) Result { - std.debug.assert(Align.supports(ids, sorted, experts, total, cumsum, block_size)); - const cfg: Config = .{ - .routes = ids.dim(0), - .experts = cumsum.dim(0) - 1, - .capacity = sorted.dim(0), - .blocks = experts.dim(0), - .block_size = block_size, - .padding = block_size - 1, - }; - const result = Kernel.call(.{ .ids = ids, .sorted = sorted, .experts = experts, .total = total, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted.shape(), .expert_ids = experts.shape(), .num_tokens_post_pad = total.shape(), .cumsum = cumsum.shape() }, .{ - .cfg = cfg, - .threads = 512, - .grid = .{ 2, 1, 1 }, - .output_operand_aliases = .{ .sorted_token_ids = .sorted, .expert_ids = .experts, .num_tokens_post_pad = .total, .cumsum = .cumsum_in }, - }); - return .{ .sorted_token_ids = result.sorted_token_ids, .expert_ids = result.expert_ids, .num_tokens_post_pad = result.num_tokens_post_pad, .cumsum = result.cumsum }; - } - - const Config = struct { - routes: i64, - experts: i64, - capacity: i64, - blocks: i64, - block_size: i64, - padding: i64, - }; - - const Kernel = zml.kernel.fly.Kernel(Config, .{ - .name = "fly_moe_align", - .inputs = &.{ "ids", "sorted", "experts", "total", "cumsum_in" }, - .outputs = &.{ "sorted_token_ids", "expert_ids", "num_tokens_post_pad", "cumsum" }, - .run = run, - }); - - fn run(b: *Builder, cfg: Config) fly.FinishError!void { - const args = Kernel.args(b); - const tid = b.threadId(.x); - const tid64 = tid.to(.i64); - const zero = b.constant(.i32, 0); - const one = b.constant(.i32, 1); - const experts = b.constant(.i32, cfg.experts); - // Distinct arrays match the histogram, inclusive ends and wave totals. - const counts = b.sharedArray(.i32, 512, 16); - const ends = b.sharedArray(.i32, 512, 16); - const waves = b.sharedArray(.i32, 8, 16); - var fill = b.openIfElse(b.blockId(.x).ne(0), .{}); - { - var positions = b.openFor(tid64, cfg.capacity, 512, .{}); - { - args.sorted_token_ids.set(positions.iv, b.constant(.i32, cfg.routes)); - positions.close(); - } - fill.elseBranch(); - } - { - counts.set(tid, zero); - b.barrier(); - var routes = b.openFor(tid64, cfg.routes, 512, .{}); - { - const id = args.ids.at(routes.iv); - var valid_id = b.openIf(id.ge(0).bitAnd(id.lt(cfg.experts))); - { - _ = b.ptrAtomicAdd(counts.emitIter().addOffset(id), one, .workgroup); - valid_id.close(); - } - routes.close(); - } - b.barrier(); - - const padded = counts.at(tid).add(cfg.padding).divUnsigned(cfg.block_size).mul(cfg.block_size); - const lane = tid.remUnsigned(64); - const wave = tid.divUnsigned(64); - var prefix = padded; - inline for (.{ 1, 2, 4, 8, 16, 32 }) |shift| { - const peer = prefix.shuffle(.up, shift, 64); - prefix = prefix.add(lane.cmpUnsigned(.ge, shift).select(peer, zero)); - } - var last_lane = b.openIf(lane.eq(63)); - { - waves.set(wave, prefix); - last_lane.close(); - } - b.barrier(); - - var sums = b.openFor(zero, 8, 1, .{ zero, zero }); - { - const value = waves.at(sums.iv); - const preceding = sums.carried[0].add(sums.iv.cmpUnsigned(.lt, wave).select(value, zero)); - const total = sums.carried[1].add(value); - sums.yield(.{ preceding, total }); - } - const end = sums.results[0].add(prefix); - const start = end.sub(padded); - ends.set(tid, end); - var has_expert = b.openIf(tid.cmpUnsigned(.lt, cfg.experts)); - { - args.cumsum.set(tid64, start); - has_expert.close(); - } - var first = b.openIf(tid.eq(0)); - { - args.cumsum.set(b.constant(.i64, cfg.experts), sums.results[1]); - args.num_tokens_post_pad.set(b.constant(.i64, 0), sums.results[1]); - first.close(); - } - b.barrier(); - - var blocks = b.openFor(tid64, cfg.blocks, 512, .{}); - { - const offset = blocks.iv.to(.i32).mul(cfg.block_size); - var block_expert = b.openIfElse(offset.cmpUnsigned(.lt, sums.results[1]), .{zero}); - { - // Upper bound of inclusive expert ends handles empty experts without a scan. - var bounds = b.openWhile(.{ zero, experts }, .{ zero, zero }); - { - bounds.yieldBefore(bounds.before_carried[0].cmpUnsigned(.lt, bounds.before_carried[1]), .{ bounds.before_carried[0], bounds.before_carried[1] }); - } - { - const lo = bounds.after_carried[0]; - const hi = bounds.after_carried[1]; - const mid = lo.add(hi).shrU(1); - const advance = ends.at(mid).cmpUnsigned(.le, offset); - bounds.yieldAfter(.{ advance.select(mid.add(1), lo), advance.select(hi, mid) }); - } - block_expert.yieldThen(.{bounds.results[0]}); - } - { - block_expert.yieldElse(.{b.constant(.i32, -1)}); - } - args.expert_ids.set(blocks.iv, block_expert.results[0]); - blocks.close(); - } - fill.close(); - } - } -}; - -pub const Sort = struct { - const Result = struct { - sorted_token_ids: Tensor, - cumsum: Tensor, - }; - - fn supports(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) bool { - return ids.rank() == 1 and sorted_token_ids.rank() == 1 and cumsum.rank() == 1 and - ids.dtype() == .i32 and sorted_token_ids.dtype() == .i32 and cumsum.dtype() == .i32 and - cumsum.dim(0) >= 2 and cumsum.dim(0) <= std.math.maxInt(i32) and - ids.dim(0) <= std.math.maxInt(i32) and sorted_token_ids.dim(0) >= ids.dim(0); - } - - /// cumsum contains one starting offset per expert followed by the total padded - /// capacity. Each valid route atomically advances its expert's cursor; ordering - /// within an expert is unspecified. Invalid expert IDs leave both buffers alone. - pub fn call(ids: Tensor, sorted_token_ids: Tensor, cumsum: Tensor) Result { - std.debug.assert(Sort.supports(ids, sorted_token_ids, cumsum)); - if (ids.dim(0) == 0) return .{ .sorted_token_ids = sorted_token_ids, .cumsum = cumsum }; - const routes = ids.dim(0); - const grid = @min(@divTrunc(routes + 255, 256), 65535); - const cfg: Config = .{ .routes = routes, .experts = cumsum.dim(0) - 1, .step = grid * 256 }; - const result = Kernel.call(.{ .ids = ids, .sorted = sorted_token_ids, .cumsum_in = cumsum }, .{ .sorted_token_ids = sorted_token_ids.shape(), .cumsum = cumsum.shape() }, .{ - .cfg = cfg, - .threads = 256, - .grid = .{ @intCast(grid), 1, 1 }, - .output_operand_aliases = .{ .sorted_token_ids = .sorted, .cumsum = .cumsum_in }, - }); - return .{ .sorted_token_ids = result.sorted_token_ids, .cumsum = result.cumsum }; - } - - const Config = struct { - routes: i64, - experts: i64, - step: i64, - }; - - const Kernel = zml.kernel.fly.Kernel(Config, .{ - .name = "fly_count_and_sort_expert_tokens", - .inputs = &.{ "ids", "sorted", "cumsum_in" }, - .outputs = &.{ "sorted_token_ids", "cumsum" }, - .run = run, - }); - - fn run(b: *Builder, cfg: Config) fly.FinishError!void { - const args = Kernel.args(b); - const start = b.blockId(.x).to(.i64).mul(256).add(b.threadId(.x).to(.i64)); - var routes = b.openFor(start, cfg.routes, cfg.step, .{}); - { - const id = args.ids.at(routes.iv); - var valid = b.openIf(id.ge(0).bitAnd(id.lt(cfg.experts))); - { - const cursor = args.cumsum.emitIter().addOffset(id.to(.i64)); - const rank = b.ptrAtomicAdd(cursor, b.constant(.i32, 1), .agent); - args.sorted_token_ids.set(rank.to(.i64), routes.iv.to(.i32)); - valid.close(); - } - routes.close(); - } - } -}; - -pub const Mfma = struct { - /// MFMA loads full 64-column by 128-reduction tiles without tail masks. - pub fn supportsShape(n: i64, k: i64) bool { - return n > 0 and k > 0 and @mod(n, 64) == 0 and @mod(k, 128) == 0; - } - - /// The caller owns route localization, activation and reduction. - /// `expert_ids` contains local bank indices, including -1 for remote experts. - /// Sorted routes must contain a real prefix followed by padding per expert. - /// `block_m` is their alignment; native tiles contain 16 or 32 rows. - pub fn call( - input: Tensor, - weight: Tensor, - scales: Tensor, - sorted_ids: Tensor, - expert_ids: Tensor, - padded_count: Tensor, - output: zml.Shape, - topk: usize, - naive: bool, - block_m: usize, - ) Tensor { - std.debug.assert(input.rank() == 2 and input.dtype() == .bf16 and weight.rank() == 3 and - (weight.dtype() == .u8 or weight.dtype() == .i8) and scales.rank() == 3 and scales.dtype() == .u8 and - sorted_ids.rank() == 1 and sorted_ids.dtype() == .i32 and expert_ids.rank() == 1 and expert_ids.dtype() == .i32 and - padded_count.count() == 1 and padded_count.dtype() == .i32 and output.rank() >= 2 and output.dtype() == .bf16 and - topk > 0 and block_m > 0 and block_m % 16 == 0); - const n = weight.dim(1); - const k = input.dim(1); - std.debug.assert(supportsShape(n, k)); - std.debug.assert(input.dim(0) > 0 and weight.dim(0) > 0 and weight.dim(2) == @divExact(k, 2) and - scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32) and - output.dim(-1) == n); - // Keep packed-word counts and route/output indices within signed i32. - std.debug.assert(weight.count() / 4 <= std.math.maxInt(i32) and output.count() <= std.math.maxInt(i32) and - sorted_ids.count() <= std.math.maxInt(i32)); - const routes: i64 = @intCast(output.count() / @as(usize, @intCast(n))); - std.debug.assert(topk <= routes and @mod(@as(usize, @intCast(routes)), topk) == 0 and - input.dim(0) == @divTrunc(routes, @as(i64, @intCast(topk)))); - const capacity: i64 = @intCast(sorted_ids.count()); - const blocks: i64 = @intCast(expert_ids.count()); - if (naive) { - std.debug.assert(blocks == routes); - } else { - std.debug.assert(capacity > 0 and block_m <= capacity and @mod(capacity, @as(i64, @intCast(block_m))) == 0 and - blocks == @divExact(capacity, @as(i64, @intCast(block_m)))); - } - if (naive and routes <= 6) return Gemv.call(input, weight, scales, expert_ids, output, topk); - return callMfma(input, weight, scales, sorted_ids, expert_ids, padded_count, output, topk, naive, block_m); - } - - fn callMfma(input: Tensor, weight: Tensor, scales: Tensor, sorted_ids: Tensor, expert_ids: Tensor, padded_count: Tensor, output: zml.Shape, topk: usize, naive: bool, block_m: usize) Tensor { - const n = weight.dim(1); - const k = input.dim(1); - const routes = input.dim(0) * @as(i64, @intCast(topk)); - const native_m: i64 = if (!naive and block_m % 32 == 0) 32 else 16; - const cfg: Config = .{ - .tokens = input.dim(0), - .experts = weight.dim(0), - .n = n, - .k = k, - .ka = @divExact(k, 4), - .kb = @divExact(k, 8), - .ks = @divExact(k, 32), - .topk = @as(i64, @intCast(topk)), - .routes = routes, - .capacity = @as(i64, @intCast(sorted_ids.count())), - .blocks = @as(i64, @intCast(expert_ids.count())), - .subtiles = @divExact(@as(i64, @intCast(block_m)), native_m), - .native_m = native_m, - .wide = @as(i64, @intFromBool(native_m == 32)), - .stage_elements = native_m * 128, - .shared_elements = native_m * 256, - .naive = @as(i64, @intFromBool(naive)), - }; - const a = input.reshape(.{ input.dim(0), @divExact(k, 4), 4 }).bitCast(.i64); - const b = weight.reshape(.{ weight.dim(0), n, @divExact(k, 8), 4 }).bitCast(.i32); - return Kernel.call(.{ .a = a, .b = b, .scales = scales, .sorted = sorted_ids, .experts = expert_ids, .padded = padded_count.reshape(.{1}) }, .{ - .output = zml.Shape.init(.{ routes, n }, .bf16), - }, .{ - .cfg = cfg, - .threads = 256, - .grid = .{ @intCast(@divExact(n, 64)), @intCast(if (naive) routes else @divExact(cfg.capacity, native_m)), 1 }, - }).output.reshape(output); - } - - const Config = struct { - tokens: i64, - experts: i64, - n: i64, - k: i64, - ka: i64, - kb: i64, - ks: i64, - topk: i64, - routes: i64, - capacity: i64, - blocks: i64, - subtiles: i64, - native_m: i64, - wide: i64, - stage_elements: i64, - shared_elements: i64, - naive: i64, - }; - - const Kernel = zml.kernel.fly.Kernel(Config, .{ - .name = "mxfp4_raw_mfma", - .inputs = &.{ "a", "b", "scales", "sorted", "experts", "padded" }, - .outputs = &.{"output"}, - .run = run, - }); - - fn run(b: *Builder, cfg: Config) fly.FinishError!void { - const args = Kernel.args(b); - const zero = b.constant(.i32, 0); - const fzero = b.constant(.f32, 0.0); - const zeros = fzero.splat(4); - const block_n = b.blockId(.x).to(.i64); - const block_m = b.blockId(.y).to(.i64); - const route_start = block_m.mul(cfg.native_m); - const has_rows = if (cfg.naive != 0) b.constant(.i1, 1) else blk: { - const padded = args.padded.at(b.constant(.i64, 0)).to(.i64); - var has_block = b.openIfElse(route_start.cmpUnsigned(.lt, padded), .{.i1}); - { - has_block.yieldThen(.{args.sorted.at(route_start).ne(cfg.routes)}); - } - { - has_block.yieldElse(.{b.constant(.i1, 0)}); - } - break :blk has_block.results[0]; - }; - var block = b.openIf(has_rows); - { - const expert_block = if (cfg.naive != 0) block_m else block_m.divUnsigned(cfg.subtiles); - const eid = args.experts.at(expert_block); - const valid_expert = eid.ge(0).bitAnd(eid.lt(cfg.experts)); - const thread = b.threadId(.x).to(.i64); - const lane = thread.remUnsigned(64); - const wave = thread.divUnsigned(64); - const mn_lane = lane.remUnsigned(16); - const k_lane = lane.divUnsigned(16); - const out_n = block_n.mul(64).add(wave.mul(16)).add(mn_lane); - // Each expert has a contiguous real prefix before route-count padding. - const second_active = if (cfg.wide != 0) blk: { - const first = args.sorted.at(route_start.add(16)); - break :blk first.ge(0).bitAnd(first.lt(cfg.routes)); - } else b.constant(.i1, 0); - - var sums = b.openIfElse(valid_expert, .{ zeros, zeros }); - { - const expert = eid.to(.i64); - const load_row = thread.divUnsigned(16); - const load_chunk = thread.remUnsigned(16); - const a_route = routeForRow(b, args.sorted, block_m, route_start, load_row, cfg.naive != 0); - const a_valid = a_route.ge(0).bitAnd(a_route.lt(cfg.routes)); - const a_base = a_route.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); - var second_route = b.openIfElse(second_active, .{zero}); - { - second_route.yieldThen(.{args.sorted.at(route_start.add(load_row).add(16))}); - } - { - second_route.yieldElse(.{b.constant(.i32, -1)}); - } - const a_route_g1 = second_route.results[0]; - const a_valid_g1 = a_route_g1.ge(0).bitAnd(a_route_g1.lt(cfg.routes)); - const a_base_g1 = a_route_g1.to(.i64).divUnsigned(cfg.topk).mul(cfg.ka); - - // One allocation, two disjoint ping-pong regions; never two identical leaves. - const shared = b.sharedArray(.i16, cfg.shared_elements, 16); - const load_a = b.copyAtom(.{ .universal = 128 }, .i64); - const load_b = b.copyAtom(.{ .universal = 128 }, .i32); - const copy_shared = b.copyAtom(.{ .universal = 128 }, .i16); - const loader: TileLoader = .{ - .a = args.a, - .weight = args.b, - .scales = args.scales, - .shared = shared, - .load_a = load_a, - .load_b = load_b, - .copy_shared = copy_shared, - .a_base = a_base, - .a_base_g1 = a_base_g1, - .a_valid = a_valid, - .a_valid_g1 = a_valid_g1, - .second_active = second_active, - .a_col = load_chunk.mul(2), - .a_store = load_row.mul(128).add(load_chunk.bitXor(load_row).mul(8)), - .b_base = expert.mul(cfg.n * cfg.kb).add(out_n.mul(cfg.kb)), - .expert = expert, - .out_n = out_n, - .k_lane = k_lane, - }; - const initial = loader.load(b, b.constant(.i64, 0), b.constant(.i64, 0)); - b.barrier(); - const atom = b.mmaAtom((fly.fly_rocdl.MmaOpCDNA3MFMAType.get(b.ctx, .{ - .m = 16, - .n = 16, - .k = 16, - .elemTyA = fly.DType.bf16.toMlir(b.ctx), - .elemTyB = fly.DType.bf16.toMlir(b.ctx), - .elemTyAcc = fly.DType.f32.toMlir(b.ctx), - }) catch return error.InvalidMlir).type_()); - var k = b.openFor(b.constant(.i64, 0), cfg.k, 128, .{ zeros, zeros, initial.words, initial.scale }); - { - const read_base = k.iv.divUnsigned(128).bitAnd(1).mul(cfg.stage_elements); - const write_base = k.iv.divUnsigned(128).bitAnd(1).bitXor(1).mul(cfg.stage_elements); - var a_fragments: [4]Value = undefined; - var a_fragments_g1: [4]Value = undefined; - inline for (0..4) |u| { - const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)); - a_fragments[u] = b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8)); - } - inline for (0..4) |u| { - const offset = read_base.add(mn_lane.mul(128)).add(k_lane.mul(4).add(u).bitXor(mn_lane).mul(8)).add(2048); - const azero = b.constant(.i16, 0).splat(8); - var second = b.openIfElse(second_active, .{azero}); - { - second.yieldThen(.{b.copyAtomLoad(copy_shared, vectorView(shared, offset, 8))}); - } - { - second.yieldElse(.{azero}); - } - a_fragments_g1[u] = second.results[0]; - } - // Prefetch the other LDS region only after all current A fragments are read. - const next_k = k.iv.add(128); - var next = b.openIfElse(next_k.cmpUnsigned(.lt, cfg.k), .{ initial.words, initial.scale }); - { - const next_tile = loader.load(b, next_k, write_base); - next.yieldThen(.{ next_tile.words, next_tile.scale }); - } - { - next.yieldElse(.{ b.constant(.i32, 0).splat(4), fzero }); - } - var acc = k.carried[0]; - var acc_g1 = k.carried[1]; - inline for (0..4) |u| { - const decoded = decodeWeight(b, k.carried[2].extract(u), k.carried[3]); - const a0 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 0, 1, 2, 3 }).bitCast(.bf16); - const a1 = a_fragments[u].shuffleVector(a_fragments[u], &.{ 4, 5, 6, 7 }).bitCast(.bf16); - const w0 = decoded.shuffleVector(decoded, &.{ 0, 1, 2, 3 }); - const w1 = decoded.shuffleVector(decoded, &.{ 4, 5, 6, 7 }); - acc = b.mmaAtomCall(atom, a0, w0, acc); - acc = b.mmaAtomCall(atom, a1, w1, acc); - var second = b.openIfElse(second_active, .{zeros}); - { - const a0_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 0, 1, 2, 3 }).bitCast(.bf16); - const a1_g1 = a_fragments_g1[u].shuffleVector(a_fragments_g1[u], &.{ 4, 5, 6, 7 }).bitCast(.bf16); - const first_acc = b.mmaAtomCall(atom, a0_g1, w0, acc_g1); - second.yieldThen(.{b.mmaAtomCall(atom, a1_g1, w1, first_acc)}); - } - { - second.yieldElse(.{acc_g1}); - } - acc_g1 = second.results[0]; - } - // All current reads finish before a later iteration reuses this LDS region. - b.barrier(); - k.yield(.{ acc, acc_g1, next.results[0], next.results[1] }); - } - sums.yieldThen(.{ k.results[0], k.results[1] }); - } - { - sums.yieldElse(.{ zeros, zeros }); - } - - inline for (0..4) |i| { - storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i), out_n, sums.results[0].extract(i), cfg); - } - var store_second = b.openIf(second_active); - { - inline for (0..4) |i| { - storeRow(b, args.output, args.sorted, block_m, route_start, k_lane.mul(4).add(i + 16), out_n, sums.results[1].extract(i), cfg); - } - store_second.close(); - } - block.close(); - } - } - - fn routeForRow(b: *Builder, sorted: Value, block: Value, start: Value, row: Value, naive: bool) Value { - return if (naive) row.eq(0).select(block.to(.i32), b.constant(.i32, -1)) else sorted.at(start.add(row)); - } - - fn storeRow(b: *Builder, output: Value, sorted: Value, block: Value, start: Value, row: Value, col: Value, value: Value, cfg: Config) void { - const route = routeForRow(b, sorted, block, start, row, cfg.naive != 0); - var valid = b.openIf(route.ge(0).bitAnd(route.lt(cfg.routes))); - { - output.set(.{ route.to(.i64), col }, value.to(.bf16)); - valid.close(); - } - } - - fn vectorView(tensor: Value, offset: Value, comptime n: i64) Value { - return tensor.emitIter().addOffset(offset).view(fly.L(n, 1)); - } - - const TileLoader = struct { - a: Value, - weight: Value, - scales: Value, - shared: Value, - load_a: Value, - load_b: Value, - copy_shared: Value, - a_base: Value, - a_base_g1: Value, - a_valid: Value, - a_valid_g1: Value, - second_active: Value, - a_col: Value, - a_store: Value, - b_base: Value, - expert: Value, - out_n: Value, - k_lane: Value, - - fn load(self: TileLoader, b: *Builder, k: Value, write_base: Value) struct { words: Value, scale: Value } { - self.storeActivation(b, self.a_base, self.a_valid, k, self.a_store.add(write_base)); - var second = b.openIf(self.second_active); - { - self.storeActivation(b, self.a_base_g1, self.a_valid_g1, k, self.a_store.add(write_base).add(2048)); - second.close(); - } - const offset = self.b_base.add(k.divUnsigned(8)).add(self.k_lane.mul(4)); - const words = b.copyAtomLoad(self.load_b, vectorView(self.weight, offset, 4)); - const byte = self.scales.at(.{ self.expert, self.out_n, k.divUnsigned(32).add(self.k_lane) }); - const scale = byte.toUnsigned(.i32); - const finite = scale.eq(0).select(b.constant(.i32, 0x00400000), scale.shl(23)); - return .{ .words = words, .scale = scale.eq(255).select(b.constant(.i32, 0x7fc00000), finite).bitCast(.f32) }; - } - - fn storeActivation(self: TileLoader, b: *Builder, base: Value, valid: Value, k: Value, destination: Value) void { - const zeros = b.constant(.i64, 0).splat(2); - var guarded = b.openIfElse(valid, .{zeros}); - { - const offset = base.add(k.divUnsigned(4)).add(self.a_col); - guarded.yieldThen(.{b.copyAtomLoad(self.load_a, vectorView(self.a, offset, 2))}); - } - { - guarded.yieldElse(.{zeros}); - } - b.copyAtomStore(self.copy_shared, guarded.results[0].bitCast(.i16), vectorView(self.shared, destination, 8)); - } - }; - - fn decodeWeight(b: *Builder, word: Value, scale: Value) Value { - const lo = b.constant(.i32, 0x44403800); - const hi = b.constant(.i32, 0x54504c48); - const even = b.perm(hi, lo, word.bitAnd(0x07070707)); - const odd = b.perm(hi, lo, word.shrU(4).bitAnd(0x07070707)); - const ev0 = b.cvtPkF32Fp8(even, false); - const od0 = b.cvtPkF32Fp8(odd, false); - const ev1 = b.cvtPkF32Fp8(even, true); - const od1 = b.cvtPkF32Fp8(odd, true); - const first = ev0.shuffleVector(od0, &.{ 0, 2, 1, 3 }); - const last = ev1.shuffleVector(od1, &.{ 0, 2, 1, 3 }); - const magnitude = first.shuffleVector(last, &.{ 0, 1, 2, 3, 4, 5, 6, 7 }); - var shifts = b.constant(.i32, 0).splat(8); - inline for (0..8) |i| shifts = shifts.insert(i, b.constant(.i32, i * 4)); - const signs = word.splat(8).shrU(shifts).bitAnd(8).shl(28); - // FNUZ has no -0: restore FP4's sign in F32 before scaling and BF16 rounding. - return magnitude.bitCast(.i32).bitOr(signs).bitCast(.f32).mul(scale).to(.bf16); - } -}; - -const Gemv = struct { - // Packed-word GEMV derived from the supplied moe-mi300x/fly_word.py algorithm. - // Several rows share a CTA, and the shorter down projection uses subwaves to - // expose independent outputs without paying for a full wave reduction per row. - const Config = struct { rows: usize, lanes: usize, unroll: usize }; - - /// One local expert ID per original route; negative/out-of-bank IDs write zero. - fn call(x: Tensor, weight: Tensor, scales: Tensor, ids: Tensor, output: zml.Shape, topk: usize) Tensor { - const cfg: Gemv.Config = if (x.dim(1) >= 4096) - .{ .rows = 4, .lanes = 64, .unroll = 1 } - else - .{ .rows = 4, .lanes = 32, .unroll = 5 }; - const rows = cfg.rows; - const lanes = cfg.lanes; - const unroll = cfg.unroll; - const n = weight.dim(1); - const k = x.dim(1); - const routes: i64 = @intCast(ids.count()); - std.debug.assert(x.dtype() == .bf16 and (weight.dtype() == .u8 or weight.dtype() == .i8)); - std.debug.assert(@mod(n, 8) == 0 and @mod(k, 32) == 0); - std.debug.assert(topk > 0 and routes == x.dim(0) * @as(i64, @intCast(topk))); - std.debug.assert(weight.dim(2) * 2 == k); - std.debug.assert(scales.dim(0) == weight.dim(0) and scales.dim(1) == n and scales.dim(2) == @divExact(k, 32)); - const tile_rows = rows * (64 / lanes); - std.debug.assert(@mod(n, @as(i64, @intCast(tile_rows * 8))) == 0); - const fields: EmitConfig = .{ .unroll = unroll, .rows = tile_rows, .lanes = lanes, .stride = unroll * lanes, .ntiles = @divExact(n, @as(i64, @intCast(tile_rows))), .tokens = x.dim(0), .experts = weight.dim(0), .n = n, .k_pairs = @divExact(k, 2), .k_words = @divExact(k, 8), .k_scales = @divExact(k, 32), .routes = routes, .grid = @divExact(n, @as(i64, @intCast(tile_rows))) * routes, .topk = @as(i64, @intCast(topk)) }; - const a = x.reshape(.{ .token = x.dim(0), .in = @divExact(k, 2), .pair = 2 }).bitCast(.i32); - const b = weight.reshape(.{ .expert = weight.dim(0), .out = n, .in = @divExact(weight.dim(2), 4), .word = 4 }).bitCast(.i32); - const matrix_output = zml.Shape.init(.{ .token = routes, .out = n }, output.dtype()); - return Gemv.Kernel.call(.{ .a = a, .b = b, .scales = scales.bitCast(.u8), .ids = ids.reshape(.{routes}) }, .{ - .output = matrix_output, - }, .{ - .cfg = fields, - .threads = @intCast(rows * 64), - .grid = .{ @intCast(fields.grid), 1, 1 }, - }).output.reshape(output); - } - - const EmitConfig = struct { - unroll: usize, - rows: usize, - lanes: usize, - stride: usize, - ntiles: i64, - tokens: i64, - experts: i64, - n: i64, - k_pairs: i64, - k_words: i64, - k_scales: i64, - routes: i64, - grid: i64, - topk: i64, - }; - - const Kernel = zml.kernel.fly.Kernel(EmitConfig, .{ - .name = "mxfp4_gemv", - .inputs = &.{ "a", "b", "scales", "ids" }, - .outputs = &.{"output"}, - .run = Gemv.run, - }); - - fn run(b: *Builder, cfg: EmitConfig) fly.FinishError!void { - const args = Gemv.Kernel.args(b); - const block = b.blockId(.x).to(.i64); - const thread = b.threadId(.x).to(.i64); - const lane = thread.remUnsigned(cfg.lanes); - const rowgroup = thread.divUnsigned(cfg.lanes); - // Preserve the eight-XCD workgroup permutation and original subwave layout. - const pid = block.remUnsigned(8).mul(@divExact(cfg.grid, 8)).add(block.divUnsigned(8)); - const row = pid.remUnsigned(cfg.ntiles).mul(cfg.rows).add(rowgroup); - const route = pid.divUnsigned(cfg.ntiles); - const expert32 = args.ids.at(route); - const expert = expert32.to(.i64); - const token = route.divUnsigned(cfg.topk); - const local = expert32.ge(0).bitAnd(expert32.lt(cfg.experts)); - const inrow = row.cmpUnsigned(.lt, cfg.n); - const fzero = b.constant(.f32, 0.0); - var sum = b.openIfElse(local.bitAnd(inrow), .{fzero}); - { - var k = b.openFor(lane, cfg.k_words, cfg.stride, .{fzero}); - { - var carried = k.carried[0]; - for (0..cfg.unroll) |u| { - const pos = k.iv.add(u * cfg.lanes); - var inbounds = b.openIfElse(pos.cmpUnsigned(.lt, cfg.k_words), .{fzero}); - { - const word = args.b.at(.{ expert, row, pos }); - const scale8 = args.scales.at(.{ expert, row, pos.divUnsigned(4) }); - const scale = decodeScale(b, scale8); - const low = word.bitAnd(0x07070707); - const high = word.shrU(4).bitAnd(0x07070707); - const even_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), low); - const odd_bits = b.perm(b.constant(.i32, 0xd4d0ccc8), b.constant(.i32, 0xc4c0b800), high); - const even = even_bits.bitAnd(word.shl(4).bitOr(0x7f7f7f7f)); - const odd = odd_bits.bitAnd(word.bitOr(0x7f7f7f7f)); - // Retain the scalar decoder and its FP32 scale -> BF16 -> FP32 boundary. - const weights = [_]Value{ - b.cvtPkF32Fp8(even, false).mul(scale).to(.bf16).to(.f32), - b.cvtPkF32Fp8(odd, false).mul(scale).to(.bf16).to(.f32), - b.cvtPkF32Fp8(even, true).mul(scale).to(.bf16).to(.f32), - b.cvtPkF32Fp8(odd, true).mul(scale).to(.bf16).to(.f32), - }; - var partial = fzero; - inline for (0..4) |pair| { - const x = args.a.at(.{ token, pos.mul(4).add(pair) }); - const xlo = x.to(.i16).bitCast(.bf16).to(.f32); - const xhi = x.shrU(16).to(.i16).bitCast(.bf16).to(.f32); - const wlo = weights[(pair / 2) * 2].extract(pair % 2); - const whi = weights[(pair / 2) * 2 + 1].extract(pair % 2); - // Odd then even FMA, followed by one add to the K-loop accumulator. - partial = b.fma(xlo, wlo, b.fma(xhi, whi, partial)); - } - inbounds.yieldThen(.{partial.add(carried)}); - } - { - inbounds.yieldElse(.{carried}); - } - carried = inbounds.results[0]; - } - k.yield(.{carried}); - } - var reduced = k.results[0]; - var shift: i32 = 1; - while (shift < @as(i32, @intCast(cfg.lanes))) : (shift *= 2) { - reduced = reduced.add(reduced.shuffleXor(shift, 64)); - } - sum.yieldThen(.{reduced}); - } - { - sum.yieldElse(.{fzero}); - } - var store = b.openIf(lane.eq(0).bitAnd(inrow)); - { - args.output.set(.{ route, row }, sum.results[0].to(.bf16)); - store.close(); - } - } - - fn decodeScale(b: *Builder, byte: Value) Value { - const scale = byte.toUnsigned(.i32); - const finite = scale.eq(0).select(b.constant(.i32, 0x00400000), scale.shl(23)); - return scale.eq(255).select(b.constant(.i32, 0x7fc00000), finite).bitCast(.f32); - } -}; diff --git a/zml/kernels/moe/metal/BUILD.bazel b/zml/kernels/moe/metal/BUILD.bazel deleted file mode 100644 index b4240b00..00000000 --- a/zml/kernels/moe/metal/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "metal", - srcs = [ - "metal.zig", - ], -) diff --git a/zml/kernels/moe/metal/metal.zig b/zml/kernels/moe/metal/metal.zig deleted file mode 100644 index f835c889..00000000 --- a/zml/kernels/moe/metal/metal.zig +++ /dev/null @@ -1,164 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const Tensor = zml.Tensor; -const Activation = @import("../api.zig").Activation; -const GlobalOptions = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; -const unpackedWeight = @import("../weights.zig").unpackedWeight; - -const log = std.log.scoped(.@"zml/moe/metal"); - -fn validateOptions(opts: GlobalOptions) void { - stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); - stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); -} - -const QuantMode = enum { none, fp8, nvfp4 }; - -fn quantMode(dtype: zml.DataType) QuantMode { - return switch (dtype) { - .bf16, .f16, .f32 => .none, - .f8e4m3fn => .fp8, - .u8, .f4e2m1 => .nvfp4, - else => stdx.debug.panic("unsupported data type: {}", .{dtype}), - }; -} - -fn moeGemm( - x_rows: Tensor, - w: Tensor, - scale: ?Tensor, - expert_ids: Tensor, - out_shape: zml.Shape, - mode: QuantMode, -) Tensor { - const side: zml.ops.CustomCallOptions = .{ .has_side_effect = false }; - return switch (mode) { - .none => zml.ops.customCall("__metal$moe_gemm", .{ x_rows, w, expert_ids }, .{out_shape}, .{}, side), - .fp8 => zml.ops.customCall("__metal$moe_gemm$f8", .{ x_rows, w, scale.?, expert_ids }, .{out_shape}, .{}, side), - .nvfp4 => zml.ops.customCall("__metal$moe_gemm$f4", .{ x_rows, w, scale.?, expert_ids }, .{out_shape}, .{}, side), - }; -} - -fn applyGateUpGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) Tensor { - const scale = global_scale orelse return output; - stdx.debug.assert(scale.dtype() == .f32, "metal backend expected gate_up scale data type to be f32, got {}", .{scale.dtype()}); - stdx.debug.assert(scale.rank() == 2 and scale.dim(.proj) == 2, "metal backend expected gate_up scale to be a 2D tensor with a .proj dim equal to 2, got {}", .{scale.shape()}); - - const mid = @divExact(output.dim(.out), 2); - const selected = scale.gather(.{ .expert = expert_ids }, .{}); - const expanded = selected.appendAxes(.{.mid}) - .broad(zml.Shape.init(.{ .r = output.dim(.r), .proj = 2, .mid = mid }, .f32)) - .merge(.{ .out = .{ .proj, .mid } }); - return output.convert(.f32).mul(expanded).convert(output.dtype()); -} - -fn applyDownGlobalScale(output: Tensor, global_scale: ?Tensor, expert_ids: Tensor) Tensor { - const scale = global_scale orelse return output; - stdx.debug.assert(scale.dtype() == .f32, "metal backend expected down scale data type to be f32, got {}", .{scale.dtype()}); - stdx.debug.assert(scale.rank() == 1, "metal backend expected down scale to be a 1D tensor, got {}", .{scale.shape()}); - - const selected = scale.gather(.{ .expert = expert_ids }, .{}) - .appendAxes(.{.d}) - .broad(output.shape().withDtype(.f32)); - return output.convert(.f32).mul(selected).convert(output.dtype()); -} - -pub fn fusedExperts( - hidden_states: zml.Tensor, - routing: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - opts: GlobalOptions, -) zml.Tensor { - validateOptions(opts); - stdx.debug.assert(gate_up_.bias == null, "metal backend doesn't support gate_up bias", .{}); - stdx.debug.assert(down_.bias == null, "metal backend doesn't support down bias", .{}); - - const gate_up_scales: ?zml.Tensor = if (gate_up_.quantization) |q| q.scales else null; - const gate_up_global_scale: ?zml.Tensor = if (gate_up_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - - const down_scales: ?zml.Tensor = if (down_.quantization) |q| q.scales else null; - const down_global_scale: ?zml.Tensor = if (down_.quantization) |q| (if (q.global_scale) |scale| scale.asMultiplier() else null) else null; - - const gate_up = unpackedWeight(gate_up_); - const down = unpackedWeight(down_); - stdx.debug.assert(gate_up.dtype() == down.dtype(), "metal backend expected gate_up and down data type to match, got {} and {}", .{ gate_up.dtype(), down.dtype() }); - const mode = quantMode(gate_up.dtype()); - switch (mode) { - .none => { - stdx.debug.assert(gate_up_scales == null and down_scales == null, "metal backend expected gate_up and down scale to be null when the weights are not quantized", .{}); - stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null when the weights are not quantized", .{}); - const is_hidden_states_valid_dtype = switch (hidden_states.dtype()) { - .bf16, .f16, .f32 => true, - else => false, - }; - stdx.debug.assert(is_hidden_states_valid_dtype, "metal backend expected hidden_states data type to be bf16, f16 or f32, got {}", .{hidden_states.dtype()}); - }, - .fp8 => { - stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(gate_up_scales.?.dtype() == .bf16 and down_scales.?.dtype() == .bf16, "metal backend expected gate_up and down scale data type to be bf16, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); - stdx.debug.assert(gate_up_global_scale == null and down_global_scale == null, "metal backend expected gate_up and down global scale to be null", .{}); - stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); - }, - .nvfp4 => { - stdx.debug.assert(gate_up_scales != null and down_scales != null, "metal backend expected gate_up and down scale to be non null", .{}); - stdx.debug.assert(gate_up_scales.?.dtype() == .f8e4m3fn and down_scales.?.dtype() == .f8e4m3fn, "metal backend expected gate_up and down scale data type to be f8e4m3fn, got {} and {}", .{ gate_up_scales.?.dtype(), down_scales.?.dtype() }); - stdx.debug.assert(hidden_states.dtype() == .bf16, "metal backend expected hidden_states data type to be bf16, got {}", .{hidden_states.dtype()}); - }, - } - - const is_topk_weights_valid_dtype = switch (routing.weights.dtype()) { - .bf16, .f16, .f32 => true, - else => false, - }; - stdx.debug.assert(is_topk_weights_valid_dtype, "metal backend expected topk_weights data type to be bf16, f16 or f32, got {}", .{routing.weights.dtype()}); - stdx.debug.assert(routing.ids.dtype() == .i32, "metal backend expected topk_ids data type to be i32, got {}", .{routing.ids.dtype()}); - - const d = hidden_states.dim(.d); - const num_tokens = hidden_states.dim(.t); - const topk = routing.ids.dim(.topk); - const num_routes = num_tokens * topk; - - const act_dtype: zml.DataType = switch (mode) { - .none => hidden_states.dtype(), - .fp8, .nvfp4 => .bf16, - }; - - const hidden = hidden_states.reshape(.{ .token = num_tokens, .d = d }).withTags(.{ .token, .d }); - const x_rows = hidden.insertAxes(.d, .{.topk}) - .broad(zml.Shape.init(.{ .token = num_tokens, .topk = topk, .d = d }, act_dtype)) - .merge(.{ .r = .{ .token, .topk } }); - const expert_ids = routing.ids.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) - .merge(.{ .r = .{ .token, .topk } }).convert(.i32); - - const gate_up_out = applyGateUpGlobalScale(moeGemm( - x_rows, - gate_up, - gate_up_scales, - expert_ids, - zml.Shape.init(.{ .r = num_routes, .out = gate_up.dim(.dout) }, act_dtype), - mode, - ), gate_up_global_scale, expert_ids); - const activated = opts.activation.applyActivation(gate_up_out, .concatenated).convert(act_dtype); - - const down_out = applyDownGlobalScale(moeGemm( - activated, - down, - down_scales, - expert_ids, - zml.Shape.init(.{ .r = num_routes, .d = down.dim(.d) }, act_dtype), - mode, - ), down_global_scale, expert_ids); - - const weights = routing.weights.reshape(.{ .token = num_tokens, .topk = topk }).withTags(.{ .token, .topk }) - .merge(.{ .r = .{ .token, .topk } }); - const weighted = down_out.mul(weights.convert(down_out.dtype()).broad(down_out.shape())); - const combined = weighted.splitAxis(.r, .{ .token = num_tokens, .topk = topk }) - .sum(.topk).squeeze(.topk); - - return combined.reshape(.{ .t = num_tokens, .d = down.dim(.d) }); -} diff --git a/zml/kernels/moe/moe.zig b/zml/kernels/moe/moe.zig deleted file mode 100644 index eb697f36..00000000 --- a/zml/kernels/moe/moe.zig +++ /dev/null @@ -1,298 +0,0 @@ -const std = @import("std"); - -const platforms = @import("platforms"); - -const zml = @import("../../zml.zig"); -const stdx = zml.stdx; -pub const Activation = @import("api.zig").Activation; -pub const ActivationKind = @import("api.zig").ActivationKind; -pub const cute = @import("cute/cute.zig"); -pub const ExpertsLayout = @import("api.zig").ExpertsLayout; -pub const flashinfer_cutlass = @import("flashinfer_cutlass/flashinfer_cutlass.zig"); -pub const fly = @import("fly/fly.zig"); -pub const LocalRouting = @import("api.zig").LocalRouting; -pub const metal = @import("metal/metal.zig"); -pub const mosaic_tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -pub const Options = @import("api.zig").Options; -pub const ProjectionLayout = @import("api.zig").ProjectionLayout; -pub const RoutingWeightPlacement = @import("api.zig").RoutingWeightPlacement; -pub const stablehlo = @import("stablehlo/stablehlo.zig"); -pub const tcl = @import("tcl/tcl.zig"); -pub const triton = @import("triton/triton.zig"); - -const log = std.log.scoped(.@"zml/moe"); - -test { - std.testing.refAllDecls(@This()); - _ = @import("tests.zig"); -} - -pub const Backend = enum { - stablehlo, - cute, - flashinfer_cutlass, - triton, - fly, - mosaic_tpu, - metal, - tcl, - - pub fn auto(platform: *const zml.Platform, scheme: ?zml.Quantization.Scheme, dtype: zml.DataType) !Backend { - return switch (platform.target) { - .cuda => b: { - const s = scheme orelse break :b .triton; - break :b switch (s) { - .mxfp4 => if (cute.isAvailable(platform)) - .cute - else - .triton, - .nvfp4 => if (cute.isNvfp4Supported(platform)) - .cute - else if (flashinfer_cutlass.isNvfp4Supported(platform)) - .flashinfer_cutlass - else - error.UnsupportedQuantization, - .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .triton, - }; - }, - .rocm => b: { - const s = scheme orelse break :b .triton; - break :b switch (s) { - .mxfp4 => if (zml.platform.rocm.computeCapability(platform) == .gfx942) .fly else .triton, - .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .triton, - .nvfp4 => error.UnsupportedQuantization, - }; - }, - .oneapi => b: { - const s = scheme orelse break :b .triton; - break :b switch (s) { - .mxfp4 => .triton, - .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => error.UnsupportedQuantization, - }; - }, - .tpu => if (scheme == null) .mosaic_tpu else error.UnsupportedQuantization, - .furiosa => b: { - const s = scheme orelse break :b .tcl; - break :b switch (s) { - .fp8_block128 => if (dtype == .f8e4m3fn) .tcl else error.UnsupportedQuantization, - .mxfp4, .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block32 => error.UnsupportedQuantization, - }; - }, - .metal => b: { - const s = scheme orelse break :b .metal; - break :b switch (s) { - .nvfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 => .metal, - .mxfp4 => error.UnsupportedQuantization, - }; - }, - else => error.UnimplementedMoEBackend, - }; - } - - pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { - return switch (backend) { - .stablehlo => true, - .cute => cute.isAvailable(platform), - .flashinfer_cutlass => flashinfer_cutlass.isAvailable(platform), - .fly => switch (platform.target) { - .rocm => zml.platform.rocm.computeCapability(platform) == .gfx942, - else => false, - }, - .triton => switch (platform.target) { - .cuda, .rocm, .oneapi => true, - else => false, - }, - .mosaic_tpu => platform.target == .tpu, - .metal => platform.target == .metal, - .tcl => platform.target == .furiosa, - }; - } - - /// The layout contract the expert weights must have for a given backend with scheme - pub fn expertsLayout(backend: Backend, scheme: ?zml.Quantization.Scheme) !ExpertsLayout { - return switch (backend) { - .stablehlo => .{ .gate_up = .concatenated, .packing = .plain }, - .cute => if (scheme) |s| switch (s) { - .mxfp4 => .{ .gate_up = .row_interleaved_8, .packing = .mxfp4_interleaved8 }, - .nvfp4 => .{ .gate_up = .row_interleaved_1, .packing = .swizzled_scales }, - else => error.UnsupportedQuantization, - } else error.UnsupportedQuantization, - .flashinfer_cutlass => if (scheme) |s| switch (s) { - .nvfp4 => .{ .gate_up = .concatenated, .packing = .flashinfer_nvfp4 }, - else => error.UnsupportedQuantization, - } else error.UnsupportedQuantization, - .mosaic_tpu, .metal, .tcl => .{ .gate_up = .concatenated, .packing = .plain }, - .triton, .fly => if (scheme == .mxfp4) .{ .gate_up = .row_interleaved_1, .packing = .plain } else .{ .gate_up = .concatenated, .packing = .plain }, - }; - } - - pub fn register(backend: Backend, platform: *zml.Platform) !void { - return switch (backend) { - .stablehlo => {}, - .cute => {}, - .flashinfer_cutlass => flashinfer_cutlass.register(platform), - .triton, .fly => {}, - .mosaic_tpu => {}, - .metal => {}, - .tcl => {}, - }; - } -}; - -/// Check the shared token-wise contract for activations and routing tensors. -pub fn validateRoutingInputs(input: zml.Tensor, ids: zml.Tensor, weights: zml.Tensor) void { - stdx.debug.assert(input.rank() == 2 and ids.rank() == 2 and weights.rank() == 2, "MoE expects input {{.t, .d}} and routing {{.t, .topk}}, got {f}, {f}, {f}", .{ input.shape(), ids.shape(), weights.shape() }); - stdx.debug.assert(input.dim(.t) == ids.dim(.t) and input.dim(.t) == weights.dim(.t), "MoE input and routing token counts must match", .{}); - stdx.debug.assert(ids.dim(.topk) == weights.dim(.topk), "MoE routing ids and weights must have matching top-k dimensions", .{}); - stdx.debug.assert(input.shape().hasTag(.t) == 0 and input.shape().hasTag(.d) == 1 and - ids.shape().hasTag(.t) == 0 and ids.shape().hasTag(.topk) == 1 and - weights.shape().hasTag(.t) == 0 and weights.shape().hasTag(.topk) == 1, "MoE expects ordered tags {{.t, .d}} for input and {{.t, .topk}} for routing", .{}); -} - -pub const Partitioning = struct { - const Logical = zml.Sharding.Partitioning.Logical; - - in_specs: struct { - input: Logical, - ids: Logical, - weights: Logical, - gate_up: Logical, - down: Logical, - }, - out_specs: Logical, - - /// Shards the `.expert` dimension of both projections over `mesh_axis`. - /// Other dimensions, input tokens, routing IDs/weights and output are replicated. - pub fn expert(comptime mesh_axis: @EnumLiteral()) Partitioning { - const experts: Logical = .parse(.{ .expert = mesh_axis }); - return .{ - .in_specs = .{ - .input = .replicated, - .ids = .replicated, - .weights = .replicated, - .gate_up = experts, - .down = experts, - }, - .out_specs = .replicated, - }; - } -}; - -/// Input has shape {.t, .d}; routing IDs and weights have shape {.t, .topk}. -/// Returns {.t, .d}. Flatten batched input with merge(.{ .t = .{ .b, .s } }). -pub fn forwardMoe( - input: zml.Tensor, - topk_ids: zml.Tensor, - topk_weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - backend: Backend, - mesh: *const zml.Mesh, - partitioning: Partitioning, - opts: Options, -) zml.Tensor { - validateRoutingInputs(input, topk_ids, topk_weights); - - const shapes = .{ - .input = input.shape().withPartitioning(mesh, partitioning.in_specs.input), - .ids = topk_ids.shape().withPartitioning(mesh, partitioning.in_specs.ids), - .weights = topk_weights.shape().withPartitioning(mesh, partitioning.in_specs.weights), - .gate_up = gate_up.weight.shape().withPartitioning(mesh, partitioning.in_specs.gate_up), - .down = down.weight.shape().withPartitioning(mesh, partitioning.in_specs.down), - }; - const output_shape = input.shape().withPartitioning(mesh, partitioning.out_specs); - - const tokens = shapes.input.partition(.t); - const experts = shapes.gate_up.partition(.expert); - // Local kernels require complete feature vectors and top-k routing lists. - stdx.debug.assert(shapes.input.partition(.d) == .replicated and shapes.ids.partition(.topk) == .replicated and shapes.weights.partition(.topk) == .replicated, "MoE feature and top-k dimensions must be replicated", .{}); - // The expert-axis reduction must add contributions for the same token rows. - // Sharding tokens along that axis would mix contributions for different tokens. - stdx.debug.assert(shapes.ids.partition(.t) == tokens and shapes.weights.partition(.t) == tokens and output_shape.partition(.t) == tokens and output_shape.partition(.d) == .replicated, "MoE input, routing and output must have matching token partitioning", .{}); - stdx.debug.assert(shapes.down.partition(.expert) == experts and (!experts.isSharded() or experts != tokens), "MoE projections must agree on expert partitioning; tokens must be replicated along the expert axis", .{}); - for ([_]zml.Shape{ shapes.gate_up, shapes.down }) |shape| { - stdx.debug.assert(!shape.containsPartitionSpec(.open), "MoE kernel weight partitioning must be closed", .{}); - for (0..shape.rank()) |axis| { - stdx.debug.assert(axis == shape.axis(.expert) or shape.partition(axis) == .replicated, "MoE kernel weights can only be partitioned along experts", .{}); - } - } - const expert_axis = experts.meshAxis(); - const parallel = if (expert_axis) |axis| mesh.grid.axis_sizes[axis] > 1 else false; - if (backend == .metal) stdx.debug.assert(!parallel, "This MoE backend does not support expert parallelism", .{}); - - const Call = struct { - input: zml.Tensor, - ids: zml.Tensor, - weights: zml.Tensor, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - backend: Backend, - opts: Options, - mesh: *const zml.Mesh, - expert_axis: ?u3, - global_experts: u32, - - fn forward(args: @This(), _: zml.Shape) zml.Tensor { - const routing: LocalRouting = .{ - .ids = args.ids, - .weights = args.weights, - .global_num_experts = args.global_experts, - .local_num_experts = @intCast(args.gate_up.weight.dim(.expert)), - .expert_start = if (args.expert_axis) |axis| zml.ops.axisIndex(args.mesh, args.mesh.logical.axes.get(axis)).convert(.i32).scale(args.gate_up.weight.dim(.expert)) else null, - }; - const output = switch (args.backend) { - .stablehlo => stablehlo.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .cute => cute.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .flashinfer_cutlass => flashinfer_cutlass.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .triton => triton.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .fly => fly.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .mosaic_tpu => mosaic_tpu.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .metal => metal.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - .tcl => tcl.fusedExperts(args.input, routing, args.gate_up, args.down, args.opts), - }; - return if (args.expert_axis) |axis| zml.ops.allReduceOnAxis(output, zml.Tensor.add, args.mesh, args.mesh.logical.axes.get(axis)) else output; - } - }; - const args: Call = .{ - .input = input, - .ids = topk_ids, - .weights = topk_weights, - .gate_up = gate_up, - .down = down, - .backend = backend, - .opts = opts, - .mesh = mesh, - .expert_axis = if (parallel) expert_axis else null, - .global_experts = @intCast(gate_up.weight.dim(.expert)), - }; - const Logical = zml.Sharding.Partitioning.Logical; - var projection_specs: [2]zml.ops.PartitionSpecs(zml.nn.Linear) = undefined; - for ([_]zml.nn.Linear{ gate_up, down }, &projection_specs) |linear, *specs| { - zml.meta.mapAlloc(struct { - fn spec(ctx: struct { experts: i64, axis: ?zml.Shape.Tag }, tensor: zml.Tensor) Logical { - const axis = ctx.axis orelse return .replicated; - if (tensor.count() == 1) return .replicated; - // Packed scales may be untagged; experts are their leading dimension. - const dim = tensor.shape().hasTag(.expert) orelse 0; - stdx.debug.assert(tensor.dim(dim) == ctx.experts, "MoE tensor must contain {} experts, got {f}", .{ ctx.experts, tensor.shape() }); - return .{ .mapping = .{ - .src = .init(&.{if (tensor.shape().hasTag(.expert) != null) zml.Shape.toTag(.expert) else zml.Shape.toTag(._0)}), - .tgt = .init(&.{axis}), - } }; - } - }.spec, zml.Compiler.current().arena.allocator(), .{ - .experts = linear.weight.dim(.expert), - .axis = if (expert_axis) |axis| mesh.logical.axes.get(axis) else null, - }, linear, specs) catch @panic("OOM"); - } - return zml.ops.manualComputation(Call.forward, args, output_shape, mesh, .{ - .in_specs = .{ - .input = partitioning.in_specs.input, - .ids = partitioning.in_specs.ids, - .weights = partitioning.in_specs.weights, - .gate_up = projection_specs[0], - .down = projection_specs[1], - }, - .out_specs = partitioning.out_specs, - }); -} diff --git a/zml/kernels/moe/mosaic_tpu/BUILD.bazel b/zml/kernels/moe/mosaic_tpu/BUILD.bazel deleted file mode 100644 index ed4dfcd9..00000000 --- a/zml/kernels/moe/mosaic_tpu/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "mosaic_tpu", - srcs = [ - "gmm_ep.zig", - "mosaic_tpu.zig", - ], -) diff --git a/zml/kernels/moe/mosaic_tpu/gmm_ep.zig b/zml/kernels/moe/mosaic_tpu/gmm_ep.zig deleted file mode 100644 index c30d805b..00000000 --- a/zml/kernels/moe/mosaic_tpu/gmm_ep.zig +++ /dev/null @@ -1,573 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../../zml.zig"); -const mtt = @import("dsl/mosaic_tpu/builder"); -const pipeline = mtt.pipeline; -const mtt_kernel = zml.kernel.mosaic_tpu; - -fn cdiv(x: i64, y: i64) i64 { - return @divFloor(x + y - 1, y); -} - -fn alignTo(x: i64, y: i64) i64 { - return cdiv(x, y) * y; -} - -fn floorModI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { - const c0 = b.constant(.i32, 0); - const r = b.remsi(x, y); - const r_nz = b.cmpi(.ne, r, c0); - const r_neg = b.cmpi(.slt, r, c0); - return b.select(b.andi(r_neg, r_nz), b.addi(r, y), r); -} - -fn floorDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { - const c0 = b.constant(.i32, 0); - const c1 = b.constant(.i32, 1); - const q = b.divsi(x, y); - const sign = b.subi( - b.extui(b.cmpi(.sgt, x, c0), .i32), - b.extui(b.cmpi(.slt, x, c0), .i32), - ); - const sign_not_pos = b.cmpi(.ne, sign, c1); - const rem_nz = b.cmpi(.ne, b.remsi(x, y), c0); - return b.select(b.andi(sign_not_pos, rem_nz), b.subi(q, c1), q); -} - -fn ceilDivPosI32(b: *mtt.Builder, x: mtt.Value, y: mtt.Value) mtt.Value { - return b.divsi(b.addi(x, b.subi(y, b.constant(.i32, 1))), y); -} - -pub const Cfg = struct { - pub const FuseAct = enum { - none, - silu, - gelu, - }; - - size_m: i64, - size_k: i64, - size_n: i64, - size_group: i64, - size_lhs_group: i64, - tile_m: i64, - tile_k: i64, - tile_n: i64, - lhs_dtype: mtt.DType = .bf16, - rhs_dtype: mtt.DType = .bf16, - out_dtype: mtt.DType = .bf16, - acc_dtype: mtt.DType = .f32, - size_lhs_sublane: i64 = 8, - fuse_act: FuseAct = .none, - - pub fn outSizeN(self: Cfg) i64 { - return switch (self.fuse_act) { - .none => self.size_n, - .silu, .gelu => @divExact(self.size_n, 2), - }; - } - - pub fn alignedN(self: Cfg) i64 { - return switch (self.fuse_act) { - .none => alignTo(self.outSizeN(), 128), - .silu, .gelu => self.outSizeN(), - }; - } - - pub fn rhsTileN(self: Cfg) i64 { - return switch (self.fuse_act) { - .none => self.tile_n, - .silu, .gelu => self.tile_n * 2, - }; - } - - pub fn maxNumGm(self: Cfg) i64 { - return self.size_group + cdiv(self.size_m, self.tile_m) - 1; - } -}; - -fn validateConfig(cfg: Cfg) void { - std.debug.assert(cfg.size_group <= cfg.size_lhs_group); - std.debug.assert(cfg.size_m > 0); - std.debug.assert(cfg.size_k > 0); - std.debug.assert(cfg.size_n > 0); - std.debug.assert(cfg.tile_m > 0); - std.debug.assert(cfg.tile_k > 0); - std.debug.assert(cfg.tile_n > 0); - std.debug.assert(cfg.size_lhs_sublane > 0); - std.debug.assert(@mod(cfg.size_m, cfg.size_lhs_sublane) == 0); - std.debug.assert(cdiv(cfg.size_k, cfg.tile_k) == 1); - std.debug.assert(@mod(cfg.tile_k, 128) == 0); - std.debug.assert(@mod(cfg.rhsTileN(), 128) == 0); - if (cfg.fuse_act != .none) { - std.debug.assert(@mod(cfg.size_n, 2) == 0); - std.debug.assert(cdiv(cfg.alignedN(), cfg.tile_n) == 1); - } else { - std.debug.assert(@mod(cfg.alignedN(), cfg.tile_n) == 0); - } - std.debug.assert(cfg.acc_dtype == .f32); -} - -const Meta = struct { - group_id: mtt.Value, - m_offset: mtt.Value, -}; - -const PipeCtx = struct { - cfg: Cfg, - meta: Meta, - num_gm: mtt.Value, -}; - -fn winRef(b: *mtt.Builder, rw: pipeline.RefWindow, shape: []const i64) mtt.Value { - const buf = rw.buf.?; - const slot = rw.slot orelse return buf; - const a = b.arena.allocator(); - const rank = shape.len + 1; - const base = a.alloc(mtt.Value, rank) catch @panic("gmm_ep winRef OOM"); - const result_shape = a.alloc(i64, rank) catch @panic("gmm_ep winRef OOM"); - base[0] = slot; - result_shape[0] = 1; - for (shape, 0..) |d, i| { - base[i + 1] = b.constant(.i32, 0); - result_shape[i + 1] = d; - } - const sliced = b.memRefSlice(buf, base, result_shape, &.{}); - return b.memRefSqueeze(sliced, shape); -} - -fn metaLoad(b: *mtt.Builder, ref: mtt.Value, idx: mtt.Value) mtt.Value { - return b.scalarLoad(ref, &.{b.toIndex(idx)}); -} - -fn lhsIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { - const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); - const cfg = ctx.cfg; - const gm_id = indices[1]; - const k_id = indices[2]; - - const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); - const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); - const sublane = b.constant(.i32, cfg.size_lhs_sublane); - const row_start = floorDivPosI32(b, m_start, sublane); - const row_end = ceilDivPosI32(b, m_end, sublane); - const row_size = b.subi(row_end, row_start); - - const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep lhsIndexMap OOM"); - out[0] = .{ .slice = .{ .start = row_start, .size = row_size } }; - out[1] = .{ .scalar = b.constant(.i32, 0) }; - out[2] = .{ .scalar = k_id }; - return out; -} - -fn rhsIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { - const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); - const group_id = metaLoad(b, ctx.meta.group_id, indices[1]); - const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep rhsIndexMap OOM"); - out[0] = .{ .scalar = group_id }; - out[1] = .{ .scalar = indices[2] }; - out[2] = .{ .scalar = indices[0] }; - return out; -} - -fn outIndexMap(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { - const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); - const cfg = ctx.cfg; - const gm_id = indices[1]; - const n_id = indices[0]; - - const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); - const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); - const sublane = b.constant(.i32, cfg.size_lhs_sublane); - const row_start = floorDivPosI32(b, m_start, sublane); - const capped_row_end = floorDivPosI32(b, m_end, sublane); - const last_row_end = ceilDivPosI32(b, m_end, sublane); - const row_end = b.select( - b.cmpi(.eq, gm_id, b.subi(ctx.num_gm, b.constant(.i32, 1))), - last_row_end, - capped_row_end, - ); - const row_size = b.subi(row_end, row_start); - - const out = b.arena.allocator().alloc(pipeline.BlockIndex, 3) catch @panic("gmm_ep outIndexMap OOM"); - out[0] = .{ .slice = .{ .start = row_start, .size = row_size } }; - out[1] = .{ .scalar = b.constant(.i32, 0) }; - out[2] = .{ .scalar = n_id }; - return out; -} - -fn indices1DTo3D(b: *mtt.Builder, indices: []const mtt.Value) []const mtt.Value { - const out = b.arena.allocator().alloc(mtt.Value, 3) catch @panic("gmm_ep indices1DTo3D OOM"); - out[0] = b.constant(.i32, 0); - out[1] = indices[0]; - out[2] = b.constant(.i32, 0); - return out; -} - -fn lhsIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { - return lhsIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); -} - -fn rhsIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { - return rhsIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); -} - -fn outIndexMap1D(b: *mtt.Builder, indices: []const mtt.Value, ctx_opaque: ?*anyopaque) []const pipeline.BlockIndex { - return outIndexMap(b, indices1DTo3D(b, indices), ctx_opaque); -} - -fn fillMetadata(b: *mtt.Builder, cfg: Cfg, a: anytype) mtt.Value { - const c0 = b.constant(.i32, 0); - const c1 = b.constant(.i32, 1); - const group_offset = b.scalarLoad(a.group_offset, &.{b.constIndex(0)}); - const max_num_group = b.addi(group_offset, b.constant(.i32, cfg.size_group)); - b.scalarStore(a.metadata_m_offset, c0, &.{b.constIndex(0)}); - - var outer = b.openFor(c0, max_num_group, c1, .{ c0, c0 }); - { - b.traceStart(10, "outer_group_loop"); - const lhs_group_id = outer.iv; - const num_gm = outer.carried[0]; - const start_m_offset = outer.carried[1]; - const group_id = b.subi(lhs_group_id, group_offset); - const group_size = b.scalarLoad(a.group_sizes, &.{b.toIndex(lhs_group_id)}); - const end_m_offset = b.addi(start_m_offset, group_size); - const sublane = b.constant(.i32, cfg.size_lhs_sublane); - const local_offset = floorModI32(b, start_m_offset, sublane); - const aligned_group_size = b.addi(group_size, local_offset); - const tile_m = b.constant(.i32, cfg.tile_m); - var curr_num_gm = b.divsi(b.addi(aligned_group_size, b.subi(tile_m, c1)), tile_m); - const should_process = b.andi(b.cmpi(.sgt, group_size, c0), b.cmpi(.sge, group_id, c0)); - curr_num_gm = b.select(should_process, curr_num_gm, c0); - const next_num_gm = b.addi(num_gm, curr_num_gm); - - var inner = b.openFor(num_gm, next_num_gm, c1, .{start_m_offset}); - { - b.traceStart(10, "inner_tm_loop"); - const tm_id = inner.iv; - const curr_m_offset = inner.carried[0]; - const local = floorModI32(b, curr_m_offset, sublane); - const tm_room = b.subi(tile_m, local); - const remaining = b.subi(end_m_offset, curr_m_offset); - const tm_size = b.minsi(tm_room, remaining); - const next_m_offset = b.addi(curr_m_offset, tm_size); - b.scalarStore(a.metadata_group_id, group_id, &.{b.toIndex(tm_id)}); - b.scalarStore(a.metadata_m_offset, curr_m_offset, &.{b.toIndex(tm_id)}); - b.scalarStore(a.metadata_m_offset, next_m_offset, &.{b.toIndex(b.addi(tm_id, c1))}); - b.traceStop(); - inner.yield(.{next_m_offset}); - } - - b.traceStop(); - outer.yield(.{ next_num_gm, end_m_offset }); - } - return outer.results[0]; -} - -fn matmulByMxuColumns(b: *mtt.Builder, cfg: Cfg, lhs: mtt.Value, rhs: mtt.Value, cols: i64) mtt.Value { - if (cfg.tile_k > cfg.size_k) { - std.debug.assert(cdiv(cfg.size_k, cfg.tile_k) == 1); - std.debug.assert(@mod(cols, 128) == 0); - - const chunks = @divExact(cols, 128); - const parts = b.arena.allocator().alloc(mtt.Value, @intCast(chunks)) catch @panic("gmm_ep matmul parts OOM"); - const tail_k = cfg.tile_k - cfg.size_k; - const lhs_valid = b.vectorExtractStridedSlice(lhs, &.{ 0, 0 }, &.{ cfg.tile_m, cfg.size_k }, &.{ 1, 1 }); - const lhs_tail = b.vectorExtractStridedSlice(lhs, &.{ 0, cfg.size_k }, &.{ cfg.tile_m, tail_k }, &.{ 1, 1 }); - - var rev: i64 = 0; - while (rev < chunks) : (rev += 1) { - const chunk = chunks - 1 - rev; - const start_n = chunk * 128; - const rhs_valid = b.vectorExtractStridedSlice(rhs, &.{ 0, start_n }, &.{ cfg.size_k, 128 }, &.{ 1, 1 }); - const rhs_tail = b.vectorExtractStridedSlice(rhs, &.{ cfg.size_k, start_n }, &.{ tail_k, 128 }, &.{ 1, 1 }); - const zero = b.zeros(&.{ cfg.tile_m, 128 }, cfg.acc_dtype); - const tail_acc = b.matmulOpts(lhs_tail, rhs_tail, zero, .{ - .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), - }); - parts[@intCast(chunk)] = b.matmulOpts(lhs_valid, rhs_valid, tail_acc, .{ - .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), - }); - } - - if (chunks == 1) return parts[0]; - - const max_concat_inputs = 16; - if (chunks <= max_concat_inputs) return b.concatenate(parts, 1); - - const groups = cdiv(chunks, max_concat_inputs); - const group_parts = b.arena.allocator().alloc(mtt.Value, @intCast(groups)) catch @panic("gmm_ep matmul group parts OOM"); - var group: i64 = 0; - while (group < groups) : (group += 1) { - const start = group * max_concat_inputs; - const end = @min(start + max_concat_inputs, chunks); - const start_usize: usize = @intCast(start); - const end_usize: usize = @intCast(end); - const group_slice = parts[start_usize..end_usize]; - group_parts[@intCast(group)] = if (group_slice.len == 1) group_slice[0] else b.concatenate(group_slice, 1); - } - return b.concatenate(group_parts, 1); - } - - if (cols <= 128) { - return b.matmulOpts(lhs, rhs, b.zeros(&.{ cfg.tile_m, cols }, cfg.acc_dtype), .{ - .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), - }); - } - - std.debug.assert(@mod(cols, 128) == 0); - const chunks = @divExact(cols, 128); - const parts = b.arena.allocator().alloc(mtt.Value, @intCast(chunks)) catch @panic("gmm_ep matmul parts OOM"); - var rev: i64 = 0; - while (rev < chunks) : (rev += 1) { - const chunk = chunks - 1 - rev; - const start_n = chunk * 128; - const rhs_chunk = b.vectorExtractStridedSlice(rhs, &.{ 0, start_n }, &.{ cfg.tile_k, 128 }, &.{ 1, 1 }); - const zero = b.zeros(&.{ cfg.tile_m, 128 }, cfg.acc_dtype); - parts[@intCast(chunk)] = b.matmulOpts(lhs, rhs_chunk, zero, .{ - .dimension_numbers = b.dotDimensionNumbers(&.{1}, &.{0}, &.{0}, &.{1}, &.{ 0, 0, 1, 1 }, &.{}, &.{}), - }); - } - - if (chunks == 1) return parts[0]; - - const max_concat_inputs = 16; - if (chunks <= max_concat_inputs) return b.concatenate(parts, 1); - - const groups = cdiv(chunks, max_concat_inputs); - const group_parts = b.arena.allocator().alloc(mtt.Value, @intCast(groups)) catch @panic("gmm_ep matmul group parts OOM"); - var group: i64 = 0; - while (group < groups) : (group += 1) { - const start = group * max_concat_inputs; - const end = @min(start + max_concat_inputs, chunks); - const start_usize: usize = @intCast(start); - const end_usize: usize = @intCast(end); - const group_slice = parts[start_usize..end_usize]; - group_parts[@intCast(group)] = if (group_slice.len == 1) group_slice[0] else b.concatenate(group_slice, 1); - } - return b.concatenate(group_parts, 1); -} - -fn geluApprox(b: *mtt.Builder, x: mtt.Value, shape: []const i64) mtt.Value { - const x2 = b.mulf(x, x); - const x3 = b.mulf(x2, x); - const cubic = b.mulf(x3, b.splat(@as(f32, 0.044715), shape, cfgAccDtype())); - const inner = b.mulf(b.addf(x, cubic), b.splat(@as(f32, 0.7978845608028654), shape, cfgAccDtype())); - return b.mulf( - b.mulf(x, b.splat(@as(f32, 0.5), shape, cfgAccDtype())), - b.addf(b.tanh(inner), b.splat(@as(f32, 1.0), shape, cfgAccDtype())), - ); -} - -fn cfgAccDtype() mtt.DType { - return .f32; -} - -fn silu(b: *mtt.Builder, x: mtt.Value, shape: []const i64) mtt.Value { - const one = b.splat(@as(f32, 1.0), shape, cfgAccDtype()); - const denom = b.addf(one, b.exp(b.negf(x))); - return b.divf(x, denom); -} - -fn applyFuseAct(b: *mtt.Builder, cfg: Cfg, acc: mtt.Value) mtt.Value { - return switch (cfg.fuse_act) { - .none => acc, - .silu, .gelu => { - const shape = &.{ cfg.tile_m, cfg.tile_n }; - const gate = b.vectorExtractStridedSlice(acc, &.{ 0, 0 }, shape, &.{ 1, 1 }); - const up = b.vectorExtractStridedSlice(acc, &.{ 0, cfg.outSizeN() }, shape, &.{ 1, 1 }); - const activated = switch (cfg.fuse_act) { - .silu => silu(b, gate, shape), - .gelu => geluApprox(b, gate, shape), - .none => unreachable, - }; - return b.mulf(activated, up); - }, - }; -} - -fn innerBody( - b: *mtt.Builder, - indices: []const mtt.Value, - refs: []const pipeline.RefWindow, - scratches: []const mtt.Value, - ctx_opaque: ?*anyopaque, -) mtt.FinishError!void { - const ctx: *PipeCtx = @ptrCast(@alignCast(ctx_opaque.?)); - const cfg = ctx.cfg; - const gm_id = indices[1]; - - const lhs_shape = &.{ cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_k }; - const out_shape = &.{ cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_n }; - const rhs_tile_n = cfg.rhsTileN(); - - b.traceStart(10, "matmul_first_last"); - const lhs_ref = winRef(b, refs[0], lhs_shape); - const lhs_ref_2d = b.memRefReshape(lhs_ref, &.{ cfg.tile_m, cfg.tile_k }); - const lhs = b.refLoad(lhs_ref_2d); - const rhs_slot = b.toIndex(refs[1].slot.?); - var rhs = b.shapeCast( - b.vectorLoadShape(refs[1].buf.?, &.{ rhs_slot, b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.tile_k, rhs_tile_n }), - &.{ cfg.tile_k, rhs_tile_n }, - ); - const valid_k = @mod(cfg.size_k, cfg.tile_k); - if (valid_k != 0) { - const k_iota = b.iota(&.{ cfg.tile_k, rhs_tile_n }, .i32, &.{0}); - const valid = b.splat(@as(i32, @intCast(valid_k)), &.{ cfg.tile_k, rhs_tile_n }, .i32); - rhs = b.select(b.cmpi(.slt, k_iota, valid), rhs, b.zeros(&.{ cfg.tile_k, rhs_tile_n }, cfg.rhs_dtype)); - } - var acc = matmulByMxuColumns(b, cfg, lhs, rhs, rhs_tile_n); - acc = applyFuseAct(b, cfg, acc); - - const m_start = metaLoad(b, ctx.meta.m_offset, gm_id); - const m_end = metaLoad(b, ctx.meta.m_offset, b.addi(gm_id, b.constant(.i32, 1))); - const sublane = b.constant(.i32, cfg.size_lhs_sublane); - const m_offset = b.subi(m_start, floorModI32(b, m_start, sublane)); - const m_start_local = b.subi(m_start, m_offset); - const m_end_local = b.subi(m_end, m_offset); - const row_iota = b.iota(&.{ cfg.tile_m, cfg.tile_n }, .i32, &.{0}); - const row_mask = b.andi( - b.cmpi(.sle, b.broadcastTo(m_start_local, &.{ cfg.tile_m, cfg.tile_n }), row_iota), - b.cmpi(.slt, row_iota, b.broadcastTo(m_end_local, &.{ cfg.tile_m, cfg.tile_n })), - ); - acc = b.select(row_mask, acc, b.zeros(&.{ cfg.tile_m, cfg.tile_n }, cfg.acc_dtype)); - - const out_slot_i32 = refs[2].slot.?; - const out_slot = b.toIndex(out_slot_i32); - b.vectorStoreAt( - refs[2].buf.?, - b.shapeCast(b.shapeCast(acc, out_shape).to(cfg.out_dtype), &.{ 1, cdiv(cfg.tile_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.tile_n }), - &.{ out_slot, b.constIndex(0), b.constIndex(0), b.constIndex(0) }, - ); - - const out_ref = winRef(b, refs[2], out_shape); - if (cfg.tile_m == cfg.size_lhs_sublane) { - const out_value = b.shapeCast(b.refLoad(out_ref), &.{ cfg.tile_m, cfg.tile_n }); - const partial_zero = b.zeros(&.{ cfg.tile_m, cfg.tile_n }, cfg.out_dtype); - const partial_prev = b.select( - b.cmpi(.eq, gm_id, b.constant(.i32, 0)), - partial_zero, - b.refLoad(scratches[0]), - ); - const out_accum = b.addf(out_value, partial_prev); - const out_ref_for_store = winRef(b, refs[2], out_shape); - b.refStore(out_ref_for_store, b.shapeCast(out_accum, out_shape)); - } else { - const first_row = b.shapeCast( - b.vectorLoadShape(out_ref, &.{ b.constIndex(0), b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), - &.{ cfg.size_lhs_sublane, cfg.tile_n }, - ); - const partial_zero = b.zeros(&.{ cfg.size_lhs_sublane, cfg.tile_n }, cfg.out_dtype); - const partial_prev = b.select( - b.cmpi(.eq, gm_id, b.constant(.i32, 0)), - partial_zero, - b.refLoad(scratches[0]), - ); - const out_accum = b.addf(first_row, partial_prev); - const out_ref_for_store = winRef(b, refs[2], out_shape); - b.vectorStoreAt( - out_ref_for_store, - b.shapeCast(out_accum, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), - &.{ b.constIndex(0), b.constIndex(0), b.constIndex(0) }, - ); - } - - const last_row = floorDivPosI32(b, m_end_local, sublane); - const m_end_mod = floorModI32(b, m_end_local, sublane); - const partial_complete = b.cmpi(.eq, m_end_mod, b.constant(.i32, 0)); - const out_ref_for_partial = winRef(b, refs[2], out_shape); - const partial_shape = if (cfg.tile_m == cfg.size_lhs_sublane) - &.{ cfg.tile_m, cfg.tile_n } - else - &.{ cfg.size_lhs_sublane, cfg.tile_n }; - const partial_zero = b.zeros(partial_shape, cfg.out_dtype); - const partial_row = b.shapeCast( - b.vectorLoadShape(out_ref_for_partial, &.{ b.toIndex(last_row), b.constIndex(0), b.constIndex(0) }, &.{ 1, cfg.size_lhs_sublane, cfg.tile_n }), - partial_shape, - ); - b.refStore(scratches[0], b.select(partial_complete, partial_zero, partial_row)); - b.traceStop(); -} - -fn innerBody1D( - b: *mtt.Builder, - indices: []const mtt.Value, - refs: []const pipeline.RefWindow, - scratches: []const mtt.Value, - ctx_opaque: ?*anyopaque, -) mtt.FinishError!void { - try innerBody(b, indices1DTo3D(b, indices), refs, scratches, ctx_opaque); -} - -pub fn run(b: *mtt.Builder, cfg: Cfg) mtt.FinishError!void { - validateConfig(cfg); - - const a = try b.declareArgsOpts(.{ - .group_sizes = .{ .ref = .{ .shape = &.{cfg.size_lhs_group}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, - .group_offset = .{ .ref = .{ .shape = &.{1}, .dtype = .i32, .memory_space = .smem, .role = .scalar_prefetch } }, - .lhs = .{ .ref = .{ .shape = &.{ cfg.size_m, cfg.size_k }, .dtype = cfg.lhs_dtype, .memory_space = .hbm } }, - .rhs = .{ .ref = .{ .shape = &.{ cfg.size_group, cfg.size_k, cfg.size_n }, .dtype = cfg.rhs_dtype, .memory_space = .hbm } }, - .out = .{ .ref = .{ .shape = &.{ cfg.size_m, cfg.alignedN() }, .dtype = cfg.out_dtype, .memory_space = .hbm, .role = .output } }, - .partial_out = .{ .ref = .{ .shape = &.{ cfg.size_lhs_sublane, cfg.tile_n }, .dtype = cfg.out_dtype, .role = .scratch } }, - .acc = .{ .ref = .{ .shape = &.{ cfg.tile_m, cfg.tile_n }, .dtype = cfg.acc_dtype, .role = .scratch } }, - .metadata_group_id = .{ .ref = .{ .shape = &.{cfg.maxNumGm()}, .dtype = .i32, .memory_space = .smem, .role = .scratch } }, - .metadata_m_offset = .{ .ref = .{ .shape = &.{cfg.maxNumGm() + 1}, .dtype = .i32, .memory_space = .smem, .role = .scratch } }, - }, &.{}, .{ - .dimension_semantics = &.{}, - .scalar_prefetch = 2, - .scratch_operands = 4, - }); - - const num_k = b.constant(.i32, cdiv(cfg.size_k, cfg.tile_k)); - const num_n = b.constant(.i32, cdiv(cfg.alignedN(), cfg.tile_n)); - const num_gm = fillMetadata(b, cfg, a); - - const ctx = b.arena.allocator().create(PipeCtx) catch @panic("gmm_ep PipeCtx OOM"); - ctx.* = .{ - .cfg = cfg, - .meta = .{ .group_id = a.metadata_group_id, .m_offset = a.metadata_m_offset }, - .num_gm = num_gm, - }; - - const lhs_spec = pipeline.PipeSpec{ - .full_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.size_k }, - .dtype = cfg.lhs_dtype, - .source_reshape_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.size_k }, - .block_shape = &.{ .{ .bounded_slice = cdiv(cfg.tile_m, cfg.size_lhs_sublane) }, .{ .blocked = cfg.size_lhs_sublane }, .{ .blocked = cfg.tile_k } }, - .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) lhsIndexMap1D else lhsIndexMap, - .index_map_ctx = ctx, - .bounded_slice_tiling = 1, - }; - const rhs_spec = pipeline.PipeSpec{ - .full_shape = &.{ cfg.size_group, cfg.size_k, cfg.size_n }, - .dtype = cfg.rhs_dtype, - .block_shape = &.{ .squeezed, .{ .blocked = cfg.tile_k }, .{ .blocked = cfg.rhsTileN() } }, - .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) rhsIndexMap1D else rhsIndexMap, - .index_map_ctx = ctx, - .buffer_count = 3, - }; - const out_spec = pipeline.PipeSpec{ - .full_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.alignedN() }, - .dtype = cfg.out_dtype, - .source_reshape_shape = &.{ @divExact(cfg.size_m, cfg.size_lhs_sublane), cfg.size_lhs_sublane, cfg.alignedN() }, - .block_shape = &.{ .{ .bounded_slice = cdiv(cfg.tile_m, cfg.size_lhs_sublane) }, .{ .blocked = cfg.size_lhs_sublane }, .{ .blocked = cfg.tile_n } }, - .index_map = if (cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m) outIndexMap1D else outIndexMap, - .index_map_ctx = ctx, - .bounded_slice_tiling = 1, - }; - const use_1d_pipeline = cdiv(cfg.alignedN(), cfg.tile_n) == 1 and cdiv(cfg.size_k, cfg.tile_k) == 1 and cfg.size_m > cfg.tile_m; - try pipeline.emitPipeline(b, &.{ a.lhs, a.rhs, a.out }, &.{ a.partial_out, a.acc, a.metadata_group_id, a.metadata_m_offset }, .{ - .body = if (use_1d_pipeline) innerBody1D else innerBody, - .body_ctx = ctx, - .grid = if (use_1d_pipeline) &.{num_gm} else &.{ num_n, num_gm, num_k }, - .in_specs = &.{ lhs_spec, rhs_spec }, - .out_specs = &.{out_spec}, - }); -} - -pub const TpuKernel = mtt_kernel.Kernel(Cfg, .{ - .name = "gmm_ep_kernel", - .inputs = &.{ "group_sizes", "group_offset", "lhs", "rhs" }, - .outputs = &.{"out"}, - .run = run, -}); diff --git a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig deleted file mode 100644 index bb60594b..00000000 --- a/zml/kernels/moe/mosaic_tpu/mosaic_tpu.zig +++ /dev/null @@ -1,301 +0,0 @@ -const std = @import("std"); - -const mtt = @import("dsl/mosaic_tpu/builder"); -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const Tensor = zml.Tensor; -const Activation = @import("../api.zig").Activation; -const GlobalOptions = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; -pub const gmm_ep = @import("gmm_ep.zig"); - -const log = std.log.scoped(.@"zml/moe/mosaic_tpu"); - -var gate_up_transpose: std.atomic.Value(bool) = .init(false); -var down_transpose: std.atomic.Value(bool) = .init(false); - -fn validateOptions(opts: GlobalOptions) void { - stdx.debug.assert(!opts.quantize_input, "Optional FP8 input quantization requires the Triton MoE backend", .{}); - stdx.debug.assert(opts.routing_weight_placement == .after_down, "Non-Triton MoE backends require routing weights after the down projection", .{}); -} - -fn validateInputs(hidden: Tensor, gate_up: Tensor, down: Tensor, weights: Tensor, ids: Tensor) !void { - switch (hidden.dtype()) { - .bf16, .f16, .f32 => {}, - else => return error.UnsupportedType, - } - switch (gate_up.dtype()) { - .bf16, .f16, .f32 => {}, - else => return error.UnsupportedType, - } - switch (down.dtype()) { - .bf16, .f16, .f32 => {}, - else => return error.UnsupportedType, - } - switch (weights.dtype()) { - .bf16, .f16, .f32 => {}, - else => return error.UnsupportedType, - } - if (ids.dtype() != .i32) return error.UnsupportedType; - if (hidden.dim(.in) != gate_up.dim(.in)) return error.InvalidShape; - if (@rem(gate_up.dim(.out), 2) != 0) return error.InvalidShape; - if (down.dim(.mid) != @divFloor(gate_up.dim(.out), 2)) return error.InvalidShape; - if (ids.dim(.token) != hidden.dim(.token) or weights.dim(.token) != hidden.dim(.token)) return error.InvalidShape; - if (ids.dim(.topk) != weights.dim(.topk)) return error.InvalidShape; - if (gate_up.dim(.expert) != down.dim(.expert)) return error.InvalidShape; -} - -fn warningTranspose(flag: *std.atomic.Value(bool), projection_name: []const u8, expected_shape: []const u8) void { - if (!flag.swap(true, .monotonic)) { - log.warn("{s} MoE weights are transposed at runtime; weights must be loaded with the correct shape {s} to avoid the transpose at runtime.", .{ projection_name, expected_shape }); - } -} - -fn canonicalizeGateUp(w1: Tensor, hidden_size: i64) !Tensor { - if (w1.rank() != 3) return error.InvalidShape; - - if (w1.dim(1) == hidden_size) return w1.withTags(.{ .expert, .in, .out }); - if (w1.dim(2) == hidden_size) { - warningTranspose(&gate_up_transpose, "gate_up_proj", "[expert, in, out]"); - return w1.withTags(.{ .expert, .out, .in }).transpose(.{ .expert, .in, .out }); - } - return error.InvalidShape; -} - -fn canonicalizeDown(w2: Tensor, hidden_size: i64, gate_up_out: i64) !Tensor { - if (w2.rank() != 3) return error.InvalidShape; - - const mid_size = @divFloor(gate_up_out, 2); - if (w2.dim(1) == mid_size and w2.dim(2) == hidden_size) return w2.withTags(.{ .expert, .mid, .out }); - if (w2.dim(1) == hidden_size and w2.dim(2) == mid_size) { - warningTranspose(&down_transpose, "down_proj", "[expert, mid, out]"); - return w2.withTags(.{ .expert, .out, .mid }).transpose(.{ .expert, .mid, .out }); - } - return error.InvalidShape; -} - -fn histogramCounts(ids: Tensor, num_bins: i64) Tensor { - const zeros = Tensor.zeroes(.init(.{ .expert = num_bins }, .i32)); - const ones = Tensor.constant(.{ .i32 = 1 }).broad(ids.shape().withDtype(.i32)); - return zeros.scatterSlices( - .{ .expert = ids }, - ones, - .{ - .update_fn = Tensor.ScatterOpts.increment, - .indices_are_unique = false, - }, - ); -} - -fn alignSortedRowsByGroup(rows: Tensor, expert_ids_sorted: Tensor, group_sizes: Tensor, tile_m: i64) struct { - rows: Tensor, - group_sizes: Tensor, - positions: Tensor, -} { - const num_groups = group_sizes.dim(.expert); - const max_padded_rows = rows.dim(.token) + num_groups * (tile_m - 1); - - const group_ends = group_sizes.cumulativeSum(.expert); - const group_starts = Tensor.concatenate(&.{ - Tensor.zeroes(.init(.{ .expert = 1 }, .i32)), - group_ends.slice(.expert, .{ .end = num_groups - 1 }), - }, .expert); - - const padded_group_sizes = group_sizes.addConstant(tile_m - 1).divByConst(tile_m).mul(Tensor.scalar(tile_m, .i32)); - const padded_group_ends = padded_group_sizes.cumulativeSum(.expert); - const padded_group_starts = Tensor.concatenate(&.{ - Tensor.zeroes(.init(.{ .expert = 1 }, .i32)), - padded_group_ends.slice(.expert, .{ .end = num_groups - 1 }), - }, .expert); - - const logical_positions = Tensor.arange(.{ .end = rows.dim(.token) }, .i32).withTags(.{.token}); - const group_start_per_token = group_starts.gather(.{ .expert = expert_ids_sorted }, .{}); - const padded_group_start_per_token = padded_group_starts.gather(.{ .expert = expert_ids_sorted }, .{}); - const padded_positions = logical_positions.sub(group_start_per_token).add(padded_group_start_per_token).withTags(.{.token}); - - const padded = Tensor.zeroes(rows.shape().setDim(.token, max_padded_rows)) - .scatterSlices( - .{ .token = padded_positions }, - rows, - .{ .indices_are_unique = true, .update_fn = Tensor.ScatterOpts.override }, - ); - - return .{ - .rows = padded, - .group_sizes = padded_group_sizes, - .positions = padded_positions, - }; -} - -fn permuteSmallBatch(hidden: Tensor, sorted_route: Tensor, topk: i64) Tensor { - const num_tokens = hidden.dim(.token); - const num_routes = num_tokens * topk; - - const token_indices = Tensor.arange(.{ .end = num_tokens }, .i32) - .withTags(.{.token}) - .repeat1d(.token, @intCast(topk)) - .rename(.{ .token = .route }); - const token_indices_sorted = token_indices.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .route }); - - const onehot_shape = zml.Shape.init(.{ .route = num_routes, .token_src = num_tokens }, .i32); - const token_indices_2d = token_indices_sorted.insertAxes(.last, .{.token_src}).broad(onehot_shape); - const source_tokens_2d = Tensor.arange(.{ .end = num_tokens }, .i32) - .withTags(.{.token_src}) - .insertAxes(0, .{.route}) - .broad(onehot_shape); - const onehot = token_indices_2d.cmp(.EQ, source_tokens_2d).convert(hidden.dtype()); - - return onehot.dot(hidden.rename(.{ .token = .token_src }), .token_src).rename(.{ .route = .token }); -} - -fn gmmDType(dtype: zml.DataType) mtt.DType { - return switch (zml.kernel.mosaic_tpu.from(dtype)) { - .bf16 => .bf16, - .f16 => .f16, - .f32 => .f32, - else => stdx.debug.panic("mosaic_tpu MoE expects bf16/f16/f32 tensors, got {}", .{dtype}), - }; -} - -pub fn callGmmEp( - lhs: Tensor, - rhs_k_n: Tensor, - group_sizes: Tensor, - out_dtype: zml.DataType, - fuse_act: gmm_ep.Cfg.FuseAct, -) Tensor { - const k = lhs.dim(1); - const n = rhs_k_n.dim(.out); - const out_n = switch (fuse_act) { - .none => n, - .silu, .gelu => @divExact(n, 2), - }; - const aligned_k = @divFloor(k + 127, 128) * 128; - const aligned_n = if (fuse_act == .none) @divFloor(out_n + 127, 128) * 128 else out_n; - - const cfg: gmm_ep.Cfg = .{ - .size_m = lhs.dim(.token), - .size_k = k, - .size_n = n, - .size_group = rhs_k_n.dim(.expert), - .size_lhs_group = group_sizes.dim(.expert), - .tile_m = 128, - .tile_k = aligned_k, - .tile_n = if (fuse_act == .none) aligned_n else out_n, - .lhs_dtype = gmmDType(lhs.dtype()), - .rhs_dtype = gmmDType(rhs_k_n.dtype()), - .out_dtype = gmmDType(out_dtype), - .acc_dtype = .f32, - .size_lhs_sublane = 8, - .fuse_act = fuse_act, - }; - - const out = gmm_ep.TpuKernel.call( - .{ - .group_sizes = group_sizes.withTags(.{.expert}), - .group_offset = Tensor.zeroes(.init(.{1}, .i32)), - .lhs = lhs, - .rhs = rhs_k_n, - }, - .{ - .out = zml.Shape.init(.{ .token = lhs.dim(.token), .out = aligned_n }, out_dtype), - }, - .{ - .cfg = cfg, - .extras = .{ - // todo leverage tpu api like this https://github.com/openxla/tokamax/blob/f65135e85acd7571531be91d4d3241e1929421f0/tokamax/_src/ops/ragged_dot/pallas_mosaic_tpu_v2_tgmm_kernel.py#L692 - .vmem_limit_bytes = 32 * 1024 * 1024, - }, - }, - ).out; - - return out.slice(.out, .{ .end = out_n }); -} - -pub fn fusedExperts( - hidden_states: zml.Tensor, - routing: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - opts: GlobalOptions, -) zml.Tensor { - validateOptions(opts); - const expert_map_ = routing.expertMap(); - - const tokens = hidden_states.dim(.t); - const hidden = hidden_states.reshape(.{ .token = tokens, .in = hidden_states.dim(.d) }); - // TODO(Corentin): Better error message - const gate_up = canonicalizeGateUp(gate_up_.weight, hidden.dim(.in)) catch |e| stdx.debug.panic("Invalid gate_up weights for fusedExperts: {}", .{e}); - const down = canonicalizeDown(down_.weight, hidden.dim(.in), gate_up.dim(.out)) catch |e| stdx.debug.panic("Invalid down weights for fusedExperts: {}", .{e}); - const weights = routing.weights.reshape(.{ .token = tokens, .topk = routing.weights.dim(.topk) }); - const ids = routing.ids.reshape(.{ .token = tokens, .topk = routing.ids.dim(.topk) }); - - const num_experts = routing.global_num_experts; - if (expert_map_) |expert_map| { - if (expert_map.dtype() != .i32) stdx.debug.panic("expert_map must be of dtype i32, got {}", .{expert_map.dtype()}); - if (expert_map.rank() != 1 or expert_map.dim(.expert) != num_experts) stdx.debug.panic("expert_map must be a 1D tensor of length num_experts, got shape {}", .{expert_map.shape()}); - } - - const flat_ids_global = ids.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); - const flat_weights = weights.transpose(.{ .topk, .token }).flatten().withTags(.{.route}); - - const flat_ids_local, const local_route_mask = if (expert_map_) |expert_map| blk: { - const mapped = expert_map.gather(.{ .expert = flat_ids_global }, .{}).withTags(.{.route}); - const route_mask = mapped.cmp(.GE, Tensor.scalar(0, .i32).broad(mapped.shape())); - const safe_ids = route_mask.select(mapped, Tensor.zeroes(mapped.shape())); - break :blk .{ safe_ids, route_mask }; - } else .{ - flat_ids_global, - Tensor.scalar(true, .bool).broad(flat_ids_global.shape()), - }; - - const sorted_route = flat_ids_local.argsort(.route, .{ .descending = false }).rename(.{ .route = .perm }); - const hidden_sorted = if (expert_map_ != null) - permuteSmallBatch(hidden, sorted_route, ids.dim(.topk)) - else blk: { - // If not sharded, we can save the sort + gather by just repeating the hidden states for each expert and gathering with the sorted route - const hidden_repeated = hidden.repeat1d(.token, @intCast(ids.dim(.topk))).rename(.{ .token = .route }); - break :blk hidden_repeated.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); - }; - const expert_ids_sorted = flat_ids_local.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); - const local_route_mask_sorted = local_route_mask.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); - const sorted_weights = flat_weights.gather(.{ .route = sorted_route }, .{}).rename(.{ .perm = .token }); - const router_weights_sorted = local_route_mask_sorted.select( - sorted_weights, - Tensor.zeroes(sorted_weights.shape()), - ); - - const group_sizes = histogramCounts(flat_ids_local, gate_up.dim(.expert)); - const tile_m: i64 = 128; - const aligned_hidden = alignSortedRowsByGroup(hidden_sorted, expert_ids_sorted, group_sizes, tile_m); - - var gate_up_out = callGmmEp(aligned_hidden.rows, gate_up, aligned_hidden.group_sizes, hidden.dtype(), .none) - .gather(.{ .token = aligned_hidden.positions.rename(.{ .token = .route }) }, .{}) - .rename(.{ .route = .token }); - - if (gate_up_.bias) |bias| { - const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); - gate_up_out = gate_up_out.add(bias_per_token); - } - - const activated = opts.activation.applyActivation(gate_up_out, .concatenated); - - const aligned_activated = alignSortedRowsByGroup(activated, expert_ids_sorted, group_sizes, tile_m); - var down_out = callGmmEp(aligned_activated.rows, down, aligned_activated.group_sizes, hidden.dtype(), .none) - .gather(.{ .token = aligned_activated.positions.rename(.{ .token = .route }) }, .{}) - .rename(.{ .route = .token }); - - if (down_.bias) |bias| { - const bias_per_token = bias.withTags(.{ .expert, .out }).gather(.{ .expert = expert_ids_sorted }, .{}); - down_out = down_out.add(bias_per_token); - } - - const weighted = down_out.mul(router_weights_sorted.convert(down_out.dtype()).broad(down_out.shape())); - - const restore_indices = sorted_route.argsort(.perm, .{ .descending = false }).rename(.{ .perm = .restore }); - const unsorted = weighted.gather(.{ .token = restore_indices }, .{}).rename(.{ .restore = .route }); - const restored = unsorted.reshape(.{ .topk = ids.dim(.topk), .token = hidden.dim(.token), .out = down.dim(.out) }); - const combined = restored.sum(.topk).squeeze(.topk); - return combined.reshape(.{ .t = tokens, .d = down.dim(.out) }); -} diff --git a/zml/kernels/moe/stablehlo/BUILD.bazel b/zml/kernels/moe/stablehlo/BUILD.bazel deleted file mode 100644 index c31b27f0..00000000 --- a/zml/kernels/moe/stablehlo/BUILD.bazel +++ /dev/null @@ -1,8 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "stablehlo", - srcs = [ - "stablehlo.zig", - ], -) diff --git a/zml/kernels/moe/stablehlo/stablehlo.zig b/zml/kernels/moe/stablehlo/stablehlo.zig deleted file mode 100644 index ba723234..00000000 --- a/zml/kernels/moe/stablehlo/stablehlo.zig +++ /dev/null @@ -1,225 +0,0 @@ -//! Deliberately simple MoE reference: no routed GEMMs, packing, or custom calls. -//! Weights use [expert, output, input] order with concatenated gate/up columns. -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const Tensor = zml.Tensor; -const Shape = zml.Shape; -const GlobalOptions = @import("../api.zig").Options; -const Activation = @import("../api.zig").Activation; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const LocalRouting = @import("../api.zig").LocalRouting; - -pub fn activate(x: Tensor, activation: Activation) Tensor { - return switch (activation) { - .silu => x.silu(), - .gelu => x.gelu(), - .relu => x.relu(), - .swiglu, .swiglu_step, .geglu, .geglu_tanh => blk: { - const mid = @divExact(x.dim(.out), 2); - var gate = x.slice(.out, .{ .end = mid }); - var up = x.slice(.out, .{ .start = mid }); - - switch (activation) { - .swiglu_step => { - gate = gate.div(gate.negate().exp().addConstant(1)); - if (activation.swiglu_step.limit) |limit| { - gate = gate.minimum(Tensor.scalar(limit, x.dtype())); - up = up.clamp(Tensor.scalar(-limit, x.dtype()), Tensor.scalar(limit, x.dtype())); - } - break :blk gate.mul(up); - }, - .swiglu => { - if (activation.swiglu.limit) |limit| { - gate = gate.minimum(Tensor.scalar(limit, x.dtype())); - up = up.clamp(Tensor.scalar(-limit, x.dtype()), Tensor.scalar(limit, x.dtype())); - } - if (activation.swiglu.bias) |bias| up = up.addConstant(bias); - break :blk gate.div(gate.negate().exp().addConstant(1)).mul(up); - }, - .geglu => break :blk gelu(gate).mul(up), - .geglu_tanh => break :blk gate.gelu().mul(up), - else => unreachable, // already handled above - } - }, - }; -} - -/// More precise GELU approximation than the default Zig implementation, which is -/// based on tanh. This is the same approximation used in JAX. -pub fn gelu(x: Tensor) Tensor { - const xf = x.convert(.f32); - const cdf = xf.scale(-1.0 / @sqrt(@as(f64, 2))) - .erfc() - .scale(0.5); - return xf.mul(cdf).convert(x.dtype()); -} - -fn expandScales(scales: Tensor, shape: Shape) Tensor { - if (scales.count() == 1) return scales.asScalar().convert(.f32).broad(shape.withDtype(.f32)); - stdx.debug.assert(scales.rank() == shape.rank(), "reference scale rank must match weights", .{}); - var result = scales.convert(.f32); - for (0..shape.rank()) |axis| { - const factor = std.math.divCeil(i64, shape.dim(axis), result.dim(axis)) catch unreachable; - const expanded = result.shape().insert(axis + 1, .{factor}); - result = result.broadcast(expanded, Shape.range(expanded.rank(), .i64).remove(axis + 1).dims()) - .reshape(result.shape().setDim(axis, result.dim(axis) * factor)) - .slice(axis, .{ .end = shape.dim(axis) }); - } - return result.withTags(shape.tags()); -} - -pub fn dequantize(linear: zml.nn.Linear) Tensor { - var weight = linear.weight; - const q = linear.quantization orelse return weight.convert(.f32); - stdx.debug.assert(!q.swizzled_scales, "StableHLO reference requires plain block scales", .{}); - if (zml.nn.isPackedFp4(q.scheme, weight.dtype())) { - // StableHLO bitcast expands the low nibble first along the input axis. - weight = weight.bitCast(.f4e2m1).merge(.{ .unpacked = .{ linear.tag, .bitcast } }).renameTag(.unpacked, linear.tag); - } - const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; - var result = weight.convert(.f32).mul(expandScales(scales, weight.shape())); - if (q.global_scale) |scale| result = result.mul(scale.asMultiplier().broad(result.shape())); - return result; -} - -// E2M1 round-to-nearest, ties-to-even expressed independently of native -// float-to-FP4 lowering. Even encoding indices win midpoint ties. -fn roundFp4(x: Tensor) Tensor { - const magnitude = x.abs(); - var rounded = Tensor.scalar(6, .f32).broad(x.shape()); - const thresholds = [_]f32{ 5, 3.5, 2.5, 1.75, 1.25, 0.75, 0.25 }; - const values = [_]f32{ 4, 3, 2, 1.5, 1, 0.5, 0 }; - inline for (thresholds, values, 0..) |threshold, value, i| { - rounded = magnitude.cmp(if (i % 2 == 0) .LE else .LT, Tensor.scalar(threshold, .f32)) - .select(Tensor.scalar(value, .f32).broad(x.shape()), rounded); - } - return x.cmp(.LT, Tensor.scalar(0, .f32)).select(rounded.negate(), rounded); -} - -// Quantize and dequantize activation groups using tensor math, independently of -// Triton's quantizer. The return value is the represented value in FP32. -fn roundedInput(x: Tensor, linear: zml.nn.Linear, opts: Options, dtype: zml.DataType) Tensor { - if (opts.input_quantization == .none) return x.convert(dtype).convert(.f32); - const q = linear.quantization orelse return x.convert(dtype).convert(.f32); - const fixed_mxfp8 = opts.input_quantization == .mxfp8 or opts.input_quantization == .mxfp8_bf16; - const scheme = if (fixed_mxfp8) .fp8_block32 else q.scheme; - if (scheme == .nvfp4) { - const global = if (q.input_scale) |scale| scale.asMultiplier() else Tensor.scalar(1, .f32); - stdx.debug.assert(global.count() == 1, "reference NVFP4 input scale must be global", .{}); - // Keep both stored rounding boundaries when XLA fuses the quantizer. - const grouped = x.convert(.bf16).optimizationBarrier().convert(.f32).div(global.asScalar()) - .splitAxis(.in, .{ .group = @divExact(x.dim(.in), 16), .element = 16 }); - const scale = grouped.abs().max(.element).scale(1.0 / 6.0) - .minimum(Tensor.scalar(448, .f32)) - .convert(.f8e4m3fn).optimizationBarrier().convert(.f32).broad(grouped.shape()); - const divisor = scale.cmp(.NE, Tensor.scalar(0, .f32)).select(scale, Tensor.scalar(1, .f32).broad(scale.shape())); - return roundFp4(grouped.mul(Tensor.scalar(1, .f32).div(divisor))).mul(scale) - .reshape(x.shape().withDtype(.f32)).mul(global.asScalar()); - } - if (opts.input_quantization == .automatic and (!opts.quantize_input or scheme == .mxfp4)) return x.convert(dtype).convert(.f32); - const output_dtype = if (fixed_mxfp8) .f8e4m3fn else linear.weight.dtype(); - const group: i64 = switch (scheme) { - .mxfp8, .fp8_block32 => 32, - .fp8_block128 => 128, - .fp8_per_channel, .fp8_per_tensor => x.dim(.in), - else => unreachable, - }; - const source = if (opts.input_quantization == .mxfp8_bf16) x.convert(.bf16) else x; - const grouped = source.convert(.f32).splitAxis(.in, .{ .group = @divExact(x.dim(.in), group), .element = group }); - const epsilon: f32 = if (scheme == .fp8_block32) 1e-4 else 1e-10; - const max_value: f32 = if (output_dtype == .f8e4m3fnuz) 224 else 448; - var scale = grouped.abs().max(.element).maximum(Tensor.scalar(epsilon, .f32)).scale(1 / max_value); - if (scheme == .mxfp8 or scheme == .fp8_block32) scale = scale.log().scale(1 / @log(@as(f32, 2))).ceil().scale(@log(@as(f32, 2))).exp(); - const broad = scale.broad(grouped.shape()); - return grouped.div(broad).clamp(Tensor.scalar(-max_value, .f32), Tensor.scalar(max_value, .f32)) - .convert(output_dtype).optimizationBarrier().convert(.f32).mul(broad).reshape(x.shape().withDtype(.f32)); -} - -fn project(x: Tensor, linear: zml.nn.Linear, ids: Tensor, opts: Options, dtype: zml.DataType) Tensor { - const weight = dequantize(linear).withTags(.{ .expert, .out, .in }); - const selected = weight.gather(.{ .expert = ids }, .{}); - var result = roundedInput(x, linear, opts, dtype).dot(selected, .in); - if (linear.bias) |bias| { - const selected_bias = if (bias.rank() == 2) - bias.withTags(.{ .expert, .out }).gather(.{ .expert = ids }, .{}) - else - bias.withTags(.{.out}); - result = result.add(selected_bias.convert(.f32).broad(result.shape())); - } - return result; -} - -pub const Options = struct { - activation: Activation, - quantize_input: bool = false, - /// Model fixed backend contracts independently of optional input quantization: - /// Metal uses weight-only quantization; specialized MXFP4 GEMMs always use MXFP8 inputs. - /// CuTe additionally rounds the intermediate activation to BF16 before quantizing. - input_quantization: enum { automatic, none, mxfp8, mxfp8_bf16 } = .automatic, - routing_weight_placement: RoutingWeightPlacement = .after_down, -}; - -/// Negative expert IDs denote nonlocal routes and contribute zero, including bias. -pub fn reference(input: Tensor, topk_ids: Tensor, topk_weights: Tensor, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: Options) Tensor { - stdx.debug.assert(topk_ids.dim(.topk) > 0, "reference top-k must be positive", .{}); - stdx.debug.assert(topk_ids.dtype() == .i32, "reference expects i32 expert ids", .{}); - stdx.debug.assert(gate_up.weight.rank() == 3 and down.weight.rank() == 3, "reference expects rank-three expert weights", .{}); - stdx.debug.assert(gate_up.weight.dim(.expert) == down.weight.dim(.expert), "reference expert counts must match", .{}); - const tokens = input.dim(.t); - const k = topk_ids.dim(.topk); - // Nonlocal routes use expert zero for safe gathers; discard their full - // contribution after the down projection, including any projection bias. - const local_routes = topk_ids.cmp(.GE, Tensor.scalar(0, .i32)); - const safe_ids = topk_ids.maximum(Tensor.scalar(0, .i32)); - const ids = safe_ids.reshape(.{ .route = tokens * k }); - const weights = topk_weights.reshape(.{ .route = tokens * k }).convert(.f32); - const hidden = input.reshape(.{ .token = tokens, .in = input.dim(.d) }) - .insertAxes(.in, .{.topk}).broad(Shape.init(.{ .token = tokens, .topk = k, .in = input.dim(.d) }, input.dtype())) - .merge(.{ .route = .{ .token, .topk } }); - // Keep the projection's rounding boundary when XLA fuses activation/routing - // arithmetic; otherwise the reference can disagree with explicit BF16 GEMMs. - const projected = project(hidden, gate_up, ids, opts, input.dtype()).convert(input.dtype()).optimizationBarrier().convert(.f32); - var activated = activate(projected, opts.activation); - if (opts.routing_weight_placement == .before_down) activated = activated.mul(weights.broad(activated.shape())); - var output = project(activated.rename(.{ .out = .in }), down, ids, opts, input.dtype()); - if (opts.routing_weight_placement == .after_down) output = output.mul(weights.broad(output.shape())); - output = local_routes.reshape(.{ .route = tokens * k }).broad(output.shape()).select(output, Tensor.scalar(0, output.dtype())); - return output.convert(input.dtype()).convert(.f32) - .reshape(.{ .t = tokens, .topk = k, .d = output.dim(.out) }) - .sum(.topk).squeeze(.topk).convert(input.dtype()); -} - -pub fn fusedExperts(input: Tensor, routing: LocalRouting, gate_up: zml.nn.Linear, down: zml.nn.Linear, opts: GlobalOptions) Tensor { - stdx.debug.assert(routing.ids.dim(.topk) > 0 and routing.ids.dim(.topk) <= routing.global_num_experts, "MoE top-k must fit the global expert count", .{}); - return reference(input, routing.localIds(), routing.weights, gate_up, down, .{ - .activation = opts.activation, - .quantize_input = opts.quantize_input, - .routing_weight_placement = opts.routing_weight_placement, - }); -} - -test "MoE compliance: FP4 midpoint rounding" { - const Local = struct { - fn call(x: Tensor) Tensor { - return roundFp4(x); - } - }; - const values = [_]f32{ -7, -5, -3.5, -2.5, -1.75, -1.25, -0.75, -0.25, 0, 0.25, 0.75, 1.25, 1.75, 2.5, 3.5, 5, 7 }; - const expected = [_]f32{ -6, -4, -4, -2, -2, -1, -1, 0, 0, 0, 1, 1, 2, 2, 4, 4, 6 }; - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - const x: Tensor = .init(.{values.len}, .f32); - var exe = try platform.compileFn(allocator, io, Local.call, .{x}, .{}); - defer exe.deinit(); - var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); - defer input.deinit(); - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - const actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - try std.testing.expectEqualSlices(f32, &expected, actual.constItems(f32)); -} diff --git a/zml/kernels/moe/tcl/BUILD.bazel b/zml/kernels/moe/tcl/BUILD.bazel deleted file mode 100644 index 44b56e08..00000000 --- a/zml/kernels/moe/tcl/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "tcl", - srcs = [ - "moe.zig", - "tcl.zig", - ], -) diff --git a/zml/kernels/moe/tcl/moe.zig b/zml/kernels/moe/tcl/moe.zig deleted file mode 100644 index 23fb2274..00000000 --- a/zml/kernels/moe/tcl/moe.zig +++ /dev/null @@ -1,457 +0,0 @@ -//! TCL kernel schedules for Furiosa RNGD fused experts. -const std = @import("std"); - -const builder = @import("dsl/tcl/builder"); - -const zml = @import("../../../zml.zig"); -const tcl = zml.kernel.tcl; -const ActivationKind = @import("../api.zig").ActivationKind; - -pub const Format = enum { bf16, fp8_block128 }; - -/// Dispatch in `tcl.zig` uses the first matching schedule. -/// -/// - `.small_batch` for FP8 up to 64 tokens before padding. Run each selected -/// expert on the whole batch to reuse its weights, with zero routing weight -/// for unrouted pairs. -/// - `.per_route` when `num_tokens * top_k <= local_num_experts`. Gather -/// weights for each route to avoid the overhead of grouping routes. -/// - `.blockwise` otherwise. Pack routed tokens into blocks by expert to -/// reuse weights within each block, then gather results back by route. -pub const Schedule = enum { per_route, blockwise, small_batch }; - -pub const Cfg = struct { - num_tokens: i64, - top_k: i64, - local_num_experts: i64, - hidden_size: i64, - expert_intermediate_size: i64, - weight_scale_dtype: tcl.DType = .bf16, - activation: ActivationKind = .swiglu, -}; - -const base_inputs: []const [:0]const u8 = &.{ "x", "ids", "w", "gate_up", "down" }; -const scale_inputs: []const [:0]const u8 = &.{ "gate_up_scale", "down_scale" }; - -pub fn Kernel(comptime format: Format, comptime schedule: Schedule) type { - if (schedule == .small_batch and format != .fp8_block128) { - @compileError("TCL MoE FP8 small-batch schedule requires block-128 FP8"); - } - return tcl.Kernel(Cfg, .{ - .name = "moe_" ++ @tagName(schedule) ++ switch (format) { - .bf16 => "", - .fp8_block128 => "_fp8", - }, - .inputs = switch (format) { - .bf16 => base_inputs, - .fp8_block128 => base_inputs ++ scale_inputs, - }, - .outputs = &.{"y"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, schedule); - const a: Axes = .init(b, cfg); - const in = try args(b, a, cfg, format); - switch (schedule) { - .per_route => try perRoute(b, a, in, cfg.activation, format), - .blockwise => try blockwise(b, a, in, cfg, format), - .small_batch => try fp8SmallBatch(b, a, in, cfg), - } - } - }.run, - }); -} - -const Axes = struct { - T: tcl.Axis, - K: tcl.Axis, - E: tcl.Axis, - D: tcl.Axis, - I: tcl.Axis, - I2: tcl.Axis, - S2: tcl.Axis, - S1: tcl.Axis, - - fn init(b: *tcl.Builder, cfg: Cfg) Axes { - return .{ - .T = b.axis("T", cfg.num_tokens), - .K = b.axis("K", cfg.top_k), - .E = b.axis("E", cfg.local_num_experts), - .D = b.axis("D", cfg.hidden_size), - .I = b.axis("I", cfg.expert_intermediate_size), - .I2 = b.axis("I2", 2 * cfg.expert_intermediate_size), - .S2 = b.axis("S2", 2), - .S1 = b.axis("S1", 1), - }; - } -}; - -fn args(b: *tcl.Builder, a: Axes, cfg: Cfg, comptime format: Format) tcl.FinishError!Args { - const weight_dtype: tcl.DType = switch (format) { - .bf16 => .bf16, - .fp8_block128 => .f8_e4, - }; - const spec = .{ - .x = .{ .dtype = .bf16, .axes = &.{ a.T, a.D } }, - .ids = .{ .dtype = .i32, .axes = &.{ a.T, a.K } }, - .w = .{ .dtype = .f32, .axes = &.{ a.T, a.K } }, - .gate_up = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.I2, a.D } }, - .down = .{ .dtype = weight_dtype, .axes = &.{ a.E, a.D, a.I } }, - }; - if (format == .bf16) { - const t = try b.declareArgs(spec); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down }; - } - const IG = b.axis("IG", @divExact(cfg.expert_intermediate_size, 128)); - const DG = b.axis("DG", @divExact(cfg.hidden_size, 128)); - const I2G = b.axis("I2G", 2 * @divExact(cfg.expert_intermediate_size, 128)); - const t = try b.declareArgs(.{ - .x = spec.x, - .ids = spec.ids, - .w = spec.w, - .gate_up = spec.gate_up, - .down = spec.down, - .gate_up_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, I2G, DG } }, - .down_scale = .{ .dtype = cfg.weight_scale_dtype, .axes = &.{ a.E, DG, IG } }, - }); - return .{ .x = t.x, .ids = t.ids, .w = t.w, .gate_up = t.gate_up, .down = t.down, .gate_up_scale = t.gate_up_scale, .down_scale = t.down_scale }; -} - -const Args = struct { - x: builder.Tensor, - ids: builder.Tensor, - w: builder.Tensor, - gate_up: builder.Tensor, - down: builder.Tensor, - gate_up_scale: ?builder.Tensor = null, - down_scale: ?builder.Tensor = null, -}; - -const GatheredScales = struct { gate_up: ?builder.Tensor = null, down: ?builder.Tensor = null }; - -fn gatherScales(b: *tcl.Builder, a: Axes, in: Args, ids: builder.Tensor, batch: []const tcl.Axis, comptime format: Format) tcl.FinishError!GatheredScales { - if (format == .bf16) return .{}; - const IG = b.axis("IG", @divExact(a.I.size, 128)); - const DG = b.axis("DG", @divExact(a.D.size, 128)); - const gate_up = try b.reshape(in.gate_up_scale.?, &.{ a.E, a.S2, IG, DG }, .{}); - var gu_axes: [8]tcl.Axis = undefined; - var dn_axes: [8]tcl.Axis = undefined; - @memcpy(gu_axes[0..batch.len], batch); - @memcpy(dn_axes[0..batch.len], batch); - @memcpy(gu_axes[batch.len..][0..3], &[_]tcl.Axis{ a.S2, IG, DG }); - @memcpy(dn_axes[batch.len..][0..2], &[_]tcl.Axis{ DG, IG }); - return .{ - .gate_up = try b.gather(gate_up, ids, a.E, gu_axes[0 .. batch.len + 3], .{}), - .down = try b.gather(in.down_scale.?, ids, a.E, dn_axes[0 .. batch.len + 2], .{}), - }; -} - -fn blockedAxes(axes: []const tcl.Axis, axis: tcl.Axis, group: tcl.Axis, lane: tcl.Axis, storage: *[16]tcl.Axis) []const tcl.Axis { - var n: usize = 0; - for (axes) |a| { - if (std.mem.eql(u8, a.name, axis.name)) { - storage[n] = group; - n += 1; - storage[n] = lane; - } else storage[n] = a; - n += 1; - } - return storage[0..n]; -} - -/// Block scales multiply each K-block partial before K-block reduction. -/// Only the requested experts are gathered; their bank remains FP8 in HBM. -fn project(b: *tcl.Builder, x: builder.Tensor, weight: builder.Tensor, scale: ?builder.Tensor, contracting: tcl.Axis, output: tcl.Axis, to: []const tcl.Axis, dtype: ?tcl.DType, comptime format: Format) tcl.FinishError!builder.Tensor { - if (format == .bf16) { - const op = b.tensorOperation(.{}); - return op.commit(op.contract(x, weight, to), .{ .dtype = dtype }); - } - const CG = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DG" else "IG", @divExact(contracting.size, 128)); - const CL = b.axis(if (std.mem.eql(u8, contracting.name, "D")) "DB" else "IB", 128); - const OG = b.axis(if (std.mem.eql(u8, output.name, "D")) "DG" else "IG", @divExact(output.size, 128)); - const OL = b.axis(if (std.mem.eql(u8, output.name, "D")) "DB" else "IB", 128); - var x_axes: [16]tcl.Axis = undefined; - var w_axes: [16]tcl.Axis = undefined; - var w2_axes: [16]tcl.Axis = undefined; - var out_axes: [16]tcl.Axis = undefined; - var partial_axes: [16]tcl.Axis = undefined; - const xb = try b.reshape(x, blockedAxes(x.axes, contracting, CG, CL, &x_axes), .{}); - const wb = try b.reshape(weight, blockedAxes(blockedAxes(weight.axes, output, OG, OL, &w_axes), contracting, CG, CL, &w2_axes), .{}); - const out = blockedAxes(to, output, OG, OL, &out_axes); - @memcpy(partial_axes[0..out.len], out); - partial_axes[out.len] = CG; - - // Furiosa's published FP8 recipe uses an additive epsilon, including zero blocks. - var op = b.tensorOperation(.{}); - const qscale = try op.commit(op.fetch(xb, .{}).unary(.absf).reduce(&.{CL}, .maxf).divf(448.0).addf(1e-12), .{ .dtype = .f32 }); - op = b.tensorOperation(.{}); - // Native F32 reciprocal accuracy is qualified separately (within one ULP). - const reciprocal = op.ve(.divf, .{ 1.0, op.fetch(qscale, .{}) }); - const quantized = try op.commit(reciprocal.mulf(op.fetch(xb, .{})), .{ .dtype = .f8_e4 }); - op = b.tensorOperation(.{}); - const scaled = op.contract(quantized, wb, partial_axes[0 .. out.len + 1]).mulf(op.fetch(qscale, .{})).mulf(op.fetch(scale.?, .{})).reduce(&.{CG}, .addf); - return b.reshape(try op.commit(scaled, .{ .dtype = dtype, .axes = out }), to, .{}); -} - -/// `silu(gate) * up` or `gelu_tanh(gate) * up` of `h = [batch..., S2, I]` -/// (gate first), times `weights` (`[batch...]`) unless null. -fn gated(b: *tcl.Builder, a: Axes, h: builder.Tensor, batch: []const tcl.Axis, weights: ?builder.Tensor, activation: ActivationKind, comptime format: Format) tcl.FinishError!builder.Tensor { - var sliced: [4]tcl.Axis = undefined; - var flat: [3]tcl.Axis = undefined; - @memcpy(sliced[0..batch.len], batch); - @memcpy(flat[0..batch.len], batch); - sliced[batch.len] = a.S1; - sliced[batch.len + 1] = a.I; - flat[batch.len] = a.I; - const s = sliced[0 .. batch.len + 2]; - const f = flat[0 .. batch.len + 1]; - const gate = try b.reshape(try b.slice(h, a.S2, 0, s, .{}), f, .{}); - const up = try b.reshape(try b.slice(h, a.S2, 1, s, .{}), f, .{}); - // The builder can infer the TCL `Interleaving` tactic for two inputs with - // matching axes, such as `gate` and `up`. That tactic allows exactly one - // binary instruction, so force `.elementwise` for this activation chain. - var op = b.tensorOperation(.{ .tactic = .elementwise }); - const g = op.fetch(gate, .{}); - const activated = if (format == .fp8_block128) blk: { - if (activation != .swiglu) std.debug.panic("TCL MoE block-128 FP8 requires SwiGLU, got {s}", .{@tagName(activation)}); - - // WORKAROUND (TCC 2026.5.0b0): native fusion elides an internal - // BF16 SiLU write, even through as_dram/as_logical. Make its RNE - // rounding observable as integer bits before multiplying by up. - // Remove once the native reproducer preserves the BF16 boundary. - const bits = try op.commit(g.sigmoid().mulf(g).unary(.reinterpret_as_int), .{ .dtype = .i32 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const v = op.fetch(bits, .{}); - const rounded = v.binary(.shr_logical, 16).binary(.bit_andi, 1).addi(32767).addi(v).binary(.bit_andi, -65536).unary(.reinterpret_as_float); - const silu = try op.commit(rounded, .{ .dtype = .bf16 }); - op = b.tensorOperation(.{ .tactic = .elementwise }); - break :blk op.fetch(silu, .{}); - } else switch (activation) { - .swiglu => g.sigmoid().mulf(g), - .geglu_tanh => geluTanh(g), - else => std.debug.panic("TCL MoE does not support activation {s}", .{@tagName(activation)}), - }; - var v = activated.mulf(op.fetch(up, .{})); - if (weights) |w| v = v.mulf(op.fetch(w, .{})); - return op.commit(v, .{ .dtype = .bf16 }); -} - -fn geluTanh(g: builder.Value) builder.Value { - const cube = g.mulf(g).mulf(g); - const inner = g.addf(cube.mulf(0.044715)).mulf(@sqrt(2.0 / std.math.pi)); - return inner.tanh().addf(1.0).mulf(g).mulf(0.5); -} - -/// Furiosa's compiler configuration for MoE kernels (`qwen3_moe/config.py`): -/// Use IO-bound tactics for per-route and FP8 small-batch schedules and -/// compute-bound tactics for blockwise. -/// Keep the compiler's default lowering mode and omit the adaptive-einsum and -/// half-VRF settings removed in TCC 2026.5. -fn configure(b: *tcl.Builder, comptime schedule: Schedule) void { - const config = comptime blk: { - const common = .{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = true } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .propagate_sparse_axis_from_op = "None", - }; - const tactics = switch (schedule) { - .blockwise => .{ - .tactic_hint = .{ .ForLlmModelComputeBound = 200 }, - .dma_preference = 1.2, - }, - .per_route, .small_batch => .{ - .tactic_hint = .{ .ForLlmModelIOBound = 200 }, - .num_transaction_simulation_per_pe = 1024, - .dma_preference = 0.8, - }, - }; - // The transaction-simulation setting is only emitted for IO-bound schedules. - const common_info = @typeInfo(@TypeOf(common)).@"struct"; - const tactics_info = @typeInfo(@TypeOf(tactics)).@"struct"; - const Config = @Struct( - .auto, - null, - common_info.field_names ++ tactics_info.field_names, - common_info.field_types ++ tactics_info.field_types, - common_info.field_attrs ++ tactics_info.field_attrs, - ); - var merged: Config = undefined; - for (common_info.field_names) |field| @field(merged, field) = @field(common, field); - for (tactics_info.field_names) |field| @field(merged, field) = @field(tactics, field); - break :blk merged; - }; - b.compilerConfig(config); -} - -/// Gather the selected experts for every token. -fn perRoute(b: *tcl.Builder, a: Axes, in: Args, activation: ActivationKind, comptime format: Format) tcl.FinishError!void { - const gate_up = try b.reshape(in.gate_up, &.{ a.E, a.S2, a.I, a.D }, .{}); - const gu = try b.gather(gate_up, in.ids, a.E, &.{ a.T, a.K, a.S2, a.I, a.D }, .{}); - const dn = try b.gather(in.down, in.ids, a.E, &.{ a.T, a.K, a.D, a.I }, .{}); - - const scales = try gatherScales(b, a, in, in.ids, &.{ a.T, a.K }, format); - const h = try project(b, in.x, gu, scales.gate_up, a.D, a.I, &.{ a.T, a.K, a.S2, a.I }, if (format == .fp8_block128) .bf16 else null, format); - const act = try gated(b, a, h, &.{ a.T, a.K }, if (format == .fp8_block128) null else in.w, activation, format); - if (format == .fp8_block128) { - const y = try project(b, act, dn, scales.down, a.I, a.D, &.{ a.T, a.K, a.D }, .f32, format); - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(y, .{}).mulf(op.fetch(in.w, .{})).reduce(&.{a.K}, .addf), .{ .dtype = .bf16 })}); - } else { - // The BF16 recipe weights activations and contracts experts together. - const op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.contract(act, dn, &.{ a.T, a.D }), .{ .dtype = .bf16 })}); - } -} - -/// Rows per expert block: the next power of two of the routes per expert. -fn blockRows(cfg: Cfg) i64 { - const per_expert = std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, cfg.local_num_experts) catch unreachable; - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@max(per_expert, 4))) catch unreachable); -} - -const blocks_per_chunk = 32; - -/// Group routes into expert-major blocks, then run both projections in a -/// device loop over chunks of blocks. -fn blockwise(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg, comptime format: Format) tcl.FinishError!void { - const bb = blockRows(cfg); - // One block per expert at least, then enough for every route. - const max_blocks = std.math.divCeil(i64, @max(cfg.num_tokens * cfg.top_k - cfg.local_num_experts, 0), bb) catch unreachable; - const chunks = std.math.divCeil(i64, max_blocks + cfg.local_num_experts, blocks_per_chunk) catch unreachable; - const BB = b.axis("BB", bb); - const BGi = b.axis("BGi", blocks_per_chunk); - const Go = b.axis("Go", chunks); - const BG = b.axis("BG", chunks * blocks_per_chunk); - const GB = b.axis("GB", chunks * blocks_per_chunk * bb); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Routing: route (t, k) goes to row `slot` of block `start[e] + rank`. - const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(onehot, .{}).cumsum(&.{ T, K }), .{}); - op = b.tensorOperation(.{}); - const log2_bb: i64 = std.math.log2_int(u64, @intCast(bb)); - const blocks = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi).addi(bb - 1).binary(.shr_arith, log2_bb), .{}); - op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{E}), .{}); - op = b.tensorOperation(.{}); - const start_rows = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})).muli(bb), .{}); - op = b.tensorOperation(.{}); - const slot = try op.commit(op.fetch(rank, .{}).addi(op.fetch(start_rows, .{})).muli(op.fetch(onehot, .{})).reduce(&.{E}, .addi).subi(1), .{}); - // The expert of block g: how many experts end at or before g. - const block_ids = try b.arange(chunks * blocks_per_chunk, .i32, &.{BG}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const ends = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(block_ids, .{})), .{}); - op = b.tensorOperation(.{}); - const ended = try op.commit(op.where(ends, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - // Padding blocks past the last expert reuse the last one. - const block_expert = try op.commit(op.fetch(ended, .{}).reduce(&.{E}, .addi).binary(.mini, cfg.local_num_experts - 1), .{}); - const used = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(block_end), blocks_per_chunk - 1), blocks_per_chunk); - - // Rows of unused slots are never read back. - const rows = try b.reshape(try b.scatter(in.x, slot, GB, &.{ GB, D }, .{}), &.{ Go, BGi, BB, D }, .{}); - const chunk_experts = try b.reshape(block_expert, &.{ Go, BGi }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(used, .{try b.scratchpad(if (format == .fp8_block128) .f32 else .bf16, &.{ Go, BGi, BB, D })}); - { - const chunk = try b.indexRead(rows, loop.iv); - const ids = try b.indexRead(chunk_experts, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ BGi, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ BGi, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{BGi}, format); - const h = try project(b, chunk, gu, scales.gate_up, D, I, &.{ BGi, BB, a.S2, I }, if (format == .fp8_block128) .bf16 else null, format); - const act = try gated(b, a, h, &.{ BGi, BB }, null, cfg.activation, format); - const y = try project(b, act, dn, scales.down, I, D, &.{ BGi, BB, D }, if (format == .fp8_block128) .f32 else .bf16, format); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - const out = loop.results[0]; - - const per_route = try b.gather(try b.reshape(out, &.{ GB, D }, .{}), slot, GB, &.{ T, K, D }, .{}); - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(per_route, .{ .typecast_to = .f32 }).mulf(op.fetch(in.w, .{})).reduce(&.{K}, .addf), .{ .dtype = .bf16 })}); -} - -/// Number of experts processed per device-loop iteration, based on the -/// batch's route count (`cfg.num_tokens * cfg.top_k`). -fn expertsPerChunk(cfg: Cfg) i64 { - const routes: u64 = @intCast(std.math.divCeil(i64, cfg.num_tokens * cfg.top_k, 16) catch unreachable); - return @intCast(std.math.clamp(std.math.ceilPowerOfTwo(u64, routes) catch unreachable, 2, 16)); -} - -/// Process the FP8 experts selected by the batch in groups sized by `expertsPerChunk`. -/// Load each selected expert once and run it for every token. Routing weights -/// scale the F32 down-projection results before summing the experts and steps. -fn fp8SmallBatch(b: *tcl.Builder, a: Axes, in: Args, cfg: Cfg) tcl.FinishError!void { - const chunk = expertsPerChunk(cfg); - const steps = std.math.divCeil(i64, @min(cfg.local_num_experts, cfg.num_tokens * cfg.top_k), chunk) catch unreachable; - const C = b.axis("C", chunk); - const Go = b.axis("Go", steps); - const J = b.axis("J", steps * chunk); - const T, const K, const E, const D, const I = .{ a.T, a.K, a.E, a.D, a.I }; - - // Each token's routing weight per expert, and the used experts. - const experts = try b.arange(cfg.local_num_experts, .i32, &.{E}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const dist = try op.commit(op.fetch(in.ids, .{}).subi(op.fetch(experts, .{})), .{}); - op = b.tensorOperation(.{}); - const gate = try op.commit(op.where(op.fetch(dist, .{}), .eq, 0, op.fetch(in.w, .{ .broadcast_to = &.{ T, K, E } }), 0.0).reduce(&.{K}, .addf), .{}); - op = b.tensorOperation(.{}); - const onehot = try op.commit(op.where(dist, .eq, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const counts = try op.commit(op.fetch(onehot, .{}).reduce(&.{ T, K }, .addi), .{}); - op = b.tensorOperation(.{}); - const used = try op.commit(op.where(counts, .gt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(used, .{}).cumsum(&.{E}), .{}); - // A used expert's rank among them, -1 for the others. - op = b.tensorOperation(.{}); - const rank = try op.commit(op.fetch(position, .{}).muli(op.fetch(used, .{})).subi(1), .{}); - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(position), chunk - 1), chunk); - - // Slot j holds the used expert of rank j and its tokens' weights; - // slots past the last hold expert 0 with no weight. - const ranks = try b.arange(steps * chunk, .i32, &.{J}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const offset = try op.commit(op.fetch(rank, .{}).subi(op.fetch(ranks, .{})), .{}); - op = b.tensorOperation(.{}); - const slot_experts = try op.commit(op.where(op.fetch(offset, .{}), .eq, 0, op.fetch(experts, .{ .broadcast_to = &.{ E, J } }), 0).reduce(&.{E}, .addi), .{ .dtype = .i32 }); - // A reduction's condition is read plainly, so broadcast it first. - op = b.tensorOperation(.{}); - const token_offset = try op.commit(op.fetch(offset, .{ .broadcast_to = &.{ T, E, J } }), .{}); - op = b.tensorOperation(.{}); - const slot_gate = try op.commit(op.where(op.fetch(token_offset, .{}), .eq, 0, op.fetch(gate, .{ .broadcast_to = &.{ T, E, J } }), 0.0).reduce(&.{E}, .addf), .{ .dtype = .f32, .axes = &.{ J, T } }); - const step_experts = try b.reshape(slot_experts, &.{ Go, C }, .{}); - const step_gate = try b.reshape(slot_gate, &.{ Go, C, T }, .{}); - const gate_up = try b.reshape(in.gate_up, &.{ E, a.S2, I, D }, .{}); - - var loop = b.openFor(trips, .{try b.full(0, .f32, &.{ Go, T, D })}); - { - const ids = try b.indexRead(step_experts, loop.iv); - const weights = try b.indexRead(step_gate, loop.iv); - const gu = try b.gather(gate_up, ids, E, &.{ C, a.S2, I, D }, .{}); - const dn = try b.gather(in.down, ids, E, &.{ C, D, I }, .{}); - const scales = try gatherScales(b, a, in, ids, &.{C}, .fp8_block128); - const h = try project(b, in.x, gu, scales.gate_up, D, I, &.{ C, T, a.S2, I }, .bf16, .fp8_block128); - const act = try gated(b, a, h, &.{ C, T }, null, cfg.activation, .fp8_block128); - const per_expert = try project(b, act, dn, scales.down, I, D, &.{ C, T, D }, .f32, .fp8_block128); - op = b.tensorOperation(.{}); - const y = try op.commit(op.fetch(per_expert, .{}).mulf(op.fetch(weights, .{})).reduce(&.{C}, .addf), .{ .dtype = .f32 }); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, y)}); - } - op = b.tensorOperation(.{}); - b.ret(&.{try op.commit(op.fetch(loop.results[0], .{}).reduce(&.{Go}, .addf), .{ .dtype = .bf16 })}); -} diff --git a/zml/kernels/moe/tcl/tcl.zig b/zml/kernels/moe/tcl/tcl.zig deleted file mode 100644 index f05727ce..00000000 --- a/zml/kernels/moe/tcl/tcl.zig +++ /dev/null @@ -1,250 +0,0 @@ -//! Furiosa RNGD fused experts in TCL, after Furiosa's production kernels. -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const Tensor = zml.Tensor; -const tcl = zml.kernel.tcl; -const kernels = @import("moe.zig"); -const api = @import("../api.zig"); - -/// `input` is `{.t, .d}`, routing IDs and weights are `{.t, .topk}`, `gate_up` -/// is `{.expert, .dout = 2 * inter (gate then up), .d}`, and `down` is -/// `{.expert, .d, .dout = inter}`. Experts are BF16 or block-128 FP8 with -/// BF16/F32 scales. Supports plain SwiGLU and BF16 GEGLU with tanh GELU. -pub fn fusedExperts(input: Tensor, routing: api.LocalRouting, gate_up_linear: zml.nn.Linear, down_linear: zml.nn.Linear, opts: api.Options) Tensor { - const topk_ids = routing.localIds(); - stdx.debug.assert(!opts.quantize_input and opts.routing_weight_placement == .after_down, "TCL MoE requires quantize_input=false and routing_weight_placement=after_down", .{}); - stdx.debug.assert(gate_up_linear.bias == null and down_linear.bias == null, "TCL MoE takes experts without bias", .{}); - stdx.debug.assert(input.dtype() == .bf16 and topk_ids.dtype() == .i32, "TCL MoE takes BF16 activations and i32 ids", .{}); - const format = expertFormat(gate_up_linear, down_linear); - const swiglu = opts.activation == .swiglu and std.meta.eql(opts.activation.swiglu, .{}); - stdx.debug.assert(swiglu or (format == .bf16 and opts.activation == .geglu_tanh), "TCL MoE supports plain SwiGLU or GEGLU-tanh with BF16 experts, got {s}", .{@tagName(opts.activation)}); - - const gate_up = gate_up_linear.weight; - const down = down_linear.weight; - const cfg: kernels.Cfg = .{ - .num_tokens = input.dim(.t), - .top_k = topk_ids.dim(.topk), - .local_num_experts = gate_up.dim(.expert), - .hidden_size = input.dim(.d), - .expert_intermediate_size = down.dim(.dout), - .weight_scale_dtype = switch (format) { - .bf16 => .bf16, - .fp8_block128 => |s| tcl.from(s.gate_up.dtype()), - }, - .activation = std.meta.activeTag(opts.activation), - }; - stdx.debug.assert(gate_up.dim(.dout) == 2 * cfg.expert_intermediate_size and down.dim(.d) == cfg.hidden_size and gate_up.dim(.d) == cfg.hidden_size, "TCL MoE: mismatched expert shapes", .{}); - - const weights = routing.weights.convert(.f32); - const local_weights = switch (format) { - .bf16 => weights, - .fp8_block128 => topk_ids.cmp(.GE, .scalar(0, .i32)).select(weights, .scalar(0, .f32)), - }; - return switch (format) { - .bf16 => selectScheduleAndRun(.bf16, cfg, .{ .x = input, .ids = topk_ids, .w = local_weights, .gate_up = gate_up, .down = down }), - .fp8_block128 => |s| selectScheduleAndRun(.fp8_block128, cfg, .{ - .x = input, - .ids = topk_ids, - .w = local_weights, - .gate_up = gate_up, - .down = down, - .gate_up_scale = s.gate_up, - .down_scale = s.down, - }), - }; -} - -const Scales = struct { gate_up: Tensor, down: Tensor }; - -const Format = union(kernels.Format) { - bf16: void, - fp8_block128: Scales, -}; - -fn expertFormat(gate_up: zml.nn.Linear, down: zml.nn.Linear) Format { - const scheme = gate_up.quantizationScheme(); - stdx.debug.assert(scheme == down.quantizationScheme(), "TCL MoE projections must have matching quantization schemes", .{}); - return switch (scheme orelse { - stdx.debug.assert(gate_up.weight.dtype() == .bf16 and down.weight.dtype() == .bf16, "TCL MoE unquantized expert weights must be BF16", .{}); - return .bf16; - }) { - .fp8_block128 => blk: { - const scales: Scales = .{ .gate_up = fp8Scales(gate_up), .down = fp8Scales(down) }; - stdx.debug.assert(scales.gate_up.dtype() == scales.down.dtype(), "TCL MoE projection scales must have matching types", .{}); - break :blk .{ .fp8_block128 = scales }; - }, - else => |unsupported| stdx.debug.panic("TCL MoE does not support quantization scheme {s}", .{@tagName(unsupported)}), - }; -} - -fn fp8Scales(linear: zml.nn.Linear) Tensor { - const q = linear.quantization.?; - stdx.debug.assert(linear.weight.dtype() == .f8e4m3fn and !q.swizzled_scales and q.global_scale == null and q.input_scale == null, "TCL MoE takes plain E4M3FN block-128 weights", .{}); - stdx.debug.assert(q.scales.dtype() == .bf16 or q.scales.dtype() == .f32, "TCL MoE FP8 scales must be BF16 or F32", .{}); - const w = linear.weight; - const scale = q.scales; - stdx.debug.assert(@mod(w.dim(.d), 128) == 0 and @mod(w.dim(.dout), 128) == 0 and scale.dim(.expert) == w.dim(.expert) and scale.dim(.d) * 128 == w.dim(.d) and scale.dim(.dout) * 128 == w.dim(.dout), "TCL MoE FP8 scales must cover 128x128 weight blocks", .{}); - return scale; -} - -// Maximum total input tokens per MoE call before padding for the FP8 -// small-batch schedule. It loads each used expert's weights once and reuses -// them across the batch. -// Larger batches fall back to per-route or blockwise dispatch. -const fp8_small_batch_max_tokens = 64; - -fn selectScheduleAndRun(comptime format: kernels.Format, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { - if (format == .fp8_block128 and cfg.num_tokens <= fp8_small_batch_max_tokens) { - return run(format, .small_batch, cfg, inputs); - } - if (cfg.num_tokens * cfg.top_k <= cfg.local_num_experts) { - return run(format, .per_route, cfg, inputs); - } - return run(format, .blockwise, cfg, inputs); -} - -fn run(comptime format: kernels.Format, comptime schedule: kernels.Schedule, cfg: kernels.Cfg, inputs: kernels.Kernel(format, .per_route).Inputs) Tensor { - const call_cfg, const prepared = if (schedule == .small_batch) - prepareFp8SmallBatch(cfg, inputs) - else - .{ cfg, prepareRoutedInputs(cfg, inputs) }; - - const shape = prepared.x.shape(); - var y = kernels.Kernel(format, schedule).call(prepared, .{ .y = shape }, .{ .cfg = call_cfg }).y; - - if (schedule == .small_batch) y = y.slice(0, .{ .end = cfg.num_tokens }); - return y.reshape(inputs.x.shape()); -} - -fn prepareRoutedInputs(cfg: kernels.Cfg, inputs: anytype) @TypeOf(inputs) { - var prepared = inputs; - prepared.x = inputs.x.reshape(.{ cfg.num_tokens, cfg.hidden_size }); - const ids = inputs.ids.reshape(.{ cfg.num_tokens, cfg.top_k }); - const w = inputs.w.reshape(.{ cfg.num_tokens, cfg.top_k }); - // These kernels gather every route, and blockwise also scatters each route - // into an expert slot. Give nonlocal routes a valid slot and zero weight. - const valid = ids.cmp(.GE, Tensor.scalar(0, .i32)).logical(.AND, ids.cmp(.LT, Tensor.scalar(cfg.local_num_experts, .i32))); - prepared.ids = valid.select(ids, Tensor.scalar(0, .i32)); - prepared.w = valid.select(w, Tensor.scalar(0, .f32)); - return prepared; -} - -fn prepareFp8SmallBatch(cfg: kernels.Cfg, inputs: kernels.Kernel(.fp8_block128, .small_batch).Inputs) struct { kernels.Cfg, @TypeOf(inputs) } { - // WORKAROUND (TCC 2026.5.0b0): a token count other than a power of two - // panics its adder tree ("Fail to cover valid range", 5 tokens padded - // to 8): pad with unrouted tokens. - var padded = cfg; - padded.num_tokens = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.num_tokens)) catch unreachable); - const pad = padded.num_tokens - cfg.num_tokens; - var prepared = inputs; - prepared.x = inputs.x.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.hidden_size }); - prepared.ids = inputs.ids.pad(-1, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); - prepared.w = inputs.w.pad(0, .{ .t = Tensor.Pad{ .high = pad } }).reshape(.{ padded.num_tokens, cfg.top_k }); - return .{ padded, prepared }; -} - -test "tcl fused experts run on furiosa" { - try testFusedExperts(.{ .swiglu = .{} }); - try testFusedExperts(.geglu_tanh); -} - -fn testFusedExperts(activation: zml.moe.Activation) !void { - const platform = zml.testing.env(); - if (platform.target != .furiosa) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - const bf16 = zml.floats.BFloat16; - - // With two routes per token and eight experts, four tokens use per-route - // dispatch and sixteen use blockwise dispatch. - for ([_]i64{ 4, 16 }) |t| { - const k = 2; - const e = 8; - const d = 256; - const i = 64; - const x: Tensor = .init(.{ .t = t, .d = d }, .bf16); - const ids: Tensor = .init(.{ .t = t, .topk = k }, .i32); - const w: Tensor = .init(.{ .t = t, .topk = k }, .f32); - const gu: Tensor = .init(.{ .expert = e, .dout = 2 * i, .d = d }, .bf16); - const dn: Tensor = .init(.{ .expert = e, .d = d, .dout = i }, .bf16); - const Mod = struct { - pub fn forward(x_: Tensor, ids_: Tensor, w_: Tensor, gu_: Tensor, dn_: Tensor, act: zml.moe.Activation) Tensor { - return zml.moe.forwardMoe(x_, ids_, w_, .init(gu_, null, .dout), .init(dn_, null, .dout), .tcl, zml.Compiler.current().platform.replicated_mesh, .expert(.replicated), .{ - .activation = act, - .quantize_input = false, - .routing_weight_placement = .after_down, - }); - } - }; - var exe = try platform.compileFn(allocator, io, Mod.forward, .{ x, ids, w, gu, dn, activation }, .{}); - defer exe.deinit(); - - var prng: std.Random.DefaultPrng = .init(@intCast(t)); - const r = prng.random(); - const hx = try allocator.alloc(bf16, @intCast(t * d)); - defer allocator.free(hx); - const hgu = try allocator.alloc(bf16, e * 2 * i * d); - defer allocator.free(hgu); - const hdn = try allocator.alloc(bf16, e * d * i); - defer allocator.free(hdn); - for (hx) |*v| v.* = .fromF32(r.floatNorm(f32) * 0.5); - for (hgu) |*v| v.* = .fromF32(r.floatNorm(f32) / 16); - for (hdn) |*v| v.* = .fromF32(r.floatNorm(f32) / 8); - const hids = try allocator.alloc(i32, @intCast(t * k)); - defer allocator.free(hids); - const hw = try allocator.alloc(f32, @intCast(t * k)); - defer allocator.free(hw); - for (0..@intCast(t)) |tok| for (0..k) |j| { - hids[tok * k + j] = @intCast((tok * 3 + j * 5) % e); - hw[tok * k + j] = if (j == 0) 0.7 else 0.3; - }; - - var bx: zml.Buffer = try .fromBytes(io, platform, x.shape(), std.mem.sliceAsBytes(hx)); - defer bx.deinit(); - var bids: zml.Buffer = try .fromBytes(io, platform, ids.shape(), std.mem.sliceAsBytes(hids)); - defer bids.deinit(); - var bw: zml.Buffer = try .fromBytes(io, platform, w.shape(), std.mem.sliceAsBytes(hw)); - defer bw.deinit(); - var bgu: zml.Buffer = try .fromBytes(io, platform, gu.shape(), std.mem.sliceAsBytes(hgu)); - defer bgu.deinit(); - var bdn: zml.Buffer = try .fromBytes(io, platform, dn.shape(), std.mem.sliceAsBytes(hdn)); - defer bdn.deinit(); - var result = try exe.eval(allocator, io, .{ bx, bids, bw, bgu, bdn }); - defer result.deinit(); - var host = try result.toSliceAlloc(allocator, io); - defer host.free(allocator); - const out = host.items(bf16); - - for (0..@intCast(t)) |tok| { - var expected: [d]f32 = @splat(0); - for (0..k) |j| { - const ex: usize = @intCast(hids[tok * k + j]); - var act: [i]f32 = undefined; - for (0..i) |n| { - var g: f32 = 0; - var u: f32 = 0; - for (0..d) |m| { - const xv = hx[tok * d + m].toF32(); - g += xv * hgu[(ex * 2 * i + n) * d + m].toF32(); - u += xv * hgu[(ex * 2 * i + i + n) * d + m].toF32(); - } - const gate = switch (activation) { - .swiglu => g / (1 + @exp(-g)), - .geglu_tanh => 0.5 * g * (1 + std.math.tanh(@sqrt(2.0 / std.math.pi) * (g + 0.044715 * g * g * g))), - else => unreachable, - }; - act[n] = gate * u; - } - for (0..d) |m| { - var y: f32 = 0; - for (0..i) |n| y += act[n] * hdn[(ex * d + m) * i + n].toF32(); - expected[m] += hw[tok * k + j] * y; - } - } - for (expected, out[tok * d ..][0..d]) |want, got| try std.testing.expectApproxEqAbs(want, got.toF32(), 2e-2 + 2e-2 * @abs(want)); - } - } -} diff --git a/zml/kernels/moe/tests.zig b/zml/kernels/moe/tests.zig deleted file mode 100644 index c74406c3..00000000 --- a/zml/kernels/moe/tests.zig +++ /dev/null @@ -1,752 +0,0 @@ -//! Shared public-entry-point tests. Hardware absence is the only runtime skip: -//! supported cases are explicit, and errors in those cases must fail the test. -const std = @import("std"); - -const zml = @import("../../zml.zig"); -const Tensor = zml.Tensor; -const Shape = zml.Shape; -const Activation = @import("api.zig").Activation; -const ExpertsLayout = @import("api.zig").ExpertsLayout; -const GlobalOptions = @import("api.zig").Options; -const Placement = @import("api.zig").RoutingWeightPlacement; -const reference = @import("stablehlo/stablehlo.zig"); - -const Case = struct { - backend: zml.moe.Backend, - activation: Activation = .{ .swiglu = .{} }, - scheme: ?zml.Quantization.Scheme = null, - activation_dtype: zml.DataType = .bf16, - quantize_input: bool = false, - placement: Placement = .after_down, - bias: bool = false, - non_unit_scales: bool = false, - tokens: i64 = 1, - topk: i64 = 1, - width: i64 = 128, - intermediate: i64 = 128, - experts: i64 = 8, - expert_mesh: *const zml.Mesh, - - fn inputShape(c: Case) Shape { - const shape = Shape.init(.{ .t = c.tokens, .d = c.width }, c.activation_dtype); - const spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; - return shape.withPartitioning(c.expert_mesh, spec); - } - - pub fn compile(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform) !zml.Exe { - std.debug.assert(c.backend.isAvailable(platform)); - const x: Tensor = .fromShape(c.inputShape()); - return platform.compileFn(allocator, io, forward, .{ x, c }, .{ .meshes = &.{c.expert_mesh} }); - } - - pub fn check(c: Case, allocator: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, exe: *const zml.Exe) !void { - const x: Tensor = .fromShape(c.inputShape()); - - var host = try zml.Slice.alloc(allocator, x.shape()); - defer host.free(allocator); - switch (c.activation_dtype) { - inline .bf16, .f16, .f32 => |dtype| { - for (host.items(dtype.toZigType()), 0..) |*value, i| { - const f = @as(f32, @floatFromInt(@as(i32, @intCast((i * 13 + i / 128) % 23)) - 11)) / 32; - value.* = if (dtype == .bf16) .fromF32(f) else @floatCast(f); - } - }, - else => unreachable, - } - - var input = try zml.Buffer.fromSlice(io, platform, host); - defer input.deinit(); - - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - var output: zml.Bufferized(Outputs) = undefined; - runner.run(io, .{input}, .{&output}, .{ .wait = true }); - defer zml.Buffer.deinitAll(Outputs, &output); - try std.testing.expectEqualDeep(c.inputShape()._sharding, output.actual.shape()._sharding); - - // BF16 intermediate rounding and different GEMM reduction orders. Every - // element must pass; near-zero values use the absolute bound. - try zml.testing.expectClose(io, output.expected, output.actual, .{ - // Share the NVFP4 comparison budget across CUDA backends. - .absolute_tolerance = if (c.scheme == .nvfp4) 0.078125 else 0.015625, - .relative_tolerance = 0.02, - .minimum_close_fraction = 1, - }); - } -}; - -/// Generate a reproducible Tensor with a simple pattern. -/// -/// Generate as many elements as the shape requires, with this formula: -/// value[i] = (((i * multiplier + 3) % 17) - 8) / divisor -/// -/// Then reshape to the requested shape and convert to f32. -fn generateDeterministicTensor(shape: Shape, multiplier: i64, divisor: f64) Tensor { - return Tensor.arange(.{ .end = @intCast(shape.count()) }, .i32) - .mul(Tensor.scalar(multiplier, .i32)).addConstant(3) - .remainder(Tensor.scalar(17, .i32)).addConstant(-8) - .convert(.f32).scale(1 / divisor).reshape(shape.withDtype(.f32)); -} - -/// Select exactly representable FP4 values with distinct even/odd sequences. -fn generateFp4Tensor(shape: Shape, multiplier: i64) Tensor { - const values = Tensor.constantTensor(.init(.{ .value = 16 }, .f32), std.mem.sliceAsBytes(&[_]f32{ - 0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6, - })); - const index = Tensor.arange(.{ .end = @intCast(shape.count()) }, .i32); - const pair = index.div(Tensor.scalar(2, .i32)); - // Prime periods distinguish experts, output rows, and gate/up halves. - const even = pair.mul(Tensor.scalar(multiplier, .i32)).addConstant(3) - .remainder(Tensor.scalar(17, .i32)).remainder(Tensor.scalar(16, .i32)); - const odd = pair.mul(Tensor.scalar(3, .i32)).addConstant(7) - .remainder(Tensor.scalar(19, .i32)).remainder(Tensor.scalar(16, .i32)); - const indices = index.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)).select(even, odd); - return values.gather(.{ .value = indices }, .{}).convert(.f4e2m1).reshape(shape.withDtype(.f4e2m1)); -} - -/// Create a Linear fixture with a reproducible pattern, optionally quantized and biased. -fn linear(shape: Shape, scheme: ?zml.Quantization.Scheme, bias: bool, multiplier: i64) zml.nn.Linear { - const fixture_bias = if (bias) generateDeterministicTensor(shape.remove(2), 3, 32).convert(shape.dtype()) else null; - - const s = scheme orelse return .{ - .weight = generateDeterministicTensor(shape, multiplier, 16).convert(shape.dtype()), - .tag = shape.tag(2), - .bias = fixture_bias, - }; - - const weight = switch (s) { - .mxfp4, .nvfp4 => generateFp4Tensor(shape, multiplier), - else => generateDeterministicTensor(shape, multiplier, 16).convert(.f8e4m3fn), - }; - - const scale_shape: Shape = switch (s) { - .mxfp4, .mxfp8 => .init(.{ shape.dim(0), shape.dim(1), @divExact(shape.dim(2), 32) }, .f8e8m0), - .nvfp4 => .init(.{ shape.dim(0), shape.dim(1), @divExact(shape.dim(2), 16) }, .f8e4m3fn), - .fp8_per_tensor => .init(.{ shape.dim(0), 1, 1 }, .f32), - .fp8_per_channel => .init(.{ shape.dim(0), shape.dim(1), 1 }, .f32), - .fp8_block128 => .init(.{ shape.dim(0), @divExact(shape.dim(1), 128), @divExact(shape.dim(2), 128) }, .f32), - .fp8_block32 => .init(.{ shape.dim(0), @divExact(shape.dim(1), 32), @divExact(shape.dim(2), 32) }, .f32), - }; - - const scales = Tensor.arange(.{ .end = @intCast(scale_shape.count()) }, .i32) - .remainder(Tensor.scalar(2, .i32)).convert(.f32).addConstant(1).scale(1.0 / 16.0) - .reshape(scale_shape.withDtype(.f32)).convert(scale_shape.dtype()); - - const input_scale: ?zml.Quantization.GlobalScale = if (s == .nvfp4) .{ .value = Tensor.scalar(1, .f32), .operation = .multiply } else null; - const global_scale: ?zml.Quantization.GlobalScale = if (s == .nvfp4) .{ .value = Tensor.scalar(1, .f32).broad(.init(.{ .expert = shape.dim(0) }, .f32)), .operation = .multiply } else null; - - return .{ - .weight = weight, - .tag = shape.tag(2), - .bias = fixture_bias, - .quantization = .{ - .scheme = s, - .scales = scales, - .input_scale = input_scale, - .global_scale = global_scale, - }, - }; -} - -const ExpertFixture = struct { - gate_up: zml.nn.Linear, - down: zml.nn.Linear, -}; - -/// Canonical weights use logical dimensions, concatenated gate/up, and plain scales. -fn makeExpertFixture(c: Case) ExpertFixture { - const projection_width: i64 = switch (c.activation) { - .silu, .gelu, .relu => c.intermediate, - else => 2 * c.intermediate, - }; - var fixture: ExpertFixture = .{ - .gate_up = linear(.init(.{ .expert = c.experts, .dout = projection_width, .d = c.width }, c.activation_dtype), c.scheme, c.bias, 7), - .down = linear(.init(.{ .expert = c.experts, .d = c.width, .dout = c.intermediate }, c.activation_dtype), c.scheme, c.bias, 11), - }; - if (c.non_unit_scales) { - for ([_]*zml.nn.Linear{ &fixture.gate_up, &fixture.down }, 0..) |projection, i| { - const q = &projection.quantization.?; - std.debug.assert(q.scheme == .nvfp4); - // Exercise both scale polarity and distinct per-expert multipliers. - q.input_scale = .{ .value = Tensor.scalar(@as(f32, if (i == 0) 2 else 0.5), .f32), .operation = .divide }; - q.global_scale = .{ - .value = Tensor.arange(.{ .end = @intCast(c.experts) }, .i32) - .remainder(Tensor.scalar(2, .i32)).addConstant(1).convert(.f32).withTags(.{.expert}), - .operation = .divide, - }; - } - } - return fixture; -} - -fn interleave(tensor: Tensor, rows: i64) Tensor { - return tensor.reshape(.{ .expert = tensor.dim(0), .projection = 2, .row_block = @divExact(tensor.dim(1), 2 * rows), .row = rows, .column = tensor.dim(2) }) - .transpose(.{ .expert, .row_block, .projection, .row, .column }).reshape(tensor.shape()); -} - -/// Blackwell block scales: [E, N/128, 4, 32, K/4, 4] -> -/// [E, N/128, K/4, 32, 4, 4], retaining the declared [E, N, K] shape. -fn swizzleScales(scales: Tensor) Tensor { - return scales.reshape(.{ .expert = scales.dim(0), .row_block = @divExact(scales.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(scales.dim(2), 4), .scale_byte = 4 }) - .transpose(.{ .expert, .row_block, .scale_word, .row_lane, .row_quad, .scale_byte }).reshape(scales.shape()); -} - -fn packFp4Bytes(weight: Tensor) Tensor { - std.debug.assert(weight.dtype() == .f4e2m1); - return weight.reshape(weight.shape().setDim(2, @divExact(weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8); -} - -fn prepareGateUp(tensor: Tensor, layout: ExpertsLayout) Tensor { - return switch (layout.gate_up) { - .concatenated => tensor, - .row_interleaved_1 => interleave(tensor, 1), - .row_interleaved_8 => interleave(tensor, 8), - }; -} - -/// Adapt a copy of both projections to the backend's storage and layout contract. -fn prepareForBackend(canonical: ExpertFixture, backend: zml.moe.Backend) ExpertFixture { - var prepared = canonical; - const scheme = canonical.gate_up.quantizationScheme(); - const layout = backend.expertsLayout(scheme) catch |err| zml.stdx.debug.panic("fixture layout: {}", .{err}); - // Plain activations have one projection, so gate/up ordering does not apply. - const gated = canonical.gate_up.weight.dim(1) == 2 * canonical.down.weight.dim(2); - if (gated) { - prepared.gate_up.weight = prepareGateUp(canonical.gate_up.weight, layout); - if (canonical.gate_up.bias) |bias| prepared.gate_up.bias = prepareGateUp(bias.appendAxes(.{.dummy}), layout).squeeze(.dummy); - if (prepared.gate_up.quantization) |*q| q.scales = prepareGateUp(q.scales, layout); - } - - for ([_]*zml.nn.Linear{ &prepared.gate_up, &prepared.down }) |projection| { - switch (layout.packing) { - .swizzled_scales, .flashinfer_nvfp4 => { - const q = &projection.quantization.?; - q.scales = swizzleScales(q.scales); - q.swizzled_scales = true; - }, - .mxfp4_interleaved8 => { - const q = &projection.quantization.?; - const sf = q.scales; - q.scales = sf.bitCast(.u8) - .reshape(.{ .expert = sf.dim(0), .row_block = @divExact(sf.dim(1), 128), .row_quad = 4, .row_lane = 32, .scale_word = @divExact(sf.dim(2), 4), .scale_byte = 4 }) - .transpose(.{ .expert, .scale_word, .row_block, .row_lane, .row_quad, .scale_byte }) - .reshape(.{ .expert = sf.dim(0), .scale_word = @divExact(sf.dim(2), 4), .row = sf.dim(1), .scale_byte = 4 }).bitCast(.i32) - .withTags(sf.shape().transpose(.{ 0, 2, 1 })); - }, - .plain => {}, - } - switch (backend) { - .cute => { - projection.weight = packFp4Bytes(projection.weight); - const q = &projection.quantization.?; - if (scheme == .nvfp4) { - q.scales = q.scales.bitCast(.u8); - // CuTe takes the input quantization multiplier and a GEMM - // output multiplier that folds in its inverse. - const input_scale = q.input_scale.?.asMultiplier(); - q.global_scale = .{ .value = q.global_scale.?.asMultiplier().mul(input_scale), .operation = .multiply }; - q.input_scale = .{ .value = Tensor.scalar(1, .f32).div(input_scale), .operation = .multiply }; - } - }, - .triton => if (scheme == .mxfp4) { - projection.weight = packFp4Bytes(projection.weight); - }, - .mosaic_tpu => { - projection.weight = projection.weight.transpose(.{ 0, 2, 1 }); - }, - .metal => if (projection.quantization) |*q| { - if (q.scheme != .nvfp4) q.scales = q.scales.convert(.bf16); - }, - else => {}, - } - } - - if (backend == .metal and scheme == .nvfp4) { - const global = &prepared.gate_up.quantization.?.global_scale.?; - global.value = global.value.appendAxes(.{.proj}).repeat1d(.proj, 2); - } - return prepared; -} - -const Outputs = struct { actual: Tensor, expected: Tensor }; - -/// Forward a fixture through the MoE implementation under test, and also through the StableHLO -/// reference implementation. Return both outputs for comparison. -fn forward(input: Tensor, c: Case) Outputs { - const canonical = makeExpertFixture(c); - const prepared = prepareForBackend(canonical, c.backend); - - const route_shape = Shape.init(.{ .t = c.tokens, .topk = c.topk }, .i32); - const route = Tensor.arange(.{ .end = @intCast(route_shape.count()) }, .i32).reshape(route_shape); - - // Only six of eight experts receive routes, with nonuniform counts. - const ids = route.mul(Tensor.scalar(5, .i32)).addConstant(2).remainder(Tensor.scalar(@min(c.experts, 6), .i32)); - - const weights = route.remainder(Tensor.scalar(3, .i32)).convert(.f32).addConstant(1).scale(0.25); - const options: GlobalOptions = .{ - .activation = c.activation, - .quantize_input = c.quantize_input, - .routing_weight_placement = c.placement, - }; - - const data_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("data") != null) .parse(.{ .t = .data }) else .replicated; - const expert_spec: zml.Sharding.Partitioning.Logical = if (c.expert_mesh.resolveLogicalAxis("experts") != null) .parse(.{ .expert = .experts }) else .replicated; - const actual = zml.moe.forwardMoe(input, ids, weights, prepared.gate_up, prepared.down, c.backend, c.expert_mesh, .{ - .in_specs = .{ - .input = data_spec, - .ids = data_spec, - .weights = data_spec, - .gate_up = expert_spec, - .down = expert_spec, - }, - .out_specs = data_spec, - }, options); - // Keep the oracle input replicated so token sharding does not change its GEMM rounding. - const expected = reference.reference(input.replicate(), ids, weights, canonical.gate_up, canonical.down, .{ - .activation = c.activation, - .quantize_input = c.quantize_input, - .routing_weight_placement = c.placement, - .input_quantization = switch (c.backend) { - .metal => .none, - .cute => if (c.scheme == .nvfp4) .automatic else .mxfp8_bf16, - .triton => if (c.scheme == .mxfp4 and zml.Compiler.current().platform.target == .cuda) .mxfp8 else .automatic, - else => .automatic, - }, - }); - - return .{ .actual = actual, .expected = expected }; -} - -const gated_activations = [_]Activation{ - .{ .swiglu = .{} }, - .{ .swiglu = .{ .limit = 0.25 } }, - .{ .swiglu = .{ .bias = 1 } }, - .{ .swiglu = .{ .limit = 0.25, .bias = 1 } }, - .{ .swiglu_step = .{} }, - .{ .swiglu_step = .{ .limit = 0.25 } }, - .geglu, - .geglu_tanh, -}; -const all_activations = gated_activations ++ [_]Activation{ .relu, .silu, .gelu }; -const clipped_swiglu = [_]Activation{ - .{ .swiglu = .{ .limit = 0.25 } }, - .{ .swiglu = .{ .limit = 7 } }, -}; - -const FixtureShape = struct { - tokens: i64 = 1, - topk: i64 = 1, - width: i64 = 128, - intermediate: i64 = 128, - experts: i64 = 8, -}; - -// Every option combination runs with 1, 17, and 34 flattened tokens. -// Shapes and floating-point parameter values are representative finite sets. -const fixture_shapes = [_]FixtureShape{ - .{}, - .{ .tokens = 17, .topk = 2 }, - .{ .tokens = 34, .topk = 3 }, -}; -const specialized_shapes = [_]FixtureShape{ - .{ .width = 5120, .intermediate = 2304, .experts = 2 }, - .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 17, .topk = 2 }, - .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 34, .topk = 2 }, -}; -// Exercise direct decode routing with multiple experts; a single route is unsupported. -const specialized_shapes_cute = [_]FixtureShape{ - .{ .width = 5120, .intermediate = 2304, .experts = 2, .tokens = 1, .topk = 2 }, -} ++ specialized_shapes[1..].*; - -const Format = struct { - non_unit_scales: bool = false, - scheme: ?zml.Quantization.Scheme = null, - activation_dtype: zml.DataType = .bf16, -}; - -const Matrix = struct { - formats: []const Format = &.{.{}}, - activations: []const Activation, - quantize_inputs: []const bool = &.{false}, - placements: []const Placement = &.{.after_down}, - biases: []const bool = &.{false}, - shapes: []const FixtureShape = &fixture_shapes, - - pub fn count(self: Matrix) usize { - return self.formats.len * self.activations.len * self.quantize_inputs.len * - self.placements.len * self.biases.len * self.shapes.len; - } - - /// Shapes vary fastest, matching the nested loops in the backend tests. - pub fn caseAt(self: Matrix, backend: zml.moe.Backend, expert_mesh: *const zml.Mesh, index: usize) Case { - std.debug.assert(index < self.count()); - var remaining = index; - const shape = self.shapes[remaining % self.shapes.len]; - remaining /= self.shapes.len; - const bias = self.biases[remaining % self.biases.len]; - remaining /= self.biases.len; - const placement = self.placements[remaining % self.placements.len]; - remaining /= self.placements.len; - const quantize_input = self.quantize_inputs[remaining % self.quantize_inputs.len]; - remaining /= self.quantize_inputs.len; - const activation = self.activations[remaining % self.activations.len]; - remaining /= self.activations.len; - const format = self.formats[remaining]; - return .{ - .backend = backend, - .activation = activation, - .scheme = format.scheme, - .activation_dtype = format.activation_dtype, - .non_unit_scales = format.non_unit_scales, - .quantize_input = quantize_input, - .placement = placement, - .bias = bias, - .tokens = shape.tokens, - .topk = shape.topk, - .width = shape.width, - .intermediate = shape.intermediate, - .experts = shape.experts, - .expert_mesh = expert_mesh, - }; - } -}; - -const Options = struct { - producers: usize = 16, - consumers: usize = 1, - case_queue_capacity: usize = 16, - executable_queue_capacity: usize = 16, -}; - -const CaseJob = struct { - index: usize, - case: Case, -}; - -const ExecutableJob = struct { - job: CaseJob, - exe: zml.Exe, -}; - -const Result = struct { - case: Case, - outcome: union(enum) { - passed, - skipped, - compilation_failed: anyerror, - check_failed: anyerror, - }, -}; - -const Pipeline = struct { - platform: *const zml.Platform, - - fn generateCases(self: *const Pipeline, io: std.Io, backend: zml.moe.Backend, matrix: Matrix, case_queue: *std.Io.Queue(CaseJob), results: []?Result) std.Io.Cancelable!void { - defer case_queue.close(io); - const available = backend.isAvailable(self.platform); - - var index: usize = 0; - for (0..matrix.count()) |local_index| { - const c = matrix.caseAt(backend, self.platform.replicated_mesh, local_index); - if (available) { - case_queue.putOne(io, .{ .index = index, .case = c }) catch |err| return switch (err) { - error.Closed => {}, - error.Canceled => error.Canceled, - }; - } else { - results[index] = .{ .case = c, .outcome = .skipped }; - } - index += 1; - } - } - - fn compileExecutablesWorker(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, case_queue: *std.Io.Queue(CaseJob), executable_queue: *std.Io.Queue(ExecutableJob), results: []?Result) std.Io.Cancelable!void { - while (true) { - const job = case_queue.getOne(io) catch |err| return switch (err) { - error.Closed => {}, - error.Canceled => error.Canceled, - }; - var exe = job.case.compile(allocator, io, self.platform) catch |err| { - if (err == error.Canceled) return error.Canceled; - results[job.index] = .{ .case = job.case, .outcome = .{ .compilation_failed = err } }; - continue; - }; - // Ownership transfers only on a successful enqueue. - executable_queue.putOne(io, .{ .job = job, .exe = exe }) catch |err| { - exe.deinit(); - return switch (err) { - error.Closed => {}, - error.Canceled => error.Canceled, - }; - }; - } - } - - fn runTestCaseWorker(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, executable_queue: *std.Io.Queue(ExecutableJob), results: []?Result) std.Io.Cancelable!void { - while (true) { - var compiled = executable_queue.getOne(io) catch |err| return switch (err) { - error.Closed => {}, - error.Canceled => error.Canceled, - }; - defer compiled.exe.deinit(); - const job = compiled.job; - job.case.check(allocator, io, self.platform, &compiled.exe) catch |err| { - if (err == error.Canceled) return error.Canceled; - results[job.index] = .{ .case = job.case, .outcome = .{ .check_failed = err } }; - continue; - }; - results[job.index] = .{ .case = job.case, .outcome = .passed }; - } - } - - fn run(self: *const Pipeline, io: std.Io, allocator: std.mem.Allocator, backend: zml.moe.Backend, matrix: Matrix, options: Options) !void { - var arena: std.heap.ArenaAllocator = .init(allocator); - defer arena.deinit(); - - const cases_buffer = try arena.allocator().alloc(CaseJob, options.case_queue_capacity); - const executables_buffer = try arena.allocator().alloc(ExecutableJob, options.executable_queue_capacity); - const results = try arena.allocator().alloc(?Result, matrix.count()); - @memset(results, null); - - var case_queue: std.Io.Queue(CaseJob) = .init(cases_buffer); - var executable_queue: std.Io.Queue(ExecutableJob) = .init(executables_buffer); - - var enumerator: std.Io.Group = .init; - var producers: std.Io.Group = .init; - var consumers: std.Io.Group = .init; - defer { - case_queue.close(io); - executable_queue.close(io); - enumerator.cancel(io); - producers.cancel(io); - consumers.cancel(io); - // On partial startup or cancellation, queued executables still own resources. - while (executable_queue.getOneUncancelable(io)) |compiled| { - compiled.exe.deinit(); - } else |_| {} - } - - for (0..options.consumers) |_| try consumers.concurrent(io, runTestCaseWorker, .{ self, io, allocator, &executable_queue, results }); - for (0..options.producers) |_| try producers.concurrent(io, compileExecutablesWorker, .{ self, io, allocator, &case_queue, &executable_queue, results }); - try enumerator.concurrent(io, generateCases, .{ self, io, backend, matrix, &case_queue, results }); - - try enumerator.await(io); - try producers.await(io); - // Closed queues drain before getOne returns error.Closed. - executable_queue.close(io); - try consumers.await(io); - - try report(results); - } - - fn report(results: []const ?Result) !void { - var failed: usize = 0; - for (results, 0..) |result, index| { - const r = result orelse return error.MissingCaseResult; - switch (r.outcome) { - .passed, .skipped => {}, - inline .compilation_failed, .check_failed => |err, stage| { - failed += 1; - std.debug.print("MoE case {}/{} {s}: {any}\nError: {}\n", .{ - index + 1, results.len, @tagName(stage), r.case, err, - }); - }, - } - } - if (failed != 0) return error.MoeComplianceFailed; - } -}; - -test "Triton compatibility matrix" { - const platform = zml.testing.env(); - - const pipeline: Pipeline = .{ .platform = platform }; - - try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ - .formats = &.{ - .{}, - .{ .scheme = .mxfp8 }, - .{ .scheme = .fp8_per_tensor }, - .{ .scheme = .fp8_per_channel }, - .{ .scheme = .fp8_block128 }, - .{ .scheme = .fp8_block32 }, - }, - .activations = &all_activations, - // BF16 ignores this option; exercise both accepted settings. - .quantize_inputs = &.{ false, true }, - .placements = &.{ .before_down, .after_down }, - .biases = &.{ false, true }, - }, .{}); - - if (platform.target != .cuda) { - // Other platforms retain the general MXFP4 path and its full contract. - try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ - .formats = &.{.{ .scheme = .mxfp4 }}, - .activations = &all_activations, - .quantize_inputs = &.{ false, true }, - .placements = &.{ .before_down, .after_down }, - .biases = &.{ false, true }, - }, .{}); - return; - } - - // Preserve the 12 supported cases from the former general MXFP4 matrix. - // Its other 252 cases require activations or projection bias unsupported - // by the simplified path. The former specialized matrix remains below. - try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ - .formats = &.{.{ .scheme = .mxfp4 }}, - .activations = &.{gated_activations[1]}, - .quantize_inputs = &.{ false, true }, - .placements = &.{ .before_down, .after_down }, - }, .{}); -} - -// Keep triton mxfp4 separate because it only supports a subset for now -test "Triton MXFP4 compatibility matrix" { - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.SkipZigTest; - const pipeline: Pipeline = .{ .platform = platform }; - - // The merged backend uses the former specialized MXFP4 path: clipped - // SwiGLU without projection bias, with unconditional MXFP8 activations. - try pipeline.run(std.testing.io, std.testing.allocator, .triton, .{ - .formats = &.{.{ .scheme = .mxfp4 }}, - .activations = &clipped_swiglu, - .quantize_inputs = &.{ false, true }, - .placements = &.{ .before_down, .after_down }, - }, .{}); -} - -test "Triton MoE prefill launch buckets fit shared memory" { - const platform = zml.testing.env(); - if (!zml.moe.Backend.triton.isAvailable(platform) or platform.target != .rocm) return error.SkipZigTest; - - for ([_]i64{ 8, 32, 96, 128, 256, 512, 1024, 2048, 3072, 4096 }) |tokens| { - const case: Case = .{ - .backend = .triton, - .expert_mesh = platform.replicated_mesh, - .tokens = tokens, - .width = 256, - .intermediate = 128, - .topk = 2, - }; - var exe = try case.compile(std.testing.allocator, std.testing.io, platform); - defer exe.deinit(); - try case.check(std.testing.allocator, std.testing.io, platform, &exe); - } -} - -test "FlashInfer compatibility matrix" { - const platform = zml.testing.env(); - - const pipeline: Pipeline = .{ .platform = platform }; - - // Only default SwiGLU, no linear bias, and routing after the down projection. - try pipeline.run(std.testing.io, std.testing.allocator, .flashinfer_cutlass, .{ - .formats = &.{.{ .scheme = .nvfp4 }}, - .activations = &.{.{ .swiglu = .{} }}, - }, .{}); -} - -test "specialized CuTe MXFP4 compatibility matrix" { - const platform = zml.testing.env(); - - const pipeline: Pipeline = .{ .platform = platform }; - - try pipeline.run(std.testing.io, std.testing.allocator, .cute, .{ - .formats = &.{.{ .scheme = .mxfp4 }}, - .activations = &clipped_swiglu, - .quantize_inputs = &.{ false, true }, - .placements = &.{.before_down}, - .shapes = &specialized_shapes_cute, - }, .{}); -} - -test "CuTe NVFP4 compatibility matrix" { - const platform = zml.testing.env(); - if (!zml.moe.cute.isNvfp4Supported(platform)) return error.SkipZigTest; - - const pipeline: Pipeline = .{ .platform = platform }; - try pipeline.run(std.testing.io, std.testing.allocator, .cute, .{ - .formats = &.{ - .{ .scheme = .nvfp4 }, - .{ .scheme = .nvfp4, .non_unit_scales = true }, - }, - .activations = &.{ .{ .swiglu = .{} }, .{ .swiglu = .{ .limit = 0.25 } } }, - .placements = &.{ .before_down, .after_down }, - .shapes = &.{ - .{ .width = 5120, .intermediate = 2304, .tokens = 1, .topk = 2 }, - .{ .width = 5120, .intermediate = 2304, .tokens = 17, .topk = 2 }, - .{ .width = 5120, .intermediate = 2304, .tokens = 34, .topk = 3 }, - }, - }, .{}); -} - -test "Fly compatibility matrix" { - const platform = zml.testing.env(); - - const pipeline: Pipeline = .{ .platform = platform }; - - // These constraints keep execution on Fly rather than its Triton fallback. - try pipeline.run(std.testing.io, std.testing.allocator, .fly, .{ - .formats = &.{.{ .scheme = .mxfp4 }}, - .activations = &gated_activations, - .quantize_inputs = &.{ false, true }, - .placements = &.{.before_down}, - .shapes = &specialized_shapes, - }, .{}); -} - -test "Fly general projection shapes" { - const pipeline: Pipeline = .{ .platform = zml.testing.env() }; - - // Cover GEMV, sorted MFMA with 16- and 32-row tiles, and wider projections. - try pipeline.run(std.testing.io, std.testing.allocator, .fly, .{ - .formats = &.{.{ .scheme = .mxfp4 }}, - .activations = &.{.{ .swiglu = .{} }}, - .placements = &.{.before_down}, - .shapes = &.{ - .{ .width = 384, .intermediate = 128, .experts = 2 }, - .{ .width = 384, .intermediate = 128, .experts = 2, .tokens = 7 }, - .{ .width = 384, .intermediate = 128, .experts = 2, .tokens = 1024 }, - .{ .width = 7168, .intermediate = 2048, .experts = 2 }, - .{ .width = 7168, .intermediate = 2048, .experts = 2, .tokens = 7 }, - }, - }, .{}); -} - -test "Metal compatibility matrix" { - const platform = zml.testing.env(); - - const pipeline: Pipeline = .{ .platform = platform }; - - // Quantized Metal kernels require BF16 activations and do not support bias. - try pipeline.run(std.testing.io, std.testing.allocator, .metal, .{ - .formats = &.{ - .{}, - .{ .activation_dtype = .f16 }, - .{ .activation_dtype = .f32 }, - .{ .scheme = .nvfp4 }, - .{ .scheme = .mxfp8 }, - .{ .scheme = .fp8_per_tensor }, - .{ .scheme = .fp8_per_channel }, - .{ .scheme = .fp8_block128 }, - .{ .scheme = .fp8_block32 }, - }, - .activations = &all_activations, - }, .{}); -} - -test "Mosaic TPU compatibility matrix" { - const platform = zml.testing.env(); - - const pipeline: Pipeline = .{ .platform = platform }; - - // canonicalizeDown currently requires a gated projection; quantized weights - // are not supported by gmmDType. Plain activations need a backend fix first. - try pipeline.run(std.testing.io, std.testing.allocator, .mosaic_tpu, .{ - .formats = &.{ .{}, .{ .activation_dtype = .f16 }, .{ .activation_dtype = .f32 } }, - .activations = &gated_activations, - .biases = &.{ false, true }, - }, .{}); -} diff --git a/zml/kernels/moe/triton/BUILD.bazel b/zml/kernels/moe/triton/BUILD.bazel deleted file mode 100644 index 8dda7f88..00000000 --- a/zml/kernels/moe/triton/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "triton", - srcs = [ - "moe.zig", - "triton.zig", - ], -) diff --git a/zml/kernels/moe/triton/moe.zig b/zml/kernels/moe/triton/moe.zig deleted file mode 100644 index 6e4006fa..00000000 --- a/zml/kernels/moe/triton/moe.zig +++ /dev/null @@ -1,1178 +0,0 @@ -//! Triton MoE kernel declarations — direct ports of the `@triton.jit` -//! functions in `triton_kernels/moe.py`. Each kernel here is a -//! `tri.Kernel(...)` declaration with its own inline config struct (no -//! defaults — every field is required at call time). Production callers -//! invoke these via `K.call(inputs, outputs, opts)`; offline tooling reaches the TTIR -//! string via `K.emit(allocator, ctx, cfg)`. - -const std = @import("std"); - -const zml = @import("../../../zml.zig"); -const tri = zml.kernel.triton; -const Builder = tri.Builder; -const Value = tri.Value; -const DType = tri.DType; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; - -const log = std.log.scoped(.@"zml/moe/triton_kernels"); - -/// Floor to a multiple of 16 — matches the Python `(v // 16) * 16` guards -/// that keep dynamic strides aligned for tt.load/store. -fn blockFloor16(v: Value) Value { - return v.div(16).mul(16); -} - -// ============================================================================= -// per_token_group_quant_fp8 -// ============================================================================= - -/// Direct port of Python `per_token_group_quant_fp8` — per-token-group -/// float8 quantization. -pub const PerTokenGroupQuantFp8 = struct { - pub const Cfg = struct { - input_dtype: DType, - output_dtype: DType, - scale_dtype: DType, - block: usize, - quant_scheme: zml.Quantization.Scheme, - }; - pub const Kernel = tri.Kernel(Cfg, .{ - .name = "per_token_group_quant_fp8", - .inputs = &.{ "y_ptr", "group_size_ptr", "y_num_columns_ptr", "y_row_stride_ptr", "eps_ptr" }, - .outputs = &.{ "y_q", "y_s" }, - .run = run, - }); - fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { - const a = try b.declareArgs(.{ - .y_ptr = .{ .ptr = cfg.input_dtype }, - .group_size_ptr = .{ .ptr = .i64 }, - .y_num_columns_ptr = .{ .ptr = .i64 }, - .y_row_stride_ptr = .{ .ptr = .i64 }, - .eps_ptr = .{ .ptr = .f32 }, - .y_q_ptr = .{ .ptr = cfg.output_dtype }, - .y_s_ptr = .{ .ptr = cfg.scale_dtype }, - }); - - const block: i64 = @intCast(cfg.block); - const out_dt = cfg.output_dtype; - const fp8_max: f32 = switch (out_dt) { - .f8e4m3fn => 448.0, - .f8e4m3fnuz => 224.0, - else => return error.InvalidMlir, - }; - const scale_dt = cfg.scale_dtype; - - const group_size = b.load(a.group_size_ptr); - const y_num_columns = b.load(a.y_num_columns_ptr); - const y_row_stride = b.load(a.y_row_stride_ptr); - const eps = b.load(a.eps_ptr); - - const groups_per_row = y_num_columns.div(group_size); - // g_id is i32 from `tl.program_id`; the i64 promotion only happens - // inside the `(row|row_g_id).to(tl.int64)` casts in the Python source. - // y_s_ptr_shifted reuses the raw i32 `g_id` directly (no extsi). - const pid_i32 = b.programId(.x); - const g_id = pid_i32.to(.i64); - - // y_ptr += (g_id // groups_per_row) * y_row_stride + (g_id % groups_per_row) * group_size - // Python emit order: divsi, remsi, muli (row * stride), muli (row_g_id * group_size), addi. - const row = g_id.div(groups_per_row); - const row_g_id = g_id.rem(groups_per_row); - const row_off = row.mul(y_row_stride); - const grp_off = row_g_id.mul(group_size); - const y_ptr_shifted = a.y_ptr.addPtr(row_off.add(grp_off)); - const y_q_ptr_shifted = a.y_q_ptr.addPtr(g_id.mul(group_size)); - // y_s_ptr += g_id (raw i32, no cast — matches `y_s_ptr += g_id` in Python). - const y_s_ptr_shifted = a.y_s_ptr.addPtr(pid_i32); - - // cols = arange(0, BLOCK) — i64 for both pointer offsets and mask - // comparison against the i64 `group_size` (matches Python's - // `cols < group_size` after Triton's auto-promotion). - const cols_i32 = b.arange(0, block, .i32); - const cols_i64 = cols_i32.to(.i64); - const mask = cols_i64.lt(group_size); - - // Python: `tl.load(y_ptr + cols, mask=mask, other=0.0)` — explicit - // 3-operand form. Use `.other` to materialize the const-zero tensor - // (without it, `tt.load` is 2-operand and Python's emit doesn't match). - const y_load_ptrs = y_ptr_shifted.addPtr(cols_i64); - const y_other = b.zeros(&.{block}, cfg.input_dtype); - const y = b.loadOpts(y_load_ptrs, .{ .mask = mask, .other = y_other }).to(.f32); - - // _absmax = max(tl.max(tl.abs(y)), eps) — `b.max(...).maximum(eps)` - // resolves to `arith.maxnumf` (matches Python's `tl.maximum`). - const absmax = b.max(b.absf(y)).maximum(eps); - - // scale_raw = _absmax * (1.0 / fp8_max) - const scale_raw = absmax.mul(1.0 / fp8_max); - const y_s: Value, const stored_scale: Value = switch (cfg.quant_scheme) { - .mxfp8 => blk: { - const exponent = b.ceil(b.log2(scale_raw)); - // E8M0 stores the biased exponent as a byte. - break :blk .{ b.exp2(exponent), exponent.add(127).to(.i32).to(.i8) }; - }, - .fp8_per_channel, .fp8_per_tensor, .fp8_block128 => .{ scale_raw, scale_raw.to(scale_dt) }, - .fp8_block32 => blk: { - // Match quantizeBlockFp8(..., .f8e8m0), but store the scales as FP32 here. - // It is unclear whether power-of-two scaling is best for this kernel, - // but it keeps activation quantization consistent with that function. - // Round up from the FP32 bits, including values just above a power of two. - const bits = scale_raw.bitCast(.i32); - const exponent = bits.div(0x800000); - const fractional = bits.bitAnd(0x7fffff).ne(0).to(.i32); - const scale = exponent.add(fractional).mul(0x800000).bitCast(.f32); - break :blk .{ scale, scale.to(scale_dt) }; - }, - .mxfp4, .nvfp4 => unreachable, - }; - - // y_q = clamp(y / y_s, fp8_min, fp8_max).to(output_dtype) - const y_div = y.div(y_s); - const clamped = b.clampf( - y_div, - b.splat(-fp8_max, &.{block}), - b.splat(fp8_max, &.{block}), - ); - const y_q = clamped.to(out_dt); - - const y_q_ptrs = y_q_ptr_shifted.addPtr(cols_i64); - b.storeOpts(y_q_ptrs, y_q, .{ .mask = mask }); - b.store(y_s_ptr_shifted, stored_scale); - } -}; - -// ============================================================================= -// write_zeros_to_output — helper called by FusedMoe -// ============================================================================= - -/// Direct port of Python `write_zeros_to_output`. Stores a masked zero-tile -/// for blocks whose expert is `-1` (no expert assigned). `stride_cn` is -/// hardcoded to 1 (contiguous last dim), so the N-axis offset is just -/// `offs_cn` itself. -fn writeZerosToOutput( - b: *Builder, - c_ptr: Value, - stride_cm: Value, - pid_n: Value, - n: Value, - offs_token: Value, - token_mask: Value, - block_size_m: i64, - block_size_n: i64, - compute_type: DType, -) void { - const accumulator = b.zeros(&.{ block_size_m, block_size_n }, compute_type); - const offs_cn = pid_n.mul(block_size_n).add(b.arange(0, block_size_n, .i64)); - - // c_ptrs in two addptrs to match Python's `c_ptr + cm[:, None] + cn[None, :]` - // left-to-right evaluation. `stride_cm * offs_token[:, None]` puts the - // splatted stride on the LHS of arith.muli (Python source order). - const offs_token_col = b.expandDims(offs_token, 1); - const cm_col = stride_cm.mul(offs_token_col); - const c_ptrs_col = b.splat(c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); - const offs_cn_row = b.expandDims(offs_cn, 0); - const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); - const cn_off_2d = b.broadcastTo(offs_cn_row, &.{ block_size_m, block_size_n }); - const c_ptrs = c_ptrs_2d.addPtr(cn_off_2d); - - // c_mask: Python emit order — expand_dims, splat, cmpi, broadcast, broadcast, andi. - const token_mask_col = b.expandDims(token_mask, 1); - const n_splat = b.splat(n, &.{ 1, block_size_n }); - const offs_cn_lt_2d = offs_cn_row.lt(n_splat); - const token_mask_full = b.broadcastTo(token_mask_col, &.{ block_size_m, block_size_n }); - const offs_cn_lt_full = b.broadcastTo(offs_cn_lt_2d, &.{ block_size_m, block_size_n }); - b.storeOpts(c_ptrs, accumulator, .{ .mask = token_mask_full.bitAnd(offs_cn_lt_full) }); -} - -// ============================================================================= -// fused_moe_kernel -// ============================================================================= - -/// Routed GEMM supporting BF16, MXFP4, MXFP8, and per-tensor, per-channel, or -/// 128x128 block-scaled FP8 weights. -pub const FusedMoe = struct { - pub const Cfg = struct { - a_dtype: DType, - b_dtype: DType, - c_dtype: DType, - a_scale_dtype: ?DType, - b_scale_dtype: ?DType, - b_bias_dtype: ?DType, - routing_weights_dtype: ?DType, - block_size_m: usize, - block_size_n: usize, - block_size_k: usize, - group_size_m: usize, - top_k: usize, - naive_block_assignment: bool, - compute_type: DType, - quant_scheme: ?zml.Quantization.Scheme, - }; - pub const Kernel = tri.Kernel(Cfg, .{ - .name = "fused_moe_kernel", - .inputs = &.{ - "a_ptr", "b_ptr", "b_bias_ptr", "a_scale_ptr", "b_scale_ptr", - "routing_weights_ptr", "sorted_token_ids_ptr", "expert_ids_ptr", "num_tokens_post_padded_ptr", "N_ptr", - "K_ptr", "EM_ptr", "num_valid_tokens_ptr", "stride_am_ptr", "stride_be_ptr", - "stride_bn_ptr", "stride_cm_ptr", "stride_asm_ptr", "stride_ask_ptr", "stride_bse_ptr", - "stride_bsk_ptr", "stride_bsn_ptr", "stride_bbe_ptr", "stride_bbn_ptr", - }, - .outputs = &.{"c"}, - .run = run, - }); - fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { - const fp8_block_size: usize = if (cfg.quant_scheme == .fp8_block32) 32 else 128; - if ((cfg.quant_scheme == .fp8_block128 or cfg.quant_scheme == .fp8_block32) and cfg.block_size_k != fp8_block_size) { - log.err("fused_moe_kernel: block-scaled FP8 requires BLOCK_SIZE_K={}", .{fp8_block_size}); - return error.InvalidMlir; - } - - if ((cfg.quant_scheme == .mxfp4 or cfg.quant_scheme == .mxfp8) and cfg.block_size_k % 32 != 0) { - log.err("fused_moe_kernel: MXFP4/MXFP8 require BLOCK_SIZE_K divisible by 32", .{}); - return error.InvalidMlir; - } - - const a = try b.declareArgs(.{ - .a_ptr = .{ .ptr = cfg.a_dtype }, - .b_ptr = .{ .ptr = cfg.b_dtype }, - // The fixed signature requires concrete pointer types even for absent, - // unused inputs. FP32 matches the caller's placeholder tensors. - .b_bias_ptr = .{ .ptr = cfg.b_bias_dtype orelse .f32 }, - .a_scale_ptr = .{ .ptr = cfg.a_scale_dtype orelse .f32 }, - .b_scale_ptr = .{ .ptr = cfg.b_scale_dtype orelse .f32 }, - .routing_weights_ptr = .{ .ptr = cfg.routing_weights_dtype orelse .f32 }, - .sorted_token_ids_ptr = .{ .ptr = .i32 }, - .expert_ids_ptr = .{ .ptr = .i32 }, - .num_tokens_post_padded_ptr = .{ .ptr = .i32 }, - .N_ptr = .{ .ptr = .i64 }, - .K_ptr = .{ .ptr = .i64 }, - .EM_ptr = .{ .ptr = .i64 }, - .num_valid_tokens_ptr = .{ .ptr = .i64 }, - .stride_am_ptr = .{ .ptr = .i64 }, - .stride_be_ptr = .{ .ptr = .i64 }, - .stride_bn_ptr = .{ .ptr = .i64 }, - .stride_cm_ptr = .{ .ptr = .i64 }, - .stride_asm_ptr = .{ .ptr = .i64 }, - .stride_ask_ptr = .{ .ptr = .i64 }, - .stride_bse_ptr = .{ .ptr = .i64 }, - .stride_bsk_ptr = .{ .ptr = .i64 }, - .stride_bsn_ptr = .{ .ptr = .i64 }, - .stride_bbe_ptr = .{ .ptr = .i64 }, - .stride_bbn_ptr = .{ .ptr = .i64 }, - .c_ptr = .{ .ptr = cfg.c_dtype }, - }); - - const block_size_m: i64 = @intCast(cfg.block_size_m); - const block_size_n: i64 = @intCast(cfg.block_size_n); - const block_size_k: i64 = @intCast(cfg.block_size_k); - const weight_packing: i64 = if (cfg.quant_scheme == .mxfp4) 2 else 1; - const block_size_bk = @divExact(block_size_k, weight_packing); - const group_size_m: i64 = @intCast(cfg.group_size_m); - const top_k: i64 = @intCast(cfg.top_k); - const compute_type = cfg.compute_type; - - // Matrix dimensions, strides, and padded routing capacity are checked - // for 16-element alignment by the caller. - const n_raw = b.load(a.N_ptr); - const k_raw = b.load(a.K_ptr); - const em_raw = b.load(a.EM_ptr); - const num_valid_tokens = b.load(a.num_valid_tokens_ptr); - const stride_am_raw = b.load(a.stride_am_ptr); - const stride_be_raw = b.load(a.stride_be_ptr); - const stride_bn_raw = b.load(a.stride_bn_ptr); - const stride_cm_raw = b.load(a.stride_cm_ptr); - const n_block = blockFloor16(n_raw); - const k_block = blockFloor16(k_raw); - const em_block = blockFloor16(em_raw); - const stride_am_block = blockFloor16(stride_am_raw); - const stride_be_block = blockFloor16(stride_be_raw); - const stride_bn_block = blockFloor16(stride_bn_raw); - const stride_cm_block = blockFloor16(stride_cm_raw); - // stride_ak / stride_bk / stride_cn are all 1 in this port — the A/B/C - // inner dimension is contiguous, so those mul-by-1 terms are elided. - - // pid grouping. Python keeps `pid` as i32 and lets the divsi auto- - // promote at the use site — emit the cdivs first, then extsi at the - // first i64 use. Manual order keeps `extsi pid` after the cdivs in TTIR. - const pid_i32 = b.programId(.x); - const num_pid_m = em_block.cdiv(block_size_m); - const num_pid_n = n_block.cdiv(block_size_n); - const num_pid_in_group = num_pid_n.mul(group_size_m); - const pid = pid_i32.to(.i64); - const group_id = pid.div(num_pid_in_group); - const first_pid_m = group_id.mul(group_size_m); - const gsm_actual = num_pid_m.sub(first_pid_m).minimum(group_size_m); - const pid_mod_in_group = pid.rem(num_pid_in_group); - const pid_m = first_pid_m.add(pid_mod_in_group.rem(gsm_actual)); - const pid_n = pid_mod_in_group.div(gsm_actual); - - // Python source order: arange first, then load + early-return. - // Match emit order so the make_range op lands before the cf.cond_br. - const offs = b.arange(0, block_size_m, .i64); - // Order: load (i32), muli, extsi (Python emits the muli before the - // i32→i64 extsi at the cmpi). - const num_tokens_post_padded_i32 = b.load(a.num_tokens_post_padded_ptr); - const pid_m_block = pid_m.mul(block_size_m); - const num_tokens_post_padded = num_tokens_post_padded_i32.to(.i64); - const out_of_range = pid_m_block.ge(num_tokens_post_padded); - - b.returnIfOpts(out_of_range, .{ .inline_return = true }); - - const offs_token = if (cfg.naive_block_assignment) - b.select( - offs.eq(0), - pid_m.splatTo(&.{block_size_m}), - num_valid_tokens.splatTo(&.{block_size_m}), - ) - else off: { - const ids_ptrs = a.sorted_token_ids_ptr - .addPtr(pid_m.mul(block_size_m).add(offs)); - break :off b.load(ids_ptrs).to(.i64); - }; - const token_mask = offs_token.lt(num_valid_tokens); - - // off_experts = expert_ids[pid_m].to(i64). pid_m is i64; Python's - // `expert_ids_ptr + pid_m` keeps the offset as i64. - const off_experts = b.load(a.expert_ids_ptr.addPtr(pid_m)).to(.i64); - const is_dead = off_experts.eq(-1); - - // Python: if off_experts == -1: write_zeros_to_output(...); return - var dead_ret = b.openReturnIfOpts(is_dead, .{ .inline_return = true }); - { - writeZerosToOutput( - b, - a.c_ptr, - stride_cm_block, - pid_n, - n_block, - offs_token, - token_mask, - block_size_m, - block_size_n, - compute_type, - ); - dead_ret.close(); - } - - // CSE `pid_n * BLOCK_SIZE_N` so it reuses across offs_bn / offs_cn. - const pid_n_block_n = pid_n.mul(block_size_n); - const offs_bn = pid_n_block_n.add(b.arange(0, block_size_n, .i64)).rem(n_block); - - // offs_k stays i32 through expand_dims; extsi to i64 happens on the 2D form. - const offs_k_i32 = b.arange(0, block_size_k, .i32); - - // a_ptrs: `offs_token[:, None] // top_k * stride_am + offs_k[None, :]`. - // (stride_ak == 1 here, so `* stride_ak` folds away.) Build with explicit - // broadcastTo to match Python's emit shape. - const offs_token_col = b.expandDims(offs_token, 1); - const am_col = offs_token_col.div(top_k).mul(stride_am_block); - const offs_k_row = b.expandDims(offs_k_i32, 0).to(.i64); - const am_term = b.broadcastTo(am_col, &.{ block_size_m, block_size_k }); - const ak_term = b.broadcastTo(offs_k_row, &.{ block_size_m, block_size_k }); - const a_ptrs_init = a.a_ptr.addPtr(am_term.add(ak_term)); - - // b_ptrs: `b_ptr + off_experts * stride_be` (scalar addptr) THEN - // `+ (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)` (2D - // addptr) — matches Python's left-to-right evaluation. Defer extsi - // for offs_k to after offs_bn's expand_dims/mul (Python emit order). - const b_ptr_shifted = a.b_ptr.addPtr(off_experts.mul(stride_be_block)); - // MXFP4 packs two K values per byte; activation K remains unpacked. - const offs_bk_i32 = b.arange(0, block_size_bk, .i32); - const offs_k_col_i32 = b.expandDims(offs_bk_i32, 1); - const offs_bn_row = b.expandDims(offs_bn, 0); - const bn_row = offs_bn_row.mul(stride_bn_block); - const offs_k_col = offs_k_col_i32.to(.i64); - const bk_term = b.broadcastTo(offs_k_col, &.{ block_size_bk, block_size_n }); - const bn_term = b.broadcastTo(bn_row, &.{ block_size_bk, block_size_n }); - const b_ptrs_init = b_ptr_shifted.addPtr(bk_term.add(bn_term)); - - const acc_init = b.zeros(&.{ block_size_m, block_size_n }, .f32); - - // Loop bounds stay i64 because `k_block` is i64. iter_args order - // matches Python's TTIR for the matrix and accumulator state. - const num_k_iters = k_block.cdiv(block_size_k); - var loop = b.openFor(0, num_k_iters, 1, .{ a_ptrs_init, b_ptrs_init, acc_init }); - { - const k_iter = loop.iv; - const a_ptrs = loop.carried[0]; - const b_ptrs = loop.carried[1]; - const acc = loop.carried[2]; - - const k_remaining = k_block.sub(k_iter.mul(block_size_k)); - - // mask_a = token_mask[:, None] & (offs_k[None, :] < k_remaining). - const token_mask_col_a = b.expandDims(token_mask, 1); - const offs_k_row_a = b.expandDims(offs_k_i32, 0).to(.i64); - const offs_k_lt_a = offs_k_row_a.lt(b.splat(k_remaining, &.{ 1, block_size_k })); - const token_mask_full_a = b.broadcastTo(token_mask_col_a, &.{ block_size_m, block_size_k }); - const offs_k_lt_a_full = b.broadcastTo(offs_k_lt_a, &.{ block_size_m, block_size_k }); - const mask_a = token_mask_full_a.bitAnd(offs_k_lt_a_full); - - const a_val = b.loadOpts(a_ptrs, .{ - .mask = mask_a, - .other = b.zeros(&.{ block_size_m, block_size_k }, cfg.a_dtype), - }); - - const offs_k_col_b = b.expandDims(offs_bk_i32, 1).to(.i64); - const offs_k_lt_b = offs_k_col_b.lt(b.splat(k_remaining.cdiv(weight_packing), &.{ block_size_bk, 1 })); - const mask_b = b.broadcastTo(offs_k_lt_b, &.{ block_size_bk, block_size_n }); - - const b_val = b.loadOpts(b_ptrs, .{ - .mask = mask_b, - .other = b.zeros(&.{ block_size_bk, block_size_n }, cfg.b_dtype), - }); - - const new_acc = if (cfg.quant_scheme) |scheme| switch (scheme) { - .mxfp4, .mxfp8 => scaled: { - const scale_k = @divExact(block_size_k, 32); - const groups = k_iter.mul(scale_k).add(b.arange(0, scale_k, .i64)); - const stride_bse = b.load(a.stride_bse_ptr); - const stride_bsk = b.load(a.stride_bsk_ptr); - const stride_bsn = b.load(a.stride_bsn_ptr); - // dot_scaled expects RHS scales in [N, K/32] order. - const rows = off_experts.mul(stride_bse).add(offs_bn.mul(stride_bsn)).expandDims(1); - const cols = groups.mul(stride_bsk).expandDims(0); - const offsets = b.broadcastTo(rows, &.{ block_size_n, scale_k }) - .add(b.broadcastTo(cols, &.{ block_size_n, scale_k })); - const n_mask = b.broadcastTo(offs_bn.lt(n_block).expandDims(1), &.{ block_size_n, scale_k }); - const k_mask = b.broadcastTo(groups.lt(k_block.cdiv(32)).expandDims(0), &.{ block_size_n, scale_k }); - const scales = b.loadOpts(a.b_scale_ptr.addPtr(offsets), .{ - .mask = n_mask.bitAnd(k_mask), - .other = b.full(&.{ block_size_n, scale_k }, 127, .i8), - }); - const a_scales: ?Value = if (cfg.a_scale_dtype != null) input_scales: { - const a_rows = offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)).expandDims(1); - const a_cols = groups.mul(b.load(a.stride_ask_ptr)).expandDims(0); - const a_offsets = b.broadcastTo(a_rows, &.{ block_size_m, scale_k }) - .add(b.broadcastTo(a_cols, &.{ block_size_m, scale_k })); - const a_mask = b.broadcastTo(token_mask.expandDims(1), &.{ block_size_m, scale_k }) - .bitAnd(b.broadcastTo(groups.lt(k_block.cdiv(32)).expandDims(0), &.{ block_size_m, scale_k })); - break :input_scales b.loadOpts(a.a_scale_ptr.addPtr(a_offsets), .{ - .mask = a_mask, - .other = b.full(&.{ block_size_m, scale_k }, 127, .i8), - }); - } else null; - break :scaled b.dotScaledOpts(a_val, b_val, acc, a_scales, scales, if (a_scales != null) .e4m3 else .bf16, if (scheme == .mxfp4) .e2m1 else .e4m3, .{}); - }, - .fp8_per_channel, .fp8_per_tensor => b.dotOpts(a_val, if (cfg.a_scale_dtype != null) b_val else b_val.to(.bf16), acc, .{ - .input_precision = .tf32, - .max_num_imprecise_acc = 0, - }), - .fp8_block32, .fp8_block128 => scaled: { - // Activation scales are [token, K/block] and weight scales are - // [expert, N/block, K/block]. Compute their addresses from the - // loop index; activation scales are optional. - const stride_bse = b.load(a.stride_bse_ptr); - const stride_bsk = b.load(a.stride_bsk_ptr); - const stride_bsn = b.load(a.stride_bsn_ptr); - const b_scale_ptrs = a.b_scale_ptr.addPtr( - off_experts.mul(stride_bse) - .add(offs_bn.div(fp8_block_size).mul(stride_bsn)) - .add(k_iter.mul(stride_bsk)), - ); - const dot = b.dotOpts(a_val, if (cfg.a_scale_dtype != null) b_val else b_val.to(.bf16), b.zeros(&.{ block_size_m, block_size_n }, .f32), .{ - .input_precision = .tf32, - .max_num_imprecise_acc = 0, - }); - const b_s = b.loadOpts(b_scale_ptrs, .{ - .mask = offs_bn.lt(n_block), - .other = b.zeros(&.{block_size_n}, cfg.b_scale_dtype orelse .f32), - }).to(.f32); - var scaled_dot = dot.mul(b_s.expandDims(0)); - if (cfg.a_scale_dtype) |dtype| { - const a_scale_ptrs = a.a_scale_ptr.addPtr( - offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)) - .add(k_iter.mul(b.load(a.stride_ask_ptr))), - ); - const a_s = b.loadOpts(a_scale_ptrs, .{ - .mask = token_mask, - .other = b.zeros(&.{block_size_m}, dtype), - }).to(.f32); - scaled_dot = scaled_dot.mul(a_s.expandDims(1)); - } - break :scaled acc.add(scaled_dot); - }, - .nvfp4 => unreachable, - } else b.dotOpts(a_val, b_val, acc, .{ - .input_precision = .tf32, - .max_num_imprecise_acc = 0, - }); - - // BLOCK_SIZE_K is `tl.constexpr` → i32 dense splat (stride_ak == 1). - const bsk_i32: i32 = @intCast(cfg.block_size_k); - const new_a_ptrs = a_ptrs.addPtr(b.splat(bsk_i32, &.{ block_size_m, block_size_k })); - const new_b_ptrs = b_ptrs.addPtr(b.splat(@as(i32, @intCast(block_size_bk)), &.{ block_size_bk, block_size_n })); - - loop.yield(.{ - new_a_ptrs, - new_b_ptrs, - new_acc, - }); - } - var accumulator = loop.results[2]; - - if (cfg.quant_scheme == .fp8_per_channel or cfg.quant_scheme == .fp8_per_tensor) { - // These scales are constant along K, so apply them after accumulation. - const expert_offset = off_experts.mul(b.load(a.stride_bse_ptr)); - const offsets = if (cfg.quant_scheme == .fp8_per_channel) - expert_offset.add(offs_bn.mul(b.load(a.stride_bsn_ptr))) - else - b.splat(expert_offset, &.{block_size_n}); - const scales = b.loadOpts(a.b_scale_ptr.addPtr(offsets), .{ - .mask = offs_bn.lt(n_block), - .other = b.zeros(&.{block_size_n}, cfg.b_scale_dtype.?), - }).to(.f32); - accumulator = accumulator.mul(scales.expandDims(0)); - if (cfg.a_scale_dtype) |dtype| { - const offsets_a = offs_token.div(top_k).mul(b.load(a.stride_asm_ptr)); - const scales_a = b.loadOpts(a.a_scale_ptr.addPtr(offsets_a), .{ - .mask = token_mask, - .other = b.zeros(&.{block_size_m}, dtype), - }).to(.f32); - accumulator = accumulator.mul(scales_a.expandDims(1)); - } - } - - if (cfg.b_bias_dtype) |dtype| { - const offsets = off_experts.mul(b.load(a.stride_bbe_ptr)) - .add(offs_bn.mul(b.load(a.stride_bbn_ptr))); - const bias = b.loadOpts(a.b_bias_ptr.addPtr(offsets), .{ - .mask = offs_bn.lt(n_block), - .other = b.zeros(&.{block_size_n}, dtype), - }).to(.f32); - // Bias is in output units and must receive the routing weight too. - accumulator = accumulator.add(bias.expandDims(0)); - } - - if (cfg.routing_weights_dtype) |dtype| { - const other = b.zeros(&.{block_size_m}, dtype); - const routing_weights = b.loadOpts(a.routing_weights_ptr.addPtr(offs_token), .{ .mask = token_mask, .other = other }); - accumulator = accumulator.mul(routing_weights.to(.f32).expandDims(1)); - } - - accumulator = accumulator.to(compute_type); - - // c_ptrs: two addptrs to match Python's `c_ptr + cm[:, None] + cn[None, :]` - // left-to-right eval. `stride_cm * offs_token[:, None]` puts the splatted - // stride on the LHS of arith.muli (Python source order). - const offs_cn = pid_n_block_n.add(b.arange(0, block_size_n, .i64)); - const offs_token_col_c = b.expandDims(offs_token, 1); - const cm_col = stride_cm_block.mul(offs_token_col_c); - const c_ptrs_col = b.splat(a.c_ptr, &.{ block_size_m, 1 }).addPtr(cm_col); - const offs_cn_row = b.expandDims(offs_cn, 0); - const c_ptrs_2d = b.broadcastTo(c_ptrs_col, &.{ block_size_m, block_size_n }); - const c_ptrs = c_ptrs_2d.addPtr(b.broadcastTo(offs_cn_row, &.{ block_size_m, block_size_n })); - - // c_mask = token_mask[:, None] & (offs_cn[None, :] < n_block). - const token_mask_col_c = b.expandDims(token_mask, 1); - const offs_cn_lt_2d = offs_cn_row.lt(b.splat(n_block, &.{ 1, block_size_n })); - const token_mask_full_c = b.broadcastTo(token_mask_col_c, &.{ block_size_m, block_size_n }); - const offs_cn_lt_full = b.broadcastTo(offs_cn_lt_2d, &.{ block_size_m, block_size_n }); - b.storeOpts(c_ptrs, accumulator, .{ .mask = token_mask_full_c.bitAnd(offs_cn_lt_full) }); - } -}; - -test "FP8 activation quantization emits float and E8M0 scales" { - for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_channel, .fp8_per_tensor, .fp8_block128, .fp8_block32 }) |scheme| { - for ([_]DType{ .bf16, .f32 }) |input_dtype| { - const ir = try PerTokenGroupQuantFp8.Kernel.emit(std.testing.allocator, .{ - .input_dtype = input_dtype, - .output_dtype = .f8e4m3fn, - .scale_dtype = if (scheme == .mxfp8) .i8 else .f32, - .block = if (scheme == .mxfp8 or scheme == .fp8_block32) 32 else 256, - .quant_scheme = scheme, - }); - defer std.testing.allocator.free(ir); - if (scheme == .mxfp8) { - try std.testing.expect(std.mem.indexOf(u8, ir, "arith.trunci") != null); - } - } - } -} - -test "FusedMoe emits each FP8 and MXFP4 scaling path" { - const allocator = std.testing.allocator; - const schemes = [_]?zml.Quantization.Scheme{ null, .mxfp4, .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128, .fp8_block32 }; - for (schemes) |scheme| { - for ([_]DType{ .f8e4m3fn, .f8e4m3fnuz }) |fp8_dtype| { - if ((scheme == .mxfp4 or scheme == .mxfp8) and fp8_dtype == .f8e4m3fnuz) continue; - for ([_]bool{ false, true }) |quantize_input| { - if ((scheme == null or scheme == .mxfp4) and quantize_input) continue; - for ([_]?DType{ null, .bf16, .f32 }) |bias_dtype| { - const ir = try FusedMoe.Kernel.emit(allocator, .{ - .a_dtype = if (quantize_input) fp8_dtype else .bf16, - .b_dtype = if (scheme == .mxfp4) .i8 else if (scheme != null) fp8_dtype else .bf16, - .c_dtype = .bf16, - .a_scale_dtype = if (quantize_input) (if (scheme == .mxfp8) .i8 else .f32) else null, - .b_scale_dtype = if (scheme == .mxfp4 or scheme == .mxfp8) .i8 else if (scheme != null) .f32 else null, - .b_bias_dtype = bias_dtype, - .routing_weights_dtype = .bf16, - .block_size_m = 16, - .block_size_n = 32, - .block_size_k = if (scheme == .fp8_block32) 32 else 128, - .group_size_m = 1, - .top_k = 2, - .naive_block_assignment = false, - .compute_type = .bf16, - .quant_scheme = scheme, - }); - defer allocator.free(ir); - if (scheme == .mxfp4 or scheme == .mxfp8) { - try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot_scaled") != null); - } else { - try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot_scaled") == null); - try std.testing.expect(std.mem.indexOf(u8, ir, "tt.dot") != null); - if (scheme != null and !quantize_input) { - try std.testing.expect(std.mem.indexOf(u8, ir, "tt.fp_to_fp") != null); - } - } - } - } - } - } -} - -// ============================================================================= -// moe_align_block_size_kernel -// ============================================================================= - -/// Direct port of Python `moe_align_block_size_kernel`. pid==0 runs the -/// histogram + cumsum + block-to-expert assignment pass; pid==1 fills -/// sorted_token_ids with NUMEL. -/// -/// Block assignment are set up to `num_tokens_post_padded` and the remaining blocks of -/// `expert_ids` are not set to a sentinel value. -pub const MoeAlignBlockSize = struct { - pub const Cfg = struct { - numel: usize, - num_experts: usize, - padded_num_experts: usize, - max_num_tokens_padded: usize, - max_num_m_blocks: usize, - block_size_m: usize, - hist_block: usize, - }; - pub const Kernel = tri.Kernel(Cfg, .{ - .name = "moe_align_block_size_kernel", - .inputs = &.{ "topk_ids_ptr", "sorted_token_ids_ptr", "expert_ids_ptr", "num_tokens_post_pad_ptr", "cumsum_ptr" }, - .outputs = &.{ "sorted_token_ids", "expert_ids", "num_tokens_post_pad", "cumsum" }, - .run = run, - }); - fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { - const a = try b.declareArgs(.{ - .topk_ids_ptr = .{ .ptr = .i32 }, - .sorted_token_ids_ptr = .{ .ptr = .i32 }, - .expert_ids_ptr = .{ .ptr = .i32 }, - .num_tokens_post_pad_ptr = .{ .ptr = .i32 }, - .cumsum_ptr = .{ .ptr = .i32 }, - .out0_ptr = .{ .ptr = .i32 }, - .out1_ptr = .{ .ptr = .i32 }, - .out2_ptr = .{ .ptr = .i32 }, - .out3_ptr = .{ .ptr = .i32 }, - }); - - const block_size_m: i32 = @intCast(cfg.block_size_m); - const numel: i32 = @intCast(cfg.numel); - const num_experts: i32 = @intCast(cfg.num_experts); - const padded_num_experts: i64 = @intCast(cfg.padded_num_experts); - const max_num_tokens_padded: i32 = @intCast(cfg.max_num_tokens_padded); - const hist_block: i64 = @intCast(cfg.hist_block); - - const pid = b.programId(.x); - const fill_offs = b.arange(0, hist_block, .i32); - - // if pid == 1: fill sorted_token_ids with NUMEL; return - var fill_branch = b.openReturnIfOpts(pid.eq(1), .{ .inline_return = true }); - { - var fill_loop = b.openFor(0, max_num_tokens_padded, hist_block, .{}); - { - const offs = fill_loop.iv.add(fill_offs); - const mask = offs.lt(max_num_tokens_padded); - b.storeOpts( - a.sorted_token_ids_ptr.addPtr(offs), - b.splat(numel, &.{hist_block}), - .{ .mask = mask }, - ); - fill_loop.close(); - } - fill_branch.close(); - } - - // pid != 1: histogram + cumsum + block assignment. - const expert_offs = b.arange(0, padded_num_experts, .i32); - const token_offs = b.arange(0, hist_block, .i32); - const expert_mask = expert_offs.lt(num_experts); - const counts_init = b.zeros(&.{padded_num_experts}, .i32); - - var hist_loop = b.openFor(0, numel, hist_block, .{counts_init}); - { - const offs = hist_loop.iv.add(token_offs); - const mask = offs.lt(numel); - const expert_vals = b.loadOpts(a.topk_ids_ptr.addPtr(offs), .{ - .mask = mask, - .other = b.splat(num_experts, &.{hist_block}), - }); - const valid = mask.bitAnd(expert_vals.lt(num_experts)); - const h = b.histogramOpts(expert_vals, padded_num_experts, .{ .mask = valid }); - hist_loop.yield(.{hist_loop.carried[0].add(h)}); - } - const counts = hist_loop.results[0]; - - // padded_counts = where(expert_mask, cdiv(counts, BLOCK_SIZE_M) * BLOCK_SIZE_M, 0) - const padded_counts = b.select( - expert_mask, - counts.cdiv(block_size_m).mul(block_size_m), - b.zeros(&.{padded_num_experts}, .i32), - ); - const padded_cumsum = b.cumsumOpts(padded_counts, .{ .axis = 0 }); - const starts = padded_cumsum.sub(padded_counts); - const total_tokens_post_pad = b.sumOpts(padded_counts, .{ .axis = 0 }); - - b.storeOpts(a.cumsum_ptr.addPtr(expert_offs), starts, .{ .mask = expert_mask }); - b.store(a.cumsum_ptr.addPtr(num_experts), total_tokens_post_pad); - b.store(a.num_tokens_post_pad_ptr, total_tokens_post_pad); - - // Each expert owns a disjoint padded range, including empty ranges. - // Scatter from register-resident prefixes; no cross-lane global reads. - const block_counts = padded_counts.div(block_size_m); - const max_blocks = b.maxOpts(block_counts, .{ .axis = 0 }); - var scatter = b.openFor(0, max_blocks, 1, .{}); - { - const block_ids = starts.div(block_size_m).add(scatter.iv); - const mask = expert_mask.bitAnd(block_counts.gt(scatter.iv)); - b.storeOpts(a.expert_ids_ptr.addPtr(block_ids), expert_offs, .{ .mask = mask }); - scatter.close(); - } - } -}; - -// ============================================================================= -// count_and_sort_expert_tokens_kernel -// ============================================================================= - -/// Direct port of Python `count_and_sort_expert_tokens_kernel`. Each program -/// atomically bumps `cumsum[expert]`, writing its block's token offsets into -/// `sorted_token_ids` at the returned rank. -pub const CountAndSortExpertTokens = struct { - pub const Cfg = struct { - numel: usize, - num_experts: usize, - sort_block_size: usize, - }; - pub const Kernel = tri.Kernel(Cfg, .{ - .name = "count_and_sort_expert_tokens_kernel", - .inputs = &.{ "topk_ids_ptr", "sorted_token_ids_ptr", "cumsum_ptr" }, - .outputs = &.{ "sorted_token_ids", "cumsum" }, - .run = run, - }); - fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { - const a = try b.declareArgs(.{ - .topk_ids_ptr = .{ .ptr = .i32 }, - .sorted_token_ids_ptr = .{ .ptr = .i32 }, - .cumsum_ptr = .{ .ptr = .i32 }, - .out0_ptr = .{ .ptr = .i32 }, - .out1_ptr = .{ .ptr = .i32 }, - }); - - const block: i64 = @intCast(cfg.sort_block_size); - const numel: i32 = @intCast(cfg.numel); - const num_experts: i32 = @intCast(cfg.num_experts); - const block_i32: i32 = @intCast(block); - - const pid = b.programId(.x); - const num_progs = b.numPrograms(.x); - const token_offs = b.makeRange(0, block); - - const token_start_init = pid.mul(block_i32); - - // while token_start < NUMEL: - var w = b.openWhile(.{token_start_init}, .{.i32}); - { - const ts = w.before_carried[0]; - w.yieldBefore(ts.lt(numel), .{ts}); - } - { - const ts = w.after_carried[0]; - - // offs = ts + token_offs; mask = offs < NUMEL - const offs = ts.add(token_offs); - const mask = offs.lt(numel); - - // expert_vals = load(topk_ids + offs, mask=mask, other=NUM_EXPERTS) - const topk_ptrs = a.topk_ids_ptr.addPtr(offs); - const expert_vals = b.loadOpts(topk_ptrs, .{ - .mask = mask, - .other = b.splat(num_experts, &.{block}), - }); - const valid = mask.bitAnd(expert_vals.lt(num_experts)); - - // rank = atomic_add(cumsum + expert_vals, 1, mask=valid, sem="relaxed") - const cumsum_ptrs = a.cumsum_ptr.addPtr(expert_vals); - const rank = b.atomicRmwOpts(.add, cumsum_ptrs, b.ones(&.{block}, .i32), .{ - .mask = valid, - .sem = .relaxed, - .scope = .gpu, - }); - - // store(sorted_token_ids + rank, offs, mask=valid) - const sorted_ptrs = a.sorted_token_ids_ptr.addPtr(rank); - b.storeOpts(sorted_ptrs, offs, .{ .mask = valid }); - - // token_start += num_programs * BLOCK_SIZE - // Compute step INSIDE the loop body (Python's source position). - // It's loop-invariant, so XLA's LICM hoists it out — the hoisted - // op-order then matches Python's IR (splats before step). - const step = num_progs.mul(block_i32); - w.yieldAfter(.{ts.add(step)}); - } - } -}; - -/// Returns the next power of two. -fn pow2(x: i64) i64 { - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(x)) catch unreachable); -} - -pub fn quantRows(tokens: i64) i64 { - return if (tokens >= 16) 16 else 1; -} - -fn ar(b: *Builder, n: i64) Value { - return b.arange(0, n, .i32); -} - -fn ld(b: *Builder, p: Value, mask: Value, dt: tri.DType, other: i64) Value { - return b.loadOpts(p, .{ .mask = mask, .other = if (p.isTensor()) b.full(p.shape().constSlice(), other, dt) else b.constant(dt, other) }); -} - -fn ci(b: *Builder, n: i64) Value { - return b.constant(.i32, n); -} - -fn quant32(b: *Builder, x: Value) struct { q: Value, s: Value } { - const rows = x.shape().constSlice()[0]; - const cols = x.shape().constSlice()[1]; - const groups = @divExact(cols, 32); - const blocks = b.reshape(x, &.{ rows * groups, 32 }); - const sf = b.exp2(b.ceil(b.log2(b.maxOpts(b.absf(blocks), .{ .axis = 1 }).maximum(1e-4).mul(1.0 / 448.0)))); - const scaled = blocks.div(sf.expandDims(1)).maximum(-448.0).minimum(448.0); - // TODO(Corentin): We should gate that behind a platform check so that we don't do useless stuff on non-SM89 platforms. - // Find a way to add that to the builder, maybe a `b.platform()` or something like that. - const q = b: { - // SM89 lowers FP32 -> FP8 through FP16 with truncation. Round to odd at - // FP16 precision first so that this intermediate cannot turn a value just - // above an FP8 midpoint into an exact tie. Direct FP8 conversion is unchanged. - const scaled_bits = scaled.bitCast(.i32); - const sticky = b.select(scaled_bits.bitAnd(0x1fff).ne(0), b.full(scaled.shape().constSlice(), 0x2000, .i32), b.zeros(scaled.shape().constSlice(), .i32)); - const prepared = scaled_bits.bitAnd(-0x2000).bitOr(sticky).bitCast(.f32); - break :b prepared.to(.f8e4m3fn); - }; - const bits = b.shrui(sf.bitCast(.i32), b.full(sf.shape().constSlice(), 23, .i32)).bitAnd(255).to(.i8); - return .{ .q = b.reshape(q, &.{ rows, cols }), .s = b.reshape(bits, &.{ rows, groups }) }; -} - -pub fn combineBlock(tokens: i64, hidden: i64) i64 { - // Keep at least 512 CTAs before increasing per-CTA reduction work. - const elements = tokens * hidden; - if (tokens > 1 and elements >= 512 * 2048) return 2048; - if (tokens > 1 and elements >= 512 * 1024) return 1024; - return 256; -} - -/// Computes an upper bound on the number of expert tiles needed. -/// When there is only one token, reserve a single tile per selected expert. -pub fn tiles(tokens: i64, topk: i64, experts: i64, bn: i64) i64 { - if (tokens == 1) return topk; - return @min(tokens * topk, @divTrunc(tokens * topk + experts * (bn - 1) + bn - 1, bn)); -} - -// ============================================================================= -// mxfp4_triton_quant -// ============================================================================= - -pub const Quant = struct { - pub const Cfg = struct { - tokens: i64, - hidden: i64, - }; - - pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_quant", .inputs = &.{"x"}, .outputs = &.{ "q", "s" }, .run = run }); - - fn run(b: *Builder, c: Cfg) tri.FinishError!void { - const a = try b.declareArgs(.{ .x = .{ .ptr = .bf16 }, .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 } }); - const rows = b.programId(.x).mul(quantRows(c.tokens)).add(ar(b, quantRows(c.tokens))); - const block = b.programId(.y); - const cols = block.mul(128).add(ar(b, 128)); - const offsets = rows.expandDims(1).mul(c.hidden).add(cols.expandDims(0)); - const mask = rows.lt(c.tokens).expandDims(1); - const x = ld(b, a.x.addPtr(offsets), mask, .bf16, 0).to(.f32); - const q = quant32(b, x); - b.storeOpts(a.q.addPtr(offsets), q.q, .{ .mask = mask }); - const scale_offsets = rows.expandDims(1).mul(@divTrunc(c.hidden, 32)).add(block.mul(4).add(ar(b, 4)).expandDims(0)); - b.storeOpts(a.s.addPtr(scale_offsets), q.s, .{ .mask = mask }); - } -}; - -// ============================================================================= -// mxfp4_triton_route -// ============================================================================= - -pub const Route = struct { - pub const Cfg = struct { - tokens: i64, - topk: i64, - expert_offset: i64, - }; - - pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_route", .inputs = &.{"ids"}, .outputs = &.{ "counts", "pos" }, .run = run }); - - fn run(b: *Builder, c: Cfg) tri.FinishError!void { - const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); - const e = b.programId(.x); - const p = ar(b, pow2(c.tokens * c.topk)); - const ids = ld(b, a.ids.addPtr(p), p.lt(c.tokens * c.topk), .i32, -1); - const valid = p.lt(c.tokens * c.topk).bitAnd(ids.eq(e.add(c.expert_offset))); - b.storeOpts(a.pos.addPtr(p), valid.to(.i32).cumsum().sub(1), .{ .mask = valid }); - b.store(a.counts.addPtr(e), valid.to(.i32).sum()); - } -}; - -// ============================================================================= -// mxfp4_triton_route_atomic -// ============================================================================= - -pub const RouteAtomic = struct { - pub const route_block = 1024; - - pub const Cfg = struct { - tokens: i64, - topk: i64, - expert_offset: i64, - experts: i64, - }; - - pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_route_atomic", .inputs = &.{ "ids", "zeros" }, .outputs = &.{ "counts", "pos" }, .run = run }); - - /// Routing for large batches, used by `scheduleForCute`. - /// Each route takes the next free slot of its expert with an atomic - /// increment of `counts` (zero-initialized through `zeros`). Slots within an - /// expert are therefore unordered, which the grouped GEMMs do not care - /// about: every routed row is computed independently. - fn run(b: *Builder, c: Cfg) tri.FinishError!void { - const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .zeros = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 } }); - const p = b.programId(.x).mul(route_block).add(ar(b, route_block)); - const in_range = p.lt(c.tokens * c.topk); - const e = ld(b, a.ids.addPtr(p), in_range, .i32, -1).sub(ci(b, c.expert_offset)); - const valid = in_range.bitAnd(e.ge(0)).bitAnd(e.lt(c.experts)); - const slot = b.atomicRmwOpts(.add, a.counts.addPtr(e.maximum(0)), b.full(&.{route_block}, 1, .i32), .{ .mask = valid, .sem = .relaxed }); - b.storeOpts(a.pos.addPtr(p), slot, .{ .mask = valid }); - } -}; - -// ============================================================================= -// mxfp4_triton_schedule -// ============================================================================= - -pub const Schedule = struct { - pub const Cfg = struct { - tokens: i64, - topk: i64, - expert_offset: i64, - experts: i64, - /// Expert groups the schedule may fill. - capacity: i64, - /// Routed rows per expert group, i.e. the consuming GEMM's tile N. - group: i64, - }; - - pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_schedule", .inputs = &.{ "ids", "counts", "pos" }, .outputs = &.{ "map", "perm", "sched" }, .run = run }); - - fn run(b: *Builder, cfg: Cfg) tri.FinishError!void { - const nt = cfg.capacity; - // Route positions are i32 tensors; keep the group size in that type. - const group: i32 = @intCast(cfg.group); - const a = try b.declareArgs(.{ .ids = .{ .ptr = .i32 }, .counts = .{ .ptr = .i32 }, .pos = .{ .ptr = .i32 }, .map = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 } }); - const p = b.programId(.x).mul(128).add(ar(b, 128)); - const e = ld(b, a.ids.addPtr(p), p.lt(cfg.tokens * cfg.topk), .i32, -1).sub(ci(b, cfg.expert_offset)); - const es = ar(b, pow2(cfg.experts)); - const counts = ld(b, a.counts.addPtr(es), es.lt(cfg.experts), .i32, 0); - const ts = counts.cdiv(group); - const starts = ts.cumsum().sub(ts); - var first = b.openIf(b.programId(.x).eq(0)); - { - b.store(a.sched.addPtr(2 * nt), ts.sum()); - first.close(); - } - const live = p.lt(cfg.tokens * cfg.topk).bitAnd(e.ge(0)).bitAnd(e.lt(cfg.experts)); - const safe = e.maximum(0).minimum(pow2(cfg.experts) - 1); - const pos = ld(b, a.pos.addPtr(p), live, .i32, 0); - const start = starts.gather(safe, 0); - const tile = start.add(pos.div(group)); - const row = pos.rem(group); - b.storeOpts(a.perm.addPtr(p), b.select(live, tile.mul(group).add(row), b.full(&.{128}, -1, .i32)), .{ .mask = p.lt(cfg.tokens * cfg.topk) }); - b.storeOpts(a.map.addPtr(tile.mul(group).add(row)), p, .{ .mask = live }); - b.storeOpts(a.sched.addPtr(tile), e, .{ .mask = live.bitAnd(row.eq(0)) }); - b.storeOpts(a.sched.addPtr(nt).addPtr(tile), counts.gather(safe, 0).sub(pos).minimum(group), .{ .mask = live.bitAnd(row.eq(0)) }); - } -}; - -// ============================================================================= -// mxfp4_triton_combine -// ============================================================================= - -pub const Combine = struct { - pub const Cfg = struct { - tokens: i64, - hidden: i64, - topk: i64, - expert_offset: i64, - experts: i64, - routing_weight_placement: RoutingWeightPlacement = .before_down, - bn: i64, - }; - pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_combine", .inputs = &.{ "d", "ids", "perm", "rw" }, .outputs = &.{"y"}, .run = run }); - - fn run(b: *Builder, c: Cfg) tri.FinishError!void { - return combineWithBn(b, c); - } - - fn combineWithBn(b: *Builder, c: Cfg) tri.FinishError!void { - const a = try b.declareArgs(.{ .d = .{ .ptr = .bf16 }, .ids = .{ .ptr = .i32 }, .perm = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .y = .{ .ptr = .bf16 } }); - const t = b.programId(.x); - const block = combineBlock(c.tokens, c.hidden); - const h = b.programId(.y).mul(block).add(ar(b, block)); - var acc = b.zeros(&.{block}, .f32); - for (0..@intCast(c.topk)) |s| { - const pair = t.mul(c.topk).add(@as(i64, @intCast(s))); - const row = if (c.tokens == 1) pair.mul(c.bn) else b.load(a.perm.addPtr(pair)); - const valid = if (c.tokens == 1) v: { - const e = b.load(a.ids.addPtr(pair)).sub(ci(b, c.expert_offset)); - break :v e.ge(0).bitAnd(e.lt(c.experts)); - } else row.ge(0); - var value = ld(b, a.d.addPtr(row.mul(c.hidden).add(h)), valid.bitAnd(h.lt(c.hidden)), .bf16, 0).to(.f32); - if (c.routing_weight_placement == .after_down) { - value = value.mul(ld(b, a.rw.addPtr(pair), valid, .f32, 0)); - } - acc = acc.add(value); - } - b.storeOpts(a.y.addPtr(t.mul(c.hidden).add(h)), acc.to(.bf16), .{ .mask = h.lt(c.hidden) }); - } -}; - -// ============================================================================= -// mxfp4 gemm -// ============================================================================= - -pub const GateUp = struct { - pub const Cfg = GemmConfig; - - pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_gate_up", .inputs = &.{ "q", "s", "w", "ws", "ids", "rw", "map", "sched" }, .outputs = &.{ "mid", "ms" }, .run = run }); - - fn run(b: *Builder, c: Cfg) tri.FinishError!void { - return gemm(b, c, .gate_up); - } -}; - -pub const Down = struct { - pub const Cfg = GemmConfig; - - pub const Kernel = tri.Kernel(Cfg, .{ .name = "mxfp4_triton_down", .inputs = &.{ "mid", "ms", "w", "ws", "ids", "sched" }, .outputs = &.{"d"}, .run = run }); - - fn run(b: *Builder, c: Cfg) tri.FinishError!void { - return gemm(b, c, .down); - } -}; - -const GemmConfig = struct { - tokens: i64, - hidden: i64, - intermediate: i64, - experts: i64, - topk: i64, - expert_offset: i64 = 0, - // Swiglu clamping. Set to 0 to disable. - swiglu_limit: f32, - scale_stride: i64, - routing_weight_placement: RoutingWeightPlacement = .before_down, - bn: i64, - bk: i64, -}; - -const Projection = enum { gate_up, down }; - -fn gemm(b: *Builder, c: GemmConfig, comptime projection: Projection) tri.FinishError!void { - const is_up = projection == .gate_up; - const a = if (is_up) try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .rw = .{ .ptr = .f32 }, .map = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .mid = .{ .ptr = .f8e4m3fn }, .ms = .{ .ptr = .i8 } }) else try b.declareArgs(.{ .q = .{ .ptr = .f8e4m3fn }, .s = .{ .ptr = .i8 }, .w = .{ .ptr = .i8 }, .ws = .{ .ptr = .i8 }, .ids = .{ .ptr = .i32 }, .sched = .{ .ptr = .i32 }, .d = .{ .ptr = .bf16 } }); - const block = b.programId(.x); - const tile = b.programId(.y); - const n = ar(b, c.bn); - const nt = tiles(c.tokens, c.topk, c.experts, c.bn); - const expert = if (c.tokens == 1) b.load(a.ids.addPtr(tile)).sub(ci(b, c.expert_offset)) else ld(b, a.sched.addPtr(tile), tile.lt(b.load(a.sched.addPtr(2 * nt))), .i32, 0); - const valid = if (c.tokens == 1) expert.ge(0).bitAnd(expert.lt(c.experts)) else tile.lt(b.load(a.sched.addPtr(2 * nt))); - var live = b.openIf(valid); - { - const count = if (c.tokens == 1) ci(b, 1) else b.load(a.sched.addPtr(nt).addPtr(tile)); - const pair = if (c.tokens == 1) b.full(&.{c.bn}, 0, .i32).add(tile) else if (is_up) ld(b, a.map.addPtr(tile.mul(c.bn).add(n)), n.lt(count), .i32, 0) else n; - const width: i64 = if (is_up and c.tokens > 8) 256 else 128; - const ksize = if (is_up) c.hidden else c.intermediate; - const rows = if (is_up) 2 * c.intermediate else c.hidden; - const m = block.mul(width).add(ar(b, width)); - const k = ar(b, c.bk); - const ks = ar(b, @divExact(c.bk, 32)); - const row = if (is_up) pair.div(c.topk) else tile.mul(c.bn).add(n); - const wd = b.makeTensorDescriptor(a.w.addPtr(expert.to(.i64).mul(@divTrunc(rows * ksize, 2))), &.{ ci(b, rows), ci(b, @divTrunc(ksize, 2)) }, &.{ b.constant(.i64, @divTrunc(ksize, 2)), b.constant(.i64, 1) }, .pad_zero, &.{ width, @divExact(c.bk, 2) }, .i8); - var loop = b.openFor(0, @divTrunc(ksize + c.bk - 1, c.bk), 1, .{b.zeros(&.{ width, c.bn }, .f32)}); - { - const sk = loop.iv.mul(@divExact(c.bk, 32)).add(ks); - const kk = loop.iv.mul(c.bk).add(k); - const w = b.descriptorLoad(wd, &.{ block.mul(ci(b, width)), loop.iv.mul(ci(b, @divExact(c.bk, 2))) }, &.{ width, @divExact(c.bk, 2) }, .i8); - const scale_offset = m.expandDims(1).mul(c.scale_stride).add(sk.expandDims(0)); - const ws = ld(b, a.ws.addPtr(expert.to(.i64).mul(rows * c.scale_stride)).addPtr(scale_offset), sk.expandDims(0).lt(@divTrunc(ksize, 32)), .i8, 127); - var mask = kk.expandDims(0).lt(ksize); - var smask = sk.expandDims(0).lt(@divTrunc(ksize, 32)); - // The small-batch epilogue leaves padded rows unwritten. - if (is_up or c.tokens <= 8) { - mask = mask.bitAnd(n.lt(count).expandDims(1)); - smask = smask.bitAnd(n.lt(count).expandDims(1)); - } - const x = ld(b, a.q.addPtr(row.expandDims(1).mul(ksize).add(kk.expandDims(0))), mask, .f8e4m3fn, 0); - const xs = ld(b, a.s.addPtr(row.expandDims(1).mul(@divTrunc(ksize, 32)).add(sk.expandDims(0))), smask, .i8, 127); - loop.yield(.{b.dotScaled(w, b.permute(x, &.{ 1, 0 }), loop.carried[0], ws, xs, .e2m1, .e4m3)}); - } - const epilogue_n: i64 = if (c.tokens == 1) 1 else c.bn; - const out_n = ar(b, epilogue_n); - const acc = if (c.tokens == 1) live_row: { - // The remaining MMA columns are padding during single-token decode. - // Extract column zero before activation/reduction and output stores. - var v = loop.results[0]; - var cols: i64 = c.bn; - while (cols > 1) : (cols = @divTrunc(cols, 2)) { - v = b.split(b.reshape(v, &.{ width, @divTrunc(cols, 2), 2 }))[0]; - } - break :live_row b.reshape(v, &.{ 1, width }).to(.bf16); - } else b.permute(loop.results[0], &.{ 1, 0 }).to(.bf16); - if (is_up) { - const columns = @divExact(width, 2); - const halves = b.split(b.reshape(acc.to(.f32), &.{ epilogue_n, columns, 2 })); - var g = halves[0]; - var u = halves[1]; - if (c.swiglu_limit > 0) { - g = g.minimum(c.swiglu_limit); - u = u.minimum(c.swiglu_limit).maximum(-c.swiglu_limit); - } - const ex = b.externElementwise(&.{g.mul(-1.0)}, &.{ epilogue_n, columns }, .f32, "libdevice", "", "__nv_expf"); - var mid = u.mul(b.divRn(g, ex.add(1.0))); - if (c.routing_weight_placement == .before_down) { - const rw = if (c.tokens == 1) b.load(a.rw.addPtr(tile)) else ld(b, a.rw.addPtr(pair), n.lt(count), .f32, 0).expandDims(1); - mid = mid.mul(rw); - } - // Compute scales and quantize directly from the FP32 SwiGLU result. - const q = quant32(b, mid); - const col = block.mul(columns).add(ar(b, columns)); - b.store(a.mid.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.intermediate).add(col.expandDims(0))), q.q); - const scale_cols = block.mul(@divExact(columns, 32)).add(ar(b, @divExact(columns, 32))); - b.store(a.ms.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(@divTrunc(c.intermediate, 32)).add(scale_cols.expandDims(0))), q.s); - } else b.store(a.d.addPtr(tile.mul(c.bn).add(out_n).expandDims(1).mul(c.hidden).add(m.expandDims(0))), acc); - live.close(); - } -} diff --git a/zml/kernels/moe/triton/triton.zig b/zml/kernels/moe/triton/triton.zig deleted file mode 100644 index 8ba799a5..00000000 --- a/zml/kernels/moe/triton/triton.zig +++ /dev/null @@ -1,1072 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const Tensor = zml.Tensor; -const Shape = zml.Shape; -const DataType = zml.DataType; -const toDType = zml.kernel.triton.from; -const ExpertsLayout = @import("../api.zig").ExpertsLayout; -const ProjectionLayout = @import("../api.zig").ProjectionLayout; -const RoutingWeightPlacement = @import("../api.zig").RoutingWeightPlacement; -const Activation = @import("../api.zig").Activation; -const Options = @import("../api.zig").Options; -const LocalRouting = @import("../api.zig").LocalRouting; -const kernels = @import("moe.zig"); - -const log = std.log.scoped(.@"zml/moe/triton"); - -fn numStages(config: LaunchConfig) i32 { - const platform = zml.Compiler.current().platform; - if (platform.target != .rocm) return config.num_stages; - - // ROCm buffers num_stages - 1 BF16 operand pairs in LDS. A single stage - // reuses scratch space between operand layout conversions. - const bytes_per_stage: i64 = @intCast(2 * config.block_size_k * (config.block_size_m + config.block_size_n)); - var stages = config.num_stages; - for (platform.devices) |device| { - const attribute = device.pjrt_desc.attribute(platform.pjrt_api, "shared_memory_per_block_optin"); - const limit = if (attribute) |value| value.int64 else 64 * 1024; - stages = @intCast(@min(stages, @divTrunc(limit, bytes_per_stage) + 1)); - } - return stages; -} - -test { - std.testing.refAllDecls(@This()); -} - -pub fn fusedExperts( - input: zml.Tensor, - routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - options: Options, -) zml.Tensor { - // HACK(Corentin): only rely on the scheme of gate_up for now. - - if (gate_up.quantization) |q| switch (q.scheme) { - .mxfp4 => if (zml.Compiler.current().platform.target == .cuda) { - return fusedExpertsMxfp4(input, routing, gate_up, down, options); - }, - else => {}, - }; - - const layout: ProjectionLayout = if (gate_up.quantizationScheme() == .mxfp4) .row_interleaved_1 else .concatenated; - return fusedExpertsBase(input, routing, gate_up, down, layout, options); -} - -pub fn fusedExpertsBase( - hidden_states: Tensor, - routing_: LocalRouting, - gate_up_: zml.nn.Linear, - down_: zml.nn.Linear, - layout: ProjectionLayout, - opts: Options, -) Tensor { - const expert_map_ = routing_.expertMap(); - const num_tokens = hidden_states.dim(.t); - var launch_config = launchConfigForTokens(num_tokens); - - const gate_up_scheme = gate_up_.quantizationScheme(); - const down_scheme = down_.quantizationScheme(); - if (gate_up_scheme) |scheme| switch (scheme) { - .nvfp4 => stdx.debug.panic("triton backend doesn't support gate_up nvfp4 quantization scheme", .{}), - .fp8_block128 => launch_config.block_size_k = 128, - .fp8_block32 => launch_config.block_size_k = 32, - .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, - }; - - var down_launch_config = launchConfigForTokens(num_tokens); - if (down_scheme) |scheme| switch (scheme) { - .nvfp4 => stdx.debug.panic("triton backend doesn't support down nvfp4 quantization scheme", .{}), - .fp8_block128 => down_launch_config.block_size_k = 128, - .fp8_block32 => down_launch_config.block_size_k = 32, - .mxfp4, .mxfp8, .fp8_per_channel, .fp8_per_tensor => {}, - }; - - const hidden = hidden_states.reshape(.{ .token = num_tokens, .in = hidden_states.dim(.d) }).withTags(.{ .token, .in }); - const gate_up = gate_up_.weight.withTags(.{ .expert, .out, .in }); - const down = down_.weight.withTags(.{ .expert, .out, .mid }); - const routing_weights = routing_.weights.reshape(.{ .token = num_tokens, .in = routing_.weights.dim(.topk) }).withTags(.{ .token, .topk }); - const ids = routing_.ids.reshape(.{ .token = num_tokens, .in = routing_.ids.dim(.topk) }).withTags(.{ .token, .topk }); - - stdx.debug.assert(hidden.dtype() == .bf16, "triton backend expected BF16 hidden states, got {}", .{hidden.dtype()}); - if (gate_up_scheme == .mxfp4) { - stdx.debug.assert(gate_up.dtype() == .u8 or gate_up.dtype() == .i8 or gate_up.dtype() == .f4e2m1, "triton backend expected u8, i8 or f4e2m1 gate_up data type when quantization scheme is mxfp4, got {}", .{gate_up.dtype()}); - } else { - stdx.debug.assert(gate_up.dtype() == .bf16 or gate_up.dtype() == .f8e4m3fn or gate_up.dtype() == .f8e4m3fnuz, "triton backend expected gate_up data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{gate_up.dtype()}); - } - if (down_scheme == .mxfp4) { - stdx.debug.assert(down.dtype() == .u8 or down.dtype() == .i8 or down.dtype() == .f4e2m1, "triton backend expected u8, i8 or f4e2m1 down data type when quantization scheme is mxfp4, got {}", .{down.dtype()}); - } else { - stdx.debug.assert(down.dtype() == .bf16 or down.dtype() == .f8e4m3fn or down.dtype() == .f8e4m3fnuz, "triton backend expected down data type to be bf16, fp8e4m3fn or f8e4m3fnuz, got {}", .{down.dtype()}); - } - - stdx.debug.assert(routing_weights.dtype() == .f32 or routing_weights.dtype() == .bf16, "triton backend expected f32 or bf16 routing weights, got {}", .{routing_weights.dtype()}); - stdx.debug.assert(ids.dtype() == .i32, "triton backend expected i32 expert ids, got {}", .{ids.dtype()}); - const gate_up_k = gate_up.dim(.in) * @as(i64, if (gate_up_scheme == .mxfp4 and gate_up.dtype() != .f4e2m1) 2 else 1); - const down_k = down.dim(.mid) * @as(i64, if (down_scheme == .mxfp4 and down.dtype() != .f4e2m1) 2 else 1); - stdx.debug.assert(hidden.dim(.in) == gate_up_k, "triton backend expected hidden width to match gate/up input width, got {} and {}", .{ hidden.dim(.in), gate_up_k }); - const activation_reduction: i64 = switch (opts.activation) { - .gelu, .relu, .silu => 1, - .swiglu, .swiglu_step, .geglu, .geglu_tanh => 2, - }; - stdx.debug.assert(@rem(gate_up.dim(.out), activation_reduction) == 0, "triton backend expected gate/up output width to be divisible by {}, got {}", .{ activation_reduction, gate_up.dim(.out) }); - stdx.debug.assert(down_k == @divFloor(gate_up.dim(.out), activation_reduction), "triton backend expected down input width to match activated width, got {} and {}", .{ down_k, @divFloor(gate_up.dim(.out), activation_reduction) }); - stdx.debug.assert(ids.dim(.token) == hidden.dim(.token) and routing_weights.dim(.token) == hidden.dim(.token), "triton backend expected routing ids and weights to match hidden token count, got {}, {} and {}", .{ ids.dim(.token), routing_weights.dim(.token), hidden.dim(.token) }); - stdx.debug.assert(ids.dim(.topk) == routing_weights.dim(.topk), "triton backend expected routing ids and weights to have matching top-k dimensions, got {} and {}", .{ ids.dim(.topk), routing_weights.dim(.topk) }); - stdx.debug.assert(gate_up.dim(.expert) == down.dim(.expert), "triton backend expected gate/up and down expert counts to match, got {} and {}", .{ gate_up.dim(.expert), down.dim(.expert) }); - if (expert_map_) |expert_map| - stdx.debug.assert(expert_map.dtype() == .i32 and expert_map.rank() == 1, "triton backend expected a rank-1 i32 expert map, got rank {} and dtype {}", .{ expert_map.rank(), expert_map.dtype() }); - - const num_experts = if (expert_map_) |expert_map| expert_map.dim(.expert) else gate_up.dim(.expert); - const routing = prepareRouting(ids, num_experts, @intCast(launch_config.block_size_m)); - - const expert_ids = if (expert_map_) |expert_map| - expert_map.gather(.{ .expert = routing.expert_ids }, .{}).withTags(.{.g}) - else - routing.expert_ids; - - const hidden_quant, const input_scale = prepareInput(hidden, gate_up_scheme, gate_up.dtype(), opts.quantize_input); - - const gate_up_out = gemm(.{ - .input = hidden_quant, - .weight = gate_up, - .bias = gate_up_.bias, - .input_scale = input_scale, - .weight_scale = gate_up_.quantizationScales(), - .routing = routing, - .expert_ids = expert_ids, - .launch_config = launch_config, - .quant_scheme = gate_up_scheme, - .top_k = @intCast(ids.dim(.topk)), - .output_shape = Shape.init(.{ .token = routing.num_assignments, .out = gate_up.dim(.out) }, .bf16), - }); - - var activated = opts.activation.applyActivation(gate_up_out, layout); - if (opts.routing_weight_placement == .before_down) { - const weights = routing_weights.reshape(.{ .token = routing.num_assignments }).convert(.f32); - activated = activated.mul(weights.broad(activated.shape())); - } - const activated_quant, const down_input_scale = prepareInput(activated, down_scheme, down.dtype(), opts.quantize_input); - - const down_out = gemm(.{ - .input = activated_quant, - .weight = down, - .bias = down_.bias, - .input_scale = down_input_scale, - .weight_scale = down_.quantizationScales(), - .routing_weights = if (opts.routing_weight_placement == .after_down) routing_weights else null, - .routing = routing, - .expert_ids = expert_ids, - .launch_config = down_launch_config, - .quant_scheme = down_scheme, - .top_k = 1, - .output_shape = Shape.init(.{ .token = num_tokens, .topk = ids.dim(.topk), .out = down.dim(.out) }, .bf16), - }); - - const output = down_out.convert(.f32).sum(.topk).squeeze(.topk).convert(.bf16); - - return output.reshape(.{ .t = num_tokens, .d = down.dim(.out) }); -} - -pub const GemmOptions = struct { - input: Tensor, - weight: Tensor, - bias: ?Tensor = null, - input_scale: ?Tensor = null, - weight_scale: ?Tensor = null, - routing_weights: ?Tensor = null, - routing: Routing, - expert_ids: Tensor, - launch_config: LaunchConfig, - quant_scheme: ?zml.Quantization.Scheme = null, - top_k: usize, - output_shape: Shape, -}; - -pub fn gemm(opts: GemmOptions) Tensor { - // Native FP4 tensors expose logical K; the Triton operand is byte-packed. - const weight = if (opts.quant_scheme == .mxfp4 and opts.weight.dtype() == .f4e2m1) - opts.weight.reshape(opts.weight.shape().setDim(2, @divExact(opts.weight.dim(2), 2)).append(.{ .nibble = 2 })).bitCast(.u8) - else - opts.weight; - const weight_k = weight.dim(2) * @as(i64, if (opts.quant_scheme == .mxfp4) 2 else 1); - stdx.debug.assert(opts.input.dim(1) == weight_k, "input width {} must match logical weight K {}", .{ opts.input.dim(1), weight_k }); - if (opts.quant_scheme == .mxfp4) { - stdx.debug.assert(@mod(weight_k, 32) == 0 and opts.weight_scale != null, "MXFP4 requires K divisible by 32 and weight scales", .{}); - stdx.debug.assert(opts.input.dtype() == .bf16 and opts.input_scale == null, "MXFP4 requires BF16 activations without input scales", .{}); - } - - if (opts.quant_scheme == .fp8_block32) { - const scales = opts.weight_scale orelse stdx.debug.panic("block32 FP8 requires weight scales", .{}); - stdx.debug.assert(weight.dtype() == .f8e4m3fn and @mod(weight.dim(1), 32) == 0 and @mod(weight_k, 32) == 0, "block32 FP8 requires E4M3FN weights with N and K divisible by 32", .{}); - stdx.debug.assert(scales.rank() == 3 and scales.dim(0) == weight.dim(0) and scales.dim(1) == @divExact(weight.dim(1), 32) and scales.dim(2) == @divExact(weight_k, 32), "block32 FP8 requires scales shaped [expert, N/32, K/32], got {f}", .{scales.shape()}); - stdx.debug.assert(scales.dtype() == .f8e8m0 or scales.dtype() == .bf16 or scales.dtype() == .f32, "unsupported block32 FP8 scale dtype {}", .{scales.dtype()}); - } - - stdx.debug.assert(opts.quant_scheme != null or (opts.input_scale == null and opts.weight_scale == null), "scales require a quantization scheme", .{}); - for ([_]i64{ weight.dim(1), weight.dim(2), opts.input.dim(1), weight.dim(1) * weight.dim(2), opts.output_shape.dim(-1) }) |dim_or_stride| { - stdx.debug.assert(dim_or_stride > 0 and @mod(dim_or_stride, 16) == 0, "FusedMoe dimensions and matrix strides must be positive multiples of 16, got {}", .{dim_or_stride}); - } - const block_size_m: i64 = @intCast(opts.launch_config.block_size_m); - const block_size_n: i64 = @intCast(opts.launch_config.block_size_n); - const m_tokens = opts.input.dim(0); - const em_effective = if (m_tokens < block_size_m) - @min(opts.routing.max_num_tokens_padded, opts.routing.num_assignments * block_size_m) - else - opts.routing.max_num_tokens_padded; - stdx.debug.assert(@mod(em_effective, block_size_m) == 0, "routing capacity {} must be a multiple of block size {}", .{ em_effective, block_size_m }); - const grid_x = - @divExact(em_effective, block_size_m) * - (std.math.divCeil(i64, weight.dim(1), block_size_n) catch unreachable); - - const weight_scale: ?Tensor = if (opts.weight_scale) |scale| blk: { - const scheme = opts.quant_scheme orelse break :blk scale; - break :blk switch (scheme) { - // dot_scaled takes E8M0 encodings as bytes, including native E8M0 tensors. - .mxfp4, .mxfp8 => scale.bitCast(.u8), - .fp8_per_channel => scale.reshape(.{ weight.dim(0), weight.dim(1), 1 }), - .fp8_per_tensor => if (scale.count() == 1) scale else scale.reshape(.{ weight.dim(0), 1, 1 }), - // Tiled FP8 uses ordinary dot products with FP32 scale arithmetic. - .fp8_block32 => scale.convert(.f32), - .fp8_block128, .nvfp4 => scale, - }; - } else null; - - const stride_asm: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) scale.dim(1) else 0) else 0; - const stride_ask: i64 = if (opts.input_scale) |scale| (if (scale.rank() == 2) 1 else 0) else 0; - const has_weight_scale_expert_stride = if (weight_scale) |scale| - !(opts.quant_scheme == .fp8_per_tensor and scale.count() == 1) - else - false; - const stride_bse: i64 = if (weight_scale) |scale| (if (scale.rank() == 3 and has_weight_scale_expert_stride) scale.dim(1) * scale.dim(2) else 0) else 0; - const stride_bsk: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) 1 else 0) else 0; - const stride_bsn: i64 = if (weight_scale) |scale| (if (scale.rank() == 3) scale.dim(2) else 0) else 0; - - return kernels.FusedMoe.Kernel.call( - .{ - .a_ptr = opts.input, - .b_ptr = weight, - .b_bias_ptr = opts.bias orelse Tensor.scalar(0, .f32), - .a_scale_ptr = opts.input_scale orelse Tensor.scalar(1.0, .f32), - .b_scale_ptr = weight_scale orelse Tensor.scalar(1.0, .f32), - .routing_weights_ptr = opts.routing_weights orelse Tensor.scalar(1.0, .f32), - .sorted_token_ids_ptr = opts.routing.sorted_token_ids, - .expert_ids_ptr = opts.expert_ids, - .num_tokens_post_padded_ptr = opts.routing.num_tokens_post_padded, - .N_ptr = Tensor.constant(.{ .i64 = weight.dim(1) }).reshape(.{1}), - .K_ptr = Tensor.constant(.{ .i64 = weight_k }).reshape(.{1}), - .EM_ptr = Tensor.constant(.{ .i64 = em_effective }).reshape(.{1}), - .num_valid_tokens_ptr = Tensor.constant(.{ .i64 = opts.routing.num_assignments }).reshape(.{1}), - .stride_am_ptr = Tensor.constant(.{ .i64 = opts.input.dim(1) }).reshape(.{1}), - .stride_be_ptr = Tensor.constant(.{ .i64 = weight.dim(1) * weight.dim(2) }).reshape(.{1}), - .stride_bn_ptr = Tensor.constant(.{ .i64 = weight.dim(2) }).reshape(.{1}), - .stride_cm_ptr = Tensor.constant(.{ .i64 = weight.dim(.out) }).reshape(.{1}), - .stride_asm_ptr = Tensor.constant(.{ .i64 = stride_asm }).reshape(.{1}), - .stride_ask_ptr = Tensor.constant(.{ .i64 = stride_ask }).reshape(.{1}), - .stride_bse_ptr = Tensor.constant(.{ .i64 = stride_bse }).reshape(.{1}), - .stride_bsk_ptr = Tensor.constant(.{ .i64 = stride_bsk }).reshape(.{1}), - .stride_bsn_ptr = Tensor.constant(.{ .i64 = stride_bsn }).reshape(.{1}), - .stride_bbe_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() == 2 and bias.dim(0) > 1) bias.dim(1) else 0) else 0 }).reshape(.{1}), - .stride_bbn_ptr = Tensor.constant(.{ .i64 = if (opts.bias) |bias| (if (bias.rank() > 0 and bias.dim(-1) > 1) 1 else 0) else 0 }).reshape(.{1}), - }, - .{ .c = opts.output_shape }, - .{ - .cfg = .{ - .a_dtype = toDType(opts.input.dtype()), - .b_dtype = toDType(weight.dtype()), - .c_dtype = toDType(opts.output_shape.dtype()), - .a_scale_dtype = if (opts.input_scale) |scale| toDType(scale.dtype()) else null, - .b_scale_dtype = if (weight_scale) |scale| toDType(scale.dtype()) else null, - .b_bias_dtype = if (opts.bias) |bias| toDType(bias.dtype()) else null, - .routing_weights_dtype = if (opts.routing_weights) |weights| toDType(weights.dtype()) else null, - .block_size_m = opts.launch_config.block_size_m, - .block_size_n = opts.launch_config.block_size_n, - .block_size_k = opts.launch_config.block_size_k, - .group_size_m = opts.launch_config.group_size_m, - .top_k = opts.top_k, - .naive_block_assignment = opts.routing.naive_block_assignment, - .compute_type = .bf16, - .quant_scheme = opts.quant_scheme, - }, - .grid = .{ @intCast(grid_x), 1, 1 }, - .num_warps = opts.launch_config.num_warps, - .num_stages = numStages(opts.launch_config), - }, - ).c; -} - -pub const Routing = struct { - sorted_token_ids: Tensor, - expert_ids: Tensor, - num_tokens_post_padded: Tensor, - max_num_tokens_padded: i64, - num_assignments: i64, - naive_block_assignment: bool, -}; - -pub fn prepareRouting(topk_ids: Tensor, num_experts: i64, block_size_m: i64) Routing { - stdx.debug.assert(block_size_m > 0 and @mod(block_size_m, 16) == 0, "routing block size must be a positive multiple of 16, got {}", .{block_size_m}); - const ids = topk_ids.withTags(.{ .token, .topk }).convert(.i32); - const num_assignments = ids.dim(.token) * ids.dim(.topk); - const sparsity_factor: i64 = 4; - const naive_block_assignment = num_assignments * sparsity_factor <= num_experts; - const max_num_tokens_padded = if (naive_block_assignment) - num_assignments * block_size_m - else if (num_assignments < num_experts) - num_assignments * block_size_m - else - (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; - - const sorted_token_ids, const expert_ids, const num_tokens_post_padded = if (naive_block_assignment) blk: { - log.debug("Using naive block assignment for MoE kernels. Num assignments: {d}, Num experts: {d}", .{ num_assignments, num_experts }); - break :blk .{ - Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)), - ids.reshape(.{ .g = num_assignments }), - Tensor.constant(.{ .i32 = @as(i32, @intCast(max_num_tokens_padded)) }).reshape(.{1}), - }; - } else alignBlockSize(ids, num_experts, block_size_m); - - return .{ - .sorted_token_ids = sorted_token_ids, - .expert_ids = expert_ids, - .num_tokens_post_padded = num_tokens_post_padded, - .max_num_tokens_padded = max_num_tokens_padded, - .num_assignments = num_assignments, - .naive_block_assignment = naive_block_assignment, - }; -} - -fn alignBlockSize(topk_ids: Tensor, num_experts: i64, block_size_m: i64) struct { Tensor, Tensor, Tensor } { - const topk_ids_ = topk_ids.withTags(.{ .token, .topk }).convert(.i32); - const num_tokens = topk_ids_.dim(.token); - const topk = topk_ids_.dim(.topk); - const num_assignments = num_tokens * topk; - const max_num_tokens_padded = if (num_assignments < num_experts) - num_assignments * block_size_m - else - (std.math.divCeil(i64, num_assignments + num_experts * (block_size_m - 1), block_size_m) catch unreachable) * block_size_m; - const max_num_m_blocks = @divExact(max_num_tokens_padded, block_size_m); - const warp_size: i64 = 32; - const padded_num_experts: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(num_experts, warp_size)))); - const sort_block_size: i64 = 256; - const sort_grid_x: i64 = @min(std.math.divCeil(i64, num_assignments, sort_block_size) catch unreachable, 65535); - - const flat_experts = topk_ids_.reshape(.{ .g = num_assignments }); - var cumsums = Tensor.zeroes(Shape.init(.{ .g = num_experts + 1 }, .i32)); - var expert_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_m_blocks }, .i32)); - var sorted_token_ids = Tensor.zeroes(Shape.init(.{ .g = max_num_tokens_padded }, .i32)); - var num_tokens_post_padded = Tensor.zeroes(Shape.init(.{ .g = 1 }, .i32)); - - { - const align_outs = kernels.MoeAlignBlockSize.Kernel.call( - .{ - .topk_ids_ptr = flat_experts, - .sorted_token_ids_ptr = sorted_token_ids, - .expert_ids_ptr = expert_ids, - .num_tokens_post_pad_ptr = num_tokens_post_padded, - .cumsum_ptr = cumsums, - }, - .{ - .sorted_token_ids = sorted_token_ids.shape(), - .expert_ids = expert_ids.shape(), - .num_tokens_post_pad = num_tokens_post_padded.shape(), - .cumsum = cumsums.shape(), - }, - .{ - .cfg = .{ - .numel = @intCast(num_assignments), - .num_experts = @intCast(num_experts), - .padded_num_experts = @intCast(padded_num_experts), - .max_num_tokens_padded = @intCast(max_num_tokens_padded), - .max_num_m_blocks = @intCast(max_num_m_blocks), - .block_size_m = @intCast(block_size_m), - .hist_block = 256, - }, - .grid = .{ 2, 1, 1 }, - .num_stages = 1, - .num_warps = 8, - .output_operand_aliases = .{ - .sorted_token_ids = .sorted_token_ids_ptr, - .expert_ids = .expert_ids_ptr, - .num_tokens_post_pad = .num_tokens_post_pad_ptr, - .cumsum = .cumsum_ptr, - }, - }, - ); - sorted_token_ids = align_outs.sorted_token_ids; - expert_ids = align_outs.expert_ids; - num_tokens_post_padded = align_outs.num_tokens_post_pad; - cumsums = align_outs.cumsum; - } - - { - const sort_outs = kernels.CountAndSortExpertTokens.Kernel.call( - .{ - .topk_ids_ptr = flat_experts, - .sorted_token_ids_ptr = sorted_token_ids, - .cumsum_ptr = cumsums, - }, - .{ - .sorted_token_ids = sorted_token_ids.shape(), - .cumsum = cumsums.shape(), - }, - .{ - .cfg = .{ - .numel = @intCast(num_assignments), - .num_experts = @intCast(num_experts), - .sort_block_size = @intCast(sort_block_size), - }, - .grid = .{ @intCast(sort_grid_x), 1, 1 }, - .num_stages = 1, - .num_warps = 4, - .output_operand_aliases = .{ - .sorted_token_ids = .sorted_token_ids_ptr, - .cumsum = .cumsum_ptr, - }, - }, - ); - sorted_token_ids = sort_outs.sorted_token_ids; - cumsums = sort_outs.cumsum; - } - - return .{ sorted_token_ids, expert_ids, num_tokens_post_padded }; -} - -pub fn prepareInput(x: Tensor, scheme: ?zml.Quantization.Scheme, output_dtype: zml.DataType, quantize: bool) struct { Tensor, ?Tensor } { - if (quantize) { - if (scheme) |scheme_| { - if (scheme_ != .mxfp4) { - const input, const scale = quantizeFp8Input(x, scheme_, output_dtype); - return .{ input, scale }; - } - } - } - return .{ x.convert(.bf16), null }; -} - -fn quantizeFp8Input(x: Tensor, scheme: zml.Quantization.Scheme, output_dtype: DataType) struct { Tensor, Tensor } { - const group_size: i64, const scale_dtype: DataType, const absmax_epsilon: f32 = switch (scheme) { - .mxfp8 => .{ 32, .u8, 1e-10 }, - .fp8_per_channel, .fp8_per_tensor => .{ x.dim(1), .f32, 1e-10 }, - .fp8_block128 => .{ 128, .f32, 1e-10 }, - .fp8_block32 => .{ 32, .f32, 1e-4 }, - .mxfp4, .nvfp4 => unreachable, - }; - stdx.debug.assert(x.rank() == 2, "expected a rank-2 activation matrix, got {f}", .{x.shape()}); - stdx.debug.assert(@mod(x.dim(1), group_size) == 0, "activation width must be divisible by group size {d}, got {d}", .{ group_size, x.dim(1) }); - - const groups_per_row = @divExact(x.dim(1), group_size); - const quantized = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .feature = x.dim(1) }, output_dtype)); - const scales = Tensor.zeroes(Shape.init(.{ .token = x.dim(0), .group = groups_per_row }, scale_dtype)); - - const outs = kernels.PerTokenGroupQuantFp8.Kernel.call( - .{ - .y_ptr = x, - .group_size_ptr = Tensor.constant(.{ .i64 = group_size }).reshape(.{1}), - .y_num_columns_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), - .y_row_stride_ptr = Tensor.constant(.{ .i64 = x.dim(1) }).reshape(.{1}), - .eps_ptr = Tensor.scalar(absmax_epsilon, .f32), - }, - .{ .y_q = quantized.shape(), .y_s = scales.shape() }, - .{ - .cfg = .{ - .input_dtype = toDType(x.dtype()), - .output_dtype = toDType(output_dtype), - .scale_dtype = toDType(scale_dtype), - .block = std.math.ceilPowerOfTwoAssert(usize, @intCast(group_size)), - .quant_scheme = scheme, - }, - .grid = .{ @intCast(x.dim(0) * groups_per_row), 1, 1 }, - .num_stages = 1, - .num_warps = 1, - }, - ); - - return .{ outs.y_q, outs.y_s }; -} - -test "SwiGLU uses FP32 math for concatenated and interleaved BF16 inputs" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - const Local = struct { - fn forward(x: Tensor, layout: ProjectionLayout, activation: Activation) Tensor { - return activation.applyActivation(x, layout); - } - }; - const x: Tensor = .init(.{ .token = 1, .out = 6 }, .bf16); - const gates = [_]f32{ 0.75, -1.25, 3.5 }; - const ups = [_]f32{ 0.875, -2.25, 4.5 }; - for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { - var values: [6]zml.floats.BFloat16 = undefined; - for (gates, ups, 0..) |gate, up, i| { - values[if (layout == .concatenated) i else 2 * i] = .fromF32(gate); - values[if (layout == .concatenated) i + 3 else 2 * i + 1] = .fromF32(up); - } - var input = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&values)); - defer input.deinit(); - for ([_]?f32{ null, 2 }) |threshold| { - const activation: Activation = .{ .swiglu = .{ .limit = threshold } }; - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, activation }, .{}); - defer exe.deinit(); - try zml.testing.expectEqualShapes(Shape.init(.{ .token = 1, .out = 3 }, .f32), exe.output_shapes[0]); - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - var actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - for (gates, ups, actual.constItems(f32)) |gate, up, value| { - const g = if (threshold) |limit| @min(gate, limit) else gate; - const u = if (threshold) |limit| std.math.clamp(up, -limit, limit) else up; - try std.testing.expectApproxEqAbs(g / (1 + @exp(-g)) * u, value, 1e-6); - } - } - } -} - -test "FP8 routed GEMM with bias matches dequantized weights" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.SkipZigTest; - - const Local = struct { - const Outputs = struct { actual: Tensor, expected: Tensor }; - - fn expandScales(scales: Tensor, shape: Shape) Tensor { - var expanded = scales.convert(.f32); - for (0..shape.rank()) |axis| { - const factor = @divExact(shape.dim(axis), expanded.dim(axis)); - const broad_shape = expanded.shape().insert(axis + 1, .{factor}); - const axes = Shape.range(broad_shape.rank(), .i64).remove(axis + 1); - expanded = expanded.broadcast(broad_shape, axes.dims()) - .reshape(expanded.shape().setDim(axis, shape.dim(axis))); - } - return expanded.withTags(shape.tags()); - } - - fn forward(x: Tensor, w: Tensor, scales: Tensor, ids: Tensor, weights: Tensor, bias: Tensor, scheme: zml.Quantization.Scheme, mx_native: bool, quantize_input: bool) Outputs { - const weight = w.convert(.f8e4m3fn); - var input = x; - var input_scale: ?Tensor = null; - var reference_input = x.convert(.f32); - if (quantize_input) { - input, input_scale = prepareInput(x, scheme, .f8e4m3fn, true); - const scale = if (scheme == .mxfp8) input_scale.?.bitCast(.f8e8m0) else input_scale.?; - reference_input = input.convert(.f32).withTags(x.shape().tags()) - .mul(expandScales(scale.convert(.f32), x.shape())); - } - const weight_scale = if (scheme == .mxfp8) blk: { - const native = scales.convert(.f8e8m0); - break :blk if (mx_native) native else native.bitCast(.u8); - } else scales; - const config = launchConfigForTokens(8); - const routing = prepareRouting(ids, w.dim(.expert), @intCast(config.block_size_m)); - const actual = gemm(.{ - .input = input, - .weight = weight, - .bias = bias, - .input_scale = input_scale, - .weight_scale = weight_scale, - .routing_weights = weights, - .routing = routing, - .expert_ids = routing.expert_ids, - .launch_config = config, - .quant_scheme = scheme, - .top_k = 1, - .output_shape = Shape.init(.{ .token = x.dim(.token), .topk = 1, .out = w.dim(.out) }, .bf16), - }).squeeze(.topk); - const expanded_weight = weight.convert(.f32).mul(expandScales(scales, weight.shape())); - const selected_weight = expanded_weight.gather(.{ .expert = ids.squeeze(.topk) }, .{}); - const dot = reference_input.dot(selected_weight, .in); - const selected_bias = if (bias.rank() == 2) - bias.gather(.{ .expert = ids.squeeze(.topk) }, .{}) - else - bias; - const unweighted = dot.add(selected_bias.convert(.f32).broad(dot.shape())); - const expected = unweighted.mul(weights.squeeze(.topk).convert(.f32).broad(unweighted.shape())).convert(.bf16); - return .{ .actual = actual, .expected = expected }; - } - }; - - for ([_]zml.Quantization.Scheme{ .mxfp8, .fp8_per_tensor, .fp8_per_channel, .fp8_block128 }) |scheme| { - for ([_]bool{ false, true }) |quantize_input| { - for ([_]i64{ 1, 17 }) |tokens| { - const n: i64 = if (scheme == .fp8_block128) 256 else 48; - const scale_dims: [3]i64 = switch (scheme) { - .mxfp8 => .{ 8, n, 8 }, - .fp8_per_tensor => if (tokens == 1) .{ 1, 1, 1 } else .{ 8, 1, 1 }, - .fp8_per_channel => .{ 8, n, 1 }, - .fp8_block128 => .{ 8, 2, 2 }, - else => unreachable, - }; - const x: Tensor = .init(.{ .token = tokens, .in = 256 }, .bf16); - const w: Tensor = .init(.{ .expert = 8, .out = n, .in = 256 }, .bf16); - const scales: Tensor = .init(.{ scale_dims[0], scale_dims[1], scale_dims[2] }, .f32); - const ids: Tensor = .init(.{ .token = tokens, .topk = 1 }, .i32); - const weights: Tensor = .init(.{ .token = tokens, .topk = 1 }, .bf16); - const bias: Tensor = if (tokens == 1) - .init(.{ .out = n }, .bf16) - else - .init(.{ .expert = 8, .out = n }, .f32); - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, ids, weights, bias, scheme, tokens == 1, quantize_input }, .{}); - defer exe.deinit(); - var buffers: [6]zml.Buffer = undefined; - var initialized: usize = 0; - defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); - for ([_]Shape{ x.shape(), w.shape(), scales.shape(), ids.shape(), weights.shape(), bias.shape() }, 0..) |shape, i| { - const slice = try zml.Slice.alloc(allocator, shape); - defer slice.free(allocator); - for (0..shape.count()) |j| { - switch (i) { - 2 => slice.items(f32)[j] = @as(f32, @floatFromInt(@as(u32, 1) << @as(u5, @intCast(j % 4)))) / 16, - 3 => slice.items(i32)[j] = @intCast((j * 3 + 5) % 8), - 5 => { - const value = @as(f32, @floatFromInt(@as(i32, @intCast(j % 17)) - 8)) / 2; - if (shape.dtype() == .f32) { - slice.items(f32)[j] = value; - } else { - slice.items(zml.floats.BFloat16)[j] = .fromF32(value); - } - }, - else => slice.items(zml.floats.BFloat16)[j] = .fromF32(if (i == 4) - @as(f32, @floatFromInt(1 + j % 3)) / 4 - else - @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4), - } - } - buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); - initialized += 1; - } - var output: zml.Bufferized(Local.Outputs) = undefined; - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - runner.run(io, .{ buffers[0], buffers[1], buffers[2], buffers[3], buffers[4], buffers[5] }, .{&output}, .{ .wait = true }); - defer zml.Buffer.deinitAll(Local.Outputs, &output); - zml.testing.expectClose(io, output.expected, output.actual, .{ .absolute_tolerance = 0.03125, .relative_tolerance = 0.01 }) catch |err| { - log.warn("FP8 routed GEMM failed for scheme={s}, quantize_input={}, tokens={}", .{ @tagName(scheme), quantize_input, tokens }); - return err; - }; - } - } - } -} - -test "fused experts support BF16 and MXFP4 layouts, bias, and routing weights" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.SkipZigTest; - const Local = struct { - fn forward(x: Tensor, layout: ProjectionLayout, placement: RoutingWeightPlacement, storage_dtype: DataType) Tensor { - const fp4 = storage_dtype != .bf16; - const columns = Tensor.arange(.{ .end = 256 }, .i32).withTags(.{.dout}); - const is_gate = switch (layout) { - .concatenated => columns.cmp(.LT, Tensor.scalar(128, .i32)), - .row_interleaved_1 => columns.remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), - .row_interleaved_8 => columns.divByConst(8).remainder(Tensor.scalar(2, .i32)).cmp(.EQ, Tensor.scalar(0, .i32)), - }; - const dtype: DataType = if (fp4) .u8 else .bf16; - // E2M1 codes 1, 2, 3 encode 0.5, 1, 1.5; each byte holds two values. - const gate_value = if (fp4) Tensor.scalar(0x11, .u8) else Tensor.scalar(0.5, .bf16); - const up_value = if (fp4) Tensor.scalar(0x33, .u8) else Tensor.scalar(1.5, .bf16); - const gate_up_values = is_gate.select(gate_value, up_value); - var gate_up: zml.nn.Linear = .{ - .weight = gate_up_values.broad(Shape.init(.{ .expert = 8, .dout = 256, .d = @as(i64, if (fp4) 64 else 128) }, dtype)), - .tag = Shape.toTag(.d), - .quantization = if (fp4) .{ - .scheme = .mxfp4, - .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .dout = 256, .d = 4 }, .u8)), - } else null, - }; - var down: zml.nn.Linear = .{ - .weight = (if (fp4) Tensor.scalar(0x22, .u8) else Tensor.scalar(1, .bf16)).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = @as(i64, if (fp4) 64 else 128) }, dtype)), - .bias = Tensor.scalar(2, .bf16).broad(Shape.init(.{ .expert = 8, .d = 128 }, .bf16)), - .tag = Shape.toTag(.dout), - .quantization = if (fp4) .{ - .scheme = .mxfp4, - .scales = Tensor.scalar(127, .u8).broad(Shape.init(.{ .expert = 8, .d = 128, .dout = 4 }, .u8)), - } else null, - }; - if (storage_dtype == .f4e2m1) { - gate_up.weight = gate_up.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .dout = 256, .d = 128 }); - down.weight = down.weight.bitCast(.f4e2m1).reshape(.{ .expert = 8, .d = 128, .dout = 128 }); - gate_up.quantization.?.scales = gate_up.quantization.?.scales.bitCast(.f8e8m0); - down.quantization.?.scales = down.quantization.?.scales.bitCast(.f8e8m0); - } - const route = Tensor.arange(.{ .end = 2 }, .i32).reshape(.{ .t = 1, .topk = 2 }) - .broad(Shape.init(.{ .t = x.dim(.t), .topk = 2 }, .i32)); - return fusedExpertsBase(x, .{ - .ids = route.addConstant(2), - .weights = route.convert(.f32).addConstant(1).scale(0.25), - .global_num_experts = 8, - .local_num_experts = 8, - }, gate_up, down, layout, .{ - .routing_weight_placement = placement, - .quantize_input = false, - .activation = .{ .swiglu = .{} }, - }); - } - }; - for ([_]DataType{ .bf16, .u8, .f4e2m1 }) |storage_dtype| { - const tokens: i64 = if (storage_dtype == .u8) 17 else 1; - const x: Tensor = .init(.{ .t = tokens, .d = 128 }, .bf16); - const host = try zml.Slice.alloc(allocator, x.shape()); - defer host.free(allocator); - @memset(host.items(zml.floats.BFloat16), .fromF32(1.0 / 128.0)); - var input = try zml.Buffer.fromSlice(io, platform, host); - defer input.deinit(); - for ([_]ProjectionLayout{ .concatenated, .row_interleaved_1 }) |layout| { - for ([_]RoutingWeightPlacement{ .before_down, .after_down }) |placement| { - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, layout, placement, storage_dtype }, .{}); - defer exe.deinit(); - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - var actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - const activation: f32 = 0.5 / (1 + @exp(@as(f32, -0.5))) * 1.5; - // The two routes have weights 0.25 and 0.5. Bias distinguishes placement. - const bias: f32 = if (placement == .before_down) 4 else 1.5; - const expected = 128 * activation * 0.75 + bias; - for (actual.constItems(zml.floats.BFloat16)) |value| { - try std.testing.expectApproxEqAbs(expected, value.toF32(), 0.5); - } - } - } - } -} - -// ============================================================================= -// Config / validation helpers -// ============================================================================= - -const LaunchConfig = struct { - block_size_m: usize, - block_size_n: usize, - block_size_k: usize, - group_size_m: usize, - num_warps: i32, - num_stages: i32, -}; - -const launch_configs = [_]struct { tokens: i64, config: LaunchConfig }{ - .{ .tokens = 1, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, - .{ .tokens = 2, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 4 } }, - .{ .tokens = 4, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 8, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 16, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 64, .group_size_m = 64, .num_warps = 4, .num_stages = 5 } }, - .{ .tokens = 24, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 32, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 48, .config = .{ .block_size_m = 16, .block_size_n = 32, .block_size_k = 128, .group_size_m = 64, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 64, .config = .{ .block_size_m = 16, .block_size_n = 64, .block_size_k = 128, .group_size_m = 1, .num_warps = 4, .num_stages = 2 } }, - .{ .tokens = 96, .config = .{ .block_size_m = 16, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 128, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 256, .config = .{ .block_size_m = 16, .block_size_n = 256, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 2 } }, - .{ .tokens = 512, .config = .{ .block_size_m = 32, .block_size_n = 128, .block_size_k = 128, .group_size_m = 1, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 1024, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 1536, .config = .{ .block_size_m = 64, .block_size_n = 128, .block_size_k = 64, .group_size_m = 1, .num_warps = 4, .num_stages = 3 } }, - .{ .tokens = 2048, .config = .{ .block_size_m = 128, .block_size_n = 128, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 3 } }, - .{ .tokens = 3072, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 1, .num_warps = 8, .num_stages = 4 } }, - .{ .tokens = 4096, .config = .{ .block_size_m = 128, .block_size_n = 256, .block_size_k = 64, .group_size_m = 16, .num_warps = 8, .num_stages = 4 } }, -}; - -fn launchConfigForTokens(num_tokens: i64) LaunchConfig { - var best = launch_configs[0]; - for (launch_configs[1..]) |candidate| { - // The table is ascending, so ties retain the smaller token bucket. - if (@abs(candidate.tokens - num_tokens) < @abs(best.tokens - num_tokens)) { - best = candidate; - } - } - return best.config; -} - -test "MoE launch config selects the nearest token bucket" { - try std.testing.expectEqualDeep(launchConfigForTokens(4), launchConfigForTokens(6)); - try std.testing.expectEqualDeep(launchConfigForTokens(8), launchConfigForTokens(7)); - try std.testing.expectEqualDeep(launchConfigForTokens(1), launchConfigForTokens(0)); - try std.testing.expectEqualDeep(launchConfigForTokens(4096), launchConfigForTokens(8192)); - try std.testing.expectEqual(@as(usize, 64), launchConfigForTokens(16).group_size_m); - try std.testing.expectEqual(@as(i32, 5), launchConfigForTokens(16).num_stages); -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - const cc = zml.platform.cuda.computeCapability(platform) orelse return false; - return cc.major == 10; -} - -pub fn validateOptions(opts: Options) void { - stdx.debug.assert(opts.activation == .swiglu, "triton backend only accepts swiglu activation, got {}", .{opts.activation}); - stdx.debug.assert(opts.activation.swiglu.limit != null, "triton backend requires swiglu limit to be set", .{}); - stdx.debug.assert(opts.activation.swiglu.bias == null, "triton backend requires swiglu bias to be null", .{}); - stdx.debug.assert(opts.activation.swiglu.scale == null, "triton backend requires swiglu scale to be null", .{}); -} - -pub const Config = struct { - tokens: i64, - hidden: i64, - intermediate: i64, - experts: i64, - global_experts: i64, - topk: i64, - expert_offset: i64 = 0, - // Swiglu clamping. Set to 0 to disable. - swiglu_limit: f32, - /// Scale SwiGLU before FP8 quantization, or scale the down result before reduction. - routing_weight_placement: RoutingWeightPlacement = .before_down, - - pub fn validate(c: Config) !void { - if (c.tokens <= 0 or - c.experts <= 0 or - c.topk <= 0 or c.topk > c.global_experts or - c.expert_offset < 0 or c.expert_offset + c.experts > c.global_experts) - { - return error.InvalidRoutingShape; - } - - if (c.hidden < 128 or @mod(c.hidden, 128) != 0 or - c.intermediate < 128 or @mod(c.intermediate, 128) != 0) - { - return error.InvalidHiddenShape; - } - - if (!std.math.isFinite(c.swiglu_limit) or c.swiglu_limit < 0) { - return error.InvalidScale; - } - } -}; - -fn validateInputs( - c: Config, - input: zml.Tensor, - ids: zml.Tensor, - expert_weights: zml.Tensor, - gate_up: zml.Tensor, - gate_up_scales: zml.Tensor, - down: zml.Tensor, - down_scales: zml.Tensor, -) void { - stdx.debug.assert(gate_up_scales.rank() == 3 and down_scales.rank() == 3, "Invalid MXFP4 tensor layout, expected gate_up_scales and down_scales to be rank 3, got {f} and {f}", .{ gate_up_scales, down_scales }); - - const shapes = .{ - .input = zml.Shape.init(.{ c.tokens, c.hidden }, .bf16), - .ids = zml.Shape.init(.{ c.tokens, c.topk }, .i32), - .expert_weights = zml.Shape.init(.{ c.tokens, c.topk }, .f32), - .gate_up = zml.Shape.init(.{ c.experts, 2 * c.intermediate, @divExact(c.hidden, 2) }, .i8), - .gate_up_scales = zml.Shape.init(.{ c.experts, 2 * c.intermediate, gate_up_scales.dim(2) }, .f8e8m0), - .down = zml.Shape.init(.{ c.experts, c.hidden, @divExact(c.intermediate, 2) }, .i8), - .down_scales = zml.Shape.init(.{ c.experts, c.hidden, down_scales.dim(2) }, .f8e8m0), - }; - - stdx.debug.assert(input.shape().eql(shapes.input) and - ids.shape().eql(shapes.ids) and - expert_weights.shape().eql(shapes.expert_weights) and - // Note: we accept both i8 and u8 for gate_up and down - // TODO: we should instead use f4e2m1 - (gate_up.shape().eql(shapes.gate_up) or gate_up.shape().eql(shapes.gate_up.withDtype(.u8))) and - gate_up_scales.shape().eql(shapes.gate_up_scales) and - (down.shape().eql(shapes.down) or down.shape().eql(shapes.down.withDtype(.u8))) and - down_scales.shape().eql(shapes.down_scales), - \\Invalid MXFP4 tensor layout: - \\ - expected input={f}, got {f} - \\ - expected ids={f}, got {f} - \\ - expected expert_weights={f}, got {f} - \\ - expected gate_up={f}, got {f} - \\ - expected gate_up_scales={f}, got {f} - \\ - expected down={f}, got {f} - \\ - expected down_scales={f}, got {f} - , .{ - shapes.input, input, // - shapes.ids, ids, // - shapes.expert_weights, expert_weights, // - shapes.gate_up, gate_up, // - shapes.gate_up_scales, gate_up_scales, // - shapes.down, down, // - shapes.down_scales, down_scales, // - }); - - stdx.debug.assert( - gate_up_scales.dim(2) >= @divTrunc(c.hidden, 32) and - down_scales.dim(2) >= @divTrunc(c.intermediate, 32), - "Invalid MXFP4 tensor layout, not enough scales, expected a ratio of at most 32 for config {}, got {f} and {f}", - .{ c, gate_up_scales, down_scales }, - ); -} - -/// Row-major MXFP4 E2M1 weights and linear E8M0 block32 scales. -/// BF16 activations are quantized on the GPU to FP8 with per-32 E8M0 scales. -pub fn fusedExpertsMxfp4( - input: zml.Tensor, - routing: LocalRouting, - gate_up: zml.nn.Linear, - down: zml.nn.Linear, - options: Options, -) zml.Tensor { - validateOptions(options); - stdx.debug.assert(input.dtype() == .bf16, "triton backend only supports bf16 inputs, got {}", .{input.dtype()}); - stdx.debug.assert(gate_up.bias == null and down.bias == null, "triton backend expects gate_up bias and down bias to be null", .{}); - - const gq = gate_up.quantization orelse @panic("triton backend requires gate_up quantization to be set"); - const dq = down.quantization orelse @panic("triton backend requires down quantization to be set"); - stdx.debug.assert(gq.scheme == .mxfp4 and dq.scheme == .mxfp4, "triton expects gate_up and down quantization scheme to be mxfp4, got {} and {}", .{ gq.scheme, dq.scheme }); - - // Weight storage for mxfp4 in HF is expressed as u8 or i8 - stdx.debug.assert(gate_up.weight.dtype() == .u8 or gate_up.weight.dtype() == .i8, "triton expects gate_up weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - stdx.debug.assert(down.weight.dtype() == .u8 or down.weight.dtype() == .i8, "triton expects down weight dtype to be u8 or i8, got {}", .{gate_up.weight.dtype()}); - - const topk = routing.ids.dim(.topk); - const experts = gate_up.weight.dim(.expert); - const hidden = down.weight.dim(1); - const intermediate = down.weight.dim(2) * 2; - const ids = routing.localIds().convert(.i32).reshape(.{ .token = .auto, .topk = topk }); - const tokens = ids.dim(.token); - - const x = input.reshape(.{ tokens, hidden }); - const scales = routing.weights.convert(.f32).reshape(.{ tokens, topk }); - - const cfg: Config = .{ - .tokens = tokens, - .hidden = hidden, - .intermediate = intermediate, - .experts = experts, - .global_experts = routing.global_num_experts, - .topk = topk, - .swiglu_limit = options.activation.swiglu.limit.?, - .routing_weight_placement = options.routing_weight_placement, - }; - - cfg.validate() catch @panic("Invalid MXFP4 MoE configuration"); - validateInputs(cfg, x, ids, scales, gate_up.weight, gq.scales, down.weight, dq.scales); - - const bn = 16; - const bk = 512; - - const tile_count = kernels.tiles(cfg.tokens, cfg.topk, cfg.experts, bn); - - // Quantize activations - const q = kernels.Quant.Kernel.call( - .{ .x = x }, - .{ - .q = .init(.{ cfg.tokens, cfg.hidden }, .f8e4m3fn), - .s = .init(.{ cfg.tokens, @divTrunc(cfg.hidden, 32) }, .u8), - }, - .{ - .cfg = .{ .tokens = cfg.tokens, .hidden = cfg.hidden }, - .grid = .{ @intCast(@divTrunc(cfg.tokens + kernels.quantRows(cfg.tokens) - 1, kernels.quantRows(cfg.tokens))), @intCast(@divTrunc(cfg.hidden, 128)), 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - - const dummy = zml.Tensor.scalar(0, .i32); - var map = dummy; - var perm = dummy; - var sched = dummy; - - if (cfg.tokens != 1) { - const r = kernels.Route.Kernel.call( - .{ .ids = ids }, - .{ .counts = .init(.{cfg.experts}, .i32), .pos = .init(.{cfg.tokens * cfg.topk}, .i32) }, - .{ .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - }, .grid = .{ @intCast(cfg.experts), 1, 1 }, .num_warps = 4, .num_stages = 1 }, - ); - - const s = kernels.Schedule.Kernel.call( - .{ .ids = ids, .counts = r.counts, .pos = r.pos }, - .{ .map = .init(.{tile_count * bn}, .i32), .perm = .init(.{cfg.tokens * cfg.topk}, .i32), .sched = .init(.{2 * tile_count + 1}, .i32) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .capacity = tile_count, - .group = bn, - }, - .grid = .{ @intCast(@divTrunc(cfg.tokens * cfg.topk + 127, 128)), 1, 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ); - - map = s.map; - perm = s.perm; - sched = s.sched; - } - - // Each narrow tile produces 64 activations: two complete scale groups. - const up_columns: i64 = if (cfg.tokens <= 8) 64 else 128; - const m = kernels.GateUp.Kernel.call( - .{ .q = q.q, .s = q.s, .w = gate_up.weight.bitCast(.u8), .ws = gq.scales.bitCast(.u8), .ids = ids, .rw = scales, .map = map, .sched = sched }, - .{ .mid = .init(.{ tile_count * bn, cfg.intermediate }, .f8e4m3fn), .ms = .init(.{ tile_count * bn, @divTrunc(cfg.intermediate, 32) }, .u8) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = gq.scales.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.intermediate, up_columns)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.intermediate, up_columns) * tile_count), - }, - ); - - const d = kernels.Down.Kernel.call( - .{ .mid = m.mid, .ms = m.ms, .w = down.weight.bitCast(.u8), .ws = dq.scales.bitCast(.u8), .ids = ids, .sched = sched }, - .{ .d = .init(.{ tile_count * bn, cfg.hidden }, .bf16) }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .intermediate = cfg.intermediate, - .experts = cfg.experts, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .swiglu_limit = cfg.swiglu_limit, - .scale_stride = dq.scales.dim(2), - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - .bk = bk, - }, - .grid = .{ @intCast(@divTrunc(cfg.hidden, 128)), @intCast(tile_count), 1 }, - .num_warps = if (cfg.tokens <= 8) 4 else 8, - .num_stages = 2, - .global_scratch_memory_size = @intCast(128 * @divTrunc(cfg.hidden, 128) * tile_count), - }, - ); - - return kernels.Combine.Kernel.call( - .{ .d = d.d, .ids = ids, .perm = perm, .rw = scales }, - .{ .y = x.shape() }, - .{ - .cfg = .{ - .tokens = cfg.tokens, - .hidden = cfg.hidden, - .topk = cfg.topk, - .expert_offset = cfg.expert_offset, - .experts = cfg.experts, - .routing_weight_placement = cfg.routing_weight_placement, - .bn = bn, - }, - .grid = .{ @intCast(cfg.tokens), @intCast(@divTrunc(cfg.hidden + kernels.combineBlock(cfg.tokens, cfg.hidden) - 1, kernels.combineBlock(cfg.tokens, cfg.hidden))), 1 }, - .num_warps = 4, - .num_stages = 1, - }, - ).y.reshape(input.shape().dims()).withTags(input.shape()); -} diff --git a/zml/kernels/moe/weights.zig b/zml/kernels/moe/weights.zig deleted file mode 100644 index 0e288856..00000000 --- a/zml/kernels/moe/weights.zig +++ /dev/null @@ -1,10 +0,0 @@ -const nn = @import("../../nn.zig"); -const Tensor = @import("../../tensor.zig").Tensor; - -pub fn unpackedWeight(linear: nn.Linear) Tensor { - const quantization = linear.quantization orelse return linear.weight; - return if (nn.isPackedFp4(quantization.scheme, linear.weight.dtype())) - nn.unpackFp4(linear.weight, linear.tag, linear.tag) - else - linear.weight; -} diff --git a/zml/kernels/paged_attention/README.md b/zml/kernels/paged_attention/README.md deleted file mode 100644 index 20ba9d7d..00000000 --- a/zml/kernels/paged_attention/README.md +++ /dev/null @@ -1,184 +0,0 @@ -# Paged attention semantics - -[paged_attention.zig](paged_attention.zig) evaluates attention over a KV cache -whose logical sequences are stored in fixed-size physical pages. Paging changes -how keys and values are addressed; softmax still normalizes over all permitted -keys in one sequence, independently for each query and query head. - -The public entry point is: - -```zig -pagedAttention(parameters, q, k, v, kv_cache, opts) -``` - -The result has the query's shape and dtype. The `k` and `v` arguments are currently -ignored: attention reads exclusively from `kv_cache`. Populate the cache, -including the current queries' keys and values, **before** calling attention. -The attention call does not update the cache or allocate pages. Projections, -RoPE, and any Q/K normalization happen before this operation; the output -projection happens afterward. - -## Tensor layouts and heads - -Let `T` be the packed query token count, `B` the sequence capacity, `N` the -physical page count, `P` the tokens per page, `M` the page-table capacity per -sequence, `Hkv` the KV head count, `G` the query heads per KV head, and `D` the -features per head. - -| Input | Tagged shape | Meaning | -| --- | --- | --- | -| `q` | `{ .b = T, .hkv = Hkv, .hg = G, .hd = D }` | Queries packed in sequence order. Here `.b` counts tokens, not sequences. | -| Split cache `k`, `v` | `{ .page = N, .k_chunk = P, .hkv = Hkv, .hd = D }` | Physical key/value storage. | -| `parameters.block_table` | `{ .b = B, .p = M }`, `i32` | Logical page to physical page mapping for each sequence. | -| `parameters.seq_lens` | `{ .b = B }`, `i32` | Valid KV token counts, including the current queries. | -| `parameters.query_start_len` | `{ .b = B + 1 }`, `i32` | Cumulative offsets into packed queries. | - -The total query head count is `Hq = Hkv * G`. Flattening the two head axes gives -`h = hkv * G + hg`, so query head `h` reads KV head `floor(h / G)`. Each query -head has its own scores and softmax. This covers MHA (`G = 1`), GQA, and MQA -(`Hkv = 1`). `D` is the feature count of one head, not the number of heads or -the sequence length. Use matching floating-point query and cache dtypes. - -The TPU backend instead requires `KvCache.dense`, a fused tensor with layout -`{ .page = N, .k_chunk = P, .hkv = 2 * Hkv, .hd = Dpad }`. The combined head -axis interleaves `K0, V0, K1, V1, ...`. `Dpad` must equal `D` rounded up to a -multiple of 128. Pad unused cache features with zeros. The wrapper pads queries, -uses the logical `D` for the default scale, and trims the output back to `D`. -Its accepted query/cache dtypes are `bf16`, `f16`, and `f32`. - -## Packed sequences and positions - -For sequence `s`, define: - -```text -start = query_start_len[s] -end = query_start_len[s + 1] -Lq = end - start -Lk = seq_lens[s] -context_len = Lk - Lq -position(i) = context_len + i # 0 <= i < Lq -``` - -Queries `q[start:end]` represent the final `Lq` tokens of that sequence. The -kernel infers their positions from lengths; there is no separate position input. -An initial prefill has `Lq = Lk`, a chunked prefill has `Lq > 1` after a cached -prefix, and a decode normally has `Lq = 1` at position `Lk - 1`. A batch may mix -prefill and decode sequences. Set `is_prefill = true` for such mixed batches; -`false` selects the all-decode execution configuration on Triton. It does not -change the attention equation. `max_seqlen_q` is a configuration bound for the -largest query chunk, not a runtime sequence length. - -Offsets must begin at zero and be nondecreasing. Active sequences require -`0 < Lq <= Lk <= M * P`. Place inactive sequences after the active sequences, -with repeated offsets and normally `seq_lens = 0`. This ordering matters: -StableHLO stops at the first sequence with no queries, and TPU passes a count of -active sequences to its kernel. The final offset is the active packed token -count; any extra query capacity is padding whose output should not be consumed. - -## Page addressing and cache updates - -A logical key position `j` in sequence `s` is addressed as: - -```text -logical_page = floor(j / P) -physical_page = block_table[s, logical_page] -offset = j % P -K(s, j) = k_cache[physical_page, offset] -V(s, j) = v_cache[physical_page, offset] -``` - -Physical pages need not be contiguous or ordered. Different sequences may map -to the same physical pages to share a prefix. Every page covering a valid token -must have a valid physical ID in `[0, N)` and populated KV data. Entries beyond -`ceil(Lk / P)` are unused; tests use `-1` there. A negative ID is not a portable -way to mask a valid logical page. `seq_lens`, rather than allocated page count, -determines which tokens in the last page are valid. - -`KvCache.update(new_k, new_v, slot_mapping, chunk_size, backend)` writes split -cache entries using physical flattened slots: - -```text -chunk_size = P -physical_page = slot_mapping[t] / P -offset = slot_mapping[t] % P -``` - -`new_k` and `new_v` have packed token/head/feature axes matching the split -cache. `slot_mapping` is a physical address, whereas `block_table` translates -logical sequence positions. The caller maintains both mappings and lengths; -`update` does not derive or update sequence metadata. Writes override existing -entries and request buffer reuse. Use valid, distinct slots for writes requiring -deterministic results; conflicting scatter writes have no ordering guarantee. -The generic update method currently panics for `.dense`. - -## Attention equation and masks - -For local query row `i`, KV head `a`, and query group head `g`: - -```text -p = Lk - Lq + i -score_j = scale * dot(q[start + i, a, g], K(s, j, a)) -weight = softmax(score over permitted j) -out = sum_j(weight_j * V(s, j, a)) -``` - -`opts.scale` defaults to `1 / sqrt(D)`. There is no dropout or returned -probability matrix. Internal paging, tiling, or splitting of the key range must -preserve one softmax normalization; summing separately normalized page outputs -would compute a different result. Accumulation order and dtype conversions can -cause numerical differences between backends. - -The default mask is `.standard = .{ .causal = true }`: - -| Mask | Permitted logical key positions | -| --- | --- | -| Default causal | `0 <= j < Lk` and `j <= p`. Includes the query's own key. | -| `AttentionMask.none` | Every `0 <= j < Lk`. | -| Causal with `sliding_window = W` | `0 <= j < Lk` and `p - W < j <= p`. | -| Non-causal with `sliding_window = W` | `0 <= j < Lk` and `p - W < j`. Includes all later valid keys. | - -Use a positive `W`; it counts the current token as part of the left window. -Use `null` for no window. A non-causal window is a left bound, not a symmetric -window around the query. - -Triton also accepts `.indices`, an `i32` tensor shaped -`{ .b = T, .topk = K }` with `K > 0`. Each packed query row supplies logical -key positions in its own sequence, shared across its heads. Positions outside -`[0, Lk)` are ignored. This replaces the standard mask, so future valid keys -are permitted. Duplicate positions are processed repeatedly and therefore -change their weight; provide unique positions to represent a set of keys. - -On Triton, `opts.sink` supplies one softmax logit per flattened query head. -It adds `exp(sink[h])` to the denominator and contributes zero to the weighted -value numerator. It is not a cache token and does not consume a page or sequence -position. Other wrappers currently ignore `sink`. For portable use, ensure each -active query has at least one permitted key; fully masked rows are not given a -shared cross-backend output contract. - -## Example - -With `P = 4`, consider a two-query chunk after a four-token prefix and a decode -after a two-token prefix, followed by one inactive sequence: - -```text -query_start_len = [0, 2, 3, 3] -seq_lens = [6, 3, 0] -block_table = [[5, 2], [8, -1], [-1, -1]] -``` - -Packed queries 0 and 1 belong to sequence 0 at positions 4 and 5. Their current -KV entries occupy physical page 2, offsets 0 and 1 (slots 8 and 9). Packed query -2 belongs to sequence 1 at position 2, physical page 8, offset 2 (slot 34). -The default causal mask permits keys 0–4, 0–5, and 0–2 respectively, always -within the query's own sequence. With a window of 3, the permitted positions -become 2–4, 3–5, and 0–2. - -## Backend implementations - -See each backend's README for execution details, supported options, and current -limitations: - -- [Triton](triton/README.md) — CUDA, ROCm, and oneAPI. -- [Mosaic TPU](mosaic_tpu/README.md) — TPU. -- [Metal](metal/README.md) — Metal. -- [StableHLO](stablehlo/README.md) — CPU and Furiosa fallback. diff --git a/zml/kernels/paged_attention/cute/BUILD.bazel b/zml/kernels/paged_attention/cute/BUILD.bazel deleted file mode 100644 index 6329ae12..00000000 --- a/zml/kernels/paged_attention/cute/BUILD.bazel +++ /dev/null @@ -1,47 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_binary", "zig_test") - -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "cute", - srcs = [ - "attention.zig", - "cute.zig", - "//zml/kernels/paged_attention/cute/sm10x", - "//zml/kernels/paged_attention/cute/sm120", - ], -) - -zig_test( - name = "correctness_test", - srcs = [ - "cases.zig", - "fixtures.zig", - "reference.zig", - ], - main = "correctness_test.zig", - tags = [ - "exclusive", - "requires-gpu", - ], - target_compatible_with = select({ - "//platforms:cuda.enabled": [], - "//conditions:default": ["@platforms//:incompatible"], - }), - deps = ["//zml"], -) - -zig_binary( - name = "benchmark", - srcs = [ - "cases.zig", - "fixtures.zig", - ], - main = "benchmark.zig", - tags = ["manual"], - target_compatible_with = select({ - "//platforms:cuda.enabled": [], - "//conditions:default": ["@platforms//:incompatible"], - }), - deps = ["//zml"], -) diff --git a/zml/kernels/paged_attention/cute/attention.zig b/zml/kernels/paged_attention/cute/attention.zig deleted file mode 100644 index 50eb2531..00000000 --- a/zml/kernels/paged_attention/cute/attention.zig +++ /dev/null @@ -1,117 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../../zml.zig"); -const cute = zml.kernel.cute; -const V = cute.Value; - -pub const Config = struct { - head_dim: i64, - tokens: i64, - batch: i64, - kv_heads: i64, - group: i64, - pages: i64, - pages_per_seq: i64, - page_size: i64, - dtype: cute.DType, - causal: bool, - window: i64, - scale: f32, - sink: bool, - indexed: bool, - topk: i64, -}; - -pub const Kernel = cute.Kernel(Config, .{ - .name = "paged_attention_cute_generic", - .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink", "indices" }, - .outputs = &.{"out"}, - .run = run, -}); - -fn run(b: *cute.Builder, c: Config) cute.FinishError!void { - switch (c.head_dim) { - 32 => try runDim(b, c, 1), - 64 => try runDim(b, c, 2), - 128 => try runDim(b, c, 4), - 256 => try runDim(b, c, 8), - 512 => try runDim(b, c, 16), - else => unreachable, - } -} - -fn runDim(b: *cute.Builder, c: Config, comptime chunks: usize) cute.FinishError!void { - const a = try b.declareArgs(.{ - .q = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, - .k = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.pages, c.page_size, c.kv_heads, c.head_dim } } }, - .v = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.pages, c.page_size, c.kv_heads, c.head_dim } } }, - .table = .{ .tensor = .{ .dtype = .i32, .shape = &.{ c.batch, c.pages_per_seq } } }, - .lengths = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch} } }, - .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, - .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, - .indices = .{ .tensor = .{ .dtype = .i32, .shape = &.{ c.tokens, c.topk } } }, - .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, - }); - const token = b.blockIdx().x; - const head = b.blockIdx().y; - const lane = b.threadIdx().x; - const zero = b.constant(.f32, 0); - // One full warp owns a query/head; all control flow is warp-uniform. - // Also define output for unused capacity at the end of the packed queries. - for (0..chunks) |i| a.out.set(.{ token, head, lane.add(i * 32) }, zero.to(c.dtype)); - var active = b.openIf(token.lt(a.starts.get(.{c.batch}))); - { - var find_seq = b.openFor(0, c.batch, 1, .{b.constant(.i32, 0)}); - { - const owns = token.ge(a.starts.get(.{find_seq.iv})).bitAnd(token.lt(a.starts.get(.{find_seq.iv.add(1)}))); - find_seq.yield(.{b.select(owns, find_seq.iv, find_seq.carried[0])}); - } - const seq = find_seq.results[0]; - const start = a.starts.get(.{seq}); - const qlen = a.starts.get(.{seq.add(1)}).sub(start); - const len = a.lengths.get(.{seq}); - const position = len.sub(qlen).add(token.sub(start)); - const begin = if (c.window > 0) position.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); - const end = if (c.causal) position.add(1).minimum(len) else len; - var query: [chunks]V = undefined; - for (&query, 0..) |*q, i| q.* = a.q.get(.{ token, head, lane.add(i * 32) }).to(.f32); - - // State is running max, normalizer, then each lane's output features. - // A sink is an initial softmax entry with zero value. - var initial: [chunks + 2]V = @splat(zero); - initial[0] = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); - initial[1] = if (c.sink) b.constant(.f32, 1) else zero; - var keys = b.openFor(if (c.indexed) b.constant(.i32, 0) else begin, if (c.indexed) b.constant(.i32, c.topk) else end, 1, initial); - { - const key = if (c.indexed) a.indices.get(.{ token, keys.iv }) else keys.iv; - const valid = key.ge(0).bitAnd(key.lt(len)); - // Invalid indexed entries must not load the page table, K, or V. Duplicates - // deliberately contribute once per occurrence, as independent softmax entries. - var entry = b.openIfElse(valid, @as([chunks + 2]V, @splat(zero))); - { - const page = a.table.get(.{ seq, key.div(c.page_size) }); - const offset = key.rem(c.page_size); - const kv_head = head.div(c.group); - var score = zero; - for (query, 0..) |q, i| score = score.add(q.mul(a.k.get(.{ page, offset, kv_head, lane.add(i * 32) }).to(.f32))); - inline for (.{ 16, 8, 4, 2, 1 }) |step| score = score.add(b.shuffleXor(score, step)); - score = score.mul(c.scale * std.math.log2e); - const maximum = keys.carried[0].maximum(score); - const alpha = b.exp2Approx(keys.carried[0].sub(maximum)); - const weight = b.exp2Approx(score.sub(maximum)); - var next: [chunks + 2]V = undefined; - next[0] = maximum; - next[1] = keys.carried[1].mul(alpha).add(weight); - for (0..chunks) |i| next[i + 2] = keys.carried[i + 2].mul(alpha).add(weight.mul(a.v.get(.{ page, offset, kv_head, lane.add(i * 32) }).to(.f32))); - entry.yieldThen(next); - } - { - entry.yieldElse(keys.carried); - } - keys.yield(entry.results); - } - const norm = b.select(keys.results[1].gt(0), b.rcpApprox(keys.results[1]), zero); - for (0..chunks) |i| a.out.set(.{ token, head, lane.add(i * 32) }, keys.results[i + 2].mul(norm).to(c.dtype)); - active.close(); - } -} diff --git a/zml/kernels/paged_attention/cute/benchmark.zig b/zml/kernels/paged_attention/cute/benchmark.zig deleted file mode 100644 index 587af69f..00000000 --- a/zml/kernels/paged_attention/cute/benchmark.zig +++ /dev/null @@ -1,145 +0,0 @@ -const std = @import("std"); -const zml = @import("zml"); -const cases = @import("cases.zig"); -const fixtures = @import("fixtures.zig"); -const Inputs = fixtures.Inputs; - -pub fn main(init: std.process.Init) !void { - const platform = try zml.Platform.auto(init.gpa, init.io, .{}); - defer platform.deinit(init.gpa, init.io); - - if (platform.target != .cuda) return error.CudaRequired; - - const mesh = try platform.registerMesh("tp", .mesh(.{ .model = .high_bandwidth })); - - const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnknownComputeCapability; - std.debug.print("PAGED_ATTENTION dtype=bf16 page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ - cc.major, - cc.minor, - platform.fmtVerbose(), - }); - - std.debug.print("timing=synchronized_invocation_us devices={d} hkv=per_device (KV heads sharded across all visible GPUs; includes runtime, output allocation, attention, merge and wait; excludes compile, upload and output destruction)\n", .{platform.devices.len}); - var failures: usize = 0; - for (cases.benchmarks) |c| { - std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} mask={s} backend=cute\n", .{ - c.name, - c.head_dim, - c.batch, - c.kv_heads, - c.group, - c.prefill, - if (c.prefill > 0) c.chunk else 1, - c.kv_len, - @tagName(c.mask), - }); - - runBenchmark(init.gpa, init.io, platform, mesh, c) catch |err| { - failures += 1; - std.debug.print("FAIL error={s}\n", .{@errorName(err)}); - }; - } - - if (failures != 0) return error.BenchmarkFailed; -} - -fn runBenchmark(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, mesh: *const zml.Mesh, c: cases.Case) !void { - // Keep number of head per GPU constant. - var global_case = c; - global_case.kv_heads *= platform.devices.len; - var inputs = Inputs.init(global_case); - inputs.q = inputs.q.withPartitioning(mesh, .{ .hkv = .model }); - inputs.k = inputs.k.withPartitioning(mesh, .{ .hkv = .model }); - inputs.v = inputs.v.withPartitioning(mesh, .{ .hkv = .model }); - inputs.sinks = inputs.sinks.withPartitioning(mesh, .{ ._0 = .model }); - - const compile_start: std.Io.Timestamp = .now(io, .awake); - const exe = try platform.compileFn(a, io, forward, .{ inputs, global_case, mesh }, .{ - .program_name = "paged_attention_harness", - .meshes = &.{mesh}, - }); - defer exe.deinit(); - const compile_us = elapsedUs(compile_start, io); - - var runner = try exe.runner(a); - defer runner.deinit(a); - - var arena = std.heap.ArenaAllocator.init(a); - defer arena.deinit(); - - const fixture = try fixtures.Fixture.init(arena.allocator(), global_case); - - var buffers = try fixtures.upload(io, platform, inputs, fixture); - defer zml.Buffer.deinitAll(Inputs, &buffers); - - var output: zml.Buffer = undefined; - - // Warmup - for (0..5) |_| { - runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); - output.deinit(); - } - - var samples: [101]f64 = undefined; - var sample_count: usize = 0; - var accumulated_us: f64 = 0; - while (sample_count < samples.len and (sample_count < 21 or accumulated_us < 100_000)) { - const call_start: std.Io.Timestamp = .now(io, .awake); - runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); - samples[sample_count] = elapsedUs(call_start, io); - accumulated_us += samples[sample_count]; - sample_count += 1; - output.deinit(); - } - - std.mem.sort(f64, samples[0..sample_count], {}, std.sort.asc(f64)); - const median = if (sample_count % 2 == 0) - (samples[sample_count / 2 - 1] + samples[sample_count / 2]) / 2 - else - samples[sample_count / 2]; - - std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compile_ms={d:.3} n={d} median_us={d:.3} p10_us={d:.3} p90_us={d:.3}\n", .{ - c.name, - c.head_dim, - c.batch, - c.kv_heads, - c.group, - c.prefill, - if (c.prefill > 0) c.chunk else 1, - c.kv_len, - compile_us / 1000, - sample_count, - median, - samples[sample_count / 10], - samples[(sample_count - 1) * 9 / 10], - }); -} - -fn forward(inputs: Inputs, c: cases.Case, mesh: *const zml.Mesh) zml.Tensor { - const Call = struct { - inputs: Inputs, - case: cases.Case, - - fn local(self: @This(), _: zml.Shape) zml.Tensor { - return fixtures.forward(self.inputs, self.case); - } - }; - const heads: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - return zml.ops.manualComputation(Call.local, .{ .inputs = inputs, .case = c }, inputs.q.shape(), mesh, .{ - .in_specs = .{ .inputs = .{ - .q = heads, - .k = heads, - .v = heads, - .table = .replicated, - .lengths = .replicated, - .starts = .replicated, - .sinks = .parse(.{ ._0 = .model }), - .indices = .replicated, - } }, - .out_specs = heads, - }); -} - -fn elapsedUs(start: std.Io.Timestamp, io: std.Io) f64 { - return @as(f64, @floatFromInt(start.untilNow(io, .awake).toNanoseconds())) / 1000; -} diff --git a/zml/kernels/paged_attention/cute/cases.zig b/zml/kernels/paged_attention/cute/cases.zig deleted file mode 100644 index e8dedeaa..00000000 --- a/zml/kernels/paged_attention/cute/cases.zig +++ /dev/null @@ -1,103 +0,0 @@ -const std = @import("std"); - -pub const page_size = 16; -pub const topk = 8; -pub const Mask = enum { causal, non_causal, window, non_causal_window, indices }; - -pub const Case = struct { - name: []const u8 = "standard", - head_dim: usize = 128, - batch: usize = 2, - kv_heads: usize = 2, - group: usize = 4, - prefill: usize = 0, - chunk: usize = 17, - kv_len: usize = 65, - kv_capacity: ?usize = null, - inactive: usize = 0, - shuffle_pages: bool = false, - skew: bool = false, - mask: Mask = .causal, - empty_indices: bool = false, - window: u32 = 17, - sink: bool = false, - scale: ?f32 = null, - - pub fn tokens(c: Case) usize { - return c.prefill * c.chunk + c.batch - c.prefill - c.inactive; - } - pub fn pages(c: Case) usize { - return std.math.divCeil(usize, c.kv_capacity orelse c.kv_len, page_size) catch unreachable; - } - pub fn heads(c: Case) usize { - return c.kv_heads * c.group; - } - pub fn queryLen(c: Case, seq: usize) usize { - return if (seq >= c.batch - c.inactive) 0 else if (seq < c.prefill) c.chunk else 1; - } - pub fn kvLen(c: Case, seq: usize) usize { - const qlen = c.queryLen(seq); - if (qlen == 0) return 0; - const reduction = if (c.skew) (seq * 17) % c.kv_len else 0; - return @max(qlen, c.kv_len - @min(reduction, c.kv_len - 1)); - } -}; - -// Eight geometry cases plus orthogonal semantics/boundary cases, not a Cartesian product. -pub const correctness = blk: { - var result: [34]Case = undefined; - var i: usize = 0; - for ([_]usize{ 64, 128, 256, 512 }) |dim| { - for ([_]bool{ false, true }) |mixed| { - result[i] = .{ .head_dim = dim, .prefill = if (mixed) 1 else 0 }; - i += 1; - } - } - result[8] = .{ .name = "mha_page_tail", .group = 1, .kv_len = 15 }; - result[9] = .{ .name = "mqa_page_exact", .kv_heads = 1, .group = 8, .kv_len = 16 }; - result[10] = .{ .name = "non_causal", .head_dim = 256, .prefill = 1, .mask = .non_causal }; - result[11] = .{ .name = "window_sink", .head_dim = 256, .prefill = 1, .mask = .window, .sink = true, .scale = 0.125 }; - result[12] = .{ .name = "indices", .head_dim = 256, .prefill = 1, .mask = .indices, .sink = true }; - result[13] = .{ .name = "long_decode", .head_dim = 512, .kv_heads = 1, .group = 2, .batch = 1, .kv_len = 2049 }; - result[14] = .{ .name = "skew", .head_dim = 256, .batch = 4, .prefill = 2, .skew = true, .kv_len = 129 }; - result[15] = .{ .name = "mixed_short_chunks", .batch = 4, .prefill = 2, .chunk = 16, .skew = true, .kv_len = 64 }; - result[16] = .{ .name = "empty_indices", .prefill = 1, .mask = .indices, .empty_indices = true }; - result[17] = .{ .name = "d32", .head_dim = 32, .group = 1, .kv_len = 33 }; - result[18] = .{ .name = "odd_group", .group = 5, .prefill = 1, .chunk = 9, .kv_len = 79 }; - result[19] = .{ .name = "group_tile_tail", .head_dim = 256, .batch = 1, .group = 19, .kv_len = 257 }; - result[20] = .{ .name = "shuffled_mixed", .head_dim = 512, .batch = 4, .prefill = 2, .group = 3, .chunk = 11, .kv_len = 129, .shuffle_pages = true }; - result[21] = .{ .name = "split_sink", .head_dim = 512, .batch = 1, .kv_len = 4097, .sink = true, .shuffle_pages = true }; - result[22] = .{ .name = "split_window_sink", .head_dim = 256, .batch = 1, .kv_len = 4097, .mask = .window, .window = 257, .sink = true }; - result[23] = .{ .name = "inactive_sequence", .batch = 4, .prefill = 1, .inactive = 1, .kv_len = 129 }; - result[24] = .{ .name = "non_causal_window", .head_dim = 64, .prefill = 1, .kv_len = 65, .mask = .non_causal_window, .window = 7 }; - result[25] = .{ .name = "empty_splits", .batch = 1, .kv_len = 1, .kv_capacity = 4096, .sink = true }; - result[26] = .{ .name = "d32_mixed", .head_dim = 32, .group = 3, .prefill = 1, .chunk = 9, .kv_len = 257, .shuffle_pages = true }; - result[27] = .{ .name = "d64_group_tail", .head_dim = 64, .group = 19, .kv_len = 257 }; - result[28] = .{ .name = "split_mixed_sink", .head_dim = 128, .batch = 1, .prefill = 1, .chunk = 3, .kv_len = 4097, .sink = true, .shuffle_pages = true }; - result[29] = .{ .name = "d512_prefill", .head_dim = 512, .batch = 2, .prefill = 2, .chunk = 65, .kv_len = 257, .sink = true, .shuffle_pages = true }; - result[30] = .{ .name = "negative_scale", .head_dim = 128, .prefill = 1, .kv_len = 257, .mask = .non_causal, .scale = -0.125 }; - result[31] = .{ .name = "zero_scale", .head_dim = 256, .prefill = 1, .kv_len = 257, .sink = true, .scale = 0 }; - result[32] = .{ .name = "d512_window_prefill", .head_dim = 512, .batch = 1, .prefill = 1, .chunk = 33, .kv_len = 513, .mask = .window, .window = 129, .sink = true }; - result[33] = .{ .name = "prefill_inactive", .head_dim = 64, .batch = 4, .prefill = 2, .inactive = 2, .kv_len = 257, .group = 5 }; - break :blk result; -}; - -pub const benchmarks = blk: { - var result: [24]Case = undefined; - var i: usize = 0; - for ([_]usize{ 64, 128, 256, 512 }) |dim| { - for ([_]usize{ 2, 8, 32 }) |batch| { - for ([_]bool{ false, true }) |mixed| { - result[i] = .{ - .head_dim = dim, - .batch = batch, - .prefill = if (mixed) @max(1, batch / 4) else 0, - .chunk = 64, - .kv_len = 2048, - }; - i += 1; - } - } - } - break :blk result; -}; diff --git a/zml/kernels/paged_attention/cute/correctness_test.zig b/zml/kernels/paged_attention/cute/correctness_test.zig deleted file mode 100644 index 436b9d3a..00000000 --- a/zml/kernels/paged_attention/cute/correctness_test.zig +++ /dev/null @@ -1,93 +0,0 @@ -const reference = @import("reference.zig"); -const std = @import("std"); -const zml = @import("zml"); -const cases = @import("cases.zig"); -const fixtures = @import("fixtures.zig"); -const Inputs = fixtures.Inputs; - -test "CPU reference self checks" { - _ = @import("reference.zig"); -} - -test "paged attention correctness" { - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.CudaRequired; - - const a = std.testing.allocator; - const io = std.testing.io; - - const cc = zml.platform.cuda.computeCapability(platform) orelse return error.UnknownComputeCapability; - std.debug.print("PAGED_ATTENTION dtype=bf16 page_size=16 seed=42 cc={d}.{d}\n{f}\n", .{ - cc.major, cc.minor, platform.fmtVerbose(), - }); - - var failures: usize = 0; - for (cases.correctness) |c| { - std.debug.print("CASE name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} mask={s} backend=cute\n", .{ - c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, @tagName(c.mask), - }); - - runCorrectness(a, io, platform, c, false) catch |err| { - failures += 1; - std.debug.print("FAIL error={s}\n", .{@errorName(err)}); - }; - } - - try std.testing.expectEqual(@as(usize, 0), failures); -} - -test "FP16 paged attention" { - const platform = zml.testing.env(); - if (platform.target != .cuda) return error.CudaRequired; - for ([_]cases.Case{ - .{ .name = "fp16_decode", .head_dim = 64, .kv_len = 257 }, - .{ .name = "fp16_mixed", .prefill = 1, .mask = .non_causal }, - .{ .name = "fp16_window_sink", .head_dim = 256, .prefill = 1, .mask = .window, .sink = true }, - .{ .name = "fp16_split_sink", .head_dim = 512, .batch = 1, .kv_len = 2049, .sink = true, .shuffle_pages = true }, - }) |c| try runCorrectness(std.testing.allocator, std.testing.io, platform, c, true); -} - -fn forwardF16(i: Inputs, c: cases.Case) zml.Tensor { - var inputs = i; - inputs.q = i.q.convert(.f16); - inputs.k = i.k.convert(.f16); - inputs.v = i.v.convert(.f16); - return fixtures.forward(inputs, c).convert(.bf16); -} - -fn runCorrectness(a: std.mem.Allocator, io: std.Io, platform: *const zml.Platform, c: cases.Case, comptime compute_f16: bool) !void { - const exe = try platform.compileFn(a, io, if (compute_f16) forwardF16 else fixtures.forward, .{ Inputs.init(c), c }, .{ .program_name = "paged_attention_harness" }); - defer exe.deinit(); - - var runner = try exe.runner(a); - defer runner.deinit(a); - - var arena = std.heap.ArenaAllocator.init(a); - defer arena.deinit(); - - const fixture = try fixtures.Fixture.init(arena.allocator(), c); - if (compute_f16) { - // Exactly representable in both formats, so the existing CPU reference - // receives precisely the values used by the FP16 tensor-core path. - for ([_][]zml.floats.BFloat16{ fixture.q, fixture.k, fixture.v }) |values| { - for (values) |*v| if (std.math.isFinite(v.toF32())) { - v.* = .fromF32(@round(v.toF32() * 64) / 64); - }; - } - } - - var buffers = try fixtures.upload(io, platform, Inputs.init(c), fixture); - defer zml.Buffer.deinitAll(Inputs, &buffers); - - var output: zml.Buffer = undefined; - runner.run(io, .{buffers}, .{&output}, .{ .wait = true }); - defer output.deinit(); - - var host = try output.toSliceAlloc(a, io); - defer host.free(a); - - const max_error = try reference.check(a, c, fixture, host.items(zml.floats.BFloat16)); - std.debug.print("RESULT name={s} d={d} b={d} hkv={d} g={d} prefill={d} chunk={d} kv={d} backend=cute compute_dtype={s} check=all_outputs max_abs={d:.6}\n", .{ - c.name, c.head_dim, c.batch, c.kv_heads, c.group, c.prefill, if (c.prefill > 0) c.chunk else 1, c.kv_len, if (compute_f16) "f16" else "bf16", max_error, - }); -} diff --git a/zml/kernels/paged_attention/cute/cute.zig b/zml/kernels/paged_attention/cute/cute.zig deleted file mode 100644 index ef78cc49..00000000 --- a/zml/kernels/paged_attention/cute/cute.zig +++ /dev/null @@ -1,115 +0,0 @@ -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -pub const AttentionOptions = @import("../options.zig").AttentionOptions; -const kernel = @import("attention.zig"); -const sm10x = @import("sm10x/sm10x.zig"); -const sm120 = @import("sm120/sm120.zig"); - -pub const Options = struct { - batch_size: usize, - max_num_pages: usize, - max_seqlen_q: usize, - is_prefill: bool, - - pub fn isPrefill(self: Options) bool { - return self.is_prefill; - } - - pub fn maxNumPages(self: Options) usize { - return self.max_num_pages; - } -}; - -pub const Parameters = struct { - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - options_: Options, - - pub fn init(options_: Options) Parameters { - return .{ - .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), - .seq_lens = .init(.{ .b = options_.batch_size }, .i32), - .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), - .options_ = options_, - }; - } - - pub fn allocationSize(self: Parameters) usize { - return self.block_table.byteSize() + self.seq_lens.byteSize() + self.query_start_len.byteSize(); - } - - pub fn options(self: Parameters) Options { - return self.options_; - } -}; - -// TODO(Corentin): This should be moved in the platform probably -pub fn multiprocessorCount(platform: *const zml.Platform) usize { - const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 1; - if (attribute.int64 <= 0) return 1; - return @intCast(attribute.int64); -} - -test { - _ = kernel; - _ = sm10x; - _ = sm120; -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - const cc = zml.platform.cuda.computeCapability(platform) orelse return false; - return cc.major > 8 or (cc.major == 8 and cc.minor >= 9); -} - -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: AttentionOptions) zml.Tensor { - stdx.debug.assert(isAvailable(zml.Compiler.current().platform), "CuTe paged attention requires SM89 or newer", .{}); - stdx.debug.assert(q.dtype() == k.dtype() and q.dtype() == v.dtype() and (q.dtype() == .bf16 or q.dtype() == .f16), "CuTe paged attention requires matching BF16 or FP16 Q/K/V", .{}); - stdx.debug.assert(q.dim(.hd) == 32 or q.dim(.hd) == 64 or q.dim(.hd) == 128 or q.dim(.hd) == 256 or q.dim(.hd) == 512, "CuTe paged attention supports head dimensions 32, 64, 128, 256 and 512", .{}); - if (sm10x.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) - return sm10x.pagedAttention(parameters, q, k, v, opts); - if (sm120.isAvailable(zml.Compiler.current().platform) and opts.mask != .indices) - return sm120.pagedAttention(parameters, q, k, v, opts); - const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); - const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const indices = switch (opts.mask) { - .indices => |indices| indices.transpose(.{ .b, .topk }), - else => zml.Tensor.zeroes(.init(.{ q_.dim(.b), 1 }, .i32)), - }; - const cfg: kernel.Config = .{ - .head_dim = q_.dim(.hd), - .tokens = q_.dim(.b), - .batch = parameters.block_table.dim(.b), - .kv_heads = k_.dim(.hkv), - .group = q_.dim(.hg), - .pages = k_.dim(.page), - .pages_per_seq = parameters.block_table.dim(.p), - .page_size = k_.dim(.k_chunk), - .dtype = zml.kernel.cute.from(q_.dtype()), - .causal = opts.mask.isCausal(), - .window = opts.mask.slidingWindow() orelse 0, - .scale = opts.scale orelse (1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd))))), - .sink = opts.sink != null, - .indexed = opts.mask == .indices, - .topk = indices.dim(-1), - }; - const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{cfg.kv_heads * cfg.group}) else zml.Tensor.zeroes(.init(.{cfg.kv_heads * cfg.group}, .f32)); - return kernel.Kernel.call(.{ - .q = q_, - .k = k_, - .v = v_, - .table = parameters.block_table, - .lengths = parameters.seq_lens, - .starts = parameters.query_start_len, - .sink = sink, - .indices = indices, - }, .{ .out = q_.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(cfg.tokens), @intCast(cfg.kv_heads * cfg.group), 1 }, - .block = .{ 32, 1, 1 }, - }).out.transpose(q.shape()); -} diff --git a/zml/kernels/paged_attention/cute/fixtures.zig b/zml/kernels/paged_attention/cute/fixtures.zig deleted file mode 100644 index c0815d1d..00000000 --- a/zml/kernels/paged_attention/cute/fixtures.zig +++ /dev/null @@ -1,141 +0,0 @@ -const std = @import("std"); - -const zml = @import("zml"); - -const cases = @import("cases.zig"); - -pub const Fixture = struct { - q: []zml.floats.BFloat16, - k: []zml.floats.BFloat16, - v: []zml.floats.BFloat16, - table: []i32, - lengths: []i32, - starts: []i32, - sinks: []f32, - indices: []i32, - - // The caller owns an arena; all arrays live until the case is complete. - pub fn init(a: std.mem.Allocator, c: cases.Case) !Fixture { - const f: Fixture = .{ - .q = try a.alloc(zml.floats.BFloat16, c.tokens() * c.heads() * c.head_dim), - .k = try a.alloc(zml.floats.BFloat16, c.batch * c.pages() * cases.page_size * c.kv_heads * c.head_dim), - .v = try a.alloc(zml.floats.BFloat16, c.batch * c.pages() * cases.page_size * c.kv_heads * c.head_dim), - .table = try a.alloc(i32, c.batch * c.pages()), - .lengths = try a.alloc(i32, c.batch), - .starts = try a.alloc(i32, c.batch + 1), - .sinks = try a.alloc(f32, c.heads()), - .indices = try a.alloc(i32, c.tokens() * cases.topk), - }; - var rng = std.Random.DefaultPrng.init(42); - const random = rng.random(); - for (f.q) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); - for (f.k) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); - for (f.v) |*v| v.* = .fromF32(random.float(f32) * 2 - 1); - for (f.sinks, 0..) |*v, h| v.* = @as(f32, @floatFromInt(h % 5)) - 2; - @memset(f.indices, -1); - f.starts[0] = 0; - for (0..c.batch) |s| { - const len = c.kvLen(s); - f.lengths[s] = @intCast(len); - f.starts[s + 1] = f.starts[s] + @as(i32, @intCast(c.queryLen(s))); - for (0..c.pages()) |p| { - f.table[s * c.pages() + p] = if (p * cases.page_size < len) @intCast(s * c.pages() + p) else -1; - } - const live_pages = std.math.divCeil(usize, len, cases.page_size) catch unreachable; - if (c.shuffle_pages) random.shuffle(i32, f.table[s * c.pages() ..][0..live_pages]); - // Unused physical slots must not contribute to PV, even via zero * NaN. - for (len..c.pages() * cases.page_size) |j| { - const page = if (j / cases.page_size < live_pages) - @as(usize, @intCast(f.table[s * c.pages() + j / cases.page_size])) - else - s * c.pages() + j / cases.page_size; - const offset = (page * cases.page_size + j % cases.page_size) * c.kv_heads * c.head_dim; - @memset(f.k[offset..][0 .. c.kv_heads * c.head_dim], .fromF32(std.math.nan(f32))); - @memset(f.v[offset..][0 .. c.kv_heads * c.head_dim], .fromF32(std.math.nan(f32))); - } - for (@intCast(f.starts[s])..@intCast(f.starts[s + 1])) |t| { - const l: i32 = @intCast(len); - @memcpy(f.indices[t * cases.topk ..][0..cases.topk], &[_]i32{ 0, l - 1, @divTrunc(l, 2), -1, l, 0, 1, l - 2 }); - } - } - if (c.empty_indices) { - for (f.indices, 0..) |*index, i| index.* = if (i % 2 == 0) -1 else @intCast(c.kv_len); - @memset(f.k, .fromF32(std.math.nan(f32))); - @memset(f.v, .fromF32(std.math.nan(f32))); - } - return f; - } - - pub fn kvOffset(f: Fixture, c: cases.Case, seq: usize, key: usize, head: usize) usize { - const page: usize = @intCast(f.table[seq * c.pages() + key / cases.page_size]); - return ((page * cases.page_size + key % cases.page_size) * c.kv_heads + head) * c.head_dim; - } -}; - -pub const Inputs = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - table: zml.Tensor, - lengths: zml.Tensor, - starts: zml.Tensor, - sinks: zml.Tensor, - indices: zml.Tensor, - - pub fn init(c: cases.Case) Inputs { - return .{ - .q = .init(.{ .b = c.tokens(), .hkv = c.kv_heads, .hg = c.group, .hd = c.head_dim }, .bf16), - .k = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), - .v = .init(.{ .page = c.batch * c.pages(), .k_chunk = cases.page_size, .hkv = c.kv_heads, .hd = c.head_dim }, .bf16), - .table = .init(.{ .b = c.batch, .p = c.pages() }, .i32), - .lengths = .init(.{ .b = c.batch }, .i32), - .starts = .init(.{ .b = c.batch + 1 }, .i32), - .sinks = .init(.{c.heads()}, .f32), - .indices = .init(.{ .b = c.tokens(), .topk = cases.topk }, .i32), - }; - } - - fn options(i: Inputs, c: cases.Case) zml.paged_attention.AttentionOptions { - return .{ - .mask = switch (c.mask) { - .causal => .{ .standard = .{} }, - .non_causal => .none, - .window => .{ .standard = .{ .sliding_window = c.window } }, - .non_causal_window => .{ .standard = .{ .causal = false, .sliding_window = c.window } }, - .indices => .{ .indices = i.indices }, - }, - .sink = if (c.sink) i.sinks else null, - .scale = c.scale, - }; - } -}; - -pub fn forward(i: Inputs, c: cases.Case) zml.Tensor { - const parameters: zml.paged_attention.Parameters = .{ - .cute = .{ - .block_table = i.table, - .seq_lens = i.lengths, - .query_start_len = i.starts, - .options_ = .{ - .batch_size = c.batch, - .max_num_pages = c.pages(), - .max_seqlen_q = if (c.prefill > 0) c.chunk else 1, - .is_prefill = c.prefill > 0, - }, - }, - }; - return zml.paged_attention.cute.pagedAttention(parameters.cute, i.q, i.k, i.v, i.options(c)); -} - -pub fn upload(io: std.Io, platform: *const zml.Platform, inputs: Inputs, f: Fixture) !zml.Bufferized(Inputs) { - var result: zml.Bufferized(Inputs) = undefined; - var initialized: usize = 0; - errdefer inline for (@typeInfo(Inputs).@"struct".field_names, 0..) |name, index| { - if (index < initialized) @field(result, name).deinit(); - }; - inline for (@typeInfo(Inputs).@"struct".field_names) |name| { - @field(result, name) = try .fromBytes(io, platform, @field(inputs, name).shape(), std.mem.sliceAsBytes(@field(f, name))); - initialized += 1; - } - return result; -} diff --git a/zml/kernels/paged_attention/cute/reference.zig b/zml/kernels/paged_attention/cute/reference.zig deleted file mode 100644 index 1a030646..00000000 --- a/zml/kernels/paged_attention/cute/reference.zig +++ /dev/null @@ -1,127 +0,0 @@ -const std = @import("std"); -const zml = @import("zml"); -const cases = @import("cases.zig"); -const fixtures = @import("fixtures.zig"); - -/// One query/head, independent of GPU tensor operations. Duplicate indices are -/// separate softmax entries; invalid indices are excluded. -pub fn row(c: cases.Case, f: fixtures.Fixture, seq: usize, token: usize, head: usize, scores: []f32, out: []f32) void { - const len: usize = @intCast(f.lengths[seq]); - const start: usize = @intCast(f.starts[seq]); - const qlen: usize = @intCast(f.starts[seq + 1] - f.starts[seq]); - const position = len - qlen + token - start; - const count = if (c.mask == .indices) cases.topk else len; - const scale = c.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(c.head_dim))); - const q = f.q[(token * c.heads() + head) * c.head_dim ..][0..c.head_dim]; - var maximum: f32 = if (c.sink) f.sinks[head] else -std.math.inf(f32); - for (0..count) |j| { - const key = keyAt(c, f, token, j); - const valid = key >= 0 and key < len and switch (c.mask) { - .indices, .non_causal => true, - .causal => key <= position, - .window => key <= position and @as(i64, @intCast(key)) > @as(i64, @intCast(position)) - c.window, - .non_causal_window => @as(i64, @intCast(key)) > @as(i64, @intCast(position)) - c.window, - }; - scores[j] = -std.math.inf(f32); - if (!valid) continue; - const offset = f.kvOffset(c, seq, @intCast(key), head / c.group); - var dot: f32 = 0; - for (q, f.k[offset..][0..c.head_dim]) |qv, kv| dot += qv.toF32() * kv.toF32(); - scores[j] = dot * scale; - maximum = @max(maximum, scores[j]); - } - @memset(out, 0); - if (maximum == -std.math.inf(f32)) return; - var denominator: f32 = if (c.sink) @exp(f.sinks[head] - maximum) else 0; - for (0..count) |j| { - if (scores[j] == -std.math.inf(f32)) continue; - const weight = @exp(scores[j] - maximum); - denominator += weight; - const offset = f.kvOffset(c, seq, @intCast(keyAt(c, f, token, j)), head / c.group); - for (out, f.v[offset..][0..c.head_dim]) |*o, v| o.* += weight * v.toF32(); - } - for (out) |*v| v.* /= denominator; -} - -fn keyAt(c: cases.Case, f: fixtures.Fixture, token: usize, j: usize) i64 { - return if (c.mask == .indices) f.indices[token * cases.topk + j] else @intCast(j); -} - -pub fn check(a: std.mem.Allocator, c: cases.Case, f: fixtures.Fixture, actual: []const zml.floats.BFloat16) !f32 { - const active_tokens: usize = @intCast(f.starts[c.batch]); - for (actual[0 .. active_tokens * c.heads() * c.head_dim]) |v| { - if (!std.math.isFinite(v.toF32())) return error.NonFiniteOutput; - } - const scores = try a.alloc(f32, @max(c.kv_len, cases.topk)); - defer a.free(scores); - const expected = try a.alloc(f32, c.head_dim); - defer a.free(expected); - var max_error: f32 = 0; - var failures: usize = 0; - for (0..c.batch) |s| { - for (@intCast(f.starts[s])..@intCast(f.starts[s + 1])) |t| { - for (0..c.heads()) |h| { - row(c, f, s, t, h, scores, expected); - const base = (t * c.heads() + h) * c.head_dim; - for (expected, actual[base..][0..c.head_dim], 0..) |want, got_bf16, d| { - const got = got_bf16.toF32(); - const err = @abs(want - got); - max_error = @max(max_error, err); - // Fixed BF16 error budget; never adapted to observed failures. - if (!std.math.isFinite(want) or err > 0.003 + 0.02 * @abs(want)) { - if (failures == 0) std.debug.print("Mismatch seq={d} token={d} head={d} dim={d}: expected={d} actual={d}\n", .{ s, t, h, d, want, got }); - failures += 1; - } - } - } - } - } - if (failures > 0) return error.ReferenceMismatch; - return max_error; -} - -test "reference single key, uniform causal scores, and sink" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - var c: cases.Case = .{ .head_dim = 64, .batch = 1, .kv_heads = 1, .group = 1, .prefill = 1, .chunk = 2, .kv_len = 2 }; - const f = try fixtures.Fixture.init(arena.allocator(), c); - @memset(f.q, .fromF32(0)); - @memset(f.v[0..64], .fromF32(2)); - @memset(f.v[64..128], .fromF32(4)); - var scores: [2]f32 = undefined; - var out: [64]f32 = undefined; - row(c, f, 0, 0, 0, &scores, &out); - for (out) |v| try std.testing.expectEqual(@as(f32, 2), v); - row(c, f, 0, 1, 0, &scores, &out); - for (out) |v| try std.testing.expectEqual(@as(f32, 3), v); - c.sink = true; - f.sinks[0] = 0; - row(c, f, 0, 1, 0, &scores, &out); - for (out) |v| try std.testing.expectEqual(@as(f32, 2), v); -} - -test "reference windows, noncausal, duplicate and invalid indices" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - var c: cases.Case = .{ .head_dim = 64, .batch = 1, .kv_heads = 1, .group = 1, .prefill = 1, .chunk = 2, .kv_len = 2 }; - const f = try fixtures.Fixture.init(arena.allocator(), c); - @memset(f.q, .fromF32(0)); - @memset(f.v[0..64], .fromF32(2)); - @memset(f.v[64..128], .fromF32(4)); - var scores: [cases.topk]f32 = undefined; - var out: [64]f32 = undefined; - c.mask = .non_causal; - row(c, f, 0, 0, 0, &scores, &out); - for (out) |v| try std.testing.expectEqual(@as(f32, 3), v); - c.mask = .window; - c.window = 1; - row(c, f, 0, 1, 0, &scores, &out); - for (out) |v| try std.testing.expectEqual(@as(f32, 4), v); - c.mask = .indices; - @memset(f.indices, -1); - @memcpy(f.indices[0..4], &[_]i32{ 0, 0, 1, 2 }); - row(c, f, 0, 0, 0, &scores, &out); - for (out) |v| try std.testing.expectApproxEqAbs(@as(f32, 8.0 / 3.0), v, 1e-6); - row(c, f, 0, 1, 0, &scores, &out); - for (out) |v| try std.testing.expectEqual(@as(f32, 0), v); -} diff --git a/zml/kernels/paged_attention/cute/sm10x/BUILD.bazel b/zml/kernels/paged_attention/cute/sm10x/BUILD.bazel deleted file mode 100644 index 05cd7edc..00000000 --- a/zml/kernels/paged_attention/cute/sm10x/BUILD.bazel +++ /dev/null @@ -1,10 +0,0 @@ -package(default_visibility = ["//zml/kernels/paged_attention/cute:__pkg__"]) - -filegroup( - name = "sm10x", - srcs = [ - "attention.zig", - "sm103.zig", - "sm10x.zig", - ], -) diff --git a/zml/kernels/paged_attention/cute/sm10x/attention.zig b/zml/kernels/paged_attention/cute/sm10x/attention.zig deleted file mode 100644 index 12d6273c..00000000 --- a/zml/kernels/paged_attention/cute/sm10x/attention.zig +++ /dev/null @@ -1,667 +0,0 @@ -const std = @import("std"); - -const zml = @import("../../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const V = cute.Value; -const sm103 = @import("sm103.zig"); - -pub const Architecture = enum { sm100, sm103 }; - -pub const Config = struct { - architecture: Architecture, - head_dim: i64, - tokens: i64, - batch: i64, - kv_heads: i64, - group: i64, - pages_per_seq: i64, - page_size: i64, - max_query: i64, - dtype: cute.DType, - decode: bool, - causal: bool, - window: i64, - scale: f32, - sink: bool, - splits: i64 = 1, - - pub fn rows(c: Config) i64 { - return if (c.decode) (if (c.group <= 8) 8 else 16) else 64; - } - pub fn keys(c: Config) i64 { - return if (c.decode) 128 else 64; - } - pub fn queryTiles(c: Config) i64 { - return @divTrunc(c.max_query * c.group + c.rows() - 1, c.rows()); - } - fn paddedDim(c: Config) i64 { - return @max(128, c.head_dim); - } - fn features(c: Config) i64 { - return @divExact(c.paddedDim(), 128); - } - fn buffers(c: Config) i64 { - return if (c.decode) 2 else 3; - } - fn kvBytes(c: Config) i64 { - return c.keys() * 128 * 2; - } - fn qBase(c: Config) i64 { - return c.buffers() * c.kvBytes(); - } - fn pBase(c: Config) i64 { - return c.qBase() + c.rows() * c.paddedDim() * 2; - } - fn barriers(c: Config) i64 { - return c.pBase() + c.rows() * c.keys() * 2; - } - fn completion(c: Config) i64 { - return c.barriers() + c.buffers() * 16; - } - fn holder(c: Config) i64 { - return c.completion() + 16; - } - fn reductions(c: Config) i64 { - return c.holder() + 16; - } - fn sharedBytes(c: Config) i64 { - return c.reductions() + 4 * 16 * 4; - } - fn scoreColumn(c: Config) i64 { - return if (c.decode) c.features() * c.rows() else @min(c.paddedDim(), 448); - } - fn columns(c: Config) u32 { - return std.math.ceilPowerOfTwoAssert(u32, @intCast(@max(32, c.scoreColumn() + (if (c.decode) c.rows() else c.keys())))); - } -}; - -pub const block: [3]i32 = .{ 256, 1, 1 }; -pub const Attention = cute.Kernel(Config, .{ - .name = "paged_attention_cute", - .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink" }, - .outputs = &.{"out"}, - .run = attention, -}); - -fn loadInt(b: *B, pointer: V, index: V) V { - return b.loadWords(1, pointer.add(index.to(.i64).mul(4)), .global, .{})[0]; -} - -// Each 64-feature plane uses a 128-byte XOR swizzle. The descriptor and -// vector copies use this same layout for Q, K, V and probabilities. -fn swizzle(row: V, col: V, rows: i64) V { - return col.div(64).mul(rows * 128).add(row.mul(128)).add(col.rem(64).div(8).bitXor(row.rem(8)).mul(16)).add(col.rem(8).mul(2)); -} - -fn descriptor(b: *B, address: V, leading_bytes: i64) V { - const bits: u64 = (64 << 32) | (1 << 46) | (@as(u64, 2) << 61) | (@as(u64, @intCast(@divExact(leading_bytes, 16))) << 16); - return address.to(.i64).shrLogical(4).bitAnd(0x3fff).bitOr(b.constant(.i64, @as(i64, @bitCast(bits)))); -} - -fn instruction(c: Config, qk: bool) u32 { - const format: u32 = if (c.dtype == .bf16) 1 else 0; - const m: u32 = if (c.decode) 128 else 64; - const n: u32 = @intCast(if (c.decode) c.rows() else if (qk) c.keys() else 128); - return (1 << 4) | (format << 7) | (format << 10) | - (@as(u32, @intFromBool(c.decode and !qk)) << 15) | - (@as(u32, @intFromBool(!c.decode and !qk)) << 16) | ((n >> 3) << 17) | ((m >> 4) << 24); -} - -const Range = struct { first: V, count: V }; - -// Used by both attention and merge: every output row must merge exactly the -// partitions produced for its query tile, including causal/window boundaries. -fn keyRange(b: *B, c: Config, first_row: V, qlen: V, klen: V) Range { - const first_pos = klen.sub(qlen).add(first_row.div(c.group)); - const last_pos = klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); - const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); - const end = if (c.causal) last_pos.add(1).minimum(klen) else klen; - const first = begin.div(c.keys()); - return .{ .first = first, .count = end.add(c.keys() - 1).div(c.keys()).sub(first).maximum(0) }; -} - -fn liveSplits(c: Config, tiles: V) V { - return tiles.add(1).div(2).maximum(1).minimum(c.splits); -} - -const Tile = struct { - b: *B, - c: Config, - q: V, - k: V, - v: V, - table: V, - sink: V, - out: V, - shared: V, - tmem: V, - tid: V, - lane: V, - warp: V, - seq: V, - head: V, - first_row: V, - start: V, - qlen: V, - klen: V, - first_key_tile: V, - tiles: V, - split: V, - - fn ready(t: Tile, id: V) V { - return t.shared.add(t.c.barriers()).add(id.rem(t.c.buffers()).mul(8)); - } - fn released(t: Tile, id: V) V { - return t.ready(id).add(t.c.buffers() * 8); - } - fn phase(t: Tile, id: V) V { - return id.div(t.c.buffers()).rem(2); - } - fn valid(t: Tile, row: V, key: V) V { - const pos = t.klen.sub(t.qlen).add(row.div(t.c.group)); - var result = row.lt(t.qlen.mul(t.c.group)).bitAnd(key.lt(t.klen)); - if (t.c.causal) result = result.bitAnd(key.le(pos)); - if (t.c.window > 0) result = result.bitAnd(key.gt(pos.sub(t.c.window))); - return result; - } - fn putProbability(t: Tile, row: V, key: V, value: V) void { - t.b.storeHalf(t.shared.add(t.c.pBase()).add(swizzle(row, key, t.c.rows())), .shared, value.to(t.c.dtype).bitCast(.i16).to(.i32)); - } - fn put(t: Tile, row: V, dim: V, value: V) void { - const token = t.start.add(row.div(t.c.group)); - var index = token.to(.i64).mul(t.c.kv_heads).add(t.head.to(.i64)).mul(t.c.group).add(row.rem(t.c.group).to(.i64)); - if (t.c.splits > 1) { - index = t.split.to(.i64).mul(t.c.tokens * t.c.kv_heads * t.c.group).add(index).mul(t.c.head_dim + 2).add(dim.to(.i64)); - t.b.storeWords(1, t.out.add(index.mul(4)), .global, .{value.bitCast(.i32)}); - } else { - index = index.mul(t.c.head_dim).add(dim.to(.i64)); - t.b.storeHalf(t.out.add(index.mul(2)), .global, value.to(t.c.dtype).bitCast(.i16).to(.i32)); - } - } - fn norm(t: Tile, row: V, maximum: V, denominator: V) V { - const b = t.b; - if (t.c.splits > 1) return b.constant(.f32, 1); - if (!t.c.sink) return b.select(denominator.gt(0), b.rcpApprox(denominator), b.constant(.f32, 0)); - const sink = loadInt(b, t.sink, t.head.mul(t.c.group).add(row.rem(t.c.group))).bitCast(.f32).mul(std.math.log2e); - const reference = maximum.maximum(sink).maximum(-std.math.floatMax(f32)); - const weight = b.exp2Approx(maximum.sub(reference)); - return weight.div(denominator.mul(weight).add(b.exp2Approx(sink.sub(reference)))); - } -}; - -fn attention(b: *B, c: Config) cute.FinishError!void { - const a = try b.declareArgs(.{ - .q = .{ .ptr = c.dtype }, - .k = .{ .ptr = c.dtype }, - .v = .{ .ptr = c.dtype }, - .table = .{ .ptr = cute.DType.i32 }, - .lengths = .{ .ptr = cute.DType.i32 }, - .starts = .{ .ptr = cute.DType.i32 }, - .sink = .{ .ptr = cute.DType.f32 }, - .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else c.dtype }, - }); - std.debug.assert(c.sharedBytes() <= 227 * 1024); - b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); - const seq = b.blockIdx().x.div(c.queryTiles()); - const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); - const starts = b.ptrToInt(a.starts, .i64); - const start = loadInt(b, starts, seq); - const qlen = loadInt(b, starts, seq.add(1)).sub(start); - const klen = loadInt(b, b.ptrToInt(a.lengths, .i64), seq); - const range = keyRange(b, c, first_row, qlen, klen); - const splits = liveSplits(c, range.count); - const split = b.blockIdx().z; - var active = b.openIf(first_row.lt(qlen.mul(c.group)).bitAnd(split.lt(splits))); - { - const tid = b.threadIdx().x; - const lo = range.count.mul(split).div(splits); - const hi = range.count.mul(split.add(1)).div(splits); - const shared = b.ptrToInt(b.allocSmem(.i8, @intCast(c.sharedBytes()), 1024), .i32); - var t: Tile = .{ - .b = b, - .c = c, - .q = b.ptrToInt(a.q, .i64), - .k = b.ptrToInt(a.k, .i64), - .v = b.ptrToInt(a.v, .i64), - .table = b.ptrToInt(a.table, .i64), - .sink = b.ptrToInt(a.sink, .i64), - .out = b.ptrToInt(a.out, .i64), - .shared = shared, - .tmem = undefined, - .tid = tid, - .lane = tid.rem(32), - .warp = tid.div(32), - .seq = seq, - .head = b.blockIdx().y, - .first_row = first_row, - .start = start, - .qlen = qlen, - .klen = klen, - .first_key_tile = range.first.add(lo), - .tiles = hi.sub(lo), - .split = split, - }; - var leader = b.openIf(tid.eq(0)); - { - for (0..@as(usize, @intCast(c.buffers()))) |i| { - b.mbarrierInit(shared.add(c.barriers() + @as(i64, @intCast(i * 8))), 128); - b.mbarrierInit(shared.add(c.barriers() + c.buffers() * 8 + @as(i64, @intCast(i * 8))), 1); - } - b.mbarrierInit(shared.add(c.completion()), 1); - b.fenceMbarrierInit(); - leader.close(); - } - var alloc = b.openIf(tid.lt(32)); - { - b.tmemAllocAt(shared.add(c.holder()), c.columns()); - b.relinquishTmemAllocPermit(.{}); - alloc.close(); - } - var queries = b.openFor(tid, c.rows() * @divExact(c.paddedDim(), 8), 256, .{}); - { - const row = queries.iv.div(@divExact(c.paddedDim(), 8)); - const col = queries.iv.rem(@divExact(c.paddedDim(), 8)).mul(8); - const packed_row = first_row.add(row); - const valid = packed_row.lt(qlen.mul(c.group)).bitAnd(col.lt(c.head_dim)); - const token = start.add(b.select(valid, packed_row.div(c.group), b.constant(.i32, 0))); - const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(c.head_dim).add(col.to(.i64)); - b.cpAsync16(shared.add(c.qBase()).add(swizzle(row, col, c.rows())), t.q.add(offset.mul(2)), valid); - queries.close(); - } - b.cpAsyncWaitAll(); - b.syncThreads(); - t.tmem = b.loadWords(1, shared.add(c.holder()), .shared, .{})[0]; - b.tcgen05Fence(.after); - var producer = b.openIf(tid.ge(128)); - { - produce(t); - producer.close(); - } - var consumer = b.openIf(tid.lt(128)); - { - if (c.decode) { - if (c.group <= 2) decode(t, 2) else if (c.group <= 4) decode(t, 4) else if (c.group <= 8) decode(t, 8) else decode(t, 16); - } else prefill(t); - consumer.close(); - } - b.tcgen05Fence(.before); - b.syncThreads(); - var free = b.openIf(tid.lt(32)); - { - b.tmemDeallocAt(t.tmem, c.columns()); - free.close(); - } - active.close(); - } -} - -fn produce(t: Tile) void { - const b = t.b; - const c = t.c; - const row0 = t.tid.sub(128).div(16); - const col = t.tid.rem(16).mul(8); - var tiles = b.openFor(0, t.tiles, 1, .{}); - { - // Paged offsets are shared by K/V and every 128-feature stage. - var offsets: [16]V = undefined; - var valid: [16]V = undefined; - for (0..@as(usize, @intCast(@divExact(c.keys(), 8)))) |i| { - const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(row0).add(i * 8); - valid[i] = key.lt(t.klen).bitAnd(col.lt(c.head_dim)); - const safe = b.select(key.lt(t.klen), key, b.constant(.i32, 0)); - const page = loadInt(b, t.table, t.seq.mul(c.pages_per_seq).add(safe.div(c.page_size))); - offsets[i] = page.to(.i64).mul(c.page_size).add(safe.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)).mul(2); - } - for (0..@as(usize, @intCast(c.features() * 2))) |stage| { - const id = tiles.iv.mul(c.features() * 2).add(stage); - var reuse = b.openIf(id.ge(c.buffers())); - { - b.mbarrierWait(t.released(id), t.phase(id).bitXor(1), .{ .scope = .cta }); - reuse.close(); - } - const source = if (stage < c.features()) t.k else t.v; - const feature_offset: i64 = @mod(@as(i64, @intCast(stage)), c.features()) * 256; - for (0..@as(usize, @intCast(@divExact(c.keys(), 8)))) |i| { - const row = row0.add(i * 8); - b.cpAsync16(t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())).add(swizzle(row, col, c.keys())), source.add(offsets[i]).add(feature_offset), valid[i]); - } - b.cpAsyncMbarrierArrive(t.ready(id), true); - } - tiles.close(); - } -} - -fn multiply(t: Tile, tile: V, comptime qk: bool) void { - const b = t.b; - const c = t.c; - var leader = b.openIf(t.tid.eq(0)); - { - for (0..@as(usize, @intCast(c.features()))) |part| { - const id = tile.mul(c.features() * 2).add(@as(i64, @intCast(part)) + if (qk) @as(i64, 0) else c.features()); - b.mbarrierWait(t.ready(id), t.phase(id), .{ .scope = .cta }); - b.tcgen05Fence(.after); - const kv = descriptor(b, t.shared.add(id.rem(c.buffers()).mul(c.kvBytes())), if (qk) 16 else c.keys() * 128); - const qp = descriptor(b, t.shared.add(if (qk) c.qBase() else c.pBase()), 16); - const steps: usize = @intCast(if (qk) 8 else @divExact(c.keys(), 16)); - for (0..steps) |step| { - const kv_offset: i64 = if (qk) @as(i64, @intCast(step / 4)) * c.keys() * 128 + @as(i64, @intCast(step % 4)) * 32 else @as(i64, @intCast(step)) * 2048; - const qp_offset: i64 = (if (qk) @as(i64, @intCast(part * 2 + step / 4)) else @as(i64, @intCast(step / 4))) * c.rows() * 128 + @as(i64, @intCast(step % 4)) * 32; - const left = if (c.decode) kv.add(@divExact(kv_offset, 16)) else qp.add(@divExact(qp_offset, 16)); - const right = if (c.decode) qp.add(@divExact(qp_offset, 16)) else kv.add(@divExact(kv_offset, 16)); - const column: i64 = if (qk) c.scoreColumn() else @as(i64, @intCast(part)) * (if (c.decode) c.rows() else 128); - const accumulate = if (qk) b.constant(.i32, @intFromBool(part > 0 or step > 0)).ne(0) else tile.gt(0).bitOr(b.constant(.i32, @intFromBool(step > 0)).ne(0)); - b.tcgen05MmaF16(t.tmem.add(column), left, right, b.constant(.i32, @as(i32, @bitCast(instruction(c, qk)))), accumulate); - } - b.tcgen05Commit(t.released(id), .{}); - } - b.tcgen05Commit(t.shared.add(c.completion()), .{}); - leader.close(); - } - b.mbarrierWait(t.shared.add(c.completion()), b.constant(.i32, if (qk) 0 else 1), .{ .scope = .cta }); - b.tcgen05Fence(.after); - // A fast warp must not wrap completion parity before every consumer has - // observed the preceding PV completion. - if (!qk) b.barrierCtaSyncAligned(1, 128); -} - -fn publishProbabilities(t: Tile) void { - t.b.fenceProxyAsyncShared(); - t.b.tcgen05Fence(.before); - t.b.barrierCtaSyncAligned(1, 128); - t.b.tcgen05Fence(.after); -} - -fn reduceRow(comptime n: usize, values: [n]V, comptime maximum: bool) V { - var tree = values; - var width: usize = n; - while (width > 1) { - width /= 2; - for (0..width) |i| tree[i] = if (maximum) tree[i * 2].maximum(tree[i * 2 + 1]) else tree[i * 2].add(tree[i * 2 + 1]); - } - return tree[0]; -} - -fn reduceHeads(t: Tile, comptime heads: usize, values: [heads]V, comptime maximum: bool) [heads]V { - const b = t.b; - for (values, 0..) |value, h| { - var reduced = value; - inline for (.{ 16, 8, 4, 2, 1 }) |step| reduced = if (maximum) reduced.maximum(b.shuffleXor(reduced, step)) else reduced.add(b.shuffleXor(reduced, step)); - var leader = b.openIf(t.lane.eq(0)); - { - b.storeWords(1, t.shared.add(t.c.reductions()).add(t.warp.mul(64)).add(h * 4), .shared, .{reduced.bitCast(.i32)}); - leader.close(); - } - } - b.barrierCtaSyncAligned(1, 128); - var result: [heads]V = undefined; - for (0..heads) |h| { - var value = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(h * 4))), .shared, .{})[0].bitCast(.f32); - for (1..4) |w| { - const next = b.loadWords(1, t.shared.add(t.c.reductions() + @as(i64, @intCast(w * 64 + h * 4))), .shared, .{})[0].bitCast(.f32); - value = if (maximum) value.maximum(next) else value.add(next); - } - result[h] = value; - } - b.barrierCtaSyncAligned(1, 128); - return result; -} - -fn decode(t: Tile, comptime heads: usize) void { - const b = t.b; - const c = t.c; - const address = t.tmem.add(t.warp.mul(32).shl(16)); - const key_row = t.warp.mul(32).add(t.lane); - const zero = b.constant(.f32, 0); - const neg_inf = b.constant(.f32, -std.math.inf(f32)); - var initial: [2 * heads]V = undefined; - for (0..heads) |h| { - initial[h] = neg_inf; - initial[heads + h] = zero; - } - var tiles = b.openFor(0, t.tiles, 1, initial); - { - multiply(t, tiles.iv, true); - const raw = b.tmemLoad32x32b(heads, address.add(c.scoreColumn())); - b.fenceTmemLoad(); - const key = t.first_key_tile.add(tiles.iv).mul(c.keys()).add(key_row); - var scores: [heads]V = undefined; - for (0..heads) |h| scores[h] = b.select(t.valid(t.first_row.add(h), key), raw[h].bitCast(.f32).mul(c.scale * std.math.log2e), neg_inf); - const maxima = reduceHeads(t, heads, scores, true); - var next: [2 * heads]V = undefined; - var alpha: [heads]V = undefined; - var probabilities: [heads]V = undefined; - for (0..heads) |h| { - next[h] = tiles.carried[h].maximum(maxima[h]); - const reference = next[h].maximum(-std.math.floatMax(f32)); - alpha[h] = b.exp2Approx(tiles.carried[h].sub(reference)); - probabilities[h] = b.exp2Approx(scores[h].sub(reference)); - } - const sums = reduceHeads(t, heads, probabilities, false); - for (0..heads) |h| { - next[heads + h] = tiles.carried[heads + h].mul(alpha[h]).add(sums[h]); - t.putProbability(b.constant(.i32, h), key_row, probabilities[h]); - } - for (heads..@as(usize, @intCast(c.rows()))) |h| t.putProbability(b.constant(.i32, h), key_row, zero); - var rescale = b.openIf(tiles.iv.gt(0)); - { - for (0..@as(usize, @intCast(c.features()))) |part| { - const addr = address.add(@as(i64, @intCast(part)) * c.rows()); - var out = b.tmemLoad32x32b(heads, addr); - b.fenceTmemLoad(); - for (0..heads) |h| out[h] = out[h].bitCast(.f32).mul(alpha[h]).bitCast(.i32); - b.tmemStore32x32b(heads, addr, out); - } - b.fenceTmemStore(); - rescale.close(); - } - publishProbabilities(t); - multiply(t, tiles.iv, false); - tiles.yield(next); - } - for (0..@as(usize, @intCast(c.features()))) |part| { - const out = b.tmemLoad32x32b(heads, address.add(@as(i64, @intCast(part)) * c.rows())); - b.fenceTmemLoad(); - const dim = key_row.add(part * 128); - for (0..heads) |h| { - const row = t.first_row.add(h); - var valid = b.openIf(row.lt(t.qlen.mul(c.group))); - { - var feature = b.openIf(dim.lt(c.head_dim)); - { - const value = b.select(tiles.results[heads + h].gt(0), out[h].bitCast(.f32).mul(t.norm(row, tiles.results[h], tiles.results[heads + h])), zero); - t.put(row, dim, value); - feature.close(); - } - if (c.splits > 1 and part == 0) { - var stats = b.openIf(t.tid.eq(0)); - { - t.put(row, b.constant(.i32, c.head_dim), tiles.results[h]); - t.put(row, b.constant(.i32, c.head_dim + 1), tiles.results[heads + h]); - stats.close(); - } - } - valid.close(); - } - } - } -} - -fn prefill(t: Tile) void { - const b = t.b; - const c = t.c; - const address = t.tmem.add(t.warp.mul(32).shl(16)); - const local_row = t.warp.mul(16).add(t.lane); - const row = t.first_row.add(local_row); - const owns = t.lane.lt(16); - const zero = b.constant(.f32, 0); - const neg_inf = b.constant(.f32, -std.math.inf(f32)); - var tiles = b.openFor(0, t.tiles, 1, .{ neg_inf, zero }); - { - // D512 fills TMEM with O. Preserve its last 64 columns in registers - // while QK reuses them for scores, avoiding duplicate QK/softmax CTAs. - var saved: [64]V = undefined; - if (c.head_dim == 512) { - var preserve = b.openIfElse(tiles.iv.gt(0), @as([64]V, @splat(zero))); - { - const tail = b.tmemLoad32x32b(64, address.add(c.scoreColumn())); - b.fenceTmemLoad(); - var words: [64]V = undefined; - for (tail, 0..) |value, i| words[i] = value.bitCast(.f32); - preserve.yieldThen(words); - } - { - preserve.yieldElse(@as([64]V, @splat(zero))); - } - @memcpy(&saved, &preserve.results); - b.barrierCtaSyncAligned(1, 128); - } - multiply(t, tiles.iv, true); - var scores: [64]V = undefined; - var hardware_max = neg_inf; - if (c.architecture == .sm103) { - const loaded = sm103.loadScores(b, address.add(c.scoreColumn())); - scores = loaded.values; - hardware_max = loaded.maximum; - } else { - const loaded = b.tmemLoad32x32b(64, address.add(c.scoreColumn())); - b.fenceTmemLoad(); - for (loaded, 0..) |value, i| scores[i] = value.bitCast(.f32); - } - const key0 = t.first_key_tile.add(tiles.iv).mul(c.keys()); - // Hardware maxima describe raw scores. Only fully visible tiles may - // use them; page tails and mask boundaries reduce post-mask values. - const full = owns.bitAnd(t.valid(row, key0)).bitAnd(t.valid(row, key0.add(63))); - for (0..64) |i| { - scores[i] = b.select(owns.bitAnd(t.valid(row, key0.add(i))), scores[i].mul(c.scale * std.math.log2e), neg_inf); - } - const tile_max = if (c.architecture == .sm103 and c.scale > 0) blk: { - var choose = b.openIfElse(full, .{zero}); - { - choose.yieldThen(.{hardware_max.mul(c.scale * std.math.log2e)}); - } - { - choose.yieldElse(.{reduceRow(64, scores, true)}); - } - break :blk choose.results[0]; - } else reduceRow(64, scores, true); - const maximum = tiles.carried[0].maximum(tile_max); - const reference = maximum.maximum(-std.math.floatMax(f32)); - const alpha = b.exp2Approx(tiles.carried[0].sub(reference)); - var probabilities: [64]V = undefined; - for (0..64) |i| { - probabilities[i] = b.exp2Approx(scores[i].sub(reference)); - } - const denominator = tiles.carried[1].mul(alpha).add(reduceRow(64, probabilities, false)); - var write = b.openIf(owns); - { - for (0..64) |i| t.putProbability(local_row, b.constant(.i32, i), probabilities[i]); - write.close(); - } - var rescale = b.openIf(tiles.iv.gt(0)); - { - for (0..@as(usize, @intCast(@divExact(if (c.head_dim == 512) c.scoreColumn() else c.paddedDim(), 32)))) |chunk| { - const addr = address.add(chunk * 32); - var out = b.tmemLoad32x32b(32, addr); - b.fenceTmemLoad(); - for (&out) |*value| value.* = value.bitCast(.f32).mul(alpha).bitCast(.i32); - b.tmemStore32x32b(32, addr, out); - } - if (c.head_dim == 512) { - for (&saved) |*value| value.* = value.mul(alpha).bitCast(.i32); - b.tmemStore32x32b(64, address.add(c.scoreColumn()), saved); - } - b.fenceTmemStore(); - rescale.close(); - } - publishProbabilities(t); - multiply(t, tiles.iv, false); - tiles.yield(.{ maximum, denominator }); - } - const norm = t.norm(row, tiles.results[0], tiles.results[1]); - for (0..@as(usize, @intCast(@divExact(c.paddedDim(), 32)))) |chunk| { - const out = b.tmemLoad32x32b(32, address.add(chunk * 32)); - b.fenceTmemLoad(); - var valid = b.openIf(owns.bitAnd(row.lt(t.qlen.mul(c.group)))); - { - for (0..32) |i| { - const dim = chunk * 32 + i; - if (dim < c.head_dim) t.put(row, b.constant(.i32, dim), b.select(tiles.results[1].gt(0), out[i].bitCast(.f32).mul(norm), zero)); - } - if (c.splits > 1 and chunk == 0) { - t.put(row, b.constant(.i32, c.head_dim), tiles.results[0]); - t.put(row, b.constant(.i32, c.head_dim + 1), tiles.results[1]); - } - valid.close(); - } - } -} - -pub const Merge = cute.Kernel(Config, .{ - .name = "paged_attention_cute_merge", - .inputs = &.{ "partials", "lengths", "starts", "sink" }, - .outputs = &.{"out"}, - .run = merge, -}); - -fn merge(b: *B, c: Config) cute.FinishError!void { - const a = try b.declareArgs(.{ - .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, - .lengths = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch} } }, - .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, - .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{if (c.sink) c.kv_heads * c.group else 1} } }, - .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, - }); - const token = b.blockIdx().x; - const head = b.blockIdx().y; - var active = b.openIf(token.lt(a.starts.get(.{c.batch}))); - { - var find = b.openFor(0, c.batch, 1, .{b.constant(.i32, 0)}); - { - const owns = token.ge(a.starts.get(.{find.iv})).bitAnd(token.lt(a.starts.get(.{find.iv.add(1)}))); - find.yield(.{b.select(owns, find.iv, find.carried[0])}); - } - const seq = find.results[0]; - const start = a.starts.get(.{seq}); - const qlen = a.starts.get(.{seq.add(1)}).sub(start); - const row = token.sub(start).mul(c.group).add(head.rem(c.group)); - const range = keyRange(b, c, row.div(c.rows()).mul(c.rows()), qlen, a.lengths.get(.{seq})); - const splits = liveSplits(c, range.count); - const zero = b.constant(.f32, 0); - const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); - var maximum = b.openFor(0, splits, 1, .{sink}); - { - maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); - } - const reference = maximum.results[0].maximum(-std.math.floatMax(f32)); - var dims = b.openFor(b.threadIdx().x, c.head_dim, 128, .{}); - { - var sum = b.openFor(0, splits, 1, .{ zero, if (c.sink) b.exp2Approx(sink.sub(reference)) else zero }); - { - const weight = b.exp2Approx(a.partials.get(.{ sum.iv, token, head, c.head_dim }).sub(reference)); - sum.yield(.{ - sum.carried[0].add(weight.mul(a.partials.get(.{ sum.iv, token, head, dims.iv }))), - sum.carried[1].add(weight.mul(a.partials.get(.{ sum.iv, token, head, c.head_dim + 1 }))), - }); - } - const norm = b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero); - a.out.set(.{ token, head, dims.iv }, sum.results[0].mul(norm).to(c.dtype)); - dims.close(); - } - active.close(); - } -} - -test "SM10x layouts and instruction feature gates" { - for ([_]i64{ 32, 64, 128, 256, 512 }) |dim| for ([_]bool{ true, false }) |decode_only| for ([_]Architecture{ .sm100, .sm103 }) |architecture| { - const c: Config = .{ .architecture = architecture, .head_dim = dim, .tokens = 80, .batch = 4, .kv_heads = 2, .group = 5, .pages_per_seq = 256, .page_size = 16, .max_query = if (decode_only) 1 else 17, .dtype = .bf16, .decode = decode_only, .causal = true, .window = 0, .scale = 0.125, .sink = true, .splits = 4 }; - try std.testing.expect(c.columns() >= 32 and c.columns() <= 512); - try std.testing.expect(c.sharedBytes() <= 227 * 1024); - const ir = try Attention.emit(std.testing.allocator, c, block); - defer std.testing.allocator.free(ir); - try std.testing.expect(std.mem.indexOf(u8, ir, "tcgen05.mma") != null); - try std.testing.expectEqual(architecture == .sm103 and !decode_only, std.mem.indexOf(u8, ir, "tcgen05.ld.red") != null); - const merge_ir = try Merge.emit(std.testing.allocator, c, .{ 128, 1, 1 }); - defer std.testing.allocator.free(merge_ir); - }; -} diff --git a/zml/kernels/paged_attention/cute/sm10x/sm103.zig b/zml/kernels/paged_attention/cute/sm10x/sm103.zig deleted file mode 100644 index fe96916d..00000000 --- a/zml/kernels/paged_attention/cute/sm10x/sm103.zig +++ /dev/null @@ -1,36 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); -const llvm = @import("mlir/dialects/llvm"); - -const cute = @import("../../../../zml.zig").kernel.cute; - -pub const Scores = struct { - values: [64]cute.Value, - maximum: cute.Value, -}; - -// SM103 adds a per-TMEM-lane reduction across the loaded columns. This is -// useful for query-major scores; swapped decode reduces across lanes instead. -pub fn loadScores(b: *cute.Builder, address: cute.Value) Scores { - const asm_parts = comptime blk: { - @setEvalBranchQuota(20000); - var registers: []const u8 = ""; - var constraints: []const u8 = ""; - for (0..65) |i| { - if (i < 64) registers = registers ++ (if (i == 0) "" else ", ") ++ std.fmt.comptimePrint("${d}", .{i}); - constraints = constraints ++ (if (i == 0) "" else ",") ++ "=f"; - } - break :blk .{ - "tcgen05.ld.red.sync.aligned.32x32b.x64.max.f32 {" ++ registers ++ "}, $64, [$65];", - constraints ++ ",r", - }; - }; - const result = b.emit(llvm.inline_asm(b.ctx, asm_parts[0], &.{address.inner}, .{ .result_type = llvm.structType(b.ctx, &@as([65]*const mlir.Type, @splat(.float(b.ctx, .f32)))), .constraints = asm_parts[1], .has_side_effects = true }, b.loc())); - var scores: Scores = undefined; - for (0..65) |i| { - const value = b.emit(llvm.extractvalue(b.ctx, result.inner, cute.DType.f32.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); - if (i < 64) scores.values[i] = value else scores.maximum = value; - } - b.fenceTmemLoad(); - return scores; -} diff --git a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig b/zml/kernels/paged_attention/cute/sm10x/sm10x.zig deleted file mode 100644 index afc4298e..00000000 --- a/zml/kernels/paged_attention/cute/sm10x/sm10x.zig +++ /dev/null @@ -1,151 +0,0 @@ -const std = @import("std"); -const stdx = @import("stdx"); - -const zml = @import("../../../../zml.zig"); -const cute = @import("../cute.zig"); -const kernel = @import("attention.zig"); - -pub const Options = cute.Options; -pub const Parameters = cute.Parameters; - -test { - _ = kernel; -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - const cc = zml.platform.cuda.computeCapability(platform) orelse return false; - return cc.major == 10; -} - -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { - const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); - const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const decode = !parameters.options_.is_prefill; - const cc = zml.platform.cuda.computeCapability(zml.Compiler.current().platform).?; - var c: kernel.Config = .{ - .architecture = if (cc.minor >= 3) .sm103 else .sm100, - .head_dim = q_.dim(.hd), - .tokens = q_.dim(.b), - .batch = parameters.block_table.dim(.b), - .kv_heads = k_.dim(.hkv), - .group = q_.dim(.hg), - .pages_per_seq = parameters.block_table.dim(.p), - .page_size = k_.dim(.k_chunk), - .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), - .dtype = zml.kernel.cute.from(q_.dtype()), - .decode = decode, - .causal = opts.mask.isCausal(), - .window = opts.mask.slidingWindow() orelse 0, - .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), - .sink = opts.sink != null, - }; - const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); - const work = query_tiles * c.kv_heads; - const bound = if (c.window > 0 and c.causal) @min(c.window + @divTrunc(c.rows() + c.group - 1, c.group), c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 2 * c.keys() - 1, 2 * c.keys())))); - // An unused scalar stays a constant buffer; broadcasting a dummy for - // every head would add a helper kernel to each attention invocation. - const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.scalar(0, .f32).reshape(.{1}); - const out = kernel.Attention.call(.{ .q = q_, .k = k_, .v = v_, .table = parameters.block_table, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ - .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape(), - }, .{ .cfg = c, .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, .block = kernel.block }).out; - const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .lengths = parameters.seq_lens, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, - .block = .{ 128, 1, 1 }, - }).out else out; - return result.transpose(q.shape()); -} - -const ArchitectureCheck = struct { - fn forward(q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, c: kernel.Config) zml.Tensor { - const table_values: [9]i32 = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; - const table = zml.Tensor.constantTensor(.init(.{ 1, 9 }, .i32), std.mem.asBytes(&table_values)); - const lengths = zml.Tensor.constantTensor(.init(.{1}, .i32), std.mem.asBytes(&[_]i32{129})); - const starts = zml.Tensor.constantTensor(.init(.{2}, .i32), std.mem.asBytes(&[_]i32{ 0, 3 })); - const sink = zml.Tensor.zeroes(.init(.{3}, .f32)); - const partials = kernel.Attention.call(.{ .q = q, .k = k, .v = v, .table = table, .lengths = lengths, .starts = starts, .sink = sink }, .{ - .out = .init(.{ 2, 3, 3, c.head_dim + 2 }, .f32), - }, .{ .cfg = c, .grid = .{ 1, 1, 2 }, .block = kernel.block }).out; - return kernel.Merge.call(.{ .partials = partials, .lengths = lengths, .starts = starts, .sink = sink }, .{ .out = q.shape() }, .{ - .cfg = c, - .grid = .{ 3, 3, 1 }, - .block = .{ 128, 1, 1 }, - }).out; - } -}; - -test "SM10x GB200 and GB300 instruction paths match CPU reference" { - const platform = zml.testing.env(); - if (!isAvailable(platform)) return error.SkipZigTest; - const cc = zml.platform.cuda.computeCapability(platform).?; - const a = std.testing.allocator; - const io = std.testing.io; - const BF16 = zml.floats.BFloat16; - const table: [9]usize = .{ 4, 2, 8, 1, 6, 0, 7, 3, 5 }; - for ([_]i64{ 128, 512 }) |head_dim| { - const d: usize = @intCast(head_dim); - const q_host = try a.alloc(BF16, 9 * d); - defer a.free(q_host); - const k_host = try a.alloc(BF16, 144 * d); - defer a.free(k_host); - const v_host = try a.alloc(BF16, 144 * d); - defer a.free(v_host); - for (q_host, 0..) |*value, i| value.* = .fromF32(@as(f32, @floatFromInt(i % 13)) / 16 - 0.375); - @memset(k_host, .fromF32(std.math.nan(f32))); - @memset(v_host, .fromF32(std.math.nan(f32))); - for (0..129) |key| for (0..d) |feature| { - const offset = (table[key / 16] * 16 + key % 16) * d + feature; - k_host[offset] = .fromF32(@as(f32, @floatFromInt((key * 7 + feature) % 17)) / 16 - 0.5); - v_host[offset] = .fromF32(@as(f32, @floatFromInt((key + feature * 3) % 23)) / 16 - 0.625); - }; - const q: zml.Tensor = .init(.{ .b = 3, .hkv = 1, .hg = 3, .hd = d }, .bf16); - const k: zml.Tensor = .init(.{ .page = 9, .k_chunk = 16, .hkv = 1, .hd = d }, .bf16); - const v: zml.Tensor = .fromShape(k.shape()); - var qb = try zml.Buffer.fromBytes(io, platform, q.shape(), std.mem.sliceAsBytes(q_host)); - defer qb.deinit(); - var kb = try zml.Buffer.fromBytes(io, platform, k.shape(), std.mem.sliceAsBytes(k_host)); - defer kb.deinit(); - var vb = try zml.Buffer.fromBytes(io, platform, k.shape(), std.mem.sliceAsBytes(v_host)); - defer vb.deinit(); - for ([_]kernel.Architecture{ .sm100, .sm103 }) |architecture| { - if (architecture == .sm103 and cc.minor < 3) continue; - const c: kernel.Config = .{ .architecture = architecture, .head_dim = head_dim, .tokens = 3, .batch = 1, .kv_heads = 1, .group = 3, .pages_per_seq = 9, .page_size = 16, .max_query = 3, .dtype = .bf16, .decode = false, .causal = true, .window = 0, .scale = 0.125, .sink = true, .splits = 2 }; - const exe = try platform.compileFn(a, io, ArchitectureCheck.forward, .{ q, k, v, c }, .{}); - defer exe.deinit(); - var output = try exe.eval(a, io, .{ qb, kb, vb }); - defer output.deinit(); - var host = try output.toSliceAlloc(a, io); - defer host.free(a); - const actual = host.items(BF16); - for (0..9) |row| { - const keys = 127 + row / 3; - var scores: [129]f32 = undefined; - var maximum: f32 = 0; // Sink score, with zero value. - for (0..keys) |key| { - const offset = (table[key / 16] * 16 + key % 16) * d; - var score: f32 = 0; - for (0..d) |feature| score += q_host[row * d + feature].toF32() * k_host[offset + feature].toF32(); - scores[key] = score * c.scale; - maximum = @max(maximum, scores[key]); - } - var denominator = @exp(-maximum); - for (0..keys) |key| { - scores[key] = @exp(scores[key] - maximum); - denominator += scores[key]; - } - for (0..d) |feature| { - var expected: f32 = 0; - for (0..keys) |key| expected += scores[key] * v_host[(table[key / 16] * 16 + key % 16) * d + feature].toF32(); - expected /= denominator; - const got = actual[row * d + feature].toF32(); - try std.testing.expect(std.math.isFinite(got)); - try std.testing.expect(@abs(got - expected) <= 0.003 + 0.02 * @abs(expected)); - } - } - } - } -} diff --git a/zml/kernels/paged_attention/cute/sm120/BUILD.bazel b/zml/kernels/paged_attention/cute/sm120/BUILD.bazel deleted file mode 100644 index f93bd5e0..00000000 --- a/zml/kernels/paged_attention/cute/sm120/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml/kernels/paged_attention/cute:__pkg__"]) - -filegroup( - name = "sm120", - srcs = [ - "attention.zig", - "sm120.zig", - ], -) diff --git a/zml/kernels/paged_attention/cute/sm120/attention.zig b/zml/kernels/paged_attention/cute/sm120/attention.zig deleted file mode 100644 index e9675252..00000000 --- a/zml/kernels/paged_attention/cute/sm120/attention.zig +++ /dev/null @@ -1,446 +0,0 @@ -const std = @import("std"); -const mlir = @import("mlir"); -const llvm = @import("mlir/dialects/llvm"); -const vector = @import("mlir/dialects/vector"); - -const zml = @import("../../../../zml.zig"); -const cute = zml.kernel.cute; -const B = cute.Builder; -const V = cute.Value; - -pub const Config = struct { - head_dim: i64, - tokens: i64, - batch: i64, - kv_heads: i64, - group: i64, - pages_per_seq: i64, - page_size: i64, - max_query: i64, - dtype: cute.DType, - warps_per_query: i64, - kv_tile: i64, - causal: bool, - window: i64, - scale: f32, - sink: bool, - splits: i64 = 1, - - pub fn rows(c: Config) i64 { - return @divExact(64, c.warps_per_query); - } - pub fn queryTiles(c: Config) i64 { - return @divTrunc(c.max_query * c.group + c.rows() - 1, c.rows()); - } - fn kvBase(c: Config) i64 { - return c.rows() * c.head_dim * 2; - } - fn pBase(c: Config) i64 { - return c.kvBase() + c.kv_tile * c.head_dim * 2; - } - fn redBase(c: Config) i64 { - return c.pBase() + c.rows() * c.kv_tile * 2; - } - fn sharedBytes(c: Config) i64 { - return c.redBase() + 2 * 64 * 4; - } -}; - -pub const Kernel = cute.Kernel(Config, .{ - .name = "paged_attention_cute_sm120", - .inputs = &.{ "q", "k", "v", "table", "lengths", "starts", "sink" }, - .outputs = &.{"out"}, - .run = run, -}); - -fn flatten(comptime n: usize, values: [n][4]V) [n * 4]V { - var result: [n * 4]V = undefined; - for (0..n) |i| @memcpy(result[i * 4 ..][0..4], &values[i]); - return result; -} - -fn packAcc(b: *B, values: [4]V) V { - return b.emit(vector.from_elements(b.ctx, &.{ values[0].inner, values[1].inner, values[2].inner, values[3].inner }, .vector(&.{4}, .float(b.ctx, .f32)), b.loc())); -} - -fn loadI32(b: *B, ptr: V, index: V) V { - return b.loadWords(1, ptr.add(index.to(.i64).mul(4)), .global, .{})[0]; -} - -// XOR whole 16-byte vectors so cp.async and ldmatrix agree on bank layout. -fn swizzle(row: V, col: V, width: i64) V { - return row.mul(width).add(col.bitXor(row.rem(@min(8, @divExact(width, 8))).mul(8))).mul(2); -} - -fn unpackStruct(b: *B, comptime n: usize, value: V, dtype: cute.DType) [n]V { - var result: [n]V = undefined; - for (&result, 0..) |*r, i| r.* = b.emit(llvm.extractvalue(b.ctx, value.inner, dtype.toMlir(b.ctx), &.{@intCast(i)}, b.loc())); - return result; -} - -fn matrixLoad(b: *B, comptime n: usize, comptime transpose: bool, address: V) [n]V { - const assembly = std.fmt.comptimePrint("ldmatrix.sync.aligned.m8n8.x{d}{s}.shared.b16 {{", .{ n, if (transpose) ".trans" else "" }) ++ - (if (n == 4) "$0, $1, $2, $3}, [$4];" else "$0, $1}, [$2];"); - const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{address.inner}, .{ .result_type = llvm.structType(b.ctx, &@as([n]*const mlir.Type, @splat(.int(b.ctx, .i32)))), .constraints = if (n == 4) "=r,=r,=r,=r,r" else "=r,=r,r", .has_side_effects = true }, b.loc())); - return unpackStruct(b, n, result, .i32); -} - -fn mma(b: *B, comptime dtype: cute.DType, a: [4]V, k: [2]V, acc: [4]V) [4]V { - const assembly = "mma.sync.aligned.m16n8k16.row.col.f32." ++ @tagName(dtype) ++ "." ++ @tagName(dtype) ++ ".f32 " ++ - "{$0, $1, $2, $3}, {$4, $5, $6, $7}, {$8, $9}, {$10, $11, $12, $13};"; - const result = b.emit(llvm.inline_asm(b.ctx, assembly, &.{ a[0].inner, a[1].inner, a[2].inner, a[3].inner, k[0].inner, k[1].inner, acc[0].inner, acc[1].inner, acc[2].inner, acc[3].inner }, .{ .result_type = llvm.structType(b.ctx, &@as([4]*const mlir.Type, @splat(.float(b.ctx, .f32)))), .constraints = "=f,=f,=f,=f,r,r,r,r,r,r,f,f,f,f" }, b.loc())); - return unpackStruct(b, 4, result, .f32); -} - -fn run(b: *B, c: Config) cute.FinishError!void { - switch (c.head_dim) { - inline 32, 64, 128, 256, 512 => |d| switch (c.warps_per_query) { - inline 2, 4 => |w| switch (c.kv_tile) { - inline 32, 64 => |n| switch (c.dtype) { - inline .bf16, .f16 => |dt| try runTile(b, c, d, w, n, dt), - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - }, - else => unreachable, - } -} - -const TileContext = struct { - b: *B, - c: Config, - q: V, - k: V, - v: V, - table: V, - sink: V, - out: V, - shared: V, - tid: V, - lane: V, - warp: V, - seq: V, - head: V, - first_row: V, - start: V, - qlen: V, - klen: V, - split: V, - - // Every load resolves its logical page on device. Tail copies are zeroed - // before either MMA, including V tails whose backing allocation holds NaN. - fn loadKV(t: TileContext, source: V, key_base: V) void { - const b = t.b; - const c = t.c; - if (c.page_size == 16 and c.head_dim >= 64) { - // A page ID is shared by all feature vectors in its 16 rows. - // Resolve it once, then issue independent vector copies instead - // of serializing a page-table load before every cp.async. - var pages = b.openFor(0, c.kv_tile, 16, .{}); - { - const first_key = key_base.add(pages.iv); - const page_valid = first_key.lt(t.klen); - const logical_page = b.select(page_valid, first_key.div(16), b.constant(.i32, 0)); - const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(logical_page)); - const base = page.to(.i64).mul(16 * c.kv_heads * c.head_dim).add(t.head.to(.i64).mul(c.head_dim)); - for (0..@as(usize, @intCast(@divExact(c.head_dim, 64)))) |i| { - const vector_index = t.tid.add(i * 128); - const row = vector_index.div(@divExact(c.head_dim, 8)); - const col = vector_index.rem(@divExact(c.head_dim, 8)).mul(8); - const offset = base.add(row.to(.i64).mul(c.kv_heads * c.head_dim)).add(col.to(.i64)); - b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(pages.iv.add(row), col, c.head_dim)), source.add(offset.mul(2)), first_key.add(row).lt(t.klen)); - } - pages.close(); - } - return; - } - var copy = b.openFor(t.tid, c.kv_tile * @divExact(c.head_dim, 8), 128, .{}); - { - const row = copy.iv.div(@divExact(c.head_dim, 8)); - const col = copy.iv.rem(@divExact(c.head_dim, 8)).mul(8); - const key = key_base.add(row); - const valid = key.lt(t.klen); - const safe_key = b.select(valid, key, b.constant(.i32, 0)); - const page = loadI32(b, t.table, t.seq.mul(c.pages_per_seq).add(safe_key.div(c.page_size))); - const offset = page.to(.i64).mul(c.page_size).add(safe_key.rem(c.page_size).to(.i64)).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.head_dim).add(col.to(.i64)); - b.cpAsync16(t.shared.add(c.kvBase()).add(swizzle(row, col, c.head_dim)), source.add(offset.mul(2)), valid); - copy.close(); - } - } - - fn publishReduction(t: TileContext, row: V, value: V, comptime maximum: bool) void { - const b = t.b; - var v = value; - inline for (.{ 2, 1 }) |step| v = if (maximum) v.maximum(b.shuffleXor(v, step)) else v.add(b.shuffleXor(v, step)); - const address = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.mul(16).add(row).mul(4)); - var leader = b.openIf(t.lane.rem(4).eq(0)); - { - b.storeWords(1, address, .shared, .{v.bitCast(.i32)}); - leader.close(); - } - // Callers synchronize once both row halves have been published. - } - - fn readReduction(t: TileContext, comptime w: usize, row: V, comptime maximum: bool) V { - const b = t.b; - const base = t.shared.add(t.c.redBase() + (if (maximum) @as(i64, 0) else 256)).add(t.warp.div(w).mul(w * 16).add(row).mul(4)); - var result = b.loadWords(1, base, .shared, .{})[0].bitCast(.f32); - for (1..w) |i| { - const v = b.loadWords(1, base.add(i * 16 * 4), .shared, .{})[0].bitCast(.f32); - result = if (maximum) result.maximum(v) else result.add(v); - } - return result; - } - - fn store(t: TileContext, row: V, dim: V, value: V) void { - const c = t.c; - const token = t.start.add(row.div(c.group)); - var offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(row.rem(c.group).to(.i64)); - if (c.splits > 1) { - offset = t.split.to(.i64).mul(c.tokens * c.kv_heads * c.group).add(offset).mul(c.head_dim + 2).add(dim.to(.i64)); - t.b.storeWords(1, t.out.add(offset.mul(4)), .global, .{value.bitCast(.i32)}); - } else { - offset = offset.mul(c.head_dim).add(dim.to(.i64)); - t.b.storeHalf(t.out.add(offset.mul(2)), .global, value.to(c.dtype).bitCast(.i16).to(.i32)); - } - } -}; - -fn runTile(b: *B, c: Config, comptime d: usize, comptime w: usize, comptime n: usize, comptime dt: cute.DType) cute.FinishError!void { - std.debug.assert(c.sharedBytes() <= 99 * 1024); - const a = try b.declareArgs(.{ - .q = .{ .ptr = dt }, - .k = .{ .ptr = dt }, - .v = .{ .ptr = dt }, - .table = .{ .ptr = cute.DType.i32 }, - .lengths = .{ .ptr = cute.DType.i32 }, - .starts = .{ .ptr = cute.DType.i32 }, - .sink = .{ .ptr = cute.DType.f32 }, - .out = .{ .ptr = if (c.splits > 1) cute.DType.f32 else dt }, - }); - b.setFunctionAttribute(.named(b.ctx, "cu_attrs", .dict(b.ctx, &.{.named(b.ctx, "max_dynamic_shared_size_bytes", b.devMaxSharedMemoryOptin())}))); - const seq = b.blockIdx().x.div(c.queryTiles()); - const first_row = b.blockIdx().x.rem(c.queryTiles()).mul(c.rows()); - const starts = b.ptrToInt(a.starts, .i64); - const start = loadI32(b, starts, seq); - const qlen = loadI32(b, starts, seq.add(1)).sub(start); - var active = b.openIf(first_row.lt(qlen.mul(c.group))); - { - const tid = b.threadIdx().x; - const t: TileContext = .{ - .b = b, - .c = c, - .q = b.ptrToInt(a.q, .i64), - .k = b.ptrToInt(a.k, .i64), - .v = b.ptrToInt(a.v, .i64), - .table = b.ptrToInt(a.table, .i64), - .sink = b.ptrToInt(a.sink, .i64), - .out = b.ptrToInt(a.out, .i64), - .shared = b.ptrToInt(b.allocSmem(.i8, c.sharedBytes(), 128), .i32), - .tid = tid, - .lane = tid.rem(32), - .warp = tid.div(32), - .seq = seq, - .head = b.blockIdx().y, - .first_row = first_row, - .start = start, - .qlen = qlen, - .klen = loadI32(b, b.ptrToInt(a.lengths, .i64), seq), - .split = b.blockIdx().z, - }; - var copy_q = b.openFor(tid, c.rows() * (d / 8), 128, .{}); - { - const row = copy_q.iv.div(d / 8); - const col = copy_q.iv.rem(d / 8).mul(8); - const packed_row = first_row.add(row); - const valid = packed_row.lt(qlen.mul(c.group)); - const token = start.add(b.select(valid, packed_row.div(c.group), b.constant(.i32, 0))); - const offset = token.to(.i64).mul(c.kv_heads).add(t.head.to(.i64)).mul(c.group).add(packed_row.rem(c.group).to(.i64)).mul(d).add(col.to(.i64)); - b.cpAsync16(t.shared.add(swizzle(row, col, d)), t.q.add(offset.mul(2)), valid); - copy_q.close(); - } - const first_pos = t.klen.sub(qlen).add(first_row.div(c.group)); - const last_pos = t.klen.sub(qlen).add(first_row.add(c.rows() - 1).div(c.group).minimum(qlen.sub(1))); - const begin = if (c.window > 0) first_pos.sub(c.window - 1).maximum(0) else b.constant(.i32, 0); - const end = if (c.causal) last_pos.add(1).minimum(t.klen) else t.klen; - const first_tile = begin.div(n); - const tiles = end.add(n - 1).div(n).sub(first_tile).maximum(0); - const lo = first_tile.add(tiles.mul(t.split).div(c.splits)); - const hi = first_tile.add(tiles.mul(t.split.add(1)).div(c.splits)); - const row_base = t.warp.div(w).mul(16); - const key_part = t.warp.rem(w).mul(n / w); - const dim_part = t.warp.rem(w).mul(d / w); - const row0 = t.lane.div(4); - const col0 = t.lane.rem(4).mul(2); - const zero = b.constant(.f32, 0); - const neg_inf = b.constant(.f32, -std.math.inf(f32)); - const nf = n / w / 8; - const of = d / w / 8; - const count = 4 + of; - var initial: [count]V = @splat(zero); - initial[0] = neg_inf; - initial[1] = neg_inf; - for (initial[4..]) |*v| v.* = packAcc(b, .{ zero, zero, zero, zero }); - b.cpAsyncWaitAll(); - b.syncThreads(); - var loop = b.openFor(lo, hi, 1, initial); - { - const key_base = loop.iv.mul(n); - t.loadKV(t.k, key_base); - b.cpAsyncWaitAll(); - b.syncThreads(); - var scores: [nf][4]V = @splat(@splat(zero)); - // Keep only MMA operands live in registers; Q remains in shared - // memory so D512 does not consume the entire per-thread register file. - var qk = b.openFor(0, d, 16, @as([nf * 4]V, @splat(zero))); - { - const qa = matrixLoad(b, 4, false, t.shared.add(swizzle(row_base.add(t.lane.rem(16)), qk.iv.add(t.lane.div(16).mul(8)), d))); - for (0..nf) |i| { - const kb = matrixLoad(b, 2, false, t.shared.add(c.kvBase()).add(swizzle(key_part.add(i * 8).add(t.lane.rem(8)), qk.iv.add(t.lane.div(8).rem(2).mul(8)), d))); - scores[i] = mma(b, dt, qa, kb, qk.carried[i * 4 ..][0..4].*); - } - qk.yield(flatten(nf, scores)); - } - for (0..nf) |i| scores[i] = qk.results[i * 4 ..][0..4].*; - var local_max: [2]V = @splat(neg_inf); - for (0..nf) |i| { - for (0..4) |j| { - const row = first_row.add(row_base).add(row0).add((j / 2) * 8); - const position = t.klen.sub(qlen).add(row.div(c.group)); - const key = key_base.add(key_part).add(i * 8).add(col0).add(j % 2); - var valid = row.lt(qlen.mul(c.group)).bitAnd(key.lt(t.klen)); - if (c.causal) valid = valid.bitAnd(key.le(position)); - if (c.window > 0) valid = valid.bitAnd(key.gt(position.sub(c.window))); - scores[i][j] = b.select(valid, scores[i][j].mul(c.scale * std.math.log2e), neg_inf); - local_max[j / 2] = local_max[j / 2].maximum(scores[i][j]); - } - } - for (0..2) |h| t.publishReduction(row0.add(h * 8), local_max[h], true); - b.syncThreads(); - // All K reads have completed. Reuse its allocation for V, and - // overlap these global loads with softmax and probability stores. - t.loadKV(t.v, key_base); - var m: [2]V = undefined; - var alpha: [2]V = undefined; - var local_sum: [2]V = @splat(zero); - for (0..2) |h| { - m[h] = loop.carried[h].maximum(t.readReduction(w, row0.add(h * 8), true)); - // Empty/fully masked row tiles have m=-inf and denominator 0. - alpha[h] = b.exp2Approx(loop.carried[h].sub(m[h].maximum(-std.math.floatMax(f32)))); - } - for (0..nf) |i| { - for (0..4) |j| { - scores[i][j] = b.exp2Approx(scores[i][j].sub(m[j / 2].maximum(-std.math.floatMax(f32)))); - local_sum[j / 2] = local_sum[j / 2].add(scores[i][j]); - } - for (0..2) |h| { - const row = row_base.add(row0).add(h * 8); - const col = key_part.add(i * 8).add(col0); - b.storeWords(1, t.shared.add(c.pBase()).add(swizzle(row, col, n)), .shared, .{b.packHalf2(scores[i][h * 2], scores[i][h * 2 + 1], dt)}); - } - } - for (0..2) |h| t.publishReduction(row0.add(h * 8), local_sum[h], false); - b.cpAsyncWaitAll(); - b.syncThreads(); - var next: [count]V = undefined; - for (0..2) |h| { - next[h] = m[h]; - next[h + 2] = loop.carried[h + 2].mul(alpha[h]).add(t.readReduction(w, row0.add(h * 8), false)); - } - var acc: [of][4]V = undefined; - for (0..of) |i| for (0..4) |j| { - acc[i][j] = b.vectorExtract(loop.carried[4 + i], @intCast(j), .f32).mul(alpha[j / 2]); - }; - var packed_acc: [of]V = undefined; - for (0..of) |i| packed_acc[i] = packAcc(b, acc[i]); - var pv = b.openFor(0, n, 16, packed_acc); - { - const pa = matrixLoad(b, 4, false, t.shared.add(c.pBase()).add(swizzle(row_base.add(t.lane.rem(16)), pv.iv.add(t.lane.div(16).mul(8)), n))); - for (0..of) |i| { - const vb = matrixLoad(b, 2, true, t.shared.add(c.kvBase()).add(swizzle(pv.iv.add(t.lane.rem(16)), dim_part.add(i * 8), d))); - var prev: [4]V = undefined; - for (0..4) |j| prev[j] = b.vectorExtract(pv.carried[i], @intCast(j), .f32); - acc[i] = mma(b, dt, pa, vb, prev); - packed_acc[i] = packAcc(b, acc[i]); - } - pv.yield(packed_acc); - } - @memcpy(next[4..], &pv.results); - b.syncThreads(); - loop.yield(next); - } - for (0..2) |h| { - const row = first_row.add(row_base).add(row0).add(h * 8); - var valid = b.openIf(row.lt(qlen.mul(c.group))); - { - const m = loop.results[h]; - const l = loop.results[h + 2]; - var norm = b.select(l.gt(0), b.rcpApprox(l), zero); - if (c.splits == 1 and c.sink) { - const sink = loadI32(b, t.sink, t.head.mul(c.group).add(row.rem(c.group))).bitCast(.f32).mul(std.math.log2e); - const maximum = m.maximum(sink); - const weight = b.exp2Approx(m.sub(maximum)); - norm = weight.div(l.mul(weight).add(b.exp2Approx(sink.sub(maximum)))); - } - for (0..of) |i| for (0..2) |j| { - const value = b.vectorExtract(loop.results[4 + i], @intCast(h * 2 + j), .f32); - t.store(row, dim_part.add(i * 8).add(col0).add(j), if (c.splits > 1) value else value.mul(norm)); - }; - if (c.splits > 1) { - var leader = b.openIf(t.warp.rem(w).eq(0).bitAnd(t.lane.rem(4).eq(0))); - { - t.store(row, b.constant(.i32, d), m); - t.store(row, b.constant(.i32, d + 1), l); - leader.close(); - } - } - valid.close(); - } - } - active.close(); - } -} - -pub const Merge = cute.Kernel(Config, .{ - .name = "paged_attention_cute_sm120_merge", - .inputs = &.{ "partials", "starts", "sink" }, - .outputs = &.{"out"}, - .run = merge, -}); - -fn merge(b: *B, c: Config) cute.FinishError!void { - const a = try b.declareArgs(.{ - .partials = .{ .tensor = .{ .dtype = .f32, .shape = &.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 } } }, - .starts = .{ .tensor = .{ .dtype = .i32, .shape = &.{c.batch + 1} } }, - .sink = .{ .tensor = .{ .dtype = .f32, .shape = &.{c.kv_heads * c.group} } }, - .out = .{ .tensor = .{ .dtype = c.dtype, .shape = &.{ c.tokens, c.kv_heads * c.group, c.head_dim } } }, - }); - const token = b.blockIdx().x; - const head = b.blockIdx().y; - var valid = b.openIf(token.lt(a.starts.get(.{c.batch}))); - { - const zero = b.constant(.f32, 0); - const sink = if (c.sink) a.sink.get(.{head}).mul(std.math.log2e) else b.constant(.f32, -std.math.inf(f32)); - var maximum = b.openFor(0, c.splits, 1, .{sink}); - { - maximum.yield(.{maximum.carried[0].maximum(a.partials.get(.{ maximum.iv, token, head, c.head_dim }))}); - } - const m = maximum.results[0].maximum(-std.math.floatMax(f32)); - var dims = b.openFor(b.threadIdx().x, c.head_dim, 128, .{}); - { - var sum = b.openFor(0, c.splits, 1, .{ zero, if (c.sink) b.exp2Approx(sink.sub(m)) else zero }); - { - const weight = b.exp2Approx(a.partials.get(.{ sum.iv, token, head, c.head_dim }).sub(m)); - sum.yield(.{ - sum.carried[0].add(weight.mul(a.partials.get(.{ sum.iv, token, head, dims.iv }))), - sum.carried[1].add(weight.mul(a.partials.get(.{ sum.iv, token, head, c.head_dim + 1 }))), - }); - } - a.out.set(.{ token, head, dims.iv }, sum.results[0].mul(b.select(sum.results[1].gt(0), b.rcpApprox(sum.results[1]), zero)).to(c.dtype)); - dims.close(); - } - valid.close(); - } -} diff --git a/zml/kernels/paged_attention/cute/sm120/sm120.zig b/zml/kernels/paged_attention/cute/sm120/sm120.zig deleted file mode 100644 index 6645abf1..00000000 --- a/zml/kernels/paged_attention/cute/sm120/sm120.zig +++ /dev/null @@ -1,69 +0,0 @@ -const stdx = @import("stdx"); - -const zml = @import("../../../../zml.zig"); -const cute = @import("../cute.zig"); -const kernel = @import("attention.zig"); - -pub const Options = cute.Options; -pub const Parameters = cute.Parameters; - -test { - _ = kernel; -} - -pub fn isAvailable(platform: *const zml.Platform) bool { - if (platform.target != .cuda) return false; - const cc = zml.platform.cuda.computeCapability(platform) orelse return false; - return cc.major == 12; -} - -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, opts: cute.AttentionOptions) zml.Tensor { - const q_ = q.transpose(.{ .b, .hkv, .hg, .hd }); - const k_ = k.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const v_ = v.transpose(.{ .page, .k_chunk, .hkv, .hd }); - stdx.debug.assert(q_.dim(.hkv) == k_.dim(.hkv) and k_.shape().eql(v_.shape()) and k_.dim(.hd) == q_.dim(.hd) and q_.dim(.hg) > 0, "CuTe paged attention requires aligned local Q/KV heads and matching caches", .{}); - const decode = !parameters.options_.is_prefill; - var c: kernel.Config = .{ - .head_dim = q_.dim(.hd), - .tokens = q_.dim(.b), - .batch = parameters.block_table.dim(.b), - .kv_heads = k_.dim(.hkv), - .group = q_.dim(.hg), - .pages_per_seq = parameters.block_table.dim(.p), - .page_size = k_.dim(.k_chunk), - .max_query = if (decode) 1 else @min(q_.dim(.b), @as(i64, @intCast(parameters.options_.max_seqlen_q))), - .dtype = zml.kernel.cute.from(q_.dtype()), - .warps_per_query = if (decode) 4 else 2, - .kv_tile = if (q_.dim(.hd) == 512) 32 else 64, - .causal = opts.mask.isCausal(), - .window = opts.mask.slidingWindow() orelse 0, - .scale = opts.scale orelse 1 / @sqrt(@as(f32, @floatFromInt(q_.dim(.hd)))), - .sink = opts.sink != null, - }; - // Mixed batches also contain single-token rows. Partition their KV - // range so one long decode cannot hold up all the prefill CTAs. - const query_tiles = if (decode) c.batch * c.queryTiles() else c.batch + @divTrunc(c.tokens * c.group + c.rows() - 1, c.rows()); - const work = query_tiles * c.kv_heads; - const bound = if (c.window > 0) @min(c.window, c.pages_per_seq * c.page_size) else c.pages_per_seq * c.page_size; - c.splits = @max(1, @min(32, @min(@divTrunc(@as(i64, @intCast(cute.multiprocessorCount(zml.Compiler.current().platform))) * 2 + work - 1, work), @divTrunc(bound + 127, 128)))); - const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{c.kv_heads * c.group}) else zml.Tensor.zeroes(.init(.{c.kv_heads * c.group}, .f32)); - const out = kernel.Kernel.call(.{ - .q = q_, - .k = k_, - .v = v_, - .table = parameters.block_table, - .lengths = parameters.seq_lens, - .starts = parameters.query_start_len, - .sink = sink, - }, .{ .out = if (c.splits > 1) zml.Shape.init(.{ c.splits, c.tokens, c.kv_heads * c.group, c.head_dim + 2 }, .f32) else q_.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.batch * c.queryTiles()), @intCast(c.kv_heads), @intCast(c.splits) }, - .block = .{ 128, 1, 1 }, - }).out; - const result = if (c.splits > 1) kernel.Merge.call(.{ .partials = out, .starts = parameters.query_start_len, .sink = sink }, .{ .out = q_.shape() }, .{ - .cfg = c, - .grid = .{ @intCast(c.tokens), @intCast(c.kv_heads * c.group), 1 }, - .block = .{ 128, 1, 1 }, - }).out else out; - return result.transpose(q.shape()); -} diff --git a/zml/kernels/paged_attention/metal/BUILD.bazel b/zml/kernels/paged_attention/metal/BUILD.bazel deleted file mode 100644 index 39d92259..00000000 --- a/zml/kernels/paged_attention/metal/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "metal", - srcs = [ - "metal.zig", - "paged_attn.zig", - ], -) diff --git a/zml/kernels/paged_attention/metal/README.md b/zml/kernels/paged_attention/metal/README.md deleted file mode 100644 index 8ccc6fd9..00000000 --- a/zml/kernels/paged_attention/metal/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Metal paged attention - -This backend is automatically selected for Metal. See the -[parent README](../README.md) for the attention equation, packed sequence -metadata, and page addressing contract. - -[The wrapper](metal.zig) requires split KV caches. It merges query head groups, -transposes queries and caches into the kernels' layouts, launches the -[Metal DSL kernels](paged_attn.zig), and restores the query's tagged layout for -the output. - -Queries and caches are `bf16` or `f16`; other dtypes panic. Causal `bf16` -decode with head dimension 32, 64, 128, 256 or 512 runs `fa_vec_paged`, which -is `bf16` only: a threadgroup holds up to four query heads of a KV head, times -splits of the KV positions, at most 256 threads. Rows split over the positions -are merged by `fa_vec_paged_reduce`; the split count follows the GPU core count -and the number of rows. Everything else, `f16` decode included, runs -`paged_attention_tiled` (head dimension 32, 64, 96, 128, 256 or 512, pages of -8, 16 or 32 positions), then `zero_padding_rows`. - -Causal and non-causal standard masks, left sliding windows, and a custom scale -are passed to the kernels. Explicit key indices are rejected, and -`opts.sink` is currently ignored. Fused caches are rejected by -ordinary paged attention. - -When sharding heads, keep complete GQA groups together with matching query and -cache head ownership. Each local computation needs the full required key range, -features, and matching sequence metadata. diff --git a/zml/kernels/paged_attention/metal/metal.zig b/zml/kernels/paged_attention/metal/metal.zig deleted file mode 100644 index 2cec68f5..00000000 --- a/zml/kernels/paged_attention/metal/metal.zig +++ /dev/null @@ -1,240 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; -const paged_attn = @import("paged_attn.zig"); - -const log = std.log.scoped(.@"zml/attention/metal"); - -test { - _ = paged_attn; -} - -pub const Options = struct { - batch_size: usize, - max_num_pages: usize, - max_seqlen_q: usize, - is_prefill: bool, - - pub fn isPrefill(self: Options) bool { - return self.is_prefill; - } - - pub fn maxNumPages(self: Options) usize { - return self.max_num_pages; - } -}; - -pub const Parameters = struct { - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - options_: Options, - - pub fn init(options_: Options) Parameters { - return .{ - .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), - .seq_lens = .init(.{ .b = options_.batch_size }, .i32), - .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), - .options_ = options_, - }; - } - - pub fn allocationSize(self: Parameters) usize { - var allocation_size: usize = 0; - - allocation_size += self.block_table.byteSize(); - allocation_size += self.seq_lens.byteSize(); - allocation_size += self.query_start_len.byteSize(); - - return allocation_size; - } - - pub fn options(self: Parameters) Options { - return self.options_; - } -}; - -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { - if (opts.mask == .indices) @panic("Sparse Attention is not implemented on Metal"); - - const num_kv_heads = q.dim(.hkv); - - const qh = q.merge(.{ .h = .{ .hkv, .hg } }).transpose(.{ .b, .h, .hd }); - const kc = k_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); - const vc = v_cache.transpose(.{ .page, .k_chunk, .hkv, .hd }); - - const sliding_window: i32 = if (opts.mask.slidingWindow()) |window| @intCast(window) else -1; - - const out = dispatch(qh, kc, vc, parameters.block_table, parameters.seq_lens, parameters.query_start_len, .{ - .is_causal = opts.mask.isCausal(), - .sliding_window = sliding_window, - .scale = opts.scale orelse 1.0 / @sqrt(@as(f32, @floatFromInt(q.dim(.hd)))), - }); - - // [total_q_tokens, num_heads, head_dim] -> {.b, .hkv, .hg, .hd} (q's order) - return out.splitAxis(.h, .{ .hkv = num_kv_heads, .hg = .auto }).transpose(q.shape()); -} - -const DispatchOptions = struct { - is_causal: bool, - /// Below 0 for global attention. - sliding_window: i32, - scale: f32, -}; - -/// Launches fa_vec_paged for causal bf16 decode and paged_attention_tiled -/// otherwise. -/// `q` and the result are `[tokens, heads, head_dim]`, the caches -/// `[pages, block_size, kv_heads, head_dim]`, `block_table` -/// `[seqs, max_blocks]`, `seq_lens` `[seqs]` and `query_start_len` -/// `[seqs + 1]`, all i32. -fn dispatch(q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, block_table: zml.Tensor, seq_lens: zml.Tensor, query_start_len: zml.Tensor, opts: DispatchOptions) zml.Tensor { - const dtype = q.dtype(); - if (dtype != .bf16 and dtype != .f16) std.debug.panic("paged attention: Metal takes bf16 or f16 queries, not {t}", .{dtype}); - if (k_cache.dtype() != dtype or v_cache.dtype() != dtype) std.debug.panic("paged attention: Metal takes caches of the queries' {t}, not {t} and {t}", .{ dtype, k_cache.dtype(), v_cache.dtype() }); - - const total_q_tokens: i32 = @intCast(q.dim(0)); - const num_heads: i32 = @intCast(q.dim(1)); - const head_dim: i32 = @intCast(q.dim(2)); - const block_size: i32 = @intCast(k_cache.dim(1)); - const num_kv_heads: i32 = @intCast(k_cache.dim(2)); - const num_seqs: i32 = @intCast(block_table.dim(0)); - const max_blocks: i32 = @intCast(block_table.dim(1)); - - const vec_decode = opts.is_causal and total_q_tokens == num_seqs and - (head_dim == 32 or head_dim == 64 or head_dim == 128 or head_dim == 256 or head_dim == 512) and - dtype == .bf16; - - if (vec_decode) { - const args: paged_attn.FaVecPaged.HostArgs = .{ - .num_heads = num_heads, - .n_kv = num_kv_heads, - .max_blocks = max_blocks, - .num_seqs = num_seqs, - .scale = opts.scale, - .sliding_window = opts.sliding_window, - }; - - const inputs: paged_attn.FaVecPaged.Kernel.Inputs = .{ - .q = q, - .k_cache = k_cache, - .v_cache = v_cache, - .block_tables = block_table, - .seq_lens = seq_lens, - .cu_seqlens_q = query_start_len, - }; - - const kv_capacity = max_blocks * block_size; - const span = if (opts.sliding_window < 0) kv_capacity else @min(kv_capacity, opts.sliding_window + 32); - const nrows = total_q_tokens * num_heads; - const launch = vecLaunch(nrows, @divExact(num_heads, num_kv_heads), span, coreCount()); - - const cfg: paged_attn.FaVecPaged.Cfg = .{ - .head_dim = head_dim, - .block_size = block_size, - .position_stride = num_kv_heads * head_dim, - .heads = launch.heads, - .splits = launch.splits, - }; - const dst = if (launch.splits == 1) q.shape() else zml.Shape.init(.{nrows * (head_dim * launch.splits + 2 * launch.splits)}, .f32); - const out = paged_attn.FaVecPaged.Kernel.call(inputs, .{ .dst = dst }, .{ - .cfg = cfg, - .grid = .{ @intCast(@divExact(launch.splits, launch.splits_per_threadgroup)), @intCast(@divExact(num_heads, launch.heads)), @intCast(total_q_tokens) }, - .threadgroup = .{ 32, @intCast(launch.heads * launch.splits_per_threadgroup), 1 }, - .constants = .{ .args = .of(args) }, - }).dst; - if (launch.splits == 1) return out; - - return paged_attn.FaVecPagedReduce.Kernel.call(.{ .htmp = out, .seq_lens = seq_lens, .cu_seqlens_q = query_start_len }, .{ .dst = q.shape() }, .{ - .cfg = .{ .splits = launch.splits, .head_dim = head_dim }, - .grid = .{ @intCast(nrows), @intCast(@divExact(head_dim, 4 * paged_attn.FaVecPagedReduce.simdgroups)), 1 }, - .threadgroup = .{ 32, paged_attn.FaVecPagedReduce.simdgroups, 1 }, - .constants = .{ .args = .of(args) }, - }).dst; - } - - const tile = paged_attn.PagedAttentionTiled.tileConfig(head_dim) orelse std.debug.panic("paged attention: head_dim {d} unsupported", .{head_dim}); - - const cfg: paged_attn.PagedAttentionTiled.Cfg = .{ - .dtype = zml.kernel.metal.from(dtype), - .head_size = head_dim, - .block_size = block_size, - .bq = tile[0], - .tile_kv = tile[1], - .num_threads = tile[2], - .causal = opts.is_causal, - }; - - const total_q_blocks = @divTrunc(total_q_tokens, tile[0]) + num_seqs; - - const out = paged_attn.PagedAttentionTiled.Kernel.call(.{ - .q = q, - .k_cache = k_cache, - .v_cache = v_cache, - .block_tables = block_table, - .context_lens = seq_lens, - .cu_seqlens_q = query_start_len, - }, .{ .out = q.shape() }, .{ - .cfg = cfg, - .grid = .{ @intCast(num_heads), @intCast(total_q_blocks), 1 }, - .threadgroup = .{ @intCast(tile[2]), 1, 1 }, - .threadgroup_memory = paged_attn.PagedAttentionTiled.threadgroupMemory(cfg), - .constants = .{ - .num_kv_heads = .of(num_kv_heads), - .scale = .of(opts.scale), - .softcapping = .of(@as(f32, 0)), - .max_num_blocks_per_seq = .of(max_blocks), - .q_stride = .of(num_heads * head_dim), - .kv_block_stride = .of(block_size * num_kv_heads * head_dim), - .kv_head_stride = .of(head_dim), - .num_seqs = .of(num_seqs), - .sliding_window = .of(opts.sliding_window), - }, - }).out; - - // The tiled kernel leaves the padding rows untouched, so they are zeroed in - // its output. Zeroing a fresh buffer first, from cu_seqlens_q alone, would - // make XLA share that buffer across every call and copy it for each. - return paged_attn.ZeroPaddingRows.Kernel.call(.{ .cu_seqlens_q = query_start_len, .out = out }, .{ .out = q.shape() }, .{ - .cfg = .{ .dtype = cfg.dtype }, - .grid = .{ @intCast(total_q_tokens), 1, 1 }, - .threadgroup = .{ 256, 1, 1 }, - .constants = .{ .num_seqs = .of(num_seqs), .row_stride = .of(num_heads * head_dim) }, - }).out; -} - -const VecLaunch = struct { - heads: i32, - splits: i32, - splits_per_threadgroup: i32, -}; - -/// At most 256 threads, which every Apple GPU pipeline accepts. Splits past a -/// row's length exit early, so `splits` follows the core and row counts. -fn vecLaunch(rows: i32, group: i32, span: i32, cores: i32) VecLaunch { - var heads: i32 = @min(group, 4); - while (@rem(group, heads) != 0) heads -= 1; - const want: i32 = if (rows <= 2 * cores) 128 else 64; - const chunks = @divFloor(span + 31, 32); - var splits: i32 = 1; - while (splits * 2 <= @min(want, chunks)) splits *= 2; - return .{ .heads = heads, .splits = splits, .splits_per_threadgroup = @min(splits, @divFloor(8, heads)) }; -} - -fn coreCount() i32 { - const platform = zml.Compiler.current().platform; - const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse - std.debug.panic("Metal device has no core_count attribute", .{}); - return @intCast(attribute.int64); -} - -test vecLaunch { - try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 128, .splits_per_threadgroup = 2 }, vecLaunch(16, 4, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 2, .splits = 64, .splits_per_threadgroup = 4 }, vecLaunch(256, 2, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 4, .splits = 64, .splits_per_threadgroup = 2 }, vecLaunch(512, 4, 128000, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 3, .splits = 8, .splits_per_threadgroup = 2 }, vecLaunch(48, 6, 288, 40)); - try std.testing.expectEqual(VecLaunch{ .heads = 1, .splits = 1, .splits_per_threadgroup = 1 }, vecLaunch(8, 1, 20, 40)); -} diff --git a/zml/kernels/paged_attention/metal/paged_attn.zig b/zml/kernels/paged_attention/metal/paged_attn.zig deleted file mode 100644 index 04f4c1ac..00000000 --- a/zml/kernels/paged_attention/metal/paged_attn.zig +++ /dev/null @@ -1,908 +0,0 @@ -//! Metal paged attention kernels: fa_vec_paged and its split-K reduce for -//! decode, and paged_attention_tiled with zero_padding_rows otherwise. - -const std = @import("std"); - -const metal = @import("../../../kernel/kernel.zig").metal; - -const ArgSpec = metal.ArgSpec; -const Builder = metal.Builder; -const Type = metal.Type; -const Value = metal.Value; - -const flt_max = std.math.floatMax(f32); -/// A masked score: below any real one yet finite, since fast math assumes -/// no infinities. Halved so subtracting a running maximum cannot overflow. -const masked_score = -flt_max / 2.0; -const max_half = 65504.0; -const simd_width = 32; - -/// fa_vec_paged: bf16 decode over paged caches. Simdgroup s takes query head -/// s % heads of the threadgroup's KV head group and position split s / heads. -pub const FaVecPaged = struct { - pub const Cfg = struct { - /// The key and value head dimension. - head_dim: i32, - /// The page size. - block_size: i32, - /// The in-page position stride, n_kv * head_dim elements. - position_stride: i32, - heads: i32, - splits: i32, - }; - - pub const Args: metal.Struct = .{ .name = "fa_vec_paged_args", .fields = &.{ - .{ .name = "num_heads", .type = .int }, - .{ .name = "n_kv", .type = .int }, - .{ .name = "max_blocks", .type = .int }, - .{ .name = "num_seqs", .type = .int }, - .{ .name = "scale", .type = .float }, - .{ .name = "sliding_window", .type = .int }, - } }; - - pub const HostArgs = extern struct { - num_heads: i32, - n_kv: i32, - max_blocks: i32, - num_seqs: i32, - scale: f32, - /// Below 0 for global attention. - sliding_window: i32, - }; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "fa_vec_paged", - .inputs = &.{ "q", "k_cache", "v_cache", "block_tables", "seq_lens", "cu_seqlens_q" }, - .outputs = &.{"dst"}, - .constants = &.{"args"}, - .run = run, - }); - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - switch (cfg.head_dim) { - 32, 64, 128, 256, 512 => {}, - else => std.debug.panic("fa_vec_paged: head_dim {d} unsupported (32, 64, 128, 256 or 512)", .{cfg.head_dim}), - } - const char_in: ArgSpec = .{ .buffer = .{ .type = .char, .const_ = true } }; - const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; - const a = try b.declareArgs(.{ - .args = .{ .constant_struct = Args }, - .q = char_in, - .k_cache = char_in, - .v_cache = char_in, - .block_tables = int_in, - .seq_lens = int_in, - .cu_seqlens_q = int_in, - .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint3 }, - .tgpg = .{ .threadgroups_per_grid = .uint3 }, - .tiisg = .{ .thread_index_in_simdgroup = .ushort }, - .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, - .nsg = .{ .simdgroups_per_threadgroup = .ushort }, - }); - impl(b, cfg, a); - } - - fn impl(b: *Builder, cfg: Cfg, a: anytype) void { - const D = cfg.head_dim; - const D4 = @divExact(D, 4); - // A K/V row spans NL lanes of NI float4s; a simdgroup reads R rows per step. - const NL = @min(D4, simd_width); - const R = @divExact(simd_width, NL); - const NI = @divExact(D4, NL); - const C = simd_width; - const BS = cfg.block_size; - const PS = cfg.position_stride; - const HG = cfg.heads; - const NWG = cfg.splits; - const full: metal.LoopOpts = .{ .unroll = .full }; - - const num_heads = a.args.field("num_heads"); - const group = num_heads.div(a.args.field("n_kv")); - const groups = group.div(HG); - const r = a.tgpig.at(2).to(.int); - const tg_head = a.tgpig.at(1).to(.int); - const ikv = tg_head.div(groups); - const iq2 = ikv.mul(group).add(tg_head.rem(groups).mul(HG)).add(a.sgitg.rem(HG).to(.int)); - const iwg = a.tgpig.at(0).to(.int).mul(a.nsg.div(HG).to(.int)).add(a.sgitg.div(HG).to(.int)); - const rid = r.to(.long).mul(num_heads.to(.uint)).add(iq2); - const nrows = a.tgpg.at(2).to(.long).mul(num_heads.to(.uint)); - - // Padding rows: FaVecPagedReduce zeroes them when there are splits. - var padding = b.openIf(r.ge(a.cu_seqlens_q.load(a.args.field("num_seqs")))); - { - if (NWG == 1) { - const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)); - var loop = b.openFor(a.tiisg.to(.short), D4, simd_width); - { - dst4.store(loop.iv, b.constant(.float4, 0).to(.bfloat4)); - loop.close(); - } - } - b.return_(); - padding.close(); - } - - const seq = sequenceOf(b, a, r); - const kv_len = seq.kv_len; - const kv_lo = seq.kv_lo; - const ic0_lo = kv_lo.div(C); - const live = if (NWG > 1) liveSplits(b, seq, NWG) else b.lift(@as(i32, 1)); - - if (NWG > 1) { - var idle = b.openIf(iwg.ge(live)); - { - b.return_(); - idle.close(); - } - } - - const pt = a.block_tables.add(seq.idx.to(.ulong).mul(a.args.field("max_blocks").to(.uint))); - const k_cache = a.k_cache.add(ikv.to(.ulong).mul(D).mul(2)); - const v_cache = a.v_cache.add(ikv.to(.ulong).mul(D).mul(2)); - - const lane = if (R == 1) a.tiisg else a.tiisg.rem(NL); - const sub = a.tiisg.div(NL).to(.int); - const q4 = a.q.add(rid.mul(D).mul(2)).to(.bfloat4).add(lane); - const qr = b.array(.float4, @intCast(NI)); - const o = b.array(.float4, @intCast(NI)); - var init = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = init.iv; - qr.store(ii, q4.load(ii.mul(NL)).to(.float4)); - o.store(ii, b.constant(.float4, 0)); - init.close(); - } - - const S = b.local(.float, 0.0); - const M = b.local(.float, -flt_max / 2.0); - - const ic0 = b.local(.int, ic0_lo.add(iwg)); - var blocks = b.openLoop(.{}); - { - const ic = ic0.mul(C); - var past = b.openIf(ic.ge(kv_len)); - { - blocks.breakLoop(); - past.close(); - } - - const s = b.local(.float, 0.0); - var qk = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); - { - const cc = qk.iv; - const p = b.min(if (R == 1) ic.add(cc) else ic.add(cc.to(.int).mul(R)).add(sub), kv_len.sub(1)); - const pk4 = pagedRow(b, k_cache, pt, p, BS, PS).to(.bfloat4).add(lane); - const d = b.local(.float, 0.0); - var dots = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = dots.iv; - d.assign(d.add(b.dot(pk4.load(ii.mul(NL)).to(.float4), qr.load(ii)))); - dots.close(); - } - if (R == 1) { - s.assign(b.ternary(a.tiisg.to(.short).eq(cc), b.simdSum(d.rvalue()), s.rvalue())); - } else { - var x = d.rvalue(); - var mask: i32 = 1; - while (mask < NL) : (mask *= 2) x = x.add(b.simdShuffleXor(x, b.constant(.ushort, mask))); - const t = b.simdShuffle(x, a.tiisg.rem(R).mul(NL)); - s.assign(b.ternary(a.tiisg.div(R).to(.short).eq(cc), t, s.rvalue())); - } - qk.close(); - } - - const pos = ic.add(a.tiisg.to(.int)); - const in_kv = pos.lt(kv_len).bitAnd(pos.ge(kv_lo)); - const score = b.ternary(in_kv, s.mul(a.args.field("scale")), b.constant(.float, -max_half)); - - const m = M.rvalue(); - M.assign(b.simdMax(b.max(M.rvalue(), score))); - const ms = b.exp(m.sub(M)); - const vs = b.exp(score.sub(M)); - S.assign(S.mul(ms).add(b.simdSum(vs))); - - var rescale = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - o.store(rescale.iv, o.load(rescale.iv).mul(ms)); - rescale.close(); - } - - var pv = b.openForOpts(b.constant(.short, 0), @divExact(C, R), 1, full); - { - const cc = pv.iv; - const row = cc.to(.int).mul(R).add(sub); - const p = b.min(if (R == 1) ic.add(cc) else ic.add(row), kv_len.sub(1)); - const pv4 = pagedRow(b, v_cache, pt, p, BS, PS).to(.bfloat4).add(lane); - const w = b.vec(.float4, .{if (R == 1) b.simdBroadcast(vs, cc.to(.ushort)) else b.simdShuffle(vs, row)}); - var dims = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const ii = dims.iv; - o.store(ii, o.load(ii).add(pv4.load(ii.mul(NL)).to(.float4).mul(w))); - dims.close(); - } - pv.close(); - } - - blocks.next(); - ic0.assign(ic0.add(live)); - blocks.close(); - } - - var mask: i32 = NL; - while (mask < simd_width) : (mask *= 2) { - var sum = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - o.store(sum.iv, o.load(sum.iv).add(b.simdShuffleXor(o.load(sum.iv), b.constant(.ushort, mask)))); - sum.close(); - } - } - var first = if (R > 1) b.openIf(sub.eq(0)) else null; - if (NWG == 1) { - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(D4)).add(lane); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - dst4.store(out.iv.mul(NL), o.load(out.iv).mul(inv_sum).to(.bfloat4)); - out.close(); - } - } else { - // [rows][D4][splits] float4 partials, then [rows][splits] (S, M). - const dst4 = a.dst.to(.float4).add(rid.mul(D4).mul(NWG)); - var out = b.openForOpts(b.constant(.short, 0), NI, 1, full); - { - const i = lane.to(.int).add(out.iv.to(.int).mul(NL)); - dst4.store(i.mul(NWG).add(iwg), o.load(out.iv)); - out.close(); - } - var lane0 = b.openIf(a.tiisg.eq(0)); - { - const dst1 = a.dst.to(.float).add(nrows.mul(D).mul(NWG)).add(rid.mul(2 * NWG)); - dst1.store(b.lift(2).mul(iwg), S); - dst1.store(b.lift(2).mul(iwg).add(1), M); - lane0.close(); - } - } - if (first) |*scope| scope.close(); - } - - /// The address of position `p`'s bf16 row in `cache`, through the page - /// table `pt`. - fn pagedRow(b: *Builder, cache: Value, pt: Value, p: Value, BS: i32, PS: i32) Value { - const page = pt.load(p.div(BS)).to(.uint).to(.ulong).mul(b.constant(.uint, BS * PS)); - const row = p.rem(BS).to(.uint).mul(b.constant(.uint, PS)); - return cache.add(page.add(row).mul(2)); - } -}; - -/// The sequence of query row `r` and its causal KV range [kv_lo, kv_len). -fn sequenceOf(b: *Builder, a: anytype, r: Value) struct { idx: Value, kv_len: Value, kv_lo: Value } { - const first_seq = b.local(.int, 0); - const last_seq = b.local(.int, a.args.field("num_seqs").sub(1)); - var search = b.openLoop(.{}); - { - search.body(first_seq.lt(last_seq)); - } - { - const mid = first_seq.add(last_seq).add(1).div(2); - var half = b.openIfElse(a.cu_seqlens_q.load(mid).le(r), .{}); - { - first_seq.assign(mid); - half.elseBranch(); - } - { - last_seq.assign(mid.sub(1)); - half.close(); - } - search.close(); - } - const idx = first_seq.rvalue(); - const q_pos = r.sub(a.cu_seqlens_q.load(idx)); - const query_len = a.cu_seqlens_q.load(idx.add(1)).sub(a.cu_seqlens_q.load(idx)); - const kv_len = a.seq_lens.load(idx).sub(query_len).add(q_pos).add(1); - const kv_lo = b.ternary(a.args.field("sliding_window").lt(0), 0, b.max(0, kv_len.sub(a.args.field("sliding_window")))); - return .{ .idx = idx, .kv_len = kv_len, .kv_lo = kv_lo }; -} - -fn liveSplits(b: *Builder, seq: anytype, splits: i32) Value { - const chunks = seq.kv_len.add(simd_width - 1).div(simd_width).sub(seq.kv_lo.div(simd_width)); - return b.min(splits, chunks); -} - -/// fa_vec_paged_reduce: merges the fa_vec_paged split partials. -pub const FaVecPagedReduce = struct { - pub const Cfg = struct { - splits: i32, - /// The head dimension. - head_dim: i32, - }; - - pub const simdgroups = 8; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "fa_vec_paged_reduce", - .inputs = &.{ "htmp", "seq_lens", "cu_seqlens_q" }, - .outputs = &.{"dst"}, - .constants = &.{"args"}, - .run = run, - }); - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - const int_in: ArgSpec = .{ .buffer = .{ .type = .int, .const_ = true } }; - const a = try b.declareArgs(.{ - .args = .{ .constant_struct = FaVecPaged.Args }, - .htmp = .{ .buffer = .{ .type = .char, .const_ = true } }, - .seq_lens = int_in, - .cu_seqlens_q = int_in, - .dst = .{ .buffer = .{ .type = .char } }, - .tgpig = .{ .threadgroup_position_in_grid = .uint2 }, - .tgpg = .{ .threadgroups_per_grid = .uint2 }, - .tiisg = .{ .thread_index_in_simdgroup = .ushort }, - .sgitg = .{ .simdgroup_index_in_threadgroup = .ushort }, - }); - const NWG = cfg.splits; - if (NWG > 4 * simd_width) std.debug.panic("fa_vec_paged_reduce: {d} splits, at most {d}", .{ NWG, 4 * simd_width }); - const NP = @divFloor(NWG + simd_width - 1, simd_width); - const DV = cfg.head_dim; - const DV4 = @divExact(DV, 4); - const full: metal.LoopOpts = .{ .unroll = .full }; - - const rid = a.tgpig.at(0).to(.ulong); - const r = a.tgpig.at(0).div(a.args.field("num_heads").to(.uint)).to(.int); - const i = a.tgpig.at(1).to(.int).mul(simdgroups).add(a.sgitg.to(.int)); - const dst4 = a.dst.to(.bfloat4).add(rid.mul(DV4)); - - const live = b.local(.int, 0); - var real = b.openIf(r.lt(a.cu_seqlens_q.load(a.args.field("num_seqs")))); - { - live.assign(liveSplits(b, sequenceOf(b, a, r), NWG)); - real.close(); - } - var empty = b.openIf(live.le(0)); - { - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst4.store(i, b.constant(.float4, 0).to(.bfloat4)); - lane0.close(); - } - b.return_(); - empty.close(); - } - - const ss = a.htmp.to(.float).add(a.tgpg.at(0).to(.ulong).mul(DV).mul(NWG)).add(rid.mul(2 * NWG)); - const Mj = b.array(.float, @intCast(NP)); - const Sj = b.array(.float, @intCast(NP)); - const mx = b.local(.float, -flt_max / 2.0); - var load = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = load.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - const ok = idx.lt(live); - Mj.store(j, b.ternary(ok, ss.load(idx.mul(2).add(1)), b.constant(.float, -flt_max / 2.0))); - Sj.store(j, b.ternary(ok, ss.load(idx.mul(2)), b.constant(.float, 0))); - mx.assign(b.max(mx.rvalue(), Mj.load(j))); - load.close(); - } - const m = b.simdMax(mx.rvalue()); - const sum = b.local(.float, 0.0); - var weigh = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = weigh.iv; - Mj.store(j, b.exp(Mj.load(j).sub(m))); - sum.assign(sum.add(Sj.load(j).mul(Mj.load(j)))); - weigh.close(); - } - const S = b.simdSum(sum.rvalue()); - const inv_sum = b.ternary(S.eq(0.0), 0.0, b.lift(1.0).div(S)); - - const htmp4 = a.htmp.to(.float4).add(rid.mul(DV4).mul(NWG)); - const acc = b.local(.float4, b.constant(.float4, 0)); - var parts = b.openForOpts(b.constant(.short, 0), NP, 1, full); - { - const j = parts.iv; - const idx = a.tiisg.to(.int).add(j.to(.int).mul(simd_width)); - var ok = b.openIf(idx.lt(live)); - { - acc.assign(acc.add(htmp4.load(i.mul(NWG).add(idx)).mul(Mj.load(j)))); - ok.close(); - } - parts.close(); - } - const v = b.simdSum(acc.rvalue()); - var lane0 = b.openIf(a.tiisg.eq(0)); - { - dst4.store(i, v.mul(inv_sum).to(.bfloat4)); - lane0.close(); - } - } -}; - -/// zero_padding_rows: zeroes the output rows past the last real token. -pub const ZeroPaddingRows = struct { - pub const Cfg = struct { dtype: Type }; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "zero_padding_rows", - .inputs = &.{ "cu_seqlens_q", "out" }, - .outputs = &.{"out"}, - .constants = &.{ "num_seqs", "row_stride" }, - .run = run, - }); - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - switch (cfg.dtype.scalar) { - .bfloat, .half => {}, - else => std.debug.panic("zero_padding_rows: {s} unsupported (half or bfloat)", .{cfg.dtype.name()}), - } - const a = try b.declareArgs(.{ - .out = .{ .buffer = .{ .type = cfg.dtype } }, - .cu_seqlens_q = .{ .buffer = .{ .type = .int, .const_ = true } }, - .num_seqs = .{ .constant = .int }, - .row_stride = .{ .constant = .int }, - .tgp = .{ .threadgroup_position_in_grid = .uint3 }, - .tpt = .{ .thread_position_in_threadgroup = .uint3 }, - .ntg = .{ .threads_per_threadgroup = .uint3 }, - }); - const row = a.tgp.x().to(.int); - b.returnIf(row.lt(a.cu_seqlens_q.load(a.num_seqs))); - const dst = a.out.add(row.to(.ulong).mul(a.row_stride.to(.ulong))); - var loop = b.openFor(a.tpt.x().to(.int), a.row_stride, a.ntg.x().to(.int)); - { - dst.store(loop.iv, b.constant(cfg.dtype, 0)); - loop.close(); - } - } -}; - -/// paged_attention_tiled: FlashAttention-2 over 8x8 simdgroup matrices, causal -/// or bidirectional, one threadgroup per (query head, query block). -pub const PagedAttentionTiled = struct { - pub const Cfg = struct { - /// half or bfloat. - dtype: Type, - head_size: i32, - block_size: i32, - bq: i32 = 32, - tile_kv: i32 = 32, - num_threads: i32 = 128, - /// Bidirectional when false. - causal: bool = true, - }; - - pub const Kernel = metal.Kernel(Cfg, .{ - .name = "paged_attention_tiled", - .inputs = &.{ "q", "k_cache", "v_cache", "block_tables", "context_lens", "cu_seqlens_q" }, - .outputs = &.{"out"}, - .constants = &.{ "num_kv_heads", "scale", "softcapping", "max_num_blocks_per_seq", "q_stride", "kv_block_stride", "kv_head_stride", "num_seqs", "sliding_window" }, - .run = run, - }); - - /// What the MSL instantiates: half or bfloat, a head size of tileConfig - /// with its tiles, and pages of 8, 16 or 32 positions. - fn supported(cfg: Cfg) bool { - const tile = tileConfig(cfg.head_size) orelse return false; - return (cfg.dtype.eql(.half) or cfg.dtype.eql(.bfloat)) and - (cfg.block_size == 8 or cfg.block_size == 16 or cfg.block_size == 32) and - cfg.bq == tile[0] and cfg.tile_kv == tile[1] and cfg.num_threads == tile[2]; - } - - /// The (bq, tile_kv, num_threads) of a head size, null when unsupported. - pub fn tileConfig(head_size: i64) ?[3]i32 { - return switch (head_size) { - 32, 64, 96, 128 => .{ 32, 32, 128 }, - 256 => .{ 16, 16, 64 }, - 512 => .{ 8, 8, 32 }, - else => null, - }; - } - - pub fn threadgroupMemory(cfg: Cfg) u64 { - const ld = cfg.head_size + @divExact(16, @as(i32, @intCast(cfg.dtype.size()))); - return @intCast((cfg.bq + 2 * cfg.tile_kv) * ld * @as(i32, @intCast(cfg.dtype.size()))); - } - - /// Lane `lane_id`'s (column, row) in an 8x8 simdgroup_matrix fragment. - fn fragCoord(b: *Builder, lane_id: Value) Value { - const qid = lane_id.to(.short).div(4).to(.short); - const fm = qid.bitAnd(4).add(lane_id.to(.short).div(2).rem(4)).to(.short); - const fn_ = qid.bitAnd(2).mul(2).add(lane_id.to(.short).rem(2).mul(2)).to(.short); - return b.vec(.short2, .{ fn_, fm }); - } - - /// Reduces `v` with `op` across the lanes that hold one row of an 8x8 - /// simdgroup_matrix fragment. - fn fragRowReduce(b: *Builder, comptime op: enum { max, sum }, v: Value) Value { - var x = v.rvalue(); - inline for (.{ 1, 8 }) |mask| { - const y = b.simdShuffleXor(x, b.constant(.ushort, mask)); - x = switch (op) { - .max => b.max(x, y), - .sum => x.add(y), - }; - } - return x; - } - - /// Sets lane `lane` of the vector `p[i]` to `x`. - fn setLane(p: Value, i: Value, lane: anytype, x: Value) void { - p.store(i, p.load(i).insert(lane, x)); - } - - fn run(b: *Builder, cfg: Cfg) metal.FinishError!void { - if (!supported(cfg)) std.debug.panic("paged_attention_tiled: {s} at head size {d}, block size {d} (8, 16 or 32) and tiles {d}, {d}, {d} unsupported", .{ cfg.dtype.name(), cfg.head_size, cfg.block_size, cfg.bq, cfg.tile_kv, cfg.num_threads }); - const T = cfg.dtype; - const HEAD_SIZE = cfg.head_size; - const BLOCK_SIZE = cfg.block_size; - const BQ = cfg.bq; - const TILE_KV = cfg.tile_kv; - const NUM_THREADS = cfg.num_threads; - const int_arg: ArgSpec = .{ .constant = .int }; - const float_arg: ArgSpec = .{ .constant = .float }; - const a = try b.declareArgs(.{ - .out = .{ .buffer = .{ .type = T } }, - .q = .{ .buffer = .{ .type = T, .const_ = true } }, - .k_cache = .{ .buffer = .{ .type = T, .const_ = true } }, - .v_cache = .{ .buffer = .{ .type = T, .const_ = true } }, - .num_kv_heads = int_arg, - .scale = float_arg, - .softcapping = float_arg, - .block_tables = .{ .buffer = .{ .type = .uint, .const_ = true } }, - .context_lens = .{ .buffer = .{ .type = .uint, .const_ = true } }, - .max_num_blocks_per_seq = int_arg, - .q_stride = int_arg, - .kv_block_stride = int_arg, - .kv_head_stride = int_arg, - .cu_seqlens_q = .{ .buffer = .{ .type = .int, .const_ = true } }, - .num_seqs = int_arg, - .sliding_window = int_arg, - .shared_mem = .{ .threadgroup = .char }, - .tgp = .{ .threadgroup_position_in_grid = .uint3 }, - .tgpg = .{ .threadgroups_per_grid = .uint3 }, - .tpt = .{ .thread_position_in_threadgroup = .uint3 }, - .sg_idx = .{ .simdgroup_index_in_threadgroup = .uint }, - .lane = .{ .thread_index_in_simdgroup = .uint }, - }); - - const NUM_SIMD_LANES = 32; - const NUM_SG = @divExact(NUM_THREADS, NUM_SIMD_LANES); - const TD = @divExact(HEAD_SIZE, 8); - const TK = @divExact(TILE_KV, 8); - if (@divExact(BQ, NUM_SG) != 8) @panic("paged_attention_tiled: ROWS_PER_SG must be 8"); - const unroll: metal.LoopOpts = .{ .unroll = .enable }; - - const thread_idx = a.tpt.x().to(.int); - const head_idx = a.tgp.x().to(.int); - const q_block_global_idx = a.tgp.y().to(.int); - const num_heads = a.tgpg.x().to(.int); - const num_queries_per_kv = num_heads.div(a.num_kv_heads); - const kv_head_idx = head_idx.div(num_queries_per_kv); - const kv_token_stride = a.num_kv_heads.mul(a.kv_head_stride); - - const first_seq = b.local(.int, 0); - const last_seq = b.local(.int, a.num_seqs); - var search = b.openLoop(.{}); - { - search.body(first_seq.lt(last_seq)); - } - { - const mid = first_seq.add(last_seq).add(1).div(2); - var half = b.openIfElse(a.cu_seqlens_q.load(mid).div(BQ).add(mid).le(q_block_global_idx), .{}); - { - first_seq.assign(mid); - half.elseBranch(); - } - { - last_seq.assign(mid.sub(1)); - half.close(); - } - search.close(); - } - // The grid is an upper bound on the query blocks: the extra ones land past - // the last sequence and clamp to it. - const seq_idx = b.min(first_seq.rvalue(), a.num_seqs.sub(1)); - - const q_seq_start = a.cu_seqlens_q.load(seq_idx); - const cur_batch_query_len = a.cu_seqlens_q.load(seq_idx.add(1)).sub(q_seq_start); - const q_block_start = q_seq_start.div(BQ).add(seq_idx); - const q_block_local = q_block_global_idx.sub(q_block_start); - const q_pos_start = q_block_local.mul(BQ); - - b.returnIf(q_pos_start.ge(cur_batch_query_len)); - - const seq_len = a.context_lens.load(seq_idx).to(.int); - const context_len = seq_len.sub(cur_batch_query_len); - const valid_q = b.min(BQ, cur_batch_query_len.sub(q_pos_start)); - - const SMEM_PAD = @divExact(16, @as(i32, @intCast(T.size()))); - const LD = HEAD_SIZE + SMEM_PAD; - const Q_ELEMS = BQ * LD; - const KV_ELEMS = TILE_KV * LD; - - const Q_smem = a.shared_mem.to(T); - const K_smem = Q_smem.add(Q_ELEMS); - const V_smem = K_smem.add(KV_ELEMS); - - const log2e: f32 = std.math.log2e; - const scale_log2 = a.scale.mul(log2e); - - const q_base = a.q.add(q_seq_start.add(q_pos_start).mul(a.q_stride)).add(head_idx.mul(HEAD_SIZE)); - var load_q = b.openFor(thread_idx, BQ * HEAD_SIZE, NUM_THREADS); - { - const i = load_q.iv; - const r = i.div(HEAD_SIZE); - const d = i.rem(HEAD_SIZE); - var value = b.openIfElse(r.lt(valid_q), .{T}); - { - value.yieldThen(.{q_base.load(r.mul(a.q_stride).add(d))}); - } - { - value.yieldElse(.{b.constant(T, 0)}); - } - Q_smem.store(r.mul(LD).add(d), value.results[0]); - load_q.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const T2: Type = .vec(T.scalar, 2); - const q_reg = b.array(T2, @intCast(TD)); - var load_q_reg = b.openForOpts(0, TD, 1, unroll); - { - const d = load_q_reg.iv; - const tmp = b.simdgroupLoad(Q_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - q_reg.store(d, tmp.threadElements()); - load_q_reg.close(); - } - - const fc = fragCoord(b, a.lane.to(.ushort)); - const fm = fc.y(); - const fn_ = fc.x(); - - const max_score = b.local(.float, masked_score); - const sum_score = b.local(.float, 0.0); - - const o_reg = b.array(.float2, @intCast(TD)); - var zero_o = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(zero_o.iv, b.constant(.float2, 0)); - zero_o.close(); - } - - const block_table = a.block_tables.add(seq_idx.mul(a.max_num_blocks_per_seq)); - const num_kv_tiles = seq_len.add(TILE_KV).sub(1).div(TILE_KV); - - var tiles = b.openFor(0, num_kv_tiles, 1); - { - const tile_idx = tiles.iv; - const tile_start = tile_idx.mul(TILE_KV); - - if (cfg.causal) { - var past = b.openIf(tile_start.gt(context_len.add(q_pos_start).add(valid_q).sub(1))); - { - tiles.breakLoop(); - past.close(); - } - } - - const VEC_BYTES: i32 = if (HEAD_SIZE >= 256) 16 else 8; - const VEC = @divExact(VEC_BYTES, @as(i32, @intCast(T.size()))); - const LoadUnit: Type = if (VEC_BYTES == 16) .uint4 else .uint2; - std.debug.assert(@rem(LD, VEC) == 0); - var rows = b.openForOpts(0, @divExact(TILE_KV, NUM_SG), 1, unroll); - { - const t = a.sg_idx.mul(@divExact(TILE_KV, NUM_SG)).add(rows.iv).to(.int); - const kv_pos = tile_start.add(t); - var present = b.openIfElse(kv_pos.lt(seq_len), .{}); - { - const pb = block_table.load(kv_pos.div(BLOCK_SIZE)).to(.long); - const off = pb.mul(a.kv_block_stride).add(kv_pos.rem(BLOCK_SIZE).mul(kv_token_stride)).add(kv_head_idx.mul(a.kv_head_stride)); - const k_ptr = a.k_cache.add(off); - const v_ptr = a.v_cache.add(off); - var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); - { - const d = loop.iv; - const k = k_ptr.add(d).to(LoadUnit).load(0); - K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, k); - const v = v_ptr.add(d).to(LoadUnit).load(0); - V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, v); - loop.close(); - } - present.elseBranch(); - } - { - const zero = b.constant(LoadUnit, 0); - var loop = b.openForOpts(a.lane.mul(VEC).to(.int), HEAD_SIZE, NUM_SIMD_LANES * VEC, unroll); - { - const d = loop.iv; - K_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); - V_smem.add(t.mul(LD).add(d)).to(LoadUnit).store(0, zero); - loop.close(); - } - present.close(); - } - rows.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const s_reg = b.array(.float2, @intCast(TK)); - var zero_s = b.openForOpts(0, TK, 1, unroll); - { - s_reg.store(zero_s.iv, b.constant(.float2, 0)); - zero_s.close(); - } - - var qk_dims = b.openForOpts(0, TD, 1, unroll); - { - const d = qk_dims.iv; - const q_frag = b.matrix(T.scalar, q_reg.load(d)); - var qk_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = qk_keys.iv; - const k_frag = b.simdgroupLoad(K_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{ .transpose = true }); - const s_frag = b.matrix(.float, s_reg.load(k)); - s_reg.store(k, b.simdgroupMultiplyAccumulate(q_frag, k_frag, s_frag).threadElements()); - qk_keys.close(); - } - qk_dims.close(); - } - - const q_abs_pos = context_len.add(q_pos_start).add(a.sg_idx.mul(8)).add(fm).to(.int); - const row_masked = a.sg_idx.mul(8).add(fm).ge(valid_q); - const min_q_abs_pos = context_len.add(q_pos_start); - var tile_no_mask = tile_start.add(TILE_KV).le(seq_len).bitAnd(a.softcapping.le(0.0)).bitAnd(a.sliding_window.lt(0)); - if (cfg.causal) tile_no_mask = tile_no_mask.bitAnd(tile_start.add(TILE_KV).sub(1).lt(min_q_abs_pos)); - - var masking = b.openIfElse(tile_no_mask.bitAnd(row_masked.eq(false)), .{}); - { - var scale = b.openForOpts(0, TK, 1, unroll); - { - const k = scale.iv; - setLane(s_reg, k, 0, s_reg.load(k).at(0).mul(scale_log2)); - setLane(s_reg, k, 1, s_reg.load(k).at(1).mul(scale_log2)); - scale.close(); - } - masking.elseBranch(); - } - { - var keys = b.openForOpts(0, TK, 1, unroll); - { - const k = keys.iv; - var pair = b.openForOpts(0, 2, 1, unroll); - { - const jj = pair.iv; - const s = b.local(.float, s_reg.load(k).at(jj).mul(scale_log2)); - var capped = b.openIf(a.softcapping.gt(0.0)); - { - const s_orig = s.div(log2e); - s.assign(a.softcapping.mul(b.precise().tanh(s_orig.div(a.softcapping))).mul(log2e)); - capped.close(); - } - const kv_pos = tile_start.add(k.mul(8)).add(fn_).add(jj); - var masked = row_masked.bitOr(kv_pos.ge(seq_len)); - if (cfg.causal) masked = masked.bitOr(kv_pos.gt(q_abs_pos)); - const before_window = a.sliding_window.ge(0).bitAnd(kv_pos.lt(q_abs_pos.add(1).sub(a.sliding_window))); - masked = masked.bitOr(before_window); - setLane(s_reg, k, jj, b.ternary(masked, masked_score, s)); - pair.close(); - } - keys.close(); - } - masking.close(); - } - - const local_max = b.local(.float, masked_score); - var max_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = max_keys.iv; - local_max.assign(b.max(local_max.rvalue(), b.max(s_reg.load(k).at(0), s_reg.load(k).at(1)))); - max_keys.close(); - } - const row_max = fragRowReduce(b, .max, local_max); - - const prev_max = max_score.rvalue(); - const grown_max = b.max(prev_max, row_max); - const grew = grown_max.gt(prev_max); - const was_empty = prev_max.eq(masked_score); - const factor = b.ternary(grew, b.ternary(was_empty, 0.0, b.fast().exp2(prev_max.sub(grown_max))), 1.0); - const new_max = b.ternary(grew.eq(false).bitAnd(was_empty), 0.0, grown_max); - // A tile masking a whole row leaves the maximum masked: zero would - // pin it, and later scores far below zero would underflow. - max_score.assign(grown_max); - - const local_sum = b.local(.float, 0.0); - var p_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = p_keys.iv; - var pair = b.openForOpts(0, 2, 1, unroll); - { - const jj = pair.iv; - const s = s_reg.load(k).at(jj); - const p = b.ternary(s.eq(masked_score), 0.0, b.fast().exp2(s.sub(new_max))); - setLane(s_reg, k, jj, p); - local_sum.assign(local_sum.add(p)); - pair.close(); - } - p_keys.close(); - } - const row_sum = fragRowReduce(b, .sum, local_sum); - - sum_score.assign(sum_score.mul(factor).add(row_sum)); - - var rescale_o = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(rescale_o.iv, o_reg.load(rescale_o.iv).mul(factor)); - rescale_o.close(); - } - - var pv_keys = b.openForOpts(0, TK, 1, unroll); - { - const k = pv_keys.iv; - const p_frag = b.matrix(.float, s_reg.load(k)); - var pv_dims = b.openForOpts(0, TD, 1, unroll); - { - const d = pv_dims.iv; - const v_frag = b.simdgroupLoad(V_smem.add(k.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - const o_frag = b.matrix(.float, o_reg.load(d)); - o_reg.store(d, b.simdgroupMultiplyAccumulate(p_frag, v_frag, o_frag).threadElements()); - pv_dims.close(); - } - pv_keys.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - tiles.close(); - } - - const inv_sum = b.lift(1.0).div(sum_score.add(@as(f32, 1e-6))); - var normalize = b.openForOpts(0, TD, 1, unroll); - { - o_reg.store(normalize.iv, o_reg.load(normalize.iv).mul(inv_sum)); - normalize.close(); - } - - const O_smem = Q_smem.to(.float); - var store_o = b.openForOpts(0, TD, 1, unroll); - { - const d = store_o.iv; - const o_frag = b.matrix(.float, o_reg.load(d)); - b.simdgroupStore(o_frag, O_smem.add(a.sg_idx.mul(8).mul(LD)).add(d.mul(8)), LD, .{}); - store_o.close(); - } - - b.threadgroupBarrier(.mem_threadgroup); - - const out_base = a.out.add(q_seq_start.add(q_pos_start).mul(a.q_stride)).add(head_idx.mul(HEAD_SIZE)); - const total = valid_q.mul(HEAD_SIZE); - var write_out = b.openFor(thread_idx, total, NUM_THREADS); - { - const i = write_out.iv; - const r = i.div(HEAD_SIZE); - const d = i.rem(HEAD_SIZE); - const value = O_smem.load(r.mul(LD).add(d)).to(T); - out_base.store(r.mul(a.q_stride).add(d), value); - write_out.close(); - } - } -}; - -test "the paged attention kernels emit verified modules" { - const allocator = std.testing.allocator; - for ([_]i32{ 32, 64, 128, 256, 512 }) |head_dim| { - for ([_][2]i32{ .{ 1, 1 }, .{ 4, 1 }, .{ 2, 8 }, .{ 4, 128 } }) |launch| { - allocator.free(try FaVecPaged.Kernel.emit(allocator, .{ .head_dim = head_dim, .block_size = 16, .position_stride = 2 * head_dim, .heads = launch[0], .splits = launch[1] })); - } - for ([_]i32{ 8, 128 }) |splits| { - allocator.free(try FaVecPagedReduce.Kernel.emit(allocator, .{ .splits = splits, .head_dim = head_dim })); - } - } - for ([_]Type{ .half, .bfloat }) |dtype| { - allocator.free(try ZeroPaddingRows.Kernel.emit(allocator, .{ .dtype = dtype })); - for ([_]i32{ 32, 64, 96, 128, 256, 512 }) |head_size| { - const tile = PagedAttentionTiled.tileConfig(head_size).?; - for ([_]bool{ true, false }) |causal| { - allocator.free(try PagedAttentionTiled.Kernel.emit(allocator, .{ .dtype = dtype, .head_size = head_size, .block_size = 16, .bq = tile[0], .tile_kv = tile[1], .num_threads = tile[2], .causal = causal })); - } - } - } -} diff --git a/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel b/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel deleted file mode 100644 index 6a0e4e87..00000000 --- a/zml/kernels/paged_attention/mosaic_tpu/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "mosaic_tpu", - srcs = [ - "mosaic_tpu.zig", - "ragged_attention.zig", - ], -) diff --git a/zml/kernels/paged_attention/mosaic_tpu/README.md b/zml/kernels/paged_attention/mosaic_tpu/README.md deleted file mode 100644 index 6fff585c..00000000 --- a/zml/kernels/paged_attention/mosaic_tpu/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Mosaic TPU paged attention - -This backend is automatically selected for TPU. See the -[parent README](../README.md) for the attention equation and packed sequence -metadata contract. - -[The wrapper](mosaic_tpu.zig) requires fused `KvCache.dense` storage with shape -`{ .page = N, .k_chunk = P, .hkv = 2 * Hkv, .hd = Dpad }`. The combined head -axis interleaves `K0, V0, K1, V1, ...`. `Dpad` must equal the logical query head -dimension rounded up to a multiple of 128, with unused cache features zeroed. -Queries are padded before execution and outputs trimmed afterward. The default -scale uses the logical head dimension. Accepted query/cache dtypes are `bf16`, -`f16`, and `f32`. - -Causal and non-causal standard masks and left sliding windows are supported. -Explicit key indices are rejected. `opts.sink` is currently ignored. Split -caches are rejected by ordinary paged attention, and the generic -`KvCache.update` method does not implement fused-cache writes. - -The wrapper partitions heads on the model mesh and replicates sequence metadata. -It counts sequences with nonempty query intervals and passes that count to the -ragged kernel, so active sequences must precede trailing inactive entries. -Each local head computation must have the full required key range and features. - -The shared split-cache comparison test does not exercise this backend. Wrapper -unit tests cover head-dimension alignment and configuration, including scale -and standard mask options; they do not establish runtime support for every -shape or dtype. diff --git a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig b/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig deleted file mode 100644 index 84716379..00000000 --- a/zml/kernels/paged_attention/mosaic_tpu/mosaic_tpu.zig +++ /dev/null @@ -1,275 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const ragged_paged = @import("platforms/tpu/ragged_paged"); -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; -const ragged_attention = @import("ragged_attention.zig"); - -const log = std.log.scoped(.tpu_attention); - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(ragged_attention); -} - -pub const mosaic_tpu = struct { - const PreparedInputs = struct { - q: zml.Tensor, - kv_pages: zml.Tensor, - seq_lens: zml.Tensor, - block_table: zml.Tensor, - query_start_len: zml.Tensor, - num_seqs: zml.Tensor, - }; - - pub fn alignedKernelHeadDim(head_dim: usize) usize { - stdx.debug.assert(head_dim > 0, "mosaic_tpu ragged paged attention head_dim must be non-zero", .{}); - const alignment: usize = 128; - return (std.math.divCeil(usize, head_dim, alignment) catch unreachable) * alignment; - } - - pub const Options = struct { - is_prefill: bool, - batch_size: usize, - max_num_pages: usize, - max_seqlen_k: usize, - max_token_count: usize, - num_heads: usize, - num_kv_heads: usize, - head_dim: usize, - pages_per_compute_block: usize = 1, - - pub fn isPrefill(self: Options) bool { - return self.is_prefill; - } - - pub fn maxNumPages(self: Options) usize { - return self.max_num_pages; - } - }; - - pub const Parameters = struct { - opts: Options, - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - - pub fn init(opts: Options) Parameters { - return .{ - .opts = opts, - .block_table = zml.Tensor.init(.{ .b = opts.batch_size, .p = opts.max_num_pages }, .i32), - .seq_lens = zml.Tensor.init(.{ .b = opts.batch_size }, .i32), - .query_start_len = zml.Tensor.init(.{ .b = opts.batch_size + 1 }, .i32), - }; - } - - pub fn options(self: Parameters) Options { - return self.opts; - } - - pub fn allocationSize(self: Parameters) usize { - var allocation_size: usize = 0; - allocation_size += self.block_table.byteSize(); - allocation_size += self.seq_lens.byteSize(); - allocation_size += self.query_start_len.byteSize(); - return allocation_size; - } - }; - - fn raggedPagedKernelCall( - q: zml.Tensor, - kv_pages: zml.Tensor, - seq_lens: zml.Tensor, - block_table: zml.Tensor, - query_start_len: zml.Tensor, - num_seqs: zml.Tensor, - cfg: ragged_paged.Cfg, - ) zml.Tensor { - const mlir_ctx = zml.Compiler.current().mlir_ctx; - const out = ragged_attention.Kernel.call( - .{ - .kv_lens = seq_lens, - .page_indices = block_table, - .cu_q_lens = query_start_len, - .seq_buf_idx = .zeroes(.init(.{2}, .i32)), - .num_seqs = num_seqs, - .q = q, - .kv_pages = kv_pages, - }, - .{ .o = q.shape() }, - .{ - .cfg = cfg, - .extras = .{ - .vmem_limit_bytes = cfg.vmem_limit_bytes, - .additional_attributes = &.{ - .named(mlir_ctx, "kernel_name", .string(mlir_ctx, "ragged_paged_attention_kernel")), - .named(mlir_ctx, "mhlo.frontend_attributes", .dict(mlir_ctx, &.{ - .named(mlir_ctx, "kernel_metadata", .string(mlir_ctx, "{}")), - })), - }, - }, - }, - ); - return out.o; - } - - fn activeSequenceCount(query_start_len: zml.Tensor) zml.Tensor { - const start = query_start_len.slice(.b, .{ .end = query_start_len.dim(.b) - 1 }); - const end = query_start_len.slice(.b, .{ .start = 1 }); - const query_lens = end.sub(start); - return query_lens - .cmp(.GT, .zeroes(query_lens.shape())) - .convert(.i32) - .sum(.b) - .reshape(.{1}); - } - - fn cfgDtype(dtype: zml.DataType) @FieldType(ragged_paged.Cfg, "q_dtype") { - const cast = zml.kernel.mosaic_tpu.from(dtype); - return switch (cast) { - .bf16, .f16, .f32 => cast, - else => stdx.debug.panic("mosaic_tpu ragged paged attention expects q/kv dtype bf16/f16/f32, got {}", .{dtype}), - }; - } - - inline fn restoreQueryHeads(q_template: zml.Tensor, q_out: zml.Tensor) zml.Tensor { - const restored = q_out.splitAxis(.h, .{ .hkv = q_template.dim(.hkv), .hg = q_template.dim(.hg) }); - if (restored.dim(.hd) == q_template.dim(.hd)) return restored; - stdx.debug.assert( - restored.dim(.hd) > q_template.dim(.hd), - "mosaic_tpu ragged paged attention cannot restore output from head_dim {} to {}", - .{ restored.dim(.hd), q_template.dim(.hd) }, - ); - return restored.slice(.hd, .{ .end = q_template.dim(.hd) }); - } - - inline fn alignQueryHeadDimForKernel(q: zml.Tensor, target_head_dim: i64) zml.Tensor { - if (q.dim(.hd) == target_head_dim) return q; - stdx.debug.assert( - q.dim(.hd) < target_head_dim, - "mosaic_tpu ragged paged attention q head_dim {} exceeds kernel/cache head_dim {}", - .{ q.dim(.hd), target_head_dim }, - ); - return q.pad(0, .{ .hd = zml.Tensor.Pad{ .high = target_head_dim - q.dim(.hd) } }); - } - - inline fn prepareInputs(parameters: Parameters, q: zml.Tensor, kv_pages: zml.Tensor) PreparedInputs { - const q_merged = q.merge(.{ .h = .{ .hkv, .hg } }); - const logical_head_dim: usize = @intCast(q_merged.dim(.hd)); - const cache_head_dim: usize = @intCast(kv_pages.dim(.hd)); - stdx.debug.assert( - cache_head_dim == alignedKernelHeadDim(logical_head_dim), - "mosaic_tpu ragged paged attention expects KV cache head_dim {} to match aligned logical head_dim {}", - .{ cache_head_dim, logical_head_dim }, - ); - const q_ragged = alignQueryHeadDimForKernel(q_merged, kv_pages.dim(.hd)); - const query_start_len = parameters.query_start_len; - - return .{ - .q = q_ragged, - .kv_pages = kv_pages, - .seq_lens = parameters.seq_lens, - .block_table = parameters.block_table, - .query_start_len = query_start_len, - .num_seqs = activeSequenceCount(query_start_len), - }; - } - - fn buildCfg(prepared_inputs: PreparedInputs, parameters: Parameters, opts: AttentionOptions) ragged_paged.Cfg { - const q_shape = prepared_inputs.q.shape(); - const kv_pages_shape = prepared_inputs.kv_pages.shape(); - const block_table_shape = prepared_inputs.block_table.shape(); - - const q_token_count = if (q_shape.hasTag(.q) != null) q_shape.dim(.q) else q_shape.dim(.b); - const max_num_seqs = block_table_shape.dim(.b); - const num_q_heads = q_shape.dim(.h); - stdx.debug.assert( - q_shape.dim(.hd) == kv_pages_shape.dim(.hd), - "mosaic_tpu ragged paged attention expects q and KV cache head_dim to match, got {} and {}", - .{ q_shape.dim(.hd), kv_pages_shape.dim(.hd) }, - ); - stdx.debug.assert(@mod(kv_pages_shape.dim(.hkv), 2) == 0, "Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); - const num_kv_heads = std.math.divExact(i64, kv_pages_shape.dim(.hkv), 2) catch - stdx.debug.panic("Expected fused KV pages .hkv dimension to be even, got {}", .{kv_pages_shape.dim(.hkv)}); - const num_queries_per_block = @max(@as(i64, 1), @min(q_token_count, max_num_seqs)); - const logical_head_dim: f64 = @floatFromInt(parameters.opts.head_dim); - - const sliding_window: ?i64 = if (opts.mask.slidingWindow()) |window| @intCast(window) else null; - - return .{ - .num_q_tokens = q_token_count, - .num_q_heads = num_q_heads, - .num_kv_heads = num_kv_heads, - .head_dim = q_shape.dim(.hd), - .total_num_pages = kv_pages_shape.dim(.page), - .page_size = kv_pages_shape.dim(.k_chunk), - .max_num_seqs = max_num_seqs, - .pages_per_seq = @intCast(parameters.opts.max_num_pages), - .num_kv_pages_per_block = @intCast(parameters.opts.pages_per_compute_block), - .num_queries_per_block = num_queries_per_block, - .q_dtype = cfgDtype(q_shape.dtype()), - .kv_dtype = cfgDtype(kv_pages_shape.dtype()), - .sm_scale = opts.scale orelse @floatCast(1.0 / @sqrt(logical_head_dim)), - .is_causal = opts.mask.isCausal(), - .sliding_window = sliding_window, - }; - } - - pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, kv_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { - if (opts.mask == .indices) @panic("Sparse Attention is not supported on TPU"); - const prepared = prepareInputs(parameters, q, kv_cache); - const cfg = buildCfg(prepared, parameters, opts); - const q_out = raggedPagedKernelCall(prepared.q, prepared.kv_pages, prepared.seq_lens, prepared.block_table, prepared.query_start_len, prepared.num_seqs, cfg); - const restored = restoreQueryHeads(q, q_out); - stdx.debug.assert(restored.shape().eql(q.shape()), "mosaic_tpu ragged paged attention output shape mismatch, got {f}, expected {f}", .{ restored.shape(), q.shape() }); - return restored; - } -}; - -test "mosaic_tpu aligns kernel head dimensions to 128" { - try std.testing.expectEqual(@as(usize, 128), mosaic_tpu.alignedKernelHeadDim(64)); - try std.testing.expectEqual(@as(usize, 128), mosaic_tpu.alignedKernelHeadDim(128)); - try std.testing.expectEqual(@as(usize, 256), mosaic_tpu.alignedKernelHeadDim(180)); - try std.testing.expectEqual(@as(usize, 256), mosaic_tpu.alignedKernelHeadDim(256)); -} - -test "mosaic_tpu cfg uses kernel head dim and logical scale" { - const parameters = mosaic_tpu.Parameters.init(.{ - .is_prefill = false, - .batch_size = 1, - .max_num_pages = 2, - .max_seqlen_k = 32, - .max_token_count = 1, - .num_heads = 8, - .num_kv_heads = 2, - .head_dim = 64, - }); - const prepared_inputs: mosaic_tpu.PreparedInputs = .{ - .q = zml.Tensor.init(.{ .b = 1, .h = 8, .hd = 128 }, .bf16), - .kv_pages = zml.Tensor.init(.{ .page = 2, .k_chunk = 16, .hkv = 4, .hd = 128 }, .bf16), - .seq_lens = parameters.seq_lens, - .block_table = parameters.block_table, - .query_start_len = parameters.query_start_len, - .num_seqs = zml.Tensor.init(.{1}, .i32), - }; - - const cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{}); - try std.testing.expectEqual(@as(i64, 128), cfg.head_dim); - try std.testing.expect(cfg.is_causal); - try std.testing.expectApproxEqAbs( - @as(f32, 1.0 / @sqrt(@as(f32, 64.0))), - cfg.sm_scale, - 0.000001, - ); - - const non_causal_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .{ .standard = .{ .causal = false } } }); - try std.testing.expect(!non_causal_cfg.is_causal); - try std.testing.expectEqual(@as(?i64, null), non_causal_cfg.sliding_window); - - const window_cfg = mosaic_tpu.buildCfg(prepared_inputs, parameters, .{ .mask = .{ .standard = .{ .sliding_window = 16 } } }); - try std.testing.expect(window_cfg.is_causal); - try std.testing.expectEqual(@as(?i64, 16), window_cfg.sliding_window); -} diff --git a/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig b/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig deleted file mode 100644 index 30536306..00000000 --- a/zml/kernels/paged_attention/mosaic_tpu/ragged_attention.zig +++ /dev/null @@ -1,14 +0,0 @@ -const ragged_paged = @import("platforms/tpu/ragged_paged"); - -const zml = @import("../../../zml.zig"); - -test { - _ = Kernel; -} - -pub const Kernel = zml.kernel.mosaic_tpu.Kernel(ragged_paged.Cfg, .{ - .name = "ragged_paged_attention_kernel", - .inputs = &.{ "kv_lens", "page_indices", "cu_q_lens", "seq_buf_idx", "num_seqs", "q", "kv_pages" }, - .outputs = &.{"o"}, - .run = ragged_paged.run, -}); diff --git a/zml/kernels/paged_attention/options.zig b/zml/kernels/paged_attention/options.zig deleted file mode 100644 index 35ad4049..00000000 --- a/zml/kernels/paged_attention/options.zig +++ /dev/null @@ -1,36 +0,0 @@ -const Tensor = @import("../../tensor.zig").Tensor; - -pub const AttentionMask = union(enum) { - standard: struct { - /// Restrict attention to keys at or before the query position. - causal: bool = true, - /// Optional left window size, including the current token. - /// Non-causal attention also includes every later valid key. - sliding_window: ?u32 = null, - }, - /// Attend to keys at specific positions. - indices: Tensor, - - /// Attend to every valid key in the sequence. - pub const none: AttentionMask = .{ .standard = .{ .causal = false } }; - - pub fn isCausal(mask: AttentionMask) bool { - return switch (mask) { - .standard => |options| options.causal, - .indices => false, - }; - } - - pub fn slidingWindow(mask: AttentionMask) ?u32 { - return switch (mask) { - .standard => |options| options.sliding_window, - .indices => null, - }; - } -}; - -pub const AttentionOptions = struct { - mask: AttentionMask = .{ .standard = .{} }, - scale: ?f32 = null, - sink: ?Tensor = null, -}; diff --git a/zml/kernels/paged_attention/paged_attention.zig b/zml/kernels/paged_attention/paged_attention.zig deleted file mode 100644 index 4a97a38e..00000000 --- a/zml/kernels/paged_attention/paged_attention.zig +++ /dev/null @@ -1,502 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../../zml.zig"); -pub const AttentionMask = @import("options.zig").AttentionMask; -pub const AttentionOptions = @import("options.zig").AttentionOptions; -pub const cute = @import("cute/cute.zig"); -const metal = @import("metal/metal.zig"); -const tpu = @import("mosaic_tpu/mosaic_tpu.zig"); -const stablehlo = @import("stablehlo/stablehlo.zig"); -pub const tcl = @import("tcl/tcl.zig"); -const triton = @import("triton/triton.zig"); - -const PagedAttention = @This(); - -test { - _ = tcl; - std.testing.refAllDecls(Backend); - std.testing.refAllDecls(Options); - std.testing.refAllDecls(Parameters); - std.testing.refAllDecls(KvCache); -} - -pub const Backend = enum { - cute, - triton, - mosaic_tpu, - metal, - stablehlo, - tcl, - - pub fn auto(platform: *const zml.Platform) Backend { - return switch (platform.target) { - .cuda => if (cute.isAvailable(platform)) .cute else .triton, - .rocm => .triton, - .oneapi => .triton, - .tpu => .mosaic_tpu, - .metal => .metal, - .cpu => .stablehlo, - .furiosa => .tcl, - .neuron => stdx.debug.panic("Paged attention is not supported on {s} yet", .{@tagName(platform.target)}), - }; - } - - pub fn isAvailable(backend: Backend, platform: *const zml.Platform) bool { - return switch (backend) { - .cute => cute.isAvailable(platform), - .stablehlo => true, - .tcl => platform.target == .furiosa, - .triton => switch (platform.target) { - .cuda, .rocm, .oneapi => true, - .cpu, .tpu, .neuron, .metal, .furiosa => false, - }, - .metal => platform.target == .metal, - .mosaic_tpu => platform.target == .tpu, - }; - } -}; - -pub const Options = union(Backend) { - cute: cute.Options, - triton: triton.paged.Options, - mosaic_tpu: tpu.mosaic_tpu.Options, - metal: metal.Options, - stablehlo: triton.paged.Options, - tcl: tcl.paged.Options, - - const Args = struct { - backend: Backend, - is_prefill: bool, - batch_size: u32, - batch_size_prefill: ?u32 = null, - batch_size_decode: ?u32 = null, - seq_len: u32, - max_num_pages: u32, - max_token_count: u32, - num_heads: u32, - num_kv_heads: u32, - head_dim: u32, - max_seqlen_q: u32, - }; - - pub fn fromBackend(args: Args) Options { - return switch (args.backend) { - .mosaic_tpu => .{ - .mosaic_tpu = .{ - .is_prefill = args.is_prefill, - .batch_size = args.batch_size, - .max_num_pages = args.max_num_pages, - .max_seqlen_k = args.seq_len, - .max_token_count = args.max_token_count, - .num_heads = args.num_heads, - .num_kv_heads = args.num_kv_heads, - .head_dim = args.head_dim, - }, - }, - inline .cute, .triton, .metal, .stablehlo, .tcl => |t| @unionInit(Options, @tagName(t), .{ - .batch_size = args.batch_size, - .max_num_pages = args.max_num_pages, - .max_seqlen_q = args.max_seqlen_q, - .is_prefill = args.is_prefill, - }), - }; - } - - pub fn isPrefill(self: Options) bool { - return switch (self) { - inline else => |v| v.isPrefill(), - }; - } - - pub fn maxNumPages(self: Options) usize { - return switch (self) { - inline else => |v| v.maxNumPages(), - }; - } -}; - -pub const Parameters = union(Backend) { - cute: cute.Parameters, - triton: triton.paged.Parameters, - mosaic_tpu: tpu.mosaic_tpu.Parameters, - metal: metal.Parameters, - stablehlo: triton.paged.Parameters, - tcl: tcl.paged.Parameters, - - pub fn init(options_: Options) Parameters { - return switch (options_) { - inline else => |opts, backend| @unionInit(Parameters, @tagName(backend), .init(opts)), - }; - } - - pub fn options(self: Parameters) Options { - return switch (self) { - inline else => |v, backend| @unionInit(Options, @tagName(backend), v.options()), - }; - } - - pub fn allocationSize(self: Parameters) usize { - return switch (self) { - inline else => |v| v.allocationSize(), - }; - } - - pub fn onMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - zml.Tensor.onMemoryAll(&self, memory); - return self; - } - - pub fn toMemory(self: Parameters, memory: zml.platform.Memory.Kind) Parameters { - return zml.Tensor.toMemoryAll(self, memory); - } -}; - -pub const KvCache = union(enum) { - split: struct { - k: zml.Tensor, - v: zml.Tensor, - }, - // TODO: rename `dense` to `fused`. - dense: zml.Tensor, - - pub fn update( - self: KvCache, - new_k: zml.Tensor, - new_v: zml.Tensor, - slot_mapping: zml.Tensor, - chunk_size: u32, - backend: Backend, - ) KvCache { - const page_index, const offset = getPageAndOffsetFromSlotMapping(slot_mapping, chunk_size); - - const kv: KvCache = switch (self) { - .split => |split| switch (backend) { - .cute, .triton, .mosaic_tpu, .metal, .stablehlo, .tcl => .{ - .split = .{ - .k = split.k.scatterSlices( - .{ .page = page_index, .k_chunk = offset }, - new_k, - .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, - ).reuseBuffer(self.split.k), - .v = split.v.scatterSlices( - .{ .page = page_index, .k_chunk = offset }, - new_v, - .{ .update_fn = zml.Tensor.ScatterOpts.override, .indices_are_unique = false, .indices_are_sorted = false }, - ).reuseBuffer(self.split.v), - }, - }, - }, - .dense => @panic("TODO"), - }; - - return kv; - } - - pub fn getPageAndOffsetFromSlotMapping(slot_mapping: zml.Tensor, page_size: u32) [2]zml.Tensor { - const page_index = slot_mapping.divByConst(page_size); - const offset = slot_mapping.remainder(.scalar(page_size, slot_mapping.dtype())); - return .{ page_index, offset }; - } -}; - -pub const Partitioning = zml.ops.ManualPartitioning(Attention.forward); - -pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache, opts: AttentionOptions, mesh: *const zml.Mesh, partitioning: Partitioning) zml.Tensor { - _ = k; - _ = v; - const args: Attention = .{ .parameters = parameters, .q = q, .kv_cache = kv_cache, .opts = opts }; - return zml.ops.manualComputation(Attention.forward, args, q.shape(), mesh, partitioning); -} - -const Attention = struct { - parameters: Parameters, - q: zml.Tensor, - kv_cache: KvCache, - opts: AttentionOptions, - - fn forward(args: @This(), _: zml.Shape) zml.Tensor { - return switch (args.parameters) { - .cute => |p| switch (args.kv_cache) { - .split => |split| cute.pagedAttention(p, args.q, split.k, split.v, args.opts), - .dense => @panic("CuTe paged attention requires split KV pages"), - }, - .triton => |triton_parameters| switch (args.kv_cache) { - .split => |split| triton.paged.pagedAttention(triton_parameters, args.q, split.k, split.v, args.opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .mosaic_tpu => |mosaic_tpu_parameters| switch (args.kv_cache) { - .split => std.debug.panic("split KV pages is not supported with the mosaic_tpu backend", .{}), - .dense => tpu.mosaic_tpu.pagedAttention(mosaic_tpu_parameters, args.q, args.kv_cache.dense, args.opts), - }, - .metal => |metal_parameters| switch (args.kv_cache) { - .split => |split| metal.pagedAttention(metal_parameters, args.q, split.k, split.v, args.opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .stablehlo => |params| switch (args.kv_cache) { - .split => |split| stablehlo.pagedAttention(params, args.q, split.k, split.v, args.opts), - .dense => std.debug.panic("fused KV pages are only supported with the mosaic_tpu backend", .{}), - }, - .tcl => |params| switch (args.kv_cache) { - .split => |split| tcl.paged.pagedAttention(params, args.q, split.k, split.v, args.opts), - .dense => std.debug.panic("TCL paged attention takes separate K and V pages", .{}), - }, - }; - } -}; - -test "Backend.auto selects mosaic_tpu on TPU" { - const platform: zml.Platform = .{ - .arena = undefined, - .target = .tpu, - .pjrt_api = undefined, - .pjrt_client = undefined, - .state = zml.platform.State.init(.tpu), - .devices = &.{}, - .memories = &.{}, - .physical_mesh = undefined, - .replicated_mesh = undefined, - .meshes = .empty, - .io_impl = .threaded, - }; - - try std.testing.expectEqual(Backend.mosaic_tpu, Backend.auto(&platform)); -} - -test "Backend.auto selects triton on oneAPI" { - const platform: zml.Platform = .{ - .arena = undefined, - .target = .oneapi, - .pjrt_api = undefined, - .pjrt_client = undefined, - .state = zml.platform.State.init(.oneapi), - .devices = &.{}, - .memories = &.{}, - .physical_mesh = undefined, - .replicated_mesh = undefined, - .meshes = .empty, - .io_impl = .threaded, - }; - - try std.testing.expectEqual(Backend.triton, Backend.auto(&platform)); -} - -test pagedAttention { - const platform = zml.testing.env(); - const reference_backend: Backend = if (Backend.triton.isAvailable(platform)) - .triton - else if (Backend.metal.isAvailable(platform)) - .stablehlo - else - return error.SkipZigTest; - const io = std.testing.io; - const allocator = std.testing.allocator; - var arena_state: std.heap.ArenaAllocator = .init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - const num_pages = 64; - const page_size = 16; - const max_num_pages = 16; - const prefill_token_count = 32; - const num_prefill = 1; - const num_decode = 6; - const active_batch_size = num_prefill + num_decode; - const batch_size = active_batch_size + 1; - const query_token_count = prefill_token_count + num_decode; - const dt: zml.DataType = .bf16; - const tp = zml.testing.tpMesh(); - const num_kv_heads = tp.numPartitionsForLogicalAxis(.model); - const partition = .{ .hkv = .model }; - const tensors: struct { q: zml.Tensor, k: zml.Tensor, v: zml.Tensor, kv_cache: KvCache } = .{ - .q = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hg = 4, .hd = 32 }, dt)).withPartitioning(tp, partition), - .k = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), - .v = zml.Tensor.fromShape(.init(.{ .b = query_token_count, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), - .kv_cache = .{ - .split = .{ - .k = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), - .v = zml.Tensor.fromShape(.init(.{ .page = num_pages, .k_chunk = page_size, .hkv = num_kv_heads, .hd = 32 }, dt)).withPartitioning(tp, partition), - }, - }, - }; - - const meshes: []const *const zml.Mesh = &.{ platform.replicated_mesh, tp }; - const rng_q = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.q.shape(), .{} }, .{ .meshes = meshes }); - defer rng_q.deinit(); - const rng_k = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.k.shape(), .{} }, .{ .meshes = meshes }); - defer rng_k.deinit(); - const rng_kv_cache = try platform.compileFn(allocator, io, zml.Tensor.Rng.normal, .{ tensors.kv_cache.split.k.shape(), .{} }, .{ .meshes = meshes }); - defer rng_kv_cache.deinit(); - - const triton_options_args: Options.Args = .{ - .backend = .triton, - .is_prefill = true, - .batch_size = batch_size, - .batch_size_prefill = num_prefill, - .batch_size_decode = num_decode, - .seq_len = max_num_pages * page_size, - .max_num_pages = max_num_pages, - .max_token_count = 16 * page_size, - .num_heads = @intCast(tensors.q.dim(.hg) * tensors.q.dim(.hkv)), - .num_kv_heads = @intCast(tensors.q.dim(.hkv)), - .head_dim = @intCast(tensors.q.dim(.hd)), - .max_seqlen_q = 16 * 2, - }; - const triton_parameters: Parameters = .init(.fromBackend(triton_options_args)); - var q = try rng_q.eval(allocator, io, {}); - defer q.deinit(); - var new_k = try rng_k.eval(allocator, io, {}); - defer new_k.deinit(); - var new_v = try rng_k.eval(allocator, io, {}); - defer new_v.deinit(); - - var kv_cache_d: zml.Bufferized(KvCache) = .{ .split = undefined }; - kv_cache_d.split.k = try rng_kv_cache.eval(allocator, io, .{}); - defer kv_cache_d.split.k.deinit(); - kv_cache_d.split.v = try rng_kv_cache.eval(allocator, io, .{}); - defer kv_cache_d.split.v.deinit(); - - const block_table: [batch_size][max_num_pages]i32 = .{ - // prefilling pages 9-10 - .{ 0, 1, 9, 10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - // generation - .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - .{ 0, 1, 2, 3, 4, 6, 7, 8, -1, -1, -1, -1, -1, -1, -1, -1 }, - .{ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63 }, - .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - .{ 0, 1, 2, 3, 4, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - .{ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 }, - }; - - const seq_lens: [batch_size]i32 = .{ - 4 * page_size, - 4 * page_size + 3, - 7 * page_size + 11, - 15 * page_size + 9, - 4 * page_size + 3, - 4 * page_size + 3, - 4 * page_size + 3, - 0, - }; - - const query_start_len: [batch_size + 1]i32 = .{ 0, 32, 33, 34, 35, 36, 37, 38, 38 }; - var triton_parameters_d: zml.Bufferized(Parameters) = .{ .triton = .{ - .block_table = try .fromBytes(io, platform, triton_parameters.triton.block_table.shape(), @ptrCast(&block_table)), - .seq_lens = try .fromBytes(io, platform, triton_parameters.triton.seq_lens.shape(), @ptrCast(&seq_lens)), - .query_start_len = try .fromBytes(io, platform, triton_parameters.triton.query_start_len.shape(), @ptrCast(&query_start_len)), - } }; - defer zml.Buffer.deinitAll(Parameters, &triton_parameters_d); - - const all_backends = std.enums.values(Backend); - // stablehlo ignores these options, mosaic_tpu requires a dense - // KV cache instead of the split cache used by this test. - const option_sensitive_backends = [_]Backend{ .triton, .cute, .metal, .tcl }; - const test_cases = [_]struct { - name: []const u8, - attention_options: AttentionOptions, - backends: []const Backend, - }{ - .{ - .name = "unbounded", - .attention_options = .{ .mask = .{ .standard = .{} } }, - .backends = all_backends, - }, - .{ - .name = "non_causal", - .attention_options = .{ .mask = .{ .standard = .{ .causal = false } } }, - .backends = &option_sensitive_backends, - }, - .{ - .name = "sliding_window", - .attention_options = .{ .mask = .{ .standard = .{ .sliding_window = page_size } } }, - .backends = &option_sensitive_backends, - }, - }; - - for (test_cases) |test_case| { - var results_per_backend: std.enums.EnumArray(Backend, ?zml.Slice) = .initFill(null); - defer { - for (std.enums.values(Backend)) |backend| { - if (results_per_backend.get(backend)) |slice| slice.free(allocator); - } - } - - for (test_case.backends) |backend| { - if (!backend.isAvailable(platform)) { - std.log.warn("paged_attention backend {t} not available", .{backend}); - continue; - } - std.log.warn("Testing paged_attention {s} with backend {t}", .{ test_case.name, backend }); - - var backend_options_args = triton_options_args; - backend_options_args.backend = backend; - const parameters: Parameters = .init(.fromBackend(backend_options_args)); - - const head_spec: zml.Sharding.Partitioning.Logical = .parse(.{ .hkv = .model }); - const partitioning: Partitioning = .{ - .in_specs = .{ - .parameters = try zml.ops.partitionSpecsAlloc(arena, parameters, .replicated), - .q = head_spec, - .kv_cache = .{ .split = .{ .k = head_spec, .v = head_spec } }, - .opts = .{ .mask = .{ .standard = {} }, .sink = null }, - }, - .out_specs = head_spec, - }; - - const exe = try platform.compileFn( - allocator, - io, - pagedAttention, - .{ parameters, tensors.q, tensors.k, tensors.v, tensors.kv_cache, test_case.attention_options, tp, partitioning }, - .{ .program_name = try std.fmt.allocPrint(arena, "paged_attention_{s}_{t}", .{ test_case.name, backend }), .meshes = meshes }, - ); - defer exe.deinit(); - - const parameters_d: zml.Bufferized(Parameters) = switch (parameters) { - .triton => triton_parameters_d, - inline .cute, .metal, .mosaic_tpu, .stablehlo, .tcl => |_, t| @unionInit(zml.Bufferized(Parameters), @tagName(t), .{ - .block_table = triton_parameters_d.triton.block_table, - .seq_lens = triton_parameters_d.triton.seq_lens, - .query_start_len = triton_parameters_d.triton.query_start_len, - }), - }; - - var output_d = try exe.eval(allocator, io, .{ parameters_d, q, new_k, new_v, kv_cache_d, .{ .mask = .{ .standard = {} }, .sink = null } }); - defer output_d.deinit(); - results_per_backend.set(backend, try output_d.toSliceAlloc(allocator, io)); - } - - const reference = results_per_backend.get(reference_backend) orelse { - std.log.warn("test pagedAttention {s} has no {t} reference", .{ test_case.name, reference_backend }); - continue; - }; - - var num_failed: u32 = 0; - for (std.enums.values(Backend)) |backend| { - if (backend == reference_backend) continue; - const output_h = results_per_backend.get(backend) orelse continue; - - const tolerance: zml.testing.CompareOpts = .{ - .absolute_tolerance = 1e-2, - .relative_tolerance = 1e-2, - .epsilon_relative = 1e-6, - }; - - zml.testing.expectClose(io, reference, output_h, tolerance) catch |err| switch (err) { - error.TestUnexpectedResult => { - num_failed += 1; - std.log.err("test pagedAttention {s} failed on backend {t} against reference {t}", .{ - test_case.name, - backend, - reference_backend, - }); - }, - else => |e| return e, - }; - } - - if (num_failed > 0) return error.TestUnexpectedResult; - } -} diff --git a/zml/kernels/paged_attention/stablehlo/BUILD.bazel b/zml/kernels/paged_attention/stablehlo/BUILD.bazel deleted file mode 100644 index f60464f4..00000000 --- a/zml/kernels/paged_attention/stablehlo/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "stablehlo", - srcs = ["stablehlo.zig"], -) diff --git a/zml/kernels/paged_attention/stablehlo/README.md b/zml/kernels/paged_attention/stablehlo/README.md deleted file mode 100644 index 34144345..00000000 --- a/zml/kernels/paged_attention/stablehlo/README.md +++ /dev/null @@ -1,30 +0,0 @@ -# StableHLO paged attention - -This fallback is automatically selected for CPU and Furiosa. See the -[parent README](../README.md) for the general attention equation, packed -sequence metadata, and page addressing contract. - -[The implementation](stablehlo.zig) iterates over sequences, query chunks, and -KV pages using StableHLO control flow. It merges partial softmax states with -`f32` maxima, exponential sums, and weighted-value accumulators, then converts -the final output to the query dtype. Query-key dot products use the input dtype. - -## Current limitations - -- Only split KV caches work. Although public dispatch forwards `.dense`, this - implementation accesses `.split`. Latent caches are rejected by dispatch. -- Attention is always causal: `causal = false` is currently ignored. Sliding - windows and explicit key indices are rejected. `opts.sink` is ignored; custom - scale is supported. -- Multi-query chunks are processed a full page at a time, and the packed output - cursor advances by the page size. Use page-multiple query chunks; arbitrary - partial query chunks are not covered by this implementation. Single-query - decode uses a separate one-row path. -- The sequence loop stops at the first empty query interval. Active sequences - must precede trailing inactive entries. - -The shared tests in [paged_attention.zig](../paged_attention.zig) use this path -as the causal comparison reference with `bf16`, page size 16, mixed -prefill/decode, shared prefix pages, partial final KV pages, and trailing -inactive sequences. Those cases do not establish support for every shape or -dtype, or conformance for the unsupported options above. diff --git a/zml/kernels/paged_attention/stablehlo/stablehlo.zig b/zml/kernels/paged_attention/stablehlo/stablehlo.zig deleted file mode 100644 index 55ee01d5..00000000 --- a/zml/kernels/paged_attention/stablehlo/stablehlo.zig +++ /dev/null @@ -1,262 +0,0 @@ -const zml = @import("../../../zml.zig"); -const AttentionOptions = @import("../options.zig").AttentionOptions; -const triton = @import("../triton/triton.zig"); - -pub fn pagedAttention( - parameters: triton.paged.Parameters, - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - opts: AttentionOptions, -) zml.Tensor { - switch (opts.mask) { - .standard => |options| if (options.sliding_window != null) @panic("Sliding window attention mask is not supported on StableHLO"), - .indices => @panic("Sparse attention mask is not supported on StableHLO"), - } - - const page_size = k.dim(.k_chunk); - const padded_q = q.pad(0, .{ .b = zml.Tensor.Pad{ .high = page_size } }); - - const final_state = zml.ops.@"while"( - AttentionLoop, - .{ .q = padded_q, .k = k, .v = v, .parameters = parameters, .opts = opts }, - .{ - // i32 to match triton conventions - .slot_id = .scalar(0, .i32), - .seq_id = .scalar(0, .i32), - .q_page_idx = .scalar(0, .i32), - .k_page_idx = .scalar(0, .i32), - // Note: we do all partial softmax in f32 - .partial_softmax_prefill = .zeroes(q.shape().setDim(.b, page_size).withDtype(.f32)), - .partial_softmax_decode = .zeroes(q.shape().setDim(.b, 1).withDtype(.f32)), - .out = .zeroes(padded_q.shape()), - }, - ); - - return final_state.out.slice(.b, .{ .end = q.dim(.b) }); -} - -const AttentionLoop = struct { - q: zml.Tensor, - k: zml.Tensor, - v: zml.Tensor, - parameters: triton.paged.Parameters, - opts: AttentionOptions, - - const While = @This(); - - pub const State = struct { - slot_id: zml.Tensor, - seq_id: zml.Tensor, - q_page_idx: zml.Tensor, - k_page_idx: zml.Tensor, - partial_softmax_prefill: PartialSoftmax, - partial_softmax_decode: PartialSoftmax, - out: zml.Tensor, - }; - - pub fn cond(self: While, state: State) zml.Tensor { - const has_more_sequences = state.seq_id.cmp(.LT, .scalar(self.parameters.seq_lens.count(), .i32)); - const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id)); - const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))); - const has_queries = query_start.cmp(.LT, query_end).asScalar(); - return has_more_sequences.logical(.AND, has_queries); - } - - pub fn body(self: While, state_: State) State { - const query_start = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id)); - const query_end = self.parameters.query_start_len.slice(0, .dynSingle(state_.seq_id.addConstant(1))); - const seq_num_queries_: zml.Tensor = .asScalar(.sub(query_end, query_start)); - const page_size_: u32 = @intCast(self.k.dim(.k_chunk)); - - return zml.ops.@"if"( - struct { - const IfPrefill = @This(); - while_body: While, - state: State, - seq_num_queries: zml.Tensor, - page_size: u32, - - /// Prefill version: num_slots == page_size, we multiply a full page of queries with a page of keys. - pub fn onTrue(if_ctx: IfPrefill) State { - const while_body = if_ctx.while_body; - const state = if_ctx.state; - const page_size = if_ctx.page_size; - const num_slots: u32 = page_size; - - // seq_lens includes the current queries; subtracting their count gives the context length. - const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)); - const q_offset = seq_len.sub(if_ctx.seq_num_queries).add(state.q_page_idx.scale(page_size)); - const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); - - // The current K page is processed in this iteration. Continue if the next page contains - // any key visible to the current query chunk, whose exclusive end is q_offset + num_slots. - const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); - const sequence_has_more_queries: zml.Tensor = .cmp(state.q_page_idx.addConstant(1).scale(num_slots), .LT, if_ctx.seq_num_queries); - - const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); - } - - /// Decode version: num_slots == 1, we multiply one query with a page of keys. - pub fn onFalse(if_ctx: IfPrefill) State { - const while_body = if_ctx.while_body; - const state = if_ctx.state; - const page_size = if_ctx.page_size; - const num_slots: u32 = 1; - - // A decode query is the final token in seq_len, so its zero-based offset is seq_len - 1. - const seq_len = while_body.parameters.seq_lens.slice(0, .dynSingle(state.seq_id)).asScalar(); - const q_offset = seq_len.sub(if_ctx.seq_num_queries); - const next_k_offset = state.k_page_idx.addConstant(1).scale(page_size); - - const query_has_more_keys: zml.Tensor = .cmp(next_k_offset, .LT, q_offset.addConstant(num_slots)); - const sequence_has_more_queries: zml.Tensor = .scalar(false, .bool); - - const partial_softmax = while_body.attentionOnePage(state, q_offset, num_slots, page_size); - return updateState(state, num_slots, query_has_more_keys, sequence_has_more_queries, partial_softmax, while_body.nextQueryStart(state)); - } - }, - .{ - .while_body = self, - .state = state_, - .seq_num_queries = seq_num_queries_, - .page_size = page_size_, - }, - .cmp(seq_num_queries_, .GT, .scalar(1, .i32)), - ); - } - - pub fn attentionOnePage(self: While, state: State, q_offset: zml.Tensor, q_chunk: u32, page_size: u32) PartialSoftmax { - const k_offset = state.k_page_idx.scale(page_size); - const active_q = self.q.slice(self.q.axis(.b), .dyn(state.slot_id, q_chunk)); - - const active_k_page_id = self.parameters.block_table.slices( - .{ .b, .p }, - &.{ .dynSingle(state.seq_id), .dynSingle(state.k_page_idx) }, - ); - - const page_axis = self.k.axis(.page); - const active_k = self.k.slice(page_axis, .dynSingle(active_k_page_id)); - const active_v = self.v.slice(page_axis, .dynSingle(active_k_page_id)); - - const dtype = self.q.dtype(); - const attn_mask: zml.Tensor = attn_mask: { - const mask_shape: zml.Shape = .init(.{ .b = q_chunk, .k_chunk = page_size }, dtype); - - const q_idx = zml.Tensor.iota(mask_shape, .b).add(q_offset); - const k_idx = zml.Tensor.iota(mask_shape, .k_chunk).add(k_offset); - - const mask = zml.Tensor.cmp(k_idx, .LE, q_idx); - const minus_inf = zml.Tensor.constant(dtype.minValue()); - break :attn_mask .select(mask, .scalar(0, dtype), minus_inf); - }; - - const scale: f32 = self.opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(active_q.dim(.hd))))); - var attn_weights = active_q.dot(active_k, .hd).scale(scale); - attn_weights = attn_weights.add(attn_mask.broad(attn_weights.shape())); - - const og_softmax = if (q_chunk == 1) state.partial_softmax_decode else state.partial_softmax_prefill; - // Note: we compute q * k using q and k dtype, but the partial softmax is in f32. - const precise_dt = og_softmax.values.dtype(); - var partial = partialSoftmax(attn_weights.convert(precise_dt), .k_chunk); - partial.values = partial.values.dot(active_v.convert(precise_dt), .k_chunk).transpose(og_softmax.values.shape()); - partial.max_value = partial.max_value.transpose(og_softmax.max_value.shape()); - partial.exp_sum = partial.exp_sum.transpose(og_softmax.exp_sum.shape()); - - return og_softmax.merge(partial); - } - - fn nextQueryStart(self: While, state: State) zml.Tensor { - return self.parameters.query_start_len.slice(0, .dynSingle(state.seq_id.addConstant(1))).asScalar(); - } - - pub fn updateState(state: State, num_slots: u32, query_has_more_keys: zml.Tensor, sequence_has_more_queries: zml.Tensor, softmax: PartialSoftmax, next_query_start: zml.Tensor) State { - return zml.ops.if2( - query_has_more_keys, - // If the query has more pages: update partial softmax with attention on current page, delay updating output. - State{ - .slot_id = state.slot_id, - .seq_id = state.seq_id, - .q_page_idx = state.q_page_idx, - .k_page_idx = state.k_page_idx.addConstant(1), - .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else softmax, - .partial_softmax_decode = if (num_slots == 1) softmax else state.partial_softmax_decode, - .out = state.out, - }, - // Query has no more pages, we commit softmax to output and increment slot - State{ - .k_page_idx = .scalar(0, .i32), - .partial_softmax_prefill = if (num_slots == 1) state.partial_softmax_prefill else .reset(softmax), - .partial_softmax_decode = if (num_slots == 1) .reset(softmax) else state.partial_softmax_decode, - .out = state.out.dynamicUpdateSlice(.{ .b = state.slot_id }, softmax.finalize().convert(state.out.dtype())), - .slot_id = .select(sequence_has_more_queries, state.slot_id.addConstant(num_slots), next_query_start), - // If the sequence has more query chunks: increment q_page_idx, otherwise start over at 0. - .q_page_idx = .select(sequence_has_more_queries, state.q_page_idx.addConstant(1), .scalar(0, .i32)), - // If the sequence has more query chunks: keep seq_id, otherwise increment seq_id. - .seq_id = .select(sequence_has_more_queries, state.seq_id, state.seq_id.addConstant(1)), - }, - ); - } -}; - -pub const PartialSoftmax = struct { - values: zml.Tensor, - exp_sum: zml.Tensor, - max_value: zml.Tensor, - - pub fn zeroes(q_shape: zml.Shape) PartialSoftmax { - return .{ - .values = .zeroes(q_shape), - .exp_sum = .zeroes(q_shape.drop(.hd)), - .max_value = .broad(.constant(q_shape.dtype().minValue()), q_shape.drop(.hd)), - }; - } - - pub fn reset(self: PartialSoftmax) PartialSoftmax { - return .zeroes(self.values.shape()); - } - - pub fn merge(self: PartialSoftmax, other: PartialSoftmax) PartialSoftmax { - // Rescale self and other using the new global_max. - const global_max = self.max_value.maximum(other.max_value.convert(self.max_value.dtype())); - const new_self = self.rescale(global_max); - const new_other = other.rescale(global_max); - - // Now that self and other are using the same scale, we can just add them: - return .{ - .max_value = global_max, - .values = new_self.values.add(new_other.values), - .exp_sum = new_self.exp_sum.add(new_other.exp_sum), - }; - } - - /// Update max_value and rescale attn and exp_sum accordingly. - pub fn rescale(self: PartialSoftmax, max_value: zml.Tensor) PartialSoftmax { - const max_diff_exp: zml.Tensor = .exp(self.max_value.sub(max_value)); - const sum_dtype = self.exp_sum.dtype(); - return .{ - .max_value = max_value, - .values = self.values.mul(max_diff_exp.broad(self.values.shape())), - .exp_sum = self.exp_sum.mul(max_diff_exp.convert(sum_dtype)), - }; - } - - /// Divides the intermediary results by the exp_sum to get the proper attention values. - pub fn finalize(self: PartialSoftmax) zml.Tensor { - return self.values.div(self.exp_sum.broad(self.values.shape()).convert(self.values.dtype())); - } -}; - -/// Compute softmax over a chunk. -/// Returns intermediary results to allow aggregating later. -pub fn partialSoftmax(self: zml.Tensor, axis: anytype) PartialSoftmax { - const a = self.axis(axis); - const max_val = self.max(a).squeeze(a).maximum(.scalar(-1e16, self.dtype())); - const out = self.sub(max_val.broad(self.shape())).exp(); - return .{ - .values = out, - .exp_sum = out.convert(.f32).sum(a).squeeze(a), - .max_value = max_val, - }; -} diff --git a/zml/kernels/paged_attention/tcl/BUILD.bazel b/zml/kernels/paged_attention/tcl/BUILD.bazel deleted file mode 100644 index 4057eea3..00000000 --- a/zml/kernels/paged_attention/tcl/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "tcl", - srcs = ["tcl.zig"], -) diff --git a/zml/kernels/paged_attention/tcl/tcl.zig b/zml/kernels/paged_attention/tcl/tcl.zig deleted file mode 100644 index d04b684e..00000000 --- a/zml/kernels/paged_attention/tcl/tcl.zig +++ /dev/null @@ -1,1781 +0,0 @@ -//! Furiosa RNGD paged attention in TCL: f32 softmax and accumulation, BF16 contractions. -//! `defaultPlan` picks fusible BF16 decode kernels, else one-shot attention or a device loop over key chunks. -//! `Repro:` files are in the reproducer repository, platform/furiosa. -const std = @import("std"); - -const builder = @import("dsl/tcl/builder"); -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const Tensor = zml.Tensor; -const tcl = zml.kernel.tcl; -const AttentionOptions = @import("../options.zig").AttentionOptions; -const triton = @import("../triton/triton.zig"); - -/// WORKAROUND (TCC 2026.5.0b0): flash decode with 8192-key chunks fails to -/// lower, so decode chunks hold 4096 keys. Repro: repro_flash_decode_8k_chunks.py. -/// Remove once 8192-key chunks compile. -const chunk_keys = 4096; -const flash_chunk_pairs = 512 * 2048; -const prefill_chunk_keys = 2048; -const prefill_block = 32; - -/// WORKAROUND (TCC 2026.5.0b0): block counts that are not a power of two slow TCC's layout -/// search or panic its allocator. Repro: repro_flash_allocator_block_group_panic.py. -/// Remove once any count compiles. -fn prefillBlocks(tokens: i64, sequences: i64, block: i64) i64 { - const blocks = @divTrunc(tokens + sequences * (block - 1), block); - return @intCast(std.math.ceilPowerOfTwo(u64, @intCast(blocks)) catch unreachable); -} - -fn prefillBlock(tokens: i64, sequences: i64) i64 { - const short = prefill_block / 2; - return if (short * prefillBlocks(tokens, sequences, short) < prefill_block * prefillBlocks(tokens, sequences, prefill_block)) short else prefill_block; -} - -/// A straight-line head over each token's first `head` keys, then a device loop over -/// `chunk`-key chunks (none when zero), with tokens in groups of `group`. -pub const Plan = struct { - head: i64, - chunk: i64, - group: i64, -}; - -pub var calibration_plan: ?Plan = null; - -pub const paged = struct { - pub const Options = triton.paged.Options; - pub const Parameters = triton.paged.Parameters; - - /// `q` is `{.b = tokens, .hkv, .hg, .hd}`, the caches `{.page, .k_chunk, .hkv, .hd}`. - /// Sinks are one logit per query head. - pub fn pagedAttention(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { - if (opts.mask == .indices) @panic("Sparse attention masks are not supported by TCL paged attention"); - stdx.debug.assert(q.rank() == 4 and k_cache.rank() == 4 and v_cache.rank() == 4 and k_cache.shape().eqlDims(v_cache.shape()), "TCL attention requires matching rank-four split K/V caches", .{}); - stdx.debug.assert(q.dtype().isFloat() and k_cache.dtype().isFloat() and v_cache.dtype().isFloat(), "TCL attention requires floating-point queries and caches", .{}); - stdx.debug.assert(q.dim(.hkv) == k_cache.dim(.hkv) and q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hg) > 0, "TCL attention query and cache head dimensions must match", .{}); - stdx.debug.assert(parameters.block_table.dtype() == .i32 and parameters.seq_lens.dtype() == .i32 and parameters.query_start_len.dtype() == .i32, "TCL attention metadata must be I32", .{}); - stdx.debug.assert(parameters.block_table.dim(.b) == parameters.seq_lens.dim(.b) and parameters.query_start_len.dim(.b) == parameters.seq_lens.dim(.b) + 1, "TCL attention metadata batch dimensions must match", .{}); - stdx.debug.assert(parameters.seq_lens.dim(.b) > 0 and parameters.block_table.dim(.p) > 0 and k_cache.dim(.page) > 0 and k_cache.dim(.k_chunk) > 0, "TCL attention requires nonempty sequence and cache capacities", .{}); - stdx.debug.assert(k_cache.dim(.page) * k_cache.dim(.k_chunk) <= std.math.maxInt(i32) and q.dim(.b) <= std.math.maxInt(i32), "TCL attention indices must fit in I32", .{}); - const compiler = zml.Compiler.current(); - stdx.debug.assert(compiler.platform.target == .furiosa, "TCL attention requires Furiosa chips", .{}); - const attributes = compiler.platform.devices[0].pjrt_device.attributes(compiler.platform.pjrt_api) catch - std.debug.panic("TCL attention could not verify the Furiosa device's PE range", .{}); - defer attributes.deinit(); - const first_pe = attributes.get("pe_start"); - const last_pe = attributes.get("pe_end"); - stdx.debug.assert(first_pe != null and first_pe.? == .int64 and first_pe.?.int64 == 0 and - last_pe != null and last_pe.? == .int64 and last_pe.?.int64 == 7, "TCL attention requires a complete 8-PE RNGD chip", .{}); - return attend(parameters, q, k_cache, v_cache, opts); - } - - fn attend(parameters: Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, opts: AttentionOptions) Tensor { - if (q.dim(.b) == 0) return Tensor.zeroes(q.shape()); - const page = k_cache.dim(.k_chunk); - // WORKAROUND (TCC 2026.5.0b0): no DPE tactic for 32-key attention, so pad to - // at least 64 keys with invalid pages. Repro: repro_decode_32_keys_no_tactic.py. - // Remove once 16- to 48-key decode compiles. - const keys = @max(parameters.block_table.dim(.p), std.math.divCeil(i64, 64, page) catch unreachable) * page; - // WORKAROUND (TCC 2026.5.0b0): TCC fails to allocate a 4080-key chunk (85 - // pages of 48), so chunks hold whole pages and a multiple of 64 keys. Repro: - // repro_flash_allocator_block_group_panic.py. Remove once such chunks compile. - const key_alignment = @divExact(page, @as(i64, @intCast(std.math.gcd(@as(u64, @intCast(page)), 64)))) * 64; - const alignment = if (key_alignment <= chunk_keys) key_alignment else page; - const decode = !parameters.options_.is_prefill and q.dim(.b) == parameters.seq_lens.dim(.b); - const block = if (decode) 1 else prefillBlock(q.dim(.b), parameters.seq_lens.dim(.b)); - const block_count = if (decode) q.dim(.b) else prefillBlocks(q.dim(.b), parameters.seq_lens.dim(.b), block); - const rows = block_count * block; - // A flash chunk's query-key pairs are bounded: the largest chunks do not finish compiling. - const chunk_limit = @max(1, @divTrunc(@min(@as(i64, if (decode) chunk_keys else prefill_chunk_keys), @divTrunc(flash_chunk_pairs, rows)), alignment)) * alignment; - const chunk = @min(keys, chunk_limit); - var cfg: Cfg = .{ - .tokens = q.dim(.b), - .sequences = parameters.seq_lens.dim(.b), - .b = block_count, - .i = block, - .a = (std.math.divCeil(i64, keys, chunk) catch unreachable) * chunk, - .chunk = chunk, - .hkv = q.dim(.hkv), - .g = q.dim(.hg), - .d = q.dim(.hd), - .pages = k_cache.dim(.page), - .pg = page, - .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), - .decode = decode, - .use_sink = opts.sink != null, - .is_causal = opts.mask.isCausal(), - .window = @intCast(opts.mask.slidingWindow() orelse 0), - }; - const plain_decode = decode and !cfg.use_sink and cfg.window == 0; - // WORKAROUND (TCC 2026.5.0b0): an in-unit cache conversion and several tokens - // over a non-power-of-two page count fail to compile, so straight decode reads - // BF16 caches and pads the table. Repro: repro_sparse_propagation_transmute.py, - // repro_metadata_page_count_cover_valid_range.py. Remove once both compile. - const straight = decode and k_cache.dtype() == .bf16 and v_cache.dtype() == .bf16; - const straight_keys = if (straight and cfg.tokens > 1) @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(@divExact(keys, page))) catch unreachable)) * page else keys; - const plan = calibration_plan orelse defaultPlan(cfg, straight_keys, parameters.block_table.dim(.p), plain_decode and straight, straight, chunk); - const sink = if (opts.sink) |s| s.convert(.f32).reshape(.{ .hkv = cfg.hkv, .hg = cfg.g }) else Tensor.zeroes(.init(.{ .hkv = cfg.hkv, .hg = cfg.g }, .f32)); - if (plan.head > 0 and plan.chunk > 0) { - stdx.debug.assert(plain_decode and cfg.tokens == 1, "TCL hybrid attention takes one plain decode token", .{}); - return hybridDecodeAttention(parameters, q, k_cache, v_cache, cfg.scale, cfg.is_causal, plan).convert(q.dtype()); - } - if (plan.chunk == 0 and straight) { - return decodeAttention(parameters, q, k_cache, v_cache, sink, .{ - .t = cfg.tokens, - .a = straight_keys, - .group = plan.group, - .hkv = cfg.hkv, - .g = cfg.g, - .d = cfg.d, - .pages = cfg.pages, - .pg = page, - .scale = cfg.scale, - .is_causal = cfg.is_causal, - .window = cfg.window, - .use_sink = cfg.use_sink, - }).convert(q.dtype()); - } - cfg.chunk = if (plan.chunk > 0) plan.chunk else keys; - cfg.a = (std.math.divCeil(i64, keys, cfg.chunk) catch unreachable) * cfg.chunk; - const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = @divExact(cfg.a, page) - parameters.block_table.dim(.p) } }); - // WORKAROUND (TCC 2026.5.0b0): attention over query groups of a size - // other than a power of two fails ("Inconsistent tensor group - // offsets"): pad the groups with zero queries, whose outputs are - // dropped. Repro: - // repro_flash_prefill_group6_inconsistent_offsets.py (zml/reproducer, - // platform/furiosa). Remove once any group size compiles. - const groups: i64 = @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.g)) catch unreachable); - const extra = groups - cfg.g; - cfg.g = groups; - const queries = q.convert(.bf16).pad(0, .{ .hg = Tensor.Pad{ .high = extra } }); - const inputs: OneShot.Inputs = .{ - .q = queries, - .pages = table, - .seq_lens = parameters.seq_lens, - .query_offsets = parameters.query_start_len, - .k_cache = k_cache.convert(.bf16), - .v_cache = v_cache.convert(.bf16), - .sink = sink.pad(0, .{ .hg = Tensor.Pad{ .high = extra } }), - }; - const outputs: OneShot.Outputs = .{ .out = queries.shape() }; - const out = if (cfg.a == cfg.chunk) - OneShot.call(inputs, outputs, .{ .cfg = cfg }).out - else - Flash.call(inputs, outputs, .{ .cfg = cfg }).out; - return out.slice(.hg, .{ .end = groups - extra }).convert(q.dtype()); - } -}; - -fn defaultPlan(cfg: Cfg, keys: i64, columns: i64, plain_decode: bool, straight: bool, chunk: i64) Plan { - // WORKAROUND (TCC 2026.5.0b0): the hybrid kernel's tail does not compile in - // reasonable time for 2 or 4 KV heads wider than 128, which run flash instead. - // Repro: repro_hybrid_decode_slow_compile.py. Remove once that tail compiles. - if (plain_decode and cfg.tokens == 1 and columns > std.math.divCeil(i64, straight_decode_keys, cfg.pg) catch unreachable) { - if (cfg.d <= 128 or cfg.hkv == 1 or cfg.hkv >= 8) return .{ .head = head_keys, .chunk = tail_chunk_keys, .group = 1 }; - return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; - } - if (straight and cfg.window > 0) { - const window_chunk = @max(window_chunk_keys, @as(i64, @intCast(std.math.ceilPowerOfTwo(u64, @intCast(cfg.window)) catch unreachable))); - const chunks: i64 = if (cfg.tokens < chunked_decode_tokens) window_flash_chunks else batch_window_flash_chunks; - if (window_chunk <= chunk and @rem(window_chunk, cfg.pg) == 0 and keys >= chunks * window_chunk) - return .{ .head = 0, .chunk = window_chunk, .group = cfg.tokens }; - } - if (straight and keys <= decode_keys and (cfg.tokens < chunked_decode_tokens or keys <= straight_batch_decode_keys)) - return .{ .head = keys, .chunk = 0, .group = @divExact(cfg.tokens, decodeGroups(cfg.tokens, keys)) }; - if (chunk == keys) return .{ .head = keys, .chunk = 0, .group = cfg.tokens }; - return .{ .head = 0, .chunk = chunk, .group = cfg.b * cfg.i }; -} - -const window_chunk_keys = 1024; -const window_flash_chunks = 16; -const batch_window_flash_chunks = 4; - -/// Widest block table the straight-line decode kernel takes for one token. -/// WORKAROUND (TCC 2026.5.0b0): several tokens do not use it, as TCC cannot -/// propagate gather bounds outside a loop over a token axis. Repro: -/// repro_sparse_propagation_loop.py. Remove once such kernels propagate. -const straight_decode_keys = 32768; - -/// Keys a straight-line decode kernel attends to in one pass. WORKAROUND (TCC -/// 2026.5.0b0): TCC fails on the decode metadata for 131072 keys in pages of 16. -/// Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once that compiles. -const decode_keys = 65536; - -const chunked_decode_tokens = 16; -const straight_batch_decode_keys = 8192; - -/// Decode contracts the gathered cache unmasked: the select gives unused entries (NaN included) zero -/// weight. WORKAROUND (TCC 2026.5.0b0): TCC panics ("slice out of bounds") on score contractions of more -/// tokens times keys than this. Repro: repro_gather_einsum_slice_panic.py. Remove once larger products compile. -const unmasked_decode_keys = 1 << 17; - -const masked_decode_keys = 4096; - -const DecodeCfg = struct { - t: i64, - a: i64, - group: i64, - hkv: i64, - g: i64, - d: i64, - pages: i64, - pg: i64, - scale: f32, - is_causal: bool, - window: i64, - use_sink: bool, -}; - -/// The token metadata depends only on parameters every layer shares, so one kernel computes it per forward pass. -fn decodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, sink: Tensor, cfg: DecodeCfg) Tensor { - const columns = @divExact(cfg.a, cfg.pg); - const table = parameters.block_table.pad(0, .{ .p = Tensor.Pad{ .high = columns - parameters.block_table.dim(.p) } }); - const meta = DecodeMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ - .pages = zml.Shape.init(.{ .t = cfg.t, .p = columns }, .i32), - .bounds = zml.Shape.init(.{ .t = cfg.t }, .i32), - .mask = zml.Shape.init(.{ .t = cfg.t, .a = cfg.a }, .i32), - .keyed = zml.Shape.init(.{ .t = cfg.t }, .f32), - }, .{ .cfg = .{ .t = cfg.t, .s = parameters.seq_lens.dim(.b), .ap = columns, .pg = cfg.pg, .pages = cfg.pages, .is_causal = cfg.is_causal, .window = cfg.window } }); - return Decode.call(.{ - .q = q.convert(.bf16), - .pages = meta.pages, - .bounds = meta.bounds, - .mask = meta.mask, - .keyed = meta.keyed, - .k_cache = k_cache.convert(.bf16), - .v_cache = v_cache.convert(.bf16), - .sink = sink, - }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; -} - -const MetaCfg = struct { t: i64, s: i64, ap: i64, pg: i64, pages: i64, is_causal: bool, window: i64 }; - -/// WORKAROUND (TCC 2026.5.0b0): one-hot row selection and `attentionMask`'s clamp arithmetic, as TCC -/// fails on small integer gathers and the plugin's comparisons. Repro: repro_scalar_gather_no_axis_to_pad.py, -/// repro_row_broadcast_past_4096.py. Remove once both compile. -const DecodeMeta = tcl.Kernel(MetaCfg, .{ - .name = "paged_decode_metadata", - .inputs = &.{ "seq_lens", "query_offsets", "table" }, - .outputs = &.{ "pages", "bounds", "mask", "keyed" }, - .run = struct { - fn run(b: *tcl.Builder, cfg: MetaCfg) tcl.FinishError!void { - const T = b.axis("T", cfg.t); - const S = b.axis("S", cfg.s); - const O = b.axis("O", cfg.s + 1); - const AP = b.axis("AP", cfg.ap); - const PG = b.axis("PG", cfg.pg); - const A = b.axis("A", cfg.ap * cfg.pg); - const t = try b.declareArgs(.{ - .seq_lens = .{ .dtype = .i32, .axes = &.{S} }, - .query_offsets = .{ .dtype = .i32, .axes = &.{O} }, - .table = .{ .dtype = .i32, .axes = &.{ S, AP } }, - }); - const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); - const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); - const seq = try countBefore(b, ends, tokens, S); - const own = try selector(b, seq, S, cfg.s); - var op = b.tensorOperation(.{}); - const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); - var visible = try selectMetadata(b, t.seq_lens, own, S); - if (cfg.is_causal) { - const end = try selectMetadata(b, ends, own, S); - op = b.tensorOperation(.{}); - visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); - } - op = b.tensorOperation(.{}); - const limit = try op.commit(op.fetch(visible, .{}).binary(.maxi, 0).binary(.mini, cfg.ap * cfg.pg).muli(op.fetch(active, .{})), .{}); - const entries = try selectMetadata(b, t.table, own, S); - const pages = try validIndices(b, entries, cfg.pages); - const firsts = try b.arange(cfg.ap * cfg.pg, .i32, &.{AP}, .{ .step = cfg.pg }); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const used = try op.commit(op.fetch(limit, .{}).subi(op.fetch(firsts, .{})).binary(.maxi, 0).binary(.mini, 1), .{}); - op = b.tensorOperation(.{}); - const bounds = try op.commit(op.fetch(used, .{}).reduce(&.{AP}, .addi), .{}); - op = b.tensorOperation(.{}); - const spread = try op.commit(op.fetch(try b.arange(cfg.ap, .i32, &.{AP}, .{}), .{}).binary(.mini, cfg.pages - 1), .{}); - op = b.tensorOperation(.{}); - const read = try op.commit(op.where(op.fetch(used, .{}), .ne, 0, op.fetch(pages.safe, .{}), op.fetch(spread, .{ .broadcast_to = &.{ T, AP } })), .{}); - op = b.tensorOperation(.{}); - const keyed = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{AP}, .maxi).toFloat(), .{}); - op = b.tensorOperation(.{}); - const valid = try op.commit(op.fetch(pages.valid, .{ .broadcast_to = &.{ T, AP, PG } }), .{}); - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(try b.arange(cfg.ap * cfg.pg, .i32, &.{A}, .{}), .{ .broadcast_to = &.{ T, A } }), .{}); - op = b.tensorOperation(.{ .tactic = .elementwise }); - var mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(try b.reshape(valid, &.{ T, A }, .{}), .{})), .{}); - if (cfg.window > 0) { - op = b.tensorOperation(.{ .tactic = .elementwise }); - mask = try op.commit(op.fetch(position, .{}).subi(op.fetch(limit, .{})).addi(cfg.window + 1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); - } - b.ret(&.{ read, bounds, mask, keyed }); - } - }.run, -}); - -const Decode = tcl.Kernel(DecodeCfg, .{ - .name = "paged_decode_attention", - .inputs = &.{ "q", "pages", "bounds", "mask", "keyed", "k_cache", "v_cache", "sink" }, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: DecodeCfg) tcl.FinishError!void { - b.fuse(); - configure(b, .{ .tokens = cfg.t, .sequences = cfg.t, .b = cfg.t, .i = 1, .a = cfg.a, .chunk = cfg.a, .hkv = cfg.hkv, .g = cfg.g, .d = cfg.d, .pages = cfg.pages, .pg = cfg.pg, .scale = cfg.scale, .decode = true }); - const T = b.axis("T", cfg.t); - const A = b.axis("A", cfg.a); - const AP = b.axis("AP", @divExact(cfg.a, cfg.pg)); - const Hkv = b.axis("Hkv", cfg.hkv); - const G = b.axis("G", cfg.g); - const D = b.axis("D", cfg.d); - const P = b.axis("P", cfg.pages); - const PG = b.axis("PG", cfg.pg); - const t = try b.declareArgs(.{ - .q = .{ .dtype = .bf16, .axes = &.{ T, Hkv, G, D } }, - .pages = .{ .dtype = .i32, .axes = &.{ T, AP } }, - .bounds = .{ .dtype = .i32, .axes = &.{T} }, - .mask = .{ .dtype = .i32, .axes = &.{ T, A } }, - .keyed = .{ .dtype = .f32, .axes = &.{T} }, - .k_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, - .v_cache = .{ .dtype = .bf16, .axes = &.{ P, PG, Hkv, D } }, - .sink = .{ .dtype = .f32, .axes = &.{ Hkv, G } }, - }); - const sink = if (cfg.use_sink) t.sink else null; - const groups = @divExact(cfg.t, cfg.group); - if (groups == 1) return b.ret(&.{try decodeTokens(b, cfg.scale, t.q, t.pages, t.bounds, t.mask, t.keyed, t.k_cache, t.v_cache, sink)}); - const NG = b.axis("NG", groups); - const TG = b.axis("TG", @divExact(cfg.t, groups)); - var loop = b.openFor(groups, .{try b.scratchpad(.bf16, &.{ NG, TG, Hkv, G, D })}); - { - const q = try b.indexRead(try b.reshape(t.q, &.{ NG, TG, Hkv, G, D }, .{}), loop.iv); - const pages = try b.indexRead(try b.reshape(t.pages, &.{ NG, TG, AP }, .{}), loop.iv); - const bounds = try b.indexRead(try b.reshape(t.bounds, &.{ NG, TG }, .{}), loop.iv); - const mask = try b.indexRead(try b.reshape(t.mask, &.{ NG, TG, A }, .{}), loop.iv); - const keyed = try b.indexRead(try b.reshape(t.keyed, &.{ NG, TG }, .{}), loop.iv); - const out = try decodeTokens(b, cfg.scale, q, pages, bounds, mask, keyed, t.k_cache, t.v_cache, sink); - try loop.yield(.{try b.indexWrite(loop.carried[0], loop.iv, out)}); - } - b.ret(&.{try b.reshape(loop.results[0], &.{ T, Hkv, G, D }, .{})}); - } - }.run, -}); - -const unmasked_group_tokens = 4; - -/// Groups of at most `unmasked_group_tokens` tokens whose tokens times keys fit `unmasked_decode_keys`. -fn decodeGroups(tokens: i64, keys: i64) i64 { - var groups: i64 = 1; - while ((@divExact(tokens, groups) > unmasked_group_tokens or @divExact(tokens, groups) * keys > unmasked_decode_keys) and groups < tokens) { - groups += 1; - while (@rem(tokens, groups) != 0) groups += 1; - } - return groups; -} - -/// `keyed` scales the softmax: zero for tokens without keys. -fn decodeTokens(b: *tcl.Builder, scale: f32, q: builder.Tensor, pages: builder.Tensor, bounds: builder.Tensor, mask: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, sink: ?builder.Tensor) tcl.FinishError!builder.Tensor { - const T = q.axes[0]; - const Hkv = q.axes[1]; - const G = q.axes[2]; - const D = q.axes[3]; - const AP = pages.axes[1]; - const P = k_cache.axes[0]; - const PG = k_cache.axes[1]; - const keys: []const tcl.Axis = &.{ AP, PG }; - const key_mask = try b.reshape(mask, &.{ T, AP, PG }, .{}); - var k = try b.gather(k_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); - var v = try b.gather(v_cache, pages, P, &.{ T, AP, PG, Hkv, D }, .{ .valid_length = bounds }); - if (T.size == 1 and AP.size * PG.size <= masked_decode_keys) { - k = try maskCacheValues(b, k, key_mask); - v = try maskCacheValues(b, v, key_mask); - } - var op = b.tensorOperation(.{}); - const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, AP, PG }).mulf(scale), .{}); - op = b.tensorOperation(.{}); - const s = op.fetch(scores, .{}); - const masked = try op.commit(op.where(op.fetch(key_mask, .{}), .ne, 0, s, score_min), .{}); - op = b.tensorOperation(.{}); - const p = if (sink) |logits| blk: { - const maximum = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(logits, .{ .broadcast_to = &.{ Hkv, G, T } })), .{}); - op = b.tensorOperation(.{}); - const e = try op.commit(op.fetch(masked, .{}).subf(op.fetch(maximum, .{})).exp(), .{}); - op = b.tensorOperation(.{}); - const sink_exp = try op.commit(op.fetch(maximum, .{}).mulf(-1).addf(op.fetch(logits, .{})).exp(), .{}); - op = b.tensorOperation(.{}); - const denominator = try op.commit(op.fetch(e, .{}).reduce(keys, .addf).addf(op.fetch(sink_exp, .{})), .{}); - op = b.tensorOperation(.{}); - break :blk try op.commit(op.fetch(e, .{}).divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); - } else blk: { - const x = op.fetch(masked, .{}); - const e = x.subf(x.reduce(keys, .maxf)).exp(); - break :blk try op.commit(e.divf(e.reduce(keys, .addf)).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); - }; - op = b.tensorOperation(.{}); - return op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{ .dtype = .bf16 }); -} - -const head_keys = 4096; -const tail_chunk_keys = 8192; - -const HybridCfg = struct { - t: i64, - s: i64, - /// Head pages, tail chunks and pages per tail chunk. - hp: i64, - nc: i64, - cp: i64, - pg: i64, - hkv: i64, - g: i64, - d: i64, - pages: i64, - scale: f32, - is_causal: bool, -}; - -fn hybridDecodeAttention(parameters: paged.Parameters, q: Tensor, k_cache: Tensor, v_cache: Tensor, scale: f32, is_causal: bool, plan: Plan) Tensor { - const pg = k_cache.dim(.k_chunk); - const t = q.dim(.b); - const s = parameters.seq_lens.dim(.b); - const hp = std.math.divCeil(i64, plan.head, pg) catch unreachable; - const cp = std.math.divCeil(i64, plan.chunk, pg) catch unreachable; - const nc = std.math.divCeil(i64, parameters.block_table.dim(.p) - hp, cp) catch unreachable; - const cfg: HybridCfg = .{ .t = t, .s = s, .hp = hp, .nc = nc, .cp = cp, .pg = pg, .hkv = q.dim(.hkv), .g = q.dim(.hg), .d = q.dim(.hd), .pages = k_cache.dim(.page), .scale = scale, .is_causal = is_causal }; - const table = parameters.block_table.pad(-1, .{ .p = Tensor.Pad{ .high = hp + nc * cp - parameters.block_table.dim(.p) } }); - const meta = HybridMeta.call(.{ .seq_lens = parameters.seq_lens, .query_offsets = parameters.query_start_len, .table = table }, .{ - .hpages = zml.Shape.init(.{ .t = t, .p = hp }, .i32), - .hmask = zml.Shape.init(.{ .t = t, .a = hp * pg }, .i32), - .hbounds = zml.Shape.init(.{ .t = t }, .i32), - .tpages = zml.Shape.init(.{ .c = nc, .t = t, .p = cp }, .i32), - .tmask = zml.Shape.init(.{ .c = nc, .t = t, .a = cp * pg }, .i32), - .tbounds = zml.Shape.init(.{ .c = nc, .t = t }, .i32), - .limits = zml.Shape.init(.{ .t = t }, .i32), - .keyed = zml.Shape.init(.{ .t = t }, .f32), - }, .{ .cfg = cfg }); - return HybridDecode.call(.{ - .q = q.convert(.bf16), - .hpages = meta.hpages, - .hmask = meta.hmask, - .hbounds = meta.hbounds, - .tpages = meta.tpages, - .tmask = meta.tmask, - .tbounds = meta.tbounds, - .limits = meta.limits, - .keyed = meta.keyed, - .k_cache = k_cache.convert(.bf16), - .v_cache = v_cache.convert(.bf16), - }, .{ .out = q.shape().withDtype(.bf16) }, .{ .cfg = cfg }).out; -} - -/// Per token, once per forward pass. Tail chunks are chunk-major, only those holding keys of the -/// longest token; `limits` is the visible key count. -const HybridMeta = tcl.Kernel(HybridCfg, .{ - .name = "paged_hybrid_metadata", - .inputs = &.{ "seq_lens", "query_offsets", "table" }, - .outputs = &.{ "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed" }, - .run = struct { - fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { - const total = (cfg.hp + cfg.nc * cfg.cp) * cfg.pg; - const T = b.axis("T", cfg.t); - const S = b.axis("S", cfg.s); - const O = b.axis("O", cfg.s + 1); - const AP = b.axis("AP", cfg.hp + cfg.nc * cfg.cp); - const HP = b.axis("HP", cfg.hp); - const RP = b.axis("RP", cfg.nc * cfg.cp); - const NC = b.axis("NC", cfg.nc); - const CP = b.axis("CP", cfg.cp); - const HA = b.axis("HA", cfg.hp * cfg.pg); - const CK = b.axis("CK", cfg.cp * cfg.pg); - const t = try b.declareArgs(.{ - .seq_lens = .{ .dtype = .i32, .axes = &.{S} }, - .query_offsets = .{ .dtype = .i32, .axes = &.{O} }, - .table = .{ .dtype = .i32, .axes = &.{ S, AP } }, - }); - const ends = try selectMetadata(b, t.query_offsets, try selector(b, try b.arange(cfg.s + 1, .i32, &.{S}, .{ .start = 1 }), O, cfg.s + 1), O); - const tokens = try b.arange(cfg.t, .i32, &.{T}, .{}); - const seq = try countBefore(b, ends, tokens, S); - const own = try selector(b, seq, S, cfg.s); - var op = b.tensorOperation(.{}); - const active = try op.commit(op.fetch(seq, .{}).muli(-1).addi(cfg.s).binary(.maxi, 0).binary(.mini, 1), .{}); - var visible = try selectMetadata(b, t.seq_lens, own, S); - if (cfg.is_causal) { - const end = try selectMetadata(b, ends, own, S); - op = b.tensorOperation(.{}); - visible = try op.commit(op.fetch(visible, .{}).subi(op.fetch(end, .{})).addi(op.fetch(tokens, .{})).addi(1), .{}); - } - op = b.tensorOperation(.{}); - const limit = try op.commit(op.fetch(visible, .{}).binary(.maxi, 0).binary(.mini, total).muli(op.fetch(active, .{})), .{}); - - // WORKAROUND (TCC 2026.5.0b0): a 3-D one-hot product computes garbage when the one-hot is read - // first, so the selection is 2-D per chunk. Repro: repro_onehot_einsum_by_ve_wrong_values_run.py. - // Remove once it is exact on RNGD. - const rows1 = try b.reshape(try b.slice(t.table, AP, cfg.hp, &.{ S, RP }, .{}), &.{ S, NC, CP }, .{}); - op = b.tensorOperation(.{}); - const rows = try op.commit(op.fetch(rows1, .{}), .{ .axes = &.{ NC, S, CP } }); - // Trips never exceed nc: `limit` is at most the table's keys. - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limit), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); - var loop = b.openFor(trips, .{ try b.scratchpad(.i32, &.{ NC, T, CP }), try b.scratchpad(.i32, &.{ NC, T, CK }), try b.scratchpad(.i32, &.{ NC, T }), try b.full(0, .i32, &.{T}) }); - { - const reads, const masks, const counts, const seen = loop.carried; - const first = try b.symExpr(.add, try b.symExpr(.mul, loop.iv, cfg.cp * cfg.pg), cfg.hp * cfg.pg); - const chunk = try hybridColumns(b, cfg, own, try b.indexRead(rows, loop.iv), limit, first, CP, CK); - op = b.tensorOperation(.{}); - const any = try op.commit(op.fetch(seen, .{}).binary(.maxi, op.fetch(chunk.any, .{})), .{}); - try loop.yield(.{ - try b.indexWrite(reads, loop.iv, chunk.pages), - try b.indexWrite(masks, loop.iv, chunk.mask), - try b.indexWrite(counts, loop.iv, chunk.bounds), - any, - }); - } - const head = try hybridColumns(b, cfg, own, try b.slice(t.table, AP, 0, &.{ S, HP }, .{}), limit, 0, HP, HA); - op = b.tensorOperation(.{}); - const keyed = try op.commit(op.fetch(head.any, .{}).binary(.maxi, op.fetch(loop.results[3], .{})).toFloat(), .{}); - b.ret(&.{ head.pages, head.mask, head.bounds, loop.results[0], loop.results[1], loop.results[2], limit, keyed }); - } - }.run, -}); - -/// Columns start at key `first`; sparse gathers never read past a token's `bounds`. -fn hybridColumns(b: *tcl.Builder, cfg: HybridCfg, own: builder.Tensor, table: builder.Tensor, limit: builder.Tensor, first: anytype, cols: tcl.Axis, keys: tcl.Axis) tcl.FinishError!struct { pages: builder.Tensor, mask: builder.Tensor, bounds: builder.Tensor, any: builder.Tensor } { - const T = own.axes[0]; - const S = own.axes[1]; - const PG = b.axis("PG", cfg.pg); - const pages = try validIndices(b, try selectMetadata(b, table, own, S), cfg.pages); - var op = b.tensorOperation(.{}); - const firsts = try op.commit(op.fetch(try b.arange(cols.size * cfg.pg, .i32, &.{cols}, .{ .step = cfg.pg }), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const used = try op.commit(op.fetch(limit, .{}).subi(op.fetch(firsts, .{})).binary(.maxi, 0).binary(.mini, 1), .{}); - op = b.tensorOperation(.{}); - const bounds = try op.commit(op.fetch(used, .{}).reduce(&.{cols}, .addi), .{}); - op = b.tensorOperation(.{}); - const any = try op.commit(op.fetch(used, .{}).muli(op.fetch(pages.valid, .{})).reduce(&.{cols}, .maxi), .{}); - op = b.tensorOperation(.{}); - const key_ids = try op.commit(op.fetch(try b.arange(keys.size, .i32, &.{keys}, .{}), .{}).addi(if (@TypeOf(first) == builder.Tensor) op.fetch(first, .{}) else first), .{}); - const to: []const tcl.Axis = &.{ T, cols, PG }; - op = b.tensorOperation(.{}); - const position = try op.commit(op.fetch(try b.reshape(key_ids, &.{ cols, PG }, .{}), .{ .broadcast_to = to }), .{}); - op = b.tensorOperation(.{}); - const valid = try op.commit(op.fetch(pages.valid, .{ .broadcast_to = to }), .{}); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const mask = try op.commit(op.fetch(position, .{}).muli(-1).addi(op.fetch(limit, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid, .{})), .{}); - return .{ .pages = pages.safe, .mask = try b.reshape(mask, &.{ T, keys }, .{}), .bounds = bounds, .any = any }; -} - -const SoftmaxState = struct { max: builder.Tensor, sum: builder.Tensor, out: builder.Tensor }; - -const HybridDecode = tcl.Kernel(HybridCfg, .{ - .name = "paged_hybrid_decode_attention", - .inputs = &.{ "q", "hpages", "hmask", "hbounds", "tpages", "tmask", "tbounds", "limits", "keyed", "k_cache", "v_cache" }, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: HybridCfg) tcl.FinishError!void { - b.fuse(); - const a = hybridAxes(b, cfg); - const t = try b.declareArgs(.{ - .q = .{ .dtype = .bf16, .axes = &.{ a.T, a.Hkv, a.G, a.D } }, - .hpages = .{ .dtype = .i32, .axes = &.{ a.T, a.HP } }, - .hmask = .{ .dtype = .i32, .axes = &.{ a.T, a.HA } }, - .hbounds = .{ .dtype = .i32, .axes = &.{a.T} }, - .tpages = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CP } }, - .tmask = .{ .dtype = .i32, .axes = &.{ a.NC, a.T, a.CK } }, - .tbounds = .{ .dtype = .i32, .axes = &.{ a.NC, a.T } }, - .limits = .{ .dtype = .i32, .axes = &.{a.T} }, - .keyed = .{ .dtype = .f32, .axes = &.{a.T} }, - .k_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, - .v_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, - }); - const tail = try hybridTail(b, cfg, a, t.q, t.tpages, t.tmask, t.tbounds, t.limits, t.k_cache, t.v_cache); - b.ret(&.{try hybridHead(b, cfg, a, t.q, t.hpages, t.hmask, t.hbounds, t.keyed, t.k_cache, t.v_cache, tail)}); - } - }.run, -}); - -const HybridAxes = struct { T: tcl.Axis, HP: tcl.Axis, HA: tcl.Axis, NC: tcl.Axis, CP: tcl.Axis, CK: tcl.Axis, Hkv: tcl.Axis, G: tcl.Axis, D: tcl.Axis, P: tcl.Axis, PG: tcl.Axis }; - -fn hybridAxes(b: *tcl.Builder, cfg: HybridCfg) HybridAxes { - return .{ - .T = b.axis("T", cfg.t), - .HP = b.axis("HP", cfg.hp), - .HA = b.axis("HA", cfg.hp * cfg.pg), - .NC = b.axis("NC", cfg.nc), - .CP = b.axis("CP", cfg.cp), - .CK = b.axis("CK", cfg.cp * cfg.pg), - .Hkv = b.axis("Hkv", cfg.hkv), - .G = b.axis("G", cfg.g), - .D = b.axis("D", cfg.d), - .P = b.axis("P", cfg.pages), - .PG = b.axis("PG", cfg.pg), - }; -} - -fn hybridTail(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, tpages: builder.Tensor, tmask: builder.Tensor, tbounds: builder.Tensor, limits: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor) tcl.FinishError!SoftmaxState { - const stats: []const tcl.Axis = &.{ a.Hkv, a.G, a.T }; - const trips = try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(limits), cfg.cp * cfg.pg - 1 - cfg.hp * cfg.pg), cfg.cp * cfg.pg); - var loop = b.openFor(trips, .{ try b.full(score_min, .f32, stats), try b.full(0, .f32, stats), try b.full(0, .f32, &.{ a.T, a.Hkv, a.G, a.D }) }); - { - const max, const sum, const out = loop.carried; - const mask = try b.reshape(try b.indexRead(tmask, loop.iv), &.{ a.T, a.CP, a.PG }, .{}); - const bounds = try b.indexRead(tbounds, loop.iv); - const pages = try b.indexRead(tpages, loop.iv); - const k = try b.gather(k_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); - const v = try b.gather(v_cache, pages, a.P, &.{ a.T, a.CP, a.PG, a.Hkv, a.D }, .{ .valid_length = bounds }); - const merged = try mergeSoftmax(b, .{ .max = max, .sum = sum, .out = out }, try chunkSoftmax(b, cfg, q, k, v, mask)); - try loop.yield(.{ merged.max, merged.sum, merged.out }); - } - return .{ .max = loop.results[0], .sum = loop.results[1], .out = loop.results[2] }; -} - -fn hybridHead(b: *tcl.Builder, cfg: HybridCfg, a: HybridAxes, q: builder.Tensor, hpages: builder.Tensor, hmask: builder.Tensor, hbounds: builder.Tensor, keyed: builder.Tensor, k_cache: builder.Tensor, v_cache: builder.Tensor, tail: SoftmaxState) tcl.FinishError!builder.Tensor { - const head_mask = try b.reshape(hmask, &.{ a.T, a.HP, a.PG }, .{}); - const head_k = try b.gather(k_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); - const head_v = try b.gather(v_cache, hpages, a.P, &.{ a.T, a.HP, a.PG, a.Hkv, a.D }, .{ .valid_length = hbounds }); - const keys: []const tcl.Axis = &.{ a.HP, a.PG }; - var op = b.tensorOperation(.{}); - const scores = try op.commit(op.contract(q, head_k, &.{ a.Hkv, a.G, a.T, a.HP, a.PG }).mulf(cfg.scale), .{}); - op = b.tensorOperation(.{}); - const sv = op.fetch(scores, .{}); - const masked = try op.commit(op.where(op.fetch(head_mask, .{}), .ne, 0, sv, score_min), .{}); - op = b.tensorOperation(.{}); - const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf).maxf(op.fetch(tail.max, .{})), .{}); - const alpha = try expDifference(b, tail.max, max); - op = b.tensorOperation(.{}); - const carried = try op.commit(op.fetch(tail.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); - op = b.tensorOperation(.{}); - const denominator = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().reduce(keys, .addf).addf(op.fetch(carried, .{})), .{}); - op = b.tensorOperation(.{}); - const p = try op.commit(op.fetch(masked, .{}).subf(op.fetch(max, .{})).exp().divf(op.fetch(denominator, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); - op = b.tensorOperation(.{}); - const head = try op.commit(op.contract(head_v, p, &.{ a.T, a.Hkv, a.G, a.D }), .{}); - op = b.tensorOperation(.{}); - const factor = try op.commit(op.fetch(alpha, .{}).divf(op.fetch(denominator, .{})), .{}); - op = b.tensorOperation(.{}); - return op.commit(op.fetch(tail.out, .{}).mulf(op.fetch(factor, .{})).addf(op.fetch(head, .{})).mulf(op.fetch(keyed, .{})), .{ .dtype = .bf16 }); -} - -fn chunkSoftmax(b: *tcl.Builder, cfg: HybridCfg, q: builder.Tensor, k: builder.Tensor, v: builder.Tensor, mask: builder.Tensor) tcl.FinishError!SoftmaxState { - const T = q.axes[0]; - const Hkv = q.axes[1]; - const G = q.axes[2]; - const D = q.axes[3]; - const keys = mask.axes[1..]; - var op = b.tensorOperation(.{}); - const scores = try op.commit(op.contract(q, k, &.{ Hkv, G, T, keys[0], keys[1] }).mulf(cfg.scale), .{}); - op = b.tensorOperation(.{}); - const sv = op.fetch(scores, .{}); - const masked = try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, sv, score_min), .{}); - op = b.tensorOperation(.{}); - const max = try op.commit(op.fetch(masked, .{}).reduce(keys, .maxf), .{}); - const exps = try maskedExp(b, masked, max, mask); - op = b.tensorOperation(.{}); - const sum = try op.commit(op.fetch(exps, .{}).reduce(keys, .addf), .{}); - op = b.tensorOperation(.{}); - const p = try op.commit(op.fetch(exps, .{}).mulf(1.0), .{ .dtype = .bf16 }); - op = b.tensorOperation(.{}); - const out = try op.commit(op.contract(v, p, &.{ T, Hkv, G, D }), .{}); - return .{ .max = max, .sum = sum, .out = out }; -} - -fn mergeSoftmax(b: *tcl.Builder, x: SoftmaxState, y: SoftmaxState) tcl.FinishError!SoftmaxState { - var op = b.tensorOperation(.{ .tactic = .elementwise }); - const max = try op.commit(op.fetch(x.max, .{}).maxf(op.fetch(y.max, .{})), .{}); - const alpha = try expDifference(b, x.max, max); - const beta = try expDifference(b, y.max, max); - op = b.tensorOperation(.{}); - const xs = try op.commit(op.fetch(x.sum, .{}).mulf(op.fetch(alpha, .{})), .{}); - op = b.tensorOperation(.{}); - const sum = try op.commit(op.fetch(y.sum, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xs, .{})), .{}); - op = b.tensorOperation(.{}); - const xo = try op.commit(op.fetch(x.out, .{}).mulf(op.fetch(alpha, .{})), .{}); - op = b.tensorOperation(.{}); - const out = try op.commit(op.fetch(y.out, .{}).mulf(op.fetch(beta, .{})).addf(op.fetch(xo, .{})), .{}); - return .{ .max = max, .sum = sum, .out = out }; -} - -const Cfg = struct { - tokens: i64, - sequences: i64, - /// Query blocks and query rows per block. - b: i64, - i: i64, - /// Keys per block (padded to whole chunks) and keys per chunk. - a: i64, - chunk: i64, - hkv: i64, - g: i64, - d: i64, - pages: i64, - pg: i64, - scale: f32, - decode: bool, - use_sink: bool = false, - is_causal: bool = true, - window: i32 = -1, -}; - -/// Prefill bounds the tactic shapes searched (fewer for narrow heads): the compute-bound hint compiles too slowly. -fn configure(b: *tcl.Builder, cfg: Cfg) void { - const shapes: i64 = if (cfg.hkv * cfg.g * cfg.d >= 1024) 16 else 8; - if (cfg.decode) b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .enable_tactic_pruning = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .propagate_sparse_axis_from_op = "Gather", - .tactic_hint = .{ .ForLlmModelIOBound = 200 }, - .num_transaction_simulation_per_pe = 1024, - .dma_preference = 0.8, - }) else b.compilerConfig(.{ - .enable_einsum_fusion = true, - .padding_policy = .{ .Small = 1.1 }, - .instruction_mem_budget = 0xB0000, - .reshape_einsum_mode = .{ .Reshape = .{ .permute = false } }, - .tensor_unit_bridge_threshold_in_page = 12, - .allow_external_operators = false, - .scheduler_beam_search = true, - .allow_reduce_by_ve_cluster_chip_reduce = false, - .allow_reduce_by_ve_cluster_chip_reduce_base_population = false, - .tactic_hint = .{ .DefaultWithMaxNumUniqueShapes = shapes }, - .dma_preference = 1.2, - }); -} - -const Axes = struct { - T: tcl.Axis, - S: tcl.Axis, - O: tcl.Axis, - AP: tcl.Axis, - B: tcl.Axis, - I: tcl.Axis, - Hkv: tcl.Axis, - G: tcl.Axis, - D: tcl.Axis, - P: tcl.Axis, - PG: tcl.Axis, - - fn init(b: *tcl.Builder, cfg: Cfg) Axes { - return .{ - .T = b.axis("T", cfg.tokens), - .S = b.axis("S", cfg.sequences), - .O = b.axis("O", cfg.sequences + 1), - .AP = b.axis("AP", @divExact(cfg.a, cfg.pg)), - .B = b.axis("B", cfg.b), - .I = b.axis("I", cfg.i), - .Hkv = b.axis("Hkv", cfg.hkv), - .G = b.axis("G", cfg.g), - .D = b.axis("D", cfg.d), - .P = b.axis("P", cfg.pages), - .PG = b.axis("PG", cfg.pg), - }; - } -}; - -const NativeInputs = struct { - q: builder.Tensor, - pages: builder.Tensor, - seq_lens: builder.Tensor, - query_offsets: builder.Tensor, - k_cache: builder.Tensor, - v_cache: builder.Tensor, - sink: builder.Tensor, -}; - -fn declareInputs(b: *tcl.Builder, a: Axes) tcl.FinishError!NativeInputs { - const t = try b.declareArgs(.{ - .q = .{ .dtype = .bf16, .axes = &.{ a.T, a.Hkv, a.G, a.D } }, - .pages = .{ .dtype = .i32, .axes = &.{ a.S, a.AP } }, - .seq_lens = .{ .dtype = .i32, .axes = &.{a.S} }, - .query_offsets = .{ .dtype = .i32, .axes = &.{a.O} }, - .k_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, - .v_cache = .{ .dtype = .bf16, .axes = &.{ a.P, a.PG, a.Hkv, a.D } }, - .sink = .{ .dtype = .f32, .axes = &.{ a.Hkv, a.G } }, - }); - return .{ .q = t.q, .pages = t.pages, .seq_lens = t.seq_lens, .query_offsets = t.query_offsets, .k_cache = t.k_cache, .v_cache = t.v_cache, .sink = t.sink }; -} - -const Prepared = struct { - q: builder.Tensor, - /// Page table rows: one per block for decode, one per sequence for prefill (blocks select theirs with `pick`). - pages: builder.Tensor, - pick: ?builder.Tensor, - qpos: builder.Tensor, - kvlen: builder.Tensor, - row: builder.Tensor, - active: builder.Tensor, - /// Per page row: the first query position and the keys. - first: builder.Tensor, - keys: builder.Tensor, - - /// WORKAROUND (TCC 2026.5.0b0): TCC has no tactic for selecting whole rows of 8192 pages, so flash - /// attention selects per chunk. Repro: repro_metadata_einsum_by_ve_no_tactic.py. Remove once such - /// products compile. - fn blockPages(self: Prepared, b: *tcl.Builder, a: Axes, pages: builder.Tensor) tcl.FinishError!builder.Tensor { - const pick = self.pick orelse return pages; - return selectMetadata(b, pages, pick, a.S); - } -}; - -fn countBefore(b: *tcl.Builder, ends: builder.Tensor, positions: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const distance = try op.commit(op.fetch(ends, .{}).subi(op.fetch(positions, .{})), .{}); - op = b.tensorOperation(.{}); - const passed = try op.commit(op.where(distance, .le, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - return op.commit(op.fetch(passed, .{}).reduce(&.{axis}, .addi), .{}); -} - -fn selector(b: *tcl.Builder, indices: builder.Tensor, axis: tcl.Axis, count: i64) tcl.FinishError!builder.Tensor { - const source = try b.arange(count, .i32, &.{axis}, .{}); - var op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const distance = try op.commit(op.fetch(indices, .{}).subi(op.fetch(source, .{})), .{}); - op = b.tensorOperation(.{}); - return op.commit(op.where(distance, .eq, 0, 1, 0), .{}); -} - -// Keep metadata exact in I32. WORKAROUND (TCC 2026.5.0b0): no native tactics for -// scalar gathers or four-byte-offset slices. Repro: repro_scalar_gather_no_axis_to_pad.py, -// repro_i32_slice_odd_offset.py. Remove once both compile. -fn selectMetadata(b: *tcl.Builder, values: builder.Tensor, selected: builder.Tensor, axis: tcl.Axis) tcl.FinishError!builder.Tensor { - const op = b.tensorOperation(.{ .tactic = if (values.axes.len > 1) .einsum_by_ve else .reduce_by_ve }); - return op.commit(op.fetch(selected, .{}).muli(op.fetch(values, .{})).reduce(&.{axis}, .addi), .{}); -} - -/// Reads adjacent I32 offsets inside the native unit: a separate `offsets[1:]` argument would -/// need alignment staging at every invocation. -fn prepareQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, t: NativeInputs) tcl.FinishError!Prepared { - const start_indices = try b.arange(cfg.sequences, .i32, &.{a.S}, .{}); - const end_indices = try b.arange(cfg.sequences + 1, .i32, &.{a.S}, .{ .start = 1 }); - const starts = try selectMetadata(b, t.query_offsets, try selector(b, start_indices, a.O, cfg.sequences + 1), a.O); - const ends = try selectMetadata(b, t.query_offsets, try selector(b, end_indices, a.O, cfg.sequences + 1), a.O); - var op = b.tensorOperation(.{}); - const qlen = try op.commit(op.fetch(ends, .{}).subi(op.fetch(starts, .{})), .{}); - const tokens = try b.arange(cfg.tokens, .i32, &.{a.T}, .{}); - const seq_of_token = try countBefore(b, ends, tokens, a.S); - // Padding tokens follow all sequence ends; this vector avoids a symbolic scalar reduction in a vector operation. - op = b.tensorOperation(.{}); - const token_tail = try op.commit(op.fetch(seq_of_token, .{}).subi(cfg.sequences), .{}); - op = b.tensorOperation(.{}); - const active = try op.commit(op.where(token_tail, .lt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const token_seq = try op.commit(op.fetch(seq_of_token, .{}).binary(.mini, cfg.sequences - 1), .{}); - if (cfg.decode) { - op = b.tensorOperation(.{}); - const live = try op.commit(op.where(qlen, .gt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const lengths = try op.commit(op.fetch(t.seq_lens, .{}).muli(op.fetch(live, .{})), .{}); - op = b.tensorOperation(.{}); - const qpos = try op.commit(op.fetch(lengths, .{}).subi(1), .{}); - op = b.tensorOperation(.{}); - const safe_start = try op.commit(op.fetch(starts, .{}).binary(.mini, cfg.tokens - 1), .{}); - const q = if (cfg.tokens == 1) t.q else try b.gather(t.q, safe_start, a.T, &.{ a.S, a.Hkv, a.G, a.D }, .{}); - return .{ - .q = try b.reshape(q, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}), - .pages = try b.reshape(t.pages, &.{ a.B, a.AP }, .{}), - .pick = null, - .qpos = try b.reshape(qpos, &.{ a.B, a.I }, .{}), - .kvlen = try b.reshape(lengths, &.{a.B}, .{}), - .row = token_seq, - .active = active, - .first = try b.reshape(qpos, &.{a.B}, .{}), - .keys = try b.reshape(lengths, &.{a.B}, .{}), - }; - } - - const shift = std.math.log2_int(u64, @intCast(cfg.i)); - op = b.tensorOperation(.{}); - const blocks = try op.commit(op.fetch(qlen, .{}).addi(cfg.i - 1).binary(.shr_arith, shift), .{}); - op = b.tensorOperation(.{}); - const block_end = try op.commit(op.fetch(blocks, .{}).cumsum(&.{a.S}), .{}); - op = b.tensorOperation(.{}); - const block_start = try op.commit(op.fetch(block_end, .{}).subi(op.fetch(blocks, .{})), .{}); - const block_ids = try b.arange(cfg.b, .i32, &.{a.B}, .{}); - const seq_of_block = try countBefore(b, block_end, block_ids, a.S); - op = b.tensorOperation(.{}); - const beyond = try op.commit(op.fetch(seq_of_block, .{}).subi(cfg.sequences), .{}); - op = b.tensorOperation(.{}); - const live_block = try op.commit(op.where(beyond, .lt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const seq = try op.commit(op.fetch(seq_of_block, .{}).binary(.mini, cfg.sequences - 1), .{}); - const pick = try selector(b, seq, a.S, cfg.sequences); - const block_begin = try selectMetadata(b, block_start, pick, a.S); - const begin = try selectMetadata(b, starts, pick, a.S); - const query_length = try selectMetadata(b, qlen, pick, a.S); - const key_length = try selectMetadata(b, t.seq_lens, pick, a.S); - const local_rows = try b.arange(cfg.i, .i32, &.{a.I}, .{}); - op = b.tensorOperation(.{}); - const first_row = try op.commit(op.fetch(block_ids, .{}).subi(op.fetch(block_begin, .{})).muli(cfg.i), .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const qi = try op.commit(op.fetch(first_row, .{}).addi(op.fetch(local_rows, .{})), .{}); - op = b.tensorOperation(.{}); - const remaining = try op.commit(op.fetch(qi, .{}).subi(op.fetch(query_length, .{})), .{}); - op = b.tensorOperation(.{}); - const inside = try op.commit(op.where(remaining, .lt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const live = try op.commit(op.fetch(inside, .{}).muli(op.fetch(live_block, .{})), .{}); - op = b.tensorOperation(.{}); - const qidx = try op.commit(op.fetch(qi, .{}).addi(op.fetch(begin, .{})).muli(op.fetch(live, .{})), .{}); - const q = try b.gather(t.q, qidx, a.T, &.{ a.B, a.I, a.Hkv, a.G, a.D }, .{}); - op = b.tensorOperation(.{}); - const pos = try op.commit(op.fetch(qi, .{}).addi(op.fetch(key_length, .{})).subi(op.fetch(query_length, .{})), .{}); - op = b.tensorOperation(.{}); - const qpos = try op.commit(op.where(live, .ne, 0, op.fetch(pos, .{}), -1), .{}); - op = b.tensorOperation(.{}); - const kvlen = try op.commit(op.fetch(key_length, .{}).muli(op.fetch(live_block, .{})), .{}); - - const token_pick = try selector(b, token_seq, a.S, cfg.sequences); - const token_begin = try selectMetadata(b, starts, token_pick, a.S); - const token_block = try selectMetadata(b, block_start, token_pick, a.S); - op = b.tensorOperation(.{}); - const row = try op.commit(op.fetch(token_block, .{}).muli(cfg.i).addi(op.fetch(tokens, .{})).subi(op.fetch(token_begin, .{})).binary(.maxi, 0).binary(.mini, cfg.b * cfg.i - 1), .{}); - op = b.tensorOperation(.{}); - const first = try op.commit(op.fetch(t.seq_lens, .{}).subi(op.fetch(qlen, .{})), .{}); - return .{ .q = q, .pages = t.pages, .pick = pick, .qpos = qpos, .kvlen = kvlen, .row = row, .active = active, .first = first, .keys = t.seq_lens }; -} - -fn restoreQueries(b: *tcl.Builder, cfg: Cfg, a: Axes, prepared: Prepared, output: builder.Tensor) tcl.FinishError!builder.Tensor { - if (cfg.decode and cfg.tokens == 1) return b.reshape(output, &.{ a.T, a.Hkv, a.G, a.D }, .{}); - const R = b.axis("R", cfg.b * cfg.i); - const rows = try b.reshape(output, &.{ R, a.Hkv, a.G, a.D }, .{}); - const ordered = try b.gather(rows, prepared.row, R, &.{ a.T, a.Hkv, a.G, a.D }, .{}); - const op = b.tensorOperation(.{ .tactic = .elementwise }); - // Establish the full data layout before broadcasting the I32 row mask. - const values = op.fetch(ordered, .{}); - return op.commit(op.where(op.fetch(prepared.active, .{ .broadcast_to = ordered.axes }), .ne, 0, values, 0.0), .{ .dtype = .bf16 }); -} - -const score_min = -std.math.floatMax(f32); - -fn validIndices(b: *tcl.Builder, indices: builder.Tensor, count: i64) tcl.FinishError!struct { safe: builder.Tensor, valid: builder.Tensor } { - var op = b.tensorOperation(.{}); - const safe = try op.commit(op.fetch(indices, .{}).binary(.maxi, 0).binary(.mini, count - 1), .{}); - op = b.tensorOperation(.{}); - const nonnegative = try op.commit(op.fetch(indices, .{}).addi(1).binary(.maxi, 0).binary(.mini, 1), .{}); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const valid = try op.commit(op.fetch(indices, .{}).muli(-1).addi(count).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(nonnegative, .{})), .{}); - return .{ .safe = safe, .valid = valid }; -} - -/// Validity is per sequence, not per query, so K/V are not replicated per row of a prefill block. -fn cacheMask(b: *tcl.Builder, valid_pages: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor) tcl.FinishError!builder.Tensor { - var op = b.tensorOperation(.{}); - const pos = try op.commit(op.fetch(position, .{ .broadcast_to = valid_pages.axes }), .{}); - op = b.tensorOperation(.{ .tactic = .elementwise }); - return op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(valid_pages, .{})), .{}); -} - -/// A select, not a multiply by zero: NaN/Inf in unwritten slots must not reach the contraction. -fn maskCacheValues(b: *tcl.Builder, values: builder.Tensor, valid: builder.Tensor) tcl.FinishError!builder.Tensor { - const op = b.tensorOperation(.{ .tactic = .elementwise }); - const data = op.fetch(values, .{}); - return op.commit(op.where(op.fetch(valid, .{ .broadcast_to = values.axes }), .ne, 0, data, 0.0), .{ .dtype = .bf16 }); -} - -fn attentionMask(b: *tcl.Builder, cfg: Cfg, qpos: builder.Tensor, kvlen: builder.Tensor, position: builder.Tensor, valid_keys: builder.Tensor) tcl.FinishError!builder.Tensor { - const axes = &.{ qpos.axes[0], qpos.axes[1], position.axes[position.axes.len - 1] }; - const read: builder.TensorOperation.FetchOptions = .{ .broadcast_to = axes }; - // Elementwise operations require their first read to cover the output - // axes, and each VE instruction must consume the current accumulator. - var op = b.tensorOperation(.{}); - const pos = try op.commit(op.fetch(position, read), .{}); - op = b.tensorOperation(.{}); - const query = try op.commit(op.fetch(qpos, read), .{}); - op = b.tensorOperation(.{}); - var mask = try op.commit(op.fetch(valid_keys, read), .{}); - op = b.tensorOperation(.{ .tactic = .elementwise }); - mask = try op.commit(op.fetch(pos, .{}).muli(-1).addi(op.fetch(kvlen, .{})).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); - op = b.tensorOperation(.{ .tactic = .elementwise }); - mask = try op.commit(op.fetch(query, .{}).addi(1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); - if (cfg.is_causal) { - op = b.tensorOperation(.{ .tactic = .elementwise }); - mask = try op.commit(op.fetch(query, .{}).subi(op.fetch(pos, .{})).addi(1).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); - } - if (cfg.window > 0) { - op = b.tensorOperation(.{ .tactic = .elementwise }); - mask = try op.commit(op.fetch(pos, .{}).subi(op.fetch(query, .{})).addi(cfg.window).binary(.maxi, 0).binary(.mini, 1).muli(op.fetch(mask, .{})), .{}); - } - return mask; -} - -fn maskedScores(b: *tcl.Builder, a: Axes, cfg: Cfg, q: builder.Tensor, k: builder.Tensor, mask: builder.Tensor, keys: tcl.Axis) tcl.FinishError!builder.Tensor { - const scores_axes = &.{ a.Hkv, a.G, a.B, a.I, keys }; - var op = b.tensorOperation(.{}); - const scores = try op.commit(op.contract(q, k, scores_axes).mulf(cfg.scale), .{}); - op = b.tensorOperation(.{}); - const s = op.fetch(scores, .{}); - return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, s, score_min), .{}); -} - -fn maskedExp(b: *tcl.Builder, scores: builder.Tensor, maximum: builder.Tensor, mask: builder.Tensor) tcl.FinishError!builder.Tensor { - const op = b.tensorOperation(.{}); - const e = op.fetch(scores, .{}).subf(op.fetch(maximum, .{})).exp(); - // A sentinel minus itself gives exp(0), so masking only the logits fails for empty rows and fully masked chunks. - return op.commit(op.where(op.fetch(mask, .{}), .ne, 0, e, 0.0), .{}); -} - -fn expDifference(b: *tcl.Builder, x: builder.Tensor, maximum: builder.Tensor) tcl.FinishError!builder.Tensor { - const op = b.tensorOperation(.{ .tactic = .elementwise }); - const m = op.fetch(maximum, .{}); - return op.commit(op.fetch(x, .{}).subf(m).exp(), .{}); -} - -fn safeDenominator(b: *tcl.Builder, sum: builder.Tensor) tcl.FinishError!builder.Tensor { - const op = b.tensorOperation(.{}); - const s = op.fetch(sum, .{}); - return op.commit(op.where(s, .gt, 0, s, 1.0), .{}); -} - -const OneShot = tcl.Kernel(Cfg, .{ - .name = "attention", - .inputs = &.{ "q", "pages", "seq_lens", "query_offsets", "k_cache", "v_cache", "sink" }, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, cfg); - const a: Axes = .init(b, cfg); - const A = b.axis("A", cfg.a); - const Nb = b.axis("Nb", cfg.pages * cfg.pg); - const t = try declareInputs(b, a); - const prepared = try prepareQueries(b, cfg, a, t); - const kc = try b.reshape(t.k_cache, &.{ Nb, a.Hkv, a.D }, .{}); - const vc = try b.reshape(t.v_cache, &.{ Nb, a.Hkv, a.D }, .{}); - const page_indices = try validIndices(b, try prepared.blockPages(b, a, prepared.pages), cfg.pages); - const within_page = try b.arange(cfg.pg, .i32, &.{a.PG}, .{}); - var slot_op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const flat_slots = try slot_op.commit(slot_op.fetch(page_indices.safe, .{}).muli(cfg.pg).addi(slot_op.fetch(within_page, .{})), .{}); - const expanded_slots = try b.reshape(flat_slots, &.{ a.B, A }, .{}); - slot_op = b.tensorOperation(.{}); - const valid = try slot_op.commit(slot_op.fetch(page_indices.valid, .{ .broadcast_to = &.{ a.B, a.AP, a.PG } }), .{}); - const valid_keys = try b.reshape(valid, &.{ a.B, A }, .{}); - slot_op = b.tensorOperation(.{}); - const slots = try slot_op.commit(slot_op.where(valid_keys, .ne, 0, slot_op.fetch(expanded_slots, .{}), 0), .{}); - const valid_length = if (cfg.decode) prepared.kvlen else null; - const position = try b.arange(cfg.a, .i32, &.{A}, .{}); - const valid_cache = try cacheMask(b, valid_keys, prepared.kvlen, position); - const k = try maskCacheValues(b, try b.gather(kc, slots, Nb, &.{ a.B, A, a.Hkv, a.D }, .{ .valid_length = valid_length }), valid_cache); - const v = try maskCacheValues(b, try b.gather(vc, slots, Nb, &.{ a.B, A, a.Hkv, a.D }, .{ .valid_length = valid_length }), valid_cache); - const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); - const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, A); - var op = b.tensorOperation(.{}); - const p = if (cfg.decode) compact: { - const exp_sink = if (cfg.use_sink) sink: { - const maximum = try op.commit(op.fetch(masked, .{}).reduce(&.{A}, .maxf).maxf(op.fetch(t.sink, .{})), .{}); - break :sink try expDifference(b, t.sink, maximum); - } else null; - op = b.tensorOperation(.{}); - const scores = op.fetch(masked, .{}); - var maximum = scores.reduce(&.{A}, .maxf); - if (cfg.use_sink) maximum = maximum.maxf(op.fetch(t.sink, .{})); - const e = scores.subf(maximum).exp(); - var sum = e.reduce(&.{A}, .addf); - if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); - const probabilities = try op.commit(e.divf(sum), .{ .dtype = .bf16 }); - // Empty rows have uniform probabilities from finite sentinels: clear masked ones before the value contraction. - op = b.tensorOperation(.{}); - break :compact try op.commit(op.where(op.fetch(mask, .{}), .ne, 0, op.fetch(probabilities, .{}), 0.0), .{ .dtype = .bf16 }); - } else tiled: { - // Separate reductions schedule better for multi-query blocks with the current compiler. - var m = op.fetch(masked, .{}).reduce(&.{A}, .maxf); - if (cfg.use_sink) m = m.maxf(op.fetch(t.sink, .{})); - const maximum = try op.commit(m, .{}); - const e = try maskedExp(b, masked, maximum, mask); - const exp_sink = if (cfg.use_sink) try expDifference(b, t.sink, maximum) else null; - op = b.tensorOperation(.{}); - var sum = op.fetch(e, .{}).reduce(&.{A}, .addf); - if (exp_sink) |s| sum = sum.addf(op.fetch(s, .{})); - const denominator = try safeDenominator(b, try op.commit(sum, .{})); - op = b.tensorOperation(.{}); - break :tiled try op.commit(op.fetch(e, .{}).divf(op.fetch(denominator, .{})), .{ .dtype = .bf16 }); - }; - op = b.tensorOperation(.{}); - const output = try op.commit(op.contract(v, p, &.{ a.B, a.I, a.Hkv, a.G, a.D }), .{ .dtype = .bf16 }); - b.ret(&.{try restoreQueries(b, cfg, a, prepared, output)}); - } - }.run, -}); - -const Flash = tcl.Kernel(Cfg, .{ - .name = "flash_attention", - .inputs = &.{ "q", "pages", "seq_lens", "query_offsets", "k_cache", "v_cache", "sink" }, - .outputs = &.{"out"}, - .run = struct { - fn run(b: *tcl.Builder, cfg: Cfg) tcl.FinishError!void { - configure(b, cfg); - const a: Axes = .init(b, cfg); - const C = b.axis("C", @divExact(cfg.a, cfg.chunk)); - const Ck = b.axis("Ck", cfg.chunk); - const Cp = b.axis("Cp", @divExact(cfg.chunk, cfg.pg)); - const t = try declareInputs(b, a); - const prepared = try prepareQueries(b, cfg, a, t); - const rows = prepared.pages.axes[0]; - const by_row = try b.reshape(prepared.pages, &.{ rows, C, Cp }, .{}); - const page_op = b.tensorOperation(.{}); - const pages = try page_op.commit(page_op.fetch(by_row, .{}), .{ .axes = &.{ C, rows, Cp } }); - const stats = &.{ a.Hkv, a.G, a.B, a.I }; - const out_axes = &.{ a.B, a.I, a.Hkv, a.G, a.D }; - var op = b.tensorOperation(.{}); - const shift = std.math.log2_int(u64, @intCast(cfg.chunk)); - // With a window, each page row's loop starts at the chunk holding its first visible key. TCL divides - // by shifts, so chunks that are not a power of two run every chunk and the mask applies the window. - const window = if (cfg.window > 0 and std.math.isPowerOfTwo(cfg.chunk)) blk: { - op = b.tensorOperation(.{}); - const start = try op.commit(op.fetch(prepared.first, .{}).subi(cfg.window - 1).binary(.maxi, 0).binary(.shr_arith, shift), .{}); - op = b.tensorOperation(.{}); - const count = try op.commit(op.fetch(prepared.keys, .{}).addi(cfg.chunk - 1).binary(.shr_arith, shift).subi(op.fetch(start, .{})), .{}); - // WORKAROUND (TCC 2026.5.0b0): TCC reads the loop index only as an index, so each trip's chunk per - // row is built before the loop. Repro: repro_loop_index_as_data.py. Remove once the loop index - // compiles as data. - const trip = try b.arange(C.size, .i32, &.{C}, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const chunks = try op.commit(op.fetch(trip, .{}).addi(op.fetch(start, .{})), .{ .axes = &.{ C, rows } }); - break :blk .{ .chunks = chunks, .trips = try b.reduceMaxI32(count) }; - } else null; - const n = if (window) |w| w.trips else try b.symExpr(.div, try b.symExpr(.add, try b.reduceMaxI32(prepared.kvlen), cfg.chunk - 1), cfg.chunk); - var loop = b.openFor(n, .{ try b.full(0, .f32, stats), try b.full(0, .f32, out_axes), try b.full(score_min, .f32, stats) }); - { - const sum_acc, const qkv_acc, const max_acc = loop.carried; - const local_pos = try b.arange(cfg.chunk, .i32, &.{Ck}, .{}); - const local_pages = try b.arange(@divExact(cfg.chunk, cfg.pg), .i32, &.{Cp}, .{}); - var row_pages: builder.Tensor = undefined; - var position: builder.Tensor = undefined; - var first_keys: builder.Tensor = undefined; - if (window) |w| { - const chunk_index = try b.indexRead(w.chunks, loop.iv); - op = b.tensorOperation(.{}); - const read_index = try op.commit(op.fetch(chunk_index, .{}).binary(.mini, @divExact(cfg.a, cfg.chunk) - 1), .{}); - // WORKAROUND (TCC 2026.5.0b0): a batched gather of the chunk does not lower - // for some shapes. Repro: repro_batched_gather_all_tactics_failed.py. Remove - // once batched gathers of several rows compile. - row_pages = try selectMetadata(b, by_row, try selector(b, read_index, C, C.size), C); - op = b.tensorOperation(.{}); - const row_offset = try op.commit(op.fetch(chunk_index, .{}).muli(cfg.chunk), .{}); - const offset = if (prepared.pick) |pick| try selectMetadata(b, row_offset, pick, a.S) else row_offset; - op = b.tensorOperation(.{}); - position = try op.commit(op.fetch(local_pos, .{ .broadcast_to = &.{ a.B, Ck } }).addi(op.fetch(offset, .{})), .{}); - op = b.tensorOperation(.{}); - first_keys = try op.commit(op.fetch(local_pages, .{ .broadcast_to = &.{ a.B, Cp } }).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); - } else { - row_pages = try b.indexRead(pages, loop.iv); - const offset = try b.symExpr(.mul, loop.iv, cfg.chunk); - op = b.tensorOperation(.{}); - position = try op.commit(op.fetch(local_pos, .{}).addi(op.fetch(offset, .{})), .{}); - op = b.tensorOperation(.{}); - first_keys = try op.commit(op.fetch(local_pages, .{}).muli(cfg.pg).addi(op.fetch(offset, .{})), .{}); - } - const indices = try validIndices(b, try prepared.blockPages(b, a, row_pages), cfg.pages); - const idx = indices.safe; - op = b.tensorOperation(.{}); - const valid_pages = try op.commit(op.fetch(indices.valid, .{ .broadcast_to = &.{ a.B, Cp, a.PG } }), .{}); - const valid_keys = try b.reshape(valid_pages, &.{ a.B, Ck }, .{}); - op = b.tensorOperation(.{ .tactic = .einsum_by_ve }); - const remaining = try op.commit(op.fetch(prepared.kvlen, .{}).subi(op.fetch(first_keys, .{})), .{}); - op = b.tensorOperation(.{}); - const live_pages = try op.commit(op.where(remaining, .gt, 0, 1, 0), .{}); - op = b.tensorOperation(.{}); - const cvl = try op.commit(op.fetch(live_pages, .{}).reduce(&.{Cp}, .addi), .{}); - const mask = try attentionMask(b, cfg, prepared.qpos, prepared.kvlen, position, valid_keys); - const k = try b.reshape(try b.gather(t.k_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); - const v = try b.reshape(try b.gather(t.v_cache, idx, a.P, &.{ a.B, Cp, a.PG, a.Hkv, a.D }, .{ .valid_length = cvl }), &.{ a.B, Ck, a.Hkv, a.D }, .{}); - const masked = try maskedScores(b, a, cfg, prepared.q, k, mask, Ck); - op = b.tensorOperation(.{}); - const max_cur = try op.commit(op.fetch(masked, .{}).reduce(&.{Ck}, .maxf), .{}); - op = b.tensorOperation(.{ .tactic = .elementwise }); - const max_new = try op.commit(op.fetch(max_cur, .{}).maxf(op.fetch(max_acc, .{})), .{}); - const rescale = try expDifference(b, max_acc, max_new); - const e = try maskedExp(b, masked, max_new, mask); - op = b.tensorOperation(.{}); - const sum_cur = try op.commit(op.fetch(e, .{}).reduce(&.{Ck}, .addf), .{}); - op = b.tensorOperation(.{}); - const sum_new = try op.commit(op.fetch(sum_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(sum_cur, .{})), .{}); - op = b.tensorOperation(.{}); - // Materialize the narrowing conversion through the vector engine. - const p = try op.commit(op.fetch(e, .{}).mulf(1.0), .{ .dtype = .bf16 }); - op = b.tensorOperation(.{}); - const qkv_cur = try op.commit(op.contract(p, v, out_axes), .{}); - op = b.tensorOperation(.{}); - const qkv_new = try op.commit(op.fetch(qkv_acc, .{}).mulf(op.fetch(rescale, .{})).addf(op.fetch(qkv_cur, .{})), .{}); - try loop.yield(.{ sum_new, qkv_new, max_new }); - } - - var sum = loop.results[0]; - var numerator = loop.results[1]; - if (cfg.use_sink) { - op = b.tensorOperation(.{}); - const maximum = try op.commit(op.fetch(loop.results[2], .{}).maxf(op.fetch(t.sink, .{})), .{}); - const alpha = try expDifference(b, loop.results[2], maximum); - const exp_sink = try expDifference(b, t.sink, maximum); - op = b.tensorOperation(.{}); - sum = try op.commit(op.fetch(sum, .{}).mulf(op.fetch(alpha, .{})).addf(op.fetch(exp_sink, .{})), .{}); - op = b.tensorOperation(.{}); - numerator = try op.commit(op.fetch(numerator, .{}).mulf(op.fetch(alpha, .{})), .{}); - } - const denominator = try safeDenominator(b, sum); - op = b.tensorOperation(.{}); - const output = try op.commit(op.fetch(numerator, .{}).divf(op.fetch(denominator, .{})), .{ .dtype = .bf16 }); - b.ret(&.{try restoreQueries(b, cfg, a, prepared, output)}); - } - }.run, -}); - -test { - std.testing.refAllDecls(attention_test); -} - -const attention_test = struct { - pub const Options = struct { - is_causal: bool = true, - sliding_window: i32 = -1, - scale: ?f32 = null, - sink: ?[]const f32 = null, - }; - - pub fn reference(allocator: std.mem.Allocator, q: zml.Slice, k: zml.Slice, v: zml.Slice, table: []const i32, lengths: []const i32, starts: []const i32, opts: Options) !zml.Slice { - const bf16 = zml.floats.BFloat16; - const hkv: usize = @intCast(q.shape.dim(.hkv)); - const groups: usize = @intCast(q.shape.dim(.hg)); - const d: usize = @intCast(q.shape.dim(.hd)); - const page: usize = @intCast(k.shape.dim(.k_chunk)); - const pages: usize = @intCast(k.shape.dim(.page)); - const max_pages = table.len / lengths.len; - const scale = opts.scale orelse 1.0 / @sqrt(@as(f32, @floatFromInt(d))); - const queries = q.constItems(bf16); - const keys = k.constItems(bf16); - const values = v.constItems(bf16); - const result = try zml.Slice.alloc(allocator, q.shape.withDtype(.f32)); - errdefer result.free(allocator); - const out = result.items(f32); - @memset(out, 0); - const scores = try allocator.alloc(f32, max_pages * page); - defer allocator.free(scores); - - for (lengths, 0..) |length, seq| { - const qlen = starts[seq + 1] - starts[seq]; - for (@intCast(starts[seq])..@intCast(starts[seq + 1])) |token| { - const qpos = length - qlen + @as(i32, @intCast(token)) - starts[seq]; - if (qpos < 0) continue; - for (0..hkv) |h| for (0..groups) |g| { - const offset = ((token * hkv + h) * groups + g) * d; - const sink = if (opts.sink) |s| s[h * groups + g] else -std.math.inf(f32); - var maximum = sink; - @memset(scores, -std.math.inf(f32)); - for (0..@intCast(length)) |key| { - const pos: i32 = @intCast(key); - if (opts.is_causal and pos > qpos) continue; - if (opts.sliding_window > 0 and qpos - pos >= opts.sliding_window) continue; - const physical = table[seq * max_pages + key / page]; - if (physical < 0 or physical >= pages) continue; - const kv_offset = ((@as(usize, @intCast(physical)) * page + key % page) * hkv + h) * d; - var dot: f32 = 0; - for (0..d) |i| dot += queries[offset + i].toF32() * keys[kv_offset + i].toF32(); - scores[key] = dot * scale; - maximum = @max(maximum, scores[key]); - } - if (maximum == -std.math.inf(f32)) continue; - var denominator = @exp(sink - maximum); - for (scores[0..@intCast(length)]) |score| denominator += @exp(score - maximum); - for (0..@intCast(length)) |key| { - if (scores[key] == -std.math.inf(f32)) continue; - const physical: usize = @intCast(table[seq * max_pages + key / page]); - const kv_offset = ((physical * page + key % page) * hkv + h) * d; - const probability = @exp(scores[key] - maximum) / denominator; - for (0..d) |i| out[offset + i] += probability * values[kv_offset + i].toF32(); - } - }; - } - } - return result; - } - - test "paged attention host reference scale and sink normalization" { - const bf16 = zml.floats.BFloat16; - const queries: [3]bf16 = @splat(.fromF32(1)); - const keys = [_]bf16{ .fromF32(0), .fromF32(1) }; - const values = [_]bf16{ .fromF32(2), .fromF32(4) }; - const q = zml.Slice.initConst(.init(.{ .b = 3, .hkv = 1, .hg = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&queries)); - const k = zml.Slice.initConst(.init(.{ .page = 1, .k_chunk = 2, .hkv = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&keys)); - const v = zml.Slice.initConst(k.shape, std.mem.sliceAsBytes(&values)); - const cases = [_]struct { opts: Options, expected: f32 }{ - .{ .opts = .{ .is_causal = false, .scale = 0 }, .expected = 3 }, - .{ .opts = .{ .is_causal = false, .scale = @log(@as(f32, 3)) }, .expected = 3.5 }, - .{ .opts = .{ .is_causal = false, .scale = @log(@as(f32, 3)), .sink = &.{0} }, .expected = 2.8 }, - }; - for (cases) |case| { - const result = try reference(std.testing.allocator, q, k, v, &.{0}, &.{2}, &.{ 0, 2 }, case.opts); - defer result.free(std.testing.allocator); - for (result.constItems(f32)[0..2]) |value| try std.testing.expectApproxEqAbs(case.expected, value, 1e-6); - try std.testing.expectEqual(@as(f32, 0), result.constItems(f32)[2]); - } - } - - pub fn expectClose(expected: zml.Slice, actual: zml.Slice) !void { - try std.testing.expectEqualSlices(i64, expected.shape.dims(), actual.shape.dims()); - for (expected.constItems(f32), 0..) |want, i| { - const got = floatAt(actual, i); - try std.testing.expect(std.math.isFinite(got)); - std.testing.expectApproxEqAbs(want, got, 3e-2 + 3e-2 * @abs(want)) catch |err| { - std.log.err("paged attention mismatch at element {d}: expected {d}, got {d}", .{ i, want, got }); - return err; - }; - } - } - - fn floatAt(slice: zml.Slice, index: usize) f32 { - return switch (slice.shape.dtype()) { - .bf16 => slice.constItems(zml.floats.BFloat16)[index].toF32(), - .f16 => @floatCast(slice.constItems(f16)[index]), - .f32 => slice.constItems(f32)[index], - else => unreachable, - }; - } - - test "paged attention host reference windows sinks and padding" { - const bf16 = zml.floats.BFloat16; - const queries: [3]bf16 = @splat(.fromF32(0)); - const keys: [2]bf16 = @splat(.fromF32(0)); - const values = [_]bf16{ .fromF32(2), .fromF32(4) }; - const q = zml.Slice.initConst(.init(.{ .b = 3, .hkv = 1, .hg = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&queries)); - const k = zml.Slice.initConst(.init(.{ .page = 1, .k_chunk = 2, .hkv = 1, .hd = 1 }, .bf16), std.mem.sliceAsBytes(&keys)); - const v = zml.Slice.initConst(k.shape, std.mem.sliceAsBytes(&values)); - const cases = [_]struct { opts: Options, expected: [3]f32 }{ - .{ .opts = .{}, .expected = .{ 2, 3, 0 } }, - .{ .opts = .{ .sliding_window = 1 }, .expected = .{ 2, 4, 0 } }, - .{ .opts = .{ .is_causal = false, .sliding_window = 1 }, .expected = .{ 3, 4, 0 } }, - .{ .opts = .{ .sink = &.{0} }, .expected = .{ 1, 2, 0 } }, - .{ .opts = .{ .sink = &.{-std.math.inf(f32)} }, .expected = .{ 2, 3, 0 } }, - }; - for (cases) |case| { - const result = try reference(std.testing.allocator, q, k, v, &.{0}, &.{2}, &.{ 0, 2 }, case.opts); - defer result.free(std.testing.allocator); - try std.testing.expectEqualSlices(f32, &case.expected, result.constItems(f32)); - } - } -}; - -const TestCase = struct { - prefill: bool, - seqs: []const [2]i32, - padding: i64 = 0, - opts: attention_test.Options = .{}, - mask_all: bool = false, - padding_page: i32 = -1, - poison: enum { none, k, v, both } = .none, - poison_value: f32 = std.math.nan(f32), -}; - -const TestDims = struct { - batch: usize = 4, - hkv: usize = 2, - g: usize = 4, - d: usize = 64, - page: usize = 16, - uniform: bool = false, - uniform_scores: bool = false, - spikes: []const usize = &.{}, - dtype: zml.DataType = .bf16, -}; - -test "tcl paged attention unit scale and dtype boundaries on furiosa" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - try runCases(16, 32, &.{.{ .prefill = true, .seqs = &@as([4][2]i32, @splat(.{ 32, 256 })), .opts = .{ .scale = 1.0 } }}, .{}); - inline for (.{ .f16, .f32 }) |dtype| { - try runCases(2, 8, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 17 }, .{ 1, 1 }, .{ 1, 3 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sink = &.{ 0, -std.math.inf(f32) } } }}, .{ .hkv = 1, .g = 2, .d = 32, .dtype = dtype }); - } -} - -test "tcl paged attention single decode row needs no rearrangement" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - try runCases(16, 32, &.{ - .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } } }, - .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 1 }, - }, .{ .batch = 1 }); - inline for (.{ 257, 320 }) |max_pages| { - try runCases(max_pages, 2 * max_pages, &.{.{ - .prefill = false, - .seqs = &.{ .{ 1, @intCast(max_pages * 16 - 1) }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, - }}, .{ .batch = 1, .hkv = 8, .g = 4, .d = 128 }); - } - inline for (.{ 5, 1025 }) |max_pages| { - try runCases(max_pages, 2 * max_pages, &.{.{ - .prefill = false, - .seqs = &.{ .{ 1, @intCast(max_pages * 48 - 1) }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, - }}, .{ .batch = 1, .hkv = 1, .g = 1, .d = 32, .page = 48 }); - } -} - -test "tcl paged attention packs inactive leading sequences and odd batches" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - try runCases(16, 32, &.{ - .{ .prefill = false, .seqs = &.{ .{ 0, 17 }, .{ 1, 63 }, .{ 0, 23 }, .{ 1, 65 } }, .padding = 2 }, - .{ .prefill = true, .seqs = &.{ .{ 0, 17 }, .{ 33, 64 }, .{ 0, 23 }, .{ 2, 3 } }, .padding = 3 }, - }, .{}); - try runCases(2, 8, &.{ - .{ .prefill = false, .seqs = &.{ .{ 0, 9 }, .{ 1, 17 }, .{ 1, 1 }, .{ 0, 0 } }, .padding = 1 }, - }, .{ .batch = 3, .hkv = 1, .g = 2, .d = 32 }); -} - -test "tcl paged attention masks inactive decode rows across all heads" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - try runCases(32, 64, &.{.{ - .prefill = false, - .seqs = &.{ .{ 1, 511 }, .{ 1, 504 }, .{ 1, 497 }, .{ 1, 490 }, .{ 0, 483 }, .{ 1, 476 }, .{ 1, 469 }, .{ 1, 462 } }, - .padding = 1, - }}, .{ .batch = 8 }); -} - -test "tcl paged attention invalid pages cannot wrap into slot zero" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - const cases = [_]TestCase{ - .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .padding_page = 1 << 28 }, - .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .padding_page = std.math.maxInt(i32) }, - }; - try runCases(2, 8, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); -} - -test "tcl paged attention emits decode and prefill kernels with and without sinks" { - inline for (.{ false, true }) |decode| { - inline for (.{ false, true }) |use_sink| { - const cfg: Cfg = .{ .tokens = if (decode) 2 else 8, .sequences = 2, .b = if (decode) 2 else 3, .i = if (decode) 1 else prefill_block, .a = 32, .chunk = 16, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .decode = decode, .use_sink = use_sink }; - const one_shot = try OneShot.emit(std.testing.allocator, cfg); - defer std.testing.allocator.free(one_shot); - const flash = try Flash.emit(std.testing.allocator, cfg); - defer std.testing.allocator.free(flash); - for ([_][]const u8{ one_shot, flash }) |ir| { - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, ir, "\"tcl.dpe\"")); - try std.testing.expect(std.mem.indexOf(u8, ir, "!tcl.logical") == null); - try std.testing.expect(std.mem.indexOf(u8, ir, "#tcl.symbol<\"I\">") == null); - - const decode = try Decode.emit(std.testing.allocator, .{ .t = 2, .a = 64, .group = 2, .hkv = 2, .g = 4, .d = 32, .pages = 8, .pg = 16, .scale = 0.125, .is_causal = true, .window = 0, .use_sink = false }); - defer std.testing.allocator.free(decode); - try std.testing.expectEqual(@as(usize, 2), std.mem.count(u8, decode, "\"tcl.dpe\"")); - try std.testing.expect(std.mem.indexOf(u8, decode, "tcl.fusible") != null); - try std.testing.expect(std.mem.indexOf(u8, decode, "propagate_sparse_axis_from_op = \"Gather\"") != null); -} - -test "tcl paged attention excludes poisoned unused cache entries on furiosa" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - const seqs = [_][2]i32{ .{ 1, 17 }, .{ 1, 31 }, .{ 0, 0 }, .{ 0, 23 } }; - try runCases(8, 32, &.{ - .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .k }, - .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .v }, - .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both }, - .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = std.math.inf(f32) }, - .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = -std.math.inf(f32) }, - .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .poison_value = 1e30 }, - // A sink selects OneShot's compact decode softmax instead of Decode. - .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .opts = .{ .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, - .{ .prefill = false, .seqs = &seqs, .padding = 2, .poison = .both, .mask_all = true, .padding_page = std.math.maxInt(i32) }, - .{ .prefill = false, .seqs = &.{ .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 }, .{ 0, 0 } }, .padding = 4, .poison = .both }, - }, .{ .hkv = 1, .g = 2, .d = 32 }); - - const mixed = [_][2]i32{ .{ 37, 65 }, .{ 0, 19 }, .{ 1, 17 }, .{ 4, 2 } }; - try runCases(8, 32, &.{ - .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .k }, - .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .v }, - .{ .prefill = true, .seqs = &mixed, .padding = 3, .poison = .both, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, - }, .{ .hkv = 1, .g = 2, .d = 32 }); - - const boundary = [_][2]i32{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 4096 }, .{ 2, 17 } }; - try runCases(257, 1028, &.{ - .{ .prefill = true, .seqs = &boundary, .padding = 3, .poison = .both }, - .{ .prefill = true, .seqs = &boundary, .padding = 3, .poison = .both, .poison_value = std.math.inf(f32), .opts = .{ .is_causal = false, .sliding_window = 33, .scale = 0.2, .sink = &.{ 0, -std.math.inf(f32) } } }, - .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 17, .sink = &.{ 0, -std.math.inf(f32) } } }, - }, .{ .hkv = 1, .g = 2, .d = 32 }); - try runCases(86, 344, &.{.{ - .prefill = true, - .seqs = &.{ .{ 3, 4033 }, .{ 1, 4032 }, .{ 0, 0 }, .{ 1, 4031 } }, - .padding = 1, - .poison = .both, - .poison_value = -std.math.inf(f32), - .opts = .{ .sliding_window = 49 }, - }}, .{ .hkv = 1, .g = 1, .d = 32, .page = 48 }); -} - -test "tcl paged attention runs on furiosa" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - inline for (.{ 16, 320 }) |max_pages| { - const cases = if (max_pages == 16) [_]TestCase{ - .{ .prefill = true, .seqs = &.{ .{ 37, 200 }, .{ 1, 77 }, .{ 20, 20 }, .{ 0, 0 } } }, - .{ .prefill = false, .seqs = &.{ .{ 1, 250 }, .{ 1, 3 }, .{ 1, 64 }, .{ 1, 129 } } }, - } else [_]TestCase{ - .{ .prefill = true, .seqs = &.{ .{ 37, 4500 }, .{ 1, 77 }, .{ 70, 70 }, .{ 0, 0 } } }, - .{ .prefill = false, .seqs = &.{ .{ 1, 5000 }, .{ 1, 3 }, .{ 1, 4097 }, .{ 1, 129 } } }, - }; - try runCases(max_pages, 2 * max_pages, &cases, .{}); - } -} - -test "tcl paged attention options and inactive queries on furiosa" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - const sinks = [_]f32{ -std.math.inf(f32), -5, 0, 2, 8, -1, 1, 20 }; - const disabled: [8]f32 = @splat(-std.math.inf(f32)); - const mixed = [4][2]i32{ .{ 37, 200 }, .{ 0, 0 }, .{ 1, 77 }, .{ 20, 20 } }; - const cases = [_]TestCase{ - .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, - .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, - .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .is_causal = false, .sliding_window = 0 } }, - .{ .prefill = true, .seqs = &mixed, .padding = 5, .opts = .{ .sliding_window = 1, .sink = &disabled } }, - .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 1 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .sink = &sinks } }, - .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 4 }, - .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true }, - .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 17 })), .mask_all = true, .opts = .{ .sink = &sinks } }, - .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 20 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .is_causal = false, .sliding_window = 17, .scale = 0.2, .sink = &sinks } }, - .{ .prefill = false, .seqs = &.{ .{ 1, 251 }, .{ 0, 73 }, .{ 1, 20 }, .{ 0, 17 } }, .padding = 2, .opts = .{ .sliding_window = 1, .sink = &disabled } }, - }; - try runCases(16, 32, &cases, .{}); -} - -test "tcl paged attention chunk boundaries on furiosa" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - const cases = [_]TestCase{ - .{ .prefill = true, .seqs = &.{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 4096 }, .{ 2, 4095 } }, .padding = 7, .opts = .{ .is_causal = false, .sliding_window = 33, .scale = 0.2, .sink = &.{ 0, 10 } } }, - .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 17, .sink = &.{ -std.math.inf(f32), 0 } } }, - .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 0, 0 })), .padding = 4 }, - .{ .prefill = false, .seqs = &@as([4][2]i32, @splat(.{ 1, 4097 })), .mask_all = true, .opts = .{ .sink = &.{ -std.math.inf(f32), 0 } } }, - }; - try runCases(257, 514, &cases, .{ .hkv = 1, .g = 2, .d = 32 }); - try runCases(256, 512, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 0, 0 }, .{ 0, 0 } }, .padding = 2 }}, .{ .hkv = 1, .g = 1, .d = 32 }); - try runCases(86, 172, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 4079 }, .{ 1, 4080 }, .{ 1, 4081 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 49 } }}, .{ .hkv = 1, .g = 1, .d = 32, .page = 48 }); -} - -test "tcl paged attention preserves constant values across chunks" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - const cases = [_]TestCase{ - .{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }, - .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1 }, - .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1, .opts = .{ .sliding_window = 1, .sink = &.{0} } }, - .{ .prefill = true, .seqs = &.{ .{ 37, 4097 }, .{ 0, 0 }, .{ 1, 77 }, .{ 2, 2 } }, .padding = 3 }, - }; - try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32, .uniform = true }); -} - -test "tcl paged attention chunked decode on furiosa" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - const cases = [_]TestCase{ - .{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }, - .{ .prefill = false, .seqs = &.{ .{ 1, 4095 }, .{ 1, 4096 }, .{ 1, 4097 }, .{ 0, 0 } }, .padding = 1 }, - }; - try runCases(257, 514, &cases, .{ .hkv = 1, .g = 1, .d = 32 }); -} - -test "tcl paged attention hybrid decode on furiosa" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - try runCases(2049, 2101, &.{ - .{ .prefill = false, .seqs = &.{.{ 1, 300 }}, .poison = .both }, - .{ .prefill = false, .seqs = &.{.{ 1, 4097 }}, .poison = .both }, - .{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both, .poison_value = std.math.inf(f32) }, - .{ .prefill = false, .seqs = &.{.{ 0, 0 }}, .padding = 1, .poison = .both }, - .{ .prefill = false, .seqs = &.{.{ 1, 17 }}, .mask_all = true, .poison = .both }, - }, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32 }); - try runCases(2049, 2101, &.{.{ .prefill = false, .seqs = &.{.{ 1, 32784 }}, .poison = .both }}, .{ .batch = 1, .hkv = 1, .g = 2, .d = 32, .uniform_scores = true, .spikes = &.{ 4095, 4096, 12287, 12288, 32783 } }); -} - -test "tcl paged attention flash decode of many tokens on furiosa" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - const seqs = [_][2]i32{ .{ 1, 32784 }, .{ 1, 4097 }, .{ 1, 1 }, .{ 0, 0 } } ++ @as([12][2]i32, @splat(.{ 1, 300 })); - try runCases(2049, 2600, &.{ - .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both }, - .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 1024 } }, - .{ .prefill = false, .seqs = &seqs, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 5000, .sink = &.{ 0, -std.math.inf(f32) } } }, - }, .{ .batch = 16, .hkv = 1, .g = 2, .d = 32 }); - try runCases(2049, 2700, &.{ - .{ .prefill = true, .seqs = &.{ .{ 37, 32784 }, .{ 5, 9000 } }, .padding = 2, .poison = .both, .opts = .{ .sliding_window = 1024 } }, - .{ .prefill = true, .seqs = &.{ .{ 70, 32784 }, .{ 0, 0 } }, .padding = 1, .poison = .both, .opts = .{ .sliding_window = 3000 } }, - }, .{ .batch = 2, .hkv = 1, .g = 2, .d = 32 }); -} - -test "tcl paged attention uniform probabilities on furiosa" { - if (zml.testing.env().target != .furiosa) return error.SkipZigTest; - try runCases(257, 514, &.{.{ .prefill = false, .seqs = &.{ .{ 1, 64 }, .{ 1, 65 }, .{ 1, 66 }, .{ 0, 0 } }, .padding = 1 }}, .{ .hkv = 1, .g = 1, .d = 32, .uniform_scores = true }); -} - -fn runCases(comptime max_pages: usize, comptime num_pages: usize, cases: []const TestCase, comptime dims: TestDims) !void { - const platform = zml.testing.env(); - const allocator = std.testing.allocator; - const io = std.testing.io; - const bf16 = zml.floats.BFloat16; - const hkv = dims.hkv; - const g = dims.g; - const d = dims.d; - const page = dims.page; - const batch = dims.batch; - var prng: std.Random.DefaultPrng = .init(7); - const r = prng.random(); - const kc = try allocator.alloc(bf16, num_pages * page * hkv * d); - defer allocator.free(kc); - const vc = try allocator.alloc(bf16, num_pages * page * hkv * d); - defer allocator.free(vc); - const k_t: Tensor = .init(.{ .page = num_pages, .k_chunk = page, .hkv = hkv, .hd = d }, dims.dtype); - - for (cases) |case| { - var tokens: i64 = case.padding; - for (case.seqs[0..batch]) |s| tokens += s[0]; - const params: paged.Parameters = .init(.{ .batch_size = batch, .max_num_pages = max_pages, .max_seqlen_q = 128, .is_prefill = case.prefill }); - const q_t: Tensor = .init(.{ .b = tokens, .hkv = hkv, .hg = g, .hd = d }, dims.dtype); - const Mod = struct { - pub fn forward(p: paged.Parameters, q: Tensor, k: Tensor, v: Tensor, opts: AttentionOptions) Tensor { - return paged.pagedAttention(p, q, k, v, opts); - } - }; - const opts: AttentionOptions = .{ - .mask = .{ .standard = .{ - .causal = case.opts.is_causal, - .sliding_window = if (case.opts.sliding_window > 0) @intCast(case.opts.sliding_window) else null, - } }, - .scale = case.opts.scale, - .sink = if (case.opts.sink != null) Tensor.init(.{ .h = hkv * g }, .f32) else null, - }; - const v_t: Tensor = .init(k_t.shape(), dims.dtype); - var exe = try zml.module.compile(allocator, io, Mod.forward, .{ params, q_t, k_t, v_t, opts }, platform, .{}); - defer exe.deinit(); - - const q = try allocator.alloc(bf16, @intCast(tokens * hkv * g * d)); - defer allocator.free(q); - for (q) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); - var table: [batch][max_pages]i32 = @splat(@splat(case.padding_page)); - var seq_lens: [batch]i32 = undefined; - var starts: [batch + 1]i32 = undefined; - starts[0] = 0; - var next_page: i32 = 0; - for (case.seqs[0..batch], 0..) |s, seq| { - seq_lens[seq] = s[1]; - starts[seq + 1] = starts[seq] + s[0]; - if (!case.mask_all) { - for (0..@intCast(std.math.divCeil(i32, s[1], page) catch unreachable)) |p| { - table[seq][p] = @mod(next_page * 7, @as(i32, num_pages)); - next_page += 1; - } - } - } - - for (kc) |*v| v.* = .fromF32(if (dims.uniform or dims.uniform_scores) 0 else r.floatNorm(f32)); - for (vc, 0..) |*v, i| v.* = .fromF32(if (dims.uniform) @as(f32, @floatFromInt(i % d)) / d else r.floatNorm(f32)); - if (dims.spikes.len > 0) { - for (case.seqs[0..batch], 0..) |seq, row| { - for (0..@intCast(seq[1])) |key| { - const slot: usize = @intCast(table[row][key / page] * @as(i32, page) + @as(i32, @intCast(key % page))); - const spike = std.mem.indexOfScalar(usize, dims.spikes, key) != null; - @memset(vc[slot * hkv * d ..][0 .. hkv * d], bf16.fromF32(if (spike) 32768 else 0)); - } - } - } - if (case.poison != .none) { - const live = try allocator.alloc(bool, num_pages * page); - defer allocator.free(live); - @memset(live, false); - for (case.seqs[0..batch], 0..) |s, seq| { - if (s[0] == 0) continue; - for (0..@intCast(s[1])) |key| { - const physical = table[seq][key / page]; - if (physical < 0 or physical >= num_pages) continue; - live[@as(usize, @intCast(physical)) * page + key % page] = true; - } - } - const poison = bf16.fromF32(case.poison_value); - for (live, 0..) |valid, slot| { - if (valid) continue; - const start = slot * hkv * d; - if (case.poison == .k or case.poison == .both) @memset(kc[start..][0 .. hkv * d], poison); - if (case.poison == .v or case.poison == .both) @memset(vc[start..][0 .. hkv * d], poison); - } - } - var bk = try testBuffer(platform, k_t.shape(), kc); - defer bk.deinit(); - var bv = try testBuffer(platform, k_t.shape(), vc); - defer bv.deinit(); - var bq = try testBuffer(platform, q_t.shape(), q); - defer bq.deinit(); - var bp: zml.Bufferized(paged.Parameters) = .{ - .block_table = try .fromBytes(io, platform, params.block_table.shape(), std.mem.sliceAsBytes(&table)), - .seq_lens = try .fromBytes(io, platform, params.seq_lens.shape(), std.mem.sliceAsBytes(&seq_lens)), - .query_start_len = try .fromBytes(io, platform, params.query_start_len.shape(), std.mem.sliceAsBytes(&starts)), - }; - defer zml.Buffer.deinitAll(paged.Parameters, &bp); - var opts_d: zml.Bufferized(AttentionOptions) = .{ - .mask = .{ .standard = {} }, - .sink = if (case.opts.sink) |s| try .fromBytes(io, platform, opts.sink.?.shape(), std.mem.sliceAsBytes(s)) else null, - }; - defer zml.Buffer.deinitAll(AttentionOptions, &opts_d); - var result = try exe.eval(allocator, io, .{ bp, bq, bk, bv, opts_d }); - defer result.deinit(); - const host = try result.toSliceAlloc(allocator, io); - defer host.free(allocator); - const expected = try attention_test.reference( - allocator, - .initConst(q_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(q)), - .initConst(k_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(kc)), - .initConst(k_t.shape().withDtype(.bf16), std.mem.sliceAsBytes(vc)), - @as([*]const i32, @ptrCast(&table))[0 .. batch * max_pages], - &seq_lens, - &starts, - case.opts, - ); - defer expected.free(allocator); - try std.testing.expectEqual(dims.dtype, host.shape.dtype()); - attention_test.expectClose(expected, host) catch |err| { - std.log.err("TCL attention fixture: {any}", .{case}); - return err; - }; - const active_elements = if (case.mask_all) 0 else @as(usize, @intCast(starts[batch])) * hkv * g * d; - for (active_elements..@intCast(tokens * hkv * g * d)) |index| try std.testing.expectEqual(@as(f32, 0), attention_test.floatAt(host, index)); - for (case.seqs[0..batch], 0..) |seq, index| { - const empty_queries: usize = @intCast(@max(0, seq[0] - seq[1])); - const begin: usize = @as(usize, @intCast(starts[index])) * hkv * g * d; - for (begin..begin + empty_queries * hkv * g * d) |element| try std.testing.expectEqual(@as(f32, 0), attention_test.floatAt(host, element)); - } - } -} - -fn testBuffer(platform: *const zml.Platform, shape: zml.Shape, values: []const zml.floats.BFloat16) !zml.Buffer { - const slice = try zml.Slice.alloc(std.testing.allocator, shape); - defer slice.free(std.testing.allocator); - switch (shape.dtype()) { - .bf16 => @memcpy(slice.items(zml.floats.BFloat16), values), - .f16 => for (slice.items(f16), values) |*out, value| { - out.* = @floatCast(value.toF32()); - }, - .f32 => for (slice.items(f32), values) |*out, value| { - out.* = value.toF32(); - }, - else => unreachable, - } - return .fromBytes(std.testing.io, platform, shape, slice.bytes); -} diff --git a/zml/kernels/paged_attention/triton/BUILD.bazel b/zml/kernels/paged_attention/triton/BUILD.bazel deleted file mode 100644 index eff7d096..00000000 --- a/zml/kernels/paged_attention/triton/BUILD.bazel +++ /dev/null @@ -1,11 +0,0 @@ -package(default_visibility = ["//zml:__pkg__"]) - -filegroup( - name = "triton", - srcs = [ - "sparse_unified_attention.zig", - "triton.zig", - "unified_attention.zig", - "unified_attention_oneapi.zig", - ], -) diff --git a/zml/kernels/paged_attention/triton/README.md b/zml/kernels/paged_attention/triton/README.md deleted file mode 100644 index 37aba88b..00000000 --- a/zml/kernels/paged_attention/triton/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Triton paged attention - -This backend is automatically selected for CUDA, ROCm, and oneAPI. See the -[parent README](../README.md) for the attention equation, packed sequence -metadata, and page addressing contract. - -[Triton's wrapper](triton.zig) uses split KV caches and supports causal and -non-causal attention, a left sliding window, explicit key indices, and softmax -sinks. Query heads are grouped by KV head. Keep complete GQA groups together -when sharding `.hkv`, with matching query and cache ownership and complete -`.hg` and `.hd` axes. Each local computation needs its full key range and the -matching sequence metadata. - -For standard masks, the wrapper chooses between a 2D kernel and a 3D split-key -kernel followed by segment reduction. Prefill uses the 2D path; all-decode -batches select a path based on batch size and KV head count. Explicit indices -use the dedicated sparse path described below. oneAPI forces split-key execution for -all-decode batches and has a specialized path for standard masks. Segment -reduction combines softmax maxima, exponential sums, and weighted values to -preserve one normalization over the key range. - -Set `is_prefill = true` for mixed prefill/decode batches. `max_seqlen_q` guides -kernel configuration. These options do not change logical query positions. - -The shared tests in [paged_attention.zig](../paged_attention.zig) exercise -`bf16`, mixed prefill/decode, shared prefix pages, partial final KV pages, -trailing inactive sequences, and option-specific comparisons. Kernel selection -and numerical behavior depend on the target; these cases do not establish -support for every shape or dtype. - -## Sparse attention - -`.indices` mask dispatches to a dedicated sparse kernel in -`sparse_unified_attention.zig`. -The `i32` tensor has shape `{ .b = T, .topk = K }`, with one list per packed -query shared across heads. Each entry is a logical position within that query's -sequence, including its cached prefix. The kernel translates each selected -position through the existing page table; it does not compute top-k selection. -Negative and out-of-sequence entries are ignored, including holes before valid -tail entries. Duplicate entries count repeatedly. Indices replace the standard -mask, so they do not implicitly enforce causality. Empty selections return zero; -optional attention sinks contribute only to the softmax denominator. - -Prefill processes one query and one KV-head group per program and writes the -normalized result directly. Decode uses 16- or 32-position gather tiles and an -occupancy-based split count bounded by selection length. Single-token decode -addresses sequence slots directly; other query layouts use packed-query lookup. -Inactive sequence slots and unused query capacity are skipped. Only segments -covering selection slots launch. Small empty segments skip Q and KV loading, -and larger segments skip wholly empty tiles. Known strides and scale are -specialized, and sparse gathers use the default cache policy. - -Split-K decode writes FP32 partials and contiguous-head statistics. Its reducer -handles at most 64 features per program, ignores unlaunched and zero-mass -segments, and never reads their unwritten partial outputs. The direct and split -paths use the same online softmax and support non-power-of-two GQA groups and -padded head dimensions. Dense attention retains its existing dispatch and kernels. diff --git a/zml/kernels/paged_attention/triton/sparse_unified_attention.zig b/zml/kernels/paged_attention/triton/sparse_unified_attention.zig deleted file mode 100644 index 79113f1e..00000000 --- a/zml/kernels/paged_attention/triton/sparse_unified_attention.zig +++ /dev/null @@ -1,326 +0,0 @@ -const std = @import("std"); -const zml = @import("../../../zml.zig"); -const tri = zml.kernel.triton; -const Builder = tri.Builder; -const Value = tri.Value; -const DType = tri.DType; - -/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. -const RCP_LN2: f32 = 1.4426950408889634; - -/// Partition selection slots, not logical KV positions. Scratch capacity may be -/// larger than the number of segments that have any selection slots to visit. -pub const SparseSegments = struct { - tiles_per_segment: i64, - span: i64, - launched: i64, - - pub fn init(count: i64, capacity: i64, tile: i64) SparseSegments { - std.debug.assert(count > 0 and capacity > 0 and tile > 0); - const tiles = std.math.divCeil(i64, count, capacity * tile) catch unreachable; - return .{ - .tiles_per_segment = tiles, - .span = tiles * tile, - .launched = std.math.divCeil(i64, count, tiles * tile) catch unreachable, - }; - } -}; - -pub const SparseAttentionConfig = struct { - dtype: DType, - num_seqs: i32, - num_query_heads: i64, - num_queries_per_kv: i64, - head_size: i64, - head_size_padded: i64, - block_m: i64, - block_size: i64, - tile_size: i64, - indices_count: i64, - num_segments: i64 = 1, - decode_sequence_grid: bool = false, - use_sinks: bool = false, - scale: f32, - block_table_stride: i64, - query_strides: [3]i64, - key_strides: [4]i64, - value_strides: [4]i64, - indices_strides: [2]i64, -}; - -fn findSeqIdx( - k: *Builder, - query_start_len_ptr: Value, - target_idx: Value, - num_seqs: Value, - block_q: i64, - use_q_block_mode: bool, -) Value { - const left_init = k.constant(.i32, 0); - - var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); - { - const left = w.before_carried[0]; - const right = w.before_carried[1]; - w.yieldBefore(left.lt(right), .{ left, right }); - } - { - const left = w.after_carried[0]; - const right = w.after_carried[1]; - const mid = left.add(right).div(2); - const val = k.load(query_start_len_ptr.addPtr(mid)); - const mid_val = if (use_q_block_mode) - val.div(@as(i32, @intCast(block_q))).add(mid) - else - val; - - var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); - { - i.yieldThen(.{ mid.add(1), right }); - } - { - i.yieldElse(.{ left, mid }); - } - w.yieldAfter(.{ i.results[0], i.results[1] }); - } - - return w.results[0].sub(1); -} - -/// One program processes all selected keys for one query/KV-head group and write the normalized output. -pub const KernelSparseAttention2dPtr = KernelSparseAttentionPtr(false); - -/// Also known as split-K in the litterature, this 3d variant uses multiple programs process different -/// portions of that same selection list. They write partial results, then a reduction kernels combines -/// them into one normalized output. -/// -/// Mostly useful during decode when there are few queries and a long top-k list. -pub const KernelSparseAttention3dPtr = KernelSparseAttentionPtr(true); - -/// One query and one GQA group per program. The direct specialization writes -/// normalized output; split decode writes FP32 partials for SparseReducePtr. -fn KernelSparseAttentionPtr(comptime split_k: bool) type { - return struct { - pub const Config = SparseAttentionConfig; - pub const Kernel = tri.Kernel(Config, .{ - .name = if (split_k) "kernel_sparse_attention_3d_ptr" else "kernel_sparse_attention_2d_ptr", - .inputs = &.{ - "query_ptr", "key_ptr", "value_ptr", "indices_ptr", - "block_tables_ptr", "seq_lens_ptr", "query_start_len_ptr", "sink_ptr", - }, - .outputs = if (split_k) &.{ "output", "segm_max", "segm_expsum" } else &.{"output"}, - .run = run, - }); - - fn run(k: *Builder, cfg: Config) tri.FinishError!void { - const a = if (split_k) try k.declareArgs(.{ - .query_ptr = .{ .ptr = cfg.dtype }, - .key_ptr = .{ .ptr = cfg.dtype }, - .value_ptr = .{ .ptr = cfg.dtype }, - .indices_ptr = .{ .ptr = .i32 }, - .block_tables_ptr = .{ .ptr = .i32 }, - .seq_lens_ptr = .{ .ptr = .i32 }, - .query_start_len_ptr = .{ .ptr = .i32 }, - .sink_ptr = .{ .ptr = .f32 }, - .output_ptr = .{ .ptr = .f32 }, - .segm_max_ptr = .{ .ptr = .f32 }, - .segm_expsum_ptr = .{ .ptr = .f32 }, - }) else try k.declareArgs(.{ - .query_ptr = .{ .ptr = cfg.dtype }, - .key_ptr = .{ .ptr = cfg.dtype }, - .value_ptr = .{ .ptr = cfg.dtype }, - .indices_ptr = .{ .ptr = .i32 }, - .block_tables_ptr = .{ .ptr = .i32 }, - .seq_lens_ptr = .{ .ptr = .i32 }, - .query_start_len_ptr = .{ .ptr = .i32 }, - .sink_ptr = .{ .ptr = .f32 }, - .output_ptr = .{ .ptr = cfg.dtype }, - }); - - std.debug.assert(cfg.block_m >= cfg.num_queries_per_kv); - std.debug.assert(split_k or cfg.num_segments == 1); - - const partition = SparseSegments.init(cfg.indices_count, cfg.num_segments, cfg.tile_size); - const program = k.programId(.x); - const num_seqs = k.constant(.i32, cfg.num_seqs); - if (!cfg.decode_sequence_grid) { - const active_queries = k.load(a.query_start_len_ptr.addPtr(num_seqs)); - k.returnIf(program.ge(active_queries)); - } - - const seq = if (cfg.decode_sequence_grid) program else findSeqIdx(k, a.query_start_len_ptr, program, num_seqs, 1, false); - const seq_len = k.load(a.seq_lens_ptr.addPtr(seq)); - const query_start = k.load(a.query_start_len_ptr.addPtr(seq)); - const query_end = k.load(a.query_start_len_ptr.addPtr(seq).addPtr(1)); - if (cfg.decode_sequence_grid) k.returnIf(query_start.eq(query_end)); - const query = if (cfg.decode_sequence_grid) query_start else program; - const kv_head = if (cfg.num_query_heads == cfg.num_queries_per_kv) k.constant(.i32, 0) else k.programId(.y); - - // Launch the partial tail first, without launching unused capacity. - const slot = if (split_k) k.programId(.z) else k.constant(.i32, 0); - const segment = if (split_k) k.where(slot.eq(0), k.constant(.i32, partition.launched - 1), slot.sub(1)) else slot; - const segment_start = segment.mul(@as(i32, @intCast(partition.span))); - const heads = k.arange(0, cfg.block_m, .i32); - const head = kv_head.mul(@as(i32, @intCast(cfg.num_queries_per_kv))).add(heads); - const head_mask = heads.lt(@as(i32, @intCast(cfg.num_queries_per_kv))); - const dims = k.arange(0, cfg.head_size_padded, .i32); - const dim_mask = dims.lt(@as(i32, @intCast(cfg.head_size))); - const indices_row = a.indices_ptr.addPtr(query.to(.i64).mul(cfg.indices_strides[0])); - - const sink = if (cfg.use_sinks) k.loadOpts(a.sink_ptr.addPtr(head), .{ - .mask = head_mask.bitAnd(segment.eq(0)), - .other = k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32), - }).mul(RCP_LN2) else k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32); - const has_sink = if (cfg.use_sinks) segment.eq(0).bitAnd(head_mask) else k.full(&.{cfg.block_m}, 0, .i1); - const initial_max = k.where(has_sink, sink, k.full(&.{cfg.block_m}, -std.math.inf(f32), .f32)); - const initial_sum = has_sink.to(.f32); - const initial_acc = k.zeros(&.{ cfg.block_m, cfg.head_size_padded }, .f32); - - if (partition.span <= 256) { - const scan_size: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(partition.span))); - const selected = segment_start.add(k.arange(0, scan_size, .i32)); - const scan_mask = selected.lt(cfg.indices_count).bitAnd(selected.lt(segment_start.add(@as(i32, @intCast(partition.span))))); - const positions = k.loadOpts(indices_row.addPtr(selected.to(.i64).mul(cfg.indices_strides[1])), .{ - .mask = scan_mask, - .other = k.full(&.{scan_size}, -1, .i32), - }); - const empty = k.sum(scan_mask.bitAnd(positions.ge(0)).bitAnd(positions.lt(seq_len)).to(.i32)).eq(0); - var branch = k.openIfElse(empty, .{}); - { - storeResult(k, cfg, a, query, head, head_mask, segment, dims, dim_mask, initial_max, initial_sum, initial_acc); - branch.elseBranch(); - } - { - branch.close(); - } - k.returnIf(empty); - } - - const q_offset = query.to(.i64).mul(cfg.query_strides[0]) - .add(head.to(.i64).mul(cfg.query_strides[1]).expandDims(1)) - .add(dims.to(.i64).mul(cfg.query_strides[2]).expandDims(0)); - const Q = k.loadOpts(a.query_ptr.addPtr(q_offset), .{ - .mask = k.mask2d(head_mask, dim_mask, cfg.block_m, cfg.head_size_padded), - .other = initial_acc.to(cfg.dtype), - }); - const table = a.block_tables_ptr.addPtr(seq.to(.i64).mul(cfg.block_table_stride)); - const offsets = k.arange(0, cfg.tile_size, .i32); - var loop = k.openFor(0, partition.tiles_per_segment, 1, .{ initial_max, initial_sum, initial_acc }); - { - const selected = segment_start.add(loop.iv.mul(@as(i32, @intCast(cfg.tile_size)))).add(offsets); - const position = k.loadOpts(indices_row.addPtr(selected.to(.i64).mul(cfg.indices_strides[1])), .{ - .mask = selected.lt(cfg.indices_count), - .other = k.full(&.{cfg.tile_size}, -1, .i32), - }); - const valid = selected.lt(cfg.indices_count).bitAnd(position.ge(0)).bitAnd(position.lt(seq_len)); - // A bounded single-tile segment was already checked above. - const nonempty = if (partition.tiles_per_segment == 1 and partition.span <= 256) k.constant(.i1, 1) else k.sum(valid.to(.i32)).gt(0); - var branch = k.openIfElse(nonempty, .{ k.tensorTy(&.{cfg.block_m}, .f32), k.tensorTy(&.{cfg.block_m}, .f32), k.tensorTy(&.{ cfg.block_m, cfg.head_size_padded }, .f32) }); - { - const safe_position = k.where(valid, position, k.zeros(&.{cfg.tile_size}, .i32)); - const page = k.loadOpts(table.addPtr(safe_position.div(@as(i32, @intCast(cfg.block_size)))), .{ - .mask = valid, - .other = k.zeros(&.{cfg.tile_size}, .i32), - }).to(.i64); - const in_page = safe_position.rem(@as(i32, @intCast(cfg.block_size))).to(.i64); - const key_offset = page.mul(cfg.key_strides[0]).add(in_page.mul(cfg.key_strides[1])).add(kv_head.to(.i64).mul(cfg.key_strides[2])).expandDims(0) - .add(dims.to(.i64).mul(cfg.key_strides[3]).expandDims(1)); - const value_offset = page.mul(cfg.value_strides[0]).add(in_page.mul(cfg.value_strides[1])).add(kv_head.to(.i64).mul(cfg.value_strides[2])).expandDims(1) - .add(dims.to(.i64).mul(cfg.value_strides[3]).expandDims(0)); - const K = k.loadOpts(a.key_ptr.addPtr(key_offset), .{ - .mask = k.mask2d(dim_mask, valid, cfg.head_size_padded, cfg.tile_size), - .other = k.zeros(&.{ cfg.head_size_padded, cfg.tile_size }, cfg.dtype), - }); - const V = k.loadOpts(a.value_ptr.addPtr(value_offset), .{ - .mask = k.mask2d(valid, dim_mask, cfg.tile_size, cfg.head_size_padded), - .other = k.zeros(&.{ cfg.tile_size, cfg.head_size_padded }, cfg.dtype), - }); - const scores = k.dot(Q, K, k.zeros(&.{ cfg.block_m, cfg.tile_size }, .f32)).mul(cfg.scale * RCP_LN2); - const S = k.where(k.mask2d(head_mask, valid, cfg.block_m, cfg.tile_size), scores, k.full(&.{ cfg.block_m, cfg.tile_size }, -std.math.inf(f32), .f32)); - const max_score = loop.carried[0].maximum(k.maxOpts(S, .{ .axis = 1 })); - const next_max = k.where(max_score.gt(-std.math.inf(f32)), max_score, k.zeros(&.{cfg.block_m}, .f32)); - const alpha = k.exp2(loop.carried[0].sub(next_max)); - const P = k.exp2(S.sub(next_max.expandDims(1))); - const sum = loop.carried[1].mul(alpha).add(k.sumOpts(P, .{ .axis = 1 })); - const acc = k.dot(P.to(cfg.dtype), V, loop.carried[2].mul(alpha.expandDims(1))); - branch.yieldThen(.{ next_max, sum, acc }); - } - { - branch.yieldElse(.{ loop.carried[0], loop.carried[1], loop.carried[2] }); - } - loop.yield(.{ branch.results[0], branch.results[1], branch.results[2] }); - } - - storeResult(k, cfg, a, query, head, head_mask, segment, dims, dim_mask, loop.results[0], loop.results[1], loop.results[2]); - } - - fn storeResult(k: *Builder, cfg: Config, a: anytype, query: Value, head: Value, head_mask: Value, segment: Value, dims: Value, dim_mask: Value, max_score: Value, sum: Value, acc: Value) void { - if (split_k) { - // Adjacent heads are written together by the producer. - const stats_offset = query.to(.i64).mul(cfg.num_segments).add(segment.to(.i64)).mul(cfg.num_query_heads).add(head.to(.i64)); - k.storeOpts(a.segm_max_ptr.addPtr(stats_offset), max_score, .{ .mask = head_mask }); - k.storeOpts(a.segm_expsum_ptr.addPtr(stats_offset), sum, .{ .mask = head_mask }); - const offset = query.to(.i64).mul(cfg.num_query_heads).add(head.to(.i64)).mul(cfg.num_segments).add(segment.to(.i64)).mul(cfg.head_size_padded).expandDims(1).add(dims.to(.i64).expandDims(0)); - // Zero-mass segments need no partial output; the reducer masks them. - k.storeOpts(a.output_ptr.addPtr(offset), acc, .{ .mask = k.mask2d(head_mask.bitAnd(sum.gt(0.0)), dim_mask, cfg.block_m, cfg.head_size_padded) }); - } else { - const denominator = k.where(sum.gt(0.0), sum, k.full(&.{cfg.block_m}, 1.0, .f32)); - const offset = query.to(.i64).mul(cfg.query_strides[0]).add(head.to(.i64).mul(cfg.query_strides[1]).expandDims(1)).add(dims.to(.i64).mul(cfg.query_strides[2]).expandDims(0)); - k.storeOpts(a.output_ptr.addPtr(offset), acc.div(denominator.expandDims(1)).to(cfg.dtype), .{ .mask = k.mask2d(head_mask, dim_mask, cfg.block_m, cfg.head_size_padded) }); - } - } - }; -} - -pub const SparseReducePtr = struct { - pub const Config = struct { - dtype: DType, - num_seqs: i32, - num_query_heads: i64, - head_size: i64, - head_size_padded: i64, - head_tile: i64, - num_segments: i64, - launched_segments: i64, - output_strides: [3]i64, - }; - pub const Kernel = tri.Kernel(Config, .{ - .name = "reduce_sparse_attention_ptr", - .inputs = &.{ "segm_output_ptr", "segm_max_ptr", "segm_expsum_ptr", "query_start_len_ptr" }, - .outputs = &.{"output"}, - .run = run, - }); - fn run(k: *Builder, cfg: Config) tri.FinishError!void { - const a = try k.declareArgs(.{ - .segm_output_ptr = .{ .ptr = .f32 }, - .segm_max_ptr = .{ .ptr = .f32 }, - .segm_expsum_ptr = .{ .ptr = .f32 }, - .query_start_len_ptr = .{ .ptr = .i32 }, - .output_ptr = .{ .ptr = cfg.dtype }, - }); - const query = k.programId(.x); - k.returnIf(query.ge(k.load(a.query_start_len_ptr.addPtr(cfg.num_seqs)))); - const head = k.programId(.y); - const segments = k.arange(0, cfg.num_segments, .i32); - const segment_mask = segments.lt(cfg.launched_segments); - const dims = k.programId(.z).mul(@as(i32, @intCast(cfg.head_tile))).add(k.arange(0, cfg.head_tile, .i32)); - const dim_mask = dims.lt(cfg.head_size); - const stats = query.to(.i64).mul(cfg.num_segments).add(segments.to(.i64)).mul(cfg.num_query_heads).add(head.to(.i64)); - const maxima = k.loadOpts(a.segm_max_ptr.addPtr(stats), .{ .mask = segment_mask, .other = k.full(&.{cfg.num_segments}, -std.math.inf(f32), .f32) }); - const sums = k.loadOpts(a.segm_expsum_ptr.addPtr(stats), .{ .mask = segment_mask, .other = k.zeros(&.{cfg.num_segments}, .f32) }); - const nonempty = segment_mask.bitAnd(sums.gt(0.0)); - const maximum = k.max(k.where(nonempty, maxima, k.full(&.{cfg.num_segments}, -std.math.inf(f32), .f32))); - const safe_maximum = k.where(maximum.gt(-std.math.inf(f32)), maximum, k.constant(.f32, 0.0)); - const rescale = k.exp2(maxima.sub(safe_maximum)); - const denominator = k.sum(sums.mul(rescale)); - const offset = query.to(.i64).mul(cfg.num_query_heads).add(head.to(.i64)).mul(cfg.num_segments).add(segments.to(.i64)).mul(cfg.head_size_padded).expandDims(1).add(dims.to(.i64).expandDims(0)); - const partial = k.loadOpts(a.segm_output_ptr.addPtr(offset), .{ - .mask = k.mask2d(nonempty, dim_mask, cfg.num_segments, cfg.head_tile), - .other = k.zeros(&.{ cfg.num_segments, cfg.head_tile }, .f32), - }); - const numerator = k.sumOpts(partial.mul(rescale.expandDims(1)), .{ .axis = 0 }); - const output = numerator.div(k.where(denominator.gt(0.0), denominator, k.constant(.f32, 1.0))); - const output_offset = query.to(.i64).mul(cfg.output_strides[0]).add(head.to(.i64).mul(cfg.output_strides[1])).add(dims.to(.i64).mul(cfg.output_strides[2])); - k.storeOpts(a.output_ptr.addPtr(output_offset), output.to(cfg.dtype), .{ .mask = dim_mask }); - } -}; diff --git a/zml/kernels/paged_attention/triton/triton.zig b/zml/kernels/paged_attention/triton/triton.zig deleted file mode 100644 index 2a12e012..00000000 --- a/zml/kernels/paged_attention/triton/triton.zig +++ /dev/null @@ -1,861 +0,0 @@ -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("../../../zml.zig"); -const triton = zml.kernel.triton; -const AttentionOptions = @import("../options.zig").AttentionOptions; -const kernels = @import("unified_attention.zig"); -const kernels_oneapi = @import("unified_attention_oneapi.zig"); -const kernels_sparse = @import("sparse_unified_attention.zig"); - -const log = std.log.scoped(.@"zml/attention/triton"); - -fn isOneapiTarget() bool { - return zml.Compiler.current().platform.target == .oneapi; -} - -fn use2dKernel(all_decode: bool, batch_size: usize, num_kv_heads: usize) bool { - // Intel decode spills the 2D whole-sequence kernel; force the 3D split-K path. - if (all_decode and isOneapiTarget()) return false; - // prefill uses 2D; decode uses 3D until the batch is large enough to - // provide at least 128 2D launch programs across KV heads. - if (all_decode) { - const seq_threshold_3d = @divFloor(128, num_kv_heads); - return batch_size > seq_threshold_3d; - } - - return true; -} - -pub const Config2D = struct { - block_m: usize, - block_q: usize, - tile_size: usize, - num_warps: usize, - num_stages: usize, - total_q_blocks: usize, -}; - -fn isCudaComputeCapability(expected: zml.platform.cuda.ComputeCapability) bool { - const platform = zml.module.CompilationContext.current().platform; - return if (zml.platform.cuda.computeCapability(platform)) |cc| cc.eql(expected) else false; -} - -/// Largest `BLOCK_M * HEAD_SIZE_PADDED` the 2D kernel survives. That product is -/// the shape of both the query tile and the f32 accumulator carried through the -/// tile loop; above it the kernel faults with CUDA_ERROR_MISALIGNED_ADDRESS. -/// -/// Measured on a GB300 with Gemma-4-31B, whose global-attention layers are -/// head_dim 512 (its sliding ones are 256): -/// -/// 64x256, 32x512, 16x512 ok 64x512, 128x512 fault -/// -/// The bound is on this product alone. Two things that do *not* move it, so do -/// not reach for them if a new head size faults: TILE_SIZE, which at 64 pushes -/// total shared memory to ~160 KB and still passes at BLOCK_M 32 while 64x512 -/// faults at 98 KB; and num_warps, where 8 halves the per-thread register share -/// and still faults -- which is also why the obvious "the accumulator outgrows -/// the register file" story cannot be the whole explanation. The underlying -/// Triton codegen bug is not diagnosed; this only keeps us out of it. -const max_query_tile_elements: usize = 16 * 1024; - -fn select2dConfig(options: paged.PagedAttentionOptions) Config2D { - const max_num_stages_2d: usize = if (options.head_dim <= 128) 4 else 2; - - // Until we test on other platforms, gate the fix to GB300 - const is_gb300 = isCudaComputeCapability(.{ .major = 10, .minor = 3 }); - - var num_stages_2d: usize, var num_warps: usize, var tile_size: usize = if (!options.all_decode) .{ 1, 2, 64 } else .{ 3, 2, options.block_size }; - - var block_m = options.block_m; - var block_q = options.block_q; - if (options.max_seqlen_q >= 256) { - if (options.head_dim >= 256) { - block_m = 64; - tile_size = 16; - } else { - block_m = 128; - } - - if (is_gb300) { - // Halving BLOCK_M for every doubling of the head keeps that product - // constant: 128 at head_dim 128, 64 at 256, 32 at 512. The first two - // are the values this branch has always picked. - const head_dim_padded = std.math.ceilPowerOfTwoAssert(usize, options.head_dim); - block_m = @min(128, max_query_tile_elements / head_dim_padded); - } - num_stages_2d = 1; - num_warps = 4; - } - block_q = block_m / options.numQueriesPerKv(); - const total_q_blocks = options.num_tokens / block_q + options.batch_size; - - return .{ - .block_m = block_m, - .block_q = block_q, - .tile_size = tile_size, - .num_warps = num_warps, - .num_stages = @min(max_num_stages_2d, num_stages_2d), - .total_q_blocks = total_q_blocks, - }; -} - -pub const Config3D = struct { - const AttentionConfig = struct { - tile_size: usize, - num_segments_per_seq: usize, - num_warps: usize, - num_stages: usize, - block_q: usize, - block_m: usize, - total_q_blocks: usize, - }; - const ReduceConfig = struct { - tile_size: usize, - num_segments_per_seq: usize, - num_warps: usize, - num_stages: usize, - block_q: usize, - block_m: usize, - }; - attention: AttentionConfig, - reduce: ReduceConfig, -}; - -fn select3dConfig(options: paged.PagedAttentionOptions) Config3D { - var reduce_num_warps: usize = 2; - // Intel decode needs more warps to spread the work and avoid register spill. - const attn_warps: usize = if (options.all_decode and isOneapiTarget()) 8 else 2; - const tile_size = options.block_size; - - //const MAX_SEGMENTS: usize = @min(128, std.math.divCeil(usize, max_seqlen_k, tile_size)); - var num_segments = std.math.divCeil(usize, options.target_num_prgms, options.num_2d_prgms) catch unreachable; - num_segments = std.math.ceilPowerOfTwoAssert(usize, num_segments); - num_segments = @min(num_segments, 128); - if (options.all_decode and !isOneapiTarget()) { - // Keep the number of segments small to then limit reduce cost - // Didn't change the computation of Intel decode at the momment - // Need to be tested - num_segments = @min(num_segments, 16); - } - const min_segments: usize = if (tile_size <= 16) 16 else 8; - num_segments = @max(num_segments, min_segments); - if (num_segments == min_segments) { - reduce_num_warps = 1; - } - - return .{ - .attention = .{ - .tile_size = tile_size, - .num_segments_per_seq = num_segments, - .num_warps = attn_warps, - .num_stages = 1, - .block_m = options.block_m, - .block_q = options.block_q, - .total_q_blocks = options.total_q_blocks, - }, - .reduce = .{ - .tile_size = tile_size, - .num_segments_per_seq = num_segments, - .num_warps = reduce_num_warps, - .num_stages = 1, - .block_m = options.block_m, - .block_q = options.block_q, - }, - }; -} - -pub fn getCuCount() usize { - const platform = zml.Compiler.current().platform; - if (platform.devices.len == 0) return 1; - const attribute = platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return 1; - if (attribute.int64 <= 0) return 1; - return @intCast(attribute.int64); -} - -const SparseConfig = struct { - tile_size: usize, - num_segments: usize, -}; - -fn selectSparseConfig(count: usize, batch_size: usize, num_kv_heads: usize, cu_count: usize, all_decode: bool) SparseConfig { - std.debug.assert(count > 0 and batch_size > 0 and num_kv_heads > 0); - if (!all_decode) return .{ .tile_size = 32, .num_segments = 1 }; - - const target_num_prgms = cu_count * 4; - const occupancy_splits = std.math.ceilPowerOfTwoAssert(usize, std.math.divCeil(usize, target_num_prgms * 4, batch_size * num_kv_heads) catch unreachable); - const tile: usize = if (occupancy_splits > 128) 16 else 32; - const tile_splits = std.math.ceilPowerOfTwoAssert(usize, std.math.divCeil(usize, count, tile) catch unreachable); - - return .{ - .tile_size = tile, - .num_segments = @min(tile_splits, @min(if (tile == 16) @as(usize, 256) else 128, @max(8, occupancy_splits))), - }; -} - -pub const paged = struct { - pub const Options = struct { - batch_size: usize, - max_num_pages: usize, - max_seqlen_q: usize, - is_prefill: bool, - - pub fn isPrefill(self: Options) bool { - return self.is_prefill; - } - - pub fn maxNumPages(self: Options) usize { - return self.max_num_pages; - } - }; - - pub const Parameters = struct { - block_table: zml.Tensor, - seq_lens: zml.Tensor, - query_start_len: zml.Tensor, - options_: Options, - - pub fn init(options_: Options) Parameters { - return .{ - .block_table = .init(.{ .b = options_.batch_size, .p = options_.max_num_pages }, .i32), - .seq_lens = .init(.{ .b = options_.batch_size }, .i32), - .query_start_len = .init(.{ .b = options_.batch_size + 1 }, .i32), - .options_ = options_, - }; - } - - pub fn allocationSize(self: Parameters) usize { - var allocation_size: usize = 0; - - allocation_size += self.block_table.byteSize(); - allocation_size += self.seq_lens.byteSize(); - allocation_size += self.query_start_len.byteSize(); - - return allocation_size; - } - - pub fn options(self: Parameters) Options { - return self.options_; - } - }; - - pub const PagedAttentionOptions = struct { - cu_count: usize, - all_decode: bool, - num_tokens: usize, - num_heads: usize, - num_kv_heads: usize, - head_dim: usize, - batch_size: usize, - block_size: usize, - num_blocks: usize, - max_num_block_per_seq: usize, - sliding_window: usize, - block_m: usize, - block_q: usize, - total_q_blocks: usize, - target_num_prgms: usize, - num_2d_prgms: usize, - max_seqlen_q: usize, - scale: ?f32, - - pub fn numQueriesPerKv(self: PagedAttentionOptions) usize { - return self.num_heads / self.num_kv_heads; - } - - pub fn maxSeqLenK(self: PagedAttentionOptions) usize { - return self.max_num_block_per_seq * self.block_size; - } - }; - - pub fn pagedAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { - if (opts.mask == .indices) { - return pagedSparseAttention(parameters, q, k_cache, v_cache, opts); - } - - const cu_count = getCuCount(); - const num_heads: usize = @intCast(q.dim(.hkv) * q.dim(.hg)); - const num_kv_heads: usize = @intCast(k_cache.dim(.hkv)); - const num_queries_per_kv: usize = num_heads / num_kv_heads; - // Intel decode: pack exactly one GQA group per tile (block_q == 1) so the - // single decode query token doesn't carry masked-out fp32 acc lanes. - // oneAPI decode keeps one GQA group per tile, padded to a power of two so tt.make_range emits legal Triton IR. - const block_m: usize = if (!parameters.options_.is_prefill and isOneapiTarget()) - std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv) - else if (num_queries_per_kv <= 16) 16 else std.math.ceilPowerOfTwoAssert(usize, num_queries_per_kv); - const block_q: usize = block_m / num_queries_per_kv; - const num_tokens: usize = @intCast(q.dim(.b)); - const num_seqs: usize = @intCast(parameters.block_table.dim(.b)); - const total_q_blocks: usize = num_tokens / block_q + num_seqs; - const target_num_prgms: usize = cu_count * 4; - const num_2d_prgms: usize = total_q_blocks * num_kv_heads; - - const sliding_window: usize = opts.mask.slidingWindow() orelse 0; - - const paged_attention_opts: PagedAttentionOptions = .{ - .cu_count = getCuCount(), - .all_decode = !parameters.options_.is_prefill, - .num_tokens = num_tokens, - .num_heads = num_heads, - .num_kv_heads = num_kv_heads, - .head_dim = @intCast(q.dim(.hd)), - .batch_size = @intCast(parameters.block_table.dim(.b)), - .block_size = @intCast(k_cache.dim(.k_chunk)), - .num_blocks = @intCast(k_cache.dim(.page)), - .max_num_block_per_seq = @intCast(parameters.block_table.dim(.p)), - .sliding_window = sliding_window, - .block_m = block_m, - .block_q = block_q, - .total_q_blocks = total_q_blocks, - .target_num_prgms = target_num_prgms, - .num_2d_prgms = num_2d_prgms, - .max_seqlen_q = parameters.options_.max_seqlen_q, - .scale = opts.scale, - }; - - const use_2d_kernel = use2dKernel( - paged_attention_opts.all_decode, - paged_attention_opts.batch_size, - paged_attention_opts.num_kv_heads, - ); - const output = if (use_2d_kernel) - pagedAttention2d(parameters, q, k_cache, v_cache, opts, paged_attention_opts) - else if (isOneapiTarget()) - pagedAttention3dOneapi(parameters, q, k_cache, v_cache, opts, paged_attention_opts) - else - pagedAttention3d(parameters, q, k_cache, v_cache, opts, paged_attention_opts); - - return output; - } - - pub fn pagedSparseAttention(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions) zml.Tensor { - stdx.debug.assert(opts.mask == .indices, "Expected indices mask but got {}", .{opts.mask}); - const indices = opts.mask.indices; - stdx.debug.assert(indices.rank() == 2 and indices.dtype() == .i32, "Sparse indices must be an i32 [query, topk] tensor, got {f}", .{indices.shape()}); - stdx.debug.assert(indices.dim(.b) == q.dim(.b) and indices.dim(.topk) > 0, "Sparse indices {f} must match queries {f} with a nonempty selection capacity", .{ indices.shape(), q.shape() }); - stdx.debug.assert(indices.shape().hasTags(.{ .b, .topk }), "Sparse indices expect to have .b, .topk tags", .{}); - stdx.debug.assert(q.dtype() == k_cache.dtype() and q.dtype() == v_cache.dtype(), "Sparse attention requires matching query and KV dtypes", .{}); - - const config = selectSparseConfig( - @intCast(opts.mask.indices.dim(.topk)), - @intCast(parameters.block_table.dim(.b)), - @intCast(q.dim(.hkv)), - getCuCount(), - !parameters.options_.is_prefill, - ); - - return if (config.num_segments == 1) - pagedSparseAttention2d(parameters, q, k_cache, v_cache, opts, config) - else - pagedSparseAttention3d(parameters, q, k_cache, v_cache, opts, config); - } - - pub fn pagedSparseAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { - std.debug.assert(tuning.num_segments == 1); - - const indices = opts.mask.indices; - - const group = q.dim(.hg); - const num_kv_heads = q.dim(.hkv); - std.debug.assert(num_kv_heads == k_cache.dim(.hkv) and num_kv_heads == v_cache.dim(.hkv)); - std.debug.assert(q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hd) == v_cache.dim(.hd)); - - const all_decode = !parameters.options_.is_prefill; - - const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); - - const qs = q_shape.computeElementStrides().constSlice(); - const ks = k_cache.shape().computeElementStrides().constSlice(); - const vs = v_cache.shape().computeElementStrides().constSlice(); - const is = indices.shape().computeElementStrides().constSlice(); - const ts = parameters.block_table.shape().computeElementStrides().constSlice(); - - const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(16, q.dim(.hd))))); - - const config: kernels_sparse.SparseAttentionConfig = .{ - .dtype = triton.from(q.dtype()), - .num_seqs = @intCast(parameters.block_table.dim(.b)), - .num_query_heads = num_kv_heads * group, - .num_queries_per_kv = group, - .head_size = q.dim(.hd), - .head_size_padded = head_size_padded, - .block_m = @intCast(@max(16, std.math.ceilPowerOfTwoAssert(u64, @intCast(group)))), - .block_size = k_cache.dim(.k_chunk), - .tile_size = @intCast(tuning.tile_size), - .indices_count = indices.dim(.topk), - .num_segments = @intCast(tuning.num_segments), - .decode_sequence_grid = all_decode and parameters.options_.max_seqlen_q == 1, - .use_sinks = opts.sink != null, - .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), - .block_table_stride = ts[parameters.block_table.axis(.b)], - .query_strides = .{ qs[q_shape.axis(.b)], qs[q_shape.axis(.h)], qs[q_shape.axis(.hd)] }, - .key_strides = .{ ks[k_cache.axis(.page)], ks[k_cache.axis(.k_chunk)], ks[k_cache.axis(.hkv)], ks[k_cache.axis(.hd)] }, - .value_strides = .{ vs[v_cache.axis(.page)], vs[v_cache.axis(.k_chunk)], vs[v_cache.axis(.hkv)], vs[v_cache.axis(.hd)] }, - .indices_strides = .{ is[indices.axis(.b)], is[indices.axis(.topk)] }, - }; - - const inputs: kernels_sparse.KernelSparseAttention2dPtr.Kernel.Inputs = .{ - .query_ptr = q, - .key_ptr = k_cache, - .value_ptr = v_cache, - .indices_ptr = indices, - .block_tables_ptr = parameters.block_table, - .seq_lens_ptr = parameters.seq_lens, - .query_start_len_ptr = parameters.query_start_len, - .sink_ptr = if (opts.sink) |sink| sink.convert(.f32) else .scalar(0, .f32), - }; - - const grid: [3]i32 = .{ - if (config.decode_sequence_grid) config.num_seqs else @intCast(q.dim(.b)), - @intCast(num_kv_heads), - 1, - }; - - const warps: i32 = if (all_decode and isOneapiTarget()) 8 else 4; - - const output = kernels_sparse.KernelSparseAttention2dPtr.Kernel.call(inputs, .{ - .output = q.shape(), - }, .{ - .cfg = config, - .grid = grid, - .num_stages = 1, - .num_warps = warps, - }); - - return output.output; - } - - pub fn pagedSparseAttention3d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, tuning: SparseConfig) zml.Tensor { - std.debug.assert(tuning.num_segments > 1); - - const indices = opts.mask.indices; - const group = q.dim(.hg); - const num_kv_heads = q.dim(.hkv); - std.debug.assert(num_kv_heads == k_cache.dim(.hkv) and num_kv_heads == v_cache.dim(.hkv)); - std.debug.assert(q.dim(.hd) == k_cache.dim(.hd) and q.dim(.hd) == v_cache.dim(.hd)); - - const all_decode = !parameters.options_.is_prefill; - - const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); - - const qs = q_shape.computeElementStrides().constSlice(); - const ks = k_cache.shape().computeElementStrides().constSlice(); - const vs = v_cache.shape().computeElementStrides().constSlice(); - const is = indices.shape().computeElementStrides().constSlice(); - const ts = parameters.block_table.shape().computeElementStrides().constSlice(); - - const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(u64, @intCast(@max(16, q.dim(.hd))))); - - const config: kernels_sparse.SparseAttentionConfig = .{ - .dtype = triton.from(q.dtype()), - .num_seqs = @intCast(parameters.block_table.dim(.b)), - .num_query_heads = num_kv_heads * group, - .num_queries_per_kv = group, - .head_size = q.dim(.hd), - .head_size_padded = head_size_padded, - .block_m = @intCast(@max(16, std.math.ceilPowerOfTwoAssert(u64, @intCast(group)))), - .block_size = k_cache.dim(.k_chunk), - .tile_size = @intCast(tuning.tile_size), - .indices_count = indices.dim(.topk), - .num_segments = @intCast(tuning.num_segments), - .decode_sequence_grid = all_decode and parameters.options_.max_seqlen_q == 1, - .use_sinks = opts.sink != null, - .scale = opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))), - .block_table_stride = ts[parameters.block_table.axis(.b)], - .query_strides = .{ qs[q_shape.axis(.b)], qs[q_shape.axis(.h)], qs[q_shape.axis(.hd)] }, - .key_strides = .{ ks[k_cache.axis(.page)], ks[k_cache.axis(.k_chunk)], ks[k_cache.axis(.hkv)], ks[k_cache.axis(.hd)] }, - .value_strides = .{ vs[v_cache.axis(.page)], vs[v_cache.axis(.k_chunk)], vs[v_cache.axis(.hkv)], vs[v_cache.axis(.hd)] }, - .indices_strides = .{ is[indices.axis(.b)], is[indices.axis(.topk)] }, - }; - - const inputs: kernels_sparse.KernelSparseAttention3dPtr.Kernel.Inputs = .{ - .query_ptr = q, - .key_ptr = k_cache, - .value_ptr = v_cache, - .indices_ptr = indices, - .block_tables_ptr = parameters.block_table, - .seq_lens_ptr = parameters.seq_lens, - .query_start_len_ptr = parameters.query_start_len, - .sink_ptr = if (opts.sink) |sink| sink.convert(.f32) else .scalar(0, .f32), - }; - - const partition = kernels_sparse.SparseSegments.init(config.indices_count, config.num_segments, config.tile_size); - const grid: [3]i32 = .{ - if (config.decode_sequence_grid) config.num_seqs else @intCast(q.dim(.b)), - @intCast(num_kv_heads), - @intCast(partition.launched), - }; - - const warps: i32 = if (all_decode) 8 else 4; - - const stats_shape = zml.Shape.init(.{ q.dim(.b), config.num_segments, config.num_query_heads }, .f32); - - const partials = kernels_sparse.KernelSparseAttention3dPtr.Kernel.call(inputs, .{ - .output = .init(.{ q.dim(.b), config.num_query_heads, config.num_segments, head_size_padded }, .f32), - .segm_max = stats_shape, - .segm_expsum = stats_shape, - }, .{ - .cfg = config, - .grid = grid, - .num_stages = 1, - .num_warps = warps, - }); - - const head_tile = @min(64, head_size_padded); - - const output = kernels_sparse.SparseReducePtr.Kernel.call(.{ - .segm_output_ptr = partials.output, - .segm_max_ptr = partials.segm_max, - .segm_expsum_ptr = partials.segm_expsum, - .query_start_len_ptr = parameters.query_start_len, - }, .{ .output = q.shape() }, .{ - .cfg = .{ - .dtype = config.dtype, - .num_seqs = config.num_seqs, - .num_query_heads = config.num_query_heads, - .head_size = config.head_size, - .head_size_padded = head_size_padded, - .head_tile = head_tile, - .num_segments = config.num_segments, - .launched_segments = partition.launched, - .output_strides = config.query_strides, - }, - .grid = .{ @intCast(q.dim(.b)), @intCast(config.num_query_heads), @intCast(@divExact(head_size_padded, head_tile)) }, - .num_stages = 1, - .num_warps = 4, - }).output; - - return output; - } - - pub fn pagedAttention2d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { - const config = select2dConfig(paged_attention_opts); - const k_strides = k_cache.shape().computeElementStrides().constSlice(); - const v_strides = v_cache.shape().computeElementStrides().constSlice(); - - const kernel_config: kernels.KernelUnifiedAttention2dPtr.Config = .{ - .q_dtype = triton.from(q.dtype()), - .kv_dtype = triton.from(k_cache.dtype()), - .k_strides = .{ k_strides[0], k_strides[1], k_strides[2] }, - .v_strides = .{ v_strides[0], v_strides[1], v_strides[2] }, - .o_dtype = triton.from(q.dtype()), - .num_query_heads = @intCast(paged_attention_opts.num_heads), - .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), - .block_size = @intCast(paged_attention_opts.block_size), - .tile_size = @intCast(config.tile_size), - .head_size = @intCast(paged_attention_opts.head_dim), - .head_size_padded = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)), - .use_alibi_slopes = false, - .use_qq_bias = false, - .use_softcap = false, - .use_sinks = (opts.sink != null), - .sliding_window = @intCast(paged_attention_opts.sliding_window), - .block_q = @intCast(config.block_q), - .block_m = @intCast(config.block_m), - .use_fp8 = false, - .all_decode = paged_attention_opts.all_decode, - .is_causal = opts.mask.isCausal(), - }; - log.debug("pagedAttention2d config: {any}", .{kernel_config}); - - const dummy: zml.Tensor = .scalar(0, .i8); - const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); - - const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); - const q_strides = q_shape.computeElementStrides().constSlice(); - - const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); - const num_seqs = parameters.block_table.dim(0); - - const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; - - const output = kernels.KernelUnifiedAttention2dPtr.Kernel.call( - .{ - .query_ptr = q, - .key_cache_ptr = k_cache, - .value_cache_ptr = v_cache, - .sink_ptr = sink, - .block_tables_ptr = parameters.block_table, - .seq_lens_ptr = parameters.seq_lens, - .alibi_slopes_ptr = dummy, - .qq_bias_ptr = dummy, - .scale_ptr = .scalar(scale, .f32), - .k_scale_ptr = dummy, - .v_scale_ptr = dummy, - .out_scale_ptr = dummy, - .softcap_ptr = dummy, - .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), - .query_stride_0_ptr = .scalar(q_strides[0], .i64), - .query_stride_1_ptr = .scalar(q_strides[1], .i64), - .output_stride_0_ptr = .scalar(q_strides[0], .i64), - .output_stride_1_ptr = .scalar(q_strides[1], .i64), - .qq_bias_stride_0_ptr = dummy, - .query_start_len_ptr = parameters.query_start_len, - .num_seqs_ptr = .scalar(num_seqs, .i32), - }, - .{ .output = q.shape() }, - .{ - .cfg = kernel_config, - .grid = .{ @intCast(paged_attention_opts.num_kv_heads), @intCast(config.total_q_blocks), 1 }, - .num_stages = @intCast(config.num_stages), - .num_warps = @intCast(config.num_warps), - }, - ); - return output.output; - } - - pub fn pagedAttention3d(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { - const config = select3dConfig(paged_attention_opts); - const k_strides = k_cache.shape().computeElementStrides().constSlice(); - const v_strides = v_cache.shape().computeElementStrides().constSlice(); - - const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); - const attn_kernel_config: kernels.KernelUnifiedAttention3dPtr.Config = .{ - .q_dtype = triton.from(q.dtype()), - .kv_dtype = triton.from(k_cache.dtype()), - .k_strides = .{ k_strides[0], k_strides[1], k_strides[2] }, - .v_strides = .{ v_strides[0], v_strides[1], v_strides[2] }, - .num_query_heads = @intCast(paged_attention_opts.num_heads), - .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), - .block_size = @intCast(paged_attention_opts.block_size), - .tile_size = @intCast(config.attention.tile_size), - .head_size = @intCast(paged_attention_opts.head_dim), - .head_size_padded = head_size_padded, - .use_alibi_slopes = false, - .use_qq_bias = false, - .use_softcap = false, - .use_sinks = (opts.sink != null), - .sliding_window = @intCast(paged_attention_opts.sliding_window), - .block_q = @intCast(config.attention.block_q), - .block_m = @intCast(config.attention.block_m), - .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), - .all_decode = paged_attention_opts.all_decode, - .is_causal = opts.mask.isCausal(), - }; - log.debug("pagedAttention3d attention config: {any}", .{attn_kernel_config}); - - const reduce_kernel_config: kernels.ReduceSegmentsPtr.Config = .{ - .o_dtype = triton.from(q.dtype()), - .num_query_heads = @intCast(paged_attention_opts.num_heads), - .tile_size = @intCast(config.reduce.tile_size), - .head_size = @intCast(paged_attention_opts.head_dim), - .head_size_padded = head_size_padded, - .block_q = @intCast(config.reduce.block_q), - .num_segments_per_seq = @intCast(config.reduce.num_segments_per_seq), - .use_fp8 = false, - }; - log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); - - const dummy: zml.Tensor = .scalar(0, .i8); - const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); - - const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); - const q_strides = q_shape.computeElementStrides().constSlice(); - - const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); - const num_seqs = parameters.block_table.dim(0); - - const attn_grid: [3]i32 = .{ - @intCast(config.attention.total_q_blocks), - @intCast(paged_attention_opts.num_kv_heads), - @intCast(config.attention.num_segments_per_seq), - }; - - const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; - - const attn_output = kernels.KernelUnifiedAttention3dPtr.Kernel.call( - .{ - .query_ptr = q, - .key_cache_ptr = k_cache, - .value_cache_ptr = v_cache, - .sink_ptr = sink, - .block_tables_ptr = parameters.block_table, - .seq_lens_ptr = parameters.seq_lens, - .alibi_slopes_ptr = dummy, - .qq_bias_ptr = dummy, - .scale_ptr = .scalar(scale, .f32), - .k_scale_ptr = dummy, - .v_scale_ptr = dummy, - .softcap_ptr = dummy, - .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), - .query_stride_0_ptr = .scalar(q_strides[0], .i64), - .query_stride_1_ptr = .scalar(q_strides[1], .i64), - .qq_bias_stride_0_ptr = dummy, - .query_start_len_ptr = parameters.query_start_len, - .num_seqs_ptr = .scalar(num_seqs, .i32), - }, - .{ - .segm_output = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), - .segm_max = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), - .segm_expsum = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), - }, - .{ - .cfg = attn_kernel_config, - .grid = attn_grid, - .num_stages = @intCast(config.attention.num_stages), - .num_warps = @intCast(config.attention.num_warps), - }, - ); - - const output = kernels.ReduceSegmentsPtr.Kernel.call( - .{ - .segm_output_ptr = attn_output.segm_output, - .segm_max_ptr = attn_output.segm_max, - .segm_expsum_ptr = attn_output.segm_expsum, - .seq_lens_ptr = parameters.seq_lens, - .num_seqs_ptr = .scalar(num_seqs, .i32), - .out_scale_inv_ptr = dummy, - .output_stride_0_ptr = .scalar(q_strides[0], .i64), - .output_stride_1_ptr = .scalar(q_strides[1], .i64), - .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), - .query_start_len_ptr = parameters.query_start_len, - }, - .{ .output = q.shape() }, - .{ - .cfg = reduce_kernel_config, - .grid = .{ - @intCast(paged_attention_opts.num_tokens), - @intCast(paged_attention_opts.num_heads), - 1, - }, - .num_stages = @intCast(config.reduce.num_stages), - .num_warps = @intCast(config.reduce.num_warps), - }, - ); - - return output.output; - } - - /// oneAPI/Intel specialization of pagedAttention3d: routes the attention pass to - /// the SIMD16-tuned kernel (compile-time KV strides, cached loads, same-page fast - /// path) while reusing the shared segment-reduce kernel. See unified_attention_oneapi.zig. - pub fn pagedAttention3dOneapi(parameters: Parameters, q: zml.Tensor, k_cache: zml.Tensor, v_cache: zml.Tensor, opts: AttentionOptions, paged_attention_opts: PagedAttentionOptions) zml.Tensor { - const config = select3dConfig(paged_attention_opts); - - const head_size_padded: i64 = @intCast(std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim)); - - const k_strides = k_cache.shape().computeElementStrides().constSlice(); - const v_strides = v_cache.shape().computeElementStrides().constSlice(); - - const attn_kernel_config: kernels_oneapi.KernelUnifiedAttention3dPtr.Config = .{ - .q_dtype = triton.from(q.dtype()), - .kv_dtype = triton.from(k_cache.dtype()), - .num_query_heads = @intCast(paged_attention_opts.num_heads), - .num_queries_per_kv = @intCast(paged_attention_opts.numQueriesPerKv()), - .block_size = @intCast(paged_attention_opts.block_size), - .tile_size = @intCast(config.attention.tile_size), - .head_size = @intCast(paged_attention_opts.head_dim), - .head_size_padded = head_size_padded, - .use_alibi_slopes = false, - .use_qq_bias = false, - .use_softcap = false, - .use_sinks = (opts.sink != null), - .sliding_window = @intCast(paged_attention_opts.sliding_window), - .is_causal = opts.mask.isCausal(), - .block_q = @intCast(config.attention.block_q), - .block_m = @intCast(config.attention.block_m), - .num_segments_per_seq = @intCast(config.attention.num_segments_per_seq), - .all_decode = paged_attention_opts.all_decode, - .stride_k_cache_0 = k_strides[k_cache.axis(.page)], - .stride_k_cache_1 = k_strides[k_cache.axis(.k_chunk)], - .stride_k_cache_2 = k_strides[k_cache.axis(.hkv)], - .stride_v_cache_0 = v_strides[v_cache.axis(.page)], - .stride_v_cache_1 = v_strides[v_cache.axis(.k_chunk)], - .stride_v_cache_2 = v_strides[v_cache.axis(.hkv)], - // Intel: keep KV loads cached (.none); the streaming .cg hint lowers to - // fully-uncached here and defeats the L2 prefetcher. - .kv_cache_modifier = .none, - }; - log.debug("pagedAttention3dOneapi attention config: {any}", .{attn_kernel_config}); - - const reduce_kernel_config: kernels.ReduceSegmentsPtr.Config = .{ - .o_dtype = triton.from(q.dtype()), - .num_query_heads = @intCast(paged_attention_opts.num_heads), - .tile_size = @intCast(config.reduce.tile_size), - .head_size = @intCast(paged_attention_opts.head_dim), - .head_size_padded = head_size_padded, - .block_q = @intCast(config.reduce.block_q), - .num_segments_per_seq = @intCast(config.reduce.num_segments_per_seq), - .use_fp8 = false, - }; - log.debug("pagedAttention3d reduce config: {any}", .{reduce_kernel_config}); - - const dummy: zml.Tensor = .scalar(0, .i8); - const block_table_strides = parameters.block_table.shape().computeElementStrides().constSlice(); - - const q_shape = q.shape().mergeAxes(.{ .h = .{ .hkv, .hg } }); - const q_strides = q_shape.computeElementStrides().constSlice(); - - const scale: f32 = paged_attention_opts.scale orelse @floatCast(1.0 / @sqrt(@as(f64, @floatFromInt(q.dim(.hd))))); - const num_seqs = parameters.block_table.dim(0); - - const attn_grid: [3]i32 = .{ @intCast(config.attention.total_q_blocks), @intCast(paged_attention_opts.num_kv_heads), @intCast(config.attention.num_segments_per_seq) }; - - const sink = if (opts.sink) |sink| sink.convert(.f32) else dummy; - - const attn_output = kernels_oneapi.KernelUnifiedAttention3dPtr.Kernel.call( - .{ - .query_ptr = q, - .key_cache_ptr = k_cache, - .value_cache_ptr = v_cache, - .sink_ptr = sink, - .block_tables_ptr = parameters.block_table, - .seq_lens_ptr = parameters.seq_lens, - .alibi_slopes_ptr = dummy, - .qq_bias_ptr = dummy, - .scale_ptr = .scalar(scale, .f32), - .k_scale_ptr = dummy, - .v_scale_ptr = dummy, - .softcap_ptr = dummy, - .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), - .query_stride_0_ptr = .scalar(q_strides[0], .i64), - .query_stride_1_ptr = .scalar(q_strides[1], .i64), - .qq_bias_stride_0_ptr = dummy, - .stride_k_cache_0_ptr = .scalar(k_strides[k_cache.axis(.page)], .i64), - .stride_k_cache_1_ptr = .scalar(k_strides[k_cache.axis(.k_chunk)], .i64), - .stride_k_cache_2_ptr = .scalar(k_strides[k_cache.axis(.hkv)], .i64), - .stride_v_cache_0_ptr = .scalar(v_strides[v_cache.axis(.page)], .i64), - .stride_v_cache_1_ptr = .scalar(v_strides[v_cache.axis(.k_chunk)], .i64), - .stride_v_cache_2_ptr = .scalar(v_strides[v_cache.axis(.hkv)], .i64), - .query_start_len_ptr = parameters.query_start_len, - .num_seqs_ptr = .scalar(num_seqs, .i32), - }, - .{ - .segm_output = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq, std.math.ceilPowerOfTwoAssert(usize, paged_attention_opts.head_dim) }, .f32), - .segm_max = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), - .segm_expsum = zml.Shape.init(.{ paged_attention_opts.num_tokens, paged_attention_opts.num_heads, config.attention.num_segments_per_seq }, .f32), - }, - .{ - .cfg = attn_kernel_config, - .grid = attn_grid, - .num_stages = @intCast(config.attention.num_stages), - .num_warps = @intCast(config.attention.num_warps), - }, - ); - - const output = kernels.ReduceSegmentsPtr.Kernel.call( - .{ - .segm_output_ptr = attn_output.segm_output, - .segm_max_ptr = attn_output.segm_max, - .segm_expsum_ptr = attn_output.segm_expsum, - .seq_lens_ptr = parameters.seq_lens, - .num_seqs_ptr = .scalar(num_seqs, .i32), - .out_scale_inv_ptr = dummy, - .output_stride_0_ptr = .scalar(q_strides[0], .i64), - .output_stride_1_ptr = .scalar(q_strides[1], .i64), - .block_table_stride_ptr = .scalar(block_table_strides[0], .i64), - .query_start_len_ptr = parameters.query_start_len, - }, - .{ .output = q.shape() }, - .{ - .cfg = reduce_kernel_config, - .grid = .{ @intCast(paged_attention_opts.num_tokens), @intCast(paged_attention_opts.num_heads), 1 }, - .num_stages = @intCast(config.reduce.num_stages), - .num_warps = @intCast(config.reduce.num_warps), - }, - ); - - return output.output; - } -}; diff --git a/zml/kernels/paged_attention/triton/unified_attention.zig b/zml/kernels/paged_attention/triton/unified_attention.zig deleted file mode 100644 index 7b2b9428..00000000 --- a/zml/kernels/paged_attention/triton/unified_attention.zig +++ /dev/null @@ -1,1121 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const ttir = @import("mlir/dialects/ttir"); - -const zml = @import("../../../zml.zig"); -const tri = zml.kernel.triton; -const Builder = tri.Builder; -const Value = tri.Value; -const DType = tri.DType; - -const log = std.log.scoped(.@"zml/attention/triton"); - -pub const FP8_E4M3_MIN: f32 = -448.0; -pub const FP8_E4M3_MAX: f32 = 448.0; - -/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. -const RCP_LN2: f32 = 1.4426950408889634; - -pub const KernelUnifiedAttention2dPtr = struct { - pub const Config = struct { - q_dtype: DType, - kv_dtype: DType, - k_strides: [3]i64, - v_strides: [3]i64, - o_dtype: DType, - scale_dtype: DType = .f32, - - num_query_heads: i64, - num_queries_per_kv: i64, - - block_size: i64, - tile_size: i64, - head_size: i64, - head_size_padded: i64, - - use_alibi_slopes: bool, - use_qq_bias: bool, - use_softcap: bool, - use_sinks: bool, - sliding_window: i64, - - stride_k_cache_3: i64 = 1, - stride_v_cache_3: i64 = 1, - - block_q: i64, - block_m: i64, - use_fp8: bool, - fp8_min: f32 = FP8_E4M3_MIN, - fp8_max: f32 = FP8_E4M3_MAX, - all_decode: bool = false, - is_causal: bool = true, - }; - - pub const Kernel = tri.Kernel(Config, .{ - .name = "kernel_unified_attention_2d_ptr", - .inputs = &.{ - "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", - "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", - "scale_ptr", "k_scale_ptr", "v_scale_ptr", "out_scale_ptr", - "softcap_ptr", "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", - "output_stride_0_ptr", "output_stride_1_ptr", "qq_bias_stride_0_ptr", "query_start_len_ptr", - "num_seqs_ptr", - }, - .outputs = &.{"output"}, - .run = run, - }); - fn run(b: *Builder, cfg: Config) tri.FinishError!void { - const a = try b.declareArgs(.{ - .query_ptr = .{ .ptr = cfg.q_dtype }, - .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, - .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, - .sink_ptr = .{ .ptr = cfg.scale_dtype }, - .block_tables_ptr = .{ .ptr = .i32 }, - .seq_lens_ptr = .{ .ptr = .i32 }, - .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, - .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, - .scale_ptr = .{ .ptr = .f32 }, - .k_scale_ptr = .{ .ptr = .f32 }, - .v_scale_ptr = .{ .ptr = .f32 }, - .out_scale_ptr = .{ .ptr = .f32 }, - .softcap_ptr = .{ .ptr = .f32 }, - .block_table_stride_ptr = .{ .ptr = .i64 }, - .query_stride_0_ptr = .{ .ptr = .i64 }, - .query_stride_1_ptr = .{ .ptr = .i64 }, - .output_stride_0_ptr = .{ .ptr = .i64 }, - .output_stride_1_ptr = .{ .ptr = .i64 }, - .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, - .query_start_len_ptr = .{ .ptr = .i32 }, - .num_seqs_ptr = .{ .ptr = .i32 }, - .output_ptr = .{ .ptr = cfg.o_dtype }, - }); - - const scale = b.load(a.scale_ptr); - const k_scale = b.load(a.k_scale_ptr); - const v_scale = b.load(a.v_scale_ptr); - const out_scale = b.load(a.out_scale_ptr); - const softcap = b.load(a.softcap_ptr); - const block_table_stride = b.load(a.block_table_stride_ptr); - const query_stride_0 = b.load(a.query_stride_0_ptr); - const query_stride_1 = b.load(a.query_stride_1_ptr); - const output_stride_0 = b.load(a.output_stride_0_ptr); - const output_stride_1 = b.load(a.output_stride_1_ptr); - const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); - const num_seqs = b.load(a.num_seqs_ptr); - - kernelUnifiedAttention2d( - b, - a.output_ptr, - a.query_ptr, - a.key_cache_ptr, - a.value_cache_ptr, - a.sink_ptr, - a.block_tables_ptr, - a.seq_lens_ptr, - a.alibi_slopes_ptr, - a.qq_bias_ptr, - scale, - k_scale, - v_scale, - out_scale, - softcap, - block_table_stride, - query_stride_0, - query_stride_1, - output_stride_0, - output_stride_1, - qq_bias_stride_0, - a.query_start_len_ptr, - num_seqs, - cfg, - ); - } -}; - -/// Direct port of Python `kernel_unified_attention_3d_ptr`. -pub const KernelUnifiedAttention3dPtr = struct { - pub const Config = struct { - q_dtype: DType, - kv_dtype: DType, - k_strides: [3]i64, - v_strides: [3]i64, - scale_dtype: DType = .f32, - - num_query_heads: i64, - num_queries_per_kv: i64, - - block_size: i64, - tile_size: i64, - head_size: i64, - head_size_padded: i64, - - use_alibi_slopes: bool, - use_qq_bias: bool, - use_softcap: bool, - use_sinks: bool, - sliding_window: i64, - - stride_k_cache_3: i64 = 1, - stride_v_cache_3: i64 = 1, - - block_q: i64, - block_m: i64, - num_segments_per_seq: i64, - all_decode: bool = false, - is_causal: bool = true, - }; - - pub const Kernel = tri.Kernel(Config, .{ - .name = "kernel_unified_attention_3d_ptr", - .inputs = &.{ - "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", - "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", - "scale_ptr", "k_scale_ptr", "v_scale_ptr", "softcap_ptr", - "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", - "query_start_len_ptr", "num_seqs_ptr", - }, - .outputs = &.{ "segm_output", "segm_max", "segm_expsum" }, - .run = run, - }); - fn run(b: *Builder, cfg: Config) tri.FinishError!void { - const a = try b.declareArgs(.{ - .query_ptr = .{ .ptr = cfg.q_dtype }, - .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, - .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, - .sink_ptr = .{ .ptr = cfg.scale_dtype }, - .block_tables_ptr = .{ .ptr = .i32 }, - .seq_lens_ptr = .{ .ptr = .i32 }, - .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, - .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, - .scale_ptr = .{ .ptr = .f32 }, - .k_scale_ptr = .{ .ptr = .f32 }, - .v_scale_ptr = .{ .ptr = .f32 }, - .softcap_ptr = .{ .ptr = .f32 }, - .block_table_stride_ptr = .{ .ptr = .i64 }, - .query_stride_0_ptr = .{ .ptr = .i64 }, - .query_stride_1_ptr = .{ .ptr = .i64 }, - .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, - .query_start_len_ptr = .{ .ptr = .i32 }, - .num_seqs_ptr = .{ .ptr = .i32 }, - .segm_output_ptr = .{ .ptr = .f32 }, - .segm_max_ptr = .{ .ptr = .f32 }, - .segm_expsum_ptr = .{ .ptr = .f32 }, - }); - - const scale = b.load(a.scale_ptr); - const k_scale = b.load(a.k_scale_ptr); - const v_scale = b.load(a.v_scale_ptr); - const softcap = b.load(a.softcap_ptr); - const block_table_stride = b.load(a.block_table_stride_ptr); - const query_stride_0 = b.load(a.query_stride_0_ptr); - const query_stride_1 = b.load(a.query_stride_1_ptr); - const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); - const num_seqs = b.load(a.num_seqs_ptr); - - kernelUnifiedAttention3d( - b, - a.segm_output_ptr, - a.segm_max_ptr, - a.segm_expsum_ptr, - a.query_ptr, - a.key_cache_ptr, - a.value_cache_ptr, - a.sink_ptr, - a.block_tables_ptr, - a.seq_lens_ptr, - a.alibi_slopes_ptr, - a.qq_bias_ptr, - scale, - k_scale, - v_scale, - softcap, - block_table_stride, - query_stride_0, - query_stride_1, - qq_bias_stride_0, - a.query_start_len_ptr, - num_seqs, - cfg, - ); - } -}; - -/// Direct port of Python `reduce_segments_ptr`. -pub const ReduceSegmentsPtr = struct { - pub const Config = struct { - o_dtype: DType, - scale_dtype: DType = .f32, - - num_query_heads: i64, - tile_size: i64, - head_size: i64, - head_size_padded: i64, - - block_q: i64, - num_segments_per_seq: i64, - - use_fp8: bool, - fp8_min: f32 = FP8_E4M3_MIN, - fp8_max: f32 = FP8_E4M3_MAX, - }; - - pub const Kernel = tri.Kernel(Config, .{ - .name = "reduce_segments_ptr", - .inputs = &.{ - "segm_output_ptr", "segm_max_ptr", "segm_expsum_ptr", "seq_lens_ptr", - "num_seqs_ptr", "out_scale_inv_ptr", "output_stride_0_ptr", "output_stride_1_ptr", - "block_table_stride_ptr", "query_start_len_ptr", - }, - .outputs = &.{"output"}, - .run = run, - }); - fn run(b: *Builder, cfg: Config) tri.FinishError!void { - const a = try b.declareArgs(.{ - .segm_output_ptr = .{ .ptr = .f32 }, - .segm_max_ptr = .{ .ptr = .f32 }, - .segm_expsum_ptr = .{ .ptr = .f32 }, - .seq_lens_ptr = .{ .ptr = .i32 }, - .num_seqs_ptr = .{ .ptr = .i32 }, - .out_scale_inv_ptr = .{ .ptr = .f32 }, - .output_stride_0_ptr = .{ .ptr = .i64 }, - .output_stride_1_ptr = .{ .ptr = .i64 }, - .block_table_stride_ptr = .{ .ptr = .i64 }, - .query_start_len_ptr = .{ .ptr = .i32 }, - .output_ptr = .{ .ptr = cfg.o_dtype }, - }); - - const num_seqs = b.load(a.num_seqs_ptr); - const out_scale_inv = b.load(a.out_scale_inv_ptr); - const output_stride_0 = b.load(a.output_stride_0_ptr); - const output_stride_1 = b.load(a.output_stride_1_ptr); - const block_table_stride = b.load(a.block_table_stride_ptr); - - reduceSegments( - b, - a.output_ptr, - a.segm_output_ptr, - a.segm_max_ptr, - a.segm_expsum_ptr, - a.seq_lens_ptr, - num_seqs, - out_scale_inv, - output_stride_0, - output_stride_1, - block_table_stride, - a.query_start_len_ptr, - cfg, - ); - } -}; - -// ============================================================================ -// Helpers -// ============================================================================ - -fn ctx() *mlir.Context { - return zml.Compiler.current().mlir_ctx; -} - -fn isFp8(dt: DType) bool { - return dt == .f8e4m3fn or dt == .f8e5m2; -} - -fn fastExp(k: *Builder, x: Value) Value { - return k.exp2(x.mul(RCP_LN2)); -} - -fn cdivFn(x: Value, y: anytype) Value { - return switch (@typeInfo(@TypeOf(y))) { - .comptime_int, .comptime_float => x.add(y - 1).div(y), - else => x.add(y).sub(1).div(y), - }; -} - -fn applySoftcap(k: *Builder, s_val: Value, x: Value) Value { - const sdiv = s_val.div(x); - const p1 = k.exp2(sdiv); - const p2 = k.exp2(k.negf(sdiv)); - return x.mul(p1.sub(p2)).div(p1.add(p2)); -} - -fn findSeqIdx( - k: *Builder, - query_start_len_ptr: Value, - target_idx: Value, - num_seqs: Value, - block_q: i64, - use_q_block_mode: bool, -) Value { - const left_init = k.constant(.i32, 0); - - var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); - { - const left = w.before_carried[0]; - const right = w.before_carried[1]; - w.yieldBefore(left.lt(right), .{ left, right }); - } - { - const left = w.after_carried[0]; - const right = w.after_carried[1]; - const mid = left.add(right).div(2); - const val = k.load(query_start_len_ptr.addPtr(mid)); - const mid_val = if (use_q_block_mode) - val.div(@as(i32, @intCast(block_q))).add(mid) - else - val; - - var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); - { - i.yieldThen(.{ mid.add(1), right }); - } - { - i.yieldElse(.{ left, mid }); - } - w.yieldAfter(.{ i.results[0], i.results[1] }); - } - - return w.results[0].sub(1); -} - -fn dequantKv(loaded: Value, scale: Value, kv_is_fp8: bool, q_dtype: DType) Value { - if (!kv_is_fp8 or isFp8(q_dtype)) return loaded; - return loaded.to(.f32).mul(scale).to(q_dtype); -} - -fn kernelUnifiedAttention2d( - k: *Builder, - output_ptr: Value, - query_ptr: Value, - key_cache_ptr: Value, - value_cache_ptr: Value, - sink_ptr: Value, - block_tables_ptr: Value, - seq_lens_ptr: Value, - alibi_slopes_ptr: Value, - qq_bias_ptr: Value, - scale: Value, - k_scale: Value, - v_scale: Value, - out_scale: Value, - softcap: Value, - block_table_stride: Value, - query_stride_0: Value, - query_stride_1: Value, - output_stride_0: Value, - output_stride_1: Value, - qq_bias_stride_0: Value, - query_start_len_ptr: Value, - num_seqs: Value, - config: KernelUnifiedAttention2dPtr.Config, -) void { - const BLOCK_M: i64 = config.block_m; - const BLOCK_Q: i64 = config.block_q; - const BLOCK_SIZE: i64 = config.block_size; - const TILE_SIZE: i64 = config.tile_size; - const HEAD_SIZE: i64 = config.head_size; - const HEAD_SIZE_PADDED: i64 = config.head_size_padded; - const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; - const NUM_QUERY_HEADS: i64 = config.num_query_heads; - const SLIDING_WINDOW: i64 = config.sliding_window; - const stride_k_cache_0 = k.constant(.i64, config.k_strides[0]); - const stride_k_cache_1 = k.constant(.i64, config.k_strides[1]); - const stride_k_cache_2 = k.constant(.i64, config.k_strides[2]); - const stride_v_cache_0 = k.constant(.i64, config.v_strides[0]); - const stride_v_cache_1 = k.constant(.i64, config.v_strides[1]); - const stride_v_cache_2 = k.constant(.i64, config.v_strides[2]); - - const kv_head_idx = k.programId(.x); - const q_block_global_idx = k.programId(.y); - - const qk_scale = scale.mul(RCP_LN2); - - const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); - - const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); - const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); - - const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); - - const cur_batch_in_all_start_index = q_start_raw; - // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. - const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); - const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); - - // Python: if q_block_local_idx * BLOCK_Q >= cur_batch_query_len: return - const out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); - k.returnIf(out_of_range); - - const offs_m = k.arange(0, BLOCK_M, .i32); - const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); - const offs_t = k.arange(0, TILE_SIZE, .i32); - - // query_pos = q_block_local_idx * BLOCK_Q + offs_m // NUM_QUERIES_PER_KV - const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); - const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); - - const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); - const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) - .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); - - const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); - const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); - const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); - - const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) - else - k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); - const query_mask_0 = query_pos.lt(cur_batch_query_len); - const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); - - // Q : (BLOCK_M, HEAD_SIZE_PADDED) - const q_mask_ab = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED); - const q_mask_c = query_mask_1.expandDims(1); - const q_mask = q_mask_ab.bitAnd(q_mask_c); - const q_cache_mod: ttir.CacheModifier = - if (config.all_decode or BLOCK_M >= NUM_QUERY_HEADS) .cg else .none; - const Q = k.loadOpts(query_ptr.addPtr(query_offset), .{ - .mask = q_mask, - .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), - .cache_modifier = q_cache_mod, - }); - - const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); - - // M init — sinks or -inf. - const m_init: Value = if (config.use_sinks) mb: { - const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ - .mask = query_mask_1, - .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), - }); - break :mb loaded.to(.f32).mul(RCP_LN2); - } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); - - const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); - const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); - - const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); - const context_len = seq_len.sub(cur_batch_query_len); - - const alibi_slope: Value = if (config.use_alibi_slopes) - k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ - .mask = query_mask_1, - .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), - }) - else - k.zeros(&.{BLOCK_M}, config.scale_dtype); - - // In the `use_qq_bias = true` branch, the tensor offset auto-splats the - // scalar ptr via addPtr. In the else branch there is no offset, so we - // need an explicit splatTo to give expandDims a tensor to work on. - const qq_bias_row_ptrs: Value = if (config.use_qq_bias) - qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) - else - qq_bias_ptr.splatTo(&.{BLOCK_M}); - - // In causal mode, each query block only scans keys up to the last query - // represented in this block. In non-causal mode, scan the whole sequence - // so draft rows can attend to all rows written for the same sequence. - const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); - const max_seq_prefix_len = if (config.is_causal) b: { - const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); - break :b max_prefix_raw.minimum(seq_len); - } else seq_len; - - const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); - - // Sliding-window tile pruning — comptime branch. - var tile_start: Value = k.constant(.i32, 0); - var tile_end: Value = num_tiles; - if (SLIDING_WINDOW > 0) { - const qpos_lo = q_local_bq; - if (config.is_causal) { - const qpos_hi_raw = qpos_lo.add(pad_term - 1); - const qpos_hi = qpos_hi_raw.minimum(cur_batch_query_len.sub(1)); - const first_allowed_key = context_len.add(qpos_lo).sub(@as(i32, @intCast(SLIDING_WINDOW - 1))); - const last_allowed_key = context_len.add(qpos_hi); - tile_start = first_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).maximum(0); - tile_end = last_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).add(1).minimum(num_tiles); - } else { - const first_allowed_key = context_len.add(qpos_lo).sub(@as(i32, @intCast(SLIDING_WINDOW - 1))); - tile_start = first_allowed_key.div(@as(i32, @intCast(TILE_SIZE))).maximum(0); - } - } - - const kv_cache_mod: ttir.CacheModifier = - if (config.all_decode) .cg else .none; - - const block_tables_ptr_shifted = block_tables_ptr.addPtr(block_table_offset); - - // Iterate through tiles (loop carries M, L, acc). - var loop = k.openFor(tile_start, tile_end, 1, .{ m_init, l_init, acc_init }); - { - const j = loop.iv; - const M = loop.carried[0]; - const L = loop.carried[1]; - const acc = loop.carried[2]; - - const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); - - const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); - - const physical_block_idx = k.load( - block_tables_ptr_shifted.addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), - ).to(.i64); - - const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) - .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) - .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) - .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 1).mul(stride_v_cache_1)); - - // k_offset : (HEAD_SIZE_PADDED, TILE_SIZE) - const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) - .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) - .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) - .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 0).mul(stride_k_cache_1)); - - // K : (HEAD_SIZE_PADDED, TILE_SIZE) - const k_mask = k.mask2d(dim_mask, tile_mask, HEAD_SIZE_PADDED, TILE_SIZE); - const K_load = k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ - .mask = k_mask, - .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), - .cache_modifier = kv_cache_mod, - }); - const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); - - // V : (TILE_SIZE, HEAD_SIZE_PADDED) - const v_mask = k.mask2d(tile_mask, dim_mask, TILE_SIZE, HEAD_SIZE_PADDED); - const V_load = k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ - .mask = v_mask, - .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), - .cache_modifier = kv_cache_mod, - }); - const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); - - // S : (BLOCK_M, TILE_SIZE) - const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); - const qk = k.dot(Q, K, acc_zero); - var S = qk_scale.mul(qk); - - if (config.use_softcap) { - S = applySoftcap(k, S, softcap).mul(RCP_LN2); - } - - const ql_2d_i32 = query_pos.expandDims(1); - const so_2d = seq_offset.expandDims(0); - const seq_mask = if (config.is_causal) b: { - // seq_offset[None, :] < context_len + query_pos[:, None] + 1 - const rhs = ql_2d_i32.add(context_len).add(1); - break :b so_2d.lt(rhs); - } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); - - // S = tl.where(qmask_1 & qmask_0 & seq_mask, S, -inf) - const qm0_2d = query_mask_0.expandDims(1); - const qm1_2d = query_mask_1.expandDims(1); - const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); - S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); - - if (SLIDING_WINDOW > 0) { - const diff = ql_2d_i32.add(context_len).sub(so_2d); - const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); - S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); - } - - if (config.use_alibi_slopes) { - const alibi_2d = alibi_slope.expandDims(1); - const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); - S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); - } - - if (config.use_qq_bias) { - const key_rel_pos = seq_offset.sub(context_len); - const is_query_key = key_rel_pos.ge(0) - .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); - const qq_ptrs = qq_bias_row_ptrs.expandDims(1) - .addPtr(key_rel_pos.to(.i64).expandDims(0)); - const qq_bias = k.loadOpts(qq_ptrs, .{ - .mask = is_query_key.expandDims(0), - .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), - }); - S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); - } - - // m_j = max(M, max(S, axis=1)) - var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); - m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); - - // P = exp2(S - m_j[:, None]); l_j = sum(P, axis=1); alpha = exp2(M - m_j) - const mj_2d = m_j.expandDims(1); - const P = k.exp2(S.sub(mj_2d)); - const l_j = k.sumOpts(P, .{ .axis = 1 }); - const alpha = k.exp2(M.sub(m_j)); - - const alpha_2d = alpha.expandDims(1); - const acc_scaled = acc.mul(alpha_2d); - const new_L = L.mul(alpha).add(l_j); - const new_M = m_j; - - const P_cast = P.to(config.q_dtype); - const new_acc = k.dot(P_cast, V, acc_scaled); - - loop.yield(.{ new_M, new_L, new_acc }); - } - const L_final = loop.results[1]; - var acc_final = loop.results[2]; - - const L_2d = k.expandDims(L_final, 1); - const one_over_L = k.full(&.{ BLOCK_M, 1 }, 1.0, .f32).div(L_2d); - acc_final = acc_final.mul(k.broadcastTo(one_over_L, &.{ BLOCK_M, HEAD_SIZE_PADDED })); - - if (config.use_fp8) { - acc_final = acc_final.mul(out_scale); - acc_final = k.clampf( - acc_final, - k.full(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.fp8_min, .f32), - k.full(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.fp8_max, .f32), - ); - } - - const oo0_2d = query_offset_0.expandDims(1).mul(output_stride_0); - const oo1_2d = query_offset_1.expandDims(1).mul(output_stride_1); - const output_offset = oo0_2d.add(oo1_2d).add(k.expandDims(offs_d, 0)); - - const store_mask = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) - .bitAnd(query_mask_1.expandDims(1)); - k.storeOpts( - output_ptr.addPtr(output_offset), - acc_final.to(config.o_dtype), - .{ .mask = store_mask }, - ); -} - -fn kernelUnifiedAttention3d( - k: *Builder, - segm_output_ptr: Value, - segm_max_ptr: Value, - segm_expsum_ptr: Value, - query_ptr: Value, - key_cache_ptr: Value, - value_cache_ptr: Value, - sink_ptr: Value, - block_tables_ptr: Value, - seq_lens_ptr: Value, - alibi_slopes_ptr: Value, - qq_bias_ptr: Value, - scale: Value, - k_scale: Value, - v_scale: Value, - softcap: Value, - block_table_stride: Value, - query_stride_0: Value, - query_stride_1: Value, - qq_bias_stride_0: Value, - query_start_len_ptr: Value, - num_seqs: Value, - config: KernelUnifiedAttention3dPtr.Config, -) void { - const BLOCK_M: i64 = config.block_m; - const BLOCK_Q: i64 = config.block_q; - const BLOCK_SIZE: i64 = config.block_size; - const TILE_SIZE: i64 = config.tile_size; - const HEAD_SIZE: i64 = config.head_size; - const HEAD_SIZE_PADDED: i64 = config.head_size_padded; - const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; - const NUM_QUERY_HEADS: i64 = config.num_query_heads; - const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; - const SLIDING_WINDOW: i64 = config.sliding_window; - const stride_k_cache_0 = k.constant(.i64, config.k_strides[0]); - const stride_k_cache_1 = k.constant(.i64, config.k_strides[1]); - const stride_k_cache_2 = k.constant(.i64, config.k_strides[2]); - const stride_v_cache_0 = k.constant(.i64, config.v_strides[0]); - const stride_v_cache_1 = k.constant(.i64, config.v_strides[1]); - const stride_v_cache_2 = k.constant(.i64, config.v_strides[2]); - - const q_block_global_idx = k.programId(.x); - const kv_head_idx = k.programId(.y); - const segm_idx = k.programId(.z); - - const qk_scale = scale.mul(RCP_LN2); - - const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); - - const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); - const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); - const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); - - const cur_batch_in_all_start_index = q_start_raw; - // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. - const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); - const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); - - const q_out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); - k.returnIf(q_out_of_range); - - const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); - const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); - - const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); - const segm_out_of_range = segm_lo.ge(seq_len); - k.returnIf(segm_out_of_range); - - const offs_m = k.arange(0, BLOCK_M, .i32); - const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); - const offs_t = k.arange(0, TILE_SIZE, .i32); - - const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); - const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); - - const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); - const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) - .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); - - // query_offset = qo_0[:, None]*qstride_0 + qo_1[:, None]*qstride_1 + offs_d[None, :] - // Inline `expandDims(offs_d, 0)` *inside* the chain so it's emitted after - // the qo0+qo1 addi - const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); - const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); - const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); - - const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) - else - k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); - const query_mask_0 = query_pos.lt(cur_batch_query_len); - const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); - - const q_mask_pre = k.expandDims(query_mask_0, 1).bitAnd(k.expandDims(query_mask_1, 1)); - const q_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - q_mask_pre.bitAnd(k.expandDims(dim_mask, 0)) - else - q_mask_pre; - const q_ptrs = query_ptr.addPtr(query_offset); - const Q = k.loadOpts(q_ptrs, .{ - .mask = k.broadcastTo(q_mask_partial, &.{ BLOCK_M, HEAD_SIZE_PADDED }), - .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), - }); - - const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); - - // M init: USE_SINKS + segm_idx==0 → loaded sinks; else -inf. - const segm_is_zero = segm_idx.eq(0); - const m_init: Value = if (config.use_sinks) mb: { - var mi = k.openIfElse(segm_is_zero, .{k.tensorTy(&.{BLOCK_M}, .f32)}); - { - const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ - .mask = query_mask_1, - .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), - }); - mi.yieldThen(.{loaded.to(.f32).mul(RCP_LN2)}); - } - { - mi.yieldElse(.{k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)}); - } - break :mb mi.results[0]; - } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); - - const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); - const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); - - const context_len = seq_len.sub(cur_batch_query_len); - - const alibi_slope: Value = if (config.use_alibi_slopes) - k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ - .mask = query_mask_1, - .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), - }) - else - k.zeros(&.{BLOCK_M}, config.scale_dtype); - - // See 2d-body note on qq_bias_row_ptrs: else branch needs an explicit splatTo - // (no sibling offset to auto-broadcast against). - const qq_bias_row_ptrs: Value = if (config.use_qq_bias) - qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) - else - qq_bias_ptr.splatTo(&.{BLOCK_M}); - - const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); - const max_seq_prefix_len = if (config.is_causal) b: { - const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); - break :b max_prefix_raw.minimum(seq_len); - } else seq_len; - const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); - - const segm_tile_lo = segm_idx.mul(tiles_per_segment); - const segm_tile_hi = segm_idx.add(1).mul(tiles_per_segment).minimum(num_tiles); - - const kv_cache_mod: ttir.CacheModifier = - if (config.all_decode) .cg else .none; - - var loop = k.openFor(segm_tile_lo, segm_tile_hi, 1, .{ m_init, l_init, acc_init }); - { - const j = loop.iv; - const M = loop.carried[0]; - const L = loop.carried[1]; - const acc = loop.carried[2]; - - const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); - const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); - - const physical_block_idx = k.load( - block_tables_ptr.addPtr(block_table_offset).addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), - ).to(.i64); - const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) - .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) - .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) - .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 1).mul(stride_v_cache_1)); - - const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) - .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) - .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) - .add(k.expandDims(seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))), 0).mul(stride_k_cache_1)); - - const k_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - k.expandDims(dim_mask, 1).bitAnd(k.expandDims(tile_mask, 0)) - else - k.expandDims(tile_mask, 0); - const K_load = k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ - .mask = k.broadcastTo(k_mask_partial, &.{ HEAD_SIZE_PADDED, TILE_SIZE }), - .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), - .cache_modifier = kv_cache_mod, - }); - const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); - - const v_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - k.expandDims(tile_mask, 1).bitAnd(k.expandDims(dim_mask, 0)) - else - k.expandDims(tile_mask, 1); - const V_load = k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ - .mask = k.broadcastTo(v_mask_partial, &.{ TILE_SIZE, HEAD_SIZE_PADDED }), - .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), - .cache_modifier = kv_cache_mod, - }); - const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); - - const ql_2d_i32 = query_pos.expandDims(1); - const so_2d = seq_offset.expandDims(0); - const seq_mask = if (config.is_causal) b: { - // seq_offset[None, :] < context_len + query_pos[:, None] + 1 - const rhs = context_len.add(ql_2d_i32).add(1); - break :b so_2d.lt(rhs); - } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); - - const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); - const qk = k.dot(Q, K, acc_zero); - var S = qk_scale.mul(qk); - - if (config.use_softcap) { - S = applySoftcap(k, S, softcap).mul(RCP_LN2); - } - - const qm0_2d = query_mask_0.expandDims(1); - const qm1_2d = query_mask_1.expandDims(1); - const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); - S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); - - if (SLIDING_WINDOW > 0) { - const diff = ql_2d_i32.add(context_len).sub(so_2d); - const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); - S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); - } - - if (config.use_alibi_slopes) { - const alibi_2d = alibi_slope.expandDims(1); - const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); - S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); - } - - if (config.use_qq_bias) { - const key_rel_pos = seq_offset.sub(context_len); - const is_query_key = key_rel_pos.ge(0) - .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); - const qq_ptrs = qq_bias_row_ptrs.expandDims(1) - .addPtr(key_rel_pos.to(.i64).expandDims(0)); - const qq_bias = k.loadOpts(qq_ptrs, .{ - .mask = is_query_key.expandDims(0), - .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), - }); - S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); - } - - var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); - m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); - - const mj_2d = m_j.expandDims(1); - const P = k.exp2(S.sub(mj_2d)); - const l_j = k.sumOpts(P, .{ .axis = 1 }); - const alpha = k.exp2(M.sub(m_j)); - - const alpha_2d = alpha.expandDims(1); - const acc_scaled = acc.mul(alpha_2d); - const new_L = L.mul(alpha).add(l_j); - - const P_cast = P.to(config.q_dtype); - const new_acc = k.dot(P_cast, V, acc_scaled); - - loop.yield(.{ m_j, new_L, new_acc }); - } - const M_final = loop.results[0]; - const L_final = loop.results[1]; - const acc_final = loop.results[2]; - - // segm_output_offset = qo_0[:, None].to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) - // + qo_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) - // + segm_idx * HEAD_SIZE_PADDED + offs_d[None, :] - const segm_out_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); - const segm_out_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; - - // Inline `segm_idx * HEAD_SIZE_PADDED` and `offs_d[None, :]` inside the chain - // so each term's ops emit in source order - const oo0_2d = k.expandDims(query_offset_0, 1).to(.i64).mul(segm_out_token_stride_i64); - const oo1_2d = k.expandDims(query_offset_1, 1).mul(segm_out_head_stride_i32); - const segm_output_offset = oo0_2d.add(oo1_2d) - .add(segm_idx.mul(@as(i32, @intCast(HEAD_SIZE_PADDED)))) - .add(k.expandDims(offs_d, 0)); - - const store_mask_2d = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) - .bitAnd(query_mask_1.expandDims(1)); - k.storeOpts( - segm_output_ptr.addPtr(segm_output_offset), - acc_final, - .{ .mask = store_mask_2d }, - ); - - // segm_offset = qo_0.to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ) - // + qo_1 * NUM_SEGMENTS_PER_SEQ + segm_idx - const segm_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); - const segm_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; - const segm_offset = query_offset_0.to(.i64).mul(segm_token_stride_i64) - .add(query_offset_1.mul(segm_head_stride_i32)) - .add(segm_idx); - const store_mask_1d = query_mask_0.bitAnd(query_mask_1); - k.storeOpts(segm_max_ptr.addPtr(segm_offset), M_final, .{ .mask = store_mask_1d }); - k.storeOpts(segm_expsum_ptr.addPtr(segm_offset), L_final, .{ .mask = store_mask_1d }); -} - -fn reduceSegments( - k: *Builder, - output_ptr: Value, - segm_output_ptr: Value, - segm_max_ptr: Value, - segm_expsum_ptr: Value, - seq_lens_ptr: Value, - num_seqs: Value, - out_scale_inv: Value, - output_stride_0: Value, - output_stride_1: Value, - block_table_stride: Value, - query_start_len_ptr: Value, - config: ReduceSegmentsPtr.Config, -) void { - _ = block_table_stride; - - const TILE_SIZE: i64 = config.tile_size; - const HEAD_SIZE: i64 = config.head_size; - const HEAD_SIZE_PADDED: i64 = config.head_size_padded; - const NUM_QUERY_HEADS: i64 = config.num_query_heads; - const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; - const BLOCK_Q: i64 = config.block_q; - - const query_token_idx = k.programId(.x); - const query_head_idx = k.programId(.y); - - const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); - const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) - else - k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); - const output_offset = query_token_idx.to(.i64).mul(output_stride_0) - .add(query_head_idx.to(.i64).mul(output_stride_1)) - .add(offs_d.to(.i64)); - - // The launch covers the padded query capacity, but the attention pass only - // writes scratch for active tokens. Zero the tail before looking up a - // sequence or dividing by its segment size (zero for inactive sequences). - const active_tokens = k.load(query_start_len_ptr.addPtr(num_seqs)); - const inactive = query_token_idx.ge(active_tokens); - k.storeOpts(output_ptr.addPtr(output_offset), k.zeros(&.{HEAD_SIZE_PADDED}, config.o_dtype), .{ - .mask = dim_mask.bitAnd(inactive), - }); - k.returnIf(inactive); - - const seq_idx = findSeqIdx(k, query_start_len_ptr, query_token_idx, num_seqs, BLOCK_Q, false); - - const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); - const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); - const act_num_segments = cdivFn(seq_len, tiles_per_segment.mul(@as(i32, @intCast(TILE_SIZE)))); - - const seg_range = k.arange(0, NUM_SEGMENTS_PER_SEQ, .i32); - const segm_mask = seg_range.lt(act_num_segments); - - // segm_offset = query_token_idx*NQH*NSPS + query_head_idx*NSPS + arange(NSPS) - const tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; - const head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); - const segm_offset = query_token_idx.to(.i64).mul(tok_stride) - .add(query_head_idx.mul(head_stride).to(.i64)) - .add(seg_range.to(.i64)); - - const segm_max = k.loadOpts(segm_max_ptr.addPtr(segm_offset), .{ - .mask = segm_mask, - .other = k.full(&.{NUM_SEGMENTS_PER_SEQ}, -std.math.inf(f32), .f32), - }); - const overall_max = k.max(segm_max); - - var segm_expsum = k.loadOpts(segm_expsum_ptr.addPtr(segm_offset), .{ - .mask = segm_mask, - .other = k.zeros(&.{NUM_SEGMENTS_PER_SEQ}, .f32), - }); - segm_expsum = segm_expsum.mul(k.exp2(segm_max.sub(overall_max))); - const overall_expsum = k.sum(segm_expsum); - - // segm_output_offset : (NSPS, HSP) - const out_tok_stride: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; - const out_head_stride: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); - const head_size_padded_i32: i32 = @intCast(HEAD_SIZE_PADDED); - const base = query_token_idx.to(.i64).mul(out_tok_stride) - .add(query_head_idx.mul(out_head_stride).to(.i64)); - const seg_off_2d = b: { - const seg_col_i32 = k.expandDims(seg_range, 1); // (NSPS, 1) i32 - const seg_col_scaled = seg_col_i32.mul(head_size_padded_i32); // (NSPS, 1) i32 - break :b seg_col_scaled.to(.i64); // extsi → (NSPS, 1) i64 - }; - const dim_off_2d = b: { - const dim_row_i32 = k.expandDims(offs_d, 0); // (1, HSP) i32 - break :b dim_row_i32.to(.i64); // extsi → (1, HSP) i64 - }; - const seg_offset_2d_full = k.broadcastTo(base.add(seg_off_2d), &.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }); - const dim_offset_2d_full = k.broadcastTo(dim_off_2d, &.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }); - const segm_output_offset = seg_offset_2d_full.add(dim_offset_2d_full); - - var segm_output = k.loadOpts( - segm_output_ptr.addPtr(segm_output_offset), - .{ - .mask = k.mask2d(segm_mask, dim_mask, NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED), - .other = k.zeros(&.{ NUM_SEGMENTS_PER_SEQ, HEAD_SIZE_PADDED }, .f32), - }, - ); - const rescale = k.exp2(segm_max.sub(overall_max)); - segm_output = segm_output.mul(rescale.expandDims(1)); - const acc_sum = k.sumOpts(segm_output, .{ .axis = 0 }); - - var acc = k.where( - overall_expsum.eq(0.0), - k.full(&.{HEAD_SIZE_PADDED}, 0.0, .f32), - acc_sum.div(overall_expsum), - ); - - if (config.use_fp8) { - acc = acc.mul(out_scale_inv); - acc = k.clampf( - acc, - k.full(&.{HEAD_SIZE_PADDED}, config.fp8_min, .f32), - k.full(&.{HEAD_SIZE_PADDED}, config.fp8_max, .f32), - ); - } - - k.storeOpts( - output_ptr.addPtr(output_offset), - acc.to(config.o_dtype), - .{ .mask = dim_mask }, - ); -} diff --git a/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig b/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig deleted file mode 100644 index b7c724f9..00000000 --- a/zml/kernels/paged_attention/triton/unified_attention_oneapi.zig +++ /dev/null @@ -1,536 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const ttir = @import("mlir/dialects/ttir"); - -const zml = @import("../../../zml.zig"); -const tri = zml.kernel.triton; -const Builder = tri.Builder; -const Value = tri.Value; -const DType = tri.DType; - -const log = std.log.scoped(.@"zml/attention/triton"); - -// oneAPI/Intel (SIMD16) specialization of the 3D split-K paged-attention kernel. -// Duplicated from unified_attention.zig so the shared kernel stays backend-neutral; -// only the Intel decode path routes here (see triton_attention.zig pagedAttention3dOneapi). - -/// `log_2(e)` — pre-multiply with this to turn `exp(x)` into `exp2(x * RCP_LN2)`. -const RCP_LN2: f32 = 1.4426950408889634; - -fn isFp8(dt: DType) bool { - return dt == .f8e4m3fn or dt == .f8e5m2; -} - -fn cdivFn(x: Value, y: anytype) Value { - return switch (@typeInfo(@TypeOf(y))) { - .comptime_int, .comptime_float => x.add(y - 1).div(y), - else => x.add(y).sub(1).div(y), - }; -} - -fn applySoftcap(k: *Builder, s_val: Value, x: Value) Value { - const sdiv = s_val.div(x); - const p1 = k.exp2(sdiv); - const p2 = k.exp2(k.negf(sdiv)); - return x.mul(p1.sub(p2)).div(p1.add(p2)); -} - -fn findSeqIdx( - k: *Builder, - query_start_len_ptr: Value, - target_idx: Value, - num_seqs: Value, - block_q: i64, - use_q_block_mode: bool, -) Value { - const left_init = k.constant(.i32, 0); - - var w = k.openWhile(.{ left_init, num_seqs }, .{ .i32, .i32 }); - { - const left = w.before_carried[0]; - const right = w.before_carried[1]; - w.yieldBefore(left.lt(right), .{ left, right }); - } - { - const left = w.after_carried[0]; - const right = w.after_carried[1]; - const mid = left.add(right).div(2); - const val = k.load(query_start_len_ptr.addPtr(mid)); - const mid_val = if (use_q_block_mode) - val.div(@as(i32, @intCast(block_q))).add(mid) - else - val; - - var i = k.openIfElse(mid_val.le(target_idx), .{ .i32, .i32 }); - { - i.yieldThen(.{ mid.add(1), right }); - } - { - i.yieldElse(.{ left, mid }); - } - w.yieldAfter(.{ i.results[0], i.results[1] }); - } - - return w.results[0].sub(1); -} - -fn dequantKv(loaded: Value, scale: Value, kv_is_fp8: bool, q_dtype: DType) Value { - if (!kv_is_fp8 or isFp8(q_dtype)) return loaded; - return loaded.to(.f32).mul(scale).to(q_dtype); -} - -pub const KernelUnifiedAttention3dPtr = struct { - pub const Config = struct { - q_dtype: DType, - kv_dtype: DType, - scale_dtype: DType = .f32, - - num_query_heads: i64, - num_queries_per_kv: i64, - - block_size: i64, - tile_size: i64, - head_size: i64, - head_size_padded: i64, - - use_alibi_slopes: bool, - use_qq_bias: bool, - use_softcap: bool, - use_sinks: bool, - sliding_window: i64, - is_causal: bool = true, - - stride_k_cache_0: i64 = 0, - stride_k_cache_1: i64 = 0, - stride_k_cache_2: i64 = 0, - stride_k_cache_3: i64 = 1, - stride_v_cache_0: i64 = 0, - stride_v_cache_1: i64 = 0, - stride_v_cache_2: i64 = 0, - stride_v_cache_3: i64 = 1, - - block_q: i64, - block_m: i64, - num_segments_per_seq: i64, - all_decode: bool = false, - kv_cache_modifier: ttir.CacheModifier = .none, - }; - - pub const Kernel = tri.Kernel(Config, .{ - .name = "kernel_unified_attention_3d_ptr_oneapi", - .inputs = &.{ - "query_ptr", "key_cache_ptr", "value_cache_ptr", "sink_ptr", - "block_tables_ptr", "seq_lens_ptr", "alibi_slopes_ptr", "qq_bias_ptr", - "scale_ptr", "k_scale_ptr", "v_scale_ptr", "softcap_ptr", - "block_table_stride_ptr", "query_stride_0_ptr", "query_stride_1_ptr", "qq_bias_stride_0_ptr", - "stride_k_cache_0_ptr", "stride_k_cache_1_ptr", "stride_k_cache_2_ptr", "stride_v_cache_0_ptr", - "stride_v_cache_1_ptr", "stride_v_cache_2_ptr", "query_start_len_ptr", "num_seqs_ptr", - }, - .outputs = &.{ "segm_output", "segm_max", "segm_expsum" }, - .run = run, - }); - fn run(b: *Builder, cfg: Config) tri.FinishError!void { - const a = try b.declareArgs(.{ - .query_ptr = .{ .ptr = cfg.q_dtype }, - .key_cache_ptr = .{ .ptr = cfg.kv_dtype }, - .value_cache_ptr = .{ .ptr = cfg.kv_dtype }, - .sink_ptr = .{ .ptr = cfg.scale_dtype }, - .block_tables_ptr = .{ .ptr = .i32 }, - .seq_lens_ptr = .{ .ptr = .i32 }, - .alibi_slopes_ptr = .{ .ptr = cfg.scale_dtype }, - .qq_bias_ptr = .{ .ptr = cfg.scale_dtype }, - .scale_ptr = .{ .ptr = .f32 }, - .k_scale_ptr = .{ .ptr = .f32 }, - .v_scale_ptr = .{ .ptr = .f32 }, - .softcap_ptr = .{ .ptr = .f32 }, - .block_table_stride_ptr = .{ .ptr = .i64 }, - .query_stride_0_ptr = .{ .ptr = .i64 }, - .query_stride_1_ptr = .{ .ptr = .i64 }, - .qq_bias_stride_0_ptr = .{ .ptr = .i64 }, - .stride_k_cache_0_ptr = .{ .ptr = .i64 }, - .stride_k_cache_1_ptr = .{ .ptr = .i64 }, - .stride_k_cache_2_ptr = .{ .ptr = .i64 }, - .stride_v_cache_0_ptr = .{ .ptr = .i64 }, - .stride_v_cache_1_ptr = .{ .ptr = .i64 }, - .stride_v_cache_2_ptr = .{ .ptr = .i64 }, - .query_start_len_ptr = .{ .ptr = .i32 }, - .num_seqs_ptr = .{ .ptr = .i32 }, - .segm_output_ptr = .{ .ptr = .f32 }, - .segm_max_ptr = .{ .ptr = .f32 }, - .segm_expsum_ptr = .{ .ptr = .f32 }, - }); - - const scale = b.load(a.scale_ptr); - const k_scale = b.load(a.k_scale_ptr); - const v_scale = b.load(a.v_scale_ptr); - const softcap = b.load(a.softcap_ptr); - const block_table_stride = b.load(a.block_table_stride_ptr); - const query_stride_0 = b.load(a.query_stride_0_ptr); - const query_stride_1 = b.load(a.query_stride_1_ptr); - const qq_bias_stride_0 = b.load(a.qq_bias_stride_0_ptr); - const stride_k_cache_0 = b.constant(.i64, cfg.stride_k_cache_0); - const stride_k_cache_1 = b.constant(.i64, cfg.stride_k_cache_1); - const stride_k_cache_2 = b.constant(.i64, cfg.stride_k_cache_2); - const stride_v_cache_0 = b.constant(.i64, cfg.stride_v_cache_0); - const stride_v_cache_1 = b.constant(.i64, cfg.stride_v_cache_1); - const stride_v_cache_2 = b.constant(.i64, cfg.stride_v_cache_2); - const num_seqs = b.load(a.num_seqs_ptr); - - kernelUnifiedAttention3d( - b, - a.segm_output_ptr, - a.segm_max_ptr, - a.segm_expsum_ptr, - a.query_ptr, - a.key_cache_ptr, - a.value_cache_ptr, - a.sink_ptr, - a.block_tables_ptr, - a.seq_lens_ptr, - a.alibi_slopes_ptr, - a.qq_bias_ptr, - scale, - k_scale, - v_scale, - softcap, - block_table_stride, - query_stride_0, - query_stride_1, - qq_bias_stride_0, - stride_k_cache_0, - stride_k_cache_1, - stride_k_cache_2, - stride_v_cache_0, - stride_v_cache_1, - stride_v_cache_2, - a.query_start_len_ptr, - num_seqs, - cfg, - ); - } -}; - -fn kernelUnifiedAttention3d( - k: *Builder, - segm_output_ptr: Value, - segm_max_ptr: Value, - segm_expsum_ptr: Value, - query_ptr: Value, - key_cache_ptr: Value, - value_cache_ptr: Value, - sink_ptr: Value, - block_tables_ptr: Value, - seq_lens_ptr: Value, - alibi_slopes_ptr: Value, - qq_bias_ptr: Value, - scale: Value, - k_scale: Value, - v_scale: Value, - softcap: Value, - block_table_stride: Value, - query_stride_0: Value, - query_stride_1: Value, - qq_bias_stride_0: Value, - stride_k_cache_0: Value, - stride_k_cache_1: Value, - stride_k_cache_2: Value, - stride_v_cache_0: Value, - stride_v_cache_1: Value, - stride_v_cache_2: Value, - query_start_len_ptr: Value, - num_seqs: Value, - config: KernelUnifiedAttention3dPtr.Config, -) void { - const BLOCK_M: i64 = config.block_m; - const BLOCK_Q: i64 = config.block_q; - const BLOCK_SIZE: i64 = config.block_size; - const TILE_SIZE: i64 = config.tile_size; - const HEAD_SIZE: i64 = config.head_size; - const HEAD_SIZE_PADDED: i64 = config.head_size_padded; - const NUM_QUERIES_PER_KV: i64 = config.num_queries_per_kv; - const NUM_QUERY_HEADS: i64 = config.num_query_heads; - const NUM_SEGMENTS_PER_SEQ: i64 = config.num_segments_per_seq; - const SLIDING_WINDOW: i64 = config.sliding_window; - - const q_block_global_idx = k.programId(.x); - const kv_head_idx = k.programId(.y); - const segm_idx = k.programId(.z); - - const qk_scale = scale.mul(RCP_LN2); - - const seq_idx = findSeqIdx(k, query_start_len_ptr, q_block_global_idx, num_seqs, BLOCK_Q, true); - - const q_start_raw = k.load(query_start_len_ptr.addPtr(seq_idx)); - const q_block_start_idx = q_start_raw.div(@as(i32, @intCast(BLOCK_Q))).add(seq_idx); - const q_block_local_idx = q_block_global_idx.sub(q_block_start_idx); - - const cur_batch_in_all_start_index = q_start_raw; - // Match Python's left-to-right `query_start_len_ptr + seq_idx + 1`: two `tt.addptr`s. - const cur_batch_in_all_stop_index = k.load(query_start_len_ptr.addPtr(seq_idx).addPtr(1)); - const cur_batch_query_len = cur_batch_in_all_stop_index.sub(cur_batch_in_all_start_index); - - const q_out_of_range = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))).ge(cur_batch_query_len); - k.returnIf(q_out_of_range); - - const seq_len = k.load(seq_lens_ptr.addPtr(seq_idx)); - const tiles_per_segment = cdivFn(seq_len, @as(i32, @intCast(NUM_SEGMENTS_PER_SEQ * TILE_SIZE))); - - const segm_lo = segm_idx.mul(tiles_per_segment).mul(@as(i32, @intCast(TILE_SIZE))); - const segm_out_of_range = segm_lo.ge(seq_len); - k.returnIf(segm_out_of_range); - - const offs_m = k.arange(0, BLOCK_M, .i32); - const offs_d = k.arange(0, HEAD_SIZE_PADDED, .i32); - const offs_t = k.arange(0, TILE_SIZE, .i32); - - const q_local_bq = q_block_local_idx.mul(@as(i32, @intCast(BLOCK_Q))); - const query_pos = q_local_bq.add(offs_m.div(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); - - const query_offset_0 = cur_batch_in_all_start_index.add(query_pos); - const query_offset_1 = kv_head_idx.mul(@as(i32, @intCast(NUM_QUERIES_PER_KV))) - .add(offs_m.rem(@as(i32, @intCast(NUM_QUERIES_PER_KV)))); - - // query_offset = qo_0[:, None]*qstride_0 + qo_1[:, None]*qstride_1 + offs_d[None, :] - // Inline `expandDims(offs_d, 0)` *inside* the chain so it's emitted after - // the qo0+qo1 addi - const qo0_2d = k.expandDims(query_offset_0, 1).mul(query_stride_0); - const qo1_2d = k.expandDims(query_offset_1, 1).mul(query_stride_1); - const query_offset = qo0_2d.add(qo1_2d).add(k.expandDims(offs_d, 0)); - - const dim_mask: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - offs_d.lt(@as(i32, @intCast(HEAD_SIZE))) - else - k.full(&.{HEAD_SIZE_PADDED}, 1, .i1); - const query_mask_0 = query_pos.lt(cur_batch_query_len); - const query_mask_1 = query_offset_1.lt(@as(i32, @intCast(NUM_QUERY_HEADS))); - - const q_mask_pre = k.expandDims(query_mask_0, 1).bitAnd(k.expandDims(query_mask_1, 1)); - const q_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - q_mask_pre.bitAnd(k.expandDims(dim_mask, 0)) - else - q_mask_pre; - const q_ptrs = query_ptr.addPtr(query_offset); - const Q = k.loadOpts(q_ptrs, .{ - .mask = k.broadcastTo(q_mask_partial, &.{ BLOCK_M, HEAD_SIZE_PADDED }), - .other = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, config.q_dtype), - }); - - const block_table_offset = seq_idx.to(.i64).mul(block_table_stride); - - // M init: USE_SINKS + segm_idx==0 → loaded sinks; else -inf. - const segm_is_zero = segm_idx.eq(0); - const m_init: Value = if (config.use_sinks) mb: { - var mi = k.openIfElse(segm_is_zero, .{k.tensorTy(&.{BLOCK_M}, .f32)}); - { - const loaded = k.loadOpts(sink_ptr.addPtr(query_offset_1), .{ - .mask = query_mask_1, - .other = k.full(&.{BLOCK_M}, -std.math.inf(f32), config.scale_dtype), - }); - mi.yieldThen(.{loaded.to(.f32).mul(RCP_LN2)}); - } - { - mi.yieldElse(.{k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32)}); - } - break :mb mi.results[0]; - } else k.full(&.{BLOCK_M}, -std.math.inf(f32), .f32); - - const l_init = k.full(&.{BLOCK_M}, 1.0, .f32); - const acc_init = k.zeros(&.{ BLOCK_M, HEAD_SIZE_PADDED }, .f32); - - const context_len = seq_len.sub(cur_batch_query_len); - - const alibi_slope: Value = if (config.use_alibi_slopes) - k.loadOpts(alibi_slopes_ptr.addPtr(query_offset_1), .{ - .mask = query_mask_1, - .other = k.full(&.{BLOCK_M}, 0.0, config.scale_dtype), - }) - else - k.zeros(&.{BLOCK_M}, config.scale_dtype); - - // See 2d-body note on qq_bias_row_ptrs: else branch needs an explicit splatTo - // (no sibling offset to auto-broadcast against). - const qq_bias_row_ptrs: Value = if (config.use_qq_bias) - qq_bias_ptr.addPtr(query_pos.to(.i64).mul(qq_bias_stride_0)) - else - qq_bias_ptr.splatTo(&.{BLOCK_M}); - - const pad_term: i32 = @intCast(@divTrunc(BLOCK_M - 1, NUM_QUERIES_PER_KV) + 1); - const max_prefix_raw = context_len.add(q_local_bq).add(pad_term); - const max_seq_prefix_len = if (config.is_causal) max_prefix_raw.minimum(seq_len) else seq_len; - const num_tiles = cdivFn(max_seq_prefix_len, @as(i32, @intCast(TILE_SIZE))); - - const segm_tile_lo = segm_idx.mul(tiles_per_segment); - const segm_tile_hi = segm_idx.add(1).mul(tiles_per_segment).minimum(num_tiles); - - const kv_cache_mod: ttir.CacheModifier = config.kv_cache_modifier; - - var loop = k.openFor(segm_tile_lo, segm_tile_hi, 1, .{ m_init, l_init, acc_init }); - { - const j = loop.iv; - const M = loop.carried[0]; - const L = loop.carried[1]; - const acc = loop.carried[2]; - - const seq_offset = j.mul(@as(i32, @intCast(TILE_SIZE))).add(offs_t); - const tile_mask: Value = seq_offset.lt(max_seq_prefix_len); - - const same_block = TILE_SIZE == BLOCK_SIZE; - const in_block_offs: Value = if (same_block) offs_t else seq_offset.rem(@as(i32, @intCast(BLOCK_SIZE))); - const physical_block_idx: Value = if (same_block) - k.load(block_tables_ptr.addPtr(block_table_offset).addPtr(j)).to(.i64).splatTo(&.{TILE_SIZE}) - else - k.load( - block_tables_ptr.addPtr(block_table_offset).addPtr(seq_offset.div(@as(i32, @intCast(BLOCK_SIZE)))), - ).to(.i64); - - const v_offset = k.expandDims(physical_block_idx, 1).mul(stride_v_cache_0) - .add(kv_head_idx.to(.i64).mul(stride_v_cache_2)) - .add(k.expandDims(offs_d, 0).mul(config.stride_v_cache_3)) - .add(k.expandDims(in_block_offs, 1).mul(stride_v_cache_1)); - - const k_offset = k.expandDims(physical_block_idx, 0).mul(stride_k_cache_0) - .add(kv_head_idx.to(.i64).mul(stride_k_cache_2)) - .add(k.expandDims(offs_d, 1).mul(config.stride_k_cache_3)) - .add(k.expandDims(in_block_offs, 0).mul(stride_k_cache_1)); - - const block_io_loads = same_block and HEAD_SIZE_PADDED == HEAD_SIZE; - - const K_load = if (block_io_loads) - k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ .cache_modifier = kv_cache_mod }) - else kb: { - const k_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - k.expandDims(dim_mask, 1).bitAnd(k.expandDims(tile_mask, 0)) - else - k.expandDims(tile_mask, 0); - break :kb k.loadOpts(key_cache_ptr.addPtr(k_offset), .{ - .mask = k.broadcastTo(k_mask_partial, &.{ HEAD_SIZE_PADDED, TILE_SIZE }), - .other = k.zeros(&.{ HEAD_SIZE_PADDED, TILE_SIZE }, config.kv_dtype), - .cache_modifier = kv_cache_mod, - }); - }; - const K = dequantKv(K_load, k_scale, isFp8(config.kv_dtype), config.q_dtype); - - const V_raw = if (block_io_loads) - k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ .cache_modifier = kv_cache_mod }) - else vb: { - const v_mask_partial: Value = if (HEAD_SIZE_PADDED != HEAD_SIZE) - k.expandDims(tile_mask, 1).bitAnd(k.expandDims(dim_mask, 0)) - else - k.expandDims(tile_mask, 1); - break :vb k.loadOpts(value_cache_ptr.addPtr(v_offset), .{ - .mask = k.broadcastTo(v_mask_partial, &.{ TILE_SIZE, HEAD_SIZE_PADDED }), - .other = k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), - .cache_modifier = kv_cache_mod, - }); - }; - const V_load = if (block_io_loads) - k.where( - k.broadcastTo(k.expandDims(tile_mask, 1), &.{ TILE_SIZE, HEAD_SIZE_PADDED }), - V_raw, - k.zeros(&.{ TILE_SIZE, HEAD_SIZE_PADDED }, config.kv_dtype), - ) - else - V_raw; - const V = dequantKv(V_load, v_scale, isFp8(config.kv_dtype), config.q_dtype); - - // seq_mask = seq_offset[None, :] < context_len + query_pos[:, None] + 1 - const ql_2d_i32 = query_pos.expandDims(1); - const so_2d = seq_offset.expandDims(0); - const seq_mask = if (config.is_causal) b: { - const rhs = context_len.add(ql_2d_i32).add(1); - break :b so_2d.lt(rhs); - } else k.broadcastTo(tile_mask.expandDims(0), &.{ BLOCK_M, TILE_SIZE }); - - const acc_zero = k.zeros(&.{ BLOCK_M, TILE_SIZE }, .f32); - const qk = k.dot(Q, K, acc_zero); - var S = qk_scale.mul(qk); - - if (config.use_softcap) { - S = applySoftcap(k, S, softcap).mul(RCP_LN2); - } - - const qm0_2d = query_mask_0.expandDims(1); - const qm1_2d = query_mask_1.expandDims(1); - const keep_mask = qm1_2d.bitAnd(qm0_2d).bitAnd(seq_mask); - S = k.where(keep_mask, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); - - if (SLIDING_WINDOW > 0) { - const diff = ql_2d_i32.add(context_len).sub(so_2d); - const in_win = diff.lt(@as(i32, @intCast(SLIDING_WINDOW))); - S = k.where(in_win, S, k.full(&.{ BLOCK_M, TILE_SIZE }, -std.math.inf(f32), .f32)); - } - - if (config.use_alibi_slopes) { - const alibi_2d = alibi_slope.expandDims(1); - const pos_diff = seq_offset.sub(context_len).to(config.scale_dtype).expandDims(0); - S = S.add(alibi_2d.mul(pos_diff).to(.f32).mul(RCP_LN2)); - } - - if (config.use_qq_bias) { - const key_rel_pos = seq_offset.sub(context_len); - const is_query_key = key_rel_pos.ge(0) - .bitAnd(key_rel_pos.to(.i64).lt(qq_bias_stride_0)); - const qq_ptrs = qq_bias_row_ptrs.expandDims(1) - .addPtr(key_rel_pos.to(.i64).expandDims(0)); - const qq_bias = k.loadOpts(qq_ptrs, .{ - .mask = is_query_key.expandDims(0), - .other = k.zeros(&.{ BLOCK_M, TILE_SIZE }, config.scale_dtype), - }); - S = S.add(qq_bias.to(.f32).mul(RCP_LN2)); - } - - var m_j = M.maximum(k.maxOpts(S, .{ .axis = 1 })); - m_j = k.where(m_j.gt(-std.math.inf(f32)), m_j, k.full(&.{BLOCK_M}, 0.0, .f32)); - - const mj_2d = m_j.expandDims(1); - const P = k.exp2(S.sub(mj_2d)); - const l_j = k.sumOpts(P, .{ .axis = 1 }); - const alpha = k.exp2(M.sub(m_j)); - - const alpha_2d = alpha.expandDims(1); - const acc_scaled = acc.mul(alpha_2d); - const new_L = L.mul(alpha).add(l_j); - - const P_cast = P.to(config.q_dtype); - const new_acc = k.dot(P_cast, V, acc_scaled); - - loop.yield(.{ m_j, new_L, new_acc }); - } - const M_final = loop.results[0]; - const L_final = loop.results[1]; - const acc_final = loop.results[2]; - - // segm_output_offset = qo_0[:, None].to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) - // + qo_1[:, None] * (NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED) - // + segm_idx * HEAD_SIZE_PADDED + offs_d[None, :] - const segm_out_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED); - const segm_out_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ * HEAD_SIZE_PADDED; - - // Inline `segm_idx * HEAD_SIZE_PADDED` and `offs_d[None, :]` inside the chain - // so each term's ops emit in source order - const oo0_2d = k.expandDims(query_offset_0, 1).to(.i64).mul(segm_out_token_stride_i64); - const oo1_2d = k.expandDims(query_offset_1, 1).mul(segm_out_head_stride_i32); - const segm_output_offset = oo0_2d.add(oo1_2d) - .add(segm_idx.mul(@as(i32, @intCast(HEAD_SIZE_PADDED)))) - .add(k.expandDims(offs_d, 0)); - - const store_mask_2d = k.mask2d(query_mask_0, dim_mask, BLOCK_M, HEAD_SIZE_PADDED) - .bitAnd(query_mask_1.expandDims(1)); - k.storeOpts( - segm_output_ptr.addPtr(segm_output_offset), - acc_final, - .{ .mask = store_mask_2d }, - ); - - // segm_offset = qo_0.to(i64) * (NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ) - // + qo_1 * NUM_SEGMENTS_PER_SEQ + segm_idx - const segm_head_stride_i32: i32 = @intCast(NUM_SEGMENTS_PER_SEQ); - const segm_token_stride_i64: i64 = NUM_QUERY_HEADS * NUM_SEGMENTS_PER_SEQ; - const segm_offset = query_offset_0.to(.i64).mul(segm_token_stride_i64) - .add(query_offset_1.mul(segm_head_stride_i32)) - .add(segm_idx); - const store_mask_1d = query_mask_0.bitAnd(query_mask_1); - k.storeOpts(segm_max_ptr.addPtr(segm_offset), M_final, .{ .mask = store_mask_1d }); - k.storeOpts(segm_expsum_ptr.addPtr(segm_offset), L_final, .{ .mask = store_mask_1d }); -} diff --git a/zml/logo.zig b/zml/logo.zig deleted file mode 100644 index 235bc43d..00000000 --- a/zml/logo.zig +++ /dev/null @@ -1,211 +0,0 @@ -const std = @import("std"); - -pub const Color = enum { - reset, - cyan, - sky, - blue, - violet, - pink, - shine_cyan, - shine_sky, - shine_blue, - shine_violet, - shine_pink, - - pub fn escape(self: Color, support: ColorSupport) []const u8 { - return switch (support) { - .true_color => self.trueColor(), - ._256 => self.xterm256(), - else => "", - }; - } - - fn trueColor(self: Color) []const u8 { - return switch (self) { - .reset => "\x1b[0m", - .cyan => "\x1b[38;2;0;255;255m", - .sky => "\x1b[38;2;0;221;255m", - .blue => "\x1b[38;2;135;143;255m", - .violet => "\x1b[38;2;215;90;219m", - .pink => "\x1b[38;2;242;48;174m", - .shine_cyan => "\x1b[1;38;2;140;255;255m", - .shine_sky => "\x1b[1;38;2;128;244;255m", - .shine_blue => "\x1b[1;38;2;205;210;255m", - .shine_violet => "\x1b[1;38;2;255;170;255m", - .shine_pink => "\x1b[1;38;2;255;150;220m", - }; - } - - fn xterm256(self: Color) []const u8 { - return switch (self) { - .reset => "\x1b[0m", - .cyan => "\x1b[38;5;51m", - .sky => "\x1b[38;5;45m", - .blue => "\x1b[38;5;105m", - .violet => "\x1b[38;5;170m", - .pink => "\x1b[38;5;205m", - .shine_cyan => "\x1b[1;38;5;123m", - .shine_sky => "\x1b[1;38;5;123m", - .shine_blue => "\x1b[1;38;5;189m", - .shine_violet => "\x1b[1;38;5;219m", - .shine_pink => "\x1b[1;38;5;212m", - }; - } -}; - -pub const Row = struct { - text: []const u8, - color: Color = .reset, - shine: Color = .reset, -}; - -pub const Block = struct { - rows: []const Row, -}; - -pub const ColorSupport = enum { - no_color, - ansi, - _256, - true_color, -}; - -pub const Options = struct { - color_support: ColorSupport, - is_shiny: *const fn ([]const u8) bool = isShineGlyph, -}; - -pub const zml_art_blocks = [_]Block{ - .{ .rows = &.{ - .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, - .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, - .{ .text = " ███████╗", .color = .cyan, .shine = .shine_cyan }, - .{ .text = " ╚══███╔╝", .color = .sky, .shine = .shine_sky }, - .{ .text = " ███╔╝ ", .color = .blue, .shine = .shine_blue }, - .{ .text = " ███╔╝ ", .color = .violet, .shine = .shine_violet }, - .{ .text = " ███████╗", .color = .pink, .shine = .shine_pink }, - .{ .text = " ╚══════╝", .color = .pink, .shine = .shine_pink }, - } }, - .{ .rows = &.{ - .{ .text = " ", .color = .cyan, .shine = .shine_cyan }, - .{ .text = "███╗ ███╗", .color = .cyan, .shine = .shine_cyan }, - .{ .text = "████╗ ████║", .color = .sky, .shine = .shine_sky }, - .{ .text = "██╔████╔██║", .color = .blue, .shine = .shine_blue }, - .{ .text = "██║╚██╔╝██║", .color = .violet, .shine = .shine_violet }, - .{ .text = "██║ ╚═╝ ██║", .color = .pink, .shine = .shine_pink }, - .{ .text = "╚═╝ ╚═╝", .color = .pink, .shine = .shine_pink }, - .{ .text = " ", .color = .pink, .shine = .shine_pink }, - } }, - .{ .rows = &.{ - .{ .text = "██╗ ", .color = .cyan, .shine = .shine_cyan }, - .{ .text = "██║ ", .color = .sky, .shine = .shine_sky }, - .{ .text = "██║ ", .color = .blue, .shine = .shine_blue }, - .{ .text = "██║ .ai", .color = .violet, .shine = .shine_violet }, - .{ .text = "███████╗", .color = .pink, .shine = .shine_pink }, - .{ .text = "╚══════╝", .color = .pink, .shine = .shine_pink }, - .{ .text = " ", .color = .pink, .shine = .shine_pink }, - .{ .text = " ", .color = .pink, .shine = .shine_pink }, - } }, -}; - -pub fn detectColorSupport(io: std.Io, file: std.Io.File, env: *const std.process.Environ.Map) ColorSupport { - var no_color: bool = false; - var clicolor_force: bool = false; - no_color = if (env.get("NO_COLOR")) |v| v.len != 0 else false; - clicolor_force = if (env.get("CLICOLOR_FORCE")) |v| v.len != 0 else false; - - const mode = std.Io.Terminal.Mode.detect(io, file, no_color, clicolor_force) catch return .no_color; - if (mode != .escape_codes) return .no_color; - - if (env.get("COLORTERM")) |value| { - if (std.ascii.eqlIgnoreCase(value, "truecolor") or - std.ascii.eqlIgnoreCase(value, "24bit")) - { - return .true_color; - } - } - - if (env.get("TERM")) |value| { - if (std.mem.endsWith(u8, value, "-direct")) return .true_color; - if (std.mem.indexOf(u8, value, "truecolor") != null) return .true_color; - - if (std.mem.indexOf(u8, value, "256color") != null) return ._256; - } - - return .ansi; -} - -pub fn writeZmlArt(writer: *std.Io.Writer, options: Options) !void { - try write(writer, zml_art_blocks[0..], options); -} - -pub fn write(writer: *std.Io.Writer, blocks: []const Block, options: Options) !void { - if (blocks.len == 0) return; - - const row_count = blocks[0].rows.len; - for (blocks[1..]) |block| { - if (block.rows.len != row_count) return error.InvalidLogoBlocks; - } - - for (0..row_count) |row_index| { - const last_block = findLastVisibleBlockIndex(blocks, row_index) orelse { - try writer.writeAll("\n"); - continue; - }; - - for (blocks[0 .. last_block + 1], 0..) |block, block_index| { - const row = block.rows[row_index]; - try writeRow(writer, row, options, block_index == last_block); - } - try writer.writeAll(Color.reset.escape(options.color_support)); - try writer.writeAll("\n"); - } -} - -pub fn writeRow(writer: *std.Io.Writer, row: Row, options: Options, rtrim_whitespace: bool) !void { - const text = if (rtrim_whitespace) trimRightWhitespace(row.text) else row.text; - if (text.len == 0) return; - - const color = row.color.escape(options.color_support); - const shine = row.shine.escape(options.color_support); - - var view: std.unicode.Utf8View = .initUnchecked(text); - var iter = view.iterator(); - var is_shiny = false; - - try writer.writeAll(color); - while (iter.nextCodepointSlice()) |codepoint| { - const should_shine = options.is_shiny(codepoint); - if (should_shine != is_shiny) { - try writer.writeAll(if (should_shine) shine else color); - is_shiny = should_shine; - } - try writer.writeAll(codepoint); - } -} - -fn findLastVisibleBlockIndex(blocks: []const Block, row_index: usize) ?usize { - var index = blocks.len; - while (index > 0) { - index -= 1; - if (trimRightWhitespace(blocks[index].rows[row_index].text).len != 0) return index; - } - return null; -} - -fn trimRightWhitespace(text: []const u8) []const u8 { - return std.mem.trimEnd(u8, text, " \t\r\n"); -} - -pub fn isShineGlyph(codepoint: []const u8) bool { - return std.mem.eql(u8, codepoint, "╔") or - std.mem.eql(u8, codepoint, "╗") or - std.mem.eql(u8, codepoint, "╚") or - std.mem.eql(u8, codepoint, "╝") or - std.mem.eql(u8, codepoint, "═") or - std.mem.eql(u8, codepoint, "║") or - std.mem.eql(u8, codepoint, ".") or - std.mem.eql(u8, codepoint, "a") or - std.mem.eql(u8, codepoint, "i"); -} diff --git a/zml/mem.zig b/zml/mem.zig index a64ae968..c12a2f61 100644 --- a/zml/mem.zig +++ b/zml/mem.zig @@ -22,8 +22,8 @@ pub const DmaAllocator = union(enum) { pub fn init(parent: std.mem.Allocator, device: *const Device) DmaAllocator { return switch (device.platform.target) { .cuda => .{ .dmam = .init(parent, device.platform) }, - .rocm, .oneapi, .tpu, .furiosa => .{ .uib = .init(device.memory(.host_pinned).?) }, - .cpu, .neuron, .metal => .{ .passthrough = parent }, + .tpu, .neuron => .{ .uib = .init(device.memory(.host_pinned)) }, + .rocm, .cpu => .{ .passthrough = parent }, }; } @@ -225,7 +225,9 @@ pub const FixedBufferPool = struct { pub const DynamicBufferPool = struct { const Node = struct { next: ?*Node }; - const alignment: std.mem.Alignment = .of(Node); + const alignment: std.mem.Alignment = .max(.of(u8), .of(Node)); + const ItemPtr = [*]align(alignment.toByteUnits()) u8; + const Slice = []align(alignment.toByteUnits()) u8; block_size: usize, max_blocks: usize, @@ -309,8 +311,8 @@ pub const DynamicBufferPool = struct { } } - fn nodeToSlice(self: DynamicBufferPool, node: *Node) []u8 { - const ptr: [*]u8 = @ptrCast(node); + fn nodeToSlice(self: *DynamicBufferPool, node: *Node) Slice { + const ptr: ItemPtr = @ptrCast(@alignCast(node)); return ptr[0..self.block_size]; } }; @@ -329,20 +331,14 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu const ModelBufferized = Bufferized(Model); if (ModelBufferized == Buffer) { - bufferized_.* = .{ - // I'm not sure I like that. I'd rather set all fields but _shards than leaving most of the m undefined. - ._shape = model._shape, - ._shards = .empty, - ._platform = undefined, - }; return; } const type_info = @typeInfo(ModelBufferized); switch (type_info) { .@"struct" => |struct_type_info| { - inline for (struct_type_info.field_names) |field_name| { - try bufferizeInner(allocator, @field(model, field_name), &@field(bufferized_, field_name)); + inline for (struct_type_info.fields) |field| { + try bufferizeInner(allocator, @field(model, field.name), &@field(bufferized_, field.name)); } }, .@"union" => { @@ -364,7 +360,7 @@ fn bufferizeInner(allocator: std.mem.Allocator, model: anytype, bufferized_: *Bu .pointer => |p| { switch (p.size) { .slice => { - bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.attrs.@"align" orelse @alignOf(p.child)), model.len); + bufferized_.* = try allocator.alignedAlloc(p.child, .fromByteUnits(p.alignment), model.len); for (model, bufferized_.*) |src, *dst| { try bufferizeInner(allocator, src, dst); } diff --git a/zml/meta.zig b/zml/meta.zig index 84fb9d36..1274761b 100644 --- a/zml/meta.zig +++ b/zml/meta.zig @@ -20,30 +20,27 @@ pub fn MapType(From: type, To: type) type { return switch (@typeInfo(T)) { .@"struct" => |struct_infos| { - const field_names = struct_infos.field_names; + const fields = struct_infos.fields; var same: bool = true; - var struct_field_names: [field_names.len][]const u8 = undefined; - var struct_field_types: [field_names.len]type = undefined; - var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; - for ( - struct_field_names[0..], - struct_field_types[0..], - struct_field_attrs[0..], - struct_infos.field_names, - struct_infos.field_types, - struct_infos.field_attrs, - ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { - if (!source_attrs.@"comptime") { - const R = map(SourceField); - if (R == SourceField) { - field_name.* = source_name; - field_type.* = SourceField; - field_attr.* = source_attrs; + var struct_field_names: [fields.len][]const u8 = undefined; + var struct_field_types: [fields.len]type = undefined; + var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (struct_field_names[0..], struct_field_types[0..], struct_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { + if (!field.is_comptime) { + const R = map(field.type); + if (R == field.type) { + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } else { - field_name.* = source_name; + field_name.* = field.name; field_type.* = R; field_attr.* = .{ - .@"comptime" = source_attrs.@"comptime", + .@"comptime" = field.is_comptime, .@"align" = @alignOf(R), }; same = false; @@ -55,35 +52,34 @@ pub fn MapType(From: type, To: type) type { } } } else { - field_name.* = source_name; - field_type.* = SourceField; - field_attr.* = source_attrs; + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } } if (same) return T; return if (struct_infos.is_tuple) @Tuple(&struct_field_types) else @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); }, .@"union" => |union_info| { - const field_names = union_info.field_names; + const fields = union_info.fields; var same: bool = true; - var union_field_names: [field_names.len][]const u8 = undefined; - var union_field_types: [field_names.len]type = undefined; - var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; - for ( - union_field_names[0..], - union_field_types[0..], - union_field_attrs[0..], - union_info.field_names, - union_info.field_types, - union_info.field_attrs, - ) |*field_name, *field_type, *field_attr, source_name, SourceField, source_attrs| { - const R = map(SourceField); - if (R == SourceField) { - field_name.* = source_name; - field_type.* = SourceField; - field_attr.* = source_attrs; + var union_field_names: [fields.len][]const u8 = undefined; + var union_field_types: [fields.len]type = undefined; + var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; + for (union_field_names[0..], union_field_types[0..], union_field_attrs[0..], fields) |*field_name, *field_type, *field_attr, field| { + const R = map(field.type); + if (R == field.type) { + field_name.* = field.name; + field_type.* = field.type; + field_attr.* = .{ + .@"align" = field.alignment, + }; } else { - field_name.* = source_name; + field_name.* = field.name; field_type.* = R; field_attr.* = .{ .@"align" = @alignOf(R), @@ -96,11 +92,11 @@ pub fn MapType(From: type, To: type) type { }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.attrs.@"const") + .slice => if (ptr_info.is_const) []const map(ptr_info.child) else []map(ptr_info.child), - .one => if (ptr_info.attrs.@"const") + .one => if (ptr_info.is_const) *const map(ptr_info.child) else *map(ptr_info.child), @@ -143,7 +139,7 @@ test MapType { /// The `to` parameter must be passed with mutable pointer, and tensor data need to be mutable if callback needs it. /// `mapAlloc` tries as much as possible to respect the conversions made by Zig itself. /// For example it can convert from a comptime array to a runtime slice. -/// `mapAlloc` allocates slices and single-item pointers needed to hold mapped values. +/// `mapAlloc` can allocate new slices to write the result if the result struct requires it. /// The caller is owning said allocations, using an `ArenaAllocator` might help tracking them. /// /// Note: to avoid infinite loop, mapAlloc doesn't look for `From` fields inside `To` struct. @@ -152,7 +148,7 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam // TODO: handle tuple to slice conversion // TODO: handle error bubbling up const From = FnParam(cb, 1); - const To = stdx.meta.FnReturn(cb); + const To = stdx.meta.FnResult(cb); const FromStruct = @TypeOf(from); const type_info_to_ptr = @typeInfo(@TypeOf(to)); if (type_info_to_ptr != .pointer) { @@ -180,34 +176,29 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam return; } - if (FromStruct == ToStruct and !comptime Contains(FromStruct, From)) { - to.* = from; - return; - } - if (@sizeOf(ToStruct) == 0) return; switch (type_info_to) { - .@"struct" => |info| inline for (info.field_names, info.field_types, info.field_attrs) |field_name, Field, field_attrs| { - if (field_attrs.@"comptime" or @sizeOf(Field) == 0) continue; - const field_type_info = @typeInfo(Field); + .@"struct" => |info| inline for (info.fields) |field| { + if (field.is_comptime or @sizeOf(field.type) == 0) continue; + const field_type_info = @typeInfo(field.type); // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. switch (field_type_info) { // .pointer => try convertType(From, To, allocator, @field(from, field.name), @field(to, field.name), Ctx, ctx, cb), - .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field_name)) { + .array, .optional, .@"union", .@"struct", .pointer => if (@hasField(FromStruct, field.name)) { try mapAlloc( cb, allocator, ctx, - @field(from, field_name), - &@field(to, field_name), + @field(from, field.name), + &@field(to, field.name), ); - } else if (field_attrs.default_value_ptr) |_| { - @field(to, field_name) = null; + } else if (field.default_value_ptr) |_| { + @field(to, field.name) = null; } else { - stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field_name, ToStruct }); + stdx.debug.compileError("Mapping {} -> {} inside {} failed. Missing field {s} in {}", .{ From, To, FromStruct, field.name, ToStruct }); }, - else => @field(to, field_name) = @field(from, field_name), + else => @field(to, field.name) = @field(from, field.name), } }, .@"union" => { @@ -231,11 +222,7 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } to.* = items; }, - else => { - const item = try allocator.create(ptr_info.child); - try mapAlloc(cb, allocator, ctx, from.*, item); - to.* = item; - }, + else => try mapAlloc(cb, allocator, ctx, from.*, to.*), }, .slice => { const items = try allocator.alloc(@typeInfo(ToStruct).pointer.child, from.len); @@ -244,10 +231,6 @@ pub fn mapAlloc(comptime cb: anytype, allocator: std.mem.Allocator, ctx: FnParam } to.* = items; }, - .many => if (FromStruct == [*:0]const u8 and ToStruct == FromStruct and From != u8) { - // Tensor axis tags are borrowed immutable strings. - to.* = from; - } else stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), else => stdx.debug.compileError("zml.meta.mapAlloc doesn't support: {}", .{FromStruct}), }, .optional => if (from) |f| { @@ -276,9 +259,7 @@ test mapAlloc { field: A, array: [2]A, slice: []const A, - pointer: *const A, other: u8, - tag: [*:0]const u8, // We want to allow conversion from comptime to runtime, because Zig type inference works like this. comptime static_val: u8 = 8, comptime static_slice: [2]A = .{ .{ .a = 11 }, .{ .a = 12 } }, @@ -288,9 +269,7 @@ test mapAlloc { field: B, array: [2]B, slice: []const B, - pointer: *const B, other: u8, - tag: [*:0]const u8, static_val: u8, static_slice: []B, field_with_empty: struct { B, Empty }, @@ -300,9 +279,7 @@ test mapAlloc { .field = .{ .a = 4 }, .array = .{ .{ .a = 5 }, .{ .a = 6 } }, .other = 7, - .tag = "feature", .slice = &.{ .{ .a = 9 }, .{ .a = 10 } }, - .pointer = &.{ .a = 13 }, .field_with_empty = .{ .{ .a = 9 }, .{} }, }; var bb: BB = undefined; @@ -310,19 +287,16 @@ test mapAlloc { try mapAlloc(A.convert, testing.allocator, {}, aa, &bb); defer testing.allocator.free(bb.slice); defer testing.allocator.free(bb.static_slice); - defer testing.allocator.destroy(bb.pointer); try testing.expectEqual(4, bb.field.b); try testing.expectEqual(5, bb.array[0].b); try testing.expectEqual(6, bb.array[1].b); try testing.expectEqual(7, bb.other); - try testing.expectEqual(aa.tag, bb.tag); try testing.expectEqual(8, bb.static_val); try testing.expectEqual(9, bb.slice[0].b); try testing.expectEqual(10, bb.slice[1].b); try testing.expectEqual(11, bb.static_slice[0].b); try testing.expectEqual(12, bb.static_slice[1].b); - try testing.expectEqual(13, bb.pointer.b); } /// Visit a given type `T` and: @@ -350,21 +324,25 @@ pub fn MapRestrict(From: type, To: type) type { .@"struct" => |struct_infos| { // We know that at least one of the struct field contains a From. // We map each field individually. Fields without From and comptime fields are removed. - const field_names = struct_infos.field_names; + const fields = struct_infos.fields; var num_fields: usize = 0; - var struct_field_names: [field_names.len][]const u8 = undefined; - var struct_field_types: [field_names.len]type = undefined; - var struct_field_attrs: [field_names.len]std.builtin.Type.Struct.FieldAttributes = undefined; - for (struct_infos.field_names, struct_infos.field_types, struct_infos.field_attrs) |field_name, Field, field_attrs| { - if (!field_attrs.@"comptime" and Contains(Field, From)) { - const R = map(Field); - if (R == Field) { - struct_field_names[num_fields] = field_name; - struct_field_types[num_fields] = Field; - struct_field_attrs[num_fields] = field_attrs; + var struct_field_names: [fields.len][]const u8 = undefined; + var struct_field_types: [fields.len]type = undefined; + var struct_field_attrs: [fields.len]std.builtin.Type.StructField.Attributes = undefined; + for (fields) |field| { + if (!field.is_comptime and Contains(field.type, From)) { + const R = map(field.type); + if (R == field.type) { + struct_field_names[num_fields] = field.name; + struct_field_types[num_fields] = field.type; + struct_field_attrs[num_fields] = .{ + .@"comptime" = field.is_comptime, + .@"align" = field.alignment, + .default_value_ptr = field.default_value_ptr, + }; } else { - const name = if (struct_infos.is_tuple) struct_infos.field_names[num_fields] else field_name; + const name = if (struct_infos.is_tuple) struct_infos.fields[num_fields].name else field.name; struct_field_names[num_fields] = name; struct_field_types[num_fields] = R; struct_field_attrs[num_fields] = .{ @@ -391,13 +369,13 @@ pub fn MapRestrict(From: type, To: type) type { .@"union" => |union_info| { // We know that at least one of the union field contains a From. // We map each field individually. Fields without From, are replaced by "void". - const field_names = union_info.field_names; - var union_field_names: [field_names.len][]const u8 = undefined; - var union_field_types: [field_names.len]type = undefined; - var union_field_attrs: [field_names.len]std.builtin.Type.Union.FieldAttributes = undefined; - for (0.., union_info.field_names, union_info.field_types) |i, field_name, Field| { - const FT = map(Field); - union_field_names[i] = field_name; + const fields = union_info.fields; + var union_field_names: [fields.len][]const u8 = undefined; + var union_field_types: [fields.len]type = undefined; + var union_field_attrs: [fields.len]std.builtin.Type.UnionField.Attributes = undefined; + for (0.., fields) |i, field| { + const FT = map(field.type); + union_field_names[i] = field.name; union_field_types[i] = FT; union_field_attrs[i] = .{ .@"align" = @alignOf(FT), @@ -407,19 +385,19 @@ pub fn MapRestrict(From: type, To: type) type { }, .array => |arr_info| [arr_info.len]map(arr_info.child), .pointer => |ptr_info| switch (ptr_info.size) { - .slice => if (ptr_info.attrs.@"const") + .slice => if (ptr_info.is_const) []const map(ptr_info.child) else []map(ptr_info.child), - .one => if (ptr_info.attrs.@"const") + .one => if (ptr_info.is_const) *const map(ptr_info.child) else *map(ptr_info.child), - .many => if (ptr_info.attrs.@"const") + .many => if (ptr_info.is_const) [*]const map(ptr_info.child) else [*]map(ptr_info.child), - .c => if (ptr_info.attrs.@"const") + .c => if (ptr_info.is_const) [*c]map(ptr_info.child) else [*c]map(ptr_info.child), @@ -460,98 +438,90 @@ test MapRestrict { /// Recursively visit the given struct and calls the callback for each K found. /// The `v` parameter must me a pointer, and tensor data need to be mutable if callbacks needs it. pub fn VisitReturn(comptime cb: anytype) type { - return if (stdx.meta.FnReturnErrorSet(cb)) |error_set| + return if (stdx.meta.FnSignature(cb, null).ReturnErrorSet) |error_set| error_set!void else void; } -const Visitor = struct { - pub const Action = enum { - callback, - recurse, - exit, - }; - - pub fn determineAction(comptime callback: anytype, comptime PtrTypeOfV: type) Action { - const ptr_info = switch (@typeInfo(PtrTypeOfV)) { - .pointer => |info| info, - else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a pointer/slice input, but received: {any}", .{ @TypeOf(callback), PtrTypeOfV }), - }; - - const TargetType = switch (@typeInfo(FnParam(callback, 1))) { - .pointer => |info| info.child, - else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), - }; - - if (ptr_info.size == .one and ptr_info.child == TargetType) { - return .callback; - } - - switch (@typeInfo(ptr_info.child)) { - .@"struct", .@"union" => { - if (@hasDecl(ptr_info.child, "zml_no_meta_visit")) return .exit; - }, - else => {}, - } - - return .recurse; +pub fn visit(comptime cb: anytype, ctx: FnParam(cb, 0), v: anytype) VisitReturn(cb) { + const Callback = @TypeOf(cb); + const Ptr = @TypeOf(v); + const type_info_v = @typeInfo(Ptr); + if (type_info_v != .pointer) { + stdx.debug.compileError("zml.meta.visit({}) is expecting a pointer/slice input, but received: {}", .{ Callback, Ptr }); } -}; + const ptr_info = type_info_v.pointer; + const Child = ptr_info.child; + const can_error = stdx.meta.FnSignature(cb, null).ReturnErrorSet != null; -pub fn visit(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) VisitReturn(callback) { - const action = comptime Visitor.determineAction(callback, @TypeOf(v)); + const K, const mutating_cb = switch (@typeInfo(FnParam(cb, 1))) { + .pointer => |info| .{ info.child, !info.is_const }, + else => stdx.debug.compileError("zml.meta.visit is expecting a callback with a pointer as second argument but found {}", .{FnParam(cb, 1)}), + }; + // Abort if v doesnt' contain any K. + if (comptime !Contains(Ptr, K)) return; - const PtrTypeOfV = @TypeOf(v); - const ptr_info_v = @typeInfo(PtrTypeOfV).pointer; - const ChildTypeV = ptr_info_v.child; + // Handle simple cases. + switch (Ptr) { + *const K, *K => return cb(ctx, v), + *const ?K, *?K => return if (v.*) |*val| cb(ctx, val) else {}, + []const K, []K => { + for (v) |*v_elem| if (can_error) try cb(ctx, v_elem) else cb(ctx, v_elem); + return; + }, + else => {}, + } - const can_error = stdx.meta.FnReturnErrorSet(callback) != null; + // Handle stdx.BoundedArray that contains uninitalized data. + if (@typeInfo(Child) == .@"struct" and @hasDecl(Child, "constSlice") and @hasDecl(Child, "slice")) { + return visit(cb, ctx, if (mutating_cb) v.slice() else v.constSlice()); + } - return switch (action) { - .exit => {}, - .callback => callback(ctx, v), - .recurse => { - const TargetType, const mutating_cb = switch (@typeInfo(FnParam(callback, 1))) { - .pointer => |info| .{ info.child, !info.attrs.@"const" }, - else => stdx.debug.compileError("zml.meta.visit({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), - }; - - if (comptime !Contains(PtrTypeOfV, TargetType)) return; - - if (@typeInfo(ChildTypeV) == .@"struct" and @hasDecl(ChildTypeV, "constSlice") and @hasDecl(ChildTypeV, "slice")) { - return visit(callback, ctx, if (mutating_cb) v.slice() else v.constSlice()); - } - - switch (ptr_info_v.size) { - .one => switch (@typeInfo(ChildTypeV)) { - .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { - if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; - if (@typeInfo(Field) == .pointer) { - if (can_error) try visit(callback, ctx, @field(v, field_name)) else visit(callback, ctx, @field(v, field_name)); + // Recursively visit fields of v. + switch (ptr_info.size) { + .one => switch (@typeInfo(Child)) { + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, K)) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + switch (field_type_info) { + .pointer => if (can_error) try visit(cb, ctx, @field(v, field.name)) else visit(cb, ctx, @field(v, field.name)), + .array, .optional, .@"union", .@"struct" => if (can_error) try visit(cb, ctx, &@field(v, field.name)) else visit(cb, ctx, &@field(v, field.name)), + else => {}, + } + }, + .array => for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), + .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), + .@"union" => switch (v.*) { + inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), + }, + else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), + }, + .slice => { + for (v) |*v_elem| { + switch (@typeInfo(Child)) { + .@"struct" => |s| inline for (s.fields) |field| { + if (field.is_comptime or comptime !Contains(field.type, K)) continue; + const field_type_info = @typeInfo(field.type); + // If the field is already a pointer, we recurse with it directly, otherwise, we recurse with a pointer to the field. + if (field_type_info == .pointer) { + if (can_error) try visit(cb, ctx, @field(v_elem, field.name)) else visit(cb, ctx, @field(v_elem, field.name)); } else { - if (can_error) try visit(callback, ctx, &@field(v, field_name)) else visit(callback, ctx, &@field(v, field_name)); + if (can_error) try visit(cb, ctx, &@field(v_elem, field.name)) else visit(cb, ctx, &@field(v_elem, field.name)); } }, - .array => for (v[0..]) |*elem| { - if (can_error) try visit(callback, ctx, elem) else visit(callback, ctx, elem); + .array => for (v) |*elem| if (can_error) try visit(cb, ctx, elem) else visit(cb, ctx, elem), + .optional => if (v.* != null) if (can_error) try visit(cb, ctx, &v.*.?) else visit(cb, ctx, &v.*.?), + .@"union" => switch (v_elem.*) { + inline else => |*v_field| if (can_error) try visit(cb, ctx, v_field) else visit(cb, ctx, v_field), }, - .optional => return if (v.*) |*opt| visit(callback, ctx, opt), - .@"union" => return switch (v.*) { - inline else => |*v_field| visit(callback, ctx, v_field), - }, - .pointer => return visit(callback, ctx, v.*), - else => stdx.debug.compileError("zml.meta.visit({any}) doesn't support visiting pointer type {any}", .{ @TypeOf(callback), ChildTypeV }), - }, - .slice => { - for (v) |*v_elem| { - if (can_error) try visit(callback, ctx, v_elem) else visit(callback, ctx, v_elem); - } - }, - .many, .c => stdx.debug.compileError("zml.meta.visit({any}) doesn't support [*] style pointers got {any}", .{ @TypeOf(callback), ChildTypeV }), + else => stdx.debug.compileError("zml.meta.visit({}) doesn't support fields of type: {}", .{ Callback, Child }), + } } }, - }; + .many, .c => stdx.debug.compileError("zml.meta.visit({}) doesn't support [*] style pointers, got: {}", .{ Callback, Ptr }), + } } test visit { @@ -560,162 +530,20 @@ test visit { const NestedAttr = struct { nested: Attr }; const NestedAttrOptional = struct { nested: ?Attr }; const SimpleStruct = struct { prop: Attr }; - const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr, prop4: []Attr }; - const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; - - const LocalContext = struct { - result: usize, - - fn cb(ctx: *@This(), attr: *const Attr) void { - ctx.result += attr.data; - } - - fn mutCb(ctx: *@This(), attr: *Attr) void { - ctx.result += attr.data; - } - }; - - { - var context: LocalContext = .{ .result = 0 }; - const container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - - visit(LocalContext.cb, &context, &container); - - try std.testing.expectEqual(1, context.result); - } - { - var context: LocalContext = .{ .result = 0 }; - var container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - - visit(LocalContext.mutCb, &context, &container); - - try std.testing.expectEqual(1, context.result); - } - { - var context: LocalContext = .{ .result = 0 }; - var more_props: [3]Attr = .{ .{ .data = 4 }, .{ .data = 5 }, .{ .data = 6 } }; - - var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null, .prop4 = &more_props }; - - visit(LocalContext.mutCb, &context, &container); - - try std.testing.expectEqual(1 + 4 + 5 + 6, context.result); - } - { - var context: LocalContext = .{ .result = 0 }; - var more_props: [3]Attr = .{ .{ .data = 4 }, .{ .data = 5 }, .{ .data = 6 } }; - const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 }, .prop4 = &more_props }; - - visit(LocalContext.cb, &context, &container); - - try std.testing.expectEqual(1 + 2 + 4 + 5 + 6, context.result); - } - { - var context: LocalContext = .{ .result = 0 }; - const prop5: stdx.BoundedArray(Attr, 8) = .{ - .buffer = @splat(.{ .data = 4 }), - .len = 2, - }; - const container: NestedTypesStruct = .{ - .prop1 = .{ .data = 1 }, - .prop2 = .{ .other = "hello" }, - .prop3 = .{ .nested = .{ .data = 2 } }, - .prop4 = .{ .nested = .{ .data = 3 } }, - .prop5 = prop5, // 4 will be counted twice. - }; - - visit(LocalContext.cb, &context, &container); - - try std.testing.expectEqual(14, context.result); - } -} - -/// Similar to `visit` but doesn't follow pointers or slices. -/// This guarantees that the given type is a flat struct. -/// -/// see: `zml.meta.visit` -pub fn visitFlatStruct(comptime callback: anytype, ctx: FnParam(callback, 0), v: anytype) VisitReturn(callback) { - const action = comptime Visitor.determineAction(callback, @TypeOf(v)); - - const TargetType = switch (@typeInfo(FnParam(callback, 1))) { - .pointer => |info| info.child, - else => stdx.debug.compileError("zml.meta.visitStruct({any}) is expecting a callback with a pointer as second argument but found {any}", .{ @TypeOf(callback), FnParam(callback, 1) }), - }; - - const PtrTypeOfV = @TypeOf(v); - const ptr_info_v = @typeInfo(PtrTypeOfV).pointer; - const ChildTypeV = ptr_info_v.child; - - const can_error = stdx.meta.FnReturnErrorSet(callback) != null; - const err_msg = "zml.meta.visitStruct only handles flat struct, found pointer in {}"; - const err_args = .{ChildTypeV}; - - switch (action) { - .exit => {}, - .callback => return callback(ctx, v), - .recurse => { - if (comptime !Contains(ChildTypeV, TargetType)) return; - - // bounded array support (like visit) - if (@typeInfo(ChildTypeV) == .@"struct" and @hasDecl(ChildTypeV, "constSlice") and @hasDecl(ChildTypeV, "slice")) { - return for (v.slice()) |*elem| { - if (can_error) - try visitFlatStruct(callback, ctx, elem) - else - visitFlatStruct(callback, ctx, elem); - }; - } - - return switch (ptr_info_v.size) { - .one => switch (@typeInfo(ChildTypeV)) { - .@"struct" => |s| inline for (s.field_names, s.field_types, s.field_attrs) |field_name, Field, field_attrs| { - if (field_attrs.@"comptime" or comptime !Contains(Field, TargetType)) continue; - stdx.debug.assertComptime(@typeInfo(Field) != .pointer, err_msg, err_args); - - if (can_error) try visitFlatStruct(callback, ctx, &@field(v, field_name)) else visitFlatStruct(callback, ctx, &@field(v, field_name)); - }, - .array => for (v[0..]) |*elem| { - if (can_error) try visitFlatStruct(callback, ctx, elem) else visitFlatStruct(callback, ctx, elem); - }, - .optional => if (v.*) |*opt| visitFlatStruct(callback, ctx, opt), - .@"union" => switch (v.*) { - inline else => |*v_field| visitFlatStruct(callback, ctx, v_field), - }, - else => stdx.debug.compileError(err_msg, err_args), - }, - else => stdx.debug.compileError(err_msg, err_args), - }; - }, - } -} - -test visitFlatStruct { - const Attr = struct { data: usize }; - const OtherAttr = struct { other: []const u8 }; - const NestedAttr = struct { nested: Attr }; - const NestedAttrOptional = struct { nested: ?Attr }; - const SimpleStruct = struct { prop: Attr }; - // Note: no slice anymore, no stdx.BoundedArray const MultipleTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: ?Attr }; const NestedTypesStruct = struct { prop1: Attr, prop2: OtherAttr, prop3: NestedAttr, prop4: NestedAttrOptional, prop5: stdx.BoundedArray(Attr, 8) }; - const LocalContext = struct { - result: usize, - - fn cb(ctx: *@This(), attr: *const Attr) void { - ctx.result += attr.data; - } - - fn mutCb(ctx: *@This(), attr: *Attr) void { - ctx.result += attr.data; - } - }; + const LocalContext = struct { result: usize }; { var context: LocalContext = .{ .result = 0 }; const container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - visitFlatStruct(LocalContext.cb, &context, &container); + visit((struct { + fn cb(ctx: *LocalContext, attr: *const Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); try std.testing.expectEqual(1, context.result); } @@ -723,7 +551,11 @@ test visitFlatStruct { var context: LocalContext = .{ .result = 0 }; var container: SimpleStruct = .{ .prop = .{ .data = 1 } }; - visitFlatStruct(LocalContext.mutCb, &context, &container); + visit((struct { + fn cb(ctx: *LocalContext, attr: *Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); try std.testing.expectEqual(1, context.result); } @@ -731,7 +563,11 @@ test visitFlatStruct { var context: LocalContext = .{ .result = 0 }; var container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = null }; - visitFlatStruct(LocalContext.mutCb, &context, &container); + visit((struct { + fn cb(ctx: *LocalContext, attr: *Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); try std.testing.expectEqual(1, context.result); } @@ -739,9 +575,13 @@ test visitFlatStruct { var context: LocalContext = .{ .result = 0 }; const container: MultipleTypesStruct = .{ .prop1 = .{ .data = 1 }, .prop2 = .{ .other = "hello" }, .prop3 = .{ .data = 2 } }; - visitFlatStruct(LocalContext.cb, &context, &container); + visit((struct { + fn cb(ctx: *LocalContext, attr: *const Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); - try std.testing.expectEqual(1 + 2, context.result); + try std.testing.expectEqual(3, context.result); } { var context: LocalContext = .{ .result = 0 }; @@ -757,11 +597,16 @@ test visitFlatStruct { .prop5 = prop5, // 4 will be counted twice. }; - visitFlatStruct(LocalContext.cb, &context, &container); + visit((struct { + fn cb(ctx: *LocalContext, attr: *const Attr) void { + ctx.result += attr.data; + } + }).cb, &context, &container); - try std.testing.expectEqual(1 + 2 + 3 + 4 + 4, context.result); + try std.testing.expectEqual(14, context.result); } } + pub fn count(T: type, value: anytype) u32 { var counter: u32 = 0; visit(struct { @@ -800,17 +645,17 @@ pub fn zip(comptime func: anytype, allocator: std.mem.Allocator, values: anytype .pointer => stdx.debug.compileError("zip only accept by value arguments. Received: {}", .{V}), .@"struct" => |struct_info| { var out: V = values[0]; - inline for (struct_info.field_names, struct_info.field_types, struct_info.field_attrs) |field_name, Field, field_attrs| { - if (field_attrs.@"comptime") continue; - if (@typeInfo(Field) == .pointer) { + inline for (struct_info.fields) |f| { + if (f.is_comptime) continue; + if (@typeInfo(f.type) == .pointer) { stdx.debug.compileError("zip doesn't follow pointers and don't accept struct containing them. Received: {}", .{V}); } - var fields = try allocator.alloc(Field, values.len); + var fields = try allocator.alloc(f.type, values.len); defer allocator.free(fields); for (values, 0..) |val, i| { - fields[i] = @field(val, field_name); + fields[i] = @field(val, f.name); } - @field(out, field_name) = try zip(func, allocator, fields, args); + @field(out, f.name) = try zip(func, allocator, fields, args); } return out; }, @@ -857,13 +702,13 @@ pub fn collectAlloc( func_ctx: _CollectCtx(func), allocator: std.mem.Allocator, obj: anytype, -) std.mem.Allocator.Error![]stdx.meta.FnReturn(func) { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); +) std.mem.Allocator.Error![]stdx.meta.FnSignature(func, null).ReturnT { + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collect expects a func with two arguments, got: {}", .{@TypeOf(func)}); const CollectAllocCtx = struct { func_ctx: _CollectCtx(func), allocator: std.mem.Allocator, - out: std.ArrayList(stdx.meta.FnReturn(func)) = .empty, + out: std.ArrayList(stdx.meta.FnSignature(func, null).ReturnT) = .empty, oom: bool = false, fn cb(ctx: *@This(), val: *const _CollectArg(func)) void { @@ -884,8 +729,8 @@ pub fn collectAlloc( /// Given a func(X) -> Y or a func(Ctx, X) -> Y, /// finds all X in the given object, and write the result of func(X) into a given slice. /// Asserts that the number of X found is equal to the slice len. -pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnReturn(func)) void { - stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".param_types.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); +pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: []stdx.meta.FnResult(func)) void { + stdx.debug.assertComptime(@typeInfo(@TypeOf(func)).@"fn".params.len <= 2, "zml.meta.collectBuf expects a func with one or two arguments, got: {}", .{@TypeOf(func)}); const LocalContext = struct { func_ctx: _CollectCtx(func), out: @TypeOf(out), @@ -904,79 +749,15 @@ pub fn collectBuf(func: anytype, func_ctx: _CollectCtx(func), obj: anytype, out: std.debug.assert(context.idx == context.out.len); } -/// Finds all X in the given object, and write their pointers into an arraylist. -pub fn collectPtrs( - T: type, - allocator: std.mem.Allocator, - obj: anytype, -) std.mem.Allocator.Error![]*const T { - const CollectPtrCtx = struct { - allocator: std.mem.Allocator, - out: std.ArrayList(*const T) = .empty, - oom: bool = false, - - fn cb(ctx: *@This(), val: *const T) void { - if (ctx.oom) return; - ctx.out.append(ctx.allocator, val) catch { - ctx.oom = true; - }; - } - }; - var context = CollectPtrCtx{ .allocator = allocator }; - visit(CollectPtrCtx.cb, &context, obj); - if (context.oom) return error.OutOfMemory; - - return context.out.toOwnedSlice(allocator); -} - -/// Finds all X in the given object, and write their mutable pointers into an arraylist. -pub fn collectMutPtrs( - T: type, - allocator: std.mem.Allocator, - obj: anytype, -) std.mem.Allocator.Error![]*T { - const ok = switch (@typeInfo(@TypeOf(obj))) { - .pointer => |ptr_info| !ptr_info.attrs.@"const", - else => false, - }; - stdx.debug.assertComptime(ok, "zml.meta.collectMutPtrs requires a mutable pointer input, got: {}", .{@TypeOf(obj)}); - return @ptrCast(try collectPtrs(T, allocator, obj)); -} - -/// Finds all X in the given object, and copy them by value into an arraylist. -pub fn collectValues( - T: type, - allocator: std.mem.Allocator, - obj: anytype, -) std.mem.Allocator.Error![]T { - const CollectValuesCtx = struct { - allocator: std.mem.Allocator, - out: std.ArrayList(T) = .empty, - oom: bool = false, - - fn cb(ctx: *@This(), val: *const T) void { - if (ctx.oom) return; - ctx.out.append(ctx.allocator, val.*) catch { - ctx.oom = true; - }; - } - }; - var context = CollectValuesCtx{ .allocator = allocator }; - visit(CollectValuesCtx.cb, &context, obj); - if (context.oom) return error.OutOfMemory; - - return context.out.toOwnedSlice(allocator); -} - fn _CollectCtx(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".param_types; + const params = @typeInfo(@TypeOf(func)).@"fn".params; if (params.len == 1) return void; - return params[0] orelse @compileError("anytype not supported in collect"); + return params[0].type orelse @compileError("anytype not supported in collect"); } fn _CollectArg(func: anytype) type { - const params = @typeInfo(@TypeOf(func)).@"fn".param_types; - return params[params.len - 1] orelse @compileError("anytype not supported in collect"); + const params = @typeInfo(@TypeOf(func)).@"fn".params; + return params[params.len - 1].type orelse @compileError("anytype not supported in collect"); } pub fn Contains(Haystack: type, T: type) bool { @@ -991,18 +772,15 @@ pub fn Contains(Haystack: type, T: type) bool { return switch (@typeInfo(Haystack)) { .@"struct" => |info| { - if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; - inline for (info.field_types, info.field_attrs) |Field, field_attrs| { - if (!field_attrs.@"comptime" and Contains(Field, T)) { + inline for (info.fields) |field| { + if (!field.is_comptime and Contains(field.type, T)) return true; - } } return false; }, .@"union" => |info| { - if (@hasDecl(Haystack, "zml_no_meta_visit")) return false; - inline for (info.field_types) |Field| { - if (Contains(Field, T)) + inline for (info.fields) |field| { + if (Contains(field.type, T)) return true; } return false; diff --git a/zml/mlirx.zig b/zml/mlirx.zig index bc92eee0..748d988c 100644 --- a/zml/mlirx.zig +++ b/zml/mlirx.zig @@ -8,40 +8,80 @@ const Shape = @import("shape.zig").Shape; const mlirx = @This(); pub const Type = struct { - pub fn rankedTensor(ctx: *mlir.Context, shape: Shape) *const mlir.Type { - return .rankedTensor(shape.dims(), mlirx.Type.fromDType(ctx, shape.dtype())); - } - pub fn fromDType(ctx: *mlir.Context, dt: dtype.DataType) *const mlir.Type { return switch (dt) { - .bool => .int(ctx, .i1), - .f4e2m1 => .float(ctx, .f4e2m1fn), - .f8e3m4 => .float(ctx, .f8e3m4), - .f8e4m3 => .float(ctx, .f8e4m3), - .f8e4m3fn => .float(ctx, .f8e4m3fn), - .f8e4m3fnuz => .float(ctx, .f8e4m3fnuz), - .f8e4m3b11fnuz => .float(ctx, .f8e4m3b11fnuz), - .f8e5m2 => .float(ctx, .f8e5m2), - .f8e5m2fnuz => .float(ctx, .f8e5m2fnuz), - .f8e8m0 => .float(ctx, .f8e8m0fnu), - .bf16 => .float(ctx, .bf16), - .f16 => .float(ctx, .f16), - .f32 => .float(ctx, .f32), - .f64 => .float(ctx, .f64), - .i2 => .int(ctx, .i2), - .i4 => .int(ctx, .i4), - .i8 => .int(ctx, .i8), - .i16 => .int(ctx, .i16), - .i32 => .int(ctx, .i32), - .i64 => .int(ctx, .i64), - .u2 => .int(ctx, .u2), - .u4 => .int(ctx, .u4), - .u8 => .int(ctx, .u8), - .u16 => .int(ctx, .u16), - .u32 => .int(ctx, .u32), - .u64 => .int(ctx, .u64), - .c64 => .complex(ctx, .c64), - .c128 => .complex(ctx, .c128), + .bool => mlir.integerType(ctx, .i1), + .f4e2m1 => mlir.floatType(ctx, .f4e2m1fn), + .f8e3m4 => mlir.floatType(ctx, .f8e3m4), + .f8e4m3 => mlir.floatType(ctx, .f8e4m3), + .f8e4m3fn => mlir.floatType(ctx, .f8e4m3fn), + .f8e4m3fnuz => mlir.floatType(ctx, .f8e4m3fnuz), + .f8e4m3b11fnuz => mlir.floatType(ctx, .f8e4m3b11fnuz), + .f8e5m2 => mlir.floatType(ctx, .f8e5m2), + .f8e5m2fnuz => mlir.floatType(ctx, .f8e5m2fnuz), + .f8e8m0 => @panic("Unsupported type"), + .bf16 => mlir.floatType(ctx, .bf16), + .f16 => mlir.floatType(ctx, .f16), + .f32 => mlir.floatType(ctx, .f32), + .f64 => mlir.floatType(ctx, .f64), + .i2 => mlir.integerType(ctx, .i2), + .i4 => mlir.integerType(ctx, .i4), + .i8 => mlir.integerType(ctx, .i8), + .i16 => mlir.integerType(ctx, .i16), + .i32 => mlir.integerType(ctx, .i32), + .i64 => mlir.integerType(ctx, .i64), + .u2 => mlir.integerType(ctx, .u2), + .u4 => mlir.integerType(ctx, .u4), + .u8 => mlir.integerType(ctx, .u8), + .u16 => mlir.integerType(ctx, .u16), + .u32 => mlir.integerType(ctx, .u32), + .u64 => mlir.integerType(ctx, .u64), + .c64 => mlir.complexType(ctx, .c64), + .c128 => mlir.complexType(ctx, .c128), }; } + + // TODO(Corentin): Maybe remove that as it now requires a mlir context. + // It's weird to have to provide a mlir context to get a zml.DataType from a mlir.Type. + pub fn toDType(ctx: *mlir.Context, mlir_type: *const mlir.Type) dtype.DataType { + const mapping = .{ + .{ .bool, mlir.integerType(ctx, .i1) }, + + .{ .f8e4m3b11fnuz, mlir.floatType(ctx, .f8e4m3b11fnuz) }, + .{ .f8e4m3fn, mlir.floatType(ctx, .f8e4m3fn) }, + .{ .f8e4m3fnuz, mlir.floatType(ctx, .f8e4m3fnuz) }, + .{ .f8e3m4, mlir.floatType(ctx, .f8e3m4) }, + .{ .f8e5m2, mlir.floatType(ctx, .f8e5m2) }, + .{ .f8e5m2fnuz, mlir.floatType(ctx, .f8e5m2fnuz) }, + .{ .f4e2m1, mlir.floatType(ctx, .f4e2m1fn) }, + .{ .bf16, mlir.floatType(ctx, .bf16) }, + .{ .f16, mlir.floatType(ctx, .f16) }, + .{ .f32, mlir.floatType(ctx, .f32) }, + .{ .f64, mlir.floatType(ctx, .f64) }, + + .{ .i4, mlir.integerType(ctx, .i4) }, + .{ .i8, mlir.integerType(ctx, .i8) }, + .{ .i16, mlir.integerType(ctx, .i16) }, + .{ .i32, mlir.integerType(ctx, .i32) }, + .{ .i64, mlir.integerType(ctx, .i64) }, + + .{ .u4, mlir.integerType(ctx, .u4) }, + .{ .u8, mlir.integerType(ctx, .u8) }, + .{ .u16, mlir.integerType(ctx, .u16) }, + .{ .u32, mlir.integerType(ctx, .u32) }, + .{ .u64, mlir.integerType(ctx, .u64) }, + + .{ .c64, mlir.complexType(ctx, .c64) }, + .{ .c128, mlir.complexType(ctx, .c128) }, + }; + + inline for (mapping) |entry| { + const dt, const mlirT = entry; + if (mlirT.eql(mlir_type)) { + return dt; + } + } + + std.debug.panic("Could not convert mlir.Type to DataType: {f}", .{mlir_type}); + } }; diff --git a/zml/module.zig b/zml/module.zig new file mode 100644 index 00000000..31637852 --- /dev/null +++ b/zml/module.zig @@ -0,0 +1,536 @@ +const std = @import("std"); + +const c = @import("c"); +const dialects = @import("mlir/dialects"); +const mlir = @import("mlir"); +const pjrt = @import("pjrt"); +const stdx = @import("stdx"); +const upb = @import("upb"); + +const Buffer = @import("buffer.zig").Buffer; +const Exe = @import("exe.zig").Exe; +const Memory = @import("platform.zig").Memory; +const meta = @import("meta.zig"); +const mlirx = @import("mlirx.zig"); +const pjrtx = @import("pjrtx.zig"); +const Platform = @import("platform.zig").Platform; +const Shape = @import("shape.zig").Shape; +const Tensor = @import("tensor.zig").Tensor; + +const log = std.log.scoped(.@"zml/module"); + +const AttributeList = stdx.BoundedArray(mlir.NamedAttribute, 3); + +pub const CompilationContext = struct { + pub const Scope = struct { + block: *mlir.Block, + id_to_argument: std.AutoArrayHashMapUnmanaged(usize, usize), + id_to_donation: std.AutoArrayHashMapUnmanaged(usize, usize), + id_to_output_memory_kind: std.AutoArrayHashMapUnmanaged(usize, Memory.Kind), + arena: std.heap.ArenaAllocator, + + pub fn initFromBlock(allocator: std.mem.Allocator, block: *mlir.Block) Scope { + const arena: std.heap.ArenaAllocator = .init(allocator); + return .{ + .block = block, + .id_to_argument = .empty, + .id_to_donation = .empty, + .id_to_output_memory_kind = .empty, + .arena = arena, + }; + } + + pub fn deinit(self: *Scope) void { + self.arena.deinit(); + } + }; + + allocator: std.mem.Allocator, + arena: std.heap.ArenaAllocator, + + mlir_registry: *mlir.DialectRegistry, + mlir_ctx: *mlir.Context, + mlir_pass_manager: *mlir.PassManager, + //mlir_op_pass_manager: *mlir.OpPassManager, + module: *mlir.Module, + platform: *const Platform, + + scopes: stdx.BoundedArray(Scope, 16) = .{}, + + threadlocal var _current: ?*CompilationContext = null; + + pub fn init(allocator: std.mem.Allocator, io: std.Io, platform: *const Platform) CompilationContext { + _ = io; // autofix + mlir.registerPasses("Transforms"); + const mlir_registry = mlir.DialectRegistry.init() catch unreachable; + inline for (.{ "func", "stablehlo" }) |d| { + mlir.DialectHandle.fromString(d).insertDialect(mlir_registry); + } + var mlir_ctx = mlir.Context.init(.{ .registry = mlir_registry, .threading = false }) catch unreachable; + mlir_ctx.loadAllAvailableDialects(); + + //const loc = mlir.Location.fromSrc(mlir_ctx, @src()).named(mlir_ctx, "main"); + const module = mlir.Module.init(.unknown(mlir_ctx)); + module.operation().setAttributeByName("sym_name", mlir.stringAttribute(mlir_ctx, "zml")); + + const pass_manager = mlir.PassManager.init(mlir_ctx); + { + var opm = pass_manager.asOpPassManager(); + const passes: []const []const u8 = &.{ + "canonicalize", + "cse", + "canonicalize", + }; + for (passes) |pass| { + opm.addPipeline(pass) catch unreachable; + } + } + + return .{ + .allocator = allocator, + .arena = std.heap.ArenaAllocator.init(allocator), + .mlir_registry = mlir_registry, + .mlir_ctx = mlir_ctx, + .mlir_pass_manager = pass_manager, + .module = module, + .platform = platform, + }; + } + + pub fn deinit(self: *CompilationContext) void { + self.mlir_pass_manager.deinit(); + self.module.deinit(); + self.mlir_ctx.deinit(); + self.mlir_registry.deinit(); + self.arena.deinit(); + } + + pub fn activate(self: *CompilationContext) void { + std.debug.assert(_current == null); + _current = self; + } + + pub fn deactivate(self: *CompilationContext) void { + _ = self; + _current = null; + } + + pub fn current() *CompilationContext { + return _current.?; + } + + pub fn currentScope(self: *CompilationContext) *Scope { + return &self.scopes.slice()[self.scopes.len - 1]; + } + + pub fn pushBlock(self: *CompilationContext, block: *mlir.Block) void { + const scope = Scope.initFromBlock(self.allocator, block); + self.scopes.appendAssumeCapacity(scope); + } + + pub fn popBlock(self: *CompilationContext) void { + var maybe_popped_scope = self.scopes.pop(); + if (maybe_popped_scope) |*popped| { + popped.deinit(); + } + } + + //pub fn addShardingAttributes( + // self: *const CompilationContext, + // mesh: Mesh, + // arg_attrs: []AttributeList, + // res_attrs: []AttributeList, + // input_shapes: []const Shape, + // input_meshes: []const ?Mesh, + // output_info: OutputInfo, + //) void { + // if (mesh.isSinglePartition()) return; // No sharding attributes for single partition. + + // const default_layout = mlir.NamedAttribute.named(self.mlir_ctx, "mhlo.layout_mode", mlir.stringAttribute(self.mlir_ctx, "default")); + // for (arg_attrs, input_shapes, input_meshes) |*attr, shape, input_mesh| { + // attr.appendAssumeCapacity(default_layout); + // if (input_mesh) |m| { + // // log.warn("addShardingAttributes for {} with input mesh: {}", .{ shape, m }); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(m, shape)))); + // } else { + // // log.warn("addShardingAttributes for {} with progran mesh: {}", .{ shape, mesh }); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + // } + // } + + // for (res_attrs, output_info.shapes) |*attr, shape| { + // attr.appendAssumeCapacity(default_layout); + // attr.appendAssumeCapacity(.named(self.mlir_ctx, "mhlo.sharding", self.getShardingAttr(Sharding.init(mesh, shape)))); + // } + //} + + //pub fn getShardingAttr(self: *const CompilationContext, sharding: Sharding) *const mlir.Attribute { + // var sharding_str: stdx.BoundedArray(u8, 128) = .{}; + // sharding.writeShardingRepresentation(sharding_str.writer()) catch unreachable; + // return mlir.stringAttribute(self.mlir_ctx, sharding_str.constSlice()); + //} +}; + +pub fn compile( + allocator: std.mem.Allocator, + io: std.Io, + comptime func: anytype, + args: std.meta.ArgsTuple(@TypeOf(func)), + platform: *const Platform, +) !Exe { + var compilation_context: CompilationContext = .init(allocator, io, platform); + defer compilation_context.deinit(); + + const result = emitMlir(&compilation_context, func, args) catch unreachable; + defer result.output_info.deinit(compilation_context.allocator); + defer result.input_info.deinit(compilation_context.allocator); + + _ = result.func.appendTo(compilation_context.module.body()); + const sharding = platform.sharding(); + compilation_context.module.operation().setAttributeByName("mhlo.num_replicas", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_replicas)); + compilation_context.module.operation().setAttributeByName("mhlo.num_partitions", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, sharding.num_partitions)); + + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const loaded_executable = compileModuleToPjrtExecutable(arena.allocator(), io, platform, compilation_context.module, null) catch unreachable; + + log.debug("\n******** ZML generated MLIR ********\n{f}", .{compilation_context.module.operation()}); + + const num_devices = sharding.num_partitions * sharding.num_replicas; + const exe = try Exe.init(allocator, platform, loaded_executable, result.input_info.shapes, result.output_info.shapes, num_devices); + errdefer exe.deinit(); + + return exe; +} + +fn collectShapes(allocator: std.mem.Allocator, v: anytype) ![]Shape { + const LocalContext = struct { + list: *std.array_list.Managed(Shape), + }; + var list = std.array_list.Managed(Shape).init(allocator); + errdefer list.deinit(); + + var context: LocalContext = .{ .list = &list }; + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.list.append(tensor.shape()); + } + }.cb, &context, v); + + return try list.toOwnedSlice(); +} + +pub const OutputInfo = struct { + shapes: []Shape, + values: []*const mlir.Value, + donations: []?usize, + output_memory_kinds: []Memory.Kind, + + pub fn deinit(self: OutputInfo, allocator: std.mem.Allocator) void { + allocator.free(self.shapes); + allocator.free(self.values); + allocator.free(self.donations); + allocator.free(self.output_memory_kinds); + } +}; + +fn collectOutputInfo(allocator: std.mem.Allocator, v: anytype) !OutputInfo { + const LocalContext = struct { + shape_list: *std.array_list.Managed(Shape), + value_list: *std.array_list.Managed(*const mlir.Value), + donation_list: *std.array_list.Managed(?usize), + output_memory_kind_list: *std.array_list.Managed(Memory.Kind), + }; + + var shape_list = std.array_list.Managed(Shape).init(allocator); + errdefer shape_list.deinit(); + var value_list = std.array_list.Managed(*const mlir.Value).init(allocator); + errdefer value_list.deinit(); + var donation_list = std.array_list.Managed(?usize).init(allocator); + errdefer donation_list.deinit(); + var output_memory_kind_list = std.array_list.Managed(Memory.Kind).init(allocator); + errdefer output_memory_kind_list.deinit(); + + var context: LocalContext = .{ + .shape_list = &shape_list, + .value_list = &value_list, + .donation_list = &donation_list, + .output_memory_kind_list = &output_memory_kind_list, + }; + + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.shape_list.append(tensor.shape()); + try ctx_.value_list.append(tensor.value()); + try ctx_.donation_list.append(tensor.donation()); + try ctx_.output_memory_kind_list.append(tensor.outputMemoryKind()); + } + }.cb, &context, v); + + return .{ + .shapes = try shape_list.toOwnedSlice(), + .values = try value_list.toOwnedSlice(), + .donations = try donation_list.toOwnedSlice(), + .output_memory_kinds = try output_memory_kind_list.toOwnedSlice(), + }; +} + +pub const InputInfo = struct { + shapes: []Shape, + + pub fn deinit(self: InputInfo, allocator: std.mem.Allocator) void { + allocator.free(self.shapes); + } +}; + +fn collectInputInfo(allocator: std.mem.Allocator, v: anytype) !InputInfo { + const LocalContext = struct { + shape_list: *std.array_list.Managed(Shape), + }; + + var shape_list = std.array_list.Managed(Shape).init(allocator); + errdefer shape_list.deinit(); + + var context: LocalContext = .{ .shape_list = &shape_list }; + + try meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) !void { + try ctx_.shape_list.append(tensor.shape()); + } + }.cb, &context, v); + + return .{ + .shapes = try shape_list.toOwnedSlice(), + }; +} + +const EmitMlirResult = struct { + func: *mlir.Operation, + input_info: InputInfo, + output_info: OutputInfo, +}; + +fn finalizeAttributeList(allocator_: std.mem.Allocator, mlir_ctx: *mlir.Context, attributes: []AttributeList) ![]*const mlir.Attribute { + const res = try allocator_.alloc(*const mlir.Attribute, attributes.len); + for (res, attributes) |*r, attr| { + r.* = mlir.dictionaryAttribute(mlir_ctx, attr.constSlice()); + } + return res; +} + +fn emitMlir(compilation_context: *CompilationContext, comptime func: anytype, args: stdx.meta.FnArgs(func)) !EmitMlirResult { + var arena = std.heap.ArenaAllocator.init(compilation_context.allocator); + defer arena.deinit(); + + const module = mlir.Module.init(.unknown(compilation_context.mlir_ctx)); + errdefer module.deinit(); + + const block = mlir.Block.init(&.{}, &.{}); + errdefer block.deinit(); + + compilation_context.pushBlock(block); + defer compilation_context.popBlock(); + + const LocalContext = struct { + compilation_context: *CompilationContext, + current_argument_id: usize = 0, + }; + var context: LocalContext = .{ + .compilation_context = compilation_context, + }; + meta.visit(struct { + fn cb(ctx_: *LocalContext, tensor: *const Tensor) void { + const mlir_type = mlir.rankedTensorType( + tensor.dims(), + mlirx.Type.fromDType(ctx_.compilation_context.mlir_ctx, tensor.dtype()), + ); + _ = ctx_.compilation_context.currentScope().block.addArgument(mlir_type, .unknown(ctx_.compilation_context.mlir_ctx)); + ctx_.compilation_context.currentScope().id_to_argument.put(ctx_.compilation_context.currentScope().arena.allocator(), tensor.id, ctx_.current_argument_id) catch unreachable; + ctx_.current_argument_id += 1; + } + }.cb, &context, &args); + + const input_info = try collectInputInfo(compilation_context.allocator, &args); + errdefer input_info.deinit(compilation_context.allocator); + + const input_attributes = try arena.allocator().alloc(AttributeList, input_info.shapes.len); + @memset(input_attributes, .{}); + + const output_info = b: { + compilation_context.activate(); + defer compilation_context.deactivate(); + + const result = @call(.auto, func, args); + + const output_info = try collectOutputInfo(compilation_context.allocator, &result); + errdefer output_info.deinit(compilation_context.allocator); + + break :b output_info; + }; + errdefer output_info.deinit(compilation_context.allocator); + + const output_attributes = try arena.allocator().alloc(AttributeList, output_info.shapes.len); + @memset(output_attributes, .{}); + + for (output_info.donations, 0..) |donation, index| if (donation) |argument_index| { + input_attributes[argument_index].appendAssumeCapacity(.named(compilation_context.mlir_ctx, "tf.aliasing_output", mlir.integerAttribute(compilation_context.mlir_ctx, .i32, index))); + }; + for (output_info.output_memory_kinds, 0..) |output_memory_kind, index| { + if (output_memory_kind == .device) continue; + output_attributes[index].appendAssumeCapacity(.named( + compilation_context.mlir_ctx, + "mhlo.memory_kind", + mlir.stringAttribute( + compilation_context.mlir_ctx, + compilation_context.platform.memoryKind(output_memory_kind), + ), + )); + } + _ = dialects.func.returns(compilation_context.mlir_ctx, output_info.values, .unknown(compilation_context.mlir_ctx)).appendTo(compilation_context.currentScope().block); + + const mlir_func = dialects.func.func(compilation_context.mlir_ctx, .{ + .name = "main", + .block = compilation_context.currentScope().block, + .location = .unknown(compilation_context.mlir_ctx), + .args_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, input_attributes), + .results_attributes = try finalizeAttributeList(arena.allocator(), compilation_context.mlir_ctx, output_attributes), + }); + + compilation_context.mlir_pass_manager.runOnOp(mlir_func) catch |err| switch (err) { + error.MlirUnexpected => { + std.log.err("Failed to canonicalize invalid mlir: {f}", .{mlir_func}); + // user errors should have triggered a panic before we reach this. + @panic("ZML generated invalid mlir. Please open a bug report"); + }, + }; + + return .{ + .func = mlir_func, + .input_info = input_info, + .output_info = output_info, + }; +} + +fn setXlaOverrideFlag(map: *c.upb_Map, flag: []const u8, value: anytype, upb_arena: *c.upb_Arena) !void { + const result = c.upb_Map_Set( + map, + .{ .str_val = upb.stringView(flag) }, + .{ .msg_val = blk: { + const field = try upb.new(c.xla_OptionOverrideProto, upb_arena); + switch (@typeInfo(@TypeOf(value))) { + .bool => c.xla_OptionOverrideProto_set_bool_field(field, value), + .comptime_int, .int => c.xla_OptionOverrideProto_set_int_field(field, @intCast(value)), + .comptime_float, .float => c.xla_OptionOverrideProto_set_double_field(field, @floatCast(value)), + else => c.xla_OptionOverrideProto_set_string_field(field, upb.stringView(value)), + } + break :blk @ptrCast(field); + } }, + upb_arena, + ); + + if (result == false) { + return std.mem.Allocator.Error.OutOfMemory; + } +} + +fn compileModuleToPjrtExecutable(arena: std.mem.Allocator, io: std.Io, platform: *const Platform, module: *const mlir.Module, xla_dump_to_: ?[]const u8) !*pjrt.LoadedExecutable { + //const tracer = Tracer.init("ai.zml.compilation"); + //const compile_frame = tracer.frameStart("pjrt compilation"); + //defer tracer.frameEnd(compile_frame, "pjrt compilation"); + + const sharding = platform.sharding(); + + var upb_alloc: upb.Allocator = .init(arena); + const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); + defer c.upb_Arena_Free(upb_arena); + + const options = blk: { + const options = try upb.new(c.xla_CompileOptionsProto, upb_arena); + c.xla_CompileOptionsProto_set_executable_build_options(options, executable_build_options_blk: { + const exec_build_options = try upb.new(c.xla_ExecutableBuildOptionsProto, upb_arena); + + c.xla_ExecutableBuildOptionsProto_set_device_ordinal(exec_build_options, -1); + c.xla_ExecutableBuildOptionsProto_set_num_replicas(exec_build_options, sharding.num_replicas); + c.xla_ExecutableBuildOptionsProto_set_num_partitions(exec_build_options, sharding.num_partitions); + c.xla_ExecutableBuildOptionsProto_set_use_spmd_partitioning(exec_build_options, sharding.num_partitions > 1 or sharding.num_replicas > 1); + + c.xla_ExecutableBuildOptionsProto_set_device_assignment(exec_build_options, device_assignment_blk: { + const device_assignment = try upb.new(c.xla_DeviceAssignmentProto, upb_arena); + + c.xla_DeviceAssignmentProto_set_replica_count(device_assignment, sharding.num_replicas); + c.xla_DeviceAssignmentProto_set_computation_count(device_assignment, sharding.num_partitions); + + const computation_devices = c.xla_DeviceAssignmentProto_resize_computation_devices(device_assignment, sharding.num_partitions, upb_arena); + for (computation_devices[0..sharding.num_partitions], 0..) |*computation_device, i| { + computation_device.* = try upb.new(c.xla_DeviceAssignmentProto_ComputationDevice, upb_arena); + _ = c.xla_DeviceAssignmentProto_ComputationDevice_add_replica_device_ids(computation_device.*, @intCast(i), upb_arena); + } + break :device_assignment_blk device_assignment; + }); + + break :executable_build_options_blk exec_build_options; + }); + + const overrides_map = c._xla_CompileOptionsProto_env_option_overrides_mutable_upb_map(options, upb_arena); + switch (platform.target) { + .cuda => { + // NVIDIA recommends these settings + // https://github.com/NVIDIA/JAX-Toolbox?tab=readme-ov-file#environment-variables + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_latency_hiding_scheduler", true, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_llvm_module_compilation_parallelism", true, upb_arena); + }, + .rocm => { + // enable only on CDNA + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_triton_gemm", gemm_blk: { + const supported: std.StaticStringMap(void) = .initComptime(.{ + .{ "gfx950", {} }, + .{ "gfx950", {} }, + .{ "gfx942", {} }, + .{ "gfx942", {} }, + .{ "gfx942", {} }, + .{ "gfx90a", {} }, + .{ "gfx90a", {} }, + .{ "gfx90a", {} }, + .{ "gfx908", {} }, + .{ "gfx906", {} }, + .{ "gfx900", {} }, + }); + break :gemm_blk supported.has(platform.devices[0].pjrt_desc.attribute(platform.pjrt_api, "compute_capability").?.string); + }, upb_arena); + // Use lld from libllvm instead of invoking the ld.lld binary. + // This saves us from having to sandbox it. + try setXlaOverrideFlag(overrides_map, "xla_gpu_use_inprocess_lld", true, upb_arena); + // Disable command buffer to avoid some weird crashes. + // This is what AMD recommended in the meantime. + try setXlaOverrideFlag(overrides_map, "xla_gpu_enable_command_buffer", "", upb_arena); + }, + else => {}, + } + + if (xla_dump_to_ orelse platform.compilation_options.xla_dump_to) |xla_dump_to| { + try setXlaOverrideFlag(overrides_map, "xla_dump_to", xla_dump_to, upb_arena); + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_as_proto", true, upb_arena); + if (platform.compilation_options.xla_dump_fusion_visualization) { + try setXlaOverrideFlag(overrides_map, "xla_dump_fusion_visualization", true, upb_arena); + } + if (platform.compilation_options.xla_dump_hlo_pass_re) |re| { + try setXlaOverrideFlag(overrides_map, "xla_dump_hlo_pass_re", re, upb_arena); + } + } + + break :blk options; + }; + + const loaded_executable = try pjrtx.Client.compile( + platform.pjrt_client, + platform.pjrt_api, + arena, + io, + module, + try upb.serialize(options, upb_arena), + ); + errdefer loaded_executable.deinit(); + + return loaded_executable; +} diff --git a/zml/nn.zig b/zml/nn.zig index eceb316b..6bbebcf9 100644 --- a/zml/nn.zig +++ b/zml/nn.zig @@ -1,17 +1,11 @@ //! Common layer definition and functions for Neural Networks (NN) const std = @import("std"); -const mlir = @import("mlir"); -const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); -const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); -const quantization = @import("quantization.zig"); -pub const Quantization = quantization.Quantization; const Shape = @import("shape.zig").Shape; const Slice = @import("slice.zig").Slice; const Tensor = @import("tensor.zig").Tensor; @@ -22,350 +16,55 @@ const log = std.log.scoped(.@"zml/nn"); pub const Linear = struct { weight: Tensor, bias: ?Tensor = null, - tag: Shape.Tag, - quantization: ?Quantization = null, + in_tag: Shape.Tag, + out_tag: Shape.Tag, pub fn init(weight: Tensor, bias: ?Tensor, tag: anytype) Linear { + stdx.debug.guard(weight.shape().hasTag(tag) != null, @src()); + // Detecting the out_tag allows us to + const axis = weight.shape().axis(tag); + const out_tag = weight.shape().tag(1 - axis); return .{ .weight = weight, .bias = bias, - .tag = zml.Shape.toTag(tag), + .in_tag = zml.Shape.toTag(tag), + .out_tag = out_tag, }; } - pub fn unloadBuffers(self: *zml.Bufferized(Linear)) void { - zml.Buffer.deinitAll(Linear, self); + pub fn unloadBuffers(linear: *zml.Bufferized(Linear)) void { + linear.weight.deinit(); + if (linear.bias) |*bias| bias.deinit(); } - pub fn quantizationScheme(self: Linear) ?Quantization.Scheme { - return if (self.quantization) |q| q.scheme else null; - } + pub fn forward(self: Linear, x: Tensor) Tensor { + var y = x.dot(self.weight, self.in_tag).renameTag(self.out_tag, self.in_tag); - pub fn quantizationScales(self: Linear) ?Tensor { - return if (self.quantization) |q| q.scales else null; - } - - pub fn forward(self: Linear, x: Tensor, out_dtype: DataType) Tensor { - if (self.quantization) |q| { - if (quantization.quantizeInput(q, x, self.tag, zml.Compiler.current().platform)) |input| { - return self.forwardQuantized(input, out_dtype); - } - return self.forwardScaled(x.convert(.bf16), null, null, out_dtype); - } - const y = x.dot(self.weight.convert(x.dtype()), self.tag).convert(out_dtype); - return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; - } - - /// Apply this layer to reusable quantized activation values and scales. - /// Convert to output_dtype before adding bias. - pub fn forwardQuantized(self: Linear, input: quantization.QuantizedInput, output_dtype: DataType) Tensor { - stdx.debug.assert(self.quantization != null, "forwardQuantized requires quantized weights", .{}); - return self.forwardScaled(input.values, input.scales, input.global_scale, output_dtype); - } - - fn forwardScaled(self: Linear, lhs: Tensor, lhs_scale: ?Tensor, input_global_scale: ?Tensor, output_dtype: DataType) Tensor { - const q = self.quantization.?; - const weight_global_scale: ?Tensor = if (q.global_scale) |s| s.asMultiplier() else null; - const weight = if (isPackedFp4(q.scheme, self.weight.dtype())) unpackFp4(self.weight, self.tag, self.tag) else self.weight; - const scales = if (q.scheme.isMx() and q.scales.dtype() == .u8) q.scales.bitCast(.f8e8m0) else q.scales; - const acc = scaledDot(lhs, weight, lhs_scale, scales, output_dtype, self.tag, .{ .rhs_scale_swizzled = q.swizzled_scales }); - const y = applyGlobalScale(acc, input_global_scale, weight_global_scale).convert(output_dtype); - return if (self.bias) |bias| y.add(bias.convert(y.dtype()).broad(y.shape())) else y; + return if (self.bias) |bias| y.add(bias.broad(y.shape())).reuseBuffer(x) else y; } }; -pub fn isPackedFp4(scheme: ?Quantization.Scheme, weight_dtype: DataType) bool { - return (scheme == .nvfp4 or scheme == .mxfp4) and (weight_dtype == .u8 or weight_dtype == .i8); -} - -/// Unpacks two f4e2m1 values per byte along `packed_tag` and names the expanded -/// axis `k_tag`. Group-size agnostic: NVFP4 (16) and MXFP4 (32) share this packing. -pub fn unpackFp4(w: Tensor, packed_tag: anytype, contracting_tag: anytype) Tensor { - stdx.debug.assert(w.dtype() == .u8 or w.dtype() == .i8, "unpackFp4 expects packed 8-bit weights, got {}", .{w.dtype()}); - const source_tag = Shape.toTag(packed_tag); - const result_tag = Shape.toTag(contracting_tag); - return w.bitCast(.f4e2m1) // bitcast inserts a tag (it respects shlo), but maybe we should simplify it - .merge(.{ .kb = .{ source_tag, .bitcast } }) - .renameTag(.kb, result_tag); -} - -test "unpackFp4 expands the requested axis" { - const platform = zml.testing.env(); - const packed_weight: Tensor = .init(.{ .out = 2, .stored = 4 }, .u8); - const Local = struct { - fn call(w: Tensor) Tensor { - return unpackFp4(w, .stored, .contracted); - } - }; - - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{packed_weight}, .{}); - defer exe.deinit(); - - try zml.testing.expectEqualShapes(.init(.{ .out = 2, .contracted = 8 }, .f4e2m1), exe.output_shapes[0]); -} - -/// Scaled matrix multiply (`xla.scaled_dot`): `acc = (lhs * lhs_scale) @ (rhs * rhs_scale)`. -/// -/// - **NVFP4**: values `.f4e2m1`, scales `.f8e4m3fn`, block 16 (weight-only bf16 lhs ok) -/// - **MXFP4**: values `.f4e2m1`, scales `.f8e8m0fnu`, block 32 -/// - **MXFP8**: values `.f8e4m3fn` / `.f8e5m2`, scales `.f8e8m0fnu`, block 32 -/// - **Block FP8**: E4M3FN / E4M3FNUZ / E8M0 values, BF16 or F32 128x128 scales -/// - TODO: INT4/8 and FP8 per tensor -/// -/// Backends: -/// XLA selects specialized GPU kernels (TileIR, Triton, or block-128 W8A8), -/// with dequant + Dot as the fallback. -/// -/// Axes follow `Tensor.dot`; XLA currently accepts one contracting axis and at -/// most one batch axis. Scales preserve operand axis order, and each scale -/// dimension must divide its corresponding value dimension. -/// -/// CPU has no specialized path. -/// -/// `out_dtype` is the element type of the returned tensor; only BF16 and F32 stay on -/// the fused path. It buys no accuracy: the kernels accumulate in F32 either way. -pub fn scaledDot( - lhs: Tensor, - rhs: Tensor, - lhs_scale: ?Tensor, - rhs_scale: Tensor, - out_dtype: DataType, - args: anytype, - opts: ScaledDotOpts, -) Tensor { - // A narrow output would be an unscaled cast of the F32 accumulator - stdx.debug.assert( - out_dtype.isFloat() and out_dtype.bitSizeOf() >= 16, - "scaledDot output dtype must be a 16-bit-or-wider float, got {s}", - .{@tagName(out_dtype)}, - ); - const dot_axes = lhs.dotAxes(rhs, args); - - const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - - var res_shape: Shape = .scalar(out_dtype); - var lhs_batching_axes: Axes = .empty; - var rhs_batching_axes: Axes = .empty; - for (dot_axes.batching.constSlice()) |b_axes| { - const l, const r = b_axes; - stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "scaledDot expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); - var t = lhs._shape.tag(l); - if (t == Shape.TagUnknown) t = rhs._shape.tag(r); - res_shape = res_shape.appendDim(lhs._shape.dim(l), t); - lhs_batching_axes.appendAssumeCapacity(lhs._shape.axis(l)); - rhs_batching_axes.appendAssumeCapacity(rhs._shape.axis(r)); - } - - var lhs_contracting_axes: Axes = .empty; - var rhs_contracting_axes: Axes = .empty; - for (dot_axes.contracting.constSlice()) |c_axes| { - const l, const r = c_axes; - stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "scaledDot expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); - lhs_contracting_axes.appendAssumeCapacity(lhs._shape.axis(l)); - rhs_contracting_axes.appendAssumeCapacity(rhs._shape.axis(r)); - } - - for (0..lhs.rank()) |l| { - if (std.mem.indexOfScalar(i64, lhs_contracting_axes.constSlice(), @intCast(l))) |_| { - continue; - } - if (std.mem.indexOfScalar(i64, lhs_batching_axes.constSlice(), @intCast(l))) |_| { - continue; - } - res_shape = res_shape.appendDim(lhs._shape.dim(l), lhs._shape.tag(l)); - } - for (0..rhs.rank()) |r| { - if (std.mem.indexOfScalar(i64, rhs_contracting_axes.constSlice(), @intCast(r))) |_| { - continue; - } - if (std.mem.indexOfScalar(i64, rhs_batching_axes.constSlice(), @intCast(r))) |_| { - continue; - } - res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); - } - - const lhs_scale_operand = lhs_scale orelse blk: { - break :blk Tensor.constantTensor(onesGrid(lhs.shape(), .bf16), DataType.bf16.one().asBytes()); - }; - - const rhs_scale_operand = if (rhs_scale.rank() != rhs.rank() and rhs_scale.shape().count() == 1) - rhs_scale.reshape(onesGrid(rhs.shape(), rhs_scale.dtype())) - else - rhs_scale; - - const mlir_ctx = zml.Compiler.current().mlir_ctx; - const dnums = mlir.Attribute.array(mlir_ctx, &.{ - .array(mlir_ctx, &.{ - .intArray(mlir_ctx, i64, lhs_contracting_axes.constSlice()), - .intArray(mlir_ctx, i64, rhs_contracting_axes.constSlice()), - }), - .array(mlir_ctx, &.{ - .intArray(mlir_ctx, i64, lhs_batching_axes.constSlice()), - .intArray(mlir_ctx, i64, rhs_batching_axes.constSlice()), - }), - }); - const attributes = [_]mlir.NamedAttribute{ - .named(mlir_ctx, "dimension_numbers", dnums), - .named(mlir_ctx, "rhs_scale_swizzled", .boolean(mlir_ctx, true)), - }; - if (opts.rhs_scale_swizzled) { - stdx.debug.assert(rhs.rank() == 2 and rhs_scale.rank() == 2 and rhs_contracting_axes.get(0) == 1, "scaledDot: a swizzled rhs scale needs a [n, k] rhs, got {f}", .{rhs}); - } - return ops.composite("xla.scaled_dot", &.{ lhs, rhs, lhs_scale_operand, rhs_scale_operand }, &.{res_shape}, scaledDotReference, res_shape, .{ - .composite_attributes = if (opts.rhs_scale_swizzled) &attributes else attributes[0..1], - })[0]; -} - -pub const ScaledDotOpts = struct { - /// The rhs scale's `[n, k / block]` grid is stored in the 128x4 blocked layout of Blackwell's block-scaled mma, - /// each device's slice on its own. - // TODO: fine for now, but we should find a better way to express how a scale is laid out. - rhs_scale_swizzled: bool = false, -}; - -test "block128 scaled dot layouts" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - // FIXME: Add rocm when we have a PJRT plugin with native fp8 dot support (by the end of Sept 2026) - if (platform.target != .cuda and platform.target != .furiosa) return error.SkipZigTest; - const dtype: DataType = if (platform.target == .rocm and @import("platform.zig").rocm.computeCapability(platform) == .gfx942) .f8e4m3fnuz else .f8e4m3fn; - const Local = struct { - const Outputs = struct { actual: Tensor, expected: Tensor, linear: Tensor, linear_expected: Tensor }; - - fn dequantize(values: Tensor, scales: Tensor) Tensor { - var expanded = scales.convert(.bf16); - for (0..values.rank()) |axis| { - const factor = @divExact(values.dim(axis), scales.dim(axis)); - const shape = expanded.shape().insert(axis + 1, .{factor}); - const axes = Shape.range(shape.rank(), .i64).remove(axis + 1); - expanded = expanded.broadcast(shape, axes.dims()).reshape(expanded.shape().setDim(axis, values.dim(axis))); - } - return values.convert(.bf16).mul(expanded); - } - - fn forward(x: Tensor, w: Tensor, scales: Tensor, fp8: DataType, prequantized: bool) Outputs { - const weight = w.convert(fp8); - const input = quantization.quantizeBlockFp8(x, .k, 128, fp8, .f32); - const linear: Linear = .{ .weight = weight, .tag = Shape.toTag(.k), .quantization = .{ .scheme = .fp8_block128, .scales = scales } }; - return .{ - .linear = linear.forward(x, .bf16), - .linear_expected = dequantize(input.values, input.scales).dot(dequantize(weight, scales), .k), - .actual = if (prequantized) - scaledDot(input.values, weight, input.scales.convert(scales.dtype()), scales, .bf16, .k, .{}) - else - scaledDot(x, weight, null, scales, .bf16, .k, .{}), - .expected = (if (prequantized) dequantize(input.values, input.scales.convert(scales.dtype())) else x) - .dot(dequantize(weight, scales), .k), - }; - } - }; - inline for (.{ - .{ .{ .m = 16, .k = 256 }, .{ .n = 256, .k = 256 } }, - .{ .{ .b = 2, .m = 16, .k = 256 }, .{ .b = 2, .n = 256, .k = 256 } }, - .{ .{ .k = 256, .b = 2, .m = 3 }, .{ .n = 256, .k = 256, .b = 2 } }, - .{ .{ .b = 2, .k = 256, .s = 2, .m = 3 }, .{ .n = 128, .b = 2, .k = 256 } }, - .{ .{ .b = 2, .m = 3, .k = 256 }, .{ .k = 256, .n = 128 } }, - .{ .{ .k = 128 }, .{ .n = 128, .k = 128 } }, - .{ .{ .m = 3, .k = 256 }, .{ .p = 2, .k = 256, .n = 128 } }, - .{ .{ .m = 3, .k = 128 }, .{ .n = 64, .k = 128 } }, - }) |layout| { - inline for (.{ DataType.f32, DataType.bf16 }) |scale_dtype| { - inline for (.{ false, true }) |prequantized| { - const x: Tensor = .init(layout[0], .bf16); - const w: Tensor = .init(layout[1], .bf16); - const scales: Tensor = .init(w.shape().setDim(.n, std.math.divCeil(i64, w.dim(.n), 128) catch unreachable).setDim(.k, @divExact(w.dim(.k), 128)).withDtype(scale_dtype), scale_dtype); - var exe = try platform.compileFn(allocator, io, Local.forward, .{ x, w, scales, dtype, prequantized }, .{}); - defer exe.deinit(); - try zml.testing.expectEqualShapes(exe.output_shapes[1], exe.output_shapes[0]); - var buffers: [3]zml.Buffer = undefined; - var initialized: usize = 0; - defer for (buffers[0..initialized]) |*buffer| buffer.deinit(); - for ([_]Shape{ x.shape(), w.shape(), scales.shape() }, 0..) |shape, i| { - const slice = try Slice.alloc(allocator, shape); - defer slice.free(allocator); - for (0..shape.count()) |j| { - const value: f32 = if (i == 2) @as(f32, @floatFromInt(1 + (j * 3 + j / 4) % 7)) / 16 else @as(f32, @floatFromInt(@as(i32, @intCast((j * 7 + j / 256) % 13)) - 6)) / 4; - if (shape.dtype() == .f32) { - slice.items(f32)[j] = value; - } else { - slice.items(zml.floats.BFloat16)[j] = .fromF32(value); - } - } - buffers[i] = try zml.Buffer.fromSlice(io, platform, slice); - initialized += 1; - } - var output: zml.Bufferized(Local.Outputs) = undefined; - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - runner.run(io, .{ buffers[0], buffers[1], buffers[2] }, .{&output}, .{ .wait = true }); - defer zml.Buffer.deinitAll(Local.Outputs, &output); - var expected = try output.expected.toSliceAlloc(allocator, io); - defer expected.free(allocator); - try zml.testing.expectClose(io, expected, output.actual, .{ .absolute_tolerance = 0.125, .relative_tolerance = 0.02 }); - var linear_expected = try output.linear_expected.toSliceAlloc(allocator, io); - defer linear_expected.free(allocator); - try zml.testing.expectClose(io, linear_expected, output.linear, .{ .absolute_tolerance = 0.125, .relative_tolerance = 0.02 }); - } - } - } -} - -/// `shape` with every dimension collapsed to 1 -fn onesGrid(shape: Shape, dt: DataType) Shape { - var res = shape.withDtype(dt); - for (0..res.rank()) |i| { - res = res.setDim(i, 1); - } - - return res; -} - -fn scaledDotReference(in: []const Tensor, out_shape: Shape) Tensor { - // Keep the unmatched-composite marker in the graph for backend lowering. - const ctx = zml.Compiler.current(); - const op = stablehlo.custom_call( - ctx.mlir_ctx, - &.{ in[0].value(), in[1].value(), in[2].value(), in[3].value() }, - &.{mlirx.Type.rankedTensor(ctx.mlir_ctx, out_shape)}, - .{ - .call_target_name = "zml$scaled_dot_unmatched", - .has_side_effect = false, - .backend_config = .{ .typed_ffi = .dict(ctx.mlir_ctx, &.{}) }, - }, - ctx.location, - ).appendTo(ctx.currentScope().block); - if (ctx.manual_computation_depth > 0 and ctx.partitioner == .gspmd) { - op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); - } - return Tensor._result(out_shape, op.result(0)); -} - -fn applyGlobalScale(acc: Tensor, igs: ?Tensor, wgs: ?Tensor) Tensor { - if (igs == null and wgs == null) { - return acc; - } - - var res = acc.convert(.f32); - if (wgs) |w| { - res = res.mul(w.convert(.f32).broad(res.shape())); - } - - if (igs) |i| { - res = res.mul(i.convert(.f32).broad(res.shape())); - } - - return res.convert(acc.dtype()); -} - pub const TokenEmbedding = struct { weight: Tensor, + pub fn init(store: zml.io.TensorStore.View) TokenEmbedding { + return .{ .weight = store.createTensor("weight").withTags(.{ .voc, .d }) }; + } + + pub fn unloadBuffers(self: *zml.Bufferized(TokenEmbedding)) void { + self.weight.deinit(); + } + pub fn forward(self: TokenEmbedding, idx: Tensor) Tensor { stdx.debug.assert(idx.dtype().isInteger(), "TokenEmbedding expects an integer input, received: {f}", .{idx}); stdx.debug.assert(self.weight.rank() == 2, "TokenEmbedding expects it's weight Tensor to be a 2D matrix, got {f}", .{self.weight}); return self.weight.withTags(.{ .voc, .d }).gather(.{ .voc = idx }, .{}); } + + pub fn unembed(self: TokenEmbedding, embeds: Tensor) Tensor { + stdx.debug.assert(embeds.shape().hasTags(.{.d}), "TokenEmbedding expects the input embeds to have a .d tag, got {f}", .{embeds.shape()}); + return self.weight.dot(embeds, .d); + } }; pub const Activation = union(enum) { @@ -452,32 +151,29 @@ test normalizeL2 { const input: zml.Tensor = .init(.{ 2, 2 }, .f32); - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, normalizeL2, .{ input, 1e-12 }, platform); defer exe.deinit(); var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9686, -1.0058, -1.7808, 0.6698 })); defer input_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, normalizeL2, .{input_buffer}); defer res.deinit(); const expectation: Slice = .init(input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6937, -0.7203, -0.9360, 0.3520 })); - try zml.testing.expectClose(std.testing.io, expectation, res, .{}); + try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); } pub const RopeOpts = struct { - layout: Layout = .real_im_pass, + layout: Layout = .sequential, + freq_base: f32 = 10_000, scaling: Scaling = .{ .default = .{} }, - /// There are 3 layouts corresponding to how to split `x` in real/imag/passthrough parts. - /// The hard part is that HF models don't specify the layout they use. - /// But HF models don't use the `interleaved` format, - /// and most models don't have a Rope passthrough, so real_im_pass and real_pass_im_pass are the same. - /// - /// * real_im_pass: [x_re0, x_re1, ..., x_im0, x_im1, ..., p_re0, p_re1, ..., p_im0, p_im1, ...] - /// * real_pass_im_pass: [x_re0, x_re1, ..., p_re0, p_re1, ..., x_im0, x_im1, ..., p_im0, p_im1, ...] - /// * interleaved: : [x_re0, x_im0, x_re1, x_im1, ..., p_re0, p_im0, p_re1, p_im1, ...] - pub const Layout = enum { real_im_pass, real_pass_im_pass, interleaved }; + /// There are two layouts corresponding to how to split `x` in real/imag parts. + /// * Interleaved means that the real/imag of each scalar is contiguous. + /// * Sequential means that you first read all real values then all imag values. + /// Typically HF models use sequential, while GGUF use interleaved. + pub const Layout = enum { interleaved, sequential }; /// There are several ways to init the scaling aka "inv_freq" pub const Scaling = union(enum) { @@ -485,7 +181,6 @@ pub const RopeOpts = struct { llama3: Llama3, yarn: Yarn, linear: Scaling.Linear, - proportional: Scaling.Proportional, pub const Default = struct { rope_theta: f32 = 10000, @@ -508,10 +203,9 @@ pub const RopeOpts = struct { mscale_all_dim: ?f32 = null, truncate: bool = true, original_max_position_embeddings: u32, - rope_theta: f32 = 10000, + rope_theta: f32, llama_4_scaling_beta: ?f32 = null, attention_factor: ?f32 = null, - partial_rotary_factor: f32 = 1.0, }; pub const Linear = struct { @@ -519,11 +213,6 @@ pub const RopeOpts = struct { rope_theta: f32 = 10000, }; - pub const Proportional = struct { - partial_rotary_factor: f32 = 1.0, - rope_theta: f32 = 10000, - }; - /// Read a Rope scaling config from HF config.json format. pub fn jsonParse(allocator: std.mem.Allocator, source: anytype, options: std.json.ParseOptions) !Scaling { return jsonParseFromValue( @@ -534,8 +223,8 @@ pub const RopeOpts = struct { } pub fn jsonParseFromValue(allocator: std.mem.Allocator, source: std.json.Value, options: std.json.ParseOptions) std.json.ParseFromValueError!Scaling { - _ = allocator; - _ = options; + _ = allocator; // autofix + _ = options; // autofix if (source == .null) return .{ .default = .{} }; if (source != .object) return error.InvalidEnumTag; @@ -546,14 +235,10 @@ pub const RopeOpts = struct { if (std.mem.eql(u8, impl.string, "llama3")) { // Note: leaky is fine here cause Llama3 struct don't need to allocate memory. return .{ .llama3 = try std.json.parseFromValueLeaky(Llama3, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; - } else if (std.mem.eql(u8, impl.string, "default")) { - return .{ .default = try std.json.parseFromValueLeaky(Default, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "yarn")) { return .{ .yarn = try std.json.parseFromValueLeaky(Yarn, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else if (std.mem.eql(u8, impl.string, "linear")) { return .{ .linear = try std.json.parseFromValueLeaky(Scaling.Linear, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; - } else if (std.mem.eql(u8, impl.string, "proportional")) { - return .{ .proportional = try std.json.parseFromValueLeaky(Scaling.Proportional, stdx.noalloc, source, .{ .ignore_unknown_fields = true }) }; } else { log.warn("Unsupported Rope implementation: {s}, will use the default one which will produce altered results", .{impl.string}); return .{ .default = .{} }; @@ -595,15 +280,6 @@ pub const RopeOpts = struct { inline else => |s| s.rope_theta, }; } - - pub fn partialRotaryDim(self: *const Scaling, head_dim: i64) u32 { - return switch (self.*) { - inline .yarn, .proportional => |s| @intFromFloat(@floor( - @as(f32, @floatFromInt(head_dim)) * std.math.clamp(s.partial_rotary_factor, 0.0, 1.0), - )), - .default, .llama3, .linear => @intCast(head_dim), - }; - } }; }; @@ -617,8 +293,7 @@ pub const RopeOpts = struct { /// - pos_idx: optional tensor which indicates which positions are needed. /// When not set `rope` return all positions from 0 to x.dim(.s) which is the max seq len. pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { - const head_dim = x.dim(.hd); - stdx.debug.assert(@mod(head_dim, 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); + stdx.debug.assert(@mod(x.dim(.hd), 2) == 0, "rope expects a even head dim (.hd), got {f}", .{x}); const idx = if (pos_idx) |idx| blk: { stdx.debug.assert(x.shape().hasTags(.{.hd}), "rope expects x argument to have .hd axes got: rope(x={f}, idx={f})", .{ x, idx }); @@ -627,15 +302,11 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { stdx.debug.assert(x.shape().hasTags(.{ .s, .hd }), "rope expects x argument to have both .s and .hd axes got: rope(x={f})", .{x}); break :blk Tensor.arange(.{ .end = x.dim(.s) }, .f32).withTags(.{.s}); }; - - const rotary_dim: u32 = opts.scaling.partialRotaryDim(head_dim); - stdx.debug.assert(rotary_dim > 0 and @mod(rotary_dim, 2) == 0, "partial rope expects a even head dim (.hd), got {d}", .{rotary_dim}); - - const x_real, const x_imag, const x_pass = zml.nn.splitRealImgPass(x, opts.layout, rotary_dim); - const inv_freq = invFreq(head_dim, opts).withTags(.{.hd}); + const x_real, const x_imag = splitRealImg(x, opts.layout); // compute sin and cos in f32 before downcasting to x type. - const inv_freq_pos = zml.Tensor.outer(idx.convert(.f32), inv_freq); + const inv_freq = invFreq(x.dim(.hd), opts).withTags(.{.hd}); + const inv_freq_pos = Tensor.outer(idx.convert(.f32), inv_freq); const scaling = opts.scaling.attentionScaling(); const cos = inv_freq_pos.cos().scale(scaling).convert(x.dtype()).broad(x_real.shape()); const sin = inv_freq_pos.sin().scale(scaling).convert(x.dtype()).broad(x_real.shape()); @@ -644,116 +315,55 @@ pub fn rope(x: Tensor, pos_idx: ?Tensor, opts: RopeOpts) Tensor { const y_real = x_real.mul(cos).sub(x_imag.mul(sin)); const y_imag = x_real.mul(sin).add(x_imag.mul(cos)); - return zml.nn.mergeRealImgPass(y_real, y_imag, x_pass, opts.layout); + // flatten last dimensions + return mergeRealImg(y_real, y_imag, opts.layout); } pub fn splitRealImg(x: Tensor, layout: RopeOpts.Layout) [2]Tensor { const n = x.dim(-1); return switch (layout) { - .real_im_pass, .real_pass_im_pass => .{ - x.slice(-1, .{ .end = @divExact(n, 2) }), - x.slice(-1, .{ .start = @divExact(n, 2), .end = n }), + .sequential => .{ + x.slice1d(-1, .{ .end = @divExact(n, 2) }), + x.slice1d(-1, .{ .start = @divExact(n, 2), .end = n }), }, .interleaved => .{ - x.slice(-1, .{ .start = 0, .step = 2 }), - x.slice(-1, .{ .start = 1, .step = 2 }), - }, - }; -} - -const Pass = union(RopeOpts.Layout) { - real_im_pass: Tensor, - real_pass_im_pass: [2]Tensor, - interleaved: Tensor, -}; - -pub fn splitRealImgPass(x: Tensor, layout: RopeOpts.Layout, rotary_dim: u32) struct { Tensor, Tensor, ?Pass } { - const ax = x.axis(.hd); - const hd = x.dim(ax); - if (rotary_dim == hd) { - const real, const img = splitRealImg(x, layout); - return .{ real, img, null }; - } - - const half = @divExact(x.dim(ax), 2); - const half_rotary = @divExact(rotary_dim, 2); - return switch (layout) { - .real_im_pass => .{ - x.slice(ax, .{ .end = half_rotary }), - x.slice(ax, .{ .start = half_rotary, .end = rotary_dim }), - .{ .real_im_pass = x.slice(ax, .{ .start = rotary_dim }) }, - }, - .real_pass_im_pass => .{ - x.slice(ax, .{ .end = half_rotary }), - x.slice(ax, .{ .start = half, .end = half + half_rotary }), - .{ .real_pass_im_pass = .{ - x.slice(ax, .{ .start = half_rotary, .end = half }), - x.slice(ax, .{ .start = half + half_rotary }), - } }, - }, - .interleaved => .{ - x.slice(ax, .{ .start = 0, .end = rotary_dim, .step = 2 }), - x.slice(ax, .{ .start = 1, .end = rotary_dim, .step = 2 }), - .{ .interleaved = x.slice(ax, .{ .start = rotary_dim }) }, + x.slice1d(-1, .{ .start = 0, .step = 2 }), + x.slice1d(-1, .{ .start = 1, .step = 2 }), }, }; } pub fn mergeRealImg(x_real: Tensor, x_imag: Tensor, layout: RopeOpts.Layout) Tensor { return switch (layout) { - .real_im_pass, .real_pass_im_pass => .concatenate(&.{ x_real, x_imag }, -1), - .interleaved => Tensor.stack(&.{ x_real, x_imag }, .last, .interleaved_real_img) - .merge(.{ .hd = .{ .hd, .interleaved_real_img } }), - }; -} - -pub fn mergeRealImgPass(x_real: Tensor, x_imag: Tensor, x_pass: ?Pass, layout: RopeOpts.Layout) Tensor { - if (x_pass == null) return mergeRealImg(x_real, x_imag, layout); - - return switch (x_pass.?) { - .real_im_pass => |pass| .concatenate(&.{ x_real, x_imag, pass }, .hd), - .real_pass_im_pass => |pass| .concatenate(&.{ x_real, pass[0], x_imag, pass[1] }, .hd), - .interleaved => |pass| .concatenate( - &.{ - Tensor.stack(&.{ x_real, x_imag }, .last, .interleaved_real_img) - .merge(.{ .hd = .{ .hd, .interleaved_real_img } }), - pass, - }, - .hd, - ), + .sequential => Tensor.concatenate(&.{ x_real, x_imag }, -1), + .interleaved => Tensor.concatenate(&.{ + x_real.appendAxes(.{.interleaved_real_img}), + x_imag.appendAxes(.{.interleaved_real_img}), + }, -1).reshape(x_imag.shape().setDim(-1, -1)), }; } /// {exp( - n * ln(10_000) / N ) | n in [0..N] } -pub fn invFreq(N: i64, opts: RopeOpts) Tensor { - const allocator = zml.Compiler.current().allocator; - const N_half: u32 = @intCast(@divExact(N, 2)); - const num_freqs: u32 = opts.scaling.partialRotaryDim(N_half); - - const inv_freq = allocator.alloc(f32, num_freqs) catch @panic("OOM"); +fn invFreq(N: i64, opts: RopeOpts) Tensor { + const allocator = zml.module.CompilationContext.current().allocator; + const N_half: usize = @intCast(@divExact(N, 2)); + const inv_freq = allocator.alloc(f32, N_half) catch @panic("OOM"); defer allocator.free(inv_freq); - _invFreq(opts, inv_freq, N_half); - return .constantTensor(.init(.{num_freqs}, .f32), @ptrCast(inv_freq)); + _invFreq(opts, inv_freq); + return zml.Tensor.constantTensor(zml.Shape.init(.{@divExact(N, 2)}, .f32), std.mem.sliceAsBytes(inv_freq)); } -fn _invFreq(opts: RopeOpts, inv_freq: []f32, half_head_dim: u32) void { - const N: u32 = @intCast(inv_freq.len); +fn _invFreq(opts: RopeOpts, inv_freq: []f32) void { + const N = inv_freq.len; // Default frequencies for (0.., inv_freq) |n, *f| { - f.* = @exp(-@log(opts.scaling.getRopeTheta()) * stdx.math.divFloat(f32, n, N)); + f.* = @exp(-@log(opts.freq_base) * stdx.math.divFloat(f32, n, N)); } switch (opts.scaling) { .default => {}, - .proportional => { - // proportional uses the original number of dimensions, - // instead of the actual number of rotated dims. - for (0.., inv_freq[0..]) |n, *f| { - f.* = @exp(-@log(opts.scaling.getRopeTheta()) * stdx.math.divFloat(f32, n, half_head_dim)); - } - }, .linear => |l| { for (inv_freq) |*f| f.* /= l.factor; }, @@ -781,15 +391,15 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32, half_head_dim: u32) void { } }, .yarn => |s| { - const N_f: f64 = @floatFromInt(N); + const N_f: f64 = @floatFromInt(inv_freq.len); const M: f64 = @floatFromInt(s.original_max_position_embeddings); const f_high = s.beta_fast * (2 * std.math.pi) / M; const f_low = s.beta_slow * (2 * std.math.pi) / M; const downscaling = 1.0 / s.factor.?; // This isn't a typo: low n have a high frequency, high n have a low frequency. - var n_low: f64 = -@log(f_high) / @log(opts.scaling.getRopeTheta()) * N_f; - var n_high: f64 = -@log(f_low) / @log(opts.scaling.getRopeTheta()) * N_f; + var n_low: f64 = -@log(f_high) / @log(opts.freq_base) * N_f; + var n_high: f64 = -@log(f_low) / @log(opts.freq_base) * N_f; if (s.truncate) { n_high = std.math.ceil(n_high); n_low = std.math.floor(n_low); @@ -814,18 +424,18 @@ fn _invFreq(opts: RopeOpts, inv_freq: []f32, half_head_dim: u32) void { test "invFreq Llama3" { // Llama 3.2-1B config const llama_conf: RopeOpts = .{ + .freq_base = 500_000, .scaling = .{ .llama3 = .{ .factor = 32, .high_freq_factor = 4, .low_freq_factor = 1, .original_max_position_embeddings = 8192, - .rope_theta = 500_000, } }, }; const llama_freq = [_]f32{ 1.000000000000e+00, 6.636012792587e-01, 4.403666257858e-01, 2.922278344631e-01, 1.939227581024e-01, 1.286873817444e-01, 8.539710193872e-02, 5.666961893439e-02, 3.760603070259e-02, 2.495540864766e-02, 1.656044088304e-02, 1.098952908069e-02, 7.292665075511e-03, 4.839421249926e-03, 3.211446106434e-03, 1.290548010729e-03, 4.295567050576e-04, 9.708286233945e-05, 1.946163865796e-05, 1.291476746701e-05, 8.570255886298e-06, 5.687232260243e-06, 3.774054448513e-06, 2.504467147446e-06, 1.661967417022e-06, 1.102883629756e-06, 7.318749339902e-07, 4.856731266045e-07, 3.222932889457e-07, 2.138742303259e-07, 1.419272024350e-07, 9.418306490261e-08 }; var inv_freq: @TypeOf(llama_freq) = undefined; - _invFreq(llama_conf, &inv_freq, 32); + _invFreq(llama_conf, &inv_freq); for (llama_freq, inv_freq, 0..) |expected, actual, i| { errdefer log.err("Mismatch at position {d}.\nExpected: {any}\nActual: {any}", .{ i, llama_freq, inv_freq }); try std.testing.expectApproxEqRel(expected, actual, 1e-5); @@ -834,19 +444,19 @@ test "invFreq Llama3" { test "invFreq Yarn" { const yarn_conf: RopeOpts = .{ + .freq_base = 150_000, .scaling = .{ .yarn = .{ .factor = 32.0, .beta_fast = 32.0, .beta_slow = 1.0, .original_max_position_embeddings = 4096, .truncate = true, - .rope_theta = 150_000, } }, }; const yarn_freq = [_]f32{ 1.000000000000e+00, 6.890442967415e-01, 4.747820496559e-01, 3.271458745003e-01, 2.254180014133e-01, 1.553229838610e-01, 1.070244237781e-01, 7.374456524849e-02, 5.081327259541e-02, 3.162075206637e-02, 1.945096626878e-02, 1.179219130427e-02, 7.015713956207e-03, 4.069554619491e-03, 2.277272054926e-03, 1.206130953506e-03, 5.809474969283e-04, 2.279478358105e-04, 3.830881178146e-05, 2.639646845637e-05, 1.818833698053e-05, 1.253256959899e-05, 8.635495760245e-06, 5.950239483354e-06, 4.099978468730e-06, 2.825066758305e-06, 1.946596285052e-06, 1.341290953860e-06, 9.242089618056e-07, 6.368209142238e-07, 4.387978549403e-07, 3.023511396805e-07 }; var inv_freq: @TypeOf(yarn_freq) = undefined; - _invFreq(yarn_conf, &inv_freq, 32); + _invFreq(yarn_conf, &inv_freq); for (yarn_freq, inv_freq, 0..) |expected, actual, i| { errdefer log.err("Mismatch at position {d}.\nExpected: {d}\nActual: {d}", .{ i, stdx.fmt.slice(&yarn_freq), stdx.fmt.slice(&inv_freq) }); try std.testing.expectApproxEqRel(expected, actual, 1e-5); @@ -854,73 +464,12 @@ test "invFreq Yarn" { try std.testing.expectApproxEqRel(1.3465735902799727, yarn_conf.scaling.attentionScaling(), 1e-5); } -test "RopeOpts.Scaling parses proportional" { - const json = - \\{ - \\ "rope_type": "proportional", - \\ "partial_rotary_factor": 0.25, - \\ "rope_theta": 1000000 - \\} - ; - var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); - defer value.deinit(); - - const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); - switch (scaling) { - .proportional => |p| { - try std.testing.expectApproxEqRel(0.25, p.partial_rotary_factor, 1e-6); - try std.testing.expectApproxEqRel(1_000_000, p.rope_theta, 1e-6); - }, - else => try std.testing.expect(false), - } -} - -test "RopeOpts.Scaling parses yarn partial rotary factor" { - const json = - \\{ - \\ "rope_type": "yarn", - \\ "factor": 32.0, - \\ "original_max_position_embeddings": 4096, - \\ "partial_rotary_factor": 0.5 - \\} - ; - var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); - defer value.deinit(); - - const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); - switch (scaling) { - .yarn => |y| { - try std.testing.expectApproxEqRel(0.5, y.partial_rotary_factor, 1e-6); - }, - else => try std.testing.expect(false), - } -} - -test "RopeOpts.Scaling parses default rope type" { - const json = - \\{ - \\ "rope_type": "default", - \\ "rope_theta": 10000 - \\} - ; - var value = try std.json.parseFromSlice(std.json.Value, std.testing.allocator, json, .{}); - defer value.deinit(); - - const scaling = try RopeOpts.Scaling.jsonParseFromValue(std.testing.allocator, value.value, .{}); - switch (scaling) { - .default => |d| { - try std.testing.expectApproxEqRel(10_000, d.rope_theta, 1e-6); - }, - else => try std.testing.expect(false), - } -} - test "real/img" { const platform = zml.testing.env(); const Fns = struct { fn testSplitMergeIsId(layout: RopeOpts.Layout) Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ .n = 5, .hd = 4 }); + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); const real, const imag = splitRealImg(x, layout); const y = mergeRealImg(real, imag, layout); const real2, const imag2 = splitRealImg(y, layout); @@ -930,8 +479,8 @@ test "real/img" { } fn testSplitSeqVoid(_: void) Tensor { - const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ .n = 5, .hd = 4 }); - const real, const imag = splitRealImg(x, .real_im_pass); + const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); + const real, const imag = splitRealImg(x, .sequential); const x_real = Tensor.concatenate(&.{ Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), @@ -948,7 +497,7 @@ test "real/img" { fn testSplitSeq() Tensor { const x = Tensor.arange(.{ .end = 20 }, .f32).reshape(.{ 5, 4 }); - const real, const imag = splitRealImg(x, .real_im_pass); + const real, const imag = splitRealImg(x, .sequential); const x_real = Tensor.concatenate(&.{ Tensor.arange(.{ .start = 0, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), Tensor.arange(.{ .start = 1, .end = 20, .step = 4 }, .f32).reshape(.{ 5, 1 }), @@ -975,47 +524,47 @@ test "real/img" { } }; { - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.interleaved}, platform); defer exe.deinit(); - var d_interleaved = try exe.eval(std.testing.allocator, std.testing.io, {}); + var d_interleaved = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); defer d_interleaved.deinit(); try std.testing.expectEqual(20, try d_interleaved.getValue(i32, std.testing.io)); } { - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.real_im_pass}, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitMergeIsId, .{.sequential}, platform); defer exe.deinit(); - var d_sequential = try exe.eval(std.testing.allocator, std.testing.io, {}); + var d_sequential = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitMergeIsId, {}); defer d_sequential.deinit(); try std.testing.expectEqual(20, try d_sequential.getValue(i32, std.testing.io)); } // test the function that accepts 1 void argument { - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeqVoid, .{{}}, platform); defer exe.deinit(); - var d_split_seq_void = try exe.eval(std.testing.allocator, std.testing.io, {}); + var d_split_seq_void = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeqVoid, {}); defer d_split_seq_void.deinit(); try std.testing.expectEqual(20, try d_split_seq_void.getValue(i32, std.testing.io)); } // test the function that takes NO arguments { - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitSeq, .{}, platform); defer exe.deinit(); - var d_split_seq = try exe.eval(std.testing.allocator, std.testing.io, {}); + var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitSeq, {}); defer d_split_seq.deinit(); try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); } { - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Fns.testSplitInterleaved, .{}, platform); defer exe.deinit(); - var d_split_seq = try exe.eval(std.testing.allocator, std.testing.io, {}); + var d_split_seq = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Fns.testSplitInterleaved, {}); defer d_split_seq.deinit(); try std.testing.expectEqual(20, try d_split_seq.getValue(i32, std.testing.io)); } @@ -1029,7 +578,7 @@ test rope { var input = x; { // Convert input to the requested format - const real, const imag = splitRealImg(input, .real_im_pass); + const real, const imag = splitRealImg(input, .sequential); input = mergeRealImg(real, imag, opts.layout); } var res = rope(input, null, opts).squeeze(0); @@ -1037,7 +586,7 @@ test rope { { // Convert back to sequential const real, const imag = splitRealImg(res, opts.layout); - res = mergeRealImg(real, imag, .real_im_pass); + res = mergeRealImg(real, imag, .sequential); } return res; } @@ -1046,118 +595,21 @@ test rope { // x is made such as the interleaved and sequential reps are the same. // So the two implementations should give the same results. const x: zml.Tensor = .init(.{ .b = 1, .s = 5, .hd = 4 }, .f32); - var exe_interleaved = try platform.compileFn(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, .{}); + var exe_interleaved = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .interleaved } }, platform); defer exe_interleaved.deinit(); - var exe_sequential = try platform.compileFn(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .real_im_pass } }, .{}); + var exe_sequential = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, RopeOpts{ .layout = .sequential } }, platform); defer exe_sequential.deinit(); - const x_values: [5][4]f32 = @splat(.{ 1.0, 0.1, -1.0, -0.5 }); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_values)); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 1.0, 0.1, -1.0, -0.5 } ** 5)); defer x_buffer.deinit(); - var res1 = try exe_interleaved.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + var res1 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_interleaved, Local._fwd, .{x_buffer}); defer res1.deinit(); - var res2 = try exe_sequential.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + var res2 = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_sequential, Local._fwd, .{x_buffer}); defer res2.deinit(); - try zml.testing.expectClose(std.testing.io, res1, res2, .{}); -} - -test "rope: Proportional" { - const io = std.testing.io; - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - - const x: zml.Tensor = .init(.{ .s = 5, .hd = 16 }, .f32); - var exe = try platform.compileFn( - allocator, - io, - rope, - .{ - x, - null, - .{ - .layout = .real_pass_im_pass, - .scaling = .{ .proportional = .{ .partial_rotary_factor = 0.25 } }, - }, - }, - .{}, - ); - defer exe.deinit(); - - const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); - var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); - defer x_buffer.deinit(); - - const expected_h: [5][16]f32 = .{ - .{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, - .{ 0.667, -0.449, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.468, 0.904, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, - .{ -0.0328, -0.708, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.814, 0.719, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, - .{ -0.703, -0.897, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, 0.412, 0.464, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, - .{ -0.727, -0.997, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.369, 0.162, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, - }; - const expected: zml.Slice = .init(x.shape(), std.mem.sliceAsBytes(&expected_h)); - - var res = try exe.eval(allocator, io, .{ x_buffer, null }); - defer res.deinit(); - const res_h = try res.toSliceAlloc(allocator, io); - defer res_h.free(allocator); - - errdefer std.log.warn("{d:20}", .{res_h}); - try zml.testing.expectClose(io, expected, res, .{}); -} - -test "rope: Yarn with partial_rotary_factor" { - const io = std.testing.io; - const allocator = std.testing.allocator; - const platform = zml.testing.env(); - - const x: zml.Tensor = .init(.{ .s = 5, .hd = 16 }, .f32); - var exe = try platform.compileFn( - allocator, - io, - rope, - .{ - x, - null, - RopeOpts{ - .layout = .real_im_pass, - .scaling = .{ .yarn = .{ - .factor = 32.0, - .beta_fast = 32.0, - .beta_slow = 1.0, - .original_max_position_embeddings = 4096, - .truncate = true, - .rope_theta = 150_000, - .partial_rotary_factor = 0.25, - } }, - }, - }, - .{}, - ); - defer exe.deinit(); - - const x_h: [5][16]f32 = @splat([_]f32{ 0.754, -0.146, 0.00443, -0.958, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }); - var x_buffer: zml.Buffer = try .fromBytes(io, platform, x.shape(), @ptrCast(&x_h)); - defer x_buffer.deinit(); - - const expected_h: [5][16]f32 = .{ - .{ 1.02, -0.197, 0.00597, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, - .{ 0.544, -0.195, 0.858, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, - .{ -0.428, -0.193, 0.921, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, - .{ -1.01, -0.191, 0.137, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, - .{ -0.659, -0.19, -0.772, -1.29, -0.533, 0.834, 0.555, 0.853, -0.309, 0.999, 0.983, -0.693, 0.979, 0.997, -0.359, 0.888 }, - }; - const expected: zml.Slice = .init(x.shape(), std.mem.sliceAsBytes(&expected_h)); - - var res = try exe.eval(allocator, io, .{ x_buffer, null }); - defer res.deinit(); - const res_h = try res.toSliceAlloc(allocator, io); - defer res_h.free(allocator); - - errdefer std.log.warn("{d:20}", .{res_h}); - try zml.testing.expectClose(io, expected, res, .{}); + try zml.testing.expectClose(std.testing.io, res1, res2, 1e-4); } pub const UpsampleMode = enum { @@ -1222,41 +674,24 @@ test nearest { // 3D Tensor (basic) { const input_3d_basic: zml.Tensor = .init(.{ 1, 1, 2 }, .i32); - var exe = try platform.compileFn( - std.testing.allocator, - std.testing.io, - upsample, - .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_basic, .{ .scale_factor = &.{3}, .mode = .nearest } }, platform); defer exe.deinit(); - var input_3d_basic_buffer: zml.Buffer = try .fromBytes( - std.testing.io, - platform, - input_3d_basic.shape(), - std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}}), - ); + var input_3d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_basic.shape(), std.mem.sliceAsBytes(&[1][1][2]i32{.{.{ 1, 2 }}})); defer input_3d_basic_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_3d_basic_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_basic_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6 }, result.shape().dims()); const expected: Slice = .init(Shape.init(.{ 1, 1, 6 }, .i32), std.mem.sliceAsBytes(&[1][1][6]i32{.{.{ 1, 1, 1, 2, 2, 2 }}})); - try zml.testing.expectClose(std.testing.io, expected, result, .{}); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } // 3D Tensor (advanced) { const input_3d_advanced: zml.Tensor = .init(.{ 2, 3, 4 }, .i32); - var exe = try platform.compileFn( - std.testing.allocator, - std.testing.io, - upsample, - .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_3d_advanced, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); defer exe.deinit(); var input_3d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_3d_advanced.shape(), std.mem.sliceAsBytes(&[2][3][4]i32{ @@ -1265,7 +700,7 @@ test nearest { })); defer input_3d_advanced_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_3d_advanced_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_3d_advanced_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 3, 8 }, result.shape().dims()); @@ -1281,25 +716,19 @@ test nearest { .{ 21, 21, 22, 22, 23, 23, 24, 24 }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, .{}); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } // 4D Tensor (basic) { const input_4d_basic: zml.Tensor = .init(.{ 1, 1, 2, 2 }, .i32); - var exe = try platform.compileFn( - std.testing.allocator, - std.testing.io, - upsample, - .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_basic, .{ .scale_factor = &.{ 3, 3 }, .mode = .nearest } }, platform); defer exe.deinit(); var input_4d_basic_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_basic.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_4d_basic_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_4d_basic_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_basic_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 6, 6 }, result.shape().dims()); @@ -1311,18 +740,12 @@ test nearest { .{ 3, 3, 3, 4, 4, 4 }, .{ 3, 3, 3, 4, 4, 4 }, }}})); - try zml.testing.expectClose(std.testing.io, expected, result, .{}); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } // 4D Tensor (advanced) { const input_4d_advanced: zml.Tensor = .init(.{ 2, 2, 2, 2 }, .i32); - var exe = try platform.compileFn( - std.testing.allocator, - std.testing.io, - upsample, - .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_4d_advanced, .{ .scale_factor = &.{ 2, 2 }, .mode = .nearest } }, platform); defer exe.deinit(); var input_4d_advanced_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_4d_advanced.shape(), std.mem.sliceAsBytes(&[2][2][2][2]i32{ .{ @@ -1334,7 +757,7 @@ test nearest { } })); defer input_4d_advanced_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_4d_advanced_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_4d_advanced_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 2, 2, 4, 4 }, result.shape().dims()); @@ -1368,24 +791,18 @@ test nearest { }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, .{}); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } // 5D Tensor (basic) { const input_5d: zml.Tensor = .init(.{ 1, 1, 1, 2, 2 }, .i32); - var exe = try platform.compileFn( - std.testing.allocator, - std.testing.io, - upsample, - .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, upsample, .{ input_5d, .{ .scale_factor = &.{2}, .mode = .nearest } }, platform); defer exe.deinit(); var input_5d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input_5d.shape(), std.mem.sliceAsBytes(&[_]i32{ 1, 2, 3, 4 })); defer input_5d_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_5d_buffer}); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, upsample, .{input_5d_buffer}); defer result.deinit(); try std.testing.expectEqualSlices(i64, &.{ 1, 1, 2, 4, 4 }, result.shape().dims()); @@ -1407,7 +824,7 @@ test nearest { }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, .{}); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } } @@ -1430,7 +847,7 @@ pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Ten for (new_size.constSlice(), tags_.constSlice()) |d, t| { const ax = image.shape().axis(t); const child_opt: ResizeOpts = .{ - .original_len = if (opt.original_len) |o| o.slice(0, .single(ax)) else null, + .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, }; out = resizeLinear1d(out, ax, d, child_opt); } @@ -1439,16 +856,15 @@ pub fn resizeBilinear(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Ten test resizeBilinear { const platform = zml.testing.env(); - // Only test shapes - var comp = zml.Compiler.init(std.testing.allocator, std.testing.io, platform, .{}); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = @import("mlir").Block.init(&.{}, &.{}); - const scope = comp.pushBlock(block); - defer scope.pop(); + comp.pushBlock(block); + defer comp.popBlock(); inline for (.{ .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, @@ -1497,7 +913,7 @@ pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tens for (new_size.constSlice(), tags_.constSlice()) |d, t| { const ax = image.shape().axis(t); const child_opt: ResizeOpts = .{ - .original_len = if (opt.original_len) |o| o.slice(0, .single(ax)) else null, + .original_len = if (opt.original_len) |o| o.choose1d(0, ax) else null, }; out = resizeCubic1d(out, ax, d, child_opt); } @@ -1506,16 +922,15 @@ pub fn resizeBicubic(image: Tensor, resized_axes: anytype, opt: ResizeOpts) Tens test resizeBicubic { const platform = zml.testing.env(); - // Only test shapes - var comp = zml.Compiler.init(std.testing.allocator, std.testing.io, platform, .{}); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = @import("mlir").Block.init(&.{}, &.{}); - const scope = comp.pushBlock(block); - defer scope.pop(); + comp.pushBlock(block); + defer comp.popBlock(); inline for (.{ .{ .{ .a = 10, .b = 10 }, .{ .a = 20 }, .{ .a = 20, .b = 10 } }, @@ -1607,7 +1022,9 @@ pub fn causalAttnMask( } if (dtype.isFloat()) { - mask = Tensor.select(mask, .scalar(0, dtype), .scalar(-std.math.inf(f32), dtype)); + const zeros = Tensor.constant(dtype.zero()).broad(mask.shape()); + const minus_inf = Tensor.constant(dtype.minValue()).broad(mask.shape()); + mask = Tensor.select(mask, zeros, minus_inf); } else { mask = mask.convert(dtype); } @@ -1619,6 +1036,9 @@ pub const SdpaOpts = struct { attn_mask: ?Tensor = null, scale: ?Tensor = null, softmax_bias: ?Tensor = null, + allow_cudnn: bool = true, + // TODO: put a callback instead of all this field, + // so that }; /// Scaled dot product attention. @@ -1644,6 +1064,11 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { stdx.debug.assert(k.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "k is missing tags {{.h, .k, .hd}}", err_args); stdx.debug.assert(v.shape().hasTags(.{ .h, .k, .hd }), err_template ++ "v is missing tags {{.h, .k, .hd}}", err_args); + // TODO(Corentin): Re-enable that + //if (opts.allow_cudnn and cuda.canUseCudnnSdpa(q.shape()) and opts.softmax_bias == null) { + // return cuda.sdpa(q, k, v, opts); + //} + // Handle different numbers of head by splitting q heads. // This is a bit error prone in the sense that it depends of the layout of q heads. // This is the Llama convention though. @@ -1658,6 +1083,7 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { k = k.mul(head_scaling.convert(k.dtype())); var attn_weights = q.dot(k, .hd); + // log.debug("attn_weights : {f}, attn_mask : {?f}", .{ attn_weights, attn_mask }); if (attn_mask) |mask| attn_weights = attn_weights.add(mask.broad(attn_weights.shape())); attn_weights = attn_weights.convert(.f32); attn_weights = if (opts.softmax_bias) |softmax_bias| attn: { @@ -1671,308 +1097,6 @@ pub fn sdpa(q_: Tensor, k_: Tensor, v_: Tensor, opts: SdpaOpts) Tensor { return attn.transpose(q.shape()).merge(.{ .h = .{ .h, .hq } }); } -pub const GatedDeltaNet = struct { - /// Query tensor .{ .s, .h, .k }. - queries: Tensor, - /// Key tensor .{ .s, .h, .k }. - keys: Tensor, - /// Value tensor .{ .s, .h, .v }. - values: Tensor, - /// Forget gate .{ .s, .h }. - alphas: Tensor, - /// Delta gate .{ .s, .h }. - betas: Tensor, - /// Number of sequence positions to process, clamped to the input length. - active_length: Tensor, - - pub const State = struct { - /// Per-head recurrent state with shape .{ .h, .v, .k }. - s: Tensor, - /// Sequence of outputs with shape .{ .s, .h, .v }. - outputs: Tensor, - - /// Current step, scalar - step: Tensor, - }; - - pub const Input = struct { - s: Tensor, - }; - - pub const Output = struct { - /// Per-head recurrent state with shape .{ .h, .v, .k }. - state: Input, - /// Sequence of outputs with shape .{ .s, .h, .v }. - outputs: Tensor, - }; - - pub fn cond(gdn: GatedDeltaNet, state: State) Tensor { - const active_length = gdn.active_length - .convert(.i32) - .maximum(.scalar(0, .i32)) - .minimum(.scalar(gdn.queries.dim(.s), .i32)); - return state.step.cmp(.LT, active_length); - } - - /// Single-step recurrent update for Gated Delta Net. - pub fn body(gdn: GatedDeltaNet, state: State) State { - // q, k: .{ .h, .k } - const q = sliceStep(gdn.queries, state.step); - const k = sliceStep(gdn.keys, state.step); - // v: .{ .h, .v } - const v = sliceStep(gdn.values, state.step); - // alpha, beta: .{ .h } - const alpha = sliceStep(gdn.alphas, state.step); - const beta = sliceStep(gdn.betas, state.step); - - const hvk = state.s.shape(); - const hv = hvk.drop(.k); - - const v_hat = state.s.dot(k, .k).mul(alpha.broad(hv)); - const delta = v.sub(v_hat).mul(beta.broad(hv)); - const delta_k = delta.broad(hvk).mul(k.broad(hvk)); - const s_new = state.s.mul(alpha.broad(hvk)).add(delta_k); - const y = s_new.dot(q, .k); - - return .{ - .s = s_new, - .outputs = state.outputs.dynamicUpdateSlice(.{ .s = state.step }, y), - .step = state.step.addConstant(1), - }; - } - - fn sliceStep(input: Tensor, step_: Tensor) Tensor { - return input.slice(.s, .dynSingle(step_)); - } - - fn validateInitialState(gdn: GatedDeltaNet, state: State) void { - const err_template = "GatedDeltaNet.step(state: {f}, q: {f}, k: {f}, v: {f}, alpha: {f}, beta: {f}) is invalid ! "; - const err_args = .{ state.s, gdn.queries, gdn.keys, gdn.values, gdn.alphas, gdn.betas }; - - stdx.debug.assert(state.s.shape().hasTags(.{ .h, .v, .k }), err_template ++ "state.s is missing tags {{.h, .v, .k}}", err_args); - stdx.debug.assert(gdn.queries.shape().hasTags(.{ .s, .h, .k }), err_template ++ "q is missing tags {{.h, .k}}", err_args); - stdx.debug.assert(gdn.keys.shape().hasTags(.{ .s, .h, .k }), err_template ++ "k is missing tags {{.h, .k}}", err_args); - stdx.debug.assert(gdn.values.shape().hasTags(.{ .s, .h, .v }), err_template ++ "v is missing tags {{.h, .v}}", err_args); - stdx.debug.assert(gdn.alphas.shape().hasTags(.{ .s, .h }), err_template ++ "alphas is missing tag {{.h}}", err_args); - stdx.debug.assert(gdn.betas.shape().hasTags(.{ .s, .h }), err_template ++ "betas is missing tag {{.h}}", err_args); - stdx.debug.assert(gdn.active_length.rank() == 0 and gdn.active_length.dtype().isInteger(), err_template ++ "active_length must be an integer scalar", err_args); - - _ = collectDims(.{.h}, &.{ state.s, gdn.queries, gdn.keys, gdn.values, gdn.alphas, gdn.betas }, .strict) catch { - stdx.debug.panic(err_template ++ "head dimensions are inconsistent.", err_args); - }; - _ = collectDims(.{.k}, &.{ state.s, gdn.queries, gdn.keys }, .strict) catch { - stdx.debug.panic(err_template ++ "key dimensions are inconsistent.", err_args); - }; - _ = collectDims(.{.v}, &.{ state.s, gdn.values }, .strict) catch { - stdx.debug.panic(err_template ++ "value dimensions are inconsistent.", err_args); - }; - } - - /// Processes a sequence with Gated Delta Net recurrence using `stablehlo.while`. - /// - /// Shapes: - /// - `queries`, `keys`: .{ .s, .h, .k } - /// - `values`, `outputs`: .{ .s, .h, .v } - /// - `alphas`, `betas`: .{ .s, .h } - /// - `initial_state.s`: .{ .h, .v, .k } - /// - `active_length`: integer scalar - pub fn forward( - queries: Tensor, - keys: Tensor, - values: Tensor, - alphas: Tensor, - betas: Tensor, - initial_state: Input, - active_length: Tensor, - ) Output { - const gdn: GatedDeltaNet = .{ - .queries = queries, - .keys = keys, - .values = values, - .alphas = alphas, - .betas = betas, - .active_length = active_length, - }; - const state: GatedDeltaNet.State = .{ - .step = .scalar(0, .i32), - .s = initial_state.s, - .outputs = .zeroes(values.shape()), - }; - validateInitialState(gdn, state); - - const final_state = if (queries.dim(.s) == 1) blk: { - // avoids a `stablehlo.while`, which costs a host sync per call on GPU. - const next = gdn.body(state); - const run = gdn.cond(state); - break :blk GatedDeltaNet.State{ - .s = run.broad(state.s.shape()).select(next.s, state.s), - .outputs = run.broad(state.outputs.shape()).select(next.outputs, state.outputs), - .step = run.select(next.step, state.step), - }; - } else ops.@"while"(GatedDeltaNet, gdn, state); - - return .{ - .outputs = final_state.outputs, - .state = .{ .s = final_state.s }, - }; - } -}; - -test "gated delta net" { - const platform = zml.testing.env(); - - const queries: zml.Tensor = .init(.{ .s = 2, .h = 2, .k = 2 }, .f32); - const keys: zml.Tensor = .init(.{ .s = 2, .h = 2, .k = 2 }, .f32); - const values: zml.Tensor = .init(.{ .s = 2, .h = 2, .v = 2 }, .f32); - const alphas: zml.Tensor = .init(.{ .s = 2, .h = 2 }, .f32); - const betas: zml.Tensor = .init(.{ .s = 2, .h = 2 }, .f32); - const initial_s: zml.Tensor = .init(.{ .h = 2, .v = 2, .k = 2 }, .f32); - const active_length: zml.Tensor = .init(.{}, .u32); - - var exe = try platform.compileFn( - std.testing.allocator, - std.testing.io, - GatedDeltaNet.forward, - .{ queries, keys, values, alphas, betas, .{ .s = initial_s }, active_length }, - .{}, - ); - defer exe.deinit(); - - var queries_buffer: zml.Buffer = try .fromBytes( - std.testing.io, - platform, - queries.shape(), - std.mem.sliceAsBytes(&[2][2][2]f32{ - .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, - .{ .{ 1.0, 1.0 }, .{ 1.0, -1.0 } }, - }), - ); - defer queries_buffer.deinit(); - var keys_buffer: zml.Buffer = try .fromBytes( - std.testing.io, - platform, - keys.shape(), - std.mem.sliceAsBytes(&[2][2][2]f32{ - .{ .{ 1.0, 2.0 }, .{ 0.0, 1.0 } }, - .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, - }), - ); - defer keys_buffer.deinit(); - var values_buffer: zml.Buffer = try .fromBytes( - std.testing.io, - platform, - values.shape(), - std.mem.sliceAsBytes(&[2][2][2]f32{ - .{ .{ 3.0, 1.0 }, .{ 2.0, 4.0 } }, - .{ .{ 1.0, 5.0 }, .{ 3.0, 0.0 } }, - }), - ); - defer values_buffer.deinit(); - var alphas_buffer: zml.Buffer = try .fromBytes( - std.testing.io, - platform, - alphas.shape(), - std.mem.sliceAsBytes(&[2][2]f32{ - .{ 0.5, 0.25 }, - .{ 0.8, 0.6 }, - }), - ); - defer alphas_buffer.deinit(); - var betas_buffer: zml.Buffer = try .fromBytes( - std.testing.io, - platform, - betas.shape(), - std.mem.sliceAsBytes(&[2][2]f32{ - .{ 1.0, 0.5 }, - .{ 0.75, 1.0 }, - }), - ); - defer betas_buffer.deinit(); - var initial_s_buffer: zml.Buffer = try .fromBytes( - std.testing.io, - platform, - initial_s.shape(), - std.mem.sliceAsBytes(&[2][2][2]f32{ - .{ .{ 1.0, 0.0 }, .{ 0.0, 1.0 } }, - .{ .{ 2.0, 1.0 }, .{ 1.0, 0.0 } }, - }), - ); - defer initial_s_buffer.deinit(); - var oversized_length_buffer = try zml.Buffer.scalar(std.testing.io, platform, @as(u32, 3), .u32); - defer oversized_length_buffer.deinit(); - - var result: zml.Bufferized(GatedDeltaNet.Output) = undefined; - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run( - std.testing.io, - .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer }, oversized_length_buffer }, - .{&result}, - .{ .wait = true }, - ); - defer result.outputs.deinit(); - defer result.state.s.deinit(); - - const expected_outputs: zml.Slice = .init( - zml.Shape.init(.{ 2, 2, 2 }, .f32), - std.mem.sliceAsBytes(&[2][2][2]f32{ - .{ .{ 3.0, 0.0 }, .{ 1.125, 2.0 } }, - .{ .{ -11.15, 10.75 }, .{ 2.325, -1.2 } }, - }), - ); - const expected_final_s: zml.Slice = .init( - zml.Shape.init(.{ 2, 2, 2 }, .f32), - std.mem.sliceAsBytes(&[2][2][2]f32{ - .{ .{ -9.3, -1.85 }, .{ 6.9, 3.85 } }, - .{ .{ 3.0, 0.675 }, .{ 0.0, 1.2 } }, - }), - ); - - try zml.testing.expectClose(std.testing.io, expected_outputs, result.outputs, .{ - .absolute_tolerance = 1e-4, - .relative_tolerance = 1e-4, - }); - try zml.testing.expectClose(std.testing.io, expected_final_s, result.state.s, .{ - .absolute_tolerance = 1e-4, - .relative_tolerance = 1e-4, - }); - - var prefix_length_buffer = try zml.Buffer.scalar(std.testing.io, platform, @as(u32, 1), .u32); - defer prefix_length_buffer.deinit(); - var prefix_result: zml.Bufferized(GatedDeltaNet.Output) = undefined; - runner.run( - std.testing.io, - .{ queries_buffer, keys_buffer, values_buffer, alphas_buffer, betas_buffer, .{ .s = initial_s_buffer }, prefix_length_buffer }, - .{&prefix_result}, - .{ .wait = true }, - ); - defer prefix_result.outputs.deinit(); - defer prefix_result.state.s.deinit(); - - const expected_prefix_outputs: zml.Slice = .init( - zml.Shape.init(.{ 2, 2, 2 }, .f32), - std.mem.sliceAsBytes(&[2][2][2]f32{ - .{ .{ 3.0, 0.0 }, .{ 1.125, 2.0 } }, - .{ .{ 0.0, 0.0 }, .{ 0.0, 0.0 } }, - }), - ); - const expected_prefix_final_s: zml.Slice = .init( - zml.Shape.init(.{ 2, 2, 2 }, .f32), - std.mem.sliceAsBytes(&[2][2][2]f32{ - .{ .{ 3.0, 5.0 }, .{ 0.0, 0.5 } }, - .{ .{ 0.5, 1.125 }, .{ 0.25, 2.0 } }, - }), - ); - try zml.testing.expectClose(std.testing.io, expected_prefix_outputs, prefix_result.outputs, .{ - .absolute_tolerance = 1e-4, - .relative_tolerance = 1e-4, - }); - try zml.testing.expectClose(std.testing.io, expected_prefix_final_s, prefix_result.state.s, .{ - .absolute_tolerance = 1e-4, - .relative_tolerance = 1e-4, - }); -} - /// Options controlling generation. The default values correspond to greedy decoding. pub const SamplingStrategy = struct { topk: u32 = 1, @@ -2016,10 +1140,10 @@ test sampleTokens { const rng: zml.Tensor.Rng = .init(); const activations: zml.Tensor = .init(.{ .voc = 4 }, .f32); - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, sampleTokens, .{ activations, .{ .topk = 4, .temperature = 2.0 }, rng }, platform); defer exe.deinit(); - var rng_buffer = try rng.initBuffer(std.testing.io, platform, 0xdeadbeef); + var rng_buffer = try zml.Tensor.Rng.initBuffer(platform, 0xdeadbeef, std.testing.io); defer rng_buffer._state.deinit(); const inf = std.math.inf(f32); @@ -2034,10 +1158,7 @@ test sampleTokens { var activations_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, activations.shape(), std.mem.sliceAsBytes(&logits)); defer activations_buffer.deinit(); - var sampled: zml.Buffer = undefined; - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{ activations_buffer, rng_buffer }, .{ &sampled, &rng_buffer }, .{ .wait = true }); + var sampled, rng_buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, sampleTokens, .{ activations_buffer, rng_buffer }); defer sampled.deinit(); try zml.testing.expectEqual(expected, try sampled.getValue(i32, std.testing.io)); @@ -2126,19 +1247,17 @@ fn fixupLogits(logits: Tensor, opts: DynamicSamplingStrategy) [2]Tensor { // After the topk, we don't have .voc indices, anymore, only topk. var x = full_topk.values.rename(.{ .voc = .topk }); // mask values above the dynamic top_k - const top_k = opts.top_k.broad(x.shape()); - x = Tensor.iota(x.shape(), .topk).cmp(.GE, top_k).select(min_inf, x); - const temperature = opts.temperature.convert(x.dtype()).broad(x.shape()); - x = x.div(temperature); + x = Tensor.iota(x.shape(), .topk).cmp(.GE, opts.top_k).select(min_inf, x); + x = x.mul(opts.temperature); // if there are high values in x, softmax can overflow and will create nans in full probs // this propagate to probs_sum and probs_max. const probs = x.softmax(.topk); const probs_sum = probs.cumulativeSum(.topk); - const probs_max = probs.slice(.topk, .{ .start = 0, .end = 1 }); + const probs_max = probs.slice1d(.topk, .{ .start = 0, .end = 1 }); - const top_p = opts.top_p.convert(x.dtype()).broad(x.shape()); - const min_p = opts.min_p.convert(x.dtype()).broad(probs_max.shape()).mul(probs_max).broad(x.shape()); + const top_p = opts.top_p.broad(x.shape()); + const min_p = probs_max.mul(opts.min_p).broad(x.shape()); // * if first candidate has very high prob, then probs_sum is always greater than top_p and candidate is full false // * if first candidate score is even bigger, the probs become Nan because of the softmax, @@ -2161,7 +1280,7 @@ test sampleTokensDynamic { const logits: zml.Tensor = .init(.{ .voc = logits_data.len }, .f32); const dynamic_sampling_strategy = DynamicSamplingStrategy.init(.f32, 0); - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits, dynamic_sampling_strategy }, platform); defer exe.deinit(); var logits_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, logits.shape(), std.mem.sliceAsBytes(&logits_data)); @@ -2172,7 +1291,7 @@ test sampleTokensDynamic { // top_k == logits.len -> just sort the input .{ .{ .top_k = 4 }, [_]f32{ @log(4.0), @log(3.0), @log(2.0), @log(1.0) } }, .{ .{ .top_k = 2 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, - .{ .{ .top_k = 2, .temperature = 0.1 }, [_]f32{ @log(4.0) / 0.1, @log(3.0) / 0.1, ___, ___ } }, + .{ .{ .top_k = 2, .temperature = 0.1 }, [_]f32{ @log(4.0) * 0.1, @log(3.0) * 0.1, ___, ___ } }, // top_k == logits.len and small top_p -> make sure at least one is returned .{ .{ .top_k = 4, .top_p = 0.1 }, [_]f32{ @log(4.0), ___, ___, ___ } }, .{ .{ .top_k = 4, .top_p = 0.701 }, [_]f32{ @log(4.0), @log(3.0), ___, ___ } }, @@ -2185,11 +1304,7 @@ test sampleTokensDynamic { const args, const expected = args_expected; var dynamic_sampling_strategy_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .f32, args); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_buffers); - var new_logits: zml.Buffer = undefined; - var indices: zml.Buffer = undefined; - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{ logits_buffer, dynamic_sampling_strategy_buffers }, .{ &new_logits, &indices }, .{ .wait = true }); + var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, fixupLogits, .{ logits_buffer, dynamic_sampling_strategy_buffers }); defer new_logits.deinit(); defer indices.deinit(); try std.testing.expectEqual(top_k_indices, try indices.getValue(@TypeOf(top_k_indices), std.testing.io)); @@ -2202,7 +1317,7 @@ test sampleTokensDynamic { const logits_bf16: zml.Tensor = .init(.{ .voc = logits_data.len }, .bf16); const dynamic_sampling_strategy_bf16 = DynamicSamplingStrategy.init(.bf16, 0); - var exe_bf16 = try platform.compileFn(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, .{}); + var exe_bf16 = try zml.module.compile(std.testing.allocator, std.testing.io, fixupLogits, .{ logits_bf16, dynamic_sampling_strategy_bf16 }, platform); defer exe_bf16.deinit(); const boost = bf16.inf; @@ -2213,13 +1328,7 @@ test sampleTokensDynamic { var dynamic_sampling_strategy_bf16_buffers = try DynamicSamplingStrategy.makeBuffers(std.testing.io, platform, .bf16, .{ .top_k = 4, .top_p = 0.9, .min_p = 0.1 }); defer DynamicSamplingStrategy.deinitBuffers(&dynamic_sampling_strategy_bf16_buffers); - var new_logits: zml.Buffer = undefined; - var indices: zml.Buffer = undefined; - var runner = try exe_bf16.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }, .{ &new_logits, &indices }, .{ .wait = true }); - defer new_logits.deinit(); - defer indices.deinit(); + var new_logits, var indices = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe_bf16, fixupLogits, .{ logits_bf16_buffer, dynamic_sampling_strategy_bf16_buffers }); try std.testing.expectEqual([_]i32{ 0, 1, 2, 3 }, try indices.getValue([4]i32, std.testing.io)); try zml.testing.expectEqual([_]bf16{ boost, nerf, nerf, nerf }, try new_logits.getValue([4]bf16, std.testing.io)); @@ -2281,13 +1390,8 @@ pub fn collectDims( return context.res; } -fn shapeToDims(shape: anytype) [std.meta.fieldNames(@TypeOf(shape)).len]i64 { - const field_names = comptime std.meta.fieldNames(@TypeOf(shape)); - var result: [field_names.len]i64 = undefined; - inline for (field_names, 0..) |field_name, i| { - result[i] = @field(shape, field_name); - } - return result; +fn shapeToDims(shape: anytype) [@divExact(@sizeOf(@TypeOf(shape)), @sizeOf(i64))]i64 { + return @bitCast(shape); } test collectDims { @@ -2342,7 +1446,7 @@ fn ShapeStruct(comptime dims: anytype) type { @setEvalBranchQuota(rank + 5); var struct_field_names: [rank][]const u8 = undefined; var struct_field_types: [rank]type = undefined; - var struct_field_attrs: [rank]std.builtin.Type.Struct.FieldAttributes = undefined; + var struct_field_attrs: [rank]std.builtin.Type.StructField.Attributes = undefined; const default: i64 = NOT_SET; for (&struct_field_names, &struct_field_types, &struct_field_attrs, dims) |*field_name, *field_type, *field_attr, axis| { field_name.* = @tagName(axis); @@ -2355,53 +1459,3 @@ fn ShapeStruct(comptime dims: anytype) type { return @Struct(.@"extern", null, &struct_field_names, &struct_field_types, &struct_field_attrs); } - -/// Smooth approximation of ReLU function: log(1 + exp(x)) -/// For numerical stability, falls back to the linear function when `x > opts.threshold`. -pub fn softplus( - x: Tensor, - opts: struct { threshold: f32 = 20.0 }, -) zml.Tensor { - // Compute log1p(z) as z * log(u) / (u - 1), where u = 1 + z. - // Using log(1 + exp(x)) directly loses precision for x < ~-16.6, causing strongly negative logits to produce zero scores. - const z = x.exp(); - const u = z.addConstant(1); - const log1p = u.cmp(.EQ, .scalar(1, x.dtype())).select(z, z.mul(u.log()).div(u.subConstant(1))); - return x.cmp(.GT, .scalar(opts.threshold, x.dtype())).select(x, log1p); -} - -test "softplus" { - const inputs = [_]f32{ -20.0, -1.0, 0.0, 1.0, 2.0, 2.1, 3.0 }; - const expected = [_]f32{ 2.0611537e-9, 0.3132617, 0.6931472, 1.3132616, 2.126928, 2.1, 3.0 }; - - const platform = zml.testing.env(); - const input: Tensor = .init(.{ .x = inputs.len }, .f32); - const Local = struct { - fn call(x: Tensor, dtype: DataType) Tensor { - return softplus(x.convert(dtype), .{ .threshold = 2.0 }).convert(x.dtype()); - } - }; - - var input_buffer: zml.Buffer = try .fromBytes( - std.testing.io, - platform, - input.shape(), - std.mem.sliceAsBytes(&inputs), - ); - defer input_buffer.deinit(); - - for ([_]DataType{ .f32, .bf16 }) |dtype| { - var exe = try platform.compileFn(std.testing.allocator, std.testing.io, Local.call, .{ input, dtype }, .{}); - defer exe.deinit(); - - var result = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); - defer result.deinit(); - - try zml.testing.expectClose( - std.testing.io, - Slice.init(input.shape(), std.mem.sliceAsBytes(&expected)), - result, - .{}, - ); - } -} diff --git a/zml/ops.zig b/zml/ops.zig index f8fdba7e..81b5022a 100644 --- a/zml/ops.zig +++ b/zml/ops.zig @@ -1,326 +1,29 @@ const std = @import("std"); -const builtin = @import("builtin"); -const func = @import("mlir/dialects/func"); +const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); -const pjrt = @import("pjrt"); -const platforms = @import("platforms"); -const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const Buffer = @import("buffer.zig").Buffer; -const Bufferized = @import("mem.zig").Bufferized; -const Compiler = @import("Compiler.zig"); +const CompilationContext = @import("module.zig").CompilationContext; const constants = @import("constants.zig"); -const CustomCallBuffer = @import("pjrtx.zig").CustomCallBuffer; -const DataType = @import("dtype.zig").DataType; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); -const manual_computation = @import("ops/manual_computation.zig"); -pub const manualComputation = manual_computation.manualComputation; -pub const ManualPartitioning = manual_computation.ManualPartitioning; -pub const PartitionSpecs = manual_computation.PartitionSpecs; -pub const partitionSpecs = manual_computation.partitionSpecs; -pub const partitionSpecsAlloc = manual_computation.partitionSpecsAlloc; -const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const ShapeToCustomCallBuffer = @import("pjrtx.zig").ShapeToCustomCallBuffer; -const Sharding = @import("Sharding.zig"); const Tensor = @import("tensor.zig").Tensor; -const TensorToCustomCallBuffer = @import("pjrtx.zig").TensorToCustomCallBuffer; - -test { - _ = manual_computation; -} - -const log = std.log.scoped(.@"zml/Compiler"); - -/// Reduces local tensor values across all processes in the computation. -/// For each tensor index, `func_` combines the values at that index from every -/// process. The result is identical on all processes. -/// -/// Inside a manual computation, `inputs` describes the local tensors at each -/// partition. For example, summing `[1, 2]` and `[3, 4]` from two partitions gives -/// `[4, 6]` at each partition. The local tensor dimensions stay unchanged. -/// -/// * `inputs` is a `Tensor`, a nonempty tuple of tensors, or an `[N]Tensor`. -/// Each input must have matching shapes and dtypes across processes. -/// * `func_` describes how to combine two scalar contributions: -/// - For one input, it must be `fn (left: Tensor, right: Tensor) Tensor`, such -/// as `Tensor.add`. This also applies to a one-element tuple or array. -/// - For multiple inputs, it takes one `ReduceArgs` per input and returns a -/// tuple or array with one scalar tensor per input, in the same order. -/// -/// Returns the same container type as `inputs`, preserving tensor attributes. -/// With one process, returns `inputs` unchanged. -/// -/// The reduction uses one group containing all processes, including mesh replicas. -/// To reduce along one logical mesh axis, use `allReduceOnAxis`. -pub fn allReduce(inputs: anytype, comptime func_: anytype) AllReduceReturnType(@TypeOf(inputs)) { - const count = Compiler.current().meshes[0].numDevices(); - var devices: [Platform.MAX_NUM_DEVICES]i64 = undefined; - for (devices[0..count], 0..) |*device, i| device.* = @intCast(i); - return allReduceGroups(inputs, func_, devices[0..count], count); -} - -/// Reduces local tensor values within process groups defined by `axis` in `mesh`. -/// A group contains processes with the same coordinates on every other logical -/// mesh axis. Separate replicas of the mesh belong to separate groups. Within -/// each group, `func_` combines corresponding elements and the result is identical -/// on all member processes. -/// -/// Use this inside a manual computation to combine partial results along `axis` -/// while preserving the independent computations along the other mesh axes. -/// -/// * `inputs` and `func_` follow `allReduce`'s tensor and reducer contracts. -/// * `mesh` is the logical mesh used by the manual computation and must be -/// included in the compilation's meshes. -/// * `axis` names an axis of that mesh, for example `.experts`. It can be an enum -/// literal or a mesh axis tag. The axis must exist in `mesh`. -/// -/// Returns the same container type with tensor attributes as -/// `inputs`. An axis of size one returns `inputs` unchanged. -/// -/// For a mesh with `.data = 2` and `.experts = 4`, the following sums four local -/// expert contributions independently for each of the two data shards: -/// -/// ```zig -/// const output = allReduceOnAxis(local_output, Tensor.add, mesh, .experts); -/// ``` -/// -/// Each expert shard now has the complete output for its data shard. To expose -/// this result through `manualComputation`, an output spec such as -/// `.parse(.{ .t = .data })` keeps tokens split across `.data` and declares the -/// result replicated along `.experts`. -pub fn allReduceOnAxis(inputs: anytype, comptime func_: anytype, mesh: *const Sharding.Mesh, axis: anytype) AllReduceReturnType(@TypeOf(inputs)) { - const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical reduction axis"); - const size = mesh.grid.axis_sizes[logical_axis]; - var groups: [Platform.MAX_NUM_DEVICES]i64 = undefined; - var replica_indices: [Platform.MAX_NUM_DEVICES]usize = @splat(0); - const replicas = mesh.numReplicas(); - for (0..mesh.numDevices()) |device| { - const coords = mesh.grid.logical_coords[device]; - var tile: usize = 0; - var group: usize = 0; - for (0..mesh.logical.axes.len) |dim| { - tile = tile * mesh.grid.axis_sizes[dim] + coords[dim]; - if (dim != logical_axis) group = group * mesh.grid.axis_sizes[dim] + coords[dim]; - } - group = group * replicas + replica_indices[tile]; - replica_indices[tile] += 1; - groups[group * size + coords[logical_axis]] = @intCast(device); - } - return allReduceGroups(inputs, func_, groups[0..mesh.numDevices()], size); -} - -/// Reduces corresponding tensor elements independently within each process group. -/// `inputs`, `func_` and the return value follow `allReduce`'s contract. -/// -/// `groups` identifies processes by their executable partition IDs, as returned -/// by `partitionId()`. Each consecutive `group_size` IDs defines one group. -/// -/// For example, `groups = &.{ 0, 2, 1, 3 }` and `group_size = 2` combine partitions -/// 0 and 2 together, and partitions 1 and 3 together. Both members of each group -/// receive that group's result. Groups of size one return `inputs` unchanged. -fn allReduceGroups(inputs: anytype, comptime func_: anytype, groups: []const i64, group_size: usize) AllReduceReturnType(@TypeOf(inputs)) { - const ctx = Compiler.current(); - const mlir_ctx = ctx.mlir_ctx; - - const InputsT = @TypeOf(inputs); - const n_inputs = switch (@typeInfo(InputsT)) { - .@"struct" => |struct_info| b: { - if (InputsT == Tensor) break :b 1; - if (!struct_info.is_tuple) { - @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"); - } - break :b struct_info.field_names.len; - }, - .array => |array_info| b: { - if (array_info.child != Tensor) { - @compileError("zml.ops.allReduce expects [N]Tensor inputs"); - } - break :b array_info.len; - }, - else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), - }; - comptime stdx.debug.assertComptime(n_inputs > 0, "zml.ops.allReduce requires at least one input tensor", .{}); - - const input_tensors: [n_inputs]Tensor = switch (@typeInfo(InputsT)) { - .@"struct" => |struct_info| b: { - if (InputsT == Tensor) { - break :b .{inputs}; - } - if (!struct_info.is_tuple) { - @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"); - } - - var flat_inputs: [n_inputs]Tensor = undefined; - meta.collectBuf((struct { - pub fn cb(t: Tensor) Tensor { - return t; - } - }).cb, {}, &inputs, &flat_inputs); - break :b flat_inputs; - }, - .array => inputs, - else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), - }; - - if (group_size <= 1) return inputs; - - const reducer_block = b: { - const ArgsTypes: [input_tensors.len]type = @splat(ReduceArgs); - var args: @Tuple(&ArgsTypes) = undefined; - var block_types: [2 * input_tensors.len]*const mlir.Type = undefined; - - inline for (0..input_tensors.len) |i| { - const scalar_shape = Shape.init(.{}, input_tensors[i].dtype()); - args[i].left = .fromShape(scalar_shape); - args[i].right = .fromShape(scalar_shape); - block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, scalar_shape); - block_types[i + input_tensors.len] = mlirx.Type.rankedTensor(mlir_ctx, scalar_shape); - } - - const block_locs: [2 * input_tensors.len]*const mlir.Location = @splat(ctx.unknown_location); - - const block = mlir.Block.init(&block_types, &block_locs); - errdefer block.deinit(); - - const reducer_scope = ctx.pushBlock(block); - defer reducer_scope.pop(); - - ctx.pushLocation(@src(), stdx.meta.fnName(func_)); - defer ctx.popLocation(); - - inline for (0..input_tensors.len) |i| { - reducer_scope.registerTensorAsBlockArgument(args[i].left.id, i); - reducer_scope.registerTensorAsBlockArgument(args[i].right.id, i + input_tensors.len); - } - - var reduced_values: [input_tensors.len]*const mlir.Value = undefined; - if (input_tensors.len == 1) { - const reduced = @call(.auto, func_, .{ args[0].left, args[0].right }); - reduced_values[0] = reduced.value(); - } else { - var reduced = @call(.auto, func_, args); - var reduced_tensors: [input_tensors.len]Tensor = undefined; - meta.collectBuf(stdx.meta.identity, {}, &reduced, &reduced_tensors); - - for (reduced_values, reduced_tensors) |*vi, ti| { - vi.* = ti.value(); - } - } - - _ = stablehlo.returns(mlir_ctx, &reduced_values, ctx.location).appendTo(block); - break :b block; - }; - - const replica_groups_attr = mlir.Attribute.denseElements( - .rankedTensor(&.{ @intCast(groups.len / group_size), @intCast(group_size) }, .int(mlir_ctx, .i64)), - groups, - ); - - const handle = ctx.nextChannelId(); - const channel_handle_attr = stablehlo.channelHandleAttr( - mlir_ctx, - handle, - .collective, - ); - - var input_values: [input_tensors.len]*const mlir.Value = undefined; - var result_types: [input_tensors.len]*const mlir.Type = undefined; - inline for (0..input_tensors.len) |i| { - input_values[i] = input_tensors[i].value(); - result_types[i] = input_values[i].type_(); - } - - const op = stablehlo.allReduce( - mlir_ctx, - &input_values, - &result_types, - reducer_block, - replica_groups_attr, - channel_handle_attr, - ctx.location, - ).appendTo(ctx.currentScope().block); - - return switch (@typeInfo(@TypeOf(inputs))) { - .@"struct" => |struct_info| b: { - if (@TypeOf(inputs) == Tensor) { - break :b Tensor._result(input_tensors[0].shape(), op.result(0)); - } - - var out: @TypeOf(inputs) = undefined; - inline for (struct_info.field_names, 0..) |field_name, i| { - @field(out, field_name) = Tensor._result(input_tensors[i].shape(), op.result(i)); - } - break :b out; - }, - .array => |array_info| b: { - var out: [array_info.len]Tensor = undefined; - inline for (0..array_info.len) |i| { - out[i] = Tensor._result(input_tensors[i].shape(), op.result(i)); - } - break :b out; - }, - else => unreachable, - }; -} - -fn AllReduceReturnType(comptime InputsT: type) type { - if (InputsT == Tensor) return Tensor; - - return switch (@typeInfo(InputsT)) { - .@"struct" => |struct_info| b: { - if (!struct_info.is_tuple) { - @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor or [N]Tensor inputs"); - } - break :b InputsT; - }, - .array => |array_info| b: { - if (array_info.child != Tensor) { - @compileError("zml.ops.allReduce expects [N]Tensor inputs"); - } - break :b InputsT; - }, - else => @compileError("zml.ops.allReduce expects Tensor, tuple of Tensor, or [N]Tensor inputs"), - }; -} - -/// Returns the current process's zero based coordinate on `axis` in `mesh` -/// as a scalar u32. -/// -/// Example: for a mesh {.data = 2, .experts = 2}, each data group has expert coordinates -/// 0 and 1. Use this coordinate inside a manual computation to identify a shard -/// along one axis while `partitionId()` identifies a partition across the whole mesh. -pub fn axisIndex(mesh: *const Sharding.Mesh, axis: anytype) Tensor { - const logical_axis = mesh.resolveLogicalAxis(Shape.toTag(axis)) orelse @panic("Unknown logical mesh axis"); - var coords: [Platform.MAX_NUM_DEVICES]u32 = undefined; - for (coords[0..mesh.numDevices()], 0..) |*coord, i| coord.* = mesh.grid.logical_coords[i][logical_axis]; - return Tensor.constantTensor(.init(.{ .device = mesh.numDevices() }, .u32), std.mem.sliceAsBytes(coords[0..mesh.numDevices()])) - .gather(.{ .device = partitionId() }, .{}); -} - -pub fn partitionId() Tensor { - const ctx = Compiler.current(); - const op = stablehlo.partition_id(ctx.mlir_ctx, ctx.location).appendTo(ctx.currentScope().block); - return Tensor._result(.init(.{}, .u32), op.result(0)); -} pub const ReduceArgs = struct { left: Tensor, right: Tensor, }; -pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func_: anytype, context: anytype) stdx.meta.FnReturn(func_) { - const compiler = Compiler.current(); - const caller_scope = compiler.currentScope(); - const mlir_ctx = compiler.mlir_ctx; +pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; - // Note: inputs and inits are infered to be tuple of ReduceArgs const reduce_block, var result = b: { - const ArgsTypes: [inits.len]type = @splat(ReduceArgs); - const ArgsType = @Tuple(&ArgsTypes); + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); var args: ArgsType = undefined; var block_types: [2 * inits.len]*const mlir.Type = undefined; @@ -328,50 +31,58 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func args[i].left = .fromShape(inits[i].shape()); args[i].right = .fromShape(inits[i].shape()); - block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); - block_types[i + inits.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); } - const block_locs: [2 * inits.len]*const mlir.Location = @splat(compiler.unknown_location); + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); const reduce_block = mlir.Block.init(&block_types, &block_locs); errdefer reduce_block.deinit(); - const reduce_scope = compiler.pushBlock(reduce_block); - defer reduce_scope.pop(); - - compiler.pushLocation(@src(), stdx.meta.fnName(func_)); - defer compiler.popLocation(); + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); + const scope = CompilationContext.current().currentScope(); inline for (0..inits.len) |i| { - reduce_scope.registerTensorAsBlockArgument(args[i].left.id, i); - reduce_scope.registerTensorAsBlockArgument(args[i].right.id, i + inits.len); + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; } - var result = @call(.auto, func_, args ++ context); + var result = @call(.auto, func, args ++ context); var result_values: [inits.len]*const mlir.Value = undefined; inline for (0..inits.len) |i| { result_values[i] = result[i].value(); } - const block_result = stablehlo.returns(mlir_ctx, &result_values, compiler.location); - _ = block_result.appendTo(reduce_block); + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); break :b .{ reduce_block, result }; }; var input_values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| input_values[i] = inputs[i].value(); + inline for (0..inputs.len) |i| { + input_values[i] = inputs[i].value(); + } var init_values: [inputs.len]*const mlir.Value = undefined; inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - const reduce_op = stablehlo.reduce(mlir_ctx, &input_values, &init_values, reduce_block, axes_, compiler.location).appendTo(caller_scope.block); + const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlir_ctx, "dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, axes_)), + }, + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); // `stablehlo.reduce` drops axes. We want to avoid that to propagate tags. // So we need to broadcast the output of `stablehlo.reduce` to the input shapes. // To that order, we initialize `result` to `inputs`, then we use stdx.meta.visit, // to find the correct mlir.Value, but we first broadcast before creating the final // Tensor struct. - var broadcasting_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var broadcasting_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; for (0..constants.MAX_RANK) |i| { if (std.mem.indexOfScalar(i64, axes_, @intCast(i)) == null) { broadcasting_axes.append(@intCast(i)) catch unreachable; @@ -384,13 +95,14 @@ pub fn reduce(inputs: anytype, inits: anytype, axes_: []const i64, comptime func reduced_shape = reduced_shape.setDim(a, 1); } - const broad_op = stablehlo.broadcast_in_dim( + const tensor_type = mlir.rankedTensorType(reduced_shape.dims(), mlirx.Type.fromDType(mlir_ctx, reduced_shape.dtype())); + const broad_op = dialects.stablehlo.broadcast_in_dim( mlir_ctx, reduce_op.result(i), broadcasting_axes.slice()[0 .. reduced_shape.rank() - axes_.len], - mlirx.Type.rankedTensor(mlir_ctx, reduced_shape), - compiler.location, - ).appendTo(caller_scope.block); + tensor_type, + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); result[i] = Tensor._result(reduced_shape, broad_op.result(0)); } @@ -407,52 +119,46 @@ pub const ReduceWindowOpts = struct { padding: []const [2]i64, }; -pub fn ReduceWindowFn(N: comptime_int) type { - return @Fn( - &@as([N]type, @splat(ReduceArgs)), - &@as([N]std.builtin.Type.Fn.ParamAttributes, @splat(.{})), - [N]Tensor, - .{}, - ); -} +pub fn reduceWindow(inputs: anytype, inits: anytype, opts: ReduceWindowOpts, comptime func: anytype, context: anytype) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); -pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: ReduceWindowOpts, func_: *const ReduceWindowFn(N)) [N]Tensor { - const compiler = Compiler.current(); - const mlir_ctx = compiler.mlir_ctx; + const mlir_ctx = CompilationContext.current().mlir_ctx; const reduce_block, var result = b: { - const Args = @Tuple(&@as([N]type, @splat(ReduceArgs))); - var args: Args = undefined; - var block_types: [2 * N]*const mlir.Type = undefined; + const ArgsType = std.meta.Tuple(&[1]type{ReduceArgs} ** inits.len); + var args: ArgsType = undefined; + var block_types: [2 * inits.len]*const mlir.Type = undefined; - inline for (0..N) |i| { + inline for (0..inits.len) |i| { args[i].left = .fromShape(inits[i].shape()); args[i].right = .fromShape(inits[i].shape()); - block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); - block_types[i + N] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); + block_types[i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[i + inits.len] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); } - const block_locs: [2 * N]*const mlir.Location = @splat(compiler.unknown_location); + const block_locs: [2 * inits.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); const reduce_block = mlir.Block.init(&block_types, &block_locs); errdefer reduce_block.deinit(); - const reduce_scope = compiler.pushBlock(reduce_block); - defer reduce_scope.pop(); + CompilationContext.current().pushBlock(reduce_block); + defer CompilationContext.current().popBlock(); - inline for (0..N) |i| { - reduce_scope.registerTensorAsBlockArgument(args[i].left.id, i); - reduce_scope.registerTensorAsBlockArgument(args[i].right.id, i + N); + const scope = CompilationContext.current().currentScope(); + inline for (0..inits.len) |i| { + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, i + inits.len) catch unreachable; } - var result = @call(.auto, func_, args); + var result = @call(.auto, func, args ++ context); - var result_values: [N]*const mlir.Value = undefined; - inline for (0..N) |i| { + var result_values: [inits.len]*const mlir.Value = undefined; + inline for (0..inits.len) |i| { result_values[i] = result[i].value(); } - _ = stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(reduce_block); + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(reduce_block); break :b .{ reduce_block, result }; }; var input_values: [inputs.len]*const mlir.Value = undefined; @@ -463,12 +169,21 @@ pub fn reduceWindow(N: comptime_int, inputs: [N]Tensor, inits: [N]Tensor, opts: var init_values: [inputs.len]*const mlir.Value = undefined; inline for (0..inits.len) |i| init_values[i] = inits[i].value(); - const reduce_op = stablehlo.reduce_window(mlir_ctx, &input_values, &init_values, reduce_block, opts.window_dimensions, .{ - .window_strides = opts.window_strides, - .base_dilations = opts.base_dilations, - .window_dilations = opts.window_dilations, - .padding = opts.padding, - }, compiler.location).appendTo(compiler.currentScope().block); + const reduce_op = mlir.Operation.make(mlir_ctx, "stablehlo.reduce_window", .{ + .operands = .{ .variadic = &.{ &input_values, &init_values } }, + .result_type_inference = true, + .blocks = &.{reduce_block}, + .attributes = &.{ + .named(mlir_ctx, "window_dimensions", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dimensions)), + .named(mlir_ctx, "window_strides", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_strides)), + .named(mlir_ctx, "base_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.base_dilations)), + .named(mlir_ctx, "window_dilations", mlir.denseArrayAttribute(mlir_ctx, .i64, opts.window_dilations)), + // Cast the [][2]i64 to []i64 (safe) + .named(mlir_ctx, "padding", mlir.denseElementsAttribute(mlir.RankedTensorType.get(&.{ @intCast(opts.padding.len), 2 }, mlir.integerType(mlir_ctx, .i64), null).shaped(), @as([]const i64, @ptrCast(opts.padding)))), + }, + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); inline for (0..result.len) |i| { result[i] = Tensor.fromMlirValue(reduce_op.result(i)).withTags(inputs[i].shape()); @@ -482,13 +197,14 @@ pub const SortArgs = struct { right: Tensor, }; -pub fn sort(inputs: anytype, axis_: i64, comptime func_: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { - const compiler = Compiler.current(); - const mlir_ctx = compiler.mlir_ctx; +pub fn sort(inputs: anytype, axis_: i64, comptime func: anytype, context: anytype, is_stable: bool) [inputs.len]Tensor { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const mlir_ctx = CompilationContext.current().mlir_ctx; const sort_block = b: { - const ArgsTypes: [inputs.len]type = @splat(SortArgs); - const ArgsType = @Tuple(&ArgsTypes); + const ArgsType = std.meta.Tuple(&[1]type{SortArgs} ** inputs.len); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -496,25 +212,26 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func_: anytype, context: anyty args[i].left = Tensor.init(.{}, inputs[i].shape().dtype()); args[i].right = Tensor.init(.{}, inputs[i].shape().dtype()); - block_types[2 * i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].left.shape()); - block_types[2 * i + 1] = mlirx.Type.rankedTensor(mlir_ctx, args[i].right.shape()); + block_types[2 * i] = mlir.rankedTensorType(args[i].left.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].left.dtype())); + block_types[2 * i + 1] = mlir.rankedTensorType(args[i].right.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].right.dtype())); } - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(compiler.unknown_location); + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); const sort_block = mlir.Block.init(&block_types, &block_locs); errdefer sort_block.deinit(); - const scope = compiler.pushBlock(sort_block); - defer scope.pop(); + CompilationContext.current().pushBlock(sort_block); + defer CompilationContext.current().popBlock(); + const scope = CompilationContext.current().currentScope(); inline for (0..inputs.len) |i| { - scope.registerTensorAsBlockArgument(args[i].left.id, 2 * i); - scope.registerTensorAsBlockArgument(args[i].right.id, 2 * i + 1); + scope.id_to_argument.put(scope.arena.allocator(), args[i].left.id, 2 * i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].right.id, 2 * i + 1) catch unreachable; } - var result = @call(.auto, func_, args ++ context); + var result = @call(.auto, func, args ++ context); - _ = stablehlo.return_(mlir_ctx, result.value(), compiler.location).appendTo(sort_block); + _ = dialects.stablehlo.return_(mlir_ctx, result.value(), .unknown(mlir_ctx)).appendTo(sort_block); break :b sort_block; }; @@ -523,10 +240,17 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func_: anytype, context: anyty input_values[i] = inputs[i].value(); } - const sort_op = stablehlo.sort(mlir_ctx, &input_values, sort_block, .{ - .dimension = axis_, - .is_stable = is_stable, - }, compiler.location).appendTo(compiler.currentScope().block); + const sort_op = mlir.Operation.make(mlir_ctx, "stablehlo.sort", .{ + .operands = .{ .variadic = &.{&input_values} }, + .result_type_inference = true, + .blocks = &.{sort_block}, + .attributes = &.{ + .named(mlir_ctx, "dimension", mlir.integerAttribute(mlir_ctx, .i64, axis_)), + .named(mlir_ctx, "is_stable", mlir.boolAttribute(mlir_ctx, is_stable)), + }, + .verify = true, + .location = .unknown(mlir_ctx), + }).appendTo(CompilationContext.current().currentScope().block); var result: [inputs.len]Tensor = undefined; inline for (0..inputs.len) |i| { @@ -536,427 +260,20 @@ pub fn sort(inputs: anytype, axis_: i64, comptime func_: anytype, context: anyty return result; } -/// Implement a while loop. -/// The given While struct contains all the parameters that are fixed through the while. -/// It must have: -/// * a State struct definition that captures all the loop state that will change on each iteration -/// * a `cond(While, State) Tensor` function returning a scalar boolean tensor -/// * a `body(While, State) State` function that preserves the state shapes. -pub fn @"while"( - While: type, - context: While, - initial_state: While.State, -) While.State { - const comp = Compiler.current(); - - var arena = std.heap.ArenaAllocator.init(comp.allocator); - defer arena.deinit(); - const allocator = arena.allocator(); - - const mlir_ctx = comp.mlir_ctx; - comp.pushLocation(@src(), @typeName(While)); - defer comp.popLocation(); - - // Force to materialize tensor.value() before we push a new scope. - var captured_context: While = undefined; - meta.mapAlloc(struct { - fn capture(_: void, tensor: Tensor) Tensor { - return Tensor._result(tensor.shape(), tensor.value()); - } - }.capture, allocator, {}, context, &captured_context) catch unreachable; - - const flat_operands: []*const Tensor = meta.collectPtrs(Tensor, allocator, &initial_state) catch @panic("OOM"); - - const OperandInfo = struct { - type: *const mlir.Type, - location: *const mlir.Location, - value: *const mlir.Value, - }; - - var operands_info = std.MultiArrayList(OperandInfo).initCapacity(allocator, flat_operands.len) catch @panic("OOM"); - - for (flat_operands) |input| { - operands_info.appendAssumeCapacity(.{ - .type = mlirx.Type.rankedTensor(mlir_ctx, input._shape), - .location = comp.location, - .value = input.value(), - }); - } - - const cond_block = b: { - const block = mlir.Block.init(operands_info.items(.type), operands_info.items(.location)); - errdefer block.deinit(); - - const scope = comp.pushBlock(block); - defer scope.pop(); - - comp.pushLocationFmt(@src(), "{s}.cond", .{@typeName(While)}); - defer comp.popLocation(); - - // Interpret initial_state as the block argument - for (0.., flat_operands) |i, input| { - scope.registerTensorAsBlockArgument(input.id, i); - } - - const cond: Tensor = captured_context.cond(initial_state); - stdx.debug.assert(cond.rank() == 0 and cond.dtype() == .bool, "zml.ops.while expects cond to return a scalar bool Tensor, got {f}", .{cond}); - - _ = stablehlo.return_(mlir_ctx, cond.value(), comp.location).appendTo(block); - break :b block; - }; - - const body_block, var result = b: { - const block = mlir.Block.init(operands_info.items(.type), operands_info.items(.location)); - errdefer block.deinit(); - - const scope = comp.pushBlock(block); - defer scope.pop(); - - comp.pushLocationFmt(@src(), "{s}.body", .{@typeName(While)}); - defer comp.popLocation(); - - // Interpret operands as the block argument - scope.id_to_argument.ensureUnusedCapacity(scope.arena.allocator(), flat_operands.len) catch @panic("OOM"); - for (0.., flat_operands) |i, input| { - scope.registerTensorAsBlockArgument(input.id, i); - } - - const result: While.State = captured_context.body(initial_state); - - if (builtin.mode == .debug) { - const result_tensors = meta.collectPtrs(Tensor, allocator, &result) catch @panic("OOM"); - stdx.debug.assert(flat_operands.len == result_tensors.len, "zml.ops.while({s}) expects body to return {d} Tensor values, got {d} values in ", .{ @typeName(While), flat_operands.len, result_tensors.len }); - for (0.., flat_operands, result_tensors) |i, in, out| { - stdx.debug.assert(in.shape().eql(out.shape()), "zml.ops.while({s}) expects body to preserve loop state shape, got {f} for operand {d} but expected {f}", .{ @typeName(While), in, i, out }); - } - } - - const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); - _ = stablehlo.returns(mlir_ctx, result_values, comp.location).appendTo(block); - break :b .{ block, result }; - }; - - const op = stablehlo.while_( - mlir_ctx, - operands_info.items(.value), - operands_info.items(.type), - cond_block, - body_block, - comp.location, - ).appendTo(comp.currentScope().block); - - const AssignResultCtx = struct { - op_: *mlir.Operation, - og_tensors: []*const Tensor, - idx: usize = 0, - - fn assign(ctx: *@This(), tensor: *Tensor) void { - tensor.* = Tensor._result(ctx.og_tensors[ctx.idx].shape(), ctx.op_.result(ctx.idx)); - ctx.idx += 1; - } - }; - var assign_result_ctx: AssignResultCtx = .{ - .op_ = op, - .og_tensors = flat_operands, - }; - meta.visit(AssignResultCtx.assign, &assign_result_ctx, &result); - - return result; -} - -pub const while_ = @"while"; - -test @"while" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const While = struct { - // While struct can contain tensor and non-tensor fields. - limit: i64, - step: Tensor, - - pub const State = struct { - i: zml.Tensor, - sum: zml.Tensor, - }; - - pub fn cond(self: @This(), state: State) Tensor { - return state.i.cmp(.LT, Tensor.scalar(self.limit, .i64)); - } - - pub fn body(self: @This(), state: State) State { - return .{ - .i = state.i.add(self.step), - .sum = state.sum.add(self.step), - }; - } - - fn forward(i: Tensor, sum: Tensor) State { - return zml.ops.@"while"( - @This(), - .{ .limit = 10, .step = .scalar(1, .i64) }, - .{ .i = i, .sum = sum }, - ); - } - }; - - const initial_i: zml.Tensor = .init(.{}, .i64); - const initial_sum: zml.Tensor = .init(.{}, .i64); - var exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - While.forward, - .{ initial_i, initial_sum }, - platform, - .{}, - ); - defer exe.deinit(); - - var i_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_i.shape(), std.mem.sliceAsBytes(&[1]i64{1})); - defer i_buffer.deinit(); - var sum_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, initial_sum.shape(), std.mem.sliceAsBytes(&[1]i64{0})); - defer sum_buffer.deinit(); - - var results: zml.Bufferized(While.State) = undefined; - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{ i_buffer, sum_buffer }, .{&results}, .{ .wait = true }); - defer results.i.deinit(); - defer results.sum.deinit(); - - var cpu_i = try results.i.toSliceAlloc(std.testing.allocator, std.testing.io); - defer cpu_i.free(std.testing.allocator); - var cpu_sum = try results.sum.toSliceAlloc(std.testing.allocator, std.testing.io); - defer cpu_sum.free(std.testing.allocator); - - try std.testing.expectEqual(@as(i64, 10), cpu_i.items(i64)[0]); - try std.testing.expectEqual(@as(i64, 9), cpu_sum.items(i64)[0]); -} - -/// Implement if/else branches. -/// The given If struct contains all the parameters that can be referenced in the if/else branches. -/// -/// It must have: -/// * an `onTrue(If) T` function returning some tensors -/// * an `onFalse(If) T` function returning similar shaped tensors -/// -/// If some expression in `onTrue` or `onFalse` has side effects (like printing), -/// stablehlo guarantees they will only be evaluated in its branch. -pub fn @"if"( - If: type, - if_captures: If, - pred: Tensor, -) stdx.meta.FnReturn(If.onTrue) { - stdx.debug.assertComptime( - stdx.meta.FnReturn(If.onTrue) == stdx.meta.FnReturn(If.onFalse), - "zml.ops.if expects same return types for onTrue and onFalse, got {s} and {s}", - .{ @typeName(stdx.meta.FnReturn(If.onTrue)), @typeName(stdx.meta.FnReturn(If.onFalse)) }, - ); - - stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); - - const compiler = Compiler.current(); - var arena = std.heap.ArenaAllocator.init(compiler.allocator); - defer arena.deinit(); - - const allocator = arena.allocator(); - var blkctx: If = undefined; - - // Explicitly capture values from the parent block. - meta.mapAlloc(struct { - fn capture(_: void, tensor: Tensor) Tensor { - return Tensor._result(tensor.shape(), tensor.value()); - } - }.capture, arena.allocator(), {}, if_captures, &blkctx) catch unreachable; - - const mlir_ctx = Compiler.current().mlir_ctx; - const loc: *const mlir.Location = compiler.location; - - compiler.pushLocation(@src(), @typeName(If)); - defer compiler.popLocation(); - - const true_branch, const true_branch_block = b: { - const block = mlir.Block.init(&.{}, &.{}); - errdefer block.deinit(); - - const scope = Compiler.current().pushBlock(block); - defer scope.pop(); - - const result = blkctx.onTrue(); - const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); - defer allocator.free(result_values); - _ = stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); - break :b .{ result, block }; - }; - - const false_branch, const false_branch_block = b: { - const block = mlir.Block.init(&.{}, &.{}); - errdefer block.deinit(); - - const scope = Compiler.current().pushBlock(block); - defer scope.pop(); - - const result = blkctx.onFalse(); - const result_values = meta.collectAlloc(Tensor.value, {}, allocator, &result) catch @panic("OOM"); - defer allocator.free(result_values); - _ = stablehlo.returns(mlir_ctx, result_values, loc).appendTo(block); - break :b .{ result, block }; - }; - // TODO check that false branch values matches right branch - _ = false_branch; - - const op = stablehlo.if_(mlir_ctx, pred.asScalar().value(), true_branch_block, false_branch_block, loc); - _ = op.appendTo(Compiler.current().currentScope().block); - - return fromMlirOperationWithTags(op, true_branch); -} - -test "if" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - const allocator = std.testing.allocator; - - const IfMod = struct { - a: Tensor, - b: Tensor, - - pub fn _fwd(pred: Tensor, a: Tensor, b: Tensor) Tensor { - return @"if"(@This(), .{ .a = a, .b = b }, pred.convert(.bool)); - } - - pub fn onTrue(ctx: @This()) Tensor { - return ctx.a.dotGeneral(ctx.b, &.{.{ 1, 0 }}, &.{}); - } - - pub fn onFalse(ctx: @This()) Tensor { - return ctx.b.dotGeneral(ctx.a, &.{.{ 1, 0 }}, &.{}); - } - }; - - { - const pred: Tensor = .init(.{}, .i32); - const a: Tensor = .init(.{ 8, 8 }, .f32); - const b: Tensor = .init(.{ 8, 8 }, .f32); - const mod = try platform.compileFn(allocator, std.testing.io, IfMod._fwd, .{ pred, a, b }, .{}); - defer mod.deinit(); - } -} - -/// Simpler variant of `zml.ops.if` that assumes code-motion is supported by the backend. -/// The two branches are evaluated before the condition, then the condition chose which branches to keep. -pub fn if2( - pred: Tensor, - on_true: anytype, - on_false: @TypeOf(on_true), -) @TypeOf(on_true) { - stdx.debug.assert(pred.dtype() == .bool and pred.count() == 1, "zml.ops.if expects the condition to have exactly one element of dtype .bool, got {f}", .{pred}); - - var compiler = Compiler.current(); - - var arena = std.heap.ArenaAllocator.init(compiler.allocator); - defer arena.deinit(); - - const allocator = arena.allocator(); - - const mlir_ctx = compiler.mlir_ctx; - const loc: *const mlir.Location = compiler.location; - - const true_values = meta.collectAlloc(Tensor.value, {}, allocator, &on_true) catch @panic("OOM"); - defer allocator.free(true_values); - const true_branch_block = b: { - const block = mlir.Block.init(&.{}, &.{}); - errdefer block.deinit(); - - const scope = compiler.pushBlock(block); - defer scope.pop(); - _ = stablehlo.returns(mlir_ctx, true_values, loc).appendTo(block); - break :b block; - }; - - const false_values = meta.collectAlloc(Tensor.value, {}, allocator, &on_false) catch @panic("OOM"); - defer allocator.free(false_values); - const false_branch_block = b: { - const block = mlir.Block.init(&.{}, &.{}); - errdefer block.deinit(); - - const scope = compiler.pushBlock(block); - defer scope.pop(); - - _ = stablehlo.returns(mlir_ctx, false_values, loc).appendTo(block); - break :b block; - }; - - const op = stablehlo.if_(mlir_ctx, pred.asScalar().value(), true_branch_block, false_branch_block, loc); - _ = op.appendTo(compiler.currentScope().block); - - return fromMlirOperationWithTags(op, on_true); -} - -test if2 { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - const allocator = std.testing.allocator; - - const IfMod = struct { - pub fn _fwd(pred: Tensor, a: Tensor, b: Tensor) Tensor { - return if2( - pred.convert(.bool), - a.dotGeneral(b, &.{.{ 1, 0 }}, &.{}), - b.dotGeneral(a, &.{.{ 1, 0 }}, &.{}), - ); - } - }; - - { - const pred: Tensor = .init(.{}, .i32); - const a: Tensor = .init(.{ 8, 8 }, .f32); - const b: Tensor = .init(.{ 8, 8 }, .f32); - const mod = try platform.compileFn(allocator, std.testing.io, IfMod._fwd, .{ pred, a, b }, .{}); - defer mod.deinit(); - } -} - -/// Create a Tensor struct similar to base, keeping base tags, -/// but using mlir value and dims from the mlir operation. -fn fromMlirOperationWithTags(op: *const mlir.Operation, base: anytype) @TypeOf(base) { - const LocalContext = struct { - index: usize, - op: *const mlir.Operation, - }; - var context = LocalContext{ .index = 0, .op = op }; - var res = base; - meta.visit((struct { - fn cb(inner_ctx: *LocalContext, tensor: *Tensor) void { - var new = Tensor.fromMlirValue(inner_ctx.op.result(inner_ctx.index)); - stdx.debug.internalAssert(new.rank() == tensor.rank(), "expected operand result to have rank {} but got {f}", .{ tensor.rank(), new }); - // Copy tags; the partitioner infers result sharding. - // some ops can change dims eg reduceWindow, so we trust mlir here. - new._shape._tags = tensor._shape._tags; - tensor.* = new; - inner_ctx.index += 1; - } - }).cb, &context, &res); - std.debug.assert(context.index == op.numResults()); - return res; -} - pub const TritonOps = struct { debug: bool = false, - name: []const u8, - ir: []const u8, + name: [:0]const u8, + ir: [:0]const u8, grid: [3]i32, num_stages: i32, num_warps: i32, - is_tma_allowed: bool = false, - global_scratch_memory_size: i32 = 0, - output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, + output_operand_aliases: []const i64 = &.{}, }; /// Generate an MLIR call to the given member function with the given tensors. pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]Tensor { - const compiler = Compiler.current(); - const mlir_ctx = compiler.mlir_ctx; - var arena = std.heap.ArenaAllocator.init(compiler.allocator); + const mlir_ctx = CompilationContext.current().mlir_ctx; + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); defer arena.deinit(); var values: [inputs.len]*const mlir.Value = undefined; @@ -966,19 +283,17 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T var res_types: [outputs.len]*const mlir.Type = undefined; inline for (outputs, 0..) |output, i| { - res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); + res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(mlir_ctx, output.dtype())); } - const backend_config: *const mlir.Attribute = .dict(mlir_ctx, &.{ - .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), - .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), - .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), - .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), - .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), - .named(mlir_ctx, "num_stages", .int(mlir_ctx, .i32, opts.num_stages)), - .named(mlir_ctx, "num_warps", .int(mlir_ctx, .i32, opts.num_warps)), - .named(mlir_ctx, "is_tma_allowed", .boolean(mlir_ctx, opts.is_tma_allowed)), - .named(mlir_ctx, "global_scratch_memory_size", .int(mlir_ctx, .i32, opts.global_scratch_memory_size)), + const backend_config = mlir.dictionaryAttribute(mlir_ctx, &.{ + mlir.NamedAttribute.named(mlir_ctx, "name", mlir.stringAttribute(mlir_ctx, opts.name)), + mlir.NamedAttribute.named(mlir_ctx, "ir", mlir.stringAttribute(mlir_ctx, opts.ir)), + mlir.NamedAttribute.named(mlir_ctx, "grid_x", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[0])), + mlir.NamedAttribute.named(mlir_ctx, "grid_y", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[1])), + mlir.NamedAttribute.named(mlir_ctx, "grid_z", mlir.integerAttribute(mlir_ctx, .i32, opts.grid[2])), + mlir.NamedAttribute.named(mlir_ctx, "num_stages", mlir.integerAttribute(mlir_ctx, .i32, opts.num_stages)), + mlir.NamedAttribute.named(mlir_ctx, "num_warps", mlir.integerAttribute(mlir_ctx, .i32, opts.num_warps)), }); var operands_layouts: [inputs.len][]const usize = undefined; @@ -991,7 +306,7 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; } - const op = stablehlo.custom_call( + const op = dialects.stablehlo.custom_call( mlir_ctx, &values, &res_types, @@ -1003,291 +318,8 @@ pub fn triton(inputs: anytype, outputs: anytype, opts: TritonOps) [outputs.len]T .result_layouts = &results_layouts, .output_operand_aliases = opts.output_operand_aliases, }, - compiler.location, - ).appendTo(compiler.currentScope().block); - - var outputs_: [outputs.len]Tensor = undefined; - inline for (outputs, 0..) |output, i| { - outputs_[i] = Tensor._result(output, op.result(i)); - } - - return outputs_; -} - -pub const FlyOps = struct { - name: []const u8, - /// The whole module text: a `gpu.func @name ... kernel` inside a - /// `gpu.module`, one `!fly.ptr` per operand then per result; or a - /// top-level `func.func @name` in the destination-passing tensor ABI. - ir: []const u8, - grid: [3]i32, - /// Threads per block = num_warps * 64 on CDNA. - num_warps: i32, - waves_per_eu: i32 = 0, - /// Required when the kernel uses `fly.get_dyn_shared`. - shared_mem_bytes: i64 = 0, - /// Indices into the full argument list, operands first, so result `i` is - /// `inputs.len + i`. A no-op inside HIP graphs: a kernel that must see - /// zeroed outputs should zero them itself. - zeroed_args: []const i32 = &.{}, - output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, -}; - -pub fn fly(inputs: anytype, outputs: anytype, opts: FlyOps) [outputs.len]Tensor { - const compiler = Compiler.current(); - const mlir_ctx = compiler.mlir_ctx; - var arena = std.heap.ArenaAllocator.init(compiler.allocator); - defer arena.deinit(); - - var values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - values[i] = inputs[i].value(); - } - - var res_types: [outputs.len]*const mlir.Type = undefined; - inline for (outputs, 0..) |output, i| { - res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); - } - - // The plugin silently ignores unknown or wrong-typed optional keys, so a - // misspelled one here is not an error, just an override that never lands. - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; - attrs.appendSliceAssumeCapacity(&.{ - .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), - .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), - .named(mlir_ctx, "num_warps", .int(mlir_ctx, .i32, opts.num_warps)), - .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), - .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), - .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), - }); - if (opts.waves_per_eu > 0) attrs.appendAssumeCapacity(.named(mlir_ctx, "waves_per_eu", .int(mlir_ctx, .i32, opts.waves_per_eu))); - if (opts.shared_mem_bytes > 0) attrs.appendAssumeCapacity(.named(mlir_ctx, "shared_mem_bytes", .int(mlir_ctx, .i64, opts.shared_mem_bytes))); - if (opts.zeroed_args.len > 0) { - const Opts = stablehlo.CustomCallOpts; - var zeroed: stdx.BoundedArray(*const mlir.Attribute, Opts.MAX_OPERANDS + Opts.MAX_RESULTS) = .empty; - for (opts.zeroed_args) |i| { - std.debug.assert(i >= 0 and i < inputs.len + outputs.len); - zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); - } - attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); - } - const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); - - // The kernel sees raw pointers and assumes row-major: pin the layouts. - var operands_layouts: [inputs.len][]const usize = undefined; - inline for (inputs, 0..) |input, i| { - operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; - } - var results_layouts: [outputs.len][]const usize = undefined; - inline for (outputs, 0..) |output, i| { - results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; - } - - const op = stablehlo.custom_call( - mlir_ctx, - &values, - &res_types, - .{ - .call_target_name = "__gpu$xla.gpu.fly", - .backend_config = .{ .typed_ffi = backend_config }, - .has_side_effect = false, - .operand_layouts = &operands_layouts, - .result_layouts = &results_layouts, - .output_operand_aliases = opts.output_operand_aliases, - }, - compiler.location, - ).appendTo(compiler.currentScope().block); - - var outputs_: [outputs.len]Tensor = undefined; - inline for (outputs, 0..) |output, i| { - outputs_[i] = Tensor._result(output, op.result(i)); - } - - return outputs_; -} - -test "fly custom call, both entry ABIs" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - if (platform.target != .rocm) return error.SkipZigTest; - - const allocator = std.testing.allocator; - - // Compile `forward` over one f32 input and bring the result back. - const call = struct { - fn f(comptime forward: anytype, p: *const zml.Platform, t: Tensor, h: []const f32) !zml.Slice { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{t}, p, .{}); - defer exe.deinit(); - var buf: zml.Buffer = try .fromBytes(std.testing.io, p, t.shape(), std.mem.sliceAsBytes(h)); - defer buf.deinit(); - var out = try exe.eval(std.testing.allocator, std.testing.io, .{buf}); - defer out.deinit(); - return out.toSliceAlloc(std.testing.allocator, std.testing.io); - } - }.f; - - // Native Fly ABI: a `gpu.func ... kernel` taking one !fly.ptr per operand - // then per result. This is what `zml.kernel.fly` emits. - { - const Mod = struct { - const ir = - \\module attributes {gpu.container_module} { - \\ gpu.module @zml_fly_kernels { - \\ gpu.func @add_one(%a: !fly.ptr, %c: !fly.ptr) kernel { - \\ %tid = gpu.thread_id x - \\ %step = gpu.block_dim x - \\ %n = arith.constant 128 : index - \\ scf.for %i = %tid to %n step %step { - \\ %t = arith.index_cast %i : index to i32 - \\ %off = fly.make_int_tuple(%t) : (i32) -> !fly.int_tuple - \\ %pa = fly.add_offset(%a, %off) : (!fly.ptr, !fly.int_tuple) -> !fly.ptr - \\ %pc = fly.add_offset(%c, %off) : (!fly.ptr, !fly.int_tuple) -> !fly.ptr - \\ %va = fly.ptr.load(%pa) : (!fly.ptr) -> f32 - \\ %one = arith.constant 1.0 : f32 - \\ %vc = arith.addf %va, %one : f32 - \\ fly.ptr.store(%vc, %pc) : (f32, !fly.ptr) -> () - \\ } - \\ gpu.return - \\ } - \\ } - \\} - ; - pub fn forward(a: Tensor) Tensor { - return fly(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir, .grid = .{ 1, 1, 1 }, .num_warps = 2 })[0]; - } - }; - - var input: [128]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - - var host = try call(Mod.forward, platform, .init(.{ .n = 128 }, .f32), &input); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(@as(f32, @floatFromInt(i)) + 1, v); - } - - { - const Mod = struct { - const ir = - \\module { - \\ func.func @add_one(%input: tensor<64xf32>, %output: tensor<64xf32>) -> tensor<64xf32> { - \\ %thread = gpu.thread_id x - \\ %last = arith.constant 63 : index - \\ %i = arith.minui %thread, %last : index - \\ %value = tensor.extract %input[%i] : tensor<64xf32> - \\ %one = arith.constant 1.0 : f32 - \\ %sum = arith.addf %value, %one : f32 - \\ %updated = tensor.insert %sum into %output[%i] : tensor<64xf32> - \\ return %updated : tensor<64xf32> - \\ } - \\} - ; - pub fn forward(a: Tensor) Tensor { - return fly(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir, .grid = .{ 1, 1, 1 }, .num_warps = 2 })[0]; - } - }; - - var input: [64]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - - var host = try call(Mod.forward, platform, .init(.{ .n = 64 }, .f32), &input); - defer host.free(allocator); - for (host.items(f32), 0..) |v, i| try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); - } -} - -pub const NeuronNkiOps = struct { - name: []const u8, - entrypoint: []const u8, - source_path: []const u8, - compiler_target: []const u8, - has_side_effect: bool = false, - output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, -}; - -/// Generate an MLIR call for a Neuron NKI kernel. -/// -/// This API is Neuron-only: the source is compiled to an -/// `AwsNeuronCustomNativeKernel` backend config while emitting the graph. -pub fn neuronNki(inputs: anytype, outputs: anytype, opts: NeuronNkiOps) [outputs.len]Tensor { - const ctx = Compiler.current(); - switch (ctx.platform.target) { - .neuron => {}, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => { - stdx.debug.panic("neuronNki is only available on Neuron, got {s}", .{@tagName(ctx.platform.target)}); - }, - } - if (comptime !platforms.isEnabled(.neuron)) unreachable; - const nki_kernel = @import("platforms/neuron/nki_kernel"); - - const mlir_ctx = ctx.mlir_ctx; - var arena = std.heap.ArenaAllocator.init(ctx.allocator); - defer arena.deinit(); - - const allocator = arena.allocator(); - - var values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - values[i] = inputs[i].value(); - } - - var res_types: [outputs.len]*const mlir.Type = undefined; - inline for (outputs, 0..) |output, i| { - res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); - } - - var operands_layouts: [inputs.len][]const usize = undefined; - inline for (inputs, 0..) |input, i| { - operands_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; - } - - var results_layouts: [outputs.len][]const usize = undefined; - inline for (outputs, 0..) |output, i| { - results_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; - } - - var input_signatures: [inputs.len]nki_kernel.TensorSignature = undefined; - inline for (inputs, 0..) |input, i| { - input_signatures[i] = .{ - .dtype = @tagName(input.dtype()), - .dims = input.dims(), - }; - } - var output_signatures: [outputs.len]nki_kernel.TensorSignature = undefined; - inline for (outputs, 0..) |output, i| { - output_signatures[i] = .{ - .dtype = @tagName(output.dtype()), - .dims = output.dims(), - }; - } - - const compiled_backend_config = nki_kernel.compileNkiKernel(ctx.allocator, ctx.io, .{ - .name = opts.name, - .entrypoint = opts.entrypoint, - .source_path = opts.source_path, - .compiler_target = opts.compiler_target, - .inputs = &input_signatures, - .outputs = &output_signatures, - }) catch |err| { - stdx.debug.panic("failed to compile Neuron NKI kernel: {}", .{err}); - }; - defer ctx.allocator.free(compiled_backend_config); - - const op = stablehlo.custom_call( - mlir_ctx, - &values, - &res_types, - .{ - .call_target_name = "AwsNeuronCustomNativeKernel", - .has_side_effect = opts.has_side_effect, - .operand_layouts = &operands_layouts, - .result_layouts = &results_layouts, - .output_operand_aliases = opts.output_operand_aliases, - .additional_attributes = &.{ - .named(mlir_ctx, "backend_config", .string(mlir_ctx, compiled_backend_config)), - }, - }, - ctx.location, - ).appendTo(ctx.currentScope().block); + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); var outputs_: [outputs.len]Tensor = undefined; inline for (outputs, 0..) |output, i| { @@ -1301,11 +333,11 @@ test "triton" { const zml = @import("zml.zig"); const platform = zml.testing.env(); - if (platform.target != .cuda and platform.target != .rocm and platform.target != .oneapi) return error.SkipZigTest; + if (platform.target != .cuda and platform.target != .rocm) return error.SkipZigTest; const ir = \\ module { - \\ tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { + \\ tt.func public @add_one(%arg0: !tt.ptr {tt.divisibility = 32 : i32}, %arg1: !tt.ptr {tt.divisibility = 32 : i32}, %arg2: !tt.ptr {tt.divisibility = 32 : i32}, %arg3: !tt.ptr {tt.divisibility = 32 : i32}) { \\ %0 = tt.get_program_id x : i32 \\ %1 = tt.load %arg0 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr \\ %2 = tt.load %arg1 {cache = 1 : i32, evict = 1 : i32, isVolatile = false} : !tt.ptr @@ -1334,7 +366,7 @@ test "triton" { const a: zml.Tensor = .init(.{}, .f32); const b: zml.Tensor = .init(.{}, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, TritonMod.forward, .{ a, b }, platform); defer exe.deinit(); var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[1]f32{1})); @@ -1342,10 +374,7 @@ test "triton" { var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[1]f32{3})); defer b_buffer.deinit(); - var results: [2]zml.Buffer = undefined; - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{ a_buffer, b_buffer }, .{&results}, .{ .wait = true }); + var results = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, TritonMod.forward, .{ a_buffer, b_buffer }); defer results[0].deinit(); defer results[1].deinit(); @@ -1361,513 +390,6 @@ test "triton" { try std.testing.expectEqual(expected_result_b, cpu_result_1.items(f32)[0]); } -pub const CudaTileOps = struct { - name: []const u8, - ir: []const u8, - grid: [3]i32, - /// The Tile IR bytecode version XLA serializes at, "MAJOR.MINOR"; XLA's - /// default is 13.3. - ir_version: ?[]const u8 = null, - /// Result indices XLA zeroes before the launch, ascending. For a kernel - /// that accumulates, or writes less than the whole output. - zeroed_outputs: []const i32 = &.{}, - output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, -}; - -/// A `__gpu$xla.gpu.cuda_tile` custom call: XLA assembles `ir` with tileiras -/// and launches `name` over `grid` tile blocks. Every operand and result -/// arrives as one raw device pointer, in order, in the default layout. -pub fn cudaTile(inputs: anytype, outputs: anytype, opts: CudaTileOps) [outputs.len]Tensor { - const compiler = Compiler.current(); - const mlir_ctx = compiler.mlir_ctx; - - var values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - values[i] = inputs[i].value(); - } - - var res_types: [outputs.len]*const mlir.Type = undefined; - inline for (outputs, 0..) |output, i| { - res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); - } - - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 10) = .empty; - attrs.appendSliceAssumeCapacity(&.{ - .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), - .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cuda_tile")), - .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), - .named(mlir_ctx, "grid_x", .int(mlir_ctx, .i32, opts.grid[0])), - .named(mlir_ctx, "grid_y", .int(mlir_ctx, .i32, opts.grid[1])), - .named(mlir_ctx, "grid_z", .int(mlir_ctx, .i32, opts.grid[2])), - }); - if (opts.ir_version) |v| attrs.appendAssumeCapacity(.named(mlir_ctx, "ir_version", .string(mlir_ctx, v))); - if (opts.zeroed_outputs.len > 0) { - var zeroed: stdx.BoundedArray(*const mlir.Attribute, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - for (opts.zeroed_outputs) |i| zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); - attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); - } - const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); - - const op = stablehlo.custom_call( - mlir_ctx, - &values, - &res_types, - .{ - .call_target_name = "__gpu$xla.gpu.cuda_tile", - .backend_config = .{ .typed_ffi = backend_config }, - .has_side_effect = false, - .output_operand_aliases = opts.output_operand_aliases, - }, - compiler.location, - ).appendTo(compiler.currentScope().block); - - var outputs_: [outputs.len]Tensor = undefined; - inline for (outputs, 0..) |output, i| { - outputs_[i] = Tensor._result(output, op.result(i)); - } - - return outputs_; -} - -pub const TclOps = struct { - ir: []const u8, - /// Outputs the function updates in place in an input's buffer. - output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, -}; - -/// A TCL function the Furiosa plugin compiles as its own native unit. -pub fn tcl(inputs: anytype, outputs: anytype, opts: TclOps) [outputs.len]Tensor { - const compiler = Compiler.current(); - const mlir_ctx = compiler.mlir_ctx; - - var values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| values[i] = inputs[i].value(); - - var res_types: [outputs.len]*const mlir.Type = undefined; - inline for (outputs, 0..) |output, i| res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); - - const op = stablehlo.custom_call(mlir_ctx, &values, &res_types, .{ - .call_target_name = "furiosa.tcl", - .backend_config = .{ .original = opts.ir }, - .has_side_effect = false, - .output_operand_aliases = opts.output_operand_aliases, - }, compiler.location).appendTo(compiler.currentScope().block); - - var outputs_: [outputs.len]Tensor = undefined; - inline for (outputs, 0..) |output, i| outputs_[i] = Tensor._result(output, op.result(i)); - return outputs_; -} - -pub const MetalOps = struct { - /// The kernel symbol. - name: []const u8, - /// The kernel's module, in the AIR dialect. - ir: []const u8, - /// Threadgroups per grid. - grid: [3]u32, - /// Threads per threadgroup. - threadgroup: [3]u32, - /// Bytes of `[[threadgroup(0)]]` memory. - threadgroup_memory: u64 = 0, - /// What each `[[buffer(n)]]` binds: `k >= 0` the XLA buffer `k` (the - /// operands, then the results), `k < 0` the constant `-1 - k`. - arguments: []const i32, - /// The bytes of the constant arguments. - constants: []const []const u8 = &.{}, - output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, -}; - -/// A `__gpu$xla.gpu.air` custom call: XLA lowers `ir` to AIR, compiles it to -/// a metallib and launches `name`. -pub fn metal(inputs: anytype, outputs: anytype, opts: MetalOps) [outputs.len]Tensor { - const compiler = Compiler.current(); - const mlir_ctx = compiler.mlir_ctx; - - var values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| values[i] = inputs[i].value(); - - var res_types: [outputs.len]*const mlir.Type = undefined; - inline for (outputs, 0..) |output, i| res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); - - const allocator = compiler.allocator; - const constant_attrs = allocator.alloc(*const mlir.Attribute, opts.constants.len) catch @panic("OOM"); - defer allocator.free(constant_attrs); - for (opts.constants, constant_attrs) |bytes, *attribute| attribute.* = .denseArray(mlir_ctx, .i8, @as([]const i8, @ptrCast(bytes))); - - const sizes = [2][3]u32{ opts.grid, opts.threadgroup }; - var dims: [2][3]*const mlir.Attribute = undefined; - for (&dims, sizes) |*elems, dim| { - for (elems, dim) |*e, size| e.* = .int(mlir_ctx, .i32, size); - } - const backend_config: *const mlir.Attribute = .dict(mlir_ctx, &.{ - .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), - .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), - .named(mlir_ctx, "grid", .array(mlir_ctx, &dims[0])), - .named(mlir_ctx, "threadgroup", .array(mlir_ctx, &dims[1])), - .named(mlir_ctx, "threadgroup_memory", .int(mlir_ctx, .i64, @as(i64, @intCast(opts.threadgroup_memory)))), - .named(mlir_ctx, "arguments", .denseArray(mlir_ctx, .i32, opts.arguments)), - .named(mlir_ctx, "constants", .array(mlir_ctx, constant_attrs)), - }); - - const op = stablehlo.custom_call(mlir_ctx, &values, &res_types, .{ - .call_target_name = "__gpu$xla.gpu.air", - .backend_config = .{ .typed_ffi = backend_config }, - .has_side_effect = false, - .output_operand_aliases = opts.output_operand_aliases, - }, compiler.location).appendTo(compiler.currentScope().block); - - var outputs_: [outputs.len]Tensor = undefined; - inline for (outputs, 0..) |output, i| outputs_[i] = Tensor._result(output, op.result(i)); - return outputs_; -} - -pub const CuteOps = struct { - /// The kernel symbol. - name: []const u8, - /// Textual CuTe DSL module: either public `func.func` kernels, launched - /// with `grid`/`block`, or a `gpu.module` plus a host launch function - /// that owns the launch configuration. - ir: []const u8, - grid: ?[3]i32 = null, - block: ?[3]i32 = null, - zeroed_outputs: []const i32 = &.{}, - scalars: []const i64 = &.{}, - symmetric_workspace: ?SymmetricWorkspace = null, - output_operand_aliases: []const stablehlo.CustomCallOpts.OutputOperandAlias = &.{}, - - /// Named per-device workspace shared across CuTe calls and executables. - /// Every rank allocates `bytes`; XLA passes peer addresses and the rank. - pub const SymmetricWorkspace = struct { - name: []const u8, - bytes: i64, - }; -}; - -/// A `__gpu$xla.gpu.cute` custom call: XLA compiles `ir` with the CuTe compiler -/// and launches `name`. Every operand and result arrives as one raw device -/// pointer, in order, in the default layout. -pub fn cute(inputs: anytype, outputs: anytype, opts: CuteOps) [outputs.len]Tensor { - const compiler = Compiler.current(); - - const mlir_ctx = compiler.mlir_ctx; - - var values: [inputs.len]*const mlir.Value = undefined; - inline for (0..inputs.len) |i| { - values[i] = inputs[i].value(); - } - - var res_types: [outputs.len]*const mlir.Type = undefined; - inline for (outputs, 0..) |output, i| { - res_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output); - } - - var attrs: stdx.BoundedArray(mlir.NamedAttribute, 8) = .empty; - attrs.appendSliceAssumeCapacity(&.{ - .named(mlir_ctx, "name", .string(mlir_ctx, opts.name)), - .named(mlir_ctx, "kernel_type", .string(mlir_ctx, "cute")), - .named(mlir_ctx, "ir", .string(mlir_ctx, opts.ir)), - }); - inline for (.{ "grid", "block" }) |key| { - if (@field(opts, key)) |dims| { - var elems: [3]*const mlir.Attribute = undefined; - for (&elems, dims) |*e, d| e.* = .int(mlir_ctx, .i32, d); - attrs.appendAssumeCapacity(.named(mlir_ctx, key, .array(mlir_ctx, &elems))); - } - } - if (opts.zeroed_outputs.len > 0) { - var zeroed: stdx.BoundedArray(*const mlir.Attribute, stablehlo.CustomCallOpts.MAX_RESULTS) = .empty; - for (opts.zeroed_outputs) |i| zeroed.appendAssumeCapacity(.int(mlir_ctx, .i32, i)); - attrs.appendAssumeCapacity(.named(mlir_ctx, "zeroed_outputs", .array(mlir_ctx, zeroed.constSlice()))); - } - if (opts.scalars.len > 0) { - const allocator = Compiler.current().allocator; - const scalar_attrs = allocator.alloc(*const mlir.Attribute, opts.scalars.len) catch @panic("OOM"); - defer allocator.free(scalar_attrs); - for (opts.scalars, scalar_attrs) |value, *attribute| attribute.* = .int(mlir_ctx, .i64, value); - attrs.appendAssumeCapacity(.named(mlir_ctx, "scalars", .array(mlir_ctx, scalar_attrs))); - } - if (opts.symmetric_workspace) |workspace| { - stdx.debug.assert(workspace.name.len > 0 and workspace.bytes > 0, "zml.ops.cute({s}): symmetric_workspace needs a name and a positive size", .{opts.name}); - attrs.appendAssumeCapacity(.named(mlir_ctx, "symmetric_workspace", .dict(mlir_ctx, &.{ - .named(mlir_ctx, "name", .string(mlir_ctx, workspace.name)), - .named(mlir_ctx, "bytes", .int(mlir_ctx, .i64, workspace.bytes)), - }))); - } - const backend_config: *const mlir.Attribute = .dict(mlir_ctx, attrs.constSlice()); - - const op = stablehlo.custom_call( - mlir_ctx, - &values, - &res_types, - .{ - .call_target_name = "__gpu$xla.gpu.cute", - .backend_config = .{ .typed_ffi = backend_config }, - .has_side_effect = false, - .output_operand_aliases = opts.output_operand_aliases, - }, - compiler.location, - ).appendTo(compiler.currentScope().block); - - var outputs_: [outputs.len]Tensor = undefined; - inline for (outputs, 0..) |output, i| { - outputs_[i] = Tensor._result(output, op.result(i)); - } - - return outputs_; -} - -test "cute" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - if (platform.target != .cuda) return error.SkipZigTest; - - // out[i] = in[i] + 1 over 128 f32, in the CuTe DSL's frontend form. - const ir = - \\module attributes {gpu.container_module} { - \\ gpu.module @kernels { - \\ cuda.kernel @add_one(%arg0: !cute.ptr>, %arg1: !cute.ptr>) attributes {cute.kernel, gpu.kernel, nvvm.reqntid = array} { - \\ %i = nvvm.read.ptx.sreg.tid.x : i32 - \\ %shape = cute.make_shape() : () -> !cute.shape<"128"> - \\ %lay = cute.make_layout(%shape) : !cute.layout<"128:1"> - \\ %coord = cute.make_coord(%i) : (i32) -> !cute.coord<"?"> - \\ %in = cute.make_view(%arg0, %lay) : !cute.memref, "128:1"> - \\ %out = cute.make_view(%arg1, %lay) : !cute.memref, "128:1"> - \\ %x = cute.memref.load(%in, %coord) : (!cute.memref, "128:1">, !cute.coord<"?">) -> f32 - \\ %one = arith.constant 1.0 : f32 - \\ %y = arith.addf %x, %one : f32 - \\ cute.memref.store(%out, %coord, %y) : (!cute.memref, "128:1">, !cute.coord<"?">, f32) -> () - \\ return - \\ } - \\ } - \\ func.func @launch(%arg0: !cute.ptr>, %arg1: !cute.ptr>) -> i32 attributes {llvm.emit_c_interface} { - \\ %smem = cute.kernel_smem_size @kernels::@add_one : i64 - \\ %c0_i64 = arith.constant 0 : i64 - \\ %stream = cuda.cast %c0_i64 : i64 -> !cuda.stream - \\ %c1 = arith.constant 1 : i32 - \\ %c128 = arith.constant 128 : i32 - \\ %cfg = cuda.launch_cfg.create (blockDim = (%c128, %c1, %c1), dynamicSmemBytes = %smem, gridDim = (%c1, %c1, %c1), stream = %stream) : i32, i32, i32, i64, i32, i32, i32, !cuda.stream -> !cuda.launch_cfg - \\ %r = cuda.launch_ex @kernels::@add_one<%cfg> (%arg0, %arg1) : !cuda.launch_cfg, (!cute.ptr>, !cute.ptr>) -> !cuda.result - \\ %status = cuda.cast %r : !cuda.result -> i32 - \\ return %status : i32 - \\ } - \\} - ; - - const Mod = struct { - pub fn forward(a: Tensor) Tensor { - return cute(.{a}, .{a.shape()}, .{ .name = "add_one", .ir = ir })[0]; - } - }; - - const a: zml.Tensor = .init(.{ .n = 128 }, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); - defer exe.deinit(); - - var input: [128]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); - defer a_buffer.deinit(); - - var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); - defer result.deinit(); - - var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); - defer host.free(std.testing.allocator); - - for (host.items(f32), 0..) |v, i| { - try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); - } -} - -test "cuda_tile" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - if (platform.target != .cuda) return error.SkipZigTest; - - // out[i] = in[i] + 1 over 128 f32; the XLA milestone kernel. - const ir = - \\cuda_tile.module @m { - \\ entry @add_one(%in : tile>, %out : tile>) { - \\ %offsets = iota : tile<128xi32> - \\ %in_r = reshape %in : tile> -> tile<1xptr> - \\ %in_b = broadcast %in_r : tile<1xptr> -> tile<128xptr> - \\ %in_p = offset %in_b, %offsets : tile<128xptr>, tile<128xi32> -> tile<128xptr> - \\ %v, %t0 = load_ptr_tko weak %in_p : tile<128xptr> -> tile<128xf32>, token - \\ %one = constant : tile - \\ %one_r = reshape %one : tile -> tile<1xf32> - \\ %one_b = broadcast %one_r : tile<1xf32> -> tile<128xf32> - \\ %sum = addf %v, %one_b rounding : tile<128xf32> - \\ %out_r = reshape %out : tile> -> tile<1xptr> - \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<128xptr> - \\ %out_p = offset %out_b, %offsets : tile<128xptr>, tile<128xi32> -> tile<128xptr> - \\ %t1 = store_ptr_tko weak %out_p, %sum : tile<128xptr>, tile<128xf32> -> token - \\ return - \\ } - \\} - ; - - const Mod = struct { - pub fn forward(a: Tensor) Tensor { - return cudaTile(.{a}, .{a.shape()}, .{ - .name = "add_one", - .ir = ir, - .grid = .{ 1, 1, 1 }, - })[0]; - } - }; - - const a: zml.Tensor = .init(.{ .n = 128 }, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); - defer exe.deinit(); - - var input: [128]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); - defer a_buffer.deinit(); - - var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); - defer result.deinit(); - - var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); - defer host.free(std.testing.allocator); - - for (host.items(f32), 0..) |v, i| { - try std.testing.expectEqual(@as(f32, @floatFromInt(i + 1)), v); - } -} - -test "cuda_tile grid" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - if (platform.target != .cuda) return error.SkipZigTest; - - // Each of the 2x3x4 tile blocks writes its linear id t = x + 2y + 6z - // over out[128*t .. 128*t+128); the input is unused. - const ir = - \\cuda_tile.module @m { - \\ entry @grid(%in : tile>, %out : tile>) { - \\ %bx, %by, %bz = get_tile_block_id : tile - \\ %c2 = constant : tile - \\ %c6 = constant : tile - \\ %c128 = constant : tile - \\ %y2 = muli %by, %c2 : tile - \\ %z6 = muli %bz, %c6 : tile - \\ %xy = addi %bx, %y2 : tile - \\ %t = addi %xy, %z6 : tile - \\ %base = muli %t, %c128 : tile - \\ %offsets = iota : tile<128xi32> - \\ %base_r = reshape %base : tile -> tile<1xi32> - \\ %base_b = broadcast %base_r : tile<1xi32> -> tile<128xi32> - \\ %idx = addi %offsets, %base_b : tile<128xi32> - \\ %out_r = reshape %out : tile> -> tile<1xptr> - \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<128xptr> - \\ %out_p = offset %out_b, %idx : tile<128xptr>, tile<128xi32> -> tile<128xptr> - \\ %tf = itof %t signed rounding : tile -> tile - \\ %tf_r = reshape %tf : tile -> tile<1xf32> - \\ %tf_b = broadcast %tf_r : tile<1xf32> -> tile<128xf32> - \\ %tok = store_ptr_tko weak %out_p, %tf_b : tile<128xptr>, tile<128xf32> -> token - \\ return - \\ } - \\} - ; - - const Mod = struct { - pub fn forward(a: Tensor) Tensor { - return cudaTile(.{a}, .{a.shape()}, .{ - .name = "grid", - .ir = ir, - .grid = .{ 2, 3, 4 }, - })[0]; - } - }; - - const n = 24 * 128; - const a: zml.Tensor = .init(.{ .n = n }, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); - defer exe.deinit(); - - const input: [n]f32 = @splat(0); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); - defer a_buffer.deinit(); - - var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); - defer result.deinit(); - - var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); - defer host.free(std.testing.allocator); - - for (host.items(f32), 0..) |v, i| { - try std.testing.expectEqual(@as(f32, @floatFromInt(i / 128)), v); - } -} - -test "cuda_tile zeroed_outputs" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - if (platform.target != .cuda) return error.SkipZigTest; - - // Writes only the first 64 of 128; `zeroed_outputs` owns the rest. - const ir = - \\cuda_tile.module @m { - \\ entry @half(%in : tile>, %out : tile>) { - \\ %offsets = iota : tile<64xi32> - \\ %in_r = reshape %in : tile> -> tile<1xptr> - \\ %in_b = broadcast %in_r : tile<1xptr> -> tile<64xptr> - \\ %in_p = offset %in_b, %offsets : tile<64xptr>, tile<64xi32> -> tile<64xptr> - \\ %v, %t0 = load_ptr_tko weak %in_p : tile<64xptr> -> tile<64xf32>, token - \\ %one = constant : tile - \\ %one_r = reshape %one : tile -> tile<1xf32> - \\ %one_b = broadcast %one_r : tile<1xf32> -> tile<64xf32> - \\ %sum = addf %v, %one_b rounding : tile<64xf32> - \\ %out_r = reshape %out : tile> -> tile<1xptr> - \\ %out_b = broadcast %out_r : tile<1xptr> -> tile<64xptr> - \\ %out_p = offset %out_b, %offsets : tile<64xptr>, tile<64xi32> -> tile<64xptr> - \\ %tok = store_ptr_tko weak %out_p, %sum : tile<64xptr>, tile<64xf32> -> token - \\ return - \\ } - \\} - ; - - const Mod = struct { - pub fn forward(a: Tensor) Tensor { - return cudaTile(.{a}, .{a.shape()}, .{ - .name = "half", - .ir = ir, - .grid = .{ 1, 1, 1 }, - .zeroed_outputs = &.{0}, - })[0]; - } - }; - - const a: zml.Tensor = .init(.{ .n = 128 }, .f32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Mod.forward, .{a}, platform, .{}); - defer exe.deinit(); - - var input: [128]f32 = undefined; - for (&input, 0..) |*v, i| v.* = @floatFromInt(i); - var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&input)); - defer a_buffer.deinit(); - - var result = try exe.eval(std.testing.allocator, std.testing.io, .{a_buffer}); - defer result.deinit(); - - var host = try result.toSliceAlloc(std.testing.allocator, std.testing.io); - defer host.free(std.testing.allocator); - - for (host.items(f32), 0..) |v, i| { - const want: f32 = if (i < 64) @floatFromInt(i + 1) else 0; - try std.testing.expectEqual(want, v); - } -} - pub const ScatterArgs = struct { input: Tensor, update: Tensor, @@ -1886,19 +408,17 @@ pub fn scatter( inputs: anytype, index_tensors: anytype, updates: anytype, - comptime func_: anytype, + comptime func: anytype, context: anytype, opts: Tensor.ScatterOpts, -) stdx.meta.FnReturn(func_) { - const compiler = Compiler.current(); - var arena = std.heap.ArenaAllocator.init(compiler.allocator); +) stdx.meta.FnResult(func) { + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); defer arena.deinit(); - const mlir_ctx = compiler.mlir_ctx; + const mlir_ctx = CompilationContext.current().mlir_ctx; const update_block, var result = b: { - const ArgsTypes: [inputs.len]type = @splat(ScatterArgs); - const ArgsType = @Tuple(&ArgsTypes); + const ArgsType = std.meta.Tuple(&[1]type{ScatterArgs} ** inputs.len); var args: ArgsType = undefined; var block_types: [2 * inputs.len]*const mlir.Type = undefined; @@ -1906,30 +426,31 @@ pub fn scatter( args[i].input = Tensor.init(.{}, inputs[i].dtype()); args[i].update = Tensor.init(.{}, inputs[i].dtype()); - block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, args[i].input.shape()); - block_types[i + inputs.len] = mlirx.Type.rankedTensor(mlir_ctx, args[i].update.shape()); + block_types[i] = mlir.rankedTensorType(args[i].input.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].input.dtype())); + block_types[i + inputs.len] = mlir.rankedTensorType(args[i].update.dims(), mlirx.Type.fromDType(mlir_ctx, args[i].update.dtype())); } - const block_locs: [2 * inputs.len]*const mlir.Location = @splat(compiler.unknown_location); + const block_locs: [2 * inputs.len]*const mlir.Location = @splat(mlir.Location.unknown(mlir_ctx)); const update_block = mlir.Block.init(&block_types, &block_locs); errdefer update_block.deinit(); - const scope = compiler.pushBlock(update_block); - defer scope.pop(); + CompilationContext.current().pushBlock(update_block); + defer CompilationContext.current().popBlock(); + const scope = CompilationContext.current().currentScope(); inline for (0..inputs.len) |i| { - scope.registerTensorAsBlockArgument(args[i].input.id, i); - scope.registerTensorAsBlockArgument(args[i].update.id, i + inputs.len); + scope.id_to_argument.put(scope.arena.allocator(), args[i].input.id, i) catch unreachable; + scope.id_to_argument.put(scope.arena.allocator(), args[i].update.id, i + inputs.len) catch unreachable; } - var result = @call(.auto, func_, args ++ context); + var result = @call(.auto, func, args ++ context); var result_values: [inputs.len]*const mlir.Value = undefined; inline for (0..inputs.len) |i| { result_values[i] = result[i].value(); } - _ = stablehlo.returns(mlir_ctx, &result_values, compiler.location).appendTo(update_block); + _ = dialects.stablehlo.returns(mlir_ctx, &result_values, .unknown(mlir_ctx)).appendTo(update_block); break :b .{ update_block, result }; }; @@ -1967,9 +488,6 @@ pub fn scatter( return .{self.dynamicUpdateSlice(index_tensors, update)}; } - var updates_values: [inputs.len]*const mlir.Value = undefined; - LoweringCompatibility.preserveScatterDropSemantics(indices_per_axis.slice(), updates, opts, &updates_values); - // TODO: ideally we should catch all possible scatter errors and provide nice error messages. var config = scatterConfig(self.shape(), update.shape(), indices_per_axis, indices_axes); const indices = scatterPrepareIndices(&config, self.shape(), update.shape(), &indices_per_axis, &indices_axes); @@ -1981,7 +499,10 @@ pub fn scatter( input_values[i] = inputs[i].value(); } - const op = stablehlo.scatter( + var updates_values: [inputs.len]*const mlir.Value = undefined; + inline for (0..updates.len) |i| updates_values[i] = updates[i].value(); + + const op = dialects.stablehlo.scatter( mlir_ctx, &input_values, &.{indices.value()}, @@ -1997,8 +518,8 @@ pub fn scatter( .indices_are_sorted = opts.indices_are_sorted, .unique_indices = opts.indices_are_unique, }, - compiler.location, - ).appendTo(compiler.currentScope().block); + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); inline for (0..result.len) |i| { result[i] = Tensor._result(inputs[i].shape(), op.result(i)); @@ -2008,11 +529,11 @@ pub fn scatter( } const ScatterConfig = struct { - op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty, - up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty, - indices_batch_axes: Shape.DimsArray = .empty, - scatter_to_operand_axes: Shape.DimsArray = .empty, - updates_transpose: Shape.AxesArray = .empty, + op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, + up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}, + indices_batch_axes: Shape.DimsArray = .{}, + scatter_to_operand_axes: Shape.DimsArray = .{}, + updates_transpose: Shape.AxesArray = .{}, }; const ScatterAxisKind = enum { batching, update_window, inserted_window, window_id }; @@ -2023,11 +544,11 @@ fn scatterConfig( indices_per_axis: stdx.BoundedArray(Tensor, constants.MAX_RANK), indices_axes: Shape.TagsArray, ) ScatterConfig { - var op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty; - var up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .empty; - var indices_batch_axes: Shape.DimsArray = .empty; - var scatter_to_operand_axes: Shape.DimsArray = .empty; - var updates_transpose: Shape.AxesArray = .empty; + var op_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; + var up_kind: stdx.BoundedArray(ScatterAxisKind, constants.MAX_RANK) = .{}; + var indices_batch_axes: Shape.DimsArray = .{}; + var scatter_to_operand_axes: Shape.DimsArray = .{}; + var updates_transpose: Shape.AxesArray = .{}; const tagged_api = indices_axes.len > 0; const indices = indices_per_axis.get(0).shape(); @@ -2110,14 +631,14 @@ test scatterConfig { const zml = @import("zml.zig"); const platform = zml.testing.env(); - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - const scope = comp.pushBlock(block); - defer scope.pop(); + comp.pushBlock(block); + defer comp.popBlock(); const Local = struct { pub fn _idx(idx_shape: anytype) Tensor { @@ -2173,12 +694,12 @@ fn scatterPrepareIndices( cfg.up_kind.buffer[update.axis(batch_tag)] = .window_id; old_scatter_to_op_axes.appendAssumeCapacity(batch_ax); } - cfg.indices_batch_axes = .empty; + cfg.indices_batch_axes = .{}; // Reorder the axes so that in indices_per_axis is ordered like in op if possible. // TODO: transpose updates if needed - var indices: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .empty; - var scatter_to_op_axes: Shape.DimsArray = .empty; + var indices: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .{}; + var scatter_to_op_axes: Shape.DimsArray = .{}; while (old_scatter_to_op_axes.len > 0) { const scatter_ax = std.sort.argMin(i64, old_scatter_to_op_axes.constSlice(), {}, std.sort.asc(i64)).?; @@ -2204,8 +725,7 @@ pub const GatherAxisKind = enum { batching, offset, collapsed, indices }; pub const GatherOpts = struct { indices_are_sorted: bool = false }; pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, opts: GatherOpts) Tensor { - const compiler = Compiler.current(); - const mlir_ctx = compiler.mlir_ctx; + const mlir_ctx = CompilationContext.current().mlir_ctx; stdx.debug.assert(idx_axes.len > 0, "gather expects 1 or more axes to operate one, received none. Example: `x.gather(.a, indices, .{{}})`", .{}); for (idx_axes, 0..) |a, i| { @@ -2213,26 +733,22 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, stdx.debug.assert(a == idx_axes[i - 1] + 1, "gather expects 'idx_axes' to be sequential. But {any} aren't sequential in {f}", .{ idx_axes, self }); } } - var indices_per_axis, _ = Shape.parseStruct(Tensor, idx_per_axis); - - var indices_shape = indices_per_axis.get(0).shape(); - for (indices_per_axis.constSlice()[1..]) |idx| { + var indices_shape = idx_per_axis[0].shape(); + for (idx_per_axis[1..]) |idx| { if (idx.rank() > indices_shape.rank()) { indices_shape = idx.shape(); } } - for (indices_per_axis.slice()) |*idx| { + for (idx_per_axis) |idx| { stdx.debug.assert(idx.shape().canBroadcastTo(indices_shape), "gather indices can't be broadcasted together {any}", .{idx_per_axis}); - idx.* = idx.broad(indices_shape); } - LoweringCompatibility.preserveGatherFillSemantics(indices_per_axis.slice()); - var idx_batch_axes: Shape.DimsArray = .empty; + var idx_batch_axes: Shape.DimsArray = .{}; var self_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{ .buffer = @splat(.offset), .len = self.rank() }; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { - const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, @intCast(self_ax), 1); + const is_gather_axis = std.mem.containsAtLeastScalar(u3, idx_axes, 1, @intCast(self_ax)); if (indices_shape.hasTag(t)) |id_ax| { // tag is both in self and indices -> it's a batching dim // Note: tags are required for batching. @@ -2250,13 +766,13 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, // compute res shape var res_shape = Shape.init(.{}, self.dtype()); - var res_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .empty; + var res_kind: stdx.BoundedArray(GatherAxisKind, constants.MAX_RANK) = .{}; for (self_kind.slice(), 0..) |kind, ax_usize| { const ax: u3 = @intCast(ax_usize); if (ax == idx_axes[0]) { // The first val_ax is special cause this is the place where we insert indices axes. for (0.., indices_shape.tags(), indices_shape.dims()) |id_axis_order, id_axis, id_inserted_dim| { - const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), @intCast(id_axis_order), 1); + const is_batching_axis = std.mem.containsAtLeastScalar(i64, idx_batch_axes.constSlice(), 1, @intCast(id_axis_order)); // Batching axis is already in self. if (is_batching_axis) continue; @@ -2277,15 +793,10 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, // Sometimes the backend recognize this pattern, but not always. // So let us handle that. if (indices_shape.count() == 1 and idx_axes.len == 1) { - return self - .slice(idx_axes[0], .dyn(indices_per_axis.get(0).asScalar(), 1)) - // Keep downstream remeshe after the slice. SPMD engines like Neuron otherwise - // may hoist an all-gather before the slice and materialize the full tensor. - .optimizationBarrier() - .reshape(res_shape); + return self.dynamicSlice1d(idx_axes[0], .{ .start = idx_per_axis[0].asScalar(), .len = 1 }).reshape(res_shape); } - var slice_dims: Shape.DimsArray = .empty; + var slice_dims: Shape.DimsArray = .{}; for (self_kind.slice(), self.dims()) |k, d| { slice_dims.appendAssumeCapacity(switch (k) { .batching, .collapsed => 1, @@ -2295,9 +806,9 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, } // TODO: try changing .last by other axis and see the perf impact. - const indices = Tensor.stack(indices_per_axis.constSlice(), .last, .coord); + const indices = Tensor.stack(idx_per_axis, .last, .coord); // scoped_log.debug("gather --> {} {any}", .{ res_shape, res_kind.constSlice() }); - const gather_op = stablehlo.gather( + const gather_op = dialects.stablehlo.gather( mlir_ctx, self.value(), indices.value(), @@ -2311,127 +822,16 @@ pub fn gather(self: Tensor, idx_axes: []const u3, idx_per_axis: []const Tensor, .index_vector_dim = indices.axis(.coord), .indices_are_sorted = opts.indices_are_sorted, }, - compiler.location, - ).appendTo(compiler.currentScope().block); + .unknown(mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); const mlir_shape = Tensor.fromMlirValue(gather_op.result(0)).shape(); stdx.debug.assert(mlir_shape.eql(res_shape), "gather expects that batching indices appear in the same order in 'self' and 'indices', got: self={f}, indices={f}. You should transpose one or the other.", .{ self, indices }); return Tensor._result(res_shape, gather_op.result(0)); } -/// Backend lowering compatibility shims live here so public tensor/model code -/// can keep expressing StableHLO semantics directly. -/// Each rewrite should -/// preserve the source operation's semantics while avoiding a backend compiler -/// or runtime path that currently mishandles that legal StableHLO. -/// Fill/drop indexed ops use max-value integer sentinels for inactive lanes; -/// those lanes must not issue indirect memory accesses after lowering. -pub const LoweringCompatibility = struct { - /// Keep scalar integer broadcasts as scalar broadcasts until StableHLO broadcast emission. - /// - /// Some backend compilers fold rank-0 integer scalars into full-rank dense - /// splats too early, which can perturb later indexed-update lowering. A - /// scalar optimization barrier preserves the source-level broadcast shape. - pub fn preserveIntegerScalarBroadcast(self: Tensor, output_shape: Shape) ?Tensor { - switch (Compiler.current().platform.target) { - .neuron => {}, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => return null, - } - - if (self.rank() != 0 or output_shape.rank() == 0 or !self.dtype().isInteger()) return null; - return self.optimizationBarrier().broadcast(output_shape, &.{}); - } - - /// Preserve gather fill semantics before backend indirect-memory lowering. - /// - /// Frontends represent inactive gather rows by passing an out-of-bounds - /// integer sentinel together with fill semantics. StableHLO says those rows - /// should produce the fill value, so the sentinel lane is not a real memory - /// access. Some backend lowerings still route every lane through a gather/DGE - /// path before applying fill behavior, so sanitize inactive lanes early while - /// keeping the active rows unchanged. Related upstream Neuron report: - /// https://github.com/aws-neuron/aws-neuron-sdk/issues/1335. - pub fn preserveGatherFillSemantics(indices: []Tensor) void { - switch (Compiler.current().platform.target) { - .neuron => {}, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => return, - } - - const active_lanes = activeLanesForFillDropIndices(indices) orelse return; - replaceInactiveIndirectIndices(indices, active_lanes); - } - - /// Preserve scatter drop semantics before backend indirect-memory lowering. - /// Same as above but for scatter drop semantics. - pub fn preserveScatterDropSemantics(indices: []Tensor, updates: anytype, opts: Tensor.ScatterOpts, update_values: *[updates.len]*const mlir.Value) void { - const active_lanes: ?Tensor = switch (Compiler.current().platform.target) { - .neuron => if (opts.update_fn == Tensor.ScatterOpts.increment) activeLanesForFillDropIndices(indices) else null, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => null, - }; - - if (active_lanes) |active| replaceInactiveIndirectIndices(indices, active); - - inline for (updates, 0..) |update, i| { - const update_tensor = if (active_lanes) |active| blk: { - const mask = active.broad(update.shape().withDtype(.bool)); - const zero = Tensor.constant(update.dtype().zero()).broad(update.shape()); - break :blk mask.select(update, zero); - } else update; - update_values[i] = update_tensor.value(); - } - } - - /// Builds the lane mask used by fill/drop compatibility rewrites. - /// - /// For normal index widths, the max representable value is the inactive - /// sentinel. For 64-bit indices, the affected lowering path narrows to the - /// i32 indirect-index domain, so only values that remain valid after that - /// narrowing are considered active. When multiple index tensors form a - /// coordinate, a lane is active only if every component is active. - fn activeLanesForFillDropIndices(indices: []const Tensor) ?Tensor { - var active: ?Tensor = null; - for (indices) |idx| { - if (!idx.dtype().isInteger()) continue; - - const idx_active = switch (idx.dtype()) { - .i64 => blk: { - const zero = Tensor.scalar(@as(i64, 0), .i64).broad(idx.shape()); - const max_i32 = Tensor.scalar(@as(i64, std.math.maxInt(i32)), .i64).broad(idx.shape()); - break :blk idx.cmp(.GE, zero).logical(.AND, idx.cmp(.LE, max_i32)); - }, - .u64 => blk: { - const max_i32 = Tensor.scalar(@as(u64, std.math.maxInt(i32)), .u64).broad(idx.shape()); - break :blk idx.cmp(.LE, max_i32); - }, - inline .i2, .i4, .i8, .i16, .i32, .u2, .u4, .u8, .u16, .u32 => blk: { - const sentinel = Tensor.constant(idx.dtype().maxValue()).broad(idx.shape()); - break :blk idx.cmp(.NE, sentinel); - }, - .bool, .f4e2m1, .f8e3m4, .f8e4m3, .f8e4m3b11fnuz, .f8e4m3fn, .f8e4m3fnuz, .f8e5m2, .f8e5m2fnuz, .f8e8m0, .bf16, .f16, .f32, .f64, .c64, .c128 => unreachable, - }; - active = if (active) |current| current.logical(.AND, idx_active) else idx_active; - } - return active; - } - - /// Replaces inactive sentinel indices with a safe index before indirect lowering. - /// - /// The replacement index is intentionally zero: it is always in-bounds for - /// non-empty operands and the corresponding scatter update is masked away by - /// the caller. This keeps the lowered indirect operation from touching the - /// sentinel address while preserving fill/drop semantics at the tensor level. - fn replaceInactiveIndirectIndices(indices: []Tensor, active_lanes: Tensor) void { - for (indices) |*idx| { - if (!idx.dtype().isInteger()) continue; - - const safe_index = Tensor.constant(idx.dtype().zero()).broad(idx.shape()); - idx.* = active_lanes.select(idx.*, safe_index); - } - } -}; - pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_RANK), value: T) stdx.BoundedArray(i64, constants.MAX_RANK) { - var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; for (bounded_array.constSlice(), 0..) |v, ax| { if (v == value) { res.appendAssumeCapacity(@intCast(ax)); @@ -2440,17 +840,16 @@ pub fn _collectAxes(T: type, bounded_array: stdx.BoundedArray(T, constants.MAX_R return res; } -fn CustomCallResultTypeFromOutputSpec(comptime OutputSpecT: type) type { - const type_info = @typeInfo(OutputSpecT); +fn TensorOrTensorArray(comptime T: type) type { + const type_info = @typeInfo(T); return switch (type_info) { - .void => void, .@"struct" => |struct_info| b: { - if (OutputSpecT == Shape) break :b Tensor; - if (!struct_info.is_tuple) @compileError("Expected tuple output shape spec"); - break :b if (struct_info.field_names.len == 1) + if (T == Tensor) break :b Tensor; + if (!struct_info.is_tuple) @compileError("Expected tuple"); + break :b if (struct_info.fields.len == 1) Tensor else - [struct_info.field_names.len]Tensor; + [struct_info.fields.len]Tensor; }, .array => |array_info| b: { break :b if (array_info.len == 1) @@ -2459,156 +858,19 @@ fn CustomCallResultTypeFromOutputSpec(comptime OutputSpecT: type) type { [array_info.len]Tensor; }, .pointer => |pointer_info| b: { - if (pointer_info.size != .slice) @compileError("Expected output shape slice"); + if (pointer_info.size != .slice) @compileError("Expected slice"); break :b []Tensor; }, - else => @compileError("Unsupported customCall output shape spec type: " ++ @typeName(OutputSpecT)), + else => @compileError("Unsupported type: " ++ @typeName(T)), }; } pub const CustomCallOptions = struct { has_side_effect: bool, - output_operand_aliases: ?[]const stablehlo.CustomCallOpts.OutputOperandAlias = null, - compute_on_host: bool = false, + output_operand_aliases: ?[]const i64 = null, }; -fn customCallAdditionalAttributes(ctx: *Compiler, opts: CustomCallOptions) []const mlir.NamedAttribute { - if (!opts.compute_on_host or ctx.platform.target == .cpu) return &.{}; - - const frontend_attributes = mlir.Attribute.dict(ctx.mlir_ctx, &.{ - .named(ctx.mlir_ctx, "_xla_compute_type", .string(ctx.mlir_ctx, "host")), - }); - - const additional_attributes = ctx.arena.allocator().alloc(mlir.NamedAttribute, 1) catch unreachable; - additional_attributes[0] = .named(ctx.mlir_ctx, "mhlo.frontend_attributes", frontend_attributes); - return additional_attributes; -} - -pub fn customCallOutputOperandAliases( - comptime I: type, - comptime O: type, - comptime aliases: ?CustomCallOutputOperandAliases(I, O), -) ?[]const stablehlo.CustomCallOpts.OutputOperandAlias { - const aliases_ = aliases orelse return null; - - const output_field_names = @typeInfo(O).@"struct".field_names; - comptime var num_aliases = 0; - inline for (output_field_names) |field_name| { - if (@field(aliases_, field_name) != null) num_aliases += 1; - } - - const output_operand_aliases = comptime blk: { - var aliases_buffer: [num_aliases]stablehlo.CustomCallOpts.OutputOperandAlias = undefined; - var i = 0; - for (output_field_names, 0..) |field_name, output_index| { - if (@field(aliases_, field_name)) |operand| { - aliases_buffer[i] = .{ - .output_index = @intCast(output_index), - .operand_index = @intCast(@backingInt(operand)), - }; - i += 1; - } - } - break :blk aliases_buffer; - }; - - return &output_operand_aliases; -} - -pub const CompositeOpts = struct { - version: i32 = 0, - composite_attributes: []const mlir.NamedAttribute = &.{}, -}; - -pub fn composite( - name: [:0]const u8, - inputs: []const Tensor, - outputs: []const Shape, - comptime decomposition: anytype, - context: anytype, - opts: CompositeOpts, -) []Tensor { - const ctx = Compiler.current(); - const mlir_ctx = ctx.mlir_ctx; - - const decomp_name = ctx.allocPrint("{s}.impl_{d}", .{ name, ctx.nextCompositeId() }); - - { - const block_types = ctx.alloc(*const mlir.Type, inputs.len); - const block_locs = ctx.alloc(*const mlir.Location, inputs.len); - @memset(block_locs, ctx.unknown_location); - - for (inputs, 0..) |t, i| { - block_types[i] = mlirx.Type.rankedTensor(mlir_ctx, t.shape()); - } - - const block = mlir.Block.init(block_types, block_locs); - - const scope = ctx.pushBlock(block); - { - const arg_tensors = ctx.alloc(Tensor, inputs.len); - for (inputs, 0..) |t, i| { - arg_tensors[i] = Tensor._result(t.shape(), block.argument(i)); - } - - var result = @call(.auto, decomposition, .{ arg_tensors, context }); - const rtensors = ctx.alloc(Tensor, outputs.len); - meta.collectBuf((struct { - pub fn func_(t: Tensor) Tensor { - return t; - } - }).func_, {}, &result, rtensors); - - const rvals = ctx.alloc(*const mlir.Value, outputs.len); - for (rtensors, 0..) |t, i| { - rvals[i] = t.value(); - } - - _ = func.returns(mlir_ctx, rvals, ctx.location).appendTo(block); - } - scope.pop(); - - _ = func.func(mlir_ctx, .{ - .name = decomp_name, - .block = block, - .location = ctx.location, - .visibility = .private, - .verify = false, - }).appendTo(ctx.module.body()); - } - - const operand_values = ctx.alloc(*const mlir.Value, inputs.len); - for (inputs, 0..) |t, i| { - operand_values[i] = t.value(); - } - - const result_types = ctx.alloc(*const mlir.Type, outputs.len); - for (outputs, 0..) |s, i| { - result_types[i] = mlirx.Type.rankedTensor(mlir_ctx, s); - } - - const op = stablehlo.composite( - mlir_ctx, - operand_values, - result_types, - .{ - .name = name, - .decomposition = decomp_name, - .composite_attributes = opts.composite_attributes, - .version = opts.version, - }, - ctx.location, - ).appendTo(ctx.currentScope().block); - - const out_tensors = ctx.alloc(Tensor, outputs.len); - for (outputs, 0..) |s, i| { - out_tensors[i] = Tensor._result(s, op.result(i)); - } - - return out_tensors; -} - -pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) CustomCallResultTypeFromOutputSpec(@TypeOf(outputs)) { +pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, metadata: anytype, opts: CustomCallOptions) TensorOrTensorArray(@TypeOf(outputs)) { // Transform generic inputs to flat slice. const inputs_: []const Tensor = switch (@typeInfo(@TypeOf(inputs))) { .@"struct" => |struct_info| b: { @@ -2616,12 +878,12 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, break :b &[1]Tensor{inputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple"); - var inputs_: [struct_info.field_names.len]Tensor = undefined; + var inputs_: [struct_info.fields.len]Tensor = undefined; meta.collectBuf((struct { - pub fn func_(t: Tensor) Tensor { + pub fn func(t: Tensor) Tensor { return t; } - }).func_, {}, &inputs, &inputs_); + }).func, {}, &inputs, &inputs_); break :b &inputs_; }, .array => &inputs, @@ -2634,18 +896,17 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, // Transform generic outputs to flat slice. const output_shapes: []const Shape = switch (@typeInfo(@TypeOf(outputs))) { - .void => &.{}, .@"struct" => |struct_info| b: { if (@TypeOf(outputs) == Shape) { break :b &[1]Shape{outputs}; } if (!struct_info.is_tuple) @compileError("Expected tuple"); - var output_shapes: [struct_info.field_names.len]Shape = undefined; + var output_shapes: [struct_info.fields.len]Shape = undefined; meta.collectBuf((struct { - pub fn func_(t: Shape) Shape { + pub fn func(t: Shape) Shape { return t; } - }).func_, {}, &outputs, &output_shapes); + }).func, {}, &outputs, &output_shapes); break :b &output_shapes; }, .array => &outputs, @@ -2656,31 +917,21 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, else => @compileError("Unsupported type: " ++ @typeName(@TypeOf(outputs))), }; - const outputs_flat = typedCustomCall( - target_name, - opts, - inputs_, - output_shapes, - metadata, - ); - if (comptime CustomCallResultTypeFromOutputSpec(@TypeOf(outputs)) == void) { - stdx.debug.assert(outputs_flat.len == 0, "customCall expected zero outputs for void return, got {}", .{outputs_flat.len}); - return; - } + const outputs_flat = customCallInternal(target_name, inputs_, output_shapes, metadata, opts); // Transform flat slice to generic outputs. return switch (@typeInfo(@TypeOf(outputs))) { .@"struct" => |struct_info| b: { if (@TypeOf(outputs) == Shape) break :b outputs_flat[0]; if (!struct_info.is_tuple) @compileError("Expected tuple"); - if (struct_info.field_names.len == 1) break :b outputs_flat[0]; - var outputs_: [struct_info.field_names.len]Tensor = undefined; + if (struct_info.fields.len == 1) break :b outputs_flat[0]; + var outputs_: [struct_info.fields.len]Tensor = undefined; @memcpy(&outputs_, outputs_flat); break :b outputs_; }, .array => |array_info| b: { if (array_info.len == 1) break :b outputs_flat[0]; - var outputs_: [array_info.len]Tensor = undefined; + var outputs_: [array_info.fields.len]Tensor = undefined; @memcpy(&outputs_, outputs_flat); break :b outputs_; }, @@ -2692,358 +943,123 @@ pub fn customCall(target_name: [:0]const u8, inputs: anytype, outputs: anytype, }; } -fn metadataIntegerFieldToMlirAttribute(mlir_ctx: *mlir.Context, int_field: std.builtin.Type.Int, value: anytype) *const mlir.Attribute { - return switch (int_field.signedness) { - .signed => switch (int_field.bits) { - 8 => .int(mlir_ctx, .i8, value), - 16 => .int(mlir_ctx, .i16, value), - 32 => .int(mlir_ctx, .i32, value), - 64 => .int(mlir_ctx, .i64, value), - else => @panic("Unsupported DataType"), - }, - .unsigned => switch (int_field.bits) { - 8 => .int(mlir_ctx, .u8, value), - 16 => .int(mlir_ctx, .u16, value), - 32 => .int(mlir_ctx, .u32, value), - 64 => .int(mlir_ctx, .u64, value), - else => @panic("Unsupported DataType"), - }, - }; -} +fn customCallInternal(target_name: [:0]const u8, inputs: []const Tensor, outputs: []const Shape, metadata: anytype, opts: CustomCallOptions) []Tensor { + const ctx = CompilationContext.current(); + var arena = std.heap.ArenaAllocator.init(ctx.allocator); + defer arena.deinit(); -fn metadataFloatFieldToMlirAttribute(mlir_ctx: *mlir.Context, float_field: std.builtin.Type.Float, value: anytype) *const mlir.Attribute { - return switch (float_field.bits) { - 16 => .float(mlir_ctx, .f16, @as(f64, value)), - 32 => .float(mlir_ctx, .f32, @as(f64, value)), - 64 => .float(mlir_ctx, .f64, @as(f64, value)), - else => @panic("Unsupported DataType"), - }; -} + const values = arena.allocator().alloc(*const mlir.Value, inputs.len) catch unreachable; + for (0..inputs.len) |i| { + values[i] = inputs[i].value(); + } -fn metadataFieldToMlirAttribute(mlir_ctx: *mlir.Context, comptime T: type, value: anytype) ?*const mlir.Attribute { - const type_info = @typeInfo(T); - return switch (type_info) { - .comptime_int => .int(mlir_ctx, .u64, @as(u64, value)), - .@"enum" => |enum_field| switch (@typeInfo(enum_field.tag_type)) { - .int => |int_tag| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_tag, @backingInt(value)), - else => @compileError("Unsupported tag type for enum metadata: " ++ @typeName(enum_field.tag_type)), - }, - .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, value), - .float => |float_field| metadataFloatFieldToMlirAttribute(mlir_ctx, float_field, value), - .bool => .boolean(mlir_ctx, value), - .pointer => |pointer_info| switch (pointer_info.size) { - .slice => if (pointer_info.child == u8) - .string(mlir_ctx, value) - else - @panic("Unsupported pointer type in metadata"), - else => @panic("Unsupported pointer type in metadata"), - }, - .optional => |optional_info| if (value) |wrapped_value| switch (@typeInfo(optional_info.child)) { - .int => |int_field| metadataIntegerFieldToMlirAttribute(mlir_ctx, int_field, wrapped_value), - .float => |float_field| metadataFloatFieldToMlirAttribute(mlir_ctx, float_field, wrapped_value), - .bool => .boolean(mlir_ctx, wrapped_value), - else => @panic("Unsupported optional child type in metadata"), - } else null, - else => @compileError("Unsupported metadata type: " ++ @typeName(T)), - }; -} + const res_types = arena.allocator().alloc(*const mlir.Type, outputs.len) catch unreachable; + for (outputs, 0..) |output, i| { + res_types[i] = mlir.rankedTensorType(output.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, output.dtype())); + } -pub fn CustomCall( - // A, I and O are assumed to be simple struct without any nesting like: { a: zml.Tensor } - I: type, - O: type, - A: type, - // Receives the custom call buffers for the input and outputs a struct with the same fields as I and O - // and the provided attributes. So if I is `{a: zml.Tensor}`, the first argument is `{a: CustomCallBuffer}` - // (TensorToCustomCallBuffer(i), ShapeToCustomCallBuffer(O), A) -> !?*pjrt.ffi.Error - comptime func_: anytype, - comptime params: struct { - /// Name of the custom call, must be unique. - name: [:0]const u8, - /// Whether the function has any side-effect, meaning that XLA cannot re-order it/optimize it out. A typical example is print. - has_side_effect: bool, - /// Similar to reuseBuffer on tensors, tells XLA that the custom call re-uses an input buffer for its output. - /// Use the input/output field names to express the mapping: `{ . = . }` - output_operand_aliases: ?CustomCallOutputOperandAliases(I, O) = null, - /// Request XLA host-compute offload for this custom call on accelerator backends. - /// Input/Output buffers will automatically be moved d2h/h2d without the need for an explicit `.toMemory(...)`. - compute_on_host: bool = false, - }, -) type { - return struct { - pub const InputTensors = I; - pub const OutputShapes = O; - pub const Attributes = A; - - /// Register the custom call so you can use it. - pub fn register(platform: *const Platform) !void { - try platform.registerFfi(.{ - .name = params.name, - .platform_name = if (params.compute_on_host) "Host" else null, - .handler = @This().handler, - .traits = .{ - .command_buffer_compatible = true, + const metadata_type_info = @typeInfo(@TypeOf(metadata)); + var metadata_attributes: [metadata_type_info.@"struct".fields.len]mlir.NamedAttribute = undefined; + inline for (metadata_type_info.@"struct".fields, 0..) |field, i| { + const attribute: *const mlir.Attribute = switch (@typeInfo(field.type)) { + .comptime_int => mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @field(metadata, field.name))), + .int => |int_field| switch (int_field.signedness) { + .signed => switch (int_field.bits) { + 8 => mlir.integerAttribute(ctx.mlir_ctx, .i8, @field(metadata, field.name)), + 16 => mlir.integerAttribute(ctx.mlir_ctx, .i16, @field(metadata, field.name)), + 32 => mlir.integerAttribute(ctx.mlir_ctx, .i32, @field(metadata, field.name)), + 64 => mlir.integerAttribute(ctx.mlir_ctx, .i64, @field(metadata, field.name)), + else => @panic("Unsupported DataType"), }, - }); - } - - pub fn call(input_tensors: I, output_shapes: O, attributes: A) ShapeToTensor(O) { - const opts: CustomCallOptions = .{ - .has_side_effect = params.has_side_effect, - .output_operand_aliases = comptime customCallOutputOperandAliases(I, O, params.output_operand_aliases), - .compute_on_host = params.compute_on_host, - }; - return typedCustomCall(params.name, opts, input_tensors, output_shapes, attributes); - } - - fn handler(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error { - if (call_frame.registeringHook()) return null; - - const input_buffers: TensorToCustomCallBuffer(I), const output_buffers: ShapeToCustomCallBuffer(O), const attributes: A = customCallArgsFromPjrtCallFrame(I, O, A, call_frame); - return func_(call_frame, input_buffers, output_buffers, attributes) catch |err| - pjrt.ffi.Error.create(call_frame.api, .unknown, @errorName(err)); - } - }; -} - -pub fn ShapeToTensor(T: type) type { - return meta.MapRestrict(Shape, Tensor).map(T); -} - -pub fn CustomCallOutputOperandAliases(I: type, O: type) type { - const field_names = @typeInfo(O).@"struct".field_names; - return @Struct( - .auto, - null, - field_names, - &@splat(?std.meta.FieldEnum(I)), - &@splat(.{ .default_value_ptr = &@as(?std.meta.FieldEnum(I), null) }), - ); -} - -pub fn typedCustomCall( - target_name: [:0]const u8, - opts: CustomCallOptions, - input: anytype, - output: anytype, - attributes: anytype, -) ShapeToTensor(@TypeOf(output)) { - const Input = @TypeOf(input); - const Output = @TypeOf(output); - const Attributes = @TypeOf(attributes); - - const ctx = Compiler.current(); - const allocator = ctx.arena.allocator(); - - stdx.debug.assert(ctx.manual_computation_depth > 0, "customCall '{s}' must be emitted inside manualComputation", .{target_name}); - - const input_tensors: []const Tensor = switch (@typeInfo(Input)) { - .@"struct" => |struct_info| b: { - var input_tensors: [struct_info.field_names.len]Tensor = undefined; - inline for (struct_info.field_names, 0..) |field_name, i| { - input_tensors[i] = @field(input, field_name); - } - break :b &input_tensors; - }, - .pointer => |pointer_info| b: { - if (pointer_info.size != .slice) @compileError("Expected input slice"); - break :b input; - }, - else => @compileError("Unsupported input type: " ++ @typeName(Input)), - }; - - const output_shapes: []const Shape = switch (@typeInfo(Output)) { - .@"struct" => |struct_info| b: { - var output_shapes: [struct_info.field_names.len]Shape = undefined; - inline for (struct_info.field_names, 0..) |field_name, i| { - output_shapes[i] = @field(output, field_name); - } - break :b &output_shapes; - }, - .pointer => |pointer_info| b: { - if (pointer_info.size != .slice) @compileError("Expected input slice"); - break :b output; - }, - else => @compileError("Unsupported output type: " ++ @typeName(Output)), - }; - - var metadata_attribute_list = switch (@typeInfo(Attributes)) { - .void => std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, 2) catch unreachable, - .@"struct" => |struct_info| std.ArrayList(mlir.NamedAttribute).initCapacity(allocator, struct_info.field_names.len + 2) catch unreachable, - else => @compileError("Unsupported type: " ++ @typeName(Attributes)), - }; - if (@typeInfo(Attributes) == .@"struct") { - inline for (@typeInfo(Attributes).@"struct".field_names, @typeInfo(Attributes).@"struct".field_types) |field_name, Field| { - if (metadataFieldToMlirAttribute(ctx.mlir_ctx, Field, @field(attributes, field_name))) |attribute| { - metadata_attribute_list.appendAssumeCapacity(mlir.NamedAttribute.named(ctx.mlir_ctx, field_name, attribute)); - } - } - } - metadata_attribute_list.appendSliceAssumeCapacity(&[_]mlir.NamedAttribute{ - .named(ctx.mlir_ctx, "pjrt_api", .int(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), - .named(ctx.mlir_ctx, "pjrt_client", .int(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), - }); - - const values = allocator.alloc(*const mlir.Value, input_tensors.len) catch unreachable; - const input_shapes = allocator.alloc(Shape, input_tensors.len) catch unreachable; - for (input_tensors, 0..) |input_tensor, i| { - values[i] = input_tensor.value(); - input_shapes[i] = input_tensor.shape(); + .unsigned => switch (int_field.bits) { + 8 => mlir.integerAttribute(ctx.mlir_ctx, .u8, @field(metadata, field.name)), + 16 => mlir.integerAttribute(ctx.mlir_ctx, .u16, @field(metadata, field.name)), + 32 => mlir.integerAttribute(ctx.mlir_ctx, .u32, @field(metadata, field.name)), + 64 => mlir.integerAttribute(ctx.mlir_ctx, .u64, @field(metadata, field.name)), + else => @panic("Unsupported DataType"), + }, + }, + .float => |float_field| switch (float_field.bits) { + 16 => mlir.floatAttribute(ctx.mlir_ctx, .f16, @as(f64, @field(metadata, field.name))), + 32 => mlir.floatAttribute(ctx.mlir_ctx, .f32, @as(f64, @field(metadata, field.name))), + 64 => mlir.floatAttribute(ctx.mlir_ctx, .f64, @as(f64, @field(metadata, field.name))), + else => @panic("Unsupported DataType"), + }, + .bool => mlir.boolAttribute(ctx.mlir_ctx, @field(metadata, field.name)), + .pointer => |pointer_info| switch (pointer_info.size) { + .slice => if (pointer_info.child == u8) + mlir.stringAttribute(ctx.mlir_ctx, @field(metadata, field.name)) + else + @panic("Unsupported pointer type in metadata"), + else => @panic("Unsupported pointer type in metadata"), + }, + else => @compileError("Unsupported metadata type: " ++ @typeName(field.type)), + }; + metadata_attributes[i] = mlir.NamedAttribute.named(ctx.mlir_ctx, field.name, attribute); } - const result_types = allocator.alloc(*const mlir.Type, output_shapes.len) catch unreachable; - for (output_shapes, 0..) |shape, i| { - result_types[i] = .rankedTensor(shape.dims(), mlirx.Type.fromDType(ctx.mlir_ctx, shape.dtype())); + const backend_config = mlir.dictionaryAttribute(ctx.mlir_ctx, &(metadata_attributes ++ [_]mlir.NamedAttribute{ + .named(ctx.mlir_ctx, "pjrt_api", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_api))))), + .named(ctx.mlir_ctx, "pjrt_client", mlir.integerAttribute(ctx.mlir_ctx, .u64, @as(u64, @bitCast(@intFromPtr(ctx.platform.pjrt_client))))), + })); + + const operands_layouts = arena.allocator().alloc([]const usize, inputs.len) catch unreachable; + for (inputs, 0..) |input, i| { + operands_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(input.rank())).constSlice()) catch unreachable; } - const backend_config: *const mlir.Attribute = .dict(ctx.mlir_ctx, metadata_attribute_list.items); - - const operand_layouts = allocator.alloc([]const usize, input_shapes.len) catch unreachable; - for (input_shapes, 0..) |shape, i| { - operand_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; + const results_layouts = arena.allocator().alloc([]const usize, outputs.len) catch unreachable; + for (outputs, 0..) |output, i| { + results_layouts[i] = arena.allocator().dupe(usize, toUsize(constants.minorToMajor(output.rank())).constSlice()) catch unreachable; } - const result_layouts = allocator.alloc([]const usize, output_shapes.len) catch unreachable; - for (output_shapes, 0..) |shape, i| { - result_layouts[i] = allocator.dupe(usize, toUsize(constants.minorToMajor(shape.rank())).constSlice()) catch unreachable; - } - - const op = stablehlo.custom_call( + const op = dialects.stablehlo.custom_call( ctx.mlir_ctx, values, - result_types, + res_types, .{ .call_target_name = target_name, .backend_config = .{ .typed_ffi = backend_config }, .has_side_effect = opts.has_side_effect, - .operand_layouts = operand_layouts, - .result_layouts = result_layouts, + .operand_layouts = operands_layouts, + .result_layouts = results_layouts, .output_operand_aliases = opts.output_operand_aliases orelse &.{}, - .additional_attributes = customCallAdditionalAttributes(ctx, opts), }, - ctx.location, - ).appendTo(ctx.currentScope().block); + .unknown(ctx.mlir_ctx), + ).appendTo(CompilationContext.current().currentScope().block); - if (ctx.partitioner == .gspmd) { - op.setAttributeByName("mhlo.sharding", .string(ctx.mlir_ctx, "{manual}")); + const outputs_ = ctx.arena.allocator().alloc(Tensor, outputs.len) catch unreachable; + for (outputs, 0..) |output, i| { + outputs_[i] = Tensor._result(output, op.result(i)); } - switch (@typeInfo(Output)) { - .@"struct" => |struct_info| { - var out: ShapeToTensor(Output) = undefined; - inline for (output_shapes, struct_info.field_names, 0..) |output_shape, field_name, i| { - @field(out, field_name) = Tensor._result(output_shape, op.result(i)); - } - return out; - }, - .pointer => |pointer_info| { - if (pointer_info.size != .slice) @compileError("Expected input slice"); - var out: []Tensor = allocator.alloc(Tensor, output_shapes.len) catch unreachable; - for (output_shapes, 0..) |output_shape, i| { - out[i] = Tensor._result(output_shape, op.result(i)); - } - return out; - }, - else => @compileError("Unsupported output type: " ++ @typeName(Output)), - } -} - -fn customCallArgsFromPjrtCallFrame(I: type, O: type, A: type, call_frame: *pjrt.ffi.CallFrame) struct { - TensorToCustomCallBuffer(I), - ShapeToCustomCallBuffer(O), - A, -} { - var input: TensorToCustomCallBuffer(I) = undefined; - inline for (@typeInfo(I).@"struct".field_names, 0..) |field_name, i| { - const buf = call_frame.args.buffers()[i]; - @field(input, field_name) = .fromPjrt(buf); - } - - var output: ShapeToCustomCallBuffer(O) = undefined; - inline for (@typeInfo(O).@"struct".field_names, 0..) |field_name, i| { - const shape_buf = call_frame.results.buffers()[i]; - @field(output, field_name) = .fromPjrt(shape_buf); - } - - const attributes: A = switch (@typeInfo(A)) { - .void => {}, - .@"struct" => blk: { - var attrs: A = undefined; - inline for (@typeInfo(A).@"struct".field_names, @typeInfo(A).@"struct".field_types) |field_name, Field| { - // TODO: Use getByIndex instead? - const attribute = call_frame.attrs.getByName(.scalar, field_name) orelse - @panic("Attribute not found: " ++ field_name); - @field(attrs, field_name) = attribute.get(Field); - } - break :blk attrs; - }, - else => @compileError("Unsupported attributes type: " ++ @typeName(A)), - }; - - return .{ input, output, attributes }; + return outputs_; } test customCall { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const tp = zml.testing.tpMesh(); - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{ .meshes = &.{tp} }); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - const scope = comp.pushBlock(block); - defer scope.pop(); + comp.pushBlock(block); + defer comp.popBlock(); - const shape = zml.Shape.init(.{128}, .bf16).withPartitioning(tp, .{ ._0 = .model }); - const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(shape); - const Local = struct { - fn body(x: Tensor, output_shape: Shape) Tensor { - return customCall("my_custom_call", .{x}, .{output_shape}, .{}, .{ - .has_side_effect = false, - .output_operand_aliases = &.{.{ .output_index = 0, .operand_index = 0 }}, - }); - } - }; - const output = manualComputation(Local.body, input, shape, tp, .{ - .in_specs = .parse(.{ ._0 = .model }), - .out_specs = .parse(.{ ._0 = .model }), + const input = Tensor.constant(zml.DataType.bf16.constant(0)).broad(Shape.init(.{128}, .bf16)); + const output = customCall("my_custom_call", .{input}, .{zml.Shape.init(.{128}, .bf16)}, .{}, .{ + .has_side_effect = false, + .output_operand_aliases = &.{0}, }); try zml.testing.expectEqualShapes(input.shape(), output.shape()); } -test "CustomCallOutputOperandAliases maps named fields to indices" { - const Input = struct { - q: Tensor, - k: Tensor, - v: Tensor, - }; - const Output = struct { - attn: Shape, - scratch: Shape, - }; - - const aliases = comptime customCallOutputOperandAliases(Input, Output, .{ - .attn = .q, - .scratch = .v, - }).?; - - try std.testing.expectEqual(@as(usize, 2), aliases.len); - try std.testing.expectEqualDeep( - stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 0, .operand_index = 0 }, - aliases[0], - ); - try std.testing.expectEqualDeep( - stablehlo.CustomCallOpts.OutputOperandAlias{ .output_index = 1, .operand_index = 2 }, - aliases[1], - ); -} - fn toUsize(values: anytype) stdx.BoundedArray(usize, constants.MAX_RANK) { - var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .empty; + var res: stdx.BoundedArray(usize, constants.MAX_RANK) = .{}; for (values) |val| res.appendAssumeCapacity(@intCast(val)); return res; } diff --git a/zml/ops/manual_computation.zig b/zml/ops/manual_computation.zig deleted file mode 100644 index a0ec0037..00000000 --- a/zml/ops/manual_computation.zig +++ /dev/null @@ -1,546 +0,0 @@ -const std = @import("std"); - -const mlir = @import("mlir"); -const shardy = @import("mlir/dialects/shardy"); -const stablehlo = @import("mlir/dialects/stablehlo"); -const stdx = @import("stdx"); - -const Compiler = @import("../Compiler.zig"); -const meta = @import("../meta.zig"); -const mlirx = @import("../mlirx.zig"); -const Shape = @import("../shape.zig").Shape; -const Sharding = @import("../Sharding.zig"); -const Tensor = @import("../tensor.zig").Tensor; - -const log = std.log.scoped(.@"zml/Compiler"); - -/// Partitioning declarations for the tensors in `T`. -/// Each `Tensor` becomes a `Sharding.Partitioning.Logical`. -/// For example, `struct { x: Tensor, bias: ?Tensor, scale: f32 }` becomes -/// `struct { x: Logical, bias: ?Logical }`. -/// -/// Initialize individual fields to declare different shardings, or use -/// `partitionSpecs` to apply one spec to every tensor in a value. -pub fn PartitionSpecs(comptime T: type) type { - return meta.MapRestrict(Tensor, Sharding.Partitioning.Logical).map(T); -} - -/// Declares the same sharding for a group of tensors at a `manualComputation` -/// boundary. -/// -/// ```zig -/// .in_specs = .{ -/// .query = .parse(.{ .h = .model }), -/// .cache = partitionSpecs(cache, .parse(.{ .h = .model })), -/// .metadata = partitionSpecs(metadata, .replicated), -/// } -/// ``` -/// -/// Requires an active compiler and uses its arena for allocations. Use -/// `partitionSpecsAlloc` outside graph construction. Use named struct fields -/// when mixing tensors with non-tensor metadata. -pub fn partitionSpecs(tensors: anytype, spec: Sharding.Partitioning.Logical) PartitionSpecs(@TypeOf(tensors)) { - return partitionSpecsAlloc(Compiler.current().arena.allocator(), tensors, spec) catch @panic("OOM"); -} - -/// Builds the same declarations as `partitionSpecs` using an explicit allocator. -pub fn partitionSpecsAlloc(allocator: std.mem.Allocator, tensors: anytype, spec: Sharding.Partitioning.Logical) std.mem.Allocator.Error!PartitionSpecs(@TypeOf(tensors)) { - var specs: PartitionSpecs(@TypeOf(tensors)) = undefined; - try meta.mapAlloc(struct { - fn assign(spec_: Sharding.Partitioning.Logical, _: Tensor) Sharding.Partitioning.Logical { - return spec_; - } - }.assign, allocator, spec, tensors, &specs); - return specs; -} - -/// Describes how the inputs and outputs of `body_fn` are partitioned across a mesh. -/// Returns the type of the `partitioning` argument to `manualComputation`. -/// -/// * `in_specs` specifies the sharding of each tensor in the body's first argument. -/// It follows the argument's structure, omitting non-tensor metadata. -/// * `out_specs` specifies the sharding of each shape in the body's second argument. -/// It follows the output shapes' structure and can be omitted for a `void` argument. -/// -/// Each spec is a `Sharding.Partitioning.Logical`. A mapping such as -/// `.parse(.{ .hidden = .model })` splits the tensor's `.hidden` dimension across -/// the mesh's `.model` axis. Omitted dimensions are replicated; `.replicated` -/// replicates the entire tensor. The specs must not contain open dimensions. -/// Slice lengths, optional presence and active union tags must match the arguments. -/// -/// For a body taking `struct { x: Tensor, bias: Tensor, scale: f32 }` and a single -/// output `Shape`, the partitioning can be declared as: -/// -/// ```zig -/// const partitioning: ManualPartitioning(body) = .{ -/// .in_specs = .{ .x = .parse(.{ .hidden = .model }), .bias = .replicated }, -/// .out_specs = .parse(.{ .hidden = .model }), -/// }; -/// ``` -pub fn ManualPartitioning(comptime body_fn: anytype) type { - const Logical = Sharding.Partitioning.Logical; - - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - - const InputSpecs = PartitionSpecs(Inputs); - const OutputSpecs = meta.MapRestrict(Shape, Logical).map(OutputShapes); - - if (OutputSpecs == void) return struct { - in_specs: InputSpecs, - out_specs: void = {}, - }; - - return struct { - in_specs: InputSpecs, - out_specs: OutputSpecs, - }; -} - -/// Describes a computation on local shards of tensors distributed across `mesh`. -/// ZML operations normally describe work on global tensors. The partitioner -/// distributes the inputs according to `in_specs`, and `body_fn` describes the work -/// performed on each shard. The results are exposed to the surrounding graph as -/// global tensors with the sharding declared by `out_specs`. -/// -/// * `body_fn` must be `fn (inputs: I, output_shapes: O) R`, where: -/// - `I` contains the local input tensors and any metadata needed by the body. -/// - `O` is `void`, `Shape`, or a container of shapes describing the local outputs. -/// - `R` is `void`, `Tensor`, or a container of tensors matching those local shapes -/// in traversal order. -/// * `inputs` contains the global tensors and metadata. The partitioner reshards -/// the tensors as needed to satisfy `in_specs`; metadata is passed to the body unchanged. -/// * `output_shapes` contains the global output shapes. The body receives their -/// local shard shapes, computed using `out_specs` and the sizes of the mesh axes. -/// * `mesh` is the logical mesh whose axis names are referenced by the specs. -/// * `partitioning` declares the input and output shardings; see `ManualPartitioning`. -/// Each partitioned dimension must be divisible by the size of its mesh axis. -/// -/// For example, splitting a dimension of size 64 across a mesh axis of size 8 gives -/// the body a dimension of size 8. A replicated dimension keeps its global size. -/// The caller supplies the global output shapes because the body only describes -/// local results: their shapes alone do not determine the global tensor shapes. -pub fn manualComputation( - comptime body_fn: anytype, - inputs: stdx.meta.FnParam(body_fn, 0), - output_shapes: stdx.meta.FnParam(body_fn, 1), - mesh: *const Sharding.Mesh, - partitioning: ManualPartitioning(body_fn), -) stdx.meta.FnReturn(body_fn) { - const Inputs = stdx.meta.FnParam(body_fn, 0); - const OutputShapes = stdx.meta.FnParam(body_fn, 1); - const Outputs = stdx.meta.FnReturn(body_fn); - - const compiler = Compiler.current(); - var arena_state: std.heap.ArenaAllocator = .init(compiler.allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - // The body may return containers borrowed from its inputs or output shapes. - const body_arena = compiler.arena.allocator(); - - var owned_inputs: Inputs = undefined; - meta.mapAlloc((struct { - fn copy(_: void, tensor: Tensor) Tensor { - return tensor; - } - }).copy, body_arena, {}, inputs, &owned_inputs) catch @panic("OOM"); - - const flat_input_tensors: []*Tensor = meta.collectMutPtrs(Tensor, arena, &owned_inputs) catch @panic("OOM"); - const in_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.in_specs) catch @panic("OOM"); - stdx.debug.assert(flat_input_tensors.len == in_specs.len, "manualComputation has {} input tensors but {} input specs", .{ flat_input_tensors.len, in_specs.len }); - - for (flat_input_tensors, in_specs) |tensor, spec| { - tensor._shape = tensor.shape().withPartitioning(mesh, spec); - stdx.debug.assert(!tensor.shape().containsPartitionSpec(.open), "manualComputation input sharding must be closed: {f}", .{tensor.shape()}); - } - - var local_output_shapes: OutputShapes = undefined; - meta.mapAlloc((struct { - fn copy(_: void, shape: Shape) Shape { - return shape; - } - }).copy, body_arena, {}, output_shapes, &local_output_shapes) catch @panic("OOM"); - - const flat_local_output_shapes = meta.collectMutPtrs(Shape, arena, &local_output_shapes) catch @panic("OOM"); - const out_specs = meta.collectValues(Sharding.Partitioning.Logical, arena, &partitioning.out_specs) catch @panic("OOM"); - stdx.debug.assert(flat_local_output_shapes.len == out_specs.len, "manualComputation has {} output shapes but {} output specs", .{ flat_local_output_shapes.len, out_specs.len }); - - const flat_output_shapes = arena.alloc(Shape, flat_local_output_shapes.len) catch @panic("OOM"); - for (flat_output_shapes, flat_local_output_shapes, out_specs) |*global, local, spec| { - global.* = local.withPartitioning(mesh, spec); - stdx.debug.assert(!global.containsPartitionSpec(.open), "manualComputation output sharding must be closed: {f}", .{global}); - local.* = Sharding.shardedShape(global.*) catch std.debug.panic("can't shard output {f} for {f}", .{ global, mesh }); - } - - const WrappedInputs = struct { - inputs: *const Inputs, - output_shapes: *const OutputShapes, - - fn wrap(allocator: std.mem.Allocator, opaque_wrapper: *const anyopaque, opaque_outputs: *anyopaque) error{OutOfMemory}![]*Tensor { - const wrapped: *const @This() = @ptrCast(@alignCast(opaque_wrapper)); - const outputs_ptr: *Outputs = @ptrCast(@alignCast(opaque_outputs)); - - outputs_ptr.* = body_fn(wrapped.inputs.*, wrapped.output_shapes.*); - - return try meta.collectMutPtrs(Tensor, allocator, outputs_ptr); - } - }; - - var outputs_struct: Outputs = undefined; - var wrapped_inputs: WrappedInputs = .{ .inputs = &owned_inputs, .output_shapes = &local_output_shapes }; - - _ = manualComputationInternal( - compiler, - arena, - mesh, - @typeName(@TypeOf(body_fn)), - &wrapped_inputs, - flat_input_tensors, - @ptrCast(&outputs_struct), - flat_output_shapes, - WrappedInputs.wrap, - ) catch |err| switch (err) { - error.OutOfMemory => @panic("OOM"), - }; - - return outputs_struct; -} - -const LocalComputationFn = *const fn (std.mem.Allocator, *const anyopaque, *anyopaque) error{OutOfMemory}![]*Tensor; - -/// Reified version of manualComputation. -/// This one is compiled exactly once, user code just need to compile the `collectPtrs` -noinline fn manualComputationInternal( - compiler: *Compiler, - arena: std.mem.Allocator, - mesh: *const Sharding.Mesh, - functor_name: []const u8, - input_struct: *anyopaque, - input_tensors: []*Tensor, - output_struct: *anyopaque, - global_output_shapes: []const Shape, - local_fn: LocalComputationFn, -) error{OutOfMemory}![]*Tensor { - const global_scope = compiler.currentScope(); - const mlir_ctx = compiler.mlir_ctx; - - log.debug("manualComputation({s}) will use mesh {s}", .{ functor_name, mesh.name }); - - const global_input_shapes = try arena.alloc(Shape, input_tensors.len); - for (global_input_shapes, input_tensors) |*shape, tensor| shape.* = tensor._shape; - - const input_types = try arena.alloc(*const mlir.Type, input_tensors.len); - for (input_types, input_tensors) |*input_ty, tensor_ptr| { - const global = tensor_ptr._shape; - const local = Sharding.shardedShape(global) catch std.debug.panic("can't shard {f} for {f}", .{ global, mesh }); - log.debug("\t- {f} -> {f}", .{ global, local }); - input_ty.* = mlirx.Type.rankedTensor(mlir_ctx, local); - } - const input_locs = try arena.alloc(*const mlir.Location, input_tensors.len); - @memset(input_locs, compiler.unknown_location); - - const input_values = try arena.alloc(*const mlir.Value, input_tensors.len); - for (input_values, input_tensors) |*input_val, tensor_ptr| { - if (compiler.partitioner == .gspmd) { - tensor_ptr.* = tensor_ptr.withPartitioning(mesh, tensor_ptr.shape()._sharding.partition); - } - input_val.* = global_scope.value(tensor_ptr.*); - } - - const manual_block = switch (compiler.partitioner) { - .shardy => mlir.Block.init(input_types, input_locs), - .gspmd => global_scope.block, - }; - errdefer if (compiler.partitioner == .shardy) manual_block.deinit(); - - const local_scope = compiler.pushBlock(manual_block); - defer local_scope.pop(); - for (0.., input_tensors, global_input_shapes) |i, tensor_ptr, global| { - const local = Sharding.shardedShape(global) catch unreachable; // we already computed this above - tensor_ptr.* = switch (compiler.partitioner) { - .shardy => Tensor._result(local, manual_block.argument(i)), - .gspmd => gspmd.globalToLocal(mlir_ctx, global_scope, tensor_ptr.*, local, compiler.location), - }; - } - - compiler.manual_computation_depth += 1; - defer compiler.manual_computation_depth -= 1; - - const local_outputs = try local_fn(arena, input_struct, output_struct); - stdx.debug.assert(local_outputs.len == global_output_shapes.len, "manualComputation body returned {} values, expected {}", .{ local_outputs.len, global_output_shapes.len }); - for (0.., local_outputs, global_output_shapes) |i, local_tensor, global_shape| { - const expected_local = Sharding.shardedShape(global_shape) catch std.debug.panic("can't shard output {f} for {f}", .{ global_shape, mesh }); - stdx.debug.assert(expected_local.eql(local_tensor._shape), "manualComputation body output {d} expected {f}, got {f}", .{ i, expected_local, local_tensor._shape }); - } - - const global_output_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); - for (global_output_shapes, 0..) |output_shape, i| { - global_output_types[i] = mlirx.Type.rankedTensor(mlir_ctx, output_shape); - } - - const local_output_values = try arena.alloc(*const mlir.Value, local_outputs.len); - for (local_output_values, local_outputs) |*output_val, tensor_ptr| { - output_val.* = local_scope.value(tensor_ptr.*); - } - - const op: *const mlir.Operation = switch (compiler.partitioner) { - .shardy => op: { - // Shardy creates a sdy.manual_computation block. - _ = shardy.return_(mlir_ctx, local_output_values, compiler.location).appendTo(manual_block); - - const out_mesh_attrs = try arena.alloc(*const shardy.TensorShardingAttr, global_output_shapes.len); - for (out_mesh_attrs, global_output_shapes) |*attr, shape| { - attr.* = try mesh.sdyShardingAttrForShape(arena, mlir_ctx, shape); - } - - break :op shardy.manual_computation( - mlir_ctx, - input_values, - global_output_types, - manual_block, - try Sharding.sdyPerValueShardingAttr(arena, mlir_ctx, global_input_shapes, mesh), - .get(mlir_ctx, out_mesh_attrs), - try Sharding.sdyManualAxesAttr(arena, mlir_ctx, global_input_shapes, global_output_shapes, mesh), - compiler.location, - ).appendTo(global_scope.block); - }, - .gspmd => op: { - // Gspmd calls SPMDShardToFullShape and wrap into an optimizationBarrier - // Shortcut to avoid empty optimization barrier - if (global_output_shapes.len == 0) return &.{}; - - const global_values = try arena.alloc(*const mlir.Value, global_output_shapes.len); - const global_types = try arena.alloc(*const mlir.Type, global_output_shapes.len); - for (global_output_shapes, 0..) |global_shape, i| { - global_values[i], global_types[i] = try gspmd.localToGlobal(compiler, arena, global_scope, local_output_values[i], global_shape); - } - - break :op stablehlo.optimizationBarrier( - mlir_ctx, - global_values, - global_types, - compiler.location, - ).appendTo(global_scope.block); - }, - }; - - // Returned tensors describe the operation's global output shapes and sharding. - for (0.., local_outputs, global_output_shapes) |o, output_tensor_ptr, output_shape| { - output_tensor_ptr.* = Tensor._result(output_shape, op.result(o)); - output_tensor_ptr._shape._sharding = output_shape._sharding; - } - return local_outputs; -} - -const gspmd = struct { - fn globalToLocal(mlir_ctx: *mlir.Context, scope: *Compiler.Scope, local_tensor: Tensor, local_shape: Shape, location: *const mlir.Location) Tensor { - const local_type = mlirx.Type.rankedTensor(mlir_ctx, local_shape); - const full_to_shard = stablehlo.custom_call( - mlir_ctx, - &.{scope.value(local_tensor)}, - &.{local_type}, - .{ - .call_target_name = "SPMDFullToShardShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(mlir_ctx, "mhlo.sharding", .string(mlir_ctx, "{manual}")), - }, - }, - location, - ).appendTo(scope.block); - return Tensor._result(local_shape, full_to_shard.result(0)); - } - - fn localToGlobal(compiler: *Compiler, arena: std.mem.Allocator, scope: *Compiler.Scope, local_value: *const mlir.Value, global_shape: Shape) !struct { *const mlir.Value, *const mlir.Type } { - const gspmd_attr = try compiler.tensorShardingAttr(arena, global_shape); - const global_type = mlirx.Type.rankedTensor(compiler.mlir_ctx, global_shape); - - const shard_to_full = stablehlo.custom_call( - compiler.mlir_ctx, - &.{local_value}, - &.{global_type}, - .{ - .call_target_name = "SPMDShardToFullShape", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(compiler.mlir_ctx, "mhlo.sharding", gspmd_attr), - }, - }, - compiler.location, - ).appendTo(scope.block); - return .{ shard_to_full.result(0), global_type }; - } -}; - -test manualComputation { - const zml = @import("../zml.zig"); - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - const mesh = zml.testing.tpMesh(); - - const LocalSum = struct { - x: Tensor, - bias: Tensor, - scale: i32, - - fn body(self: @This(), output_shape: Shape) Tensor { - return self.x.sum(.x).scale(self.scale).add(self.bias).broad(output_shape); - } - - fn forward(x: Tensor, mesh_: *const Sharding.Mesh) Tensor { - return manualComputation( - body, - .{ .x = x, .bias = .scalar(10, .i32), .scale = 2 }, - x.shape(), - mesh_, - .{ .in_specs = .{ .x = .parse(.{ .x = .model }), .bias = .replicated }, .out_specs = .parse(.{ .x = .model }) }, - ); - } - }; - - // Each shard sums two elements and broadcasts the result to its local output shape. - const shape: Shape = .init(.{ .x = 2 * mesh.numPartitionsForLogicalAxis(.model) }, .i32); - const values = try allocator.alloc(i32, shape.count()); - defer allocator.free(values); - - for (values, 0..) |*value, i| value.* = @intCast(i + 1); - - var input = try zml.Buffer.fromBytes(io, platform, shape, std.mem.sliceAsBytes(values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, LocalSum.forward, .{ .fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - const actual = try output.toSliceAlloc(allocator, io); - defer actual.free(allocator); - - for (actual.constItems(i32), 0..) |value, i| { - const pair = values[i / 2 * 2 ..][0..2]; - try std.testing.expectEqual((pair[0] + pair[1]) * 2 + 10, value); - } - - try std.testing.expect(output.shape().eqlWithTags(shape)); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } -} - -test "manualComputation composes across logical and physical meshes" { - const zml = @import("../zml.zig"); - if (comptime !zml.Target.cpu.isEnabled()) return error.SkipZigTest; - const allocator = std.testing.allocator; - const io = std.testing.io; - - const Topologies = enum { flat, torus }; - - const Model = struct { - fn increment(x: Tensor, _: Shape) Tensor { - return x.addConstant(1); - } - - fn forward(x: Tensor, mesh: *const Sharding.Mesh) Tensor { - const partitioning: ManualPartitioning(increment) = .{ - .in_specs = partitionSpecs(x, .parse(.{ .x = .model })), - .out_specs = .parse(.{ .x = .model }), - }; - - const first = manualComputation(increment, x, x.shape(), mesh, partitioning); - const converted = first.convert(.f32); - return manualComputation(increment, converted, converted.shape(), mesh, partitioning); - } - }; - - const shape: Shape = .init(.{ .x = 16 }, .i32); - - var values: [16]i32 = undefined; - var expected: [16]f32 = undefined; - for (&values, &expected, 0..) |*value, *result, i| { - value.* = @intCast(i); - result.* = @floatFromInt(i + 2); - } - - for ([_]Topologies{ .flat, .torus }) |topology| { - const platform = try zml.Platform.init(allocator, io, .cpu, .{ - .cpu = .{ .device_count = 8 }, - .physical_mesh = if (topology == .torus) .{ .custom = Sharding.PhysicalMesh.torus3d } else .auto, - }); - defer platform.deinit(allocator, io); - - // Fold three physical axes into one logical axis, or split one into three. - const logical: Sharding.LogicalMesh = if (topology == .torus) - try .init(.{.model}, &.{.balanced}, &.{8}) - else - try Sharding.LogicalMesh.init(.{ .data, .model, .pipeline }, &.{ .low_bandwidth, .high_bandwidth, .balanced }, &.{ 2, 2, 2 }); - - const mesh = try platform.registerMesh("manual", logical); - - var input: zml.Buffer = try .fromBytes(io, platform, shape, std.mem.asBytes(&values)); - defer input.deinit(); - - for ([_]Sharding.Partitioner{ .shardy, .gspmd }) |partitioner| { - const exe = try platform.compileFn(allocator, io, Model.forward, .{ Tensor.fromShape(shape), mesh }, .{ - .meshes = &.{ platform.replicated_mesh, mesh }, - .partitioner = partitioner, - }); - defer exe.deinit(); - - var output = try exe.eval(allocator, io, .{input}); - defer output.deinit(); - - try std.testing.expectEqual(expected, try output.getValue([16]f32, io)); - try std.testing.expect(output.shape().eqlWithTags(shape.withDtype(.f32))); - try std.testing.expectEqualDeep(shape.withPartitioning(mesh, .{ .x = .model })._sharding, output.shape()._sharding); - } - } -} - -test "partitionSpecs preserves optional tensors and active cache variants" { - const zml = @import("../zml.zig"); - const Logical = Sharding.Partitioning.Logical; - const Inputs = struct { - layers: []const Tensor, - bias: ?Tensor, - cache: union(enum) { disabled, values: [2]Tensor }, - scale: f32, - }; - - var arena: std.heap.ArenaAllocator = .init(std.testing.allocator); - defer arena.deinit(); - - const tensor: Tensor = .fromShape(.init(.{ .h = 8, .d = 16 }, .f32)); - var inputs: Inputs = .{ - .layers = &.{ tensor, tensor }, - .bias = null, - .cache = .{ .values = .{ tensor, tensor } }, - .scale = 2, - }; - const heads: Logical = .parse(.{ .h = .model }); - const specs = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, heads); - const expected: zml.ops.PartitionSpecs(Inputs) = .{ - .layers = &.{ heads, heads }, - .bias = null, - .cache = .{ .values = .{ heads, heads } }, - }; - try std.testing.expectEqualDeep(expected, specs); - - inputs.layers = &.{}; - inputs.bias = tensor; - inputs.cache = .disabled; - const replicated = try zml.ops.partitionSpecsAlloc(arena.allocator(), inputs, .replicated); - const expected_replicated: zml.ops.PartitionSpecs(Inputs) = .{ - .layers = &.{}, - .bias = .replicated, - .cache = .disabled, - }; - try std.testing.expectEqualDeep(expected_replicated, replicated); -} diff --git a/zml/pjrtx.zig b/zml/pjrtx.zig index 824d2bd1..6cb4ca3b 100644 --- a/zml/pjrtx.zig +++ b/zml/pjrtx.zig @@ -1,13 +1,10 @@ const std = @import("std"); +const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); const pjrt = @import("pjrt"); -const stablehlo = @import("mlir/dialects/stablehlo"); pub const DataType = @import("dtype.zig").DataType; -const meta = @import("meta.zig"); -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; const log = std.log.scoped(.@"zml/pjrtx"); @@ -15,21 +12,26 @@ pub const Client = opaque { pub const CompileError = std.mem.Allocator.Error || error{InvalidMlirBytecodeVersion} || pjrt.ApiError; pub fn compile(client: *const pjrt.Client, api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, module: *const mlir.Module, compile_options_pb: []const u8) CompileError!*pjrt.LoadedExecutable { - const stablehlo_version = blk: { - if (api.stablehloCurrentVersion()) |requested_version| { - break :blk stablehlo.smallerVersion(requested_version, stablehlo.currentVersion()); - } - break :blk stablehlo.minimumVersion(); + var bytecode: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); + defer bytecode.deinit(); + module.operation().writeBytecode(.{ .desired_emit_version = 1 }, &bytecode.writer) catch |err| { + log.err("failed to write module bytecode: {}", .{err}); + return switch (err) { + error.WriteFailed => error.OutOfMemory, + else => |e| e, + }; }; - const cloned_op = module.operation().clone(); - var cloned_module = mlir.Module.fromOperation(cloned_op); - defer cloned_module.deinit(); + const stablehlo_version = blk: { + if (api.stablehloCurrentVersion()) |requested_version| { + break :blk dialects.stablehlo.smallerVersion(requested_version, dialects.stablehlo.currentVersion()); + } + break :blk dialects.stablehlo.minimumVersion(); + }; var serialized_buffer: std.Io.Writer.Allocating = try .initCapacity(allocator, 4096); defer serialized_buffer.deinit(); - - stablehlo.serializePortableArtifact2(cloned_module, stablehlo_version, &serialized_buffer.writer) catch |err| { + dialects.stablehlo.serializePortableArtifact(bytecode.written(), stablehlo_version, &serialized_buffer.writer) catch |err| { log.err("failed to serialize to portable artifact: {}", .{err}); return switch (err) { std.Io.Writer.Error.WriteFailed => error.OutOfMemory, @@ -45,64 +47,6 @@ pub const Client = opaque { } }; -pub fn TensorToCustomCallBuffer(T: type) type { - return meta.MapRestrict(Tensor, CustomCallBuffer).map(T); -} - -pub fn ShapeToCustomCallBuffer(T: type) type { - return meta.MapRestrict(Shape, CustomCallBuffer).map(T); -} - -pub const CustomCallBuffer = struct { - shape: Shape, - ptr: *anyopaque, - - pub fn format( - self: @This(), - writer: *std.Io.Writer, - ) std.Io.Writer.Error!void { - try writer.print("{f}@{*}", .{ self.shape, self.ptr }); - } - - pub fn fromPjrt(buffer: *const pjrt.ffi.Buffer) @This() { - const dt = switch (buffer.dtype) { - .invalid => @panic("Found an invalid pjrt buffer"), - .token => @panic("Found an token pjrt buffer"), - inline else => |tag| @field(DataType, @tagName(tag)), - }; - return .{ - .ptr = buffer.data, - .shape = .init(buffer.dims(), dt), - }; - } - - // XLA: - // Align to 64-bytes, to mimic tsl::Allocator::kAllocatorAlignment. - // - // Preferred XLA:CPU alignment for buffers. XLA:CPU itself aligns intermediate - // buffers to this boundary (slices inside the preallocated temp buffer). - // https://github.com/openxla/xla/blob/fa2b7cfd909e8ffd343da1b7c5c341ace5e53aaa/xla/backends/cpu/alignment.h#L4 - pub const ALIGN = 64; - - // Assumes as host-side buffer - pub fn asHostSlice(self: CustomCallBuffer) []align(ALIGN) u8 { - const bytes: [*]align(ALIGN) u8 = @ptrCast(@alignCast(self.ptr)); - return bytes[0..self.shape.byteSize()]; - } - - // Assumes as host-side buffer - pub fn asHostConstSlice(self: CustomCallBuffer) []align(ALIGN) const u8 { - const bytes: [*]align(ALIGN) const u8 = @ptrCast(@alignCast(self.ptr)); - return bytes[0..self.shape.byteSize()]; - } - - // Assumes as host-side buffer - pub fn asHostScalar(self: CustomCallBuffer, T: type) T { - std.debug.assert(self.shape.byteSize() == @sizeOf(T)); - return std.mem.bytesAsValue(T, self.asHostConstSlice()[0..@sizeOf(T)]).*; - } -}; - pub fn bufferTypeFromDtype(dt: DataType) pjrt.BufferType { return switch (dt) { inline else => |tag| @field(pjrt.BufferType, @tagName(tag)), @@ -117,13 +61,13 @@ pub fn dtypeFromBufferType(pjrt_type: pjrt.BufferType) DataType { } test bufferTypeFromDtype { - inline for (@typeInfo(DataType).@"enum".field_values) |field_value| { - const dt: DataType = @fromBackingInt(@intCast(field_value)); + inline for (@typeInfo(DataType).@"enum".fields) |field| { + const dt: DataType = @enumFromInt(field.value); try std.testing.expectEqual(dt, dtypeFromBufferType(bufferTypeFromDtype(dt))); } - inline for (@typeInfo(pjrt.BufferType).@"enum".field_values) |field_value| { - const dt: pjrt.BufferType = @fromBackingInt(@intCast(field_value)); + inline for (@typeInfo(pjrt.BufferType).@"enum".fields) |field| { + const dt: pjrt.BufferType = @enumFromInt(field.value); if (dt == .invalid) continue; try std.testing.expectEqual(dt, bufferTypeFromDtype(dtypeFromBufferType(dt))); } diff --git a/zml/platform.zig b/zml/platform.zig index 54fb90e4..daa2f9fe 100644 --- a/zml/platform.zig +++ b/zml/platform.zig @@ -1,23 +1,38 @@ const std = @import("std"); -const builtin = @import("builtin"); const c = @import("c"); const pjrt = @import("pjrt"); +const platforms = @import("platforms"); +pub const Target = platforms.Platform; const stdx = @import("stdx"); -pub const Target = @import("platforms").Platform; -const constants = @import("constants.zig"); const Exe = @import("exe.zig").Exe; -const flashinfer_cutlass_contract = @import("kernels/moe/flashinfer_cutlass/contract.zig"); -const flashinfer_cutlass_runtime = @import("kernels/moe/flashinfer_cutlass/runtime.zig"); const pjrtx = @import("pjrtx.zig"); -const profiler_ = @import("profiling/profiler.zig"); -const Sharding = @import("Sharding.zig"); const zml = @import("zml.zig"); const log = std.log.scoped(.zml); -var api_map: std.enums.EnumArray(Target, ?*const pjrt.Api) = .initFill(null); +pub const CompilationOptions = struct { + xla_dump_to: ?[]const u8 = null, + xla_dump_fusion_visualization: bool = false, + xla_dump_hlo_pass_re: ?[]const u8 = null, + sharding_enabled: bool = false, + sharding_axes: stdx.BoundedArray([*:0]const u8, 8) = .{}, + device_memory_size: u64 = 0, +}; + +fn StaticPlatformMap(comptime E: type, comptime T: type) type { + const tag_count = @typeInfo(E).@"enum".fields.len; + var struct_field_names: [tag_count][]const u8 = undefined; + var struct_field_types: [tag_count]type = @splat(T); + const is_optional = @typeInfo(T) == .optional; + const default_value_ptr: ?*const anyopaque = if (is_optional) @ptrCast(&@as(T, null)) else null; + var struct_field_attrs: [tag_count]std.builtin.Type.StructField.Attributes = @splat(.{ .default_value_ptr = default_value_ptr }); + inline for (@typeInfo(E).@"enum".fields, 0..) |f, i| struct_field_names[i] = f.name; + return @Struct(.auto, null, &struct_field_names, &struct_field_types, &struct_field_attrs); +} + +var api_map: StaticPlatformMap(Target, ?*const pjrt.Api) = .{}; fn disableXlaLogs() void { // https://deepreg.readthedocs.io/en/latest/docs/logging.html#tensorflow-logging @@ -34,27 +49,12 @@ fn disableXlaLogs() void { ); } -fn validateDeviceCount(target: Target, num_devices: usize) !void { - if (num_devices == 0) { - log.err("The selected platform requires at least 1 device, got {}", .{num_devices}); - return error.MissingDevices; - } - switch (target) { - .cpu, .cuda, .rocm, .tpu, .neuron, .metal, .oneapi, .furiosa => { - if (!std.math.isPowerOfTwo(num_devices)) { - log.err("Platform {} requires a power-of-two device count, got {}", .{ target, num_devices }); - return error.InvalidDeviceCount; - } - }, - } -} - fn loadOrGetApi(allocator: std.mem.Allocator, io: std.Io, target: Target) !*const pjrt.Api { return switch (target) { - inline else => |tgt| api_map.get(tgt) orelse b: { + inline else => |tag| @field(api_map, @tagName(tag)) orelse b: { disableXlaLogs(); - const api = try tgt.load(allocator, io); - api_map.set(tgt, api); + const api = try platforms.load(allocator, io, tag); + @field(api_map, @tagName(tag)) = api; break :b api; }, }; @@ -72,9 +72,12 @@ pub const Memory = struct { platform: *const Platform, addressable_by_devices: []*const Device, - fn init(allocator: std.mem.Allocator, pjrt_memory: *const pjrt.Memory, platform: *Platform) !Memory { + fn init(allocator: std.mem.Allocator, pjrt_memory: *const pjrt.Memory, platform: *Platform, all_devices: []const Device) !Memory { const pjrt_addressable_by_devices = pjrt_memory.addressableByDevices(platform.pjrt_api); const addressable_by_devices = try allocator.alloc(*const Device, pjrt_addressable_by_devices.len); + for (pjrt_addressable_by_devices, addressable_by_devices) |pjrt_device, *addressable_by_device| { + addressable_by_device.* = &all_devices[pjrt_device.localHardwareId(platform.pjrt_api)]; + } return .{ .pjrt_memory = pjrt_memory, @@ -89,7 +92,7 @@ pub const Memory = struct { pub fn isOfKind(self: Memory, kind_: Kind) bool { switch (self.platform.target) { - .cuda, .rocm, .oneapi, .tpu, .furiosa => { + .cuda, .rocm => { const zml_kind: Memory.Kind = switch (self.kind().len) { "device".len => .device, "pinned_host".len => .host_pinned, @@ -98,20 +101,13 @@ pub const Memory = struct { }; return zml_kind == kind_; }, - .cpu, .neuron, .metal => return true, + .cpu, .tpu, .neuron => return true, } } fn deinit(self: *Memory, allocator: std.mem.Allocator) void { allocator.free(self.addressable_devices); } - - fn populateAddressableByDevices(self: *Memory) void { - const pjrt_addressable_by_devices = self.pjrt_memory.addressableByDevices(self.platform.pjrt_api); - for (pjrt_addressable_by_devices, self.addressable_by_devices) |pjrt_device, *addressable_by_device| { - addressable_by_device.* = self.platform.deviceFromPjrt(pjrt_device); - } - } }; pub const Device = struct { @@ -119,30 +115,19 @@ pub const Device = struct { pjrt_device: *const pjrt.Device, pjrt_desc: *const pjrt.DeviceDescription, addressable_memories: []*const Memory, - memory_by_kind: std.EnumArray(Memory.Kind, ?*const Memory), - fn init(allocator: std.mem.Allocator, pjrt_device_: *const pjrt.Device, platform: *const Platform) !Device { + fn init(allocator: std.mem.Allocator, pjrt_device_: *const pjrt.Device, platform: *const Platform, all_addressable_memories: []const Memory) !Device { const pjrt_addressable_memories = pjrt_device_.addressableMemories(platform.pjrt_api); const addressable_memories = try allocator.alloc(*const Memory, pjrt_addressable_memories.len); for (pjrt_addressable_memories, addressable_memories) |pjrt_memory, *addressable_memory| { - addressable_memory.* = platform.memoryFromPjrt(pjrt_memory); + addressable_memory.* = &all_addressable_memories[pjrt_memory.id(platform.pjrt_api)]; } - // Cache memory lookups since they are expensive - const default_memory: *const Memory = resolveDefaultMemory(pjrt_device_, platform, addressable_memories); - const memory_by_kind: std.EnumArray(Memory.Kind, ?*const Memory) = .init(.{ - .default = default_memory, - .device = resolveMemory(addressable_memories, .device), - .host_pinned = resolveMemory(addressable_memories, .host_pinned), - .host_unpinned = resolveMemory(addressable_memories, .host_unpinned), - }); - return .{ .platform = platform, .pjrt_device = pjrt_device_, .pjrt_desc = pjrt_device_.getDescription(platform.pjrt_api), .addressable_memories = addressable_memories, - .memory_by_kind = memory_by_kind, }; } @@ -150,27 +135,8 @@ pub const Device = struct { allocator.free(self.addressable_memories); } - fn resolveDefaultMemory(pjrt_device_: *const pjrt.Device, platform: *const Platform, addressable_memories: []*const Memory) *const Memory { - const pjrt_memory = pjrt_device_.defaultMemory(platform.pjrt_api); - for (addressable_memories) |mem| { - if (mem.pjrt_memory == pjrt_memory) return mem; - } - return platform.memoryFromPjrt(pjrt_memory); - } - - fn resolveMemory(addressable_memories: []*const Memory, memory_kind: Memory.Kind) ?*const Memory { - std.debug.assert(memory_kind != .default); - - for (addressable_memories) |mem| { - if (mem.isOfKind(memory_kind)) { - return mem; - } - } - return null; - } - - pub fn id(self: Device) u32 { - return @intCast(self.pjrt_desc.id(self.platform.pjrt_api)); + pub fn id(self: Device) i32 { + return self.pjrt_desc.id(self.platform.pjrt_api); } pub fn processIndex(self: Device) i32 { @@ -178,7 +144,7 @@ pub const Device = struct { } pub fn localHardwareId(self: Device) i32 { - return @intCast(self.pjrt_device.localHardwareId(self.platform.pjrt_api)); + return self.device.localHardwareId(self.platform.pjrt_api); } pub fn kind(self: Device) []const u8 { @@ -197,19 +163,6 @@ pub const Device = struct { return self.pjrt_device.memoryStats(self.platform.pjrt_api) catch .zeroes; } - /// NUMA node closest to the device, when the plugin reports one. - pub fn numaNode(self: Device) ?u32 { - const attributes = self.pjrt_device.attributes(self.platform.pjrt_api) catch return null; - defer attributes.deinit(); - - const value = attributes.get("numa_node") orelse return null; - - return switch (value) { - .int64 => |node| std.math.cast(u32, node), - else => null, - }; - } - pub fn format(self: Device, writer: *std.Io.Writer) std.Io.Writer.Error!void { try writer.print("{s} ({s})", .{ self.pjrt_desc.kind(self.platform.pjrt_api), @@ -217,98 +170,36 @@ pub const Device = struct { }); } - pub fn memory(self: *const Device, memory_kind: Memory.Kind) ?*const Memory { - return self.memory_by_kind.values[@backingInt(memory_kind)]; - } -}; - -fn platformDeviceSortId(target: Target, device: Device) usize { - return switch (target) { - .neuron => @intCast(device.localHardwareId()), - .cuda, .rocm, .tpu, .cpu, .oneapi, .metal, .furiosa => device.id(), - }; -} - -fn sortDevicesById(target: Target, devices: []Device) void { - const Context = struct { - target: Target, - - fn lessThan(ctx: @This(), lhs: Device, rhs: Device) bool { - return platformDeviceSortId(ctx.target, lhs) < platformDeviceSortId(ctx.target, rhs); + pub fn memory(self: Device, memory_kind: Memory.Kind) *const Memory { + if (memory_kind == .default) { + const mem = self.pjrt_device.defaultMemory(self.platform.pjrt_api); + return &self.platform.memories[mem.id(self.platform.pjrt_api)]; } - }; - std.mem.sort(Device, devices, Context{ .target = target }, Context.lessThan); - - if (builtin.mode == .debug) { - for (devices, 0..) |device, expected_id| { - std.debug.assert(platformDeviceSortId(target, device) == expected_id); - } - } -} - -// State union tagged on target platform to handle related resources -pub const State = union(Target) { - cpu: void, - cuda: CudaState, - rocm: void, - tpu: void, - neuron: void, - oneapi: void, - metal: void, - furiosa: void, - - pub const CudaState = struct { - flashinfer_cutlass_moe_runners: ?*flashinfer_cutlass_runtime.Runners = null, - - fn deinit(self: *CudaState) void { - if (self.flashinfer_cutlass_moe_runners) |runners| { - runners.deinit(); - self.flashinfer_cutlass_moe_runners = null; + for (self.addressable_memories) |mem| { + if (mem.isOfKind(memory_kind)) { + return mem; } } - }; - - pub fn init(target: Target) State { - return switch (target) { - .cpu => .{ .cpu = {} }, - .cuda => .{ .cuda = .{} }, - .rocm => .{ .rocm = {} }, - .tpu => .{ .tpu = {} }, - .neuron => .{ .neuron = {} }, - .oneapi => .{ .oneapi = {} }, - .metal => .{ .metal = {} }, - .furiosa => .{ .furiosa = {} }, - }; - } - - pub fn deinit(self: *State) void { - switch (self.*) { - .cuda => |*cuda_state| cuda_state.deinit(), - else => {}, - } + unreachable; } }; pub const Platform = struct { - arena: std.heap.ArenaAllocator, + arena_state: std.heap.ArenaAllocator.State = .{}, target: Target, pjrt_api: *const pjrt.Api, pjrt_client: *pjrt.Client, - state: State, devices: []const Device, memories: []const Memory, - physical_mesh: zml.Sharding.PhysicalMesh, - meshes: std.StringArrayHashMapUnmanaged(*const zml.Mesh), - replicated_mesh: *const zml.Mesh, - io_impl: IoImpl, - pub const MAX_NUM_DEVICES: u16 = if (Target.tpu.isEnabled()) 64 else 32; + // This make the pjrt struct quite fat, but is only used during compilation. + // TODO: Reconsider having it here, and maybe pass explicitly to compile, + // or create an intermediary struct: + // `const comp = platform.compiler(compile_opts); const exe = comp.compile(...);` + compilation_options: CompilationOptions = .{}, - pub const IoImpl = enum { - threaded, - zio, - }; + pub const MAX_NUM_DEVICES: u8 = if (platforms.isEnabled(.tpu)) 32 else 8; pub fn init(allocator: std.mem.Allocator, io: std.Io, target: Target, options: CreateOptions) !*Platform { const api = try loadOrGetApi(allocator, io, target); @@ -316,89 +207,61 @@ pub const Platform = struct { var named_values_buf: [16]pjrt.NamedValue = undefined; const pjrt_client = try pjrt.Client.init(api, options.toNamedValues(target, &named_values_buf)); const pjrt_devices = pjrt_client.addressableDevices(api); - try validateDeviceCount(target, pjrt_devices.len); if (pjrt_devices.len > MAX_NUM_DEVICES) { log.warn("platform {} got {} devices, but ZML only support up to {} devices. Some devices won't be used.", .{ target, pjrt_devices.len, MAX_NUM_DEVICES }); } + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + + const devices = try arena.allocator().alloc(Device, pjrt_devices.len); + const pjrt_memories = pjrt_client.addressableMemories(api); + const memories = try arena.allocator().alloc(Memory, pjrt_memories.len); - // Note: Platform is a self-owning struct. It contains the arena that created it in the first place - // But it does mean we have to be careful to pass the arena state that contains the node - const platform: *Platform = platform: { - var initial_arena = std.heap.ArenaAllocator.init(allocator); - errdefer initial_arena.deinit(); - - var initial_allocator = initial_arena.allocator(); - // Pre-heat the arena, to avoid fragmentation. - initial_allocator.free(try initial_allocator.alloc(u8, 8 * 1024)); - - const platform = try initial_allocator.create(Platform); - platform.* = .{ - .arena = initial_arena, - .target = target, - .pjrt_api = api, - .pjrt_client = pjrt_client, - .state = State.init(target), - .meshes = .empty, - .replicated_mesh = undefined, - // set below - .devices = undefined, - .memories = undefined, - .physical_mesh = undefined, - .io_impl = options.io_impl, - }; - break :platform platform; + const platform = try arena.allocator().create(Platform); + platform.* = .{ + .target = target, + .pjrt_api = api, + .pjrt_client = pjrt_client, + .compilation_options = .{}, + .devices = devices, + .memories = memories, }; - - const arena = platform.arena.allocator(); - errdefer platform.arena.deinit(); - try platform.meshes.ensureTotalCapacity(arena, 8); + defer platform.arena_state = arena.state; { - const devices = try arena.alloc(Device, pjrt_devices.len); - platform.devices = devices; - const memories = try arena.alloc(Memory, pjrt_memories.len); - platform.memories = memories; - - // TODO: part of the complication here is that we layout the data in spaghetti mode, - // where devices and memories point to each other and also point to the platform. - for (pjrt_memories, memories) |pjrt_memory, *platform_memory| { - platform_memory.* = try .init(arena, pjrt_memory, platform); - } for (pjrt_devices, devices) |pjrt_device, *platform_device| { - platform_device.* = try .init(arena, pjrt_device, platform); + platform_device.* = try .init(arena.allocator(), pjrt_device, platform, memories); } - sortDevicesById(target, devices); - for (memories) |*platform_memory| { - platform_memory.populateAddressableByDevices(); + for (pjrt_memories, memories) |pjrt_memory, *platform_memory| { + platform_memory.* = try .init(arena.allocator(), pjrt_memory, platform, devices); } - - platform.physical_mesh = try switch (options.physical_mesh) { - .auto => zml.Sharding.PhysicalMesh.auto(arena, target, devices), - .custom => |builder| builder(arena, target, devices), - }; - platform.replicated_mesh = try platform.registerMesh("replicated", .mesh(.{ .all = .high_bandwidth })); } switch (target) { .cuda => { - cuda.initFlashinferCutlass(platform, io) catch |err| { - log.warn("Failed to initialize Flashinfer CUTLASS MoE: {}", .{err}); + zml.attention.flashattn.load(arena.allocator(), io) catch { + log.warn("Failed to load flashattn", .{}); + }; + zml.attention.flashattn.register(platform) catch { + log.warn("Failed to register flashattn custom call", .{}); }; }, else => {}, } - platform.registerFfi( - .{ - .name = "zml$print", - .handler = printCallback, - .traits = .{ .command_buffer_compatible = false }, - }, - ) catch |err| { - log.warn("Failed to register FFI custom call \"zml$print\", error: {}", .{err}); - }; + if (platform.pjrt_api.ffi()) |ffi| b: { + const platform_name = switch (target) { + .cuda => "cuda", + .rocm => "rocm", + .cpu => "host", + else => break :b, + }; + ffi.register(platform.pjrt_api, "zml$print", platform_name, printCallback, .{ .command_buffer_compatible = false }) catch |e| { + log.warn("Failed to register \"print\" callback, error: {}", .{e}); + }; + } return platform; } @@ -409,9 +272,6 @@ pub const Platform = struct { .neuron, .rocm, .cuda, - .oneapi, - .metal, - .furiosa, .cpu, }; return for (ordered_targets) |target| { @@ -419,14 +279,14 @@ pub const Platform = struct { } else error.Unavailable; } - pub fn formatWithAttributes(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { + pub fn formatWithDevices(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { const tee = "├─ "; const line = "│ "; const langle = "└─ "; try writer.print("platform: {s}\n", .{@tagName(self.target)}); try writer.print("version: {f}\n", .{self.pjrt_api.version()}); - + try writer.print("stablehlo_version: {?s}\n", .{self.pjrt_api.stablehloCurrentVersion()}); try writer.print("extensions:\n", .{}); { var it = self.pjrt_api.extensions(); @@ -435,115 +295,42 @@ pub const Platform = struct { } } - try writer.print("plugin attributes:\n", .{}); - { - const attributes = self.pjrt_api.pluginAttributes(); - if (attributes.len == 0) { - try writer.print("{s}(none)\n", .{langle}); - } else { - for (attributes, 0..) |attr, i| { - const is_last_attr = i == attributes.len - 1; - try writer.print("{s}{s}", .{ if (is_last_attr) langle else tee, attr.name() }); - switch (attr.value()) { - .string => |v| try writer.print(": \"{s}\"", .{v}), - .int64 => |v| try writer.print(": {d}", .{v}), - .int64list => |v| { - for (v, 0..) |item, j| { - if (j == 0) { - try writer.print(": {d}", .{item}); - } else { - try writer.print(".{d}", .{item}); - } - } - }, - .float => |v| try writer.print(": {d}", .{v}), - .bool => |v| try writer.print(": {}", .{v}), - } - try writer.writeAll("\n"); - } - } - } - try writer.print("devices:\n", .{}); for (self.devices, 0..) |device, i| { - const is_last_device = i == self.devices.len - 1; - const child_indent = if (is_last_device) " " else line; - - try writer.print("{s}{f}\n", .{ if (is_last_device) langle else tee, device }); - - { - const device_attrs = device.pjrt_desc.attributes(self.pjrt_api); - if (device_attrs.len > 0) { - var last_name: ?[]const u8 = null; - var remaining: usize = device_attrs.len; - - while (remaining > 0) : (remaining -= 1) { - var next_index: ?usize = null; - for (device_attrs, 0..) |attr, j| { - const name = attr.name(); - if (last_name) |last| { - if (std.mem.order(u8, name, last) != .gt) continue; - } - if (next_index) |ni| { - if (std.mem.order(u8, name, device_attrs[ni].name()) == .lt) { - next_index = j; - } - } else { - next_index = j; - } - } - - if (next_index == null) break; - const attr = device_attrs[next_index.?]; - - try writer.print("{s}{s}{s}", .{ child_indent, tee, attr.name() }); - switch (attr.value()) { - .string => |v| try writer.print(": \"{s}\"", .{v}), - .int64 => |v| try writer.print(": {d}", .{v}), - .int64list => |v| try writer.print(": {any}", .{v}), - .float => |v| try writer.print(": {d}", .{v}), - .bool => |v| try writer.print(": {}", .{v}), - } - try writer.writeAll("\n"); - - last_name = attr.name(); - } - } - } - - try writer.print("{s}{s}memories:\n", .{ child_indent, langle }); - { - const memory_indent = " "; - if (device.addressable_memories.len == 0) { - try writer.print("{s}{s}{s}(none)\n", .{ child_indent, memory_indent, langle }); - } else { - for (device.addressable_memories, 0..) |mem, j| { - const is_last_mem = j == device.addressable_memories.len - 1; - try writer.print("{s}{s}{s}memory: {s}\n", .{ - child_indent, - memory_indent, - if (is_last_mem) langle else tee, - mem.pjrt_memory.debugString(self.pjrt_api), - }); - } - } + const is_last_device = i < self.devices.len - 1; + try writer.print("{s}{f}\n", .{ if (is_last_device) tee else langle, device }); + for (device.addressable_memories, 0..) |mem, j| { + try writer.print("{s}{s}memory: {s}\n", .{ + if (is_last_device) line else " ", + if (j < device.addressable_memories.len - 1) tee else langle, + mem.kind(), + }); } } } - pub fn fmtVerbose(self: *const Platform) std.fmt.Alt(*const Platform, formatWithAttributes) { + pub fn fmtVerbose(self: *const Platform) std.fmt.Alt(*const Platform, formatWithDevices) { return .{ .data = self }; } - pub fn deinit(self: *Platform, allocator: std.mem.Allocator, io: std.Io) void { - _ = io; - _ = allocator; - if (comptime Target.cuda.isEnabled()) { - self.state.deinit(); - } - self.physical_mesh.deinit(self.arena.allocator()); + pub const Sharding = struct { num_replicas: u8, num_partitions: u8 }; + + pub fn sharding(self: *const Platform) Sharding { + // replicas run the same function but with different inputs, + // while partitions contribute to one evaluation over a shared input. + // Inside an inference process, we generally don't want replicas, + // as it's best to fully isolate replicas on different processes. + // For now we hardcode num_replicas = 1. + const num_devices: u8 = @intCast(self.devices.len); + return if (self.compilation_options.sharding_enabled) + .{ .num_replicas = 1, .num_partitions = num_devices } + else + .{ .num_replicas = 1, .num_partitions = 1 }; + } + + pub fn deinit(self: *Platform, allocator: std.mem.Allocator) void { self.pjrt_client.deinit(self.pjrt_api); - self.arena.deinit(); + self.arena_state.promote(allocator).deinit(); } pub fn compile( @@ -555,14 +342,12 @@ pub const Platform = struct { args: stdx.meta.Tail( std.meta.ArgsTuple(@TypeOf(@field(@TypeOf(model_), @tagName(func)))), ), - opts: zml.Compiler.Options, ) !Exe { return self.compileFn( allocator, io, @field(@TypeOf(model_), @tagName(func)), .{model_} ++ args, - opts, ); } @@ -573,9 +358,8 @@ pub const Platform = struct { comptime func: anytype, model: stdx.meta.Head(std.meta.ArgsTuple(@TypeOf(func))), args: stdx.meta.Tail(std.meta.ArgsTuple(@TypeOf(func))), - opts: zml.Compiler.Options, ) !Exe { - return self.compileFn(allocator, io, func, .{model} ++ args, opts); + return self.compileFn(allocator, io, func, .{model} ++ args); } pub fn compileFn( @@ -584,9 +368,8 @@ pub const Platform = struct { io: std.Io, comptime func: anytype, args: std.meta.ArgsTuple(@TypeOf(func)), - opts: zml.Compiler.Options, ) !Exe { - return zml.Compiler.compile(allocator, io, self, func, args, opts); + return zml.module.compile(allocator, io, func, args, self); } pub fn format(self: *const Platform, writer: *std.Io.Writer) std.Io.Writer.Error!void { @@ -606,138 +389,18 @@ pub const Platform = struct { } unreachable; } - - pub const FfiRegistration = struct { - name: []const u8, - handler: *const pjrt.ffi.Handler, - traits: pjrt.ffi.HandlerTraits = .{ .command_buffer_compatible = false }, - platform_name: ?[]const u8 = null, - }; - - pub fn registerFfi(self: *const zml.Platform, registration: FfiRegistration) !void { - const platform_name = registration.platform_name orelse self.pjrt_client.platformName(self.pjrt_api); - if (self.pjrt_api.ffi()) |ffi| { - try ffi.register(self.pjrt_api, registration.name, platform_name, registration.handler, registration.traits); - } else { - log.warn("PJRT FFI extension not available for {s}", .{@tagName(self.target)}); - } - } - - pub const Profiler = profiler_.Profiler; - pub const ProfilerOptions = profiler_.ProfilerOptions; - - pub fn profiler(self: *const Platform, allocator: std.mem.Allocator, io: std.Io, options: ProfilerOptions) !Profiler { - return try profiler_.profiler(self.pjrt_api, allocator, io, options); - } - - /// Create a Mesh based on the given logical mesh and the default strategy. - /// Memory is owned by the platform, making it safe to copy around. - pub fn registerMesh(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh) error{OutOfMemory}!*const Sharding.Mesh { - return platform.registerMeshWithStrategy( - name, - logical, - .suggest(logical, &platform.physical_mesh), - ) catch |err| switch (err) { - error.InvalidPhysicalMesh, error.InvalidStrategy, error.InvalidPhysicalAxis, error.OverlappingLogicalAxes => { - std.debug.panic("ZML failed to create a valid mesh for logical mesh: {f}\nand physical_mesh: {f}\nPlease report this bug.", .{ logical, platform.physical_mesh }); - }, - error.InvalidPartialBinding, error.InvalidLogicalDimension, error.IncompatibleLogicalDimension => { - std.debug.panic("ZML cannot satisfy the dimensions of logical mesh {f} on physical mesh {f}", .{ logical, platform.physical_mesh }); - }, - error.OutOfMemory => |e| return e, - }; - } - - /// Create a Mesh based on the given logical mesh and a strategy. - /// Memory is owned by the platform, making it safe to copy around. - pub fn registerMeshWithStrategy(platform: *Platform, name: []const u8, logical: Sharding.LogicalMesh, strategy: Sharding.Strategy) !*const Sharding.Mesh { - const arena = platform.arena.allocator(); - const entry = try platform.meshes.getOrPut(arena, name); - if (entry.found_existing) { - std.debug.panic("Another mesh already exists with this name: {s}", .{name}); - } - - const owned_name = try arena.dupe(u8, name); - const owned_mesh = try arena.create(Sharding.Mesh); - owned_mesh.* = try .init(owned_name, &platform.physical_mesh, logical, strategy); - entry.key_ptr.* = owned_name; - entry.value_ptr.* = owned_mesh; - - return owned_mesh; - } - - fn memoryFromPjrt(self: *const Platform, pjrt_memory: *const pjrt.Memory) *const Memory { - for (self.memories) |*mem| { - if (mem.pjrt_memory == pjrt_memory) return mem; - } - unreachable; - } - - fn deviceFromPjrt(self: *const Platform, pjrt_device: *const pjrt.Device) *const Device { - for (self.devices) |*device| { - if (device.pjrt_device == pjrt_device) return device; - } - unreachable; - } - - pub inline fn defaultMemoryLayout(platform: *const Platform, dims: []const i64, dtype: zml.DataType) pjrt.MemoryLayout { - // inline cause `default` is a huge ass struct allocated on the stack, - // and toMemoryLayout returns slices into it. - // There is probably a better way of doing this, - // The queried layout can include backend-specific tiling. - return switch (platform.target) { - .tpu, .furiosa => { - if (comptime !Target.tpu.isEnabled() and !Target.furiosa.isEnabled()) unreachable; - const element_type = pjrtx.bufferTypeFromDtype(dtype); - const default = platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, element_type, dims) catch @panic("Failed to get default memory layout"); - return default.toMemoryLayout(); - }, - .cuda, .rocm, .neuron, .oneapi, .cpu, .metal => .{ - // If this is the default layout on the platform, there is no point calling PJRT - .tiled = .{ - .minor_to_major = constants.minorToMajor(@intCast(dims.len)), - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }, - }; - } }; pub const CreateOptions = struct { - pub const CreatePhysicalMeshFn = *const fn ( - allocator: std.mem.Allocator, - target: Target, - devices: []const Device, - ) anyerror!zml.Sharding.PhysicalMesh; - - pub const PhysicalMesh = union(enum) { - auto, - custom: CreatePhysicalMeshFn, - }; - - physical_mesh: PhysicalMesh = .auto, cpu: Cpu = .{ .device_count = 4 }, // bump memory fraction from XLA defaults of 75% to 90%. - // Even on a 8GB GPU it should leave enough space for the platform driver/runtime. + // Even on a 8GB GPU it should leave enough space for the Cuda driver // https://github.com/openxla/xla/blob/3e87afa11a865cf91137522492918ad18bfe5b7c/xla/pjrt/plugin/xla_gpu/xla_gpu_allocator_config.h#L25-L60 - rocm: XlaGpu = .{ .allocator = .{ .vmm = .{ .memory_fraction = 0.90 } } }, - cuda: XlaGpu = .{ .allocator = .{ .vmm = .{ .memory_fraction = 0.90 } } }, + cuda: Cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, + rocm: struct {} = .{}, tpu: struct {} = .{}, neuron: struct {} = .{}, - furiosa: Furiosa = .{}, - oneapi: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, - metal: XlaGpu = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.90 } } }, - io_impl: Platform.IoImpl = .threaded, - - pub const Furiosa = struct { - pe_count: u8 = 8, - - fn writeNamedValues(self: Furiosa, values: *std.ArrayList(pjrt.NamedValue)) void { - values.appendAssumeCapacity(.init(.int64, "pe_count", self.pe_count)); - } - }; pub const Cpu = struct { device_count: u32, @@ -747,11 +410,8 @@ pub const CreateOptions = struct { } }; - pub const XlaGpu = struct { + pub const Cuda = struct { allocator: Allocator = .{ .bfc = .{} }, - /// The PJRT C API still exposes this under legacy - /// `use_tfrt_gpu_client` name. - gpu_async_dispatch: bool = false, // TODO support all of https://github.com/openxla/xla/blob/3d31c48c719d331d432132b3e0c2c5ce52650675/xla/pjrt/c/pjrt_c_api_gpu_internal.cc#L76-L86 // visible_devices: []const i64 = &.{}, // node_id @@ -764,8 +424,6 @@ pub const CreateOptions = struct { bfc: Options, /// use cudaMallocAsync async: Options, - /// use virtual memory allocator - vmm: VmmOptions, /// use raw cuMalloc platform, @@ -774,13 +432,9 @@ pub const CreateOptions = struct { memory_fraction: f32 = 0.90, collective_memory_size_mb: i64 = 0, }; - - pub const VmmOptions = struct { - memory_fraction: f32 = 0.90, - }; }; - fn writeNamedValues(self: XlaGpu, target: Target, values: *std.ArrayList(pjrt.NamedValue)) void { + fn writeNamedValues(self: Cuda, values: *std.ArrayList(pjrt.NamedValue)) void { switch (self.allocator) { .platform => { values.appendAssumeCapacity(.init(.string, "allocator", "platform")); @@ -789,7 +443,7 @@ pub const CreateOptions = struct { values.appendAssumeCapacity(.init(.string, "allocator", switch (self.allocator) { .bfc => "bfc", .async => "cuda_async", - .platform, .vmm => unreachable, + .platform => unreachable, })); values.appendAssumeCapacity(.init(.bool, "preallocate", opt.preallocate)); if (opt.memory_fraction > 0) { @@ -799,16 +453,6 @@ pub const CreateOptions = struct { values.appendAssumeCapacity(.init(.int64, "collective_memory_size", opt.collective_memory_size_mb * 1024 * 1024)); } }, - .vmm => |opt| { - values.appendAssumeCapacity(.init(.string, "allocator", "vmm")); - if (opt.memory_fraction > 0) { - values.appendAssumeCapacity(.init(.float, "memory_fraction", opt.memory_fraction)); - } - }, - } - switch (target) { - .cuda => values.appendAssumeCapacity(.init(.bool, "use_tfrt_gpu_client", self.gpu_async_dispatch)), - else => {}, } } }; @@ -818,11 +462,7 @@ pub const CreateOptions = struct { values.shrinkRetainingCapacity(0); switch (target) { .cpu => self.cpu.writeNamedValues(&values), - .furiosa => self.furiosa.writeNamedValues(&values), - .cuda => self.cuda.writeNamedValues(target, &values), - .rocm => self.rocm.writeNamedValues(target, &values), - .oneapi => self.oneapi.writeNamedValues(target, &values), - .metal => self.metal.writeNamedValues(target, &values), + .cuda => self.cuda.writeNamedValues(&values), inline else => |t| { stdx.debug.assertComptime(@hasField(CreateOptions, @tagName(t)), "zml.platform.CreateOptions doesn't list target {s}", .{@tagName(t)}); const options = @field(self, @tagName(t)); @@ -835,136 +475,14 @@ pub const CreateOptions = struct { // TODO(Corendos): Consider moving that in its own file if its size increase too much. pub const cuda = struct { - pub const ComputeCapability = struct { - major: u8, - minor: u8, - - pub fn parse(text: []const u8) ?ComputeCapability { - var parts = std.mem.splitScalar(u8, text, '.'); - return .{ - .major = std.fmt.parseInt(u8, parts.first(), 10) catch return null, - .minor = std.fmt.parseInt(u8, parts.next() orelse "0", 10) catch return null, - }; - } - - pub fn eql(self: ComputeCapability, other: ComputeCapability) bool { - return self.major == other.major and self.minor == other.minor; - } - - pub fn atLeast(self: ComputeCapability, other: ComputeCapability) bool { - return self.major > other.major or (self.major == other.major and self.minor >= other.minor); - } - - pub fn sm(self: ComputeCapability) u16 { - return @as(u16, self.major) * 10 + self.minor; - } - }; - - /// SM count of an addressable CUDA device; unavailable attributes return null. - pub fn coreCount(platform: *const zml.Platform, device_index: usize) ?u32 { - const attribute = platform.devices[device_index].pjrt_desc.attribute(platform.pjrt_api, "core_count") orelse return null; - return std.math.cast(u32, attribute.int64); - } - - pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { - if (platform.target != .cuda) return null; - const devices = platform.pjrt_client.devices(platform.pjrt_api); - if (devices.len == 0) return null; - - const attributes = devices[0].getDescription(platform.pjrt_api).attributes(platform.pjrt_api); - return for (attributes) |attr| { - if (std.mem.eql(u8, attr.name(), "compute_capability")) { - break ComputeCapability.parse(attr.value().string); - } - } else null; - } - - fn initFlashinferCutlass(platform: *Platform, io: std.Io) !void { - if (comptime !Target.cuda.isEnabled()) return error.UnsupportedPlatform; - const cc = computeCapability(platform) orelse return error.UnsupportedArchitecture; - - const allocator = platform.arena.allocator(); - const runners = try allocator.create(flashinfer_cutlass_runtime.Runners); - errdefer allocator.destroy(runners); - - try flashinfer_cutlass_runtime.load(allocator, io, cc.sm()); - - runners.* = .{}; - errdefer runners.deinit(); - - try platform.registerFfi(.{ - .name = flashinfer_cutlass_contract.name, - .handler = flashinfer_cutlass_runtime.handler, - .traits = .{ .command_buffer_compatible = true }, - .platform_name = if (flashinfer_cutlass_contract.compute_on_host) "Host" else null, - }); - - platform.state.cuda.flashinfer_cutlass_moe_runners = runners; - } -}; - -pub const rocm = struct { - pub const ComputeCapability = enum { - /// CDNA1: MI100. - gfx908, - /// CDNA2: MI200 series. - gfx90a, - /// CDNA3: MI300 series. - gfx942, - /// CDNA4: MI350 series. - gfx950, - /// RDNA2: RX 6000 series. - gfx1030, - /// RDNA3 (Navi 31): RX 7900 series. - gfx1100, - /// RDNA3 (Navi 32): RX 7800 and RX 7700 series. - gfx1101, - /// RDNA3 (Navi 33): RX 7600 series. - gfx1102, - /// RDNA3 APU: Phoenix. - gfx1103, - /// RDNA3.5: newer Ryzen AI APUs. - gfx1150, - /// RDNA4 (Navi 44): RX 9060 family. - gfx1200, - /// RDNA4 (Navi 48): RX 9070 family. - gfx1201, - - pub const Architecture = enum { - cdna1, - cdna2, - cdna3, - cdna4, - rdna2, - rdna3, - rdna3_5, - rdna4, - }; - - pub fn architecture(self: ComputeCapability) Architecture { - return switch (self) { - .gfx908 => .cdna1, - .gfx90a => .cdna2, - .gfx942 => .cdna3, - .gfx950 => .cdna4, - .gfx1030 => .rdna2, - .gfx1100, .gfx1101, .gfx1102, .gfx1103 => .rdna3, - .gfx1150 => .rdna3_5, - .gfx1200, .gfx1201 => .rdna4, - }; - } - }; - - /// Assumes homogeneous devices. - pub fn computeCapability(platform: *const zml.Platform) ?ComputeCapability { - stdx.debug.assert(platform.target == .rocm, "computeCapability expects .rocm platform, got {}", .{platform.target}); - const device = platform.pjrt_client.devices(platform.pjrt_api)[0]; + pub fn tryGetComputeCapabilities(platform: *const zml.Platform, device: *const pjrt.Device) ?[]const u8 { + stdx.debug.assert(platform.target == .cuda, "tryGetComputeCapabilities expects .cuda platform, got {}", .{platform.target}); const description = device.getDescription(platform.pjrt_api); const attributes = description.attributes(platform.pjrt_api); return for (attributes) |attr| { if (std.mem.eql(u8, attr.name(), "compute_capability")) { - break std.meta.stringToEnum(ComputeCapability, std.mem.sliceTo(attr.value().string, ':')); + break attr.value().string; } } else null; } @@ -1015,15 +533,12 @@ fn printCallback(call_frame: *pjrt.ffi.CallFrame) callconv(.c) ?*pjrt.ffi.Error fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { if (call_frame.registeringHook()) return null; - const buffers = call_frame.args.buffers(); - if (buffers.len <= 0) return null; - const pjrt_api: *pjrt.Api = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_api").?); const pjrt_client: *pjrt.Client = @ptrFromInt(getScalarAttributeAs(u64, call_frame, "pjrt_client").?); const device_ordinal: usize = @intCast(try call_frame.ctx.getDeviceOrdinal(call_frame.api)); - const buffer = buffers[0]; + const buffer = call_frame.args.buffers()[0]; const shape = shapeFromFfiBuffer(buffer); // NOTE(Corentin): This is a hack. We take the first non device memory, hoping that it's host visible, @@ -1035,7 +550,7 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { } else return error.MemoryNotFound; var pjrt_buffer = try pjrt_client.createViewOfDeviceBuffer(pjrt_api, .{ - .device_buffer_ptr = buffer.data, + .data = buffer.data, .dims = shape.dims(), .element_type = pjrtx.bufferTypeFromDtype(shape.dtype()), .device = device, @@ -1047,51 +562,16 @@ fn printCallbackInner(call_frame: *pjrt.ffi.CallFrame) !?*pjrt.ffi.Error { }, }, }); - defer pjrt_buffer.deinit(pjrt_api); pjrt_buffer = try pjrt_buffer.copyToMemory(pjrt_api, first_non_device_memory); - - const event = pjrt_buffer.readyEvent(pjrt_api); - defer event.deinit(pjrt_api); - try event.awaitRaw(pjrt_api); + try pjrt_buffer.readyEvent(pjrt_api).awaitRaw(pjrt_api); const host_visible_data: [*]u8 = @ptrCast(@alignCast(try pjrt_buffer.opaqueDeviceMemoryDataPointer(pjrt_api))); const slice: zml.Slice = .init(shape, host_visible_data[0..shape.byteSize()]); const name = call_frame.attrs.getByName(.string, "name").?.slice(); - std.debug.print("{s} [device={d}]: {d}\n", .{ name, device_ordinal, slice }); + log.info("{s}: {d}", .{ name, slice }); return null; } - -test "platform defaultMemoryLayout is boring" { - const platform = zml.testing.env(); - - const shapes = [_][]const i64{ - &.{4096}, - &.{ 4096, 4096 }, - &.{ 4096, 4096, 4096 }, - }; - for (shapes) |dims| { - // Checks that the PJRT client always return the same thing than `platform.defaultMemoryLayout` - // This allows to bypass the PJRT calls and the string of pjrt_client.defaultMemoryLayout. - const default_layout = try platform.pjrt_client.defaultMemoryLayout(platform.pjrt_api, pjrtx.bufferTypeFromDtype(.f32), dims); - const mem_layout = default_layout.toMemoryLayout(); - - if (platform.target == .furiosa) { - try std.testing.expectEqualDeep(mem_layout, platform.defaultMemoryLayout(dims, .f32)); - continue; - } - - // Note: I'm not just calling platform.defaultMemoryLayout because I'm investigating - // wether TPU requires its special branch. - try std.testing.expectEqualDeep(mem_layout, pjrt.MemoryLayout{ - .tiled = .{ - .minor_to_major = constants.minorToMajor(@intCast(dims.len)), - .tile_dims = &.{}, - .tile_dims_sizes = &.{}, - }, - }); - } -} diff --git a/zml/profiling/profiler.zig b/zml/profiling/profiler.zig deleted file mode 100644 index d46a0c4c..00000000 --- a/zml/profiling/profiler.zig +++ /dev/null @@ -1,287 +0,0 @@ -const std = @import("std"); -const log = std.log; - -const c = @import("c"); -const pjrt = @import("pjrt"); -const upb = @import("upb"); -const xspace_to_perfetto = @import("tools/xspace_to_perfetto"); -const zffi = @import("ffi"); - -pub fn profiler(api: *const pjrt.Api, allocator: std.mem.Allocator, io: std.Io, options: ProfilerOptions) !Profiler { - var arena = std.heap.ArenaAllocator.init(allocator); - defer arena.deinit(); - - var upb_alloc: upb.Allocator = .init(arena.allocator()); - const upb_arena = c.upb_Arena_Init(null, 0, upb_alloc.inner()); - defer c.upb_Arena_Free(upb_arena); - - const profile_options = try upb.new(c.tensorflow_ProfileOptions, upb_arena); - try options.writeProto(upb_arena, profile_options); - - const serialized_options = try upb.serialize(profile_options, upb_arena); - const inner = try api.profiler(serialized_options); - - return try .init(allocator, io, inner, options); -} - -pub const ProfilerOptions = struct { - pub const default_version: u32 = 1; - - pub const DeviceType = enum(i32) { - unspecified = c.tensorflow_ProfileOptions_UNSPECIFIED, - cpu = c.tensorflow_ProfileOptions_CPU, - gpu = c.tensorflow_ProfileOptions_GPU, - tpu = c.tensorflow_ProfileOptions_TPU, - pluggable_device = c.tensorflow_ProfileOptions_PLUGGABLE_DEVICE, - }; - - pub const TraceOptions = struct { - host_traceme_filter_mask: ?u64 = null, - - fn isEmpty(self: TraceOptions) bool { - return self.host_traceme_filter_mask == null; - } - - fn writeProto(self: TraceOptions, proto: *c.tensorflow_ProfileOptions_TraceOptions) void { - if (self.host_traceme_filter_mask) |value| { - c.tensorflow_ProfileOptions_TraceOptions_set_host_traceme_filter_mask(proto, value); - } - } - }; - - pub const AdvancedConfigValue = union(enum) { - string: []const u8, - boolean: bool, - int64: i64, - - fn writeProto(self: AdvancedConfigValue, proto: *c.tensorflow_ProfileOptions_AdvancedConfigValue) void { - switch (self) { - .string => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_string_value(proto, upb.stringView(value)), - .boolean => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_bool_value(proto, value), - .int64 => |value| c.tensorflow_ProfileOptions_AdvancedConfigValue_set_int64_value(proto, value), - } - } - }; - - pub const AdvancedConfiguration = struct { - key: []const u8, - value: AdvancedConfigValue, - }; - - pub const defaults: ProfilerOptions = .{}; - - version: u32 = default_version, - repository_path: []const u8 = "/tmp/xprof", - session_id: []const u8 = "profiling", - device_type: ?DeviceType = .unspecified, - include_dataset_ops: ?bool = true, - host_tracer_level: ?u32 = 3, - device_tracer_level: ?u32 = 3, - python_tracer_level: ?u32 = 1, - enable_hlo_proto: ?bool = true, - start_timestamp_ns: ?u64 = null, - duration_ms: ?u64 = null, - trace_options: ?TraceOptions = null, - advanced_configuration: []const AdvancedConfiguration = &.{}, - raise_error_on_start_failure: ?bool = null, - override_hostname: ?[]const u8 = null, - - pub fn writeProto(self: ProfilerOptions, upb_arena: *c.upb_Arena, proto: *c.tensorflow_ProfileOptions) !void { - c.tensorflow_ProfileOptions_set_version(proto, self.version); - c.tensorflow_ProfileOptions_set_session_id(proto, upb.stringView(self.session_id)); - c.tensorflow_ProfileOptions_set_repository_path(proto, upb.stringView(self.repository_path)); - - if (self.device_type) |value| { - c.tensorflow_ProfileOptions_set_device_type(proto, @backingInt(value)); - } - if (self.include_dataset_ops) |value| { - c.tensorflow_ProfileOptions_set_include_dataset_ops(proto, value); - } - if (self.host_tracer_level) |value| { - c.tensorflow_ProfileOptions_set_host_tracer_level(proto, value); - } - if (self.device_tracer_level) |value| { - c.tensorflow_ProfileOptions_set_device_tracer_level(proto, value); - } - if (self.python_tracer_level) |value| { - c.tensorflow_ProfileOptions_set_python_tracer_level(proto, value); - } - if (self.enable_hlo_proto) |value| { - c.tensorflow_ProfileOptions_set_enable_hlo_proto(proto, value); - } - if (self.start_timestamp_ns) |value| { - c.tensorflow_ProfileOptions_set_start_timestamp_ns(proto, value); - } - if (self.duration_ms) |value| { - c.tensorflow_ProfileOptions_set_duration_ms(proto, value); - } - if (self.trace_options) |trace_options| { - if (!trace_options.isEmpty()) { - const trace_proto = try upb.new(c.tensorflow_ProfileOptions_TraceOptions, upb_arena); - trace_options.writeProto(trace_proto); - c.tensorflow_ProfileOptions_set_trace_options(proto, trace_proto); - } - } - for (self.advanced_configuration) |entry| { - const value_proto = try upb.new(c.tensorflow_ProfileOptions_AdvancedConfigValue, upb_arena); - entry.value.writeProto(value_proto); - if (!c.tensorflow_ProfileOptions_advanced_configuration_set(proto, upb.stringView(entry.key), value_proto, upb_arena)) { - return error.OutOfMemory; - } - } - if (self.raise_error_on_start_failure) |value| { - c.tensorflow_ProfileOptions_set_raise_error_on_start_failure(proto, value); - } - if (self.override_hostname) |value| { - c.tensorflow_ProfileOptions_set_override_hostname(proto, upb.stringView(value)); - } - } -}; - -pub const Profiler = struct { - arena: std.heap.ArenaAllocator, - allocator: std.mem.Allocator, - io: std.Io, - inner: ?pjrt.Profiler, - start_timestamp_ns: ?u64, - stop_timestamp_ns: ?u64, - raise_error_on_start_failure: bool, - local_host_tracer_level: ?u32, - local_host_traceme_filter_mask: ?u64, - session_dir: []const u8, - profile: Profile, - - pub const Profile = struct { - protobuf_path: []const u8, - perfetto_path: []const u8, - }; - - pub fn init(allocator: std.mem.Allocator, io: std.Io, inner: ?pjrt.Profiler, options: ProfilerOptions) !Profiler { - var arena = std.heap.ArenaAllocator.init(allocator); - errdefer arena.deinit(); - - const session_dir = try std.Io.Dir.path.join(allocator, &.{ options.repository_path, "plugins", "profile", options.session_id }); - errdefer allocator.free(session_dir); - - try std.Io.Dir.cwd().createDirPath(io, session_dir); - - const protobuf_path = try std.Io.Dir.path.join(allocator, &.{ session_dir, "profiling.xplane.pb" }); - errdefer allocator.free(protobuf_path); - - const perfetto_path = try std.Io.Dir.path.join(allocator, &.{ session_dir, "profiling.trace.json" }); - errdefer allocator.free(perfetto_path); - - return .{ - .arena = arena, - .allocator = allocator, - .io = io, - .inner = inner, - .start_timestamp_ns = null, - .stop_timestamp_ns = null, - .raise_error_on_start_failure = options.raise_error_on_start_failure orelse false, - .local_host_tracer_level = options.host_tracer_level, - .local_host_traceme_filter_mask = if (options.trace_options) |trace_options| trace_options.host_traceme_filter_mask else null, - .session_dir = session_dir, - .profile = .{ - .protobuf_path = protobuf_path, - .perfetto_path = perfetto_path, - }, - }; - } - - pub fn deinit(self: *Profiler) void { - self.allocator.free(self.profile.protobuf_path); - self.allocator.free(self.profile.perfetto_path); - self.allocator.free(self.session_dir); - self.arena.deinit(); - } - - pub fn start(self: *Profiler) !void { - self.start_timestamp_ns = @intCast(std.Io.Timestamp.now(self.io, .real).nanoseconds); - self.stop_timestamp_ns = null; - - if (shouldSkipPjrtProfiler()) { - log.warn("Detected SKIP_PJRT_PROFILER=true; disabling nested backend profiling and keeping local host tracing only.", .{}); - self.inner = null; - } else if (self.inner) |*inner| { - inner.start() catch |err| { - if (self.raise_error_on_start_failure) { - return err; - } - log.warn("Profiler backend failed to start; continuing with local host tracing only: {}", .{err}); - self.inner = null; - }; - } - if (self.local_host_tracer_level) |level| { - c.zml_traceme_session_start( - @intCast(level), - self.local_host_traceme_filter_mask orelse 0, - self.local_host_traceme_filter_mask != null, - ); - } - } - - pub fn stop(self: *Profiler) !?Profile { - self.stop_timestamp_ns = @intCast(std.Io.Timestamp.now(self.io, .real).nanoseconds); - - const protobuf = if (self.inner) |saved_inner| blk: { - var inner = saved_inner; - self.inner = null; - try inner.stop(); - break :blk try inner.collectData(self.arena.allocator()); - } else &.{}; - const merged_protobuf = c.zml_traceme_session_merge(zffi.ZigSlice.from(protobuf)); - defer if (merged_protobuf.len != 0) { - c.zml_traceme_str_free(merged_protobuf); - }; - const final_protobuf = if (merged_protobuf.len != 0) zffi.ZigSlice.to(u8, merged_protobuf) else protobuf; - if (final_protobuf.len == 0) { - return null; - } - - const normalized_protobuf = c.zml_traceme_session_postprocess( - zffi.ZigSlice.from(final_protobuf), - self.start_timestamp_ns orelse self.stop_timestamp_ns.?, - self.stop_timestamp_ns.?, - ); - defer if (normalized_protobuf.len != 0) { - c.zml_traceme_str_free(normalized_protobuf); - }; - if (normalized_protobuf.len == 0) { - log.err("Failed to normalize profile protobuf", .{}); - return error.ProfileTraceNormalizationFailed; - } - - const final_normalized_protobuf = zffi.ZigSlice.to(u8, normalized_protobuf); - - try self.writeFile(self.profile.protobuf_path, final_normalized_protobuf); - - xspace_to_perfetto.dumpXSpaceProtoToTraceJsonFile( - self.allocator, - self.io, - final_normalized_protobuf, - self.profile.perfetto_path, - ) catch |err| { - log.err("Failed to convert profile protobuf to Perfetto trace: {}", .{err}); - return error.ProfileTraceConversionFailed; - }; - - log.info("Profile dumped: {s} and {s}", .{ - self.profile.protobuf_path, - self.profile.perfetto_path, - }); - return self.profile; - } - - fn writeFile(self: *const Profiler, path: []const u8, contents: []const u8) !void { - const file = try std.Io.Dir.createFile(.cwd(), self.io, path, .{}); - defer file.close(self.io); - - try file.writePositionalAll(self.io, contents, 0); - } -}; - -fn shouldSkipPjrtProfiler() bool { - const value = std.c.getenv("SKIP_PJRT_PROFILER") orelse return false; - return std.mem.eql(u8, std.mem.span(value), "true"); -} diff --git a/zml/profiling/traceme.h b/zml/profiling/traceme.h deleted file mode 100644 index a1df3d04..00000000 --- a/zml/profiling/traceme.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include -#include - -#include "ffi/zig_slice.h" - -#ifdef __cplusplus -extern "C" { -#endif - -typedef struct zml_traceme zml_traceme; - -zml_traceme *zml_traceme_start(zig_slice name); -void zml_traceme_stop(zml_traceme *traceme); -bool zml_traceme_enabled(void); -void zml_traceme_session_start(int level, uint64_t filter_mask, - bool enable_filter); -zig_slice zml_traceme_session_merge(zig_slice xspace); -zig_slice zml_traceme_session_postprocess(zig_slice xspace, - uint64_t start_time_ns, - uint64_t stop_time_ns); -void zml_traceme_str_free(zig_slice text); - -#ifdef __cplusplus -} -#endif diff --git a/zml/profiling/traceme_lib.cc b/zml/profiling/traceme_lib.cc deleted file mode 100644 index ca37dee7..00000000 --- a/zml/profiling/traceme_lib.cc +++ /dev/null @@ -1,252 +0,0 @@ -#include "zml/profiling/traceme.h" - -#include -#include -#include - -#include "xla/tsl/profiler/convert/post_process_single_host_xplane.h" -#include "xla/tsl/profiler/backends/cpu/host_tracer_utils.h" -#include "xla/tsl/profiler/backends/cpu/traceme_recorder.h" -#include "xla/tsl/profiler/utils/time_utils.h" -#include "xla/tsl/profiler/utils/xplane_utils.h" -#include "tsl/profiler/lib/traceme.h" -#include "tsl/profiler/protobuf/xplane.pb.h" - -#if defined(__APPLE__) -#include -#include -#endif - -#if defined(ZML_RUNTIME_CUDA) -#include "nvtx3/nvToolsExt.h" -#endif - -#if defined(ZML_RUNTIME_ROCM) -#include "rocprofiler-sdk-roctx/roctx.h" -#endif - -namespace { - -struct LocalHostTraceSession { - bool active = false; - uint64_t start_timestamp_ns = 0; -}; - -LocalHostTraceSession g_local_host_trace_session; - -tensorflow::profiler::XPlane* FindOrAddHostPlane( - tensorflow::profiler::XSpace* space) { - for (auto& plane : *space->mutable_planes()) { - if (plane.name() == "/host:CPU") { - return &plane; - } - } - tensorflow::profiler::XPlane* plane = space->add_planes(); - plane->set_name("/host:CPU"); - return plane; -} - -zig_slice CopyToOwnedSlice(const std::string& data) { - if (data.empty()) { - return {}; - } - void* ptr = std::malloc(data.size()); - if (ptr == nullptr) { - return {}; - } - std::memcpy(ptr, data.data(), data.size()); - return {ptr, data.size()}; -} - -#if defined(ZML_RUNTIME_CUDA) -struct CudaNvtxRangeBackend { - explicit CudaNvtxRangeBackend(const std::string& name) - : domain(TslNvtxDomain()), active(domain != nullptr) { - if (!active) { - return; - } - - nvtxEventAttributes_t attrs{}; - attrs.version = NVTX_VERSION; - attrs.size = NVTX_EVENT_ATTRIB_STRUCT_SIZE; - attrs.messageType = NVTX_MESSAGE_TYPE_ASCII; - attrs.message.ascii = name.c_str(); - nvtxDomainRangePushEx(domain, &attrs); - } - - ~CudaNvtxRangeBackend() { - if (active) { - nvtxDomainRangePop(domain); - } - } - - static bool IsEnabled() { return TslNvtxDomain() != nullptr; } - - private: - static nvtxDomainHandle_t TslNvtxDomain() { - static nvtxDomainHandle_t domain = nvtxDomainCreateA("TSL"); - return domain; - } - - nvtxDomainHandle_t domain = nullptr; - bool active = false; -}; -#else -struct CudaNvtxRangeBackend { - explicit CudaNvtxRangeBackend(const std::string&) {} - static bool IsEnabled() { return false; } -}; -#endif - -#if defined(ZML_RUNTIME_ROCM) -struct RocmRoctxRangeBackend { - explicit RocmRoctxRangeBackend(const std::string& name) - : active(RangePush() != nullptr && RangePop() != nullptr) { - if (!active) { - return; - } - - const int level = RangePush()(name.c_str()); - active = level >= 0; - } - - ~RocmRoctxRangeBackend() { - if (active) { - RangePop()(); - } - } - - static bool IsEnabled() { - return RangePush() != nullptr && RangePop() != nullptr; - } - - private: - using RangePushFn = decltype(&roctxRangePushA); - using RangePopFn = decltype(&roctxRangePop); - - static RangePushFn RangePush() { - return reinterpret_cast(dlsym(RTLD_DEFAULT, "roctxRangePushA")); - } - - static RangePopFn RangePop() { - return reinterpret_cast(dlsym(RTLD_DEFAULT, "roctxRangePop")); - } - - bool active = false; -}; -#else -struct RocmRoctxRangeBackend { - explicit RocmRoctxRangeBackend(const std::string&) {} - static bool IsEnabled() { return false; } -}; -#endif - -#if defined(__APPLE__) -struct MacOsSignpostRangeBackend { - explicit MacOsSignpostRangeBackend(const std::string& name) - : log(Logger()), signpost_id(os_signpost_id_generate(log)) { - os_signpost_interval_begin(log, signpost_id, "zml.scope", "%{public}s", - name.c_str()); - } - - ~MacOsSignpostRangeBackend() { - os_signpost_interval_end(log, signpost_id, "zml.scope", "%{public}s", - ""); - } - - static bool IsEnabled() { return true; } - - private: - static os_log_t Logger() { - static os_log_t log = - os_log_create("ai.zml", OS_LOG_CATEGORY_POINTS_OF_INTEREST); - return log; - } - - os_log_t log; - os_signpost_id_t signpost_id; -}; -#else -struct MacOsSignpostRangeBackend { - explicit MacOsSignpostRangeBackend(const std::string&) {} - static bool IsEnabled() { return false; } -}; -#endif - -} // namespace - -struct zml_traceme { - explicit zml_traceme(std::string encoded_name) - : name(std::move(encoded_name)), - traceme([&] { return name; }), - cuda_backend(name), - rocm_backend(name), - macos_backend(name) {} - - std::string name; - tsl::profiler::TraceMe traceme; - CudaNvtxRangeBackend cuda_backend; - RocmRoctxRangeBackend rocm_backend; - MacOsSignpostRangeBackend macos_backend; -}; - -extern "C" zml_traceme* zml_traceme_start(zig_slice name) { - return new zml_traceme( - std::string(static_cast(name.ptr), name.len)); -} - -extern "C" void zml_traceme_stop(zml_traceme* traceme) { delete traceme; } - -extern "C" bool zml_traceme_enabled(void) { - return tsl::profiler::TraceMe::Active() || - CudaNvtxRangeBackend::IsEnabled() || - RocmRoctxRangeBackend::IsEnabled() || - MacOsSignpostRangeBackend::IsEnabled(); -} - -extern "C" void zml_traceme_session_start(int level, uint64_t filter_mask, - bool enable_filter) { - g_local_host_trace_session.start_timestamp_ns = - tsl::profiler::GetCurrentTimeNanos(); - g_local_host_trace_session.active = - enable_filter ? tsl::profiler::TraceMeRecorder::Start(level, filter_mask) - : tsl::profiler::TraceMeRecorder::Start(level); -} - -extern "C" zig_slice zml_traceme_session_merge(zig_slice xspace) { - tensorflow::profiler::XSpace space; - if (xspace.len != 0 && - !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { - return CopyToOwnedSlice( - std::string(static_cast(xspace.ptr), xspace.len)); - } - - if (g_local_host_trace_session.active) { - auto events = tsl::profiler::TraceMeRecorder::Stop(); - g_local_host_trace_session.active = false; - if (!events.empty()) { - tsl::profiler::ConvertCompleteEventsToXPlane( - g_local_host_trace_session.start_timestamp_ns, std::move(events), - FindOrAddHostPlane(&space)); - } - } - - return CopyToOwnedSlice(space.SerializeAsString()); -} - -extern "C" zig_slice zml_traceme_session_postprocess(zig_slice xspace, - uint64_t start_time_ns, - uint64_t stop_time_ns) { - tensorflow::profiler::XSpace space; - if (xspace.len != 0 && - !space.ParseFromArray(xspace.ptr, static_cast(xspace.len))) { - return {}; - } - - tsl::profiler::SetXSpacePidIfNotSet(space, getpid()); - tsl::profiler::PostProcessSingleHostXSpace(&space, start_time_ns, - stop_time_ns); - return CopyToOwnedSlice(space.SerializeAsString()); -} - -extern "C" void zml_traceme_str_free(zig_slice text) { std::free(text.ptr); } diff --git a/zml/profiling/tracer.zig b/zml/profiling/tracer.zig deleted file mode 100644 index 5aed01df..00000000 --- a/zml/profiling/tracer.zig +++ /dev/null @@ -1,235 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); - -const c = @import("c"); -const platforms = @import("platforms"); -const stdx = @import("stdx"); -const zffi = @import("ffi"); - -pub const Span = struct { - inner: ?*c.zml_traceme = null, - - /// Name is copied underneath - pub fn start(name: []const u8) Span { - return .{ - .inner = c.zml_traceme_start(zffi.ZigSlice.from(name)), - }; - } - - pub fn end(self: *Span) void { - if (self.inner) |inner| { - c.zml_traceme_stop(inner); - self.inner = null; - } - } -}; - -// TraceMe is always the base host tracing path. The C tracing bridge adds NVTX -// ranges on CUDA/Linux builds, roctx ranges on ROCm/Linux builds, and -// os_signpost intervals on macOS. -pub const supportsDeviceAnnotations = switch (builtin.os.tag) { - .macos => true, - .linux => platforms.target == .cuda or platforms.target == .rocm, - else => false, -}; - -pub fn enabled() bool { - return c.zml_traceme_enabled(); -} - -/// Creates a Span with encoded metadata. The metadata encoded size must be known at compile time. -/// If it's not your case, use `formatSpanName` to generate the span name before and use void for the metadata. -pub fn span(comptime name: []const u8, metadata: anytype) Span { - switch (@typeInfo(@TypeOf(metadata))) { - .void => { - return .start(name); - }, - .@"struct" => |info| { - if (info.field_names.len == 0) { - return .start(name); - } - - var buffer: [computeEncodedNameLen(name, @TypeOf(metadata))]u8 = undefined; - var encoded: std.ArrayList(u8) = .initBuffer(&buffer); - - appendEncodedName(&encoded, .failing, name, metadata) catch unreachable; - return .start(encoded.items); - }, - else => @compileError("Unsupported metadata."), - } -} - -/// Returns an allocator-owned encoded span name for metadata that cannot be -/// bounded by `span` at comptime. The caller owns the returned slice. -pub fn formatSpanName(allocator: std.mem.Allocator, name: []const u8, metadata: anytype) ![]const u8 { - switch (@typeInfo(@TypeOf(metadata))) { - .@"struct" => { - var encoded: std.ArrayList(u8) = .empty; - errdefer encoded.deinit(allocator); - - try appendEncodedName(&encoded, allocator, name, metadata); - return try encoded.toOwnedSlice(allocator); - }, - else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), - } -} - -fn appendEncodedName(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, name: []const u8, metadata: anytype) !void { - try buffer.appendSlice(allocator, name); - - var field_count: usize = 0; - try buffer.append(allocator, '#'); - inline for (comptime std.meta.fieldNames(@TypeOf(metadata))) |field_name| { - if (shouldEncodeField(field_name, @field(metadata, field_name))) { - if (field_count != 0) { - try buffer.append(allocator, ','); - } - field_count += 1; - - const key = comptime metadataKey(field_name); - validateMetadataToken(key); - try buffer.appendSlice(allocator, key); - try buffer.append(allocator, '='); - try appendMetadataValue(buffer, allocator, @field(metadata, field_name), field_name); - } - } - - // We may not have written any fields because of optional values. - if (field_count == 0) { - buffer.items.len = name.len; - } else { - try buffer.append(allocator, '#'); - } -} - -fn validateMetadataToken(token: []const u8) void { - if (std.mem.indexOfScalar(u8, token, '#') != null) { - @panic("trace metadata cannot contain '#'"); - } -} - -fn metadataKey(comptime field_name: []const u8) []const u8 { - return if (comptime std.mem.eql(u8, field_name, "root")) "_r" else field_name; -} - -fn computeEncodedNameLen(comptime name: []const u8, comptime Metadata: type) usize { - switch (@typeInfo(Metadata)) { - .@"struct" => |info| { - if (info.field_names.len == 0) return name.len; - - // name#[attrs]# - var max_len = name.len + 2; - inline for ( - info.field_names, - info.field_types, - info.field_attrs, - ) |field_name, FieldType, field_attr| { - const key = comptime metadataKey(field_name); - validateMetadataToken(key); - - const default: ?FieldType = field_attr.defaultValue(FieldType); - // [key]=[value], - max_len += key.len + 2; - max_len += switch (@typeInfo(FieldType)) { - .bool => 1, - .int => @max( - std.fmt.comptimePrint("{}", .{std.math.minInt(FieldType)}).len, - std.fmt.comptimePrint("{}", .{std.math.maxInt(FieldType)}).len, - ), - .@"enum" => |ty| b: { - var max_enum_len: usize = 0; - for (ty.field_names) |enum_field_name| { - max_enum_len = @max(max_enum_len, enum_field_name.len); - } - break :b max_enum_len; - }, - .comptime_int => std.fmt.comptimePrint("{}", .{default.?}).len, - .comptime_float => std.fmt.comptimePrint("{d}", .{default.?}).len, - .enum_literal => @tagName(default.?).len, - else => @compileError("trace metadata field '" ++ field_name ++ "' is not statically bounded; use formatspanName"), - }; - } - return max_len; - }, - else => @compileError("trace metadata must be a struct literal like .{ .step_num = 42 }"), - } -} - -fn shouldEncodeField(comptime field_name: []const u8, value: anytype) bool { - const FieldType = @TypeOf(value); - return switch (@typeInfo(FieldType)) { - .optional => value != null, - .bool => if (comptime std.mem.eql(u8, field_name, "root")) value else true, - else => true, - }; -} - -fn appendMetadataValue(buffer: *std.ArrayList(u8), allocator: std.mem.Allocator, value: anytype, comptime field_name: []const u8) !void { - const T = @TypeOf(value); - switch (@typeInfo(T)) { - .optional => { - try appendMetadataValue(buffer, allocator, value.?, field_name); - }, - .bool => { - if (comptime std.mem.eql(u8, field_name, "root")) { - try buffer.append(allocator, '1'); - } else { - try buffer.appendSlice(allocator, if (value) "1" else "0"); - } - }, - .int, .comptime_int => try buffer.print(allocator, "{}", .{value}), - .float, .comptime_float => try buffer.print(allocator, "{d}", .{value}), - .enum_literal => try buffer.appendSlice(allocator, @tagName(value)), - .@"enum" => try buffer.appendSlice(allocator, @tagName(value)), - .pointer => |ptr| { - if (ptr.size == .slice and ptr.child == u8) { - validateMetadataToken(value); - try buffer.appendSlice(allocator, value); - return; - } - - if (ptr.size == .one) switch (@typeInfo(ptr.child)) { - .array => |arr| { - if (arr.child == u8) { - const slice = value[0..]; - validateMetadataToken(slice); - try buffer.appendSlice(allocator, slice); - return; - } - }, - else => {}, - }; - - @compileError("trace metadata pointers must be []const u8 or string literals"); - }, - .array => |arr| { - if (arr.child != u8) { - @compileError("trace metadata arrays must be [N]u8"); - } - validateMetadataToken(value[0..]); - try buffer.appendSlice(allocator, value[0..]); - }, - else => @compileError("unsupported trace metadata type for field '" ++ field_name ++ "'"), - } -} - -test "formatspanName encodes metadata" { - const formatted = try formatSpanName(std.testing.allocator, "test.span", .{ - .root = true, - .count = @as(u8, 42), - .label = "batch", - }); - defer std.testing.allocator.free(formatted); - - try std.testing.expectEqualStrings("test.span#_r=1,count=42,label=batch#", formatted); -} - -test "formatspanName omits empty metadata" { - const formatted = try formatSpanName(std.testing.allocator, "test.span", .{ - .root = false, - .optional = @as(?u8, null), - }); - defer std.testing.allocator.free(formatted); - - try std.testing.expectEqualStrings("test.span", formatted); -} diff --git a/zml/quantization.zig b/zml/quantization.zig deleted file mode 100644 index db20619a..00000000 --- a/zml/quantization.zig +++ /dev/null @@ -1,519 +0,0 @@ -//! Quantization metadata and scheme classification. -const std = @import("std"); - -const stdx = @import("stdx"); - -const zml = @import("zml.zig"); - -const DataType = @import("dtype.zig").DataType; -const platform_mod = @import("platform.zig"); -const Platform = platform_mod.Platform; -const Shape = @import("shape.zig").Shape; -const Tensor = @import("tensor.zig").Tensor; - -pub const nvfp4_block_size = 16; -pub const mx_block_size = 32; - -pub const Quantization = struct { - scheme: Scheme, // decided when the checkpoint is read - scales: Tensor, - global_scale: ?GlobalScale = null, - input_scale: ?GlobalScale = null, - /// HACK: `scales` is stored in the 128x4 blocked layout of Blackwell's block-scaled mma. - // Something cleaner will need to be done to differentiate different kinds of swizzling - swizzled_scales: bool = false, - - /// A per-tensor scale and the polarity its producer wrote it in - pub const GlobalScale = struct { - value: Tensor, - operation: Operation, - - pub const Operation = enum { multiply, divide }; - - pub fn asMultiplier(self: GlobalScale) Tensor { - const s = self.value.convert(.f32); - const flat = if (s.shape().count() == 1) s.asScalar() else s; // essentially normalization - - return switch (self.operation) { - .multiply => flat, - .divide => Tensor.scalar(1.0, .f32).broad(flat.shape()).div(flat), - }; - } - }; - - pub const Scheme = enum { - /// f4e2m1 values (`u8`-packed or native), f8e4m3fn scale per 16 contracted values. - /// Emitted by llm-compressor and by NVIDIA's ModelOpt. - nvfp4, - /// f8e4m3fn values, one e8m0 (power-of-two) scale per 32 contracted values - mxfp8, - /// f4e2m1 values (`u8`-packed or native), e8m0 scale per 32 contracted values. - // GPT-OSS like - mxfp4, - /// f8e4m3fn values, one bf16 or f32 scale per output channel, constant along the contraction. - /// Emitted by llm-compressor, including for the layers an NVFP4 recipe leaves in FP8. - fp8_per_channel, - /// f8e4m3fn, f8e4m3fnuz, or f8e8m0 values, one bf16 or f32 scale per 128x128 tile. - /// The DeepSeek-style FP8 that model vendors publish themselves, under `weight_scale_inv`. - fp8_block128, - /// E4M3FN or E4M3FNUZ weights with E8M0, BF16 or F32 scales per 32x32 tile. - fp8_block32, - /// f8e4m3fn values, one scale for the whole tensor. Spelled `[1, 1]` rather than as a - /// scalar: XLA's composite rewriter requires the scale to have the operand's rank. - fp8_per_tensor, - - pub fn accepts(self: Scheme, weight: Shape, scale: Shape) bool { - if (weight.rank() != 2) return false; - - const n = weight.dim(0); - const k = if (isPackedFp4(self, weight.dtype())) 2 * weight.dim(1) else weight.dim(1); - - return switch (self) { - .nvfp4 => (weight.dtype() == .u8 or weight.dtype() == .f4e2m1) and - scale.dtype() == .f8e4m3fn and scale.rank() == 2 and - scale.dim(0) == n and scale.dim(1) * nvfp4_block_size == k, - .mxfp8 => weight.dtype() == .f8e4m3fn and isMxScale(scale) and - scale.dim(0) == n and scale.dim(1) * mx_block_size == k, - .mxfp4 => (weight.dtype() == .u8 or weight.dtype() == .i8 or weight.dtype() == .f4e2m1) and - isMxScale(scale) and - scale.dim(0) == n and scale.dim(1) * mx_block_size == k, - .fp8_per_tensor => weight.dtype() == .f8e4m3fn and scale.count() == 1, - .fp8_per_channel => weight.dtype() == .f8e4m3fn and - (scale.dtype() == .bf16 or scale.dtype() == .f32) and - scale.count() > 1 and scale.rank() == 2 and - scale.dim(0) == n and scale.dim(1) == 1, - .fp8_block32 => (weight.dtype() == .f8e4m3fn or weight.dtype() == .f8e4m3fnuz) and - (scale.dtype() == .f8e8m0 or ((scale.dtype() == .bf16 or scale.dtype() == .f32) and scale.count() > 1)) and - scale.rank() == 2 and scale.dim(0) * 32 == n and scale.dim(1) * 32 == k, - .fp8_block128 => (weight.dtype() == .f8e4m3fn or weight.dtype() == .f8e4m3fnuz or weight.dtype() == .f8e8m0) and - (scale.dtype() == .bf16 or scale.dtype() == .f32) and - scale.count() > 1 and scale.rank() == 2 and - scale.dim(0) == (std.math.divCeil(i64, n, 128) catch unreachable) and - scale.dim(1) == (std.math.divCeil(i64, k, 128) catch unreachable), - }; - } - - pub fn classify(weight: Shape, scale: Shape) ?Scheme { - for (std.enums.values(Scheme)) |scheme| { - if (scheme.accepts(weight, scale)) return scheme; - } - - return null; - } - - pub fn isMx(self: Scheme) bool { - return self == .mxfp8 or self == .mxfp4; - } - }; -}; - -pub const QuantizedInput = struct { - values: Tensor, - scales: Tensor, - global_scale: ?Tensor, - - pub fn reuseBuffer(self: QuantizedInput, other: QuantizedInput) QuantizedInput { - return .{ - .values = self.values.reuseBuffer(other.values), - .scales = self.scales.reuseBuffer(other.scales), - .global_scale = if (self.global_scale != null and other.global_scale != null) self.global_scale.?.reuseBuffer(other.global_scale.?) else self.global_scale, - }; - } -}; - -pub fn quantizeInput(quantization: Quantization, input: Tensor, axis: Shape.Tag, platform: *const Platform) ?QuantizedInput { - const global_scale: ?Tensor = if (quantization.input_scale) |scale| scale.asMultiplier() else null; - - return switch (quantization.scheme) { - .nvfp4 => if (supportsNvfp4InputQuantization(platform)) quantizeNvfp4(input.convert(.bf16), global_scale, axis) else null, - .fp8_block32, .fp8_block128 => blk: { - const dtype: DataType = switch (platform.target) { - .cuda => .f8e4m3fn, - // RNGD contracts matching FP8 pairs in 128-wide K blocks. - .furiosa => if (quantization.scheme == .fp8_block128) .f8e4m3fn else return null, - .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { - .cdna4, .rdna4 => .f8e4m3fn, - .cdna3 => .f8e4m3fnuz, - .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => return null, - } else return null, - else => return null, - }; - - break :blk switch (quantization.scheme) { - .fp8_block32 => quantizeBlockFp8(input, axis, 32, dtype, .f8e8m0), - .fp8_block128 => quantizeBlockFp8(input.convert(.bf16), axis, 128, dtype, .f32), - else => unreachable, - }; - }, - .mxfp8, .mxfp4, .fp8_per_channel, .fp8_per_tensor => null, - }; -} - -pub fn quantizeMxfp4(x: Tensor, axis: anytype) QuantizedInput { - stdx.debug.assert(@mod(x.dim(axis), mx_block_size) == 0, "MXFP4 activation width must be divisible by {}, got {f}", .{ mx_block_size, x.shape() }); - - const grouped = x.convert(.f32).splitAxis(axis, .{ .mx_ks = -1, .mx_block = mx_block_size }); - const raw_scale = grouped.abs().max(.mx_block).maximum(.scalar(6 * 0x1p-126, .f32)).scale(1.0 / 6.0); - const scales = ceilPowerOfTwo(raw_scale); - - return .{ - .values = grouped.div(scales.broad(grouped.shape())) - .clamp(.scalar(-6, .f32), .scalar(6, .f32)) - .convert(.f4e2m1) - .reshape(x.shape().withDtype(.f4e2m1)), - .scales = scales.reshape(x.shape().setDim(axis, @divExact(x.dim(axis), mx_block_size)).withDtype(.f32)).convert(.f8e8m0), - .global_scale = null, - }; -} - -pub fn quantizeMxfp8(x: Tensor, axis: anytype) QuantizedInput { - const platform = zml.Compiler.current().platform; - - const dtype: ?DataType = switch (platform.target) { - .cuda, .cpu, .tpu => .f8e4m3fn, - .rocm => if (platform_mod.rocm.computeCapability(platform)) |capability| switch (capability.architecture()) { - .cdna4, .rdna4 => .f8e4m3fn, - .cdna3 => .f8e4m3fnuz, - .cdna1, .cdna2, .rdna2, .rdna3, .rdna3_5 => null, - } else null, - else => null, - }; - - if (dtype == null) std.debug.panic("MXFP8 is not supported on {}", .{platform.target}); - return quantizeBlockFp8(x, axis, mx_block_size, dtype.?, .f8e8m0); -} - -fn ceilPowerOfTwo(raw_scale: Tensor) Tensor { - const bits = raw_scale.bitCast(.u32); - return bits.addConstant(0x7fffff).logical(.AND, .scalar(0xff800000, .u32)).bitCast(.f32); -} - -/// Quantize activation blocks, preserving the input's axes. -/// E8M0 scales round upward to a power of two to avoid overflowing FP8 values. -pub fn quantizeBlockFp8(x: Tensor, axis: anytype, block_size: i64, dtype: DataType, scale_dtype: DataType) QuantizedInput { - stdx.debug.assert(block_size > 0, "block FP8 size must be positive, got {}", .{block_size}); - stdx.debug.assert(@mod(x.dim(axis), block_size) == 0, "block FP8 activation width must be divisible by {}, got {f}", .{ block_size, x.shape() }); - // Match the routed-MoE quantizer's FNUZ range. - const fp8_max: f32 = switch (dtype) { - .f8e4m3fn => 448.0, - .f8e4m3fnuz => 224.0, - else => stdx.debug.panic("expected E4M3 FP8 dtype, got {s}", .{@tagName(dtype)}), - }; - const grouped = x.convert(.f32).splitAxis(axis, .{ .fp8_ks = -1, .fp8_block = block_size }); - const scales = switch (scale_dtype) { - // Put scale arithmetic before the reduction so it can fuse into the producer. - .f32 => grouped.abs() - .maximum(.scalar(1e-10, .f32)) - .scale(1.0 / fp8_max) - .max(.fp8_block), - .f8e8m0 => blk: { - const raw_scale = grouped.abs().maximum(.scalar(1e-4, .f32)).scale(1.0 / fp8_max).max(.fp8_block); - break :blk ceilPowerOfTwo(raw_scale); - }, - else => stdx.debug.panic("expected F32 or E8M0 scale dtype, got {s}", .{@tagName(scale_dtype)}), - }; - const values = grouped.div(scales.broad(grouped.shape())) - .clamp(.scalar(-fp8_max, .f32), .scalar(fp8_max, .f32)) - .convert(dtype) - .reshape(x.shape().withDtype(dtype)); - return .{ - .values = values, - .scales = scales.reshape(x.shape().setDim(axis, @divExact(x.dim(axis), block_size)).withDtype(.f32)).convert(scale_dtype), - .global_scale = null, - }; -} - -pub fn quantizeNvfp4(x: Tensor, input_global_scale: ?Tensor, axis: anytype) QuantizedInput { - const value_max = 6.0; - const scale_min_normal = 0x1p-6; - const scale_max = DataType.f8e4m3fn.maxValue().as(f32); - - stdx.debug.assert(x.shape().hasTag(axis) != null, "quantizeNvfp4 expects x to have {any} tag, got {f}", .{ axis, x.shape() }); - stdx.debug.assert(@rem(x.dim(axis), nvfp4_block_size) == 0, "quantizeNvfp4 expects {any} to be a multiple of {}, got {f}", .{ axis, nvfp4_block_size, x.shape() }); - - const dt = x.dtype(); - const scaled = if (input_global_scale) |igs| - x.div(igs.convert(dt).broad(x.shape())) - else - x; - const grouped = scaled.splitAxis(axis, .{ .sc = -1, .blk = nvfp4_block_size }); - - const amax = grouped.abs() - .scale(1.0 / value_max) - .clamp(.scalar(scale_min_normal, dt), .scalar(scale_max, dt)) - .max(.blk); - - const scales = amax.convert(.f8e4m3fn); - - const divisor = scales.convert(dt) - .maximum(.scalar(scale_min_normal, dt)) - .broad(grouped.shape()); - - const values = grouped.div(divisor) - .convert(.f4e2m1) - .reshape(x.shape().withDtype(.f4e2m1)); - - return .{ - .values = values, - .scales = scales.squeeze(.blk), - .global_scale = input_global_scale, - }; -} - -fn supportsNvfp4InputQuantization(platform: *const Platform) bool { - return if (platform_mod.cuda.computeCapability(platform)) |cc| cc.atLeast(.{ .major = 10, .minor = 0 }) else false; -} - -fn isPackedFp4(scheme: ?Quantization.Scheme, weight_dtype: DataType) bool { - return (scheme == .nvfp4 or scheme == .mxfp4) and (weight_dtype == .u8 or weight_dtype == .i8); -} - -fn isMxScale(scale: Shape) bool { - return scale.rank() == 2 and (scale.dtype() == .f8e8m0 or scale.dtype() == .u8); -} - -// Note: this test will evolve as we support more (INT4/8 and FP8 block-128 as a -// scaled-dot are still open; see the scaledDot doc comment). -test "Quantization.Scheme.classify" { - const expect = std.testing.expectEqual; - - // unsloth/Qwen3.6-27B-NVFP4: compressed-tensors, carrying both of its schemes under - // `weight_scale` -- NVFP4 on the MLPs of layers 0-55, FP8 per-channel everywhere else. - const nvfp4_packed: Shape = .init(.{ .dout = 17408, .d = 2560 }, .u8); - try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn))); - try expect(@as(?Quantization.Scheme, .fp8_per_channel), Quantization.Scheme.classify( - .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), - .init(.{ .dout = 17408, .sc = 1 }, .bf16), - )); - - // nvidia/Gemma-4-31B-IT-NVFP4: ModelOpt, packed values under the plain `weight` name. - try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify( - .init(.{ .dout = 21504, .d = 2688 }, .u8), - .init(.{ .dout = 21504, .sc = 336 }, .f8e4m3fn), - )); - - // Native (unpacked) f4e2m1, with K no longer doubled. - try expect(@as(?Quantization.Scheme, .nvfp4), Quantization.Scheme.classify( - .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), - .init(.{ .dout = 17408, .sc = 320 }, .f8e4m3fn), - )); - - // Qwen/Qwen3.6-27B-FP8: block 128, spelled `weight_scale_inv`. - try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( - .init(.{ .dout = 17408, .d = 5120 }, .f8e4m3fn), - .init(.{ .dout = 136, .sc = 40 }, .bf16), - )); - try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( - .init(.{ .dout = 5120, .d = 6144 }, .f8e4m3fn), - .init(.{ .dout = 40, .sc = 48 }, .bf16), - )); - - inline for (.{ .f8e4m3fn, .f8e4m3fnuz }) |dtype| { - try expect(@as(?Quantization.Scheme, .fp8_block32), Quantization.Scheme.classify( - .init(.{ .dout = 1280, .d = 5120 }, dtype), - .init(.{ .dout = 40, .sc = 160 }, .f8e8m0), - )); - } - - // GLM uses f32 scales and permits a partial final 128-row tile. - try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( - .init(.{ .dout = 2048, .d = 6144 }, .f8e4m3fn), - .init(.{ .dout = 16, .sc = 48 }, .f32), - )); - try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( - .init(.{ .dout = 576, .d = 6144 }, .f8e4m3fn), - .init(.{ .dout = 5, .sc = 48 }, .f32), - )); - try expect(@as(?Quantization.Scheme, .fp8_block128), Quantization.Scheme.classify( - .init(.{ .dout = 576, .d = 6144 }, .f8e8m0), - .init(.{ .dout = 5, .sc = 48 }, .f32), - )); - - // Mistral's per-tensor FP8: one scale for the whole tensor, rank 0 or [1]. - try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{}, .f32))); - try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify(.init(.{ .dout = 4096, .d = 4096 }, .f8e4m3fn), .init(.{ .g = 1 }, .f32))); - - // RadixArk/Muse-Glimmer-NVFP4: ModelOpt's MIXED_PRECISION recipe, whose MXFP8 half - // lands on down_proj and lm_head. safetensors has no e8m0 dtype, so the scale arrives - // as u8 under `weight_scale_inv` -- the shape below is verbatim from the checkpoint. - try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( - .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), - .init(.{ .dout = 6656, .sc = 624 }, .u8), - )); - try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( - .init(.{ .dout = 202048, .d = 6656 }, .f8e4m3fn), - .init(.{ .dout = 202048, .sc = 208 }, .u8), - )); - // The same tensor from a store that types the scale natively. - try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( - .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), - .init(.{ .dout = 6656, .sc = 624 }, .f8e8m0), - )); - // MXFP8 rejects any group but 32: 19968 / 16 = 1248, / 64 = 312. - try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( - .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), - .init(.{ .dout = 6656, .sc = 1248 }, .u8), - )); - try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( - .init(.{ .dout = 6656, .d = 19968 }, .f8e4m3fn), - .init(.{ .dout = 6656, .sc = 312 }, .u8), - )); - - // MXFP4: the same u8 packing as NVFP4 (two nibbles per byte, K halved on disk), with - // an e8m0 scale per 32 -- so K = 2 * 2560 = 5120 and 5120 / 32 = 160. - try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .u8))); - try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0))); - // DeepSeek V4 stores the same packed FP4 bits in signed bytes. - try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify( - .init(.{ .dout = 2048, .d = 2048 }, .i8), - .init(.{ .dout = 2048, .sc = 128 }, .f8e8m0), - )); - // Native (unpacked) f4e2m1, K no longer halved. - try expect(@as(?Quantization.Scheme, .mxfp4), Quantization.Scheme.classify( - .init(.{ .dout = 17408, .d = 5120 }, .f4e2m1), - .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), - )); - // An e8m0 scale never turns a non-fp4/fp8 weight into MX. - try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( - .init(.{ .dout = 17408, .d = 5120 }, .bf16), - .init(.{ .dout = 17408, .sc = 160 }, .f8e8m0), - )); - - // Rejected: everything no backend here can express. - const fp8: Shape = .init(.{ .dout = 10240, .d = 5120 }, .f8e4m3fn); - // Group 32 with an e4m3 scale: neither NVFP4 (wrong group) nor MX (wrong scale type). - try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 160 }, .f8e4m3fn))); - // Group 16 with an e8m0 scale: the mirror image, and equally unclaimed. - try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(nvfp4_packed, .init(.{ .dout = 17408, .sc = 320 }, .f8e8m0))); - // Per-channel in f32: what fusing per-tensor projections produces. The Metal - // per-channel kernels have f32 entries, so this is a real scheme, not a decline. - try expect(@as(?Quantization.Scheme, .fp8_per_channel), Quantization.Scheme.classify(fp8, .init(.{ .dout = 10240, .sc = 1 }, .f32))); - // A block grid that is neither per-channel nor 128x128. - try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify(fp8, .init(.{ .dout = 160, .sc = 80 }, .bf16))); - // Stacked MoE weights are rank 3 and go through the MoE path, not this one. - try expect(@as(?Quantization.Scheme, null), Quantization.Scheme.classify( - .init(.{ .expert = 128, .dout = 1408, .d = 1408 }, .u8), - .init(.{ .expert = 128, .dout = 1408, .sc = 176 }, .f8e4m3fn), - )); - - // `fp8_per_tensor` never checks the scale dtype, so a [1, 1] u8 scale on a [1, 32] - // weight satisfies it as well as `mxfp8`. Declaration order is what decides, and - // `mxfp8` is declared first - try expect(@as(?Quantization.Scheme, .mxfp8), Quantization.Scheme.classify( - .init(.{ .dout = 1, .d = 32 }, .f8e4m3fn), - .init(.{ .dout = 1, .sc = 1 }, .u8), - )); - - // A one-tile block grid is indistinguishable from a per-tensor scale by shape alone; the - // `count() > 1` guard on the grids is what makes per-tensor win. - try expect(@as(?Quantization.Scheme, .fp8_per_tensor), Quantization.Scheme.classify( - .init(.{ .dout = 128, .d = 128 }, .f8e4m3fn), - .init(.{ .dout = 1, .sc = 1 }, .bf16), - )); -} - -test "block FP8 quantization preserves axes and reconstructs constant blocks in FN and FNUZ" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - inline for (.{ DataType.f8e4m3fn, DataType.f8e4m3fnuz }) |dtype| { - const Local = struct { - const Outputs = struct { values: Tensor, scales: Tensor }; - fn forward(x: Tensor) Outputs { - const input = quantizeBlockFp8(x, .k, 128, dtype, .f32); - return .{ .values = input.values.convert(.f32), .scales = input.scales }; - } - }; - const x: Tensor = .init(.{ .b = 2, .k = 256, .s = 3 }, .bf16); - var exe = try platform.compileFn(allocator, io, Local.forward, .{x}, .{}); - defer exe.deinit(); - try zml.testing.expectEqualShapes(x.shape().withDtype(.f32), exe.output_shapes[0]); - try zml.testing.expectEqualShapes(x.shape().setDim(.k, 2).withDtype(.f32), exe.output_shapes[1]); - var host: [2 * 256 * 3]zml.floats.BFloat16 = undefined; - for (&host, 0..) |*value, i| { - const magnitude: f32 = if ((i / 3) % 256 < 128) 1.0 else 2.0; - value.* = .fromF32(switch (i % 3) { - 0 => 0.0, - 1 => magnitude, - else => -magnitude, - }); - } - var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); - defer buffer.deinit(); - var output: zml.Bufferized(Local.Outputs) = undefined; - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - runner.run(io, .{buffer}, .{&output}, .{ .wait = true }); - defer zml.Buffer.deinitAll(Local.Outputs, &output); - var values = try output.values.toSliceAlloc(allocator, io); - defer values.free(allocator); - var scales = try output.scales.toSliceAlloc(allocator, io); - defer scales.free(allocator); - for (scales.constItems(f32)) |scale| try std.testing.expect(std.math.isFinite(scale) and scale > 0); - for (values.constItems(f32), 0..) |value, i| { - const scale_index = (i / (256 * 3)) * 6 + ((i / 3) % 256) / 128 * 3 + i % 3; - try std.testing.expectApproxEqAbs(host[i].toF32(), value * scales.constItems(f32)[scale_index], 1e-6); - } - } -} - -test "block FP8 E8M0 scales preserve power-of-two boundaries and nonminor axes" { - const allocator = std.testing.allocator; - const io = std.testing.io; - const platform = zml.testing.env(); - inline for (.{ DataType.f8e4m3fn, DataType.f8e4m3fnuz }) |dtype| { - const Local = struct { - const Outputs = struct { values: Tensor, scales: Tensor }; - fn forward(x: Tensor) Outputs { - const input = quantizeBlockFp8(x, .k, 32, dtype, .f8e8m0); - return .{ .values = input.values.convert(.f32), .scales = input.scales.convert(.f32) }; - } - }; - const x: Tensor = .init(.{ .b = 2, .k = 256, .s = 3 }, .f32); - var exe = try platform.compileFn(allocator, io, Local.forward, .{x}, .{}); - defer exe.deinit(); - try zml.testing.expectEqualShapes(x.shape().withDtype(.f32), exe.output_shapes[0]); - try zml.testing.expectEqualShapes(x.shape().setDim(.k, 8).withDtype(.f32), exe.output_shapes[1]); - const fp8_max: f32 = if (dtype == .f8e4m3fn) 448.0 else 224.0; - var host: [2 * 256 * 3]f32 = undefined; - var maxima: [2 * 8 * 3]f32 = @splat(0); - for (&host, 0..) |*value, i| { - const block = (i / 3) % 256 / 32; - const scale_index = (i / (256 * 3)) * 24 + block * 3 + i % 3; - const boundary = fp8_max * std.math.pow(f32, 2, @as(f32, @floatFromInt(block)) - 4); - const bits: u32 = @bitCast(boundary); - const magnitude: f32 = switch (i % 3) { - 0 => @bitCast(bits - 1), - 1 => boundary, - else => @bitCast(bits + 1), - }; - value.* = if (block == 0) 0 else if (block == 1) 1e-30 else magnitude * (if (i % 2 == 0) @as(f32, 1) else -1); - maxima[scale_index] = @max(maxima[scale_index], @abs(value.*)); - } - var expected_scales: [2 * 8 * 3]f32 = undefined; - for (&expected_scales, maxima) |*scale, maximum| { - const raw = @max(maximum, 1e-4) * (1.0 / fp8_max); - const bits: u32 = @bitCast(raw); - const exponent = (bits >> 23) + @intFromBool(bits & 0x7fffff != 0); - scale.* = @bitCast(exponent << 23); - } - var buffer = try zml.Buffer.fromBytes(io, platform, x.shape(), std.mem.asBytes(&host)); - defer buffer.deinit(); - var output: zml.Bufferized(Local.Outputs) = undefined; - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - runner.run(io, .{buffer}, .{&output}, .{ .wait = true }); - defer zml.Buffer.deinitAll(Local.Outputs, &output); - var values = try output.values.toSliceAlloc(allocator, io); - defer values.free(allocator); - var scales = try output.scales.toSliceAlloc(allocator, io); - defer scales.free(allocator); - try std.testing.expectEqualSlices(f32, &expected_scales, scales.constItems(f32)); - for (values.constItems(f32), 0..) |value, i| { - const scale_index = (i / (256 * 3)) * 24 + ((i / 3) % 256) / 32 * 3 + i % 3; - const normalized = std.math.clamp(host[i] / expected_scales[scale_index], -fp8_max, fp8_max); - const expected = if (dtype == .f8e4m3fn) zml.floats.Float8E4M3FN.fromF32(normalized).toF32() else zml.floats.Float8E4M3FNUZ.fromF32(normalized).toF32(); - try std.testing.expectEqual(expected, value); - } - } -} diff --git a/zml/safetensors.zig b/zml/safetensors.zig index 3827fe27..5e4ff6ea 100644 --- a/zml/safetensors.zig +++ b/zml/safetensors.zig @@ -2,390 +2,98 @@ const std = @import("std"); const DataType = @import("dtype.zig").DataType; const Shape = @import("shape.zig").Shape; +const Dims = Shape.DimsArray; + +const StringBuilder = std.ArrayListUnmanaged(u8); + +pub const Tensors = std.StringArrayHashMapUnmanaged(Tensor); +pub const Metadatas = std.StringArrayHashMapUnmanaged(Metadata); const log = std.log.scoped(.@"zml/safetensors"); -// TODO: consider promoting this to the top level struct -// TODO: Feels like we are missing a mapping file -> tensors that would be helfpful for loading weights -/// Maps tensor names to their shape and positions in different sub files. -pub const TensorRegistry = struct { - arena: std.heap.ArenaAllocator, +const BYTES_HEADER = 8; - tensors: std.StringArrayHashMapUnmanaged(Tensor), - - /// Metadata found in `__metadata__` field of the safetensors header. - /// This is rarely use and the information there is not standardized, so we let user code handle it. - /// Here we just store the plain bytes, reach out if you need this to be already parsed. - metadata: []const u8, - - pub fn init(allocator: std.mem.Allocator) TensorRegistry { - return .{ - .arena = std.heap.ArenaAllocator.init(allocator), - .tensors = .{}, - .metadata = "", - }; - } - - pub fn deinit(self: *TensorRegistry) void { - self.arena.deinit(); - } - - pub fn fromRepo(allocator: std.mem.Allocator, io: std.Io, repo: std.Io.Dir) !TensorRegistry { - const file_type, const file = try resolveModelEntrypoint(io, repo); - defer file.close(io); - - var registry: TensorRegistry = .init(allocator); - switch (file_type) { - .safetensors => try registry.parseSafetensors(io, allocator, file), - .index => try registry.parseIndexJson(io, allocator, repo, file), - .full_index => try registry.parseFullIndexJson(io, allocator, repo, file), - .unknown => return error.InvalidPath, - } - return registry; - } - - pub fn fromPath(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !TensorRegistry { - const file_type: FileType = .resolve(path); - switch (file_type) { - .safetensors => { - // open directly - const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); - var registry: TensorRegistry = .init(allocator); - try registry.parseSafetensors(io, allocator, file); - return registry; - }, - .full_index, - .index, - => { - // resolve repo as parent folder then open - const repo_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - const repo = std.Io.Dir.openDir(.cwd(), io, repo_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); - - var registry: TensorRegistry = .init(allocator); - try switch (file_type) { - .index => registry.parseIndexJson(io, allocator, repo, file), - .full_index => registry.parseFullIndexJson(io, allocator, repo, file), - .safetensors, .unknown => unreachable, - }; - return registry; - }, - .unknown => { - // the path is possibly a folder, let's try that - const repo = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - return try fromRepo(allocator, io, repo); - }, - } - } - - /// Register a tensor, duplicate tensor name and uri. - /// If you create a lot of tensors sharing the same uri, - /// considere calling `registry.tensors` directly and avoid duplicated uris. - pub fn registerTensor(self: *TensorRegistry, tensor: Tensor) !void { - const allocator = self.arena.allocator(); - - const owned_name = try allocator.dupe(u8, tensor.name); - errdefer allocator.free(owned_name); - const owned_uri = try allocator.dupe(u8, tensor.file_uri); - errdefer allocator.free(owned_uri); - - const owned_tensor: Tensor = .{ - .name = owned_name, - .file_uri = owned_uri, - .shape = tensor.shape, - .offset = tensor.offset, - }; - try self.tensors.put(allocator, owned_name, owned_tensor); - } - - pub fn reader( - self: *TensorRegistry, - io: std.Io, - tensor_name: []const u8, - buffer: []u8, - ) !TensorReader { - const tensor = self.tensors.get(tensor_name) orelse { - log.err("Tensor {s} not found in registry", .{tensor_name}); - return error.TensorNotFound; - }; - - return try .init(io, tensor, buffer, .{}); - } - - pub fn totalBytes(self: *TensorRegistry) u64 { - var total: u64 = 0; - - var it = self.tensors.iterator(); - while (it.next()) |entry| { - total += entry.value_ptr.byteSize(); - } - - return total; - } - - /// Parse a given .safetensors file and create entries for each tensor - pub fn parseSafetensors( - self: *TensorRegistry, - io: std.Io, - scratch_allocator: std.mem.Allocator, - file: std.Io.File, - ) !void { - var tmp_state = std.heap.ArenaAllocator.init(scratch_allocator); - defer tmp_state.deinit(); - const tmp_arena = tmp_state.allocator(); - - const self_arena = self.arena.allocator(); - var owned_file_name = try self_arena.alloc(u8, std.fs.max_path_bytes); - const file_name_len = try file.realPath(io, owned_file_name); - // Try to release extra memory, but we don't care if it fails - _ = self_arena.resize(owned_file_name, file_name_len); - owned_file_name = owned_file_name[0..file_name_len]; - - const header_buf = try tmp_arena.alloc(u8, 64 * 1024); - var file_reader: std.Io.File.Reader = file.reader(io, header_buf); - - const json_header_length: u64 = try file_reader.interface.takeInt(u64, .little); - const BYTES_HEADER = 8; - const data_start_offset = BYTES_HEADER + json_header_length; - const json_data = try tmp_arena.alloc(u8, @intCast(json_header_length)); - - try file_reader.interface.readSliceAll(json_data); - const headers: std.json.Value = try std.json.parseFromSliceLeaky( - std.json.Value, - tmp_arena, - json_data, - .{}, - ); - - try self.tensors.ensureUnusedCapacity(self_arena, headers.object.count()); - for (headers.object.keys(), headers.object.values()) |key, value| { - if (std.mem.eql(u8, key, "__metadata__")) { - // value belong to tmp arena, duplicate it into self arena. - var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); - try std.json.fmt(value, .{ .whitespace = .minified }).format(&owned_metadata.writer); - self.metadata = owned_metadata.written(); - continue; - } - - const owned_name = try self_arena.dupe(u8, key); - const tensor: Tensor = try .parse(value.object, owned_file_name, owned_name, data_start_offset); - self.tensors.putAssumeCapacity(owned_name, tensor); - } - } - - pub fn parseIndexJson( - self: *TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - index_json_file: std.Io.File, - ) !void { - var tmp_state: std.heap.ArenaAllocator = .init(gpa); - defer tmp_state.deinit(); - const tmp = tmp_state.allocator(); - - const file_reader_buf = try tmp.alloc(u8, 64 * 1024); - var file_reader: std.Io.File.Reader = index_json_file.reader(io, file_reader_buf); - var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - tmp, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - var safetensors_files: std.StringArrayHashMapUnmanaged(void) = .empty; - try safetensors_files.ensureUnusedCapacity(tmp, 64); - if (index.object.get("weight_map")) |wm| { - for (wm.object.values()) |file_name| { - try safetensors_files.put(tmp, file_name.string, {}); - } - } else { - log.warn("No weight_map attribute found in index", .{}); - } - - for (safetensors_files.keys()) |file_name| { - const file = try repo.openFile(io, file_name, .{}); - defer file.close(io); - try self.parseSafetensors(io, tmp, file); - } - } - - pub fn parseFullIndexJson( - self: *TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - index_json_file: std.Io.File, - ) !void { - var tmp_state: std.heap.ArenaAllocator = .init(gpa); - defer tmp_state.deinit(); - const tmp = tmp_state.allocator(); - - const self_arena = self.arena.allocator(); - - var repo_uri_buf = try tmp.alloc(u8, 1024); - const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; - - // Note: this is relatively inefficient, we read the full file, copy most of it's content, - // then reduce to TensorRegistry format. - // We could update TensorRegistry as we stream the json. - // Zig std json API are a bit awkward around streaming though. - const file_reader_buf = try tmp.alloc(u8, 64 * 1024); - var file_reader = index_json_file.reader(io, file_reader_buf); - var json_reader: std.json.Reader = .init(tmp, &file_reader.interface); - const index = try std.json.parseFromTokenSourceLeaky( - std.json.Value, - tmp, - &json_reader, - .{ .allocate = .alloc_if_needed }, - ); - - var realpaths: std.StringArrayHashMapUnmanaged([]const u8) = .empty; - try realpaths.ensureUnusedCapacity(tmp, 64); - const wm = index.object.get("weight_map") orelse return error.InvalidJson; - - try self.tensors.ensureUnusedCapacity(self_arena, wm.object.count()); - for (wm.object.keys(), wm.object.values()) |key, value| { - const tensor_name = try self_arena.dupe(u8, key); - const object = value.object; - const filename = value.object.get("filename").?.string; - const filename_entry = try realpaths.getOrPut(tmp, filename); - const file_uri = if (filename_entry.found_existing) - filename_entry.value_ptr.* - else uri: { - const uri = try std.fs.path.join(self_arena, &.{ repo_name, filename }); - filename_entry.value_ptr.* = uri; - break :uri uri; - }; - - const tensor_info: Tensor = try .parse(object, file_uri, tensor_name, 0); - self.tensors.putAssumeCapacity(tensor_name, tensor_info); - } - - if (index.object.get("metadata")) |metadata| { - // value belong to tmp arena, duplicate it into self arena. - var owned_metadata: std.Io.Writer.Allocating = .init(self_arena); - try std.json.fmt(metadata, .{ .whitespace = .minified }).format(&owned_metadata.writer); - self.metadata = owned_metadata.written(); - } - } - - pub fn saveAsFullIndexJson( - self: *const TensorRegistry, - io: std.Io, - gpa: std.mem.Allocator, - repo: std.Io.Dir, - out: *std.Io.Writer, - ) !void { - var repo_uri_buf = try gpa.alloc(u8, std.fs.max_path_bytes); - defer gpa.free(repo_uri_buf); - const repo_name = repo_uri_buf[0..try repo.realPath(io, repo_uri_buf)]; - var total_size: u64 = 0; - for (self.tensors.values()) |value| total_size += value.byteSize(); - - try out.print( - \\{{ - \\"metadata": {{"num_tensors":{d},"total_size":{d}}}, - \\"weight_map": {{ - \\ - , .{ self.tensors.count(), total_size }); - - const n = self.tensors.count(); - for (0.., self.tensors.keys(), self.tensors.values()) |i, key, value| { - std.debug.assert(std.mem.startsWith(u8, value.file_uri, repo_name)); - const local_name = value.file_uri[repo_name.len + 1 ..]; - const end_offset = value.offset + value.byteSize(); - const safetensors_dtype: []const u8 = dt: { - for (safetensors_dtype_to_zml_dtype.keys(), safetensors_dtype_to_zml_dtype.values()) |st_dt, zml_dt| { - if (zml_dt == value.shape._dtype) break :dt st_dt; - } - // Note: we could bitcast to u8, - // but reaching this panic implies the user didn't loaded this TensorRegistry - // from a safetensors file, and manually created the dict. - // In which case, they know what they are doing, - // and can easily insert the bitcast themselves. - std.debug.panic("Safetensors doesn't support data type {}", .{value.shape._dtype}); - }; - const comma = if (i + 1 == n) "" else ","; - try out.print( - \\ "{s}": {{"filename":"{s}","shape":{f},"data_offsets":[{d},{d}],"dtype":"{s}"}}{s} - \\ - , .{ key, local_name, std.json.fmt(value.shape.dims(), .{ .whitespace = .minified }), value.offset, end_offset, safetensors_dtype, comma }); - } - - try out.writeAll( - \\} - \\} - \\ - ); - try out.flush(); - } +pub const FileType = enum { + index, + safetensors, + unknown, }; +pub fn fetchRegistry( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + model: std.Io.File, +) !TensorRegistry { + const file_type = try resolveFiletype(io, model); + + return switch (file_type) { + .index => blk: { + const file_reader_buf = try allocator.alloc(u8, 64 * 1024); + defer allocator.free(file_reader_buf); + + var index_reader = model.reader(io, file_reader_buf); + + var safetensors_index = try parseSafetensorsIndex( + allocator, + &index_reader.interface, + ); + defer safetensors_index.deinit(); + + var registry: TensorRegistry = try .initWithMetadata(allocator, safetensors_index.metadata); + + try parseSafetensorsIndexFiles( + allocator, + io, + ®istry, + safetensors_index, + repo, + ); + + break :blk registry; + }, + .safetensors => blk: { + var registry: TensorRegistry = .init(allocator); + + try parseSafetensors(allocator, io, ®istry, model); + + break :blk registry; + }, + else => return error.InvalidPath, + }; +} + pub const TensorReader = struct { - tensor: Tensor, file: std.Io.File, file_reader: std.Io.File.Reader, - alignment: ?std.mem.Alignment, remaining: u64, - padding_remaining: u64, - io: std.Io, interface: std.Io.Reader, + tensor: Tensor, - pub const InitOpts = struct { - alignment: ?std.mem.Alignment = null, - }; + pub const Error = error{TensorNotFound} || std.Io.File.OpenError || std.Io.File.Reader.SeekError || std.mem.Allocator.Error; pub fn init( io: std.Io, tensor: Tensor, buffer: []u8, - opts: InitOpts, - ) !TensorReader { + ) Error!TensorReader { const file = try std.Io.Dir.openFile(.cwd(), io, tensor.file_uri, .{ .mode = .read_only }); errdefer file.close(io); - const offset, const padding_remaining = if (opts.alignment) |alignment| blk: { - const aligned_offset = std.mem.alignBackward(u64, tensor.offset, alignment.toByteUnits()); - break :blk .{ aligned_offset, tensor.offset - aligned_offset }; - } else blk: { - break :blk .{ tensor.offset, 0 }; - }; - var file_reader = file.reader(io, buffer); - try file_reader.seekTo(offset); + try file_reader.seekTo(tensor.offset); return .{ - .tensor = tensor, .file = file, .file_reader = file_reader, - .alignment = opts.alignment, .remaining = tensor.byteSize(), - .padding_remaining = padding_remaining, + .tensor = tensor, .io = io, .interface = .{ - .vtable = if (opts.alignment) |_| blk: { - break :blk &.{ - .stream = streamAligned, - .discard = discardAligned, - }; - } else blk: { - break :blk &.{ - .stream = stream, - .discard = discard, - }; + .vtable = &.{ + .stream = stream, + .discard = discard, }, .buffer = &.{}, .seek = 0, @@ -398,17 +106,13 @@ pub const TensorReader = struct { self.file.close(self.io); } - // Limit single reads to 1GB to avoid issues on macOS where pread returns - // INT_MAX when the requested size is larger than 2GB - const max_single_read_bytes: u64 = 1 << 30; - fn stream(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { const self: *TensorReader = @fieldParentPtr("interface", r); if (self.remaining == 0) { return error.EndOfStream; } - const combined_limit = limit.min(.limited64(self.remaining)).min(.limited64(max_single_read_bytes)); + const combined_limit = limit.min(.limited64(self.remaining)); const n = try self.file_reader.interface.stream(w, combined_limit); self.remaining -= n; return n; @@ -417,6 +121,7 @@ pub const TensorReader = struct { fn discard(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { const self: *TensorReader = @fieldParentPtr("interface", r); if (self.remaining == 0) { + std.log.info("EOS2 !", .{}); return error.EndOfStream; } @@ -425,72 +130,6 @@ pub const TensorReader = struct { self.remaining -= n; return n; } - - fn streamAligned(r: *std.Io.Reader, w: *std.Io.Writer, limit: std.Io.Limit) std.Io.Reader.StreamError!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) { - return error.EndOfStream; - } - - const requested_bytes = @min(limit.minInt64(self.remaining), max_single_read_bytes); - - if (self.padding_remaining > 0) { - _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); - self.padding_remaining = 0; - } - - const align_bytes = self.alignment.?.toByteUnits(); - const file_offset = self.file_reader.pos - std.mem.alignBackward(u64, self.file_reader.pos, align_bytes); - const writer_addr = @intFromPtr(w.buffer.ptr) + w.end; - const writer_offset = writer_addr - std.mem.alignBackward(usize, writer_addr, align_bytes); - - if (file_offset != writer_offset) { - const skip = if (file_offset >= writer_offset) blk: { - break :blk file_offset - writer_offset; - } else blk: { - break :blk align_bytes - writer_offset + file_offset; - }; - - w.advance(skip); - } - - if (w.writableSliceGreedy(align_bytes)) |dest| { - const phys_buf = dest.ptr - file_offset; - const phys_pos = self.file_reader.pos - file_offset; - const phys_size = std.mem.alignBackward(usize, @min(dest.len, requested_bytes) + file_offset, align_bytes); - - if (phys_size > file_offset) { - const phys_read = self.file.readPositional(self.io, &.{phys_buf[0..phys_size]}, phys_pos) catch unreachable; - if (phys_read > file_offset) { - const bytes_read = phys_read - file_offset; - w.advance(bytes_read); - self.file_reader.pos += bytes_read; - self.remaining -= bytes_read; - return bytes_read; - } - } - } else |_| {} - - const bytes_read = try self.file_reader.interface.stream(w, .limited64(requested_bytes)); - self.remaining -= bytes_read; - - return bytes_read; - } - - fn discardAligned(r: *std.Io.Reader, limit: std.Io.Limit) std.Io.Reader.Error!usize { - const self: *TensorReader = @fieldParentPtr("interface", r); - if (self.remaining == 0) return 0; - - if (self.padding_remaining > 0) { - _ = try self.file_reader.interface.discard(.limited64(self.padding_remaining)); - self.padding_remaining = 0; - } - - const combined_limit = limit.min(.limited64(self.remaining)); - const n = try self.file_reader.interface.discard(combined_limit); - self.remaining -= n; - return n; - } }; pub const Tensor = struct { @@ -503,34 +142,6 @@ pub const Tensor = struct { return self.shape.byteSize(); } - pub fn reader(self: Tensor, io: std.Io, buffer: []u8, opts: TensorReader.InitOpts) !TensorReader { - return TensorReader.init(io, self, buffer, opts); - } - - pub fn parse(object: std.json.ObjectMap, file_uri: []const u8, name: []const u8, base_offset: u64) !Tensor { - const shape_field = object.get("shape").?.array; - if (shape_field.items.len > Shape.MAX_RANK) { - log.warn("Can't load tensor {s}, too many dims: {}", .{ name, shape_field.items.len }); - return error.TooManyDimensions; - } - - const offset_field = object.get("data_offsets").?; - const start: u64 = @intCast(offset_field.array.items[0].integer); - const dtype = try stringToDtype(object.get("dtype").?.string); - - var dims: Shape.DimsArray = .empty; - for (shape_field.items) |d| { - dims.appendAssumeCapacity(d.integer); - } - - return .{ - .file_uri = file_uri, - .name = name, - .shape = .init(dims.slice(), dtype), - .offset = base_offset + start, - }; - } - pub fn format(self: Tensor, writer: *std.Io.Writer) !void { try writer.print("Tensor(name={s} shape={f} size={d}, offset={d}, file_uri={s})", .{ self.name, @@ -542,100 +153,621 @@ pub const Tensor = struct { } }; -pub const FileType = enum { - full_index, - index, - safetensors, - unknown, +pub const TensorRegistry = struct { + arena: std.heap.ArenaAllocator, - pub fn resolve(path: []const u8) FileType { - if (std.mem.endsWith(u8, path, ".safetensors.index.json")) { - return .index; - } else if (std.mem.endsWith(u8, path, ".safetensors.full_index.json")) { - return .full_index; - } else if (std.mem.endsWith(u8, path, ".safetensors")) { - return .safetensors; + tensors: Tensors, + metadata: Metadatas, + + mutex: std.Io.Mutex = .init, + + pub fn init(allocator: std.mem.Allocator) TensorRegistry { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = .{}, + }; + } + + pub fn initWithMetadata( + allocator: std.mem.Allocator, + metadata: Metadatas, + ) !TensorRegistry { + var self: TensorRegistry = .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .tensors = .{}, + .metadata = .empty, + }; + + try self.mergeMetadata(metadata); + + return self; + } + + pub fn fromRepo( + allocator: std.mem.Allocator, + io: std.Io, + repo: std.Io.Dir, + ) !TensorRegistry { + const entrypoint = try resolveModelEntrypoint(io, repo); + return try fetchRegistry(allocator, io, repo, entrypoint); + } + + pub fn fromPath( + allocator: std.mem.Allocator, + io: std.Io, + path: []const u8, + ) !TensorRegistry { + var repo = try resolveModelRepo(io, path); + + if (std.mem.endsWith(u8, path, ".safetensors.index.json") or + std.mem.endsWith(u8, path, ".safetensors")) + { + return try fetchRegistry(allocator, io, repo, try repo.openFile(io, path, .{ .mode = .read_only })); } else { - return .unknown; + const entrypoint = try resolveModelEntrypoint(io, repo); + return try fetchRegistry(allocator, io, repo, entrypoint); } } + + pub fn deinit(self: *TensorRegistry) void { + const allocator = self.arena.allocator(); + self.tensors.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } + + pub fn mergeMetadata( + self: *TensorRegistry, + other: Metadatas, + ) !void { + // self.mutex.lock(); + // defer self.mutex.unlock(); + + const allocator = self.arena.allocator(); + + var it = other.iterator(); + while (it.next()) |entry| { + const key = try allocator.dupe(u8, entry.key_ptr.*); + const value = try entry.value_ptr.*.clone(allocator); + + const gop = try self.metadata.getOrPut(allocator, key); + if (gop.found_existing) { + gop.value_ptr.*.deinit(allocator); + gop.value_ptr.* = value; + log.info("Overwrote existing metadata key={s} with value={f}", .{ key, value }); + } else { + gop.value_ptr.* = value; + log.debug("Added new metadata key={s} with value={f}", .{ key, value }); + } + } + } + + pub fn registerTensor( + self: *TensorRegistry, + tensor: Tensor, + ) !void { + const allocator = self.arena.allocator(); + + // self.mutex.lock(); + // defer self.mutex.unlock(); + + var tensor_copy = tensor; + + tensor_copy.name = try allocator.dupe(u8, tensor.name); + tensor_copy.file_uri = try allocator.dupe(u8, tensor.file_uri); + + try self.tensors.put(allocator, tensor_copy.name, tensor_copy); + } + + pub fn reader( + self: *TensorRegistry, + io: std.Io, + tensor_name: []const u8, + buffer: []u8, + ) TensorReader.Error!TensorReader { + const tensor = self.tensors.get(tensor_name) orelse { + log.err("Tensor {s} not found in registry", .{tensor_name}); + return TensorReader.Error.TensorNotFound; + }; + + return try .init(io, tensor, buffer); + } + + pub fn iterator(self: *TensorRegistry) Tensors.Iterator { + return self.tensors.iterator(); + } + + pub fn totalBytes(self: *TensorRegistry) u64 { + var total: u64 = 0; + + var it = self.tensors.iterator(); + while (it.next()) |entry| { + total += entry.value_ptr.byteSize(); + } + + return total; + } }; +pub fn parseSafetensors( + allocator: std.mem.Allocator, + io: std.Io, + registry: *TensorRegistry, + file: std.Io.File, +) !void { + var arena = std.heap.ArenaAllocator.init(allocator); + defer arena.deinit(); + + const header_buf = try allocator.alloc(u8, 64 * 1024); + defer allocator.free(header_buf); + + var reader = file.reader(io, header_buf); + + var file_uri_buf: [1024]u8 = undefined; + const file_uri_len = try file.realPath(io, &file_uri_buf); + const file_uri = file_uri_buf[0..file_uri_len]; + + const json_header_length: u64 = try reader.interface.takeInt(u64, .little); + const data_start_offset = BYTES_HEADER + json_header_length; + const json_data = try arena.allocator().alloc(u8, @intCast(json_header_length)); + + try reader.interface.readSliceAll(json_data); + + const headers: std.json.Value = try std.json.parseFromSliceLeaky( + std.json.Value, + arena.allocator(), + json_data, + .{}, + ); + + var it = headers.object.iterator(); + while (it.next()) |entry| { + const key = entry.key_ptr.*; + const value = entry.value_ptr.*; + + if (std.mem.eql(u8, key, "__metadata__")) { + // Parse metadata into a local arena to avoid concurrent allocations + // against registry.arena (which is not thread-safe). + var local_arena = std.heap.ArenaAllocator.init(allocator); + defer local_arena.deinit(); + + const metas = try parseMetadata(local_arena.allocator(), value); + try registry.mergeMetadata(metas); + + continue; + } + + const shape_field = value.object.get("shape").?.array; + + if (shape_field.items.len > Shape.MAX_RANK) { + log.warn("Can't load tensor {s}, too many dims: {}", .{ key, shape_field.items.len }); + return error.TooManyDimensions; + } + + const offset_field = value.object.get("data_offsets").?; + const start: u64 = @intCast(offset_field.array.items[0].integer); + const dtype = try stringToDtype(value.object.get("dtype").?.string); + + var dims: Dims = .{}; + for (shape_field.items) |d| { + dims.appendAssumeCapacity(d.integer); + } + + const tensor: Tensor = .{ + .file_uri = file_uri, + .name = key, + .shape = .init(dims.slice(), dtype), + .offset = data_start_offset + start, + }; + + try registry.registerTensor(tensor); + } +} + +const SafetensorsIndex = struct { + pub const Map = std.StringArrayHashMapUnmanaged(std.ArrayList([]const u8)); + + arena: std.heap.ArenaAllocator, + + map: Map, + metadata: Metadatas, + + pub fn init(allocator: std.mem.Allocator) SafetensorsIndex { + return .{ + .arena = std.heap.ArenaAllocator.init(allocator), + .map = .empty, + .metadata = .empty, + }; + } + + pub fn iterator(self: *SafetensorsIndex) Map.Iterator { + return self.map.iterator(); + } + + pub fn deinit(self: *SafetensorsIndex) void { + const allocator = self.arena.allocator(); + self.map.deinit(allocator); + self.metadata.deinit(allocator); + self.arena.deinit(); + } +}; + +fn parseSafetensorsIndex( + allocator: std.mem.Allocator, + reader: *std.Io.Reader, +) !SafetensorsIndex { + var safetensors_index: SafetensorsIndex = .init(allocator); + errdefer safetensors_index.deinit(); + + const arena_allocator = safetensors_index.arena.allocator(); + var json_reader: std.json.Reader = .init(arena_allocator, reader); + + const index = try std.json.parseFromTokenSourceLeaky( + std.json.Value, + arena_allocator, + &json_reader, + .{ .allocate = .alloc_if_needed }, + ); + + const weight_map = index.object.get("weight_map"); + + if (weight_map) |wm| { + var it = wm.object.iterator(); + + while (it.next()) |entry| { + const weight_name = entry.key_ptr.*; + const filename = entry.value_ptr.string; + + const map_entry = try safetensors_index.map.getOrPut(arena_allocator, filename); + + if (!map_entry.found_existing) { + map_entry.value_ptr.* = .{}; + } + + try map_entry.value_ptr.append(arena_allocator, try arena_allocator.dupe(u8, weight_name)); + } + } else { + log.warn("No weight_map attribute found in index", .{}); + } + + if (index.object.get("metadata")) |metadata_val| { + safetensors_index.metadata = try parseMetadata(arena_allocator, metadata_val); + } + + return safetensors_index; +} + +fn parseSafetensorsIndexFiles( + allocator: std.mem.Allocator, + io: std.Io, + registry: *TensorRegistry, + safetensors_index: SafetensorsIndex, + repo: std.Io.Dir, +) !void { + const filenames = safetensors_index.map.keys(); + for (filenames) |filename| { + const file = try repo.openFile(io, filename, .{}); + defer file.close(io); + try parseSafetensors(allocator, io, registry, file); + } +} + +pub fn resolveFiletype(io: std.Io, file: std.Io.File) !FileType { + var path_buf: [1024]u8 = undefined; + const path_len = try file.realPath(io, &path_buf); + + if (std.mem.endsWith(u8, path_buf[0..path_len], ".safetensors.index.json")) { + return .index; + } else if (std.mem.endsWith(u8, path_buf[0..path_len], ".safetensors")) { + return .safetensors; + } else { + return .unknown; + } +} + pub const ModelPathResolutionError = error{ FileNotFound, InvalidPath, } || std.mem.Allocator.Error; -pub fn resolveModelRepo(io: std.Io, path: []const u8) ModelPathResolutionError!std.Io.Dir { - switch (FileType.resolve(path)) { - .full_index, .index, .safetensors => { - // The path is a file, let's open it's folder - const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; +pub fn resolveModelRepo( + io: std.Io, + path: []const u8, +) ModelPathResolutionError!std.Io.Dir { + if (std.mem.endsWith(u8, path, ".safetensors.index.json") or + std.mem.endsWith(u8, path, ".safetensors")) + { + const dir_path = std.fs.path.dirname(path) orelse return ModelPathResolutionError.InvalidPath; - const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; + const dir = std.Io.Dir.openDir(.cwd(), io, dir_path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, }; + }; - return dir; - }, - .unknown => { - // The path is probably a directory, let's try to open it as such - const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { - return switch (e) { - error.FileNotFound => ModelPathResolutionError.FileNotFound, - else => ModelPathResolutionError.InvalidPath, - }; - }; - return dir; - }, + return dir; } -} -pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!struct { FileType, std.Io.File } { - const default_entrypoints: []const struct { FileType, []const u8 } = &.{ - .{ .full_index, "model.safetensors.full_index.json" }, - .{ .full_index, "consolidated.safetensors.full_index.json" }, - .{ .index, "model.safetensors.index.json" }, - .{ .index, "consolidated.safetensors.index.json" }, - .{ .safetensors, "model.safetensors" }, - .{ .safetensors, "consolidated.safetensors" }, + const dir = std.Io.Dir.openDir(.cwd(), io, path, .{}) catch |e| { + return switch (e) { + error.FileNotFound => ModelPathResolutionError.FileNotFound, + else => ModelPathResolutionError.InvalidPath, + }; }; - for (default_entrypoints) |entrypoint| { - const file_type, const file_name = entrypoint; - const file = repo.openFile(io, file_name, .{ .mode = .read_only }) catch continue; - - return .{ file_type, file }; - } - - return error.FileNotFound; + return dir; } -const safetensors_dtype_to_zml_dtype = std.StaticStringMap(DataType).initComptime(.{ - .{ "F64", .f64 }, - .{ "F32", .f32 }, - .{ "F16", .f16 }, - .{ "BF16", .bf16 }, - .{ "F8_E8M0", .f8e8m0 }, - .{ "F8_E4M3", .f8e4m3fn }, - .{ "F4_E2M1", .f4e2m1 }, - .{ "I64", .i64 }, - .{ "I32", .i32 }, - .{ "I16", .i16 }, - .{ "I8", .i8 }, - .{ "U64", .u64 }, - .{ "U32", .u32 }, - .{ "U16", .u16 }, - .{ "U8", .u8 }, - .{ "BOOL", .bool }, -}); +pub fn resolveModelEntrypoint(io: std.Io, repo: std.Io.Dir) ModelPathResolutionError!std.Io.File { + { + const index_name = "model.safetensors.index.json"; + if (repo.openFile(io, index_name, .{ .mode = .read_only })) |index_file| { + return index_file; + } else |_| {} + } -fn stringToDtype(safetensor_dtype: []const u8) !DataType { - return safetensors_dtype_to_zml_dtype.get(safetensor_dtype) orelse { - log.err("Unsupported safetensor data type: {s}", .{safetensor_dtype}); + { + const model_name = "model.safetensors"; + if (repo.openFile(io, model_name, .{ .mode = .read_only })) |model_file| { + return model_file; + } else |_| {} + } + + return ModelPathResolutionError.FileNotFound; +} + +const Metadata = union(enum) { + null: void, + int: i64, + float: f64, + bool: bool, + string: []const u8, + + array_bool: []const bool, + array_int: []const i64, + array_float: []const f64, + array_string: []const []const u8, + + pub const ItemType = enum { + int, + float, + bool, + string, + + pub fn toZigType(comptime kind: ItemType) type { + return switch (kind) { + .int => i64, + .float => f64, + .bool => bool, + .string => []const u8, + }; + } + }; + + pub fn wrap(x: anytype) Metadata { + return switch (@TypeOf(x)) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => .{ .int = @intCast(x) }, + inline f16, f32, f64 => .{ .float = @floatCast(x) }, + bool => .{ .bool = x }, + []const u8 => .{ .string = x }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(x))), + }; + } + + pub fn copySlice(allocator: std.mem.Allocator, any_slice: anytype) !Metadata { + return switch (@TypeOf(any_slice[0])) { + inline u8, i8, u16, i16, u32, i32, u64, i64 => { + const res = try allocator.alloc(i64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @intCast(val); + return .{ .array_int = res }; + }, + inline f16, f32, f64 => { + const res = try allocator.alloc(f64, any_slice.len); + for (res, any_slice) |*r, val| r.* = @floatCast(val); + return .{ .array_float = res }; + }, + bool => .{ .array_bool = try allocator.dupe(bool, any_slice) }, + []const u8 => .{ .array_string = try allocator.dupe([]const u8, @alignCast(any_slice)) }, + else => @panic("Unsupported type for Value: " ++ @typeName(@TypeOf(any_slice))), + }; + } + + pub fn format( + self: Metadata, + writer: *std.Io.Writer, + ) !void { + switch (self) { + .null => _ = try writer.write("null"), + .string => |s| try writer.print("{s}", .{s}), + .bool => |b| try writer.print("{}", .{b}), + .int => |i| try writer.print("{d}", .{i}), + .float => |f| try writer.print("{d}", .{f}), + .array_bool => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{}", .{v}); + } + try writer.writeByte(']'); + }, + .array_int => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_float => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("{d}", .{v}); + } + try writer.writeByte(']'); + }, + .array_string => |arr| { + try writer.writeByte('['); + for (arr, 0..) |v, i| { + if (i > 0) try writer.writeAll(", "); + try writer.print("\"{s}\"", .{v}); + } + try writer.writeByte(']'); + }, + } + } + + pub fn deinit(self: Metadata, allocator: std.mem.Allocator) void { + switch (self) { + .string => |s| allocator.free(s), + .array_bool => |s| allocator.free(s), + .array_int => |s| allocator.free(s), + .array_float => |s| allocator.free(s), + .array_string => |s| { + for (s) |str| allocator.free(str); + allocator.free(s); + }, + else => {}, + } + } + + pub fn clone(self: Metadata, allocator: std.mem.Allocator) !Metadata { + return switch (self) { + .null => .null, + .int => |v| .{ .int = v }, + .float => |v| .{ .float = v }, + .bool => |v| .{ .bool = v }, + .string => |s| .{ .string = try allocator.dupe(u8, s) }, + .array_bool => |s| .{ .array_bool = try allocator.dupe(bool, s) }, + .array_int => |s| .{ .array_int = try allocator.dupe(i64, s) }, + .array_float => |s| .{ .array_float = try allocator.dupe(f64, s) }, + .array_string => |s| blk: { + const new_slice = try allocator.alloc([]const u8, s.len); + errdefer allocator.free(new_slice); + for (s, 0..) |str, i| { + new_slice[i] = try allocator.dupe(u8, str); + } + break :blk .{ .array_string = new_slice }; + }, + }; + } +}; + +fn stringToDtype(safetensor_type: []const u8) !DataType { + const map = std.StaticStringMap(DataType).initComptime(.{ + .{ "F64", .f64 }, + .{ "F32", .f32 }, + .{ "F16", .f16 }, + .{ "BF16", .bf16 }, + .{ "F8_E4M3", .f8e4m3fn }, + .{ "I64", .i64 }, + .{ "I32", .i32 }, + .{ "I16", .i16 }, + .{ "I8", .i8 }, + .{ "U64", .u64 }, + .{ "U32", .u32 }, + .{ "U16", .u16 }, + .{ "U8", .u8 }, + .{ "BOOL", .bool }, + }); + + return map.get(safetensor_type) orelse { + log.err("Unsupported safetensor data type: {s}", .{safetensor_type}); return error.UnsupportedDataType; }; } + +fn parseMetadata(allocator: std.mem.Allocator, val: std.json.Value) !Metadatas { + var metadatas: Metadatas = .{}; + var prefix_buf: [1024]u8 = undefined; + var prefix = StringBuilder.initBuffer(&prefix_buf); + + try populateMetadata(allocator, &prefix, val, &metadatas); + + return metadatas; +} + +fn populateMetadata(allocator: std.mem.Allocator, prefix: *StringBuilder, val: std.json.Value, metadatas: *Metadatas) !void { + const key = prefix.items; + return switch (val) { + .null => try metadatas.put(allocator, try allocator.dupe(u8, key), .null), + .bool => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .bool = v }), + .integer => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .int = v }), + .float => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .float = v }), + .number_string, .string => |v| try metadatas.put(allocator, try allocator.dupe(u8, key), .{ .string = try allocator.dupe(u8, v) }), + .array => |v| { + if (v.items.len == 0) return; + if (validSlice(v)) |item_type| { + const data: Metadata = switch (item_type) { + .bool => blk: { + const values = try allocator.alloc(bool, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.bool; + break :blk .{ .array_bool = values }; + }, + .integer => blk: { + const values = try allocator.alloc(i64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.integer; + break :blk .{ .array_int = values }; + }, + .float => blk: { + const values = try allocator.alloc(f64, v.items.len); + for (v.items, 0..) |item, i| values[i] = item.float; + break :blk .{ .array_float = values }; + }, + inline .string, .number_string => |tag| blk: { + const values = try allocator.alloc([]const u8, v.items.len); + for (v.items, 0..) |item, i| { + values[i] = try allocator.dupe(u8, @field(item, @tagName(tag))); + } + break :blk .{ .array_string = values }; + }, + .null, .array, .object => unreachable, + }; + try metadatas.put(allocator, try allocator.dupe(u8, key), data); + } else { + for (v.items, 0..) |item, i| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.items.len += std.fmt.printInt(prefix.unusedCapacitySlice(), i, 10, .lower, .{}); + try populateMetadata(allocator, prefix, item, metadatas); + prefix.items.len = old_len; + } + } + }, + .object => |v| { + var obj_iter = v.iterator(); + while (obj_iter.next()) |entry| { + const old_len = prefix.items.len; + if (prefix.items.len > 0) { + prefix.appendAssumeCapacity('.'); + } + prefix.appendSliceAssumeCapacity(entry.key_ptr.*); + try populateMetadata(allocator, prefix, entry.value_ptr.*, metadatas); + prefix.items.len = old_len; + } + }, + }; +} + +/// We can only create a Zig slice out of json array, if all values +/// in the array have the same type. +fn validSlice(v: std.json.Array) ?std.meta.Tag(std.json.Value) { + if (v.items.len == 0) return null; + + const item_type: std.meta.Tag(std.json.Value) = v.items[0]; + switch (item_type) { + .null, .array, .object => return null, + else => {}, + } + + for (v.items[1..]) |item| { + if (item != item_type) + return null; + } + + return item_type; +} diff --git a/zml/shape.zig b/zml/shape.zig index 386b594e..56c724e0 100644 --- a/zml/shape.zig +++ b/zml/shape.zig @@ -6,26 +6,141 @@ const stdx = @import("stdx"); const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; -const Sharding = @import("Sharding.zig"); const log = std.log.scoped(.shape); -test { - std.testing.refAllDecls(Shape); -} - /// Represent the shape of a tensor. -/// Contains also the sharding information, -/// so it describes exactly how the corresponding tensor/buffer will be layed out -/// across all devices of the current Platform. -/// -/// By default all created Shape are replicated, meaning the corresponding tensor / buffer will be copied on each device. -/// `.withPartitioning` allows to specify how it should be split. pub const Shape = struct { - _dtype: DataType, - _dims: DimsArray, - _tags: TagsArray, - _sharding: Sharding, + pub const MAX_RANK: u8 = 8; + + pub const PartitionSpec = union(enum) { + axis: Tag, + open, + replicated, + unknown, + + pub fn init(comptime value: anytype) PartitionSpec { + const T = @TypeOf(value); + + if (T == PartitionSpec) { + return value; + } + + if (comptime T == @EnumLiteral()) { + const name = @tagName(value); + inline for (std.meta.fields(PartitionSpec)) |field| { + if (field.type == void and std.mem.eql(u8, field.name, name)) { + return @field(PartitionSpec, field.name); + } + } + } + + if (comptime isTagConvertible(T)) { + const tag_ = Shape.toTag(value); + + return .{ .axis = tag_ }; + } + + stdx.debug.compileError("PartitionSpec.init expected a tag, got: {any}", .{T}); + } + + test "PartitionSpec.init" { + const spec_axis: PartitionSpec = .{ .axis = Shape.toTag(.a) }; + try testing.expectEqual(spec_axis, PartitionSpec.init(.a)); + + const spec_axis_open: PartitionSpec = .open; + try testing.expect(spec_axis_open.eql(.init(.open))); + } + + pub fn eql(self: PartitionSpec, other: PartitionSpec) bool { + if (@as(u4, @intFromEnum(self)) != @as(u4, @intFromEnum(other))) { + return false; + } + + switch (self) { + .axis => |t1| { + const t2 = other.axis; + return std.mem.eql(u8, std.mem.span(t1), std.mem.span(t2)); + }, + else => return true, + } + } + + test "PartitionSpec.eql" { + const spec_axis: PartitionSpec = .init(.a); + const spec_axis_b: PartitionSpec = .init(.b); + const spec_open: PartitionSpec = .open; + const spec_replicated: PartitionSpec = .replicated; + const spec_unknown: PartitionSpec = .unknown; + + try testing.expect(spec_axis.eql(.{ .axis = Shape.toTag(.a) })); + try testing.expect(!spec_axis.eql(spec_axis_b)); + try testing.expect(!spec_axis.eql(.open)); + try testing.expect(!spec_axis.eql(.replicated)); + try testing.expect(!spec_axis.eql(.unknown)); + + try testing.expect(spec_open.eql(.open)); + try testing.expect(!spec_open.eql(spec_axis)); + try testing.expect(!spec_open.eql(.replicated)); + try testing.expect(!spec_open.eql(.unknown)); + + try testing.expect(spec_replicated.eql(.replicated)); + try testing.expect(!spec_replicated.eql(spec_axis)); + try testing.expect(!spec_replicated.eql(.open)); + try testing.expect(!spec_replicated.eql(.unknown)); + + try testing.expect(spec_unknown.eql(.unknown)); + try testing.expect(!spec_unknown.eql(spec_axis)); + try testing.expect(!spec_unknown.eql(.open)); + try testing.expect(!spec_unknown.eql(.replicated)); + } + + pub fn isClosed(self: PartitionSpec) bool { + return switch (self) { + .axis => true, + .open => false, + .replicated => true, + .unknown => false, + }; + } + + test isClosed { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.isClosed()); + + const spec_open: PartitionSpec = .open; + try testing.expect(!spec_open.isClosed()); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.isClosed()); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(!spec_unknown.isClosed()); + } + + pub fn toTag(self: PartitionSpec) Tag { + return switch (self) { + .axis => |t| return t, + .open => TagOpen, + .replicated => TagReplicated, + .unknown => TagUnknown, + }; + } + + test "PartitionSpec.toTag" { + const spec_axis: PartitionSpec = .init(.a); + try testing.expect(spec_axis.toTag() == Shape.toTag(.a)); + + const spec_open: PartitionSpec = .open; + try testing.expect(spec_open.toTag() == TagOpen); + + const spec_replicated: PartitionSpec = .replicated; + try testing.expect(spec_replicated.toTag() == TagReplicated); + + const spec_unknown: PartitionSpec = .unknown; + try testing.expect(spec_unknown.toTag() == TagUnknown); + } + }; pub const Tag = [*:0]const u8; pub const TagUnknown = "_".ptr; @@ -33,10 +148,17 @@ pub const Shape = struct { const TagReplicated = "replicated".ptr; const TagOpen = "open".ptr; - pub const MAX_RANK: u8 = constants.MAX_RANK; pub const DimsArray = stdx.BoundedArray(i64, constants.MAX_RANK); pub const TagsArray = stdx.BoundedArray(Tag, constants.MAX_RANK); pub const AxesArray = stdx.BoundedArray(u3, constants.MAX_RANK); + pub const PartitionArray = stdx.BoundedArray(PartitionSpec, constants.MAX_RANK); + + const UnknownTags: TagsArray = .{ .len = 0, .buffer = [_]Tag{TagUnknown} ** constants.MAX_RANK }; + + _dtype: DataType, + _dims: DimsArray = .{}, + _tags: TagsArray = UnknownTags, + _partitioning: PartitionArray = .{}, pub fn parseDimensions(v: anytype) struct { DimsArray, TagsArray } { const T = @TypeOf(v); @@ -46,8 +168,8 @@ pub const Shape = struct { } if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger)) { - var dims_: DimsArray = .empty; - var tags_: TagsArray = .empty; + var dims_ = DimsArray.init(0) catch unreachable; + var tags_ = TagsArray.init(0) catch unreachable; for (v) |d| { dims_.appendAssumeCapacity(@intCast(d)); tags_.appendAssumeCapacity(TagUnknown); @@ -56,21 +178,21 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(T)) { - var dims_: DimsArray = .empty; - var tags_: TagsArray = .empty; - inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { - const fv = @field(v, field_name); - if (comptime stdx.meta.isInteger(Field)) { + var dims_: DimsArray = .{}; + var tags_: TagsArray = .{}; + inline for (std.meta.fields(T)) |field| { + const fv = @field(v, field.name); + if (comptime stdx.meta.isInteger(field.type)) { dims_.appendAssumeCapacity(@intCast(fv)); } else if (@TypeOf(fv) == @EnumLiteral() and comptime isAutoDim(fv)) { dims_.appendAssumeCapacity(-1); } else { - stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field_name, Field }); + stdx.debug.compileError("Field {s} should be an integer or an auto dimension, got {}", .{ field.name, field.type }); } if (comptime stdx.meta.isTuple(T)) { tags_.appendAssumeCapacity(TagUnknown); } else { - tags_.appendAssumeCapacity(toTag(field_name)); + tags_.appendAssumeCapacity(toTag(field)); } } @@ -95,8 +217,8 @@ pub const Shape = struct { return self.parseAxes(self.tags()); } - var axes_: AxesArray = .empty; - var tags_: TagsArray = .empty; + var axes_ = AxesArray.init(0) catch unreachable; + var tags_ = TagsArray.init(0) catch unreachable; if (comptime stdx.meta.isSliceOfAny(T, isAxisConvertible)) { for (v) |d| { @@ -107,9 +229,9 @@ pub const Shape = struct { } if (comptime stdx.meta.isTupleOfAny(T, isAxisConvertible)) { - inline for (comptime std.meta.fieldNames(T)) |field_name| { - axes_.appendAssumeCapacity(self.axis(@field(v, field_name))); - tags_.appendAssumeCapacity(self.tag(@field(v, field_name))); + inline for (std.meta.fields(T)) |field| { + axes_.appendAssumeCapacity(self.axis(@field(v, field.name))); + tags_.appendAssumeCapacity(self.tag(@field(v, field.name))); } return .{ axes_, tags_ }; } @@ -117,69 +239,41 @@ pub const Shape = struct { stdx.debug.compileError("Wrong type, got {}. Expected .{{.a, .b}}", .{T}); } - pub fn parseTags(v: anytype) []const Shape.Tag { + pub fn parseTags(v: anytype) TagsArray { const T = @TypeOf(v); - if (T == []const Shape.Tag) return v; - comptime validateTags(T); - return comptime frozen: { - var tagz: [v.len]Shape.Tag = undefined; - for (v, 0..) |field, i| tagz[i] = @tagName(field).ptr; - // Force the creation of a comptime constant, so that we can return a pointer to it. - const frozen_tags = tagz; - break :frozen &frozen_tags; - }; - } - - fn validateTags(T: type) void { - const message = "Expected an enum-literal tuple such as .{ .b, .d }, got " ++ @typeName(T); - switch (@typeInfo(T)) { - .@"struct" => |info| { - if (!info.is_tuple) @compileError(message); - for (info.field_types) |Field| { - if (Field != @EnumLiteral()) @compileError(message); - } - if (info.field_types.len > Shape.MAX_RANK) @compileError("Too many shape tags (maximum 8)"); - }, - else => @compileError(message), + stdx.debug.assertComptime(stdx.meta.isTupleOf(T, @EnumLiteral()), "Wrong type, got {}. Expected .{{ .a, .b }}", .{T}); + var tags_ = TagsArray.init(0) catch unreachable; + inline for (v) |field| { + tags_.appendAssumeCapacity(toTag(field)); } + return tags_; } /// Create a shape from a struct literal, eg: /// Shape.init(.{ .h = 1024, .w = 512, .c = 3 }); /// Shape.init(.{ 1024, 512, 3 }); - /// - /// The created shape is replicated. pub fn init(dimz: anytype, dt: DataType) Shape { - const _dims, const _tags = parseDimensions(dimz); - return .{ - ._dtype = dt, - ._dims = _dims, - ._tags = _tags, - ._sharding = .replicated, - }; + var res: Shape = .{ ._dtype = dt }; + res._dims, res._tags = parseDimensions(dimz); + res._partitioning.appendNTimes(.unknown, res._dims.len) catch @panic("Rank too large"); + + return res; } pub fn scalar(dt: DataType) Shape { - return .{ - ._dtype = dt, - ._dims = .empty, - ._tags = .{ .len = 0, .buffer = @splat(TagUnknown) }, - ._sharding = .replicated, - }; + return .{ ._dtype = dt }; } /// Creates a Shape with dims set to `.{0, 1, 2, ..., rank-1}`. pub fn range(rank_: usize, dt: DataType) Shape { - stdx.debug.assert(rank_ <= MAX_RANK, "zml.Shape.range expects a maximum rank of {d}, got {}", .{ MAX_RANK, rank_ }); - var res: Shape = .{ - ._dtype = dt, - ._dims = .empty, - ._tags = .{ .buffer = @splat(TagUnknown), .len = rank_ }, - ._sharding = .replicated, - }; + var res: Shape = .{ ._dtype = dt }; for (0..rank_) |i| { - res._dims.appendAssumeCapacity(@intCast(i)); + res._dims.append(@intCast(i)) catch { + stdx.debug.panic("Too many dimensions! Max: {d}, passed: {d}", .{ res._dims.capacity(), rank_ }); + }; + res._tags.append(TagUnknown) catch unreachable; } + res._partitioning.appendNTimes(.unknown, rank_) catch @panic("Rank too large"); return res; } @@ -208,7 +302,9 @@ pub const Shape = struct { fn isTagConvertible(comptime T: type) bool { return switch (T) { - @EnumLiteral(), [:0]const u8, Tag => true, + @EnumLiteral() => true, + std.builtin.Type.StructField => true, + Tag => true, else => false, }; } @@ -217,16 +313,14 @@ pub const Shape = struct { const T = @TypeOf(v); return switch (T) { @EnumLiteral() => @tagName(v).ptr, - [:0]const u8 => v.ptr, + std.builtin.Type.StructField => v.name.ptr, Tag => v, - else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a field name, got {}", .{T}), + else => stdx.debug.compileError("Shape tag should be an @EnumLiteral(), a Shape.Tag or a StructField, got {}", .{T}), }; } - fn ensureAttributesAreSync(self: Shape) void { - if (builtin.mode == .debug) { - stdx.debug.assert(self._dims.len == self._tags.len, "Tags, dims and partitioning have diverged! dims={d} tags={d}", .{ self._dims.len, self._tags.len }); - } + inline fn ensureAttributesAreSync(self: Shape) void { + stdx.debug.assert(self._dims.len == self._tags.len and self._dims.len == self._partitioning.len, "Tags, dims and partitioning have diverged! dims={d} tags={d} partitioning={d}", .{ self._dims.len, self._tags.len, self._partitioning.len }); } pub fn tag(self: Shape, ax: anytype) Tag { @@ -255,7 +349,7 @@ pub const Shape = struct { } pub fn hasTag(self: Shape, tag_: anytype) ?u3 { - return axisFromTagMaybe(self.tags(), toTag(tag_)); + return self.axisFromTagMaybe(toTag(tag_)); } pub fn hasTags(self: Shape, tagz: anytype) bool { @@ -293,21 +387,15 @@ pub const Shape = struct { self.ensureAttributesAreSync(); const T = @TypeOf(axis_); - // Special handling of unsigned int, to remove runtime computations and favor inlining - return switch (@typeInfo(T)) { - .int => |info| if (info.signedness == .signed) - self.axisFromInt(axis_) - else - @intCast(axis_), - .comptime_int => if (axis_ < 0) - self.axisFromInt(axis_) - else - axis_, - else => if (comptime isTagConvertible(T)) - self.axisFromTag(toTag(axis_)) - else - stdx.debug.compileError("Wrong axis type, expected .literal, or an integer, got: {any}", .{T}), - }; + if (comptime stdx.meta.isInteger(T)) { + return self.axisFromInt(@intCast(axis_)); + } + + if (comptime isTagConvertible(T)) { + return self.axisFromTag(toTag(axis_)); + } + + stdx.debug.compileError("Wrong axis type, expected .literal, or an integer, got: {any}", .{T}); } pub fn axes(self: Shape, axes_: anytype) AxesArray { @@ -319,7 +407,7 @@ pub const Shape = struct { return self.axes(axes_.tags()); } - var res: AxesArray = .empty; + var res = AxesArray.init(0) catch unreachable; if (comptime stdx.meta.isSliceOfAny(T, stdx.meta.isInteger) or stdx.meta.isSliceOf(T, Tag)) { for (axes_) |ax| { @@ -329,8 +417,8 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(T)) { - inline for (comptime std.meta.fieldNames(T)) |field_name| { - res.appendAssumeCapacity(self.axis(@field(axes_, field_name))); + inline for (std.meta.fields(T)) |field| { + res.appendAssumeCapacity(self.axis(@field(axes_, field.name))); } return res; } @@ -338,7 +426,7 @@ pub const Shape = struct { stdx.debug.compileError("axes expects an int-tuple or a tuple of enum literal, got {}", .{T}); } - pub fn axisFromInt(self: Shape, a: isize) u3 { + fn axisFromInt(self: Shape, a: isize) u3 { const rk: i8 = self.rank(); if (a < -rk or a > rk) { stdx.debug.panic("Tensor {f} doesn't have dimension: {d}", .{ self, a }); @@ -349,7 +437,7 @@ pub const Shape = struct { @intCast(a); } - pub fn axisFromTagMaybe(tagz: []const Tag, t: Tag) ?u3 { + fn axisFromTagMaybe(self: Shape, t: Tag) ?u3 { if (t == TagUnknown) return null; if (axisFromLiteralInt(t)) |ax| return ax; @@ -357,7 +445,7 @@ pub const Shape = struct { if (@inComptime()) { // At comptime two duplicated strings may have two different representations const t_bytes: []const u8 = std.mem.span(t); - for (tagz, 0..) |self_tag, ax| { + for (self.tags(), 0..) |self_tag, ax| { if (std.mem.eql(u8, t_bytes, std.mem.span(self_tag))) { return @truncate(ax); } @@ -366,7 +454,7 @@ pub const Shape = struct { } // But at runtime the comptime strings have been deduplicated and ptr match is enough. - if (std.mem.indexOfScalar(Tag, tagz, t)) |ax| { + if (std.mem.indexOfScalar(Tag, self.tags(), t)) |ax| { return @truncate(ax); } return null; @@ -383,7 +471,7 @@ pub const Shape = struct { fn axisFromTag(self: Shape, d: Tag) u3 { stdx.debug.assert(d != TagUnknown, "The unknown tag .{s} can't be used to fetch axis in {f}", .{ d, self }); - return axisFromTagMaybe(self.tags(), d) orelse { + return self.axisFromTagMaybe(d) orelse { stdx.debug.panic("Tensor {f} doesn't have dimension with tag: {s}", .{ self, d }); }; } @@ -398,15 +486,15 @@ pub const Shape = struct { pub fn count(self: Shape) usize { var res: i64 = 1; for (self.dims()) |d| { - stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {f}", .{&self}); + stdx.debug.assert(d >= 0, "Can't count elements in shape with negative dimension: {f}", .{self}); res *= d; } return @intCast(res); } - /// Total size in bytes needed to represent this shape on host + /// Total size in bytes needed to represent this shape. pub fn byteSize(self: Shape) usize { - return std.math.divCeil(usize, self.count() * self.dtype().bitSizeOf(), 8) catch unreachable; + return self.dtype().sizeOf() * self.count(); } /// Compares the two shapes described, ignoring tagging. @@ -425,7 +513,8 @@ pub const Shape = struct { } /// Format the shape. - /// "Shape({.a=10, .b=20}, dtype=.f32)" + /// Default format: "Shape({.a=10, .b=20}, dtype=.f32)" + /// Bare format {_}: "{.a=10, .b=20}, dtype=.f32" pub fn format(self: Shape, writer: *std.Io.Writer) !void { _ = try writer.writeByte('{'); @@ -439,16 +528,9 @@ pub const Shape = struct { try writer.print("{d}", .{d}); } - const partitioning = self._sharding.get(i); - switch (partitioning) { - .replicated => {}, - .open => try writer.writeAll("/?"), - // this state is invalid, but don't panic cause we may be called from a panic - .out_of_bound => try writer.writeAll("/!"), - else => { - const mesh_ax = partitioning.meshAxis().?; - try writer.print("/{s}", .{self._sharding.mesh.?.logical.axes.get(mesh_ax)}); - }, + const part = self._partitioning.get(i); + if (part.toTag() != TagUnknown) { + try writer.print("/{s}", .{part.toTag()}); } need_comma = true; @@ -456,10 +538,6 @@ pub const Shape = struct { if (need_comma) try writer.writeByte(','); _ = try writer.write(@tagName(self.dtype())); _ = try writer.writeByte('}'); - if (self._sharding.mesh) |m| { - try writer.writeByte('@'); - try writer.writeAll(m.name); - } } /// Broadcasts a Tensor to the given shape, extending dimensions if needed. @@ -485,24 +563,16 @@ pub const Shape = struct { return true; } - /// Reshape a given shape but keep the same total number of elements. - /// Partitioning information is generally NOT preserved, - /// unless input was fully replicated. pub fn reshape(self: Shape, new_shape_: anytype) Shape { - const _dims, const _tags = parseDimensions(new_shape_); - var new_shape: Shape = .{ - ._dtype = self._dtype, - ._dims = _dims, - ._tags = _tags, - ._sharding = .{ .mesh = self._sharding.mesh, .partition = if (self.isFullyReplicated()) .replicated(_dims.len) else .open(_dims.len) }, - }; - new_shape.inferMissingAxis(self.count()) catch |err| { - std.debug.panic("Can't reshape {any} to {any}: {t}", .{ self.dims(), new_shape.dims(), err }); - }; + var new_shape: Shape = .{ ._dtype = self.dtype() }; + new_shape._dims, new_shape._tags = parseDimensions(new_shape_); + new_shape._partitioning.appendNTimes(.unknown, new_shape._dims.len) catch @panic("Rank too large"); + new_shape.inferMissingAxis(self.count()); + stdx.debug.assert(self.count() == new_shape.count(), "Can't reshape {any} to {any}", .{ self.dims(), new_shape.dims() }); return new_shape; } - fn inferMissingAxis(self: *Shape, n_: usize) error{ UnexpectedRemainder, TooBig }!void { + fn inferMissingAxis(self: *Shape, n_: usize) void { stdx.debug.assert(std.mem.count(i64, self.dims(), &.{-1}) < 2, "Cannot infer multiple dimensions when reshaping to: {f}", .{self.*}); const inferred_ax = std.mem.indexOfScalar(i64, self.dims(), -1) orelse return; @@ -514,10 +584,10 @@ pub const Shape = struct { } } const n: i64 = @intCast(n_); - if (tmp_count > n) return error.TooBig; // Abort, `reshape` will panic with more context. - if (@mod(n, tmp_count) != 0) return error.UnexpectedRemainder; - + if (@mod(n, tmp_count) != 0) { + return; + } self._dims.set(inferred_ax, @divExact(n, tmp_count)); } @@ -535,11 +605,6 @@ pub const Shape = struct { const res = x.reshape(.{-1}); try testing.expectEqualSlices(i64, &.{30}, res.dims()); } - { - var y: Shape = .init(.{ 2, 5, -1 }, .f32); - try testing.expectError(error.UnexpectedRemainder, inferMissingAxis(&y, 15)); - try testing.expectError(error.TooBig, inferMissingAxis(&y, 8)); - } } pub fn setDim(self: Shape, ax: anytype, d: i64) Shape { @@ -578,7 +643,7 @@ pub const Shape = struct { res._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; for (0..dims_.len) |_| { - res._sharding = res._sharding.insert(ax, .replicated); + res._partitioning.insert(ax, .unknown) catch unreachable; } return res; @@ -601,7 +666,7 @@ pub const Shape = struct { var res = self; res._dims.insert(ax, d) catch unreachable; res._tags.insert(ax, toTag(tag_)) catch unreachable; - res._sharding = res._sharding.insert(ax, .replicated); + res._partitioning.insert(ax, .unknown) catch unreachable; return res; } @@ -611,7 +676,7 @@ pub const Shape = struct { const dims_, const tags_ = parseDimensions(v); res._dims.appendSliceAssumeCapacity(dims_.constSlice()); res._tags.appendSliceAssumeCapacity(tags_.constSlice()); - for (self.rank()..res.rank()) |ax| res._sharding = res._sharding.set(ax, .replicated); + res._partitioning.appendNTimesAssumeCapacity(.unknown, dims_.len); return res; } @@ -633,7 +698,7 @@ pub const Shape = struct { var res = self; res._dims.appendAssumeCapacity(d); res._tags.appendAssumeCapacity(if (tag_) |t| t else TagUnknown); - res._sharding = res._sharding.set(self.rank(), .replicated); + res._partitioning.appendAssumeCapacity(.unknown); return res; } @@ -642,7 +707,7 @@ pub const Shape = struct { const a = self.axis(axis_); _ = res._dims.orderedRemove(a); _ = res._tags.orderedRemove(a); - res._sharding = res._sharding.orderedRemove(a); + _ = res._partitioning.orderedRemove(a); return res; } @@ -659,7 +724,6 @@ pub const Shape = struct { std.mem.sort(u3, to_remove.slice(), {}, std.sort.asc(u3)); var sh: Shape = self; - sh._sharding.partition = .out_of_bound; const rk = self.rank(); var res_ax: u32 = 0; for (0..rk) |ax| { @@ -669,11 +733,12 @@ pub const Shape = struct { sh._dims.buffer[res_ax] = self._dims.buffer[ax]; sh._tags.buffer[res_ax] = self._tags.buffer[ax]; - sh._sharding = sh._sharding.set(res_ax, self._sharding.get(ax)); + sh._partitioning.buffer[res_ax] = self._partitioning.buffer[ax]; res_ax += 1; } sh._dims.len = rk - to_remove.len; sh._tags.len = rk - to_remove.len; + sh._partitioning.len = rk - to_remove.len; return sh; } @@ -696,12 +761,12 @@ pub const Shape = struct { var res = self; const original_dims = self._dims; const original_tags = self._tags; - const original_parts = self._sharding; + const original_parts = self._partitioning; for (permutations_.constSlice(), 0..) |permutation, i| { res._dims.set(i, original_dims.get(permutation)); res._tags.set(i, original_tags.get(permutation)); - res._sharding = res._sharding.set(i, original_parts.get(permutation)); + res._partitioning.set(i, original_parts.get(permutation)); } return res; @@ -720,9 +785,9 @@ pub const Shape = struct { ), ); - const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); + const shape = Shape.init(.{ .a = 10, .c = 12, .b = 11 }, .f32).withPartitioning(.{ .a = .batch, .c = .colors, .b = .open }).transpose(.{ 0, 2, 1 }); try testing.expect(shape.eqlWithTags(Shape.init(.{ .a = 10, .b = 11, .c = 12 }, .f32))); - try testing.expectEqualDeep(Sharding.Partitioning.init(&.{ .mesh_axis_0, .open, .mesh_axis_2 }), shape._sharding.partition); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.colors) }, shape._partitioning.constSlice()); } /// Tag each ax of this shape with tags from a tuple. @@ -751,7 +816,7 @@ pub const Shape = struct { return res; } - stdx.debug.compileError("Expected a tuple of enum literals eg: .{{ .a, .b, .c }} got: {any}", .{@TypeOf(tagz)}); + stdx.debug.compileError("Expected a tuple of enum literals eg: .{ .a, .b, .c } got: {any}", .{@TypeOf(tagz)}); } test withTags { @@ -825,151 +890,160 @@ pub const Shape = struct { } } - pub fn assertTags(self: Shape, tagz: anytype) void { - const T = @TypeOf(tagz); - if (T == Shape) { - return self.assertTags(tagz.tags()); - } else if (T != []const Shape.Tag) { - return self.assertTags(comptime parseTags(tagz)); - } - - stdx.debug.assert(tagz.len == self.rank(), "Not enough tags for shape {f}, got {}", .{ self, stdx.fmt.stringsZ(tagz) }); - stdx.debug.assert(std.mem.eql(Shape.Tag, tagz, self.tags()), "Expected tags {f}, got {f}", .{ stdx.fmt.stringsZ(tagz), self }); - } - pub fn withDtype(self: Shape, dt: DataType) Shape { var res = self; res._dtype = dt; return res; } - // These tests only resolve logical names; placement does not use this fixture. - const test_mesh: Sharding.Mesh = .{ - .name = "test_mesh", - .physical = undefined, - .logical = .mesh(.{ .batch = .balanced, .feature = .balanced, .colors = .balanced }), - .bindings = .init(&.{ - .init(&.{.link_x}), - .init(&.{.link_y}), - .init(&.{.link_z}), - }), - .grid = undefined, - }; + pub fn withPartitioning(self: Shape, specs: anytype) Shape { + const T = @TypeOf(specs); - /// Returns shape metadata with the requested sharding on `mesh`. - /// Omitted dimensions in a mapping are replicated, including during compilation. - /// Use `Tensor.withPartitioning` to emit a constraint on a graph value. - pub fn withPartitioning(self: Shape, mesh: *const Sharding.Mesh, partitioning: anytype) Shape { - if (@TypeOf(partitioning) != Sharding.Partitioning) { - return self.withPartitioning(mesh, Sharding.Partitioning.parse(self.tags(), mesh, .replicated, partitioning)); - } - var res = self; - res._sharding = .{ .mesh = mesh, .partition = partitioning }; - for (0..self.rank()) |ax| { - if (res._sharding.get(ax) == .out_of_bound) res._sharding = res._sharding.set(ax, .replicated); - } - for (self.rank()..MAX_RANK) |ax| res._sharding.partition = res._sharding.partition.set(ax, .out_of_bound); + var res = self.withDefaultPartitioning(); // todo add test for this new change - stdx.debug.assert(res._sharding.partition.hasUniqueAxes(), "{f}.withPartitioning({s}, ...) expects mesh axes to be used a most once, got {any}", .{ self, mesh.name, partitioning.toArray()[0..self.rank()] }); + if (stdx.meta.isStruct(T)) { + inline for (std.meta.fields(T)) |field| { + const partition_axis = @field(specs, field.name); + const axis_ = res.axisFromTagMaybe(toTag(field)); + + if (axis_) |ax| { + res._partitioning.set(ax, PartitionSpec.init(partition_axis)); + } else { + stdx.debug.panic("Partitioning axis {s} not found", .{field.name}); + } + } + } else { + stdx.debug.panic("Expected a struct of enum literals, got: {any}", .{T}); + } + + // Check that no mesh axis is used to partition multiple tensor dimensions. + var used_mesh_axes: stdx.BoundedArray(Tag, constants.MAX_RANK) = .{ .len = 0 }; + + for (res._partitioning.constSlice()) |spec| { + const axis_tag = spec.toTag(); + + if (std.mem.indexOfScalar(Tag, used_mesh_axes.constSlice(), axis_tag)) |_| { + stdx.debug.panic("Mesh axis '{s}' used to partition multiple tensor dimensions. This is not allowed.", .{axis_tag}); + } + + switch (spec) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => used_mesh_axes.appendAssumeCapacity(axis_tag), + } + } return res; } test withPartitioning { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .c = .feature, .a = .batch }); - try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .replicated, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .c = .feature, .a = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .unknown, .init(.feature) }, shape._partitioning.constSlice()); shape = shape.withDefaultPartitioning(); - try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); - shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .sharded(0), .open, .sharded(1) }, shape._sharding.toArray()[0..shape.rank()]); + shape = shape.withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqualSlices(PartitionSpec, &.{ .init(.batch), .open, .init(.feature) }, shape._partitioning.constSlice()); } - pub fn mapPartitioningAxes(self: Shape, mesh: *const Sharding.Mesh, mapping: anytype) Shape { + pub fn mapPartitioningAxes(self: Shape, mapping: anytype) Shape { const T = @TypeOf(mapping); stdx.debug.assertComptime(stdx.meta.isStruct(T), "Mapping must be a struct, e.g., .{ .old_axis = .new_axis }", .{}); + var res = self; - for (0..self.rank()) |ax| { - const mesh_axis = self.partition(ax).meshAxis() orelse continue; - inline for (comptime std.meta.fieldNames(T)) |field_name| { - const old_axis = mesh.resolveLogicalAxis(toTag(field_name)) orelse - stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, field_name }); - if (mesh_axis == old_axis) { - const new_tag = toTag(@field(mapping, field_name)); - const new_axis = mesh.resolveLogicalAxis(new_tag) orelse - stdx.debug.panic("Mesh '{s}' has no logical axis '{s}'", .{ mesh.name, new_tag }); - res._sharding = res._sharding.set(ax, .sharded(@intCast(new_axis))); - break; - } + for (res._partitioning.slice()) |*spec| { + switch (spec.*) { + .unknown => {}, + .open => {}, + .replicated => {}, + .axis => |*old_axis_tag| { + inline for (std.meta.fields(T)) |field| { + const logical_axis_in_map = toTag(field); + + if (std.mem.eql(u8, std.mem.span(old_axis_tag.*), std.mem.span(logical_axis_in_map))) { + const physical_axis_from_map = @field(mapping, field.name); + spec.* = PartitionSpec.init(physical_axis_from_map); + break; + } + } + }, } } return res; } test mapPartitioningAxes { - const shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32) - .withPartitioning(&test_mesh, .{ .batch = .batch, .seq = .replicated, .features = .feature }); - const mapped = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .feature, .feature = .batch }); - try testing.expectEqual(Sharding.PartitionSpec.sharded(1), mapped.partition(.batch)); - try testing.expectEqual(Sharding.PartitionSpec.sharded(0), mapped.partition(.features)); - try testing.expectEqual(Sharding.PartitionSpec.replicated, mapped.partition(.seq)); - const partial = shape.mapPartitioningAxes(&test_mesh, .{ .batch = .colors }); - try testing.expectEqual(Sharding.PartitionSpec.sharded(2), partial.partition(.batch)); - try testing.expectEqual(shape.partition(.features), partial.partition(.features)); + const logical_shape = Shape.init(.{ .batch = 64, .seq = 128, .features = 512 }, .f32).withPartitioning(.{ + .batch = .data, + .seq = .replicated, + .features = .model, + }); + + const physical_shape = logical_shape.mapPartitioningAxes(.{ .data = .x, .model = .y }); + + try testing.expect(physical_shape.partition(.batch).eql(.init(.x))); + try testing.expect(physical_shape.partition(.features).eql(.init(.y))); + try testing.expect(physical_shape.partition(.seq).eql(.replicated)); + + const partial_mapped_shape = logical_shape.mapPartitioningAxes(.{ .data = .z }); + try testing.expect(partial_mapped_shape.partition(.batch).eql(.init(.z))); + try testing.expect(partial_mapped_shape.partition(.features).eql(.init(.model))); } pub fn withDefaultPartitioning(self: Shape) Shape { var res = self; - res._sharding.partition = .replicated(self.rank()); + res._partitioning.clear(); + res._partitioning.appendNTimes(.unknown, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); return res; } test withDefaultPartitioning { var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape.rank()); - try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); - shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); - try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); shape = shape.withDefaultPartitioning(); - try testing.expectEqual(3, shape.rank()); - try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); } pub fn withReplicatedPartitioning(self: Shape) Shape { var res = self; - res._sharding.partition = .replicated(self._dims.len); + res._partitioning.clear(); + res._partitioning.appendNTimes(.replicated, self._dims.len) catch stdx.debug.panic("Too many partitioning axes, max: {d}", .{MAX_RANK}); return res; } test withReplicatedPartitioning { var shape = Shape.init(.{ 10, 20, 30 }, .f32); - try testing.expectEqual(3, shape.rank()); - try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .unknown, .unknown }, shape._partitioning.constSlice()); - shape = shape.withPartitioning(&test_mesh, .{ ._1 = .batch }); - try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .sharded(0), .replicated }, shape._sharding.toArray()[0..shape.rank()]); + shape = shape.withPartitioning(.{ ._1 = .batch }); + try testing.expectEqualSlices(PartitionSpec, &.{ .unknown, .init(.batch), .unknown }, shape._partitioning.constSlice()); shape = shape.withReplicatedPartitioning(); - try testing.expectEqual(3, shape.rank()); - try testing.expectEqualSlices(Sharding.PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._sharding.toArray()[0..shape.rank()]); + try testing.expectEqual(3, shape._partitioning.len); + try testing.expectEqualSlices(PartitionSpec, &.{ .replicated, .replicated, .replicated }, shape._partitioning.constSlice()); } pub fn hasAtLeastOnePartitionedAxis(self: Shape) bool { - for (0..self.rank()) |ax| { - const spec = self._sharding.get(ax); - if (spec.isSharded()) return true; + for (self._partitioning.constSlice()) |spec| { + if (spec == .axis) return true; } return false; } test hasAtLeastOnePartitionedAxis { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .replicated }); try testing.expect(shape.hasAtLeastOnePartitionedAxis()); - shape = shape.withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); + shape = shape.withPartitioning(.{ .a = .replicated, .b = .replicated }); try testing.expect(!shape.hasAtLeastOnePartitionedAxis()); shape = shape.withDefaultPartitioning(); @@ -977,18 +1051,20 @@ pub const Shape = struct { } pub fn isFullyPartitioned(self: Shape) bool { - for (0..self.rank()) |ax| { - const spec = self._sharding.get(ax); - if (!spec.isSharded()) return false; + for (self._partitioning.constSlice()) |spec| { + switch (spec) { + .replicated, .open, .unknown => return false, + .axis => {}, + } } return true; } test isFullyPartitioned { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .feature }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .x, .b = .y }); try testing.expect(shape.isFullyPartitioned()); - shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); try testing.expect(!shape.isFullyPartitioned()); shape = shape.withDefaultPartitioning(); @@ -996,47 +1072,54 @@ pub const Shape = struct { } pub fn isFullyReplicated(self: Shape) bool { - return self._sharding.isFullyReplicated(); + for (self._partitioning.constSlice()) |spec| { + if (spec != .replicated and spec != .unknown) { + return false; + } + } + return true; } test isFullyReplicated { - var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated }); + var shape = Shape.init(.{ .a = 10, .b = 20 }, .f32).withPartitioning(.{ .a = .replicated, .b = .replicated }); try testing.expect(shape.isFullyReplicated()); - shape = shape.withPartitioning(&test_mesh, .{ .a = .batch, .b = .replicated }); + shape = shape.withPartitioning(.{ .a = .replicated, .b = .unknown }); + try testing.expect(shape.isFullyReplicated()); + + shape = shape.withPartitioning(.{ .a = .x, .b = .replicated }); try testing.expect(!shape.isFullyReplicated()); } - pub fn partition(self: Shape, ax: anytype) Sharding.PartitionSpec { - return self._sharding.get(self.axis(ax)); + pub fn partition(self: Shape, ax: anytype) PartitionSpec { + return self._partitioning.get(self.axis(ax)); } test partition { - const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); - try testing.expectEqual(Sharding.PartitionSpec.sharded(0), shape.partition(.a)); + const shape = Shape.init(.{ .a = 10, .b = 20, .c = 30 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expectEqual(PartitionSpec.init(.batch), shape.partition(.a)); try testing.expectEqual(.open, shape.partition(.b)); - try testing.expectEqual(Sharding.PartitionSpec.sharded(1), shape.partition(.c)); + try testing.expectEqual(PartitionSpec.init(.feature), shape.partition(.c)); } - pub fn containsPartitionSpec(self: Shape, part: Sharding.PartitionSpec) bool { - for (0..self.rank()) |ax| { - const dim_part = self._sharding.get(ax); - if (dim_part == part) return true; + pub fn containsPartitionSpec(self: Shape, part: PartitionSpec) bool { + for (self._partitioning.constSlice()[0..self.rank()]) |dim_part| { + if (dim_part.eql(part)) return true; } return false; } test containsPartitionSpec { - var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(&test_mesh, .{ .a = .batch, .b = .open, .c = .feature }); - try testing.expect(shape.containsPartitionSpec(.sharded(0))); + var shape = Shape.init(.{ .a = 10, .b = 20, .c = 30, .d = 40 }, .f32).withPartitioning(.{ .a = .batch, .b = .open, .c = .feature }); + try testing.expect(shape.containsPartitionSpec(.init(.batch))); try testing.expect(shape.containsPartitionSpec(.open)); - try testing.expect(shape.containsPartitionSpec(.replicated)); - try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); + try testing.expect(shape.containsPartitionSpec(.unknown)); shape = shape.withDefaultPartitioning(); - try testing.expect(!shape.containsPartitionSpec(.sharded(0))); - try testing.expect(!shape.containsPartitionSpec(.out_of_bound)); - try testing.expect(shape.containsPartitionSpec(.replicated)); + try testing.expect(!shape.containsPartitionSpec(.init(.batch))); + try testing.expect(shape.containsPartitionSpec(.unknown)); + try testing.expect(!shape.containsPartitionSpec(.replicated)); try testing.expect(!shape.containsPartitionSpec(.open)); } @@ -1046,10 +1129,10 @@ pub const Shape = struct { const T = @TypeOf(renames); stdx.debug.assertComptime(stdx.meta.isStructOfAny(T, isAxisConvertible), "Must pass a struct of enum literals. Passed: {any}", .{T}); var res = self; - inline for (comptime std.meta.fieldNames(T)) |field_name| { - const new_field = @field(renames, field_name); + inline for (std.meta.fields(T)) |field| { + const new_field = @field(renames, field.name); stdx.debug.assert(self.hasTag(new_field) == null, "{f}.rename({any}) failed because of duplicated axis {any}", .{ self, renames, new_field }); - res._tags.set(self.axis(field_name), toTag(new_field)); + res._tags.set(self.axis(field), toTag(new_field)); } return res; } @@ -1164,28 +1247,21 @@ pub const Shape = struct { } } - test "byte strides handle zero dimensions and maximum rank" { - const empty = Shape.init(.{ 2, 0, 5 }, .f32).computeByteStrides(); - try std.testing.expectEqualSlices(i64, &.{ 0, 20, 4 }, empty.constSlice()); - const full_rank = Shape.init(.{ 2, 3, 5, 7, 11, 13, 17, 19 }, .u8).computeByteStrides(); - try std.testing.expectEqualSlices(i64, &.{ 4849845, 1616615, 323323, 46189, 4199, 323, 19, 1 }, full_rank.constSlice()); - const large = Shape.init(.{std.math.maxInt(i64)}, .f64).computeByteStrides(); - try std.testing.expectEqualSlices(i64, &.{8}, large.constSlice()); - } - fn computeStrides(self: Shape, element_size: i64) stdx.BoundedArray(i64, constants.MAX_RANK) { - @setRuntimeSafety(false); const rk = self.rank(); - var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .buffer = undefined, .len = rk }; - var stride = element_size; - var ax: usize = rk; - while (ax > 0) { - ax -= 1; - strides.buffer[ax] = stride; - // Axis 0 is not part of any stride; avoid computing the total size. - if (ax > 0) stride *= self._dims.buffer[ax]; - } + var strides: stdx.BoundedArray(i64, constants.MAX_RANK) = .{ .len = rk }; + if (rk == 0) return strides; + const V = @Vector(constants.MAX_RANK, i64); + const rank_mask = std.simd.iota(u8, constants.MAX_RANK) < @as(@Vector(constants.MAX_RANK, u8), @splat(rk)); + // For each axis compute the product of all following dimensions + // and the element size in bytes. + var d: V = @bitCast(self._dims.buffer); + d = @select(i64, rank_mask, d, @as(V, @splat(1))); + d = std.simd.shiftElementsLeft(d, 1, element_size); + d = std.simd.prefixScan(.Mul, -1, d); + + strides.buffer = @bitCast(d); return strides; } @@ -1193,7 +1269,7 @@ pub const Shape = struct { /// so that the given axes are contiguous. pub fn contiguousPerm(self: Shape, axes_: anytype) AxesArray { const axes__, _ = self.parseAxes(axes_); - var perms: AxesArray = .empty; + var perms = AxesArray.init(0) catch unreachable; for (0..self.rank()) |i| { if (std.mem.indexOfScalar(u3, axes__.constSlice(), @intCast(i))) |_| { continue; @@ -1241,27 +1317,26 @@ pub const Shape = struct { var new_shape = self; new_shape._dims.replaceRange(ax, 1, dims_.constSlice()) catch unreachable; new_shape._tags.replaceRange(ax, 1, tags_.constSlice()) catch unreachable; - new_shape._sharding = new_shape._sharding.orderedRemove(ax); + _ = new_shape._partitioning.orderedRemove(ax); for (0..dims_.len) |_| { - new_shape._sharding = new_shape._sharding.insert(ax, .replicated); + new_shape._partitioning.insert(ax, .unknown) catch unreachable; } - new_shape.inferMissingAxis(self.count()) catch |err| { - std.debug.panic("Can't split {any} along axis {d} into {any}: {t}", .{ self.dims(), ax, new_shape.dims(), err }); - }; - + new_shape.inferMissingAxis(self.count()); return new_shape; } test splitAxis { - try expectEqualShapes( - .init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32), - .splitAxis(.init(.{ .a = 10, .b = 3 }, .f32), .a, .{ .a1 = 5, .a2 = 2 }), + try testing.expect( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( + Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = 5, .a2 = 2 }), + ), ); - try expectEqualShapes( - .init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32), - .splitAxis(.init(.{ .a = 10, .b = 3 }, .f32), .a, .{ .a1 = .auto, .a2 = 2 }), + try testing.expect( + Shape.init(.{ .a1 = 5, .a2 = 2, .b = 3 }, .f32).eql( + Shape.init(.{ .a = 10, .b = 3 }, .f32).splitAxis(.a, .{ .a1 = .auto, .a2 = 2 }), + ), ); } @@ -1270,8 +1345,8 @@ pub const Shape = struct { stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (comptime std.meta.fieldNames(T)) |field_name| { - res = res.splitAxis(field_name, @field(axes_, field_name)); + inline for (std.meta.fields(T)) |field| { + res = res.splitAxis(field, @field(axes_, field.name)); } return res; } @@ -1295,34 +1370,39 @@ pub const Shape = struct { const axes_to_merge = self.axes(axes_); stdx.debug.assert(axes_to_merge.len > 1, "Must merge at least two axes, got {any}", .{axes_}); + var new_dim: i64 = 1; // Merging partitioned axes is complex. If ANY axis being merged is partitioned, - // the resulting mesh is ambiguous. A propagation pass must resolve this. + // the resulting sharding is ambiguous. A propagation pass must resolve this. // Marking it 'open' is the correct signal. If all are replicated/unknown, the // result is also replicated/unknown. - // rule: .replicated wins over .out_of_bound. .open wins over everything. + // rule: .replicated wins over .unknown. .open wins over everything. + var resulting_partition_spec: PartitionSpec = .unknown; + + for (0..axes_to_merge.len) |i| { + const current_axis = axes_to_merge.get(i); + new_dim *= self.dim(current_axis); + + if (i > 0) { + stdx.debug.assert(current_axis == axes_to_merge.get(i - 1) + 1, "Can't merge shape {f} along non-contiguous axes {any}", .{ self, axes_ }); + } + + switch (self.partition(current_axis)) { + .axis, .open => resulting_partition_spec = .open, + .replicated => if (resulting_partition_spec != .open) { + resulting_partition_spec = .replicated; + }, + .unknown => {}, + } + } const first_axis_to_merge = axes_to_merge.get(0); const num_axes_to_merge = axes_to_merge.len; - var merged_dim: i64 = 1; - var merged_spec: Sharding.PartitionSpec = .out_of_bound; - for (axes_to_merge.constSlice()) |ax| { - merged_dim *= self.dim(ax); - const spec = self.partition(ax); - if (spec.isSharded() or spec == .open) { - merged_spec = .open; - } else if (spec == .replicated and merged_spec != .open) { - merged_spec = .replicated; - } - } - var new_shape = self; - for (1..num_axes_to_merge) |_| { - new_shape._sharding = new_shape._sharding.orderedRemove(first_axis_to_merge); - } - new_shape._sharding = new_shape._sharding.set(first_axis_to_merge, merged_spec); - new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{merged_dim}) catch @panic("mergeAxis failed on dims"); + + new_shape._dims.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{new_dim}) catch @panic("mergeAxis failed on dims"); new_shape._tags.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{toTag(axis_)}) catch @panic("mergeAxis failed on tags"); + new_shape._partitioning.replaceRange(first_axis_to_merge, num_axes_to_merge, &.{resulting_partition_spec}) catch @panic("mergeAxis failed on partitioning"); return new_shape; } @@ -1358,23 +1438,23 @@ pub const Shape = struct { test "mergeAxis partitioning" { // Merging a partitioned axis with a replicated one -> open const shape1 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .batch, .c = .replicated }); + .withPartitioning(.{ .a = .replicated, .b = .x, .c = .replicated }); const merged1 = shape1.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged1.partition(.ab) == .open); + try testing.expect(merged1.partition(.ab).eql(.open)); try testing.expect(merged1.dim(.ab) == 10); try testing.expect(merged1.rank() == 2); // Merging two replicated axes -> replicated const shape2 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32) - .withPartitioning(&test_mesh, .{ .a = .replicated, .b = .replicated, .c = .feature }); + .withPartitioning(.{ .a = .replicated, .b = .replicated, .c = .y }); const merged2 = shape2.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged2.partition(.ab) == .replicated); - try testing.expect(merged2.partition(.c) == Sharding.PartitionSpec.sharded(1)); + try testing.expect(merged2.partition(.ab).eql(.replicated)); + try testing.expect(merged2.partition(.c).eql(.init(.y))); // Merging two unknown axes -> unknown - const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32).withDefaultPartitioning(); + const shape3 = Shape.init(.{ .a = 5, .b = 2, .c = 3 }, .f32); // default is .unknown const merged3 = shape3.mergeAxis(.ab, .{ .a, .b }); - try testing.expect(merged3.partition(.ab) == .replicated); + try testing.expect(merged3.partition(.ab).eql(.unknown)); } pub fn mergeAxes(self: Shape, axes_: anytype) Shape { @@ -1382,9 +1462,9 @@ pub const Shape = struct { stdx.debug.assertComptime(stdx.meta.isStruct(T), "Must pass struct of enum literals like .{ .a = .{ .a1, .a2 } }. Passed: {any}", .{T}); var res = self; - inline for (comptime std.meta.fieldNames(T), comptime std.meta.fieldTypes(T)) |field_name, Field| { - stdx.debug.assertComptime(stdx.meta.isTupleOfAny(Field, isAxisConvertible) or stdx.meta.isSliceOfAny(Field, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{Field}); - res = res.mergeAxis(field_name, @field(axes_, field_name)); + inline for (std.meta.fields(T)) |field| { + stdx.debug.assertComptime(stdx.meta.isTupleOfAny(field.type, isAxisConvertible) or stdx.meta.isSliceOfAny(field.type, isAxisConvertible), "Must pass struct of axes. Passed: {any}", .{field.type}); + res = res.mergeAxis(field, @field(axes_, field.name)); } return res; } @@ -1413,7 +1493,7 @@ pub const Shape = struct { } fn intersectTags(a: []const Tag, b: []const Tag) TagsArray { - var res: TagsArray = .empty; + var res = TagsArray.init(0) catch unreachable; for (a) |tag_| { if (std.mem.indexOfScalar(Tag, b, tag_)) { res.appendAssumeCapacity(tag_); @@ -1425,8 +1505,8 @@ pub const Shape = struct { pub fn parseStruct(T: type, v: anytype) struct { stdx.BoundedArray(T, constants.MAX_RANK), TagsArray } { const V = @TypeOf(v); - var vals_: stdx.BoundedArray(T, constants.MAX_RANK) = .empty; - var tags_: TagsArray = .empty; + var vals_: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; + var tags_: TagsArray = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { for (v) |d| { @@ -1436,14 +1516,14 @@ pub const Shape = struct { } if (comptime stdx.meta.isStruct(V)) { - const field_names = comptime std.meta.fieldNames(V); - stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, field_names.len, constants.MAX_RANK }); - inline for (field_names) |field_name| { - const fv = @field(v, field_name); + const fields = std.meta.fields(V); + stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "Too many fields in struct {} ({d}). Max supported is {d}.", .{ V, fields.len, constants.MAX_RANK }); + inline for (fields) |field| { + const fv = @field(v, field.name); vals_.appendAssumeCapacity(fv); if (!comptime stdx.meta.isTuple(V)) { - tags_.appendAssumeCapacity(toTag(field_name)); + tags_.appendAssumeCapacity(toTag(field)); } } return .{ vals_, tags_ }; @@ -1463,7 +1543,7 @@ pub const Shape = struct { pub fn parseAxesOptions(self: Shape, T: type, options: anytype, default: T) stdx.BoundedArray(T, constants.MAX_RANK) { const V = @TypeOf(options); - var res: stdx.BoundedArray(T, constants.MAX_RANK) = .empty; + var res: stdx.BoundedArray(T, constants.MAX_RANK) = .{}; if (comptime stdx.meta.isSliceOf(V, T)) { stdx.debug.assert(options.len == self.rank(), "expects exactly {} options in slice, for {} got {}", .{ self.rank(), self, options.len }); for (options) |d| { @@ -1473,11 +1553,11 @@ pub const Shape = struct { if (comptime stdx.meta.isStruct(V)) { for (0..self.rank()) |_| res.appendAssumeCapacity(default); - const field_names = comptime std.meta.fieldNames(V); - stdx.debug.assertComptime(field_names.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, field_names.len }); - inline for (field_names) |field_name| { - const a = self.axis(field_name); - res.buffer[a] = @field(options, field_name); + const fields = std.meta.fields(V); + stdx.debug.assertComptime(fields.len <= constants.MAX_RANK, "expects up to {} options struct literal, got {}", .{ V, constants.MAX_RANK, fields.len }); + inline for (fields) |field| { + const a = self.axis(field); + res.buffer[a] = @field(options, field.name); } return res; } @@ -1528,57 +1608,10 @@ pub const Shape = struct { pub fn iterator(self: Shape) MultiDimIterator { return .{ .shape = self, - .current_coords = @splat(0), + .current_coords = .{0} ** Shape.MAX_RANK, .flat_index = 0, }; } - - /// Generate the packed shape visible to PJRT - pub fn packedShape(sh: Shape) Shape { - const bit_size = sh.dtype().bitSizeOf(); - if (bit_size >= 8) return sh; - - const items_per_byte = @divExact(8, bit_size); - var res: Shape = sh.setDim(-1, @divExact(sh.dim(-1), items_per_byte)); - res._dtype = .u8; - return res; - } - - test packedShape { - const x: Shape = .init(.{ 4, 8 }, .u2); - try expectEqualShapes(.init(.{ 4, 2 }, .u8), x.packedShape()); - } - - test "shape transformations preserve partition specs and clear unused slots" { - var shape = Shape.init(.{ 2, 3, 4 }, .f32); - shape._sharding = .{ .mesh = &test_mesh, .partition = Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .open).set(2, .mesh_axis_1) }; - for ([_]Shape{ - shape.insert(1, .{5}), - shape.append(.{5}), - shape.remove(1), - shape.removeMany(.{ 0, 1 }), - shape.transpose(.{ 2, 0, 1 }), - shape.reshape(.{24}), - shape.splitAxis(1, .{ 1, 3 }), - shape.mergeAxis(.merged, .{ 0, 1 }), - shape.withDefaultPartitioning(), - }) |transformed| { - try testing.expectEqual(&test_mesh, transformed._sharding.mesh); - } - const parts = shape._sharding.partition; - try testing.expectEqual(parts.insert(1, .replicated), shape.insert(1, .{5})._sharding.partition); - try testing.expectEqual(parts.insert(1, .replicated), shape.insertTag(1, 5, .new)._sharding.partition); - try testing.expectEqual(parts.set(3, .replicated), shape.append(.{5})._sharding.partition); - try testing.expectEqual(parts.set(3, .replicated), shape.appendDim(5, null)._sharding.partition); - try testing.expectEqual(parts.orderedRemove(1), shape.remove(1)._sharding.partition); - const removed = shape.removeMany(.{ 0, 1 }); - try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1), removed._sharding.partition); - try testing.expectEqual(Sharding.PartitionSpec.replicated, removed.append(.{5}).partition(1)); - try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_1).set(1, .mesh_axis_0).set(2, .open), shape.transpose(.{ 2, 0, 1 })._sharding.partition); - try testing.expectEqual(Sharding.Partitioning.out_of_bound.set(0, .mesh_axis_0).set(1, .replicated).set(2, .replicated).set(3, .mesh_axis_1), shape.splitAxis(1, .{ 1, 3 })._sharding.partition); - try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withDefaultPartitioning()._sharding.partition); - try testing.expectEqual(Sharding.Partitioning.replicated(3), shape.withReplicatedPartitioning()._sharding.partition); - } }; /// Iterates over all coordinates of a shape, yielding the multi-dimensional @@ -1703,10 +1736,3 @@ test "MultiDimIterator / Shape with dimension of size 1" { } try std.testing.expectEqual(6, count); } - -fn expectEqualShapes(expected: Shape, actual: Shape) error{TestExpectedEqual}!void { - if (expected.eqlWithTags(actual)) return; - - std.debug.print("Expected {f}, got {f}", .{ expected, actual }); - return error.TestExpectedEqual; -} diff --git a/zml/slice.zig b/zml/slice.zig index 9451165b..e33316db 100644 --- a/zml/slice.zig +++ b/zml/slice.zig @@ -2,11 +2,9 @@ const std = @import("std"); const stdx = @import("stdx"); -const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; const floats = @import("floats.zig"); const Shape = @import("shape.zig").Shape; -const builtin = @import("builtin"); pub fn isBytes(comptime T: type) bool { const type_info = @typeInfo(T); @@ -19,35 +17,20 @@ pub fn isBytes(comptime T: type) bool { } pub const Slice = struct { - bytes: []const u8, - mutable: bool, + inner_data: union(enum) { + mutable: []u8, + immutable: []const u8, + }, shape: Shape, - offset_bytes: usize = 0, - byte_strides: stdx.BoundedArray(i64, constants.MAX_RANK), pub fn init(shape: Shape, bytes: anytype) Slice { stdx.debug.assertComptime(isBytes(@TypeOf(bytes)), "Expected \"bytes\" to be a []u8 or []const u8, got {s}", .{@typeName(@TypeOf(bytes))}); stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); const type_info = @typeInfo(@TypeOf(bytes)); - const byte_strides = shape.computeByteStrides(); - return .{ - .bytes = bytes, - .mutable = !type_info.pointer.attrs.@"const", - .shape = shape, - .offset_bytes = 0, - .byte_strides = byte_strides, - }; - } - - pub fn initConst(shape: Shape, bytes: []const u8) Slice { - stdx.debug.assert(bytes.len == shape.byteSize(), "Expected \"bytes\" to have the same length as shape.byteSize() ({}), got {}", .{ shape.byteSize(), bytes.len }); - return .{ - .bytes = bytes, - .mutable = false, - .shape = shape, - .offset_bytes = 0, - .byte_strides = shape.computeByteStrides(), - }; + return if (type_info.pointer.is_const) + .{ .inner_data = .{ .immutable = bytes }, .shape = shape } + else + .{ .inner_data = .{ .mutable = bytes }, .shape = shape }; } pub fn alloc(allocator: std.mem.Allocator, shape_: Shape) !Slice { @@ -63,18 +46,12 @@ pub const Slice = struct { else => |v| stdx.debug.panic("Unsupported alignment: {}", .{v}), }; - return .{ - .bytes = bytes, - .mutable = true, - .shape = shape_, - .offset_bytes = 0, - .byte_strides = shape_.computeByteStrides(), - }; + return .{ .inner_data = .{ .mutable = bytes }, .shape = shape_ }; } pub fn free(slice: Slice, allocator: std.mem.Allocator) void { - const d = slice.bytes; - // TODO: It would be better to store the aligment, this can be error prone in the presence of type casting + const d = slice.constData(); + switch (slice.shape.dtype().alignOf()) { 1 => allocator.free(@as([]align(1) const u8, @alignCast(d))), 2 => allocator.free(@as([]align(2) const u8, @alignCast(d))), @@ -91,158 +68,32 @@ pub const Slice = struct { return slice.shape.dtype(); } - pub fn isContiguous(slice: Slice) bool { - const expected = slice.shape.computeByteStrides(); - for (expected.slice(), slice.byte_strides.slice()) |exp, actual| { - if (exp != actual) return false; - } - return true; - } - - /// Returns the actual element-wise strides for this specific slice. - /// This respects transpositions and non-contiguous views. - pub fn elementStrides(self: Slice) stdx.BoundedArray(i64, constants.MAX_RANK) { - const element_size = @as(i64, @intCast(self.dtype().sizeOf())); - var res = self.byte_strides; - for (res.slice()) |*stride| { - // We divide the actual byte strides by the size of the data type - stride.* = @divExact(stride.*, element_size); - } - return res; - } - - pub fn subSlice(slice: Slice, axis: anytype, start: i64, size: i64) Slice { - const dim = slice.shape.dim(axis); - - stdx.debug.assert(start >= 0 and size >= 0 and start + size <= dim, "subSlice out of bounds: axis {d} start {d} size {d} dim {d}", .{ axis, start, size, dim }); - - const stride_bytes = slice.byte_strides.get(axis); - const new_offset = @as(i64, @intCast(slice.offset_bytes)) + start * stride_bytes; - stdx.debug.assert(new_offset >= 0, "subSlice produced negative offset: {d}", .{new_offset}); - - var res = slice; - res.offset_bytes = @intCast(new_offset); - res.shape = slice.shape.set(axis, size); - return res; - } - - pub fn squeeze(slice: Slice, axis_: anytype) Slice { - const axis = slice.shape.axis(axis_); - var res = slice; - res.shape = slice.shape.drop(axis); - _ = res.byte_strides.orderedRemove(@intCast(axis)); - return res; - } - pub fn data(slice: Slice) []u8 { - std.debug.assert(slice.mutable); - if (builtin.mode == .debug) std.debug.assert(slice.isContiguous()); - const end = slice.offset_bytes + slice.shape.byteSize(); - return @constCast(slice.bytes[slice.offset_bytes..end]); + return switch (slice.inner_data) { + .mutable => |d| d, + else => stdx.debug.panic("Expected slice to be mutable but it's immutable", .{}), + }; } pub fn constData(slice: Slice) []const u8 { - if (builtin.mode == .debug) std.debug.assert(slice.isContiguous()); - const end = slice.offset_bytes + slice.shape.byteSize(); - return slice.bytes[slice.offset_bytes..end]; + return switch (slice.inner_data) { + inline else => |d| d, + }; } pub fn items(slice: Slice, comptime T: type) []T { - stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like stdx.Packed({0}). Got: {0}", .{T}); - return @ptrCast(@alignCast(slice.data())); + return @alignCast(std.mem.bytesAsSlice(T, slice.data())); } pub fn constItems(slice: Slice, comptime T: type) []const T { - stdx.debug.assertComptime(T == bool or @bitSizeOf(T) >= 8, "zml.Slice stores packed sub-bytes type so you need to pass a packed struct here like zml.Float4E2M1.Packed. Got: {}", .{T}); - return @ptrCast(@alignCast(slice.constData())); - } - - pub fn contiguousItemsIterator(slice: *const Slice, comptime T: type) ContiguousItemsIterator(T) { - std.debug.assert(slice.dtype() == DataType.fromZigType(T)); - return .init(slice); - } - - fn ContiguousItemsIterator(comptime T: type) type { - return struct { - slice: *const Slice, - contiguous_axis_start: u4, - contiguous_item_count: usize, - outer_index: usize, - outer_count: usize, - outer_coords: [constants.MAX_RANK]usize, - next_offset_bytes: usize, - - const Iterator = @This(); - - fn init(slice: *const Slice) Iterator { - const rank = slice.shape.rank(); - const total_count = slice.shape.count(); - var contiguous_axis_start: u4 = rank; - var contiguous_item_count: usize = 1; - var expected_stride: i64 = @intCast(@sizeOf(T)); - - var axis = rank; - while (axis > 0) { - axis -= 1; - const dim = slice.shape.dim(axis); - const stride = slice.byte_strides.get(axis); - if (dim != 1 and stride != expected_stride) break; - contiguous_axis_start = axis; - contiguous_item_count *= @intCast(dim); - expected_stride *= dim; - } - - if (total_count == 0) { - contiguous_item_count = 0; - } - - return .{ - .slice = slice, - .contiguous_axis_start = contiguous_axis_start, - .contiguous_item_count = contiguous_item_count, - .outer_index = 0, - .outer_count = if (contiguous_item_count == 0) 0 else total_count / contiguous_item_count, - .outer_coords = @splat(0), - .next_offset_bytes = slice.offset_bytes, - }; - } - - pub fn next(self: *Iterator) ?[]const T { - if (self.outer_index >= self.outer_count) return null; - - const start = self.next_offset_bytes; - self.outer_index += 1; - if (self.outer_index < self.outer_count) { - self.advanceOuterCoords(); - } - - const byte_len = self.contiguous_item_count * @sizeOf(T); - return @ptrCast(@alignCast(self.slice.bytes[start..][0..byte_len])); - } - - fn advanceOuterCoords(self: *Iterator) void { - var axis = self.contiguous_axis_start; - while (axis > 0) { - axis -= 1; - - self.outer_coords[axis] += 1; - self.next_offset_bytes = @intCast(@as(i64, @intCast(self.next_offset_bytes)) + self.slice.byte_strides.get(axis)); - - const dim: usize = @intCast(self.slice.shape.dim(axis)); - if (self.outer_coords[axis] < dim) return; - - self.outer_coords[axis] = 0; - self.next_offset_bytes = @intCast(@as(i64, @intCast(self.next_offset_bytes)) - @as(i64, @intCast(dim)) * self.slice.byte_strides.get(axis)); - } - } - }; + return @alignCast(std.mem.bytesAsSlice(T, slice.constData())); } pub fn format( slice: @This(), writer: *std.Io.Writer, ) std.Io.Writer.Error!void { - return writer.print("Slice({f})@0x{x} [mut={}, off={d}, strides={any}]", .{ slice.shape, @intFromPtr(slice.bytes.ptr), slice.mutable, slice.offset_bytes, slice.byte_strides.slice() }); + return writer.print("{any}", .{slice}); } pub fn formatNumber(slice: Slice, writer: *std.Io.Writer, n: std.fmt.Number) std.Io.Writer.Error!void { @@ -258,8 +109,7 @@ pub const Slice = struct { // Special case input tensor is a scalar return switch (slice.dtype()) { inline else => |dt| { - const single_ptr: *const dt.toZigType() = @ptrCast(@alignCast(slice.bytes.ptr)); - const val = single_ptr.*; + const val: dt.toZigType() = slice.constItems(dt.toZigType())[0]; return switch (comptime dt.class()) { // Since we have custom floats, we need to explicitly convert to float32 ourselves. .float => stdx.fmt.formatFloat(floats.floatCast(f32, val), options, writer), @@ -277,28 +127,15 @@ pub const Slice = struct { try writer.splatByteAll(' ', indent_level); switch (slice.dtype()) { inline else => |dt| { - const T = dt.toPackedZigType(); - const n = slice.shape.dim(0); - const elem_per_bytes: comptime_int = @max(1, 8 / comptime dt.bitSizeOf()); - const stride = @divExact(slice.byte_strides.get(0), @as(i64, @sizeOf(T))); - - // Fetch n next elements, accounting for strides and packing - const needed_len: usize = @as(usize, if (n == 0) 0 else @intCast(@abs((n - 1) * stride) + 1)) / elem_per_bytes; - - // The formatter consumes this physical tail using `stride`, so this - // rank-1 path can read raw items without requiring contiguity. - const raw_values: []const T = @ptrCast(@alignCast(slice.bytes[slice.offset_bytes..])); - const values = raw_values[0..needed_len]; - - if (comptime dt.bitSizeOf() < 8) { - return try stdx.fmt.formatSliceAny(values, options, stride, writer); - } - + const T = dt.toZigType(); + // TODO: handle negative strides + const elem_strides: i64 = slice.shape.computeElementStrides().get(0); + const values = slice.constItems(T); switch (comptime dt.class()) { - .float => try stdx.fmt.formatFloatSlice(values, options, stride, writer), - .integer => try stdx.fmt.formatIntSlice(values, options, stride, writer), - .complex => try stdx.fmt.formatComplexSlice(values, options, stride, writer), - .bool => try stdx.fmt.formatBoolSlice(values, options, stride, writer), + .float => try stdx.fmt.formatFloatSlice(values, options, elem_strides, writer), + .integer => try stdx.fmt.formatIntSlice(values, options, elem_strides, writer), + .complex => try stdx.fmt.formatComplexSlice(values, options, elem_strides, writer), + .bool => try stdx.fmt.formatBoolSlice(values, options, elem_strides, writer), } }, } @@ -342,83 +179,8 @@ pub const Slice = struct { try writer.splatByteAll(' ', indent_level); _ = try writer.write("}"); } - - pub fn copy(destination: Slice, source: []const u8) void { - const total_bytes = destination.shape.byteSize(); - stdx.debug.assert(source.len >= total_bytes, "Source buffer size ({}) smaller than destination shape ({})", .{ source.len, total_bytes }); - - if (total_bytes == 0) return; - - if (destination.isContiguous()) { - @memcpy(destination.data()[0..total_bytes], source[0..total_bytes]); - return; - } - - const rank = destination.shape.rank(); - stdx.debug.assert(rank > 0, "Non-contiguous slice with rank 0 is unexpected", .{}); - - const outer_len: usize = @intCast(destination.shape.dim(0)); - stdx.debug.assert(outer_len > 0, "Invalid dimension size: {d}", .{outer_len}); - - stdx.debug.assert(total_bytes % outer_len == 0, "Non-even split for contiguous source", .{}); - const outer_span_bytes = total_bytes / outer_len; - - for (0..outer_len) |i| { - const destination_outer = destination.subSlice(0, @intCast(i), 1).squeeze(0); - const source_outer_start = i * outer_span_bytes; - const source_outer_end = source_outer_start + outer_span_bytes; - destination_outer.copy(source[source_outer_start..source_outer_end]); - } - } }; -test "slice expectClose compares dense and strided slices" { - const testing = @import("testing.zig"); - - const dense_data: [2][2]f32 = .{ - .{ 1, 2 }, - .{ 5, 6 }, - }; - const strided_data: [2][4]f32 = .{ - .{ 1, 2, 99, 98 }, - .{ 5, 6, 97, 96 }, - }; - - const dense = Slice.init(.init(.{ 2, 2 }, .f32), std.mem.asBytes(&dense_data)); - const storage = Slice.init(.init(.{ 2, 4 }, .f32), std.mem.asBytes(&strided_data)); - const strided = storage.subSlice(1, 0, 2); - - try std.testing.expect(!strided.isContiguous()); - try testing.expectClose(std.testing.io, dense, strided, .exact_match); - try testing.expectClose(std.testing.io, strided, dense, .exact_match); -} - -test "slice expectClose applies close fraction across strided slices" { - const testing = @import("testing.zig"); - - const dense_data: [2][2]f32 = .{ - .{ 1, 2 }, - .{ 5, 6 }, - }; - const strided_data: [2][4]f32 = .{ - .{ 1, 3, 99, 98 }, - .{ 5, 6, 97, 96 }, - }; - - const dense = Slice.init(.init(.{ 2, 2 }, .f32), std.mem.asBytes(&dense_data)); - const storage = Slice.init(.init(.{ 2, 4 }, .f32), std.mem.asBytes(&strided_data)); - const strided = storage.subSlice(1, 0, 2); - const opts: testing.CompareOpts = .{ - .absolute_tolerance = 0, - .relative_tolerance = 0, - .minimum_close_fraction = 0.75, - }; - - try std.testing.expect(!strided.isContiguous()); - try testing.expectClose(std.testing.io, dense, strided, opts); - try testing.expectClose(std.testing.io, strided, dense, opts); -} - test "slice pretty print rank 3" { const data: [4][4][4]i32 = .{ .{ @@ -504,14 +266,6 @@ test "slice pretty print rank 0" { try std.testing.expectFmt(expected, "{d}", .{slice}); } -test "slice pretty print bool" { - const data: [2]bool = .{ true, false }; - const slice = Slice.init(.init(.{2}, .bool), std.mem.asBytes(&data)); - const expected = "{1,0}"; - try std.testing.expectEqualSlices(bool, &data, slice.constItems(bool)); - try std.testing.expectFmt(expected, "{d}", .{slice}); -} - test "slice pretty print ellipsis" { const data: [9][1]i32 = .{ .{0}, .{1}, .{2}, .{3}, .{4}, .{5}, .{6}, .{7}, .{8} }; const slice = Slice.init(.init(.{ 9, 1 }, .i32), std.mem.asBytes(&data)); @@ -530,12 +284,3 @@ test "slice pretty print ellipsis" { ; try std.testing.expectFmt(expected, "{d}", .{slice}); } - -test "slice pretty print f4" { - const x_f4_packed = [_]floats.Float4E2M1.Packed{ .fromF32(0, 0.5), .fromF32(1, 1.5), .fromF32(2, 3), .fromF32(4, 6) }; - const slice = Slice.initConst(.init(.{8}, .f4e2m1), @ptrCast(&x_f4_packed)); - - const expected = "{0,0.5,1,1.5,2,3,4,6}"; - - try std.testing.expectFmt(expected, "{d}", .{slice}); -} diff --git a/zml/tensor.zig b/zml/tensor.zig index ee084835..ba0dcaf8 100644 --- a/zml/tensor.zig +++ b/zml/tensor.zig @@ -1,49 +1,36 @@ const std = @import("std"); const builtin = @import("builtin"); +const dialects = @import("mlir/dialects"); const mlir = @import("mlir"); -const shardy = @import("mlir/dialects/shardy"); -const stablehlo = @import("mlir/dialects/stablehlo"); const stdx = @import("stdx"); -const Compiler = @import("Compiler.zig"); +const Buffer = @import("buffer.zig").Buffer; +const Bufferized = @import("zml.zig").Bufferized; +const CompilationContext = @import("module.zig").CompilationContext; const constants = @import("constants.zig"); const DataType = @import("dtype.zig").DataType; -const mem = @import("mem.zig"); const Memory = @import("platform.zig").Memory; const meta = @import("meta.zig"); const mlirx = @import("mlirx.zig"); const ops = @import("ops.zig"); const Platform = @import("platform.zig").Platform; const Shape = @import("shape.zig").Shape; -const Sharding = @import("Sharding.zig"); - -test { - std.testing.refAllDecls(@This()); - std.testing.refAllDecls(Tensor); -} pub const Tensor = struct { - pub const Id = enum(u64) { _ }; - var current_id: std.atomic.Value(u64) = .init(1); + var current_id: std.atomic.Value(usize) = .{ .raw = 1 }; - id: Tensor.Id, + id: usize, auto_broadcast: bool = false, _shape: Shape, _value: ?*const mlir.Value = null, - const ResolvedAxis = u3; - pub fn init(shape_like: anytype, dt: DataType) Tensor { return .fromShape(.init(shape_like, dt)); } - fn nextTensorId() Id { - return @fromBackingInt(@intCast(Tensor.current_id.fetchAdd(1, .seq_cst))); - } - pub fn fromShape(shape_: Shape) Tensor { - return .{ .id = nextTensorId(), ._shape = shape_ }; + return .{ .id = Tensor.current_id.fetchAdd(1, .seq_cst), ._shape = shape_ }; } pub fn format(self: Tensor, writer: *std.Io.Writer) !void { @@ -75,50 +62,42 @@ pub const Tensor = struct { return self._shape.byteSize(); } - /// Internal: creates a graph tensor with dimensions, dtype and tags from `sh`, - /// and open sharding for the partitioner to infer from graph constraints. + /// Internal use + /// + /// Creates a tensor from a Shape and an mlir.Value. pub fn _result(sh: Shape, val: *const mlir.Value) Tensor { - var res: Tensor = .{ ._shape = sh, ._value = val, .id = nextTensorId() }; - res._shape._sharding = .open; + const res: Tensor = .{ + ._shape = sh, + ._value = val, + .id = Tensor.current_id.fetchAdd(1, .seq_cst), + }; - if (builtin.mode == .debug) { + if (builtin.mode == .Debug) { // Check that the MLIR value actually have the same shape. const other = fromMlirValue(val); - const eql_dtype = sh.dtype() == other.dtype(); - const eql_dims = std.mem.eql(i64, sh.dims(), other.dims()); - stdx.debug.internalAssert(eql_dims and eql_dtype, "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); + stdx.debug.internalAssert(sh.eql(other._shape), "Created a {f} from Mlir value but expected {f}", .{ other._shape, res._shape }); } return res; } - /// Internal: creates a constructor result and honors `sh`'s partitioning when - /// it specifies a mesh. - pub fn _resultPropagateSharding(sh: Shape, val: *const mlir.Value) Tensor { - const result = _result(sh, val); - const mesh = sh._sharding.mesh orelse return result; - return result.withPartitioning(mesh, sh._sharding.partition); - } - - /// Wraps an MLIR tensor value, deriving dimensions and dtype from its type. - /// Dimension tags are unknown and sharding is open for partitioner inference. + /// Creates a Tensor from a mlir.Value + /// + /// The shape is derived from the type of the mlir.Value. pub fn fromMlirValue(val: *const mlir.Value) Tensor { - const ctx = Compiler.current(); const ranked_tensor = val.type_().isA(mlir.RankedTensorType).?; const n = ranked_tensor.rank(); stdx.debug.assert(n <= constants.MAX_RANK, "Can't represent MLIR tensor of rank {}, max supported rank is {}.", .{ n, constants.MAX_RANK }); - var sh: Shape = .{ - ._dtype = ctx.dtype(ranked_tensor.elementType()), - ._dims = .empty, - ._tags = .{ .buffer = @splat(Shape.TagUnknown), .len = n }, - ._sharding = .open, - }; + var sh: Shape = .{ ._dtype = mlirx.Type.toDType(mlirCtx(), ranked_tensor.elementType()) }; for (0..n) |i| { sh._dims.appendAssumeCapacity(ranked_tensor.dimension(i)); } - return .{ ._shape = sh, ._value = val, .id = nextTensorId() }; + sh._tags.resize(n) catch unreachable; + sh._partitioning.resize(n) catch unreachable; + + return .{ ._shape = sh, ._value = val, .id = Tensor.current_id.fetchAdd(1, .seq_cst) }; } /// Returns the dimension of axis 'axis_'. @@ -136,14 +115,14 @@ pub const Tensor = struct { /// Returns the index of axis 'axis_'. /// /// 'axis_' can be an integer or a tag. - pub fn axis(self: Tensor, axis_: anytype) ResolvedAxis { + pub fn axis(self: Tensor, axis_: anytype) u3 { return self._shape.axis(axis_); } /// Returns the indices of each of the given axes. /// /// 'axis_' can be an integer or a tag. - pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(ResolvedAxis, constants.MAX_RANK) { + pub fn axes(self: Tensor, axes_: anytype) stdx.BoundedArray(u3, constants.MAX_RANK) { return self._shape.axes(axes_); } @@ -163,144 +142,25 @@ pub const Tensor = struct { return res; } - /// Declares this tensor's sharding on `mesh_`. - /// Pass a `*const Mesh`, or a mesh name such as `.tp` during compilation. - /// `partitioning` accepts a `Sharding.Partitioning` or a mapping from dimension - /// tags to logical mesh axes, `.replicated`, or `.open`. - /// - /// During compilation, emits a graph constraint. For example, - /// `x.withPartitioning(.tp, .{ .h = .model })` splits dimension `h` over `model` - /// and leaves omitted dimensions open for inference. Use `.replicated` to - /// keep a dimension whole on each shard. - /// - /// Outside compilation, updates the tensor's shape to describe an executable - /// input's sharding. Omitted dimensions are replicated and no graph constraint is - /// emitted. An explicit `Sharding.Partitioning` uses its supplied specs in both contexts. - /// - /// Executable inputs and explicitly annotated outputs must have no open - /// dimensions. An output with no declared sharding defaults to replication. - pub fn withPartitioning(self: Tensor, mesh_: anytype, partitioning: anytype) Tensor { - if (@TypeOf(mesh_) == @EnumLiteral()) { - const compiler = Compiler.currentOrNull() orelse @panic("Out side of compilation, withPartitioning expects an explicit *const zml.Mesh object as input"); - return self.withPartitioning(compiler.mesh(mesh_), partitioning); - } - - if (@TypeOf(partitioning) != Sharding.Partitioning) { - const default: Sharding.PartitionSpec = if (Compiler.currentOrNull() == null) .replicated else .open; - const parsed = Sharding.Partitioning.parse(self._shape.tags(), mesh_, default, partitioning); - return self.withPartitioning(mesh_, parsed); - } - - const partitioned_shape = self._shape.withPartitioning(mesh_, partitioning); - - const ctx = Compiler.currentOrNull() orelse { - var res = self; - res._shape = partitioned_shape; - return res; - }; - - const attr = ctx.tensorShardingAttr(ctx.allocator, partitioned_shape) catch @panic("OOM"); - - const op_result = switch (ctx.partitioner) { - .shardy => blk: { - const op = shardy.sharding_constraint( - ctx.mlir_ctx, - self.value(), - attr.isA(shardy.TensorShardingAttr).?, - ctx.location, - ).appendTo(currentBlock()); - break :blk op.result(0); - }, - .gspmd => blk: { - const op = stablehlo.custom_call( - ctx.mlir_ctx, - &.{self.value()}, - &.{self.value().type_()}, - .{ - .call_target_name = "Sharding", - .has_side_effect = false, - .backend_config = .{ .original = "" }, - .additional_attributes = &.{ - .named(ctx.mlir_ctx, "mhlo.sharding", attr), - }, - }, - ctx.location, - ).appendTo(currentBlock()); - break :blk op.result(0); - }, - }; - - var res = _result(partitioned_shape, op_result); - res._shape = partitioned_shape; - return res; + // TODO(Corentin) + pub fn withSharding(self: Tensor, axes_: anytype) Tensor { + _ = self; + _ = axes_; + @panic("Unimplemented"); } - /// Emits a graph constraint requiring a complete copy on every device. - /// Requires active compilation. For a replicated input or buffer description, - /// use the default sharding from `Tensor.init` or `Shape.init`. - pub fn replicate(self: Tensor) Tensor { - const compiler = Compiler.current(); - const partitioning: Sharding.Partitioning = .repeat(.replicated, self.rank()); - return self.withPartitioning(compiler.platform.replicated_mesh, partitioning); - } + pub fn toMemory(self: Tensor, kind: Memory) Tensor { + const ctx = CompilationContext.current(); + if (ctx.platform.target == .cpu) return self; - test withPartitioning { - // Create two similar meshes, - // Then check that tensor.withPartitioning don't confuse the meshes - const dp_mp: Sharding.Mesh = .{ - .name = "dp_mp", - .physical = undefined, - .logical = .mesh(.{ .data = .low_bandwidth, .model = .high_bandwidth }), - .bindings = .init(&.{.init(&.{.link_x})}), - .grid = undefined, - }; - - const mp_dp: Sharding.Mesh = .{ - .name = "mp_dp", - .physical = undefined, - .logical = .mesh(.{ .model = .low_bandwidth, .data = .high_bandwidth }), - .bindings = .init(&.{.init(&.{.link_x})}), - .grid = undefined, - }; - - const x = Tensor.init(.{ .h = 8 }, .f32).withPartitioning(&dp_mp, .{ .h = .model }); - try std.testing.expectEqual(&dp_mp, x.shape()._sharding.mesh); - try std.testing.expectEqual(Sharding.PartitionSpec.sharded(1), x.shape().partition(.h)); - try std.testing.expectEqual(&dp_mp, x.shape().reshape(.{ 2, 4 })._sharding.mesh); - - const x2 = x.withPartitioning(&mp_dp, .{ .h = .model }); - try std.testing.expectEqual(&mp_dp, x2.shape()._sharding.mesh); - try std.testing.expectEqual(Sharding.PartitionSpec.sharded(0), x2.shape().partition(.h)); - try std.testing.expectEqual(&mp_dp, x2.shape().reshape(.{ 2, 4 })._sharding.mesh); - - const x2_replicated = x.withPartitioning(&mp_dp, .{ .h = .replicated }); - try std.testing.expectEqual(&mp_dp, x2_replicated.shape()._sharding.mesh); - try std.testing.expectEqual(Sharding.PartitionSpec.replicated, x2_replicated.shape().partition(.h)); - } - - /// Copy the given tensor to the specified memory, preserving its sharding. - pub fn toMemory(self: Tensor, kind: Memory.Kind) Tensor { - const ctx = Compiler.current(); - switch (ctx.platform.target) { - .cpu, .neuron, .metal => { - // A single memory needs no copy, but the result must still be a distinct - // tensor: `reuseBuffer` donations are keyed by tensor id. - var res = _result(self._shape, self.value()); - res._shape._sharding = self._shape._sharding; - return res; - }, - .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, - } - - const frontend_attributes: *const mlir.Attribute = .dict(ctx.mlir_ctx, &.{ - .named( + const frontend_attributes = mlir.dictionaryAttribute(ctx.mlir_ctx, &.{ + .named(ctx.mlir_ctx, "_xla_buffer_placement", mlir.stringAttribute( ctx.mlir_ctx, - "_xla_buffer_placement", - .string(ctx.mlir_ctx, ctx.platform.memoryKind(kind)), - ), + ctx.platform.memoryKind(kind), + )), }); - const op = stablehlo.custom_call( + const op = dialects.stablehlo.custom_call( ctx.mlir_ctx, &.{self.value()}, &.{self.value().type_()}, @@ -312,97 +172,14 @@ pub const Tensor = struct { .named(ctx.mlir_ctx, "mhlo.frontend_attributes", frontend_attributes), }, }, - ctx.location, + .unknown(ctx.mlir_ctx), ).appendTo(currentBlock()); var res = _result(self._shape, op.result(0)); - res._shape._sharding = self._shape._sharding; - ctx.currentScope().id_to_memory.putNoClobber(ctx.currentScope().arena.allocator(), res.id, kind) catch @panic("OOM"); + ctx.currentScope().id_to_output_memory_kind.put(ctx.currentScope().arena.allocator(), res.id, kind) catch unreachable; return res; } - test toMemory { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - const io = std.testing.io; - - const inputs: [8]f32 = .{ -3.0, -2, -1, 1, 2, 3, 5, -5 }; - const x_t = Tensor.init(.{8}, .f32); - - const Local = struct { - fn memcpyH2D(x: Tensor) Tensor { - return x.onMemory(.host_pinned).toMemory(.device); - } - }; - - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcpyH2D, .{x_t}, platform, .{}); - defer exe.deinit(); - - var x_h = try zml.Buffer.fromBytesOpts(io, platform, x_t.shape(), @ptrCast(&inputs), .{ .memory = .host_pinned }); - defer x_h.deinit(); - - const x_h_ptr: [*]f32 = @ptrCast(@alignCast(x_h.opaqueDevicePtr(0))); - try std.testing.expectEqualSlices(f32, &inputs, x_h_ptr[0..8]); - - var x_d = try exe.eval(std.testing.allocator, io, .{x_h}); - defer x_d.deinit(); - - try zml.testing.expectClose(std.testing.io, x_h, x_d, .exact_match); - } - - /// Copy all the given tensor to the specified memory. - /// The input struct is copied on the stack, so it must be a simple flat struct without pointers. - pub fn toMemoryAll(flat_tensors: anytype, kind: Memory.Kind) @TypeOf(flat_tensors) { - const ctx = Compiler.current(); - switch (ctx.platform.target) { - .cpu, .neuron, .metal => return flat_tensors, - .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, - } - - var copy = flat_tensors; - meta.visitFlatStruct(struct { - fn toMemory(k: Memory.Kind, x: *Tensor) void { - x.* = x.toMemory(k); - } - }.toMemory, kind, ©); - return copy; - } - - /// Mark the given input tensor as being physically located on a specific memory. - /// Has no effect if the input tensor is not an executable input. - pub fn onMemory(self: Tensor, kind: Memory.Kind) Tensor { - const ctx = Compiler.current(); - switch (ctx.platform.target) { - .cpu, .neuron, .metal => return self, - .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, - } - - if (ctx.currentScope().id_to_argument.get(self.id) == null) { - return self; - } - - ctx.currentScope().id_to_memory.put(ctx.currentScope().arena.allocator(), self.id, kind) catch unreachable; - - return self; - } - - /// Mark all the given input tensors as being physically located on a specific memory - /// see `zml.Tensor.onMemory` - pub fn onMemoryAll(tensors: anytype, kind: Memory.Kind) void { - const ctx = Compiler.current(); - switch (ctx.platform.target) { - // Only one memory kind on those platform - .cpu, .neuron, .metal => return, - .cuda, .rocm, .tpu, .oneapi, .furiosa => {}, - } - - meta.visit(struct { - fn onMemory(k: Memory.Kind, x: *const Tensor) void { - _ = x.onMemory(k); - } - }.onMemory, kind, &tensors); - } - /// Returns a Tensor with new tag names. pub fn rename(self: Tensor, renames: anytype) Tensor { var res = self; @@ -424,12 +201,13 @@ pub const Tensor = struct { /// Returns the mlir.Value associated with the Tensor. /// - /// This will fail if used outside of a Compiler context. + /// This will fail if used outside of a compilation context. pub fn value(self: Tensor) *const mlir.Value { - const scope = Compiler.current().currentScope(); - return scope.id_to_argument.get(self.id) orelse - self._value orelse - @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); + if (CompilationContext.current().currentScope().id_to_argument.get(self.id)) |argument_index| { + return CompilationContext.current().currentScope().block.argument(argument_index); + } else if (self._value) |v| { + return v; + } else @panic("Something went really wrong, tensor is not an argument nor has an mlir.Value"); } /// Tell PJRT compiler that memory should be reuse between the two tensors. @@ -437,108 +215,24 @@ pub const Tensor = struct { /// but this API allows to reuse buffer between input and output arguments /// of a given function. /// Note this is visible from the outside. The caller of a function with donations - /// cannot use the donated input's contents after execution. Runners destroy its - /// handle automatically; raw `Exe.call` leaves it for the caller to deinit. + /// is not allowed to reuse the donated input buffer after the call. /// For `reuseBuffer` to be effective, it needs to propagate all the way through the output. - /// - /// The sharding of the origin tensor is propagated to this one. pub fn reuseBuffer(self: Tensor, origin: Tensor) Tensor { - const compilation_context = Compiler.current(); + const compilation_context = CompilationContext.current(); const scope = compilation_context.currentScope(); - if (scope.id_to_donation.get(origin.id)) |og_donation| { - // reuseBuffer is making the donation explicit - const donation: Compiler.Donation = switch (og_donation) { - .implicit => |input| .{ .explicit = input }, - .explicit => og_donation, - }; - scope.id_to_donation.put(scope.arena.allocator(), self.id, donation) catch @panic("OOM"); - - // The origin is mapped to a physical buffer either because it's an input or through `reuseBuffer` - // We can now look at its sharding. - if (origin._shape.isFullyReplicated()) { - stdx.debug.assert(self.byteSize() == origin.byteSize(), ".reuseBuffer expects two tensors with same byteSize, got {f} ({B}) and {f} ({B})", .{ self, self.byteSize(), origin, origin.byteSize() }); - } else { - // Origin is sharded, we can't just use the global byte size, we actually need the same byte size per shard. - // We are stricter here and require the same shape. - stdx.debug.assert(self._shape.eql(origin._shape), ".reuseBuffer expects two sharded tensors with same shape, got {f} and {f}", .{ self, origin }); - - // Normally self sharding is unspecified and we can just propagate origin sharding, - // but if the user called `.withPartitioning` explicitly it would be weird to override it silently, hence the check. - stdx.debug.assert(self._shape._sharding.isCompatible(origin._shape._sharding), ".reuseBuffer expects compatible shardings, got {f} and {f}", .{ self, origin }); - } - - var res = self; - res._shape._sharding = origin._shape._sharding; - return res; - } else { - // Origin tensor doesn't has any physical buffer associated so we can move on. - // This is allowed to allow a function like KVCache.update to call `reuseBuffer`, - // even though the caller may have constructed the inputs internally. - return self; - } - } - - test reuseBuffer { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - const io = std.testing.io; - - const inputs: [6]stdx.Packed(i4) = .{ .pack(-3.0, -2), .pack(-1, 1), .pack(2, 3), .pack(1, 2), .pack(3, 4), .pack(5, -5) }; - const x_t = Tensor.init(.{ 6, 2 }, .i4); - const y_t = Tensor.init(.{ 6, 2 }, .i4); - const z_t = Tensor.init(.{ 6, 2 }, .i4); - - const Local = struct { - fn memcopy(x: Tensor, y: Tensor, z: Tensor) [2]Tensor { - // the first output reuses the given y, but the second ouput does NOT reuse z. - return .{ x.addConstant(1).reuseBuffer(y), z }; - } - }; - - const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.memcopy, .{ x_t, y_t, z_t }, platform, .{}); - defer exe.deinit(); - - var x_d = try zml.Buffer.fromBytes(io, platform, x_t.shape(), @ptrCast(&inputs)); - defer x_d.deinit(); - - var y_d = try zml.Buffer.uninitialized(io, platform, y_t.shape(), .{}); - defer y_d.deinit(); - - var z_d = try zml.Buffer.uninitialized(io, platform, z_t.shape(), .{}); - defer z_d.deinit(); - - var y_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; - for (0..y_d.numShards()) |dev| { - y_memory[dev] = y_d.opaqueDevicePtr(dev); - } - - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - - for ([_]bool{ false, true }) |wait| { - var z2_d: zml.Buffer = undefined; - runner.run(io, .{ x_d, y_d, z_d }, .{ &y_d, &z2_d }, .{ .wait = wait }); - defer z2_d.deinit(); - try y_d.await(io); - - var output_memory: [Platform.MAX_NUM_DEVICES]*anyopaque = undefined; - for (0..y_d.numShards()) |dev| { - output_memory[dev] = y_d.opaqueDevicePtr(dev); - } - - // Check that the replacement y_d reuses the donated device memory. - try std.testing.expectEqualSlices(*anyopaque, y_memory[0..y_d.numShards()], output_memory[0..y_d.numShards()]); - - // Check that z_d and z2_d DO NOT POINT to the same addresses - for (0..z2_d.numShards()) |dev| { - try std.testing.expect(z2_d.opaqueDevicePtr(dev) != z_d.opaqueDevicePtr(dev)); - } + if (scope.id_to_argument.get(origin.id)) |argument_index| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = argument_index; + } else if (scope.id_to_donation.get(origin.id)) |origin_donation| { + const gop = scope.id_to_donation.getOrPut(scope.arena.allocator(), self.id) catch unreachable; + gop.value_ptr.* = origin_donation; } + return self; } /// Returns a Tensor containing the absolute value of each element of the input Tensor. pub fn abs(self: Tensor) Tensor { - const op = stablehlo.abs(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.abs(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); const dt = switch (self.dtype()) { .c64 => .f32, .c128 => .f64, @@ -569,11 +263,11 @@ pub const Tensor = struct { res_shape = res_shape.withDtype(dt); - const op = stablehlo.bitcast_convert( + const op = dialects.stablehlo.bitcast_convert( mlirCtx(), self.value(), - mlirx.Type.rankedTensor(mlirCtx(), res_shape), - currentLoc(), + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); @@ -586,26 +280,26 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise number of leading 0 bits in the input Tensor. pub fn countLeadingZeros(self: Tensor) Tensor { - const op = stablehlo.count_leading_zeros(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.count_leading_zeros(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing booleans indicating if each element of the input Tensor is finite. pub fn isFinite(self: Tensor) Tensor { - const op = stablehlo.is_finite(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.is_finite(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); } /// Returns a Tensor containing the element-wise number of bits set in the input Tensor. pub fn popcnt(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isInteger(), "popcnt expects tensor type to be an integer, got {}", .{self.dtype()}); - const op = stablehlo.popcnt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.popcnt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the sign of the input Tensor element-wise. pub fn sign(self: Tensor) Tensor { - const op = stablehlo.sign(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.sign(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -627,39 +321,31 @@ pub const Tensor = struct { inline for (inputs, expectations, divisors) |i, e, d| { const input: zml.Tensor = .init(.{6}, .f32); - const exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Tensor.fmod, - .{ input, d }, - platform, - .{}, - ); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.fmod, .{ input, d }, platform); defer exe.deinit(); var input_buffer = try zml.Buffer.fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&i)); defer input_buffer.deinit(); - var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); - defer output.deinit(); + const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.fmod, .{input_buffer}); - try zml.testing.expectClose(std.testing.io, zml.Slice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, .{}); + try zml.testing.expectClose(std.testing.io, zml.Slice.init(zml.Shape.init(.{6}, .f32), std.mem.sliceAsBytes(&e)), output, 1e-4); } } /// Returns a Tensor containing the element-wise left-shift operation of 'self' by 'other'. pub fn shiftLeft(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftLeft", stablehlo.shift_left)(self, other); + return binaryOp("shiftLeft", dialects.stablehlo.shift_left)(self, other); } /// Returns a Tensor containing the element-wise arithmetic right-shift operation of 'self' by 'other'. pub fn shiftRightArithmetic(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftRightArithmetic", stablehlo.shift_right_arithmetic)(self, other); + return binaryOp("shiftRightArithmetic", dialects.stablehlo.shift_right_arithmetic)(self, other); } /// Returns a Tensor containing the element-wise logical right-shift operation of 'self' by 'other'. pub fn shiftRightLogical(self: Tensor, other: Tensor) Tensor { - return binaryOp("shiftRightLogical", stablehlo.shift_right_logical)(self, other); + return binaryOp("shiftRightLogical", dialects.stablehlo.shift_right_logical)(self, other); } /// Returns the Cholesky decomposition of the input Tensor. @@ -669,24 +355,24 @@ pub const Tensor = struct { pub fn cholesky(self: Tensor, lower: bool) Tensor { stdx.debug.assert(self.rank() <= 2, "cholesky expects tensor rank to be <= 2, got {}", .{self.rank()}); - const op = stablehlo.cholesky(mlirCtx(), self.value(), lower, currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.cholesky(mlirCtx(), self.value(), lower, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Solves the system of linear equations formed by the input tensors. - pub fn triangularSolve(self: Tensor, other: Tensor, opts: stablehlo.TriangularSolveOpts) Tensor { + pub fn triangularSolve(self: Tensor, other: Tensor, opts: dialects.stablehlo.TriangularSolveOpts) Tensor { stdx.debug.assert(self.dtype() == other.dtype(), "triangularSolve expects tensors to be of the same type, got {} and {}", .{ self.dtype(), other.dtype() }); - stdx.debug.assert(self.rank() >= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank >= 2, got {} and {}", .{ self.rank(), other.rank() }); + stdx.debug.assert(self.rank() <= 2 and self.rank() == other.rank(), "triangularSolve expects tensors to have the same rank and be <= 2, got {} and {}", .{ self.rank(), other.rank() }); - const op = stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, currentLoc()).appendTo(currentBlock()); - return _result(other._shape, op.result(0)); + const op = dialects.stablehlo.triangular_solve(mlirCtx(), self.value(), other.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise rounding towards the nearest integer, breaking ties away from zero, of the input Tensor. pub fn roundNearestAfz(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestAfz expects tensor type to be a float, got {}", .{self.dtype()}); - const op = stablehlo.round_nearest_afz(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.round_nearest_afz(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -694,7 +380,7 @@ pub const Tensor = struct { pub fn roundNearestEven(self: Tensor) Tensor { stdx.debug.assert(self.dtype().isFloat(), "roundNearestEven expects tensor type to be a float, got {}", .{self.dtype()}); - const op = stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -703,7 +389,7 @@ pub const Tensor = struct { stdx.debug.assert(re._shape.eql(im._shape), "complex expects tensor shapes to match, got {f} and {f}", .{ re._shape, im._shape }); stdx.debug.assert(re.dtype() == .f32 or re.dtype() == .f64, "complex expects tensors type to be f32 or f64, got {}", .{re.dtype()}); - const op = stablehlo.complex(mlirCtx(), re.value(), im.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.complex(mlirCtx(), re.value(), im.value(), .unknown(mlirCtx())).appendTo(currentBlock()); const dt: DataType = if (re.dtype() == .f32) .c64 else .c128; return _result(re._shape.withDtype(dt), op.result(0)); } @@ -723,7 +409,7 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const op = stablehlo.real(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.real(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } @@ -744,12 +430,12 @@ pub const Tensor = struct { .c128 => .f64, else => unreachable, }; - const op = stablehlo.imag(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.imag(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape.withDtype(dt), op.result(0)); } /// Returns the Fast Fourier Transform (FFT) of the input Tensor. - pub fn fft(self: Tensor, opts: stablehlo.FftOpts) Tensor { + pub fn fft(self: Tensor, opts: dialects.stablehlo.FftOpts) Tensor { // TODO: support tagged API. stdx.debug.assert(1 <= opts.length.len and opts.length.len <= 3, "fft expects 'opts.length' length to be between 1 and 3 (inclusive), got {}", .{opts.length.len}); @@ -769,78 +455,41 @@ pub const Tensor = struct { .f32 => .c64, else => .c128, }; - const shape_ = self._shape.setDim(-1, rfftOutputLen(self.dim(-1))); + const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1), 2) + 1); break :blk shape_.withDtype(dt); }, .IRFFT => blk: { stdx.debug.assert(self.dtype().isComplex(), "fft({any}) expects tensor type to be complex, got {}", .{ opts, self.dtype() }); - const fft_dims = self.dims()[self.rank() - opts.length.len ..]; - const fft_last_dim_index = opts.length.len - 1; - stdx.debug.assert(std.mem.eql(i64, fft_dims[0..fft_last_dim_index], opts.length[0..fft_last_dim_index]), "fft({any}) expects non-innermost tensor FFT dimensions to match given lengths, got {any} and {any}", .{ opts, fft_dims[0..fft_last_dim_index], opts.length[0..fft_last_dim_index] }); - stdx.debug.assert(self.dim(-1) == rfftOutputLen(opts.length[fft_last_dim_index]), "fft({any}) expects IRFFT innermost dimension to match the RFFT output length, got {} for FFT length {}", .{ opts, self.dim(-1), opts.length[fft_last_dim_index] }); + stdx.debug.assert(std.mem.eql(i64, self.dims()[self.rank() - opts.length.len ..], opts.length), "fft({any}) expects tensor last dimensions to match given lengths, got {} and {}", .{ opts, self.dims()[self.rank() - opts.length.len ..].len, opts.length.len }); const dt: DataType = switch (self.dtype()) { .c64 => .f32, else => .f64, }; - const shape_ = self._shape.setDim(-1, opts.length[fft_last_dim_index]); + const shape_ = self._shape.setDim(-1, @divExact(self.dim(-1) - 1, 2)); break :blk shape_.withDtype(dt); }, }; - const op = stablehlo.fft(mlirCtx(), self.value(), opts, currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.fft(mlirCtx(), self.value(), opts, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(sh, op.result(0)); } - fn rfftOutputLen(input_len: i64) i64 { - const rfft_len_divisor = 2; - const rfft_edge_bin_count = 1; - return @divTrunc(input_len, rfft_len_divisor) + rfft_edge_bin_count; - } - - test "Tensor.fft IRFFT restores RFFT last dimension" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - const rfft_input_len: i64 = 10; - - const Local = struct { - pub fn _fwd(x: Tensor) Tensor { - const freq = x.fft(.{ .kind = .RFFT, .length = &.{rfft_input_len} }); - std.debug.assert(freq.dim(-1) == rfftOutputLen(rfft_input_len)); - const time = freq.fft(.{ .kind = .IRFFT, .length = &.{rfft_input_len} }); - std.debug.assert(time.dim(-1) == x.dim(-1)); - return time; - } - }; - - var exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Local._fwd, - .{Tensor.init(.{rfft_input_len}, .f32)}, - platform, - .{}, - ); - defer exe.deinit(); - } - pub const Rng = struct { _state: Tensor, - algorithm: stablehlo.RngAlgorithmAttr.Type = .DEFAULT, - - pub const Buffer = mem.Bufferized(Rng); + algorithm: dialects.stablehlo.RngAlgorithm.Type = .DEFAULT, pub fn init() Rng { return .{ ._state = .init(.{2}, .u64) }; } - pub fn initBuffer(rng: Rng, io: std.Io, platform: *const Platform, seed: u128) !Buffer { + pub fn initBuffer(platform: *const Platform, seed: u128, io: std.Io) !Bufferized(Rng) { return .{ - ._state = try .fromBytes(io, platform, rng._state._shape, std.mem.asBytes(&seed)), + ._state = try .fromBytes(io, platform, Shape.init(.{2}, .u64), std.mem.asBytes(&seed)), }; } - pub fn deinitBuffer(self: *Buffer) void { + pub fn deinitBuffer(self: *Bufferized(Rng)) void { self._state.deinit(); } @@ -850,15 +499,15 @@ pub const Tensor = struct { /// The output is guaranteed to be deterministic function of `self` Rng state, /// but it is not guaranteed to be deterministic between implementations. pub fn bitGenerator(self: Rng, sh: Shape) struct { Rng, Tensor } { - const op = stablehlo.rng_bit_generator( + const op = dialects.stablehlo.rng_bit_generator( mlirCtx(), self.algorithm, self._state.value(), - mlirx.Type.rankedTensor(mlirCtx(), self._state.shape()), - mlirx.Type.rankedTensor(mlirCtx(), sh), - currentLoc(), + mlir.rankedTensorType(self._state.dims(), mlirx.Type.fromDType(mlirCtx(), self._state.dtype())), + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), ).appendTo(currentBlock()); - return .{ self.update(op.result(0)), _resultPropagateSharding(sh, op.result(1)) }; + return .{ self.update(op.result(0)), _result(sh, op.result(1)) }; } fn update(self: Rng, new_state: *const mlir.Value) Rng { @@ -904,8 +553,7 @@ pub const Tensor = struct { floats = floats.mul(scalar(opts.max - opts.min, dt)).addConstant(opts.min); // Convert back to integer if needed. - const result = floats.convert(shape_.dtype()); - return .{ rng, _resultPropagateSharding(shape_, result.value()) }; + return .{ rng, floats.convert(shape_.dtype()) }; } test uniform { @@ -937,23 +585,21 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Compute stats over a uniform distribution on [-2, 10]. - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.uniformStats, .{ Rng.init(), Shape.init(.{1024}, .f32), .{ .min = -2, .max = 10 } }, platform); defer exe.deinit(); - var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); + var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); defer rng_buffer._state.deinit(); - var stats: zml.Bufferized(Stats) = undefined; - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{rng_buffer}, .{ &rng_buffer, &stats }, .{ .wait = true }); + var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.uniformStats, .{rng_buffer}); + defer rand._state.deinit(); defer stats.mean.deinit(); defer stats.variance.deinit(); defer stats.min.deinit(); defer stats.max.deinit(); // Check the Rng state has been modified. - try std.testing.expect(try rng_buffer._state.getValue(u128, std.testing.io) != 1234); + try std.testing.expect(try rand._state.getValue(u128, std.testing.io) != 1234); // Check the mean and variance are close to theoritical values. const mean_ = try stats.mean.getValue(f32, std.testing.io); @@ -983,8 +629,8 @@ pub const Tensor = struct { const a = Tensor.constant(DataType.Value.init(sh.dtype(), opts.mean)); const b = Tensor.constant(DataType.Value.init(sh.dtype(), opts.stddev)); const res_tensor_shape = Tensor.constantTensor(Shape.init(.{sh.rank()}, .i64), std.mem.sliceAsBytes(sh.dims())); - const op = stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, currentLoc()).appendTo(currentBlock()); - return _resultPropagateSharding(sh, op.result(0)); + const op = dialects.stablehlo.rng(mlirCtx(), a.value(), b.value(), res_tensor_shape.value(), .NORMAL, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(sh, op.result(0)); } /// Returns a Tensor of the given shape, filled with floating point numbers sampled from a Gumbel distribution, and a new Rng state. @@ -1002,8 +648,7 @@ pub const Tensor = struct { // We don't want 0 to be sampled otherwise `log` will return -inf. .{ .min = std.math.floatEps(f32), .max = 1 }, ); - const result = u.log().scale(-1).log().scale(-1).convert(shape_.dtype()); - return .{ rand, _resultPropagateSharding(shape_, result.value()) }; + return .{ rand, u.log().scale(-1).log().scale(-1).convert(shape_.dtype()) }; } test gumbel { @@ -1044,28 +689,25 @@ pub const Tensor = struct { }; const platform = zml.testing.env(); - const tgt_dist_data = [_]f32{ 2.0, 1.0, 4.0, 3.0 }; const tgt_dist: Tensor = .init(.{tgt_dist_data.len}, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Stats.gumbelStats, .{ Rng.init(), tgt_dist }, platform); defer exe.deinit(); - var rng_buffer = try Rng.init().initBuffer(std.testing.io, platform, 1234); + var rng_buffer = try Rng.initBuffer(platform, 1234, std.testing.io); defer rng_buffer._state.deinit(); - var tgt_dist_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), @ptrCast(&tgt_dist_data)); + var tgt_dist_buffer: Buffer = try .fromBytes(std.testing.io, platform, tgt_dist.shape(), std.mem.sliceAsBytes(&tgt_dist_data)); defer tgt_dist_buffer.deinit(); - var stats: zml.Bufferized(Stats) = undefined; - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{ rng_buffer, tgt_dist_buffer }, .{ &rng_buffer, &stats }, .{ .wait = true }); + var rand, var stats = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Stats.gumbelStats, .{ rng_buffer, tgt_dist_buffer }); + defer rand._state.deinit(); defer stats.mean.deinit(); defer stats.variance.deinit(); defer stats.actual_dist.deinit(); // Check the Rng state has been modified. - try std.testing.expect(try rng_buffer._state.getValue(i128, std.testing.io) != 1234); + try std.testing.expect(try rand._state.getValue(i128, std.testing.io) != 1234); // Check the mean and variance are close to theoritical values. const mean_ = try stats.mean.getValue(f32, std.testing.io); @@ -1092,11 +734,11 @@ pub const Tensor = struct { stdx.debug.assert(1 <= exponent_bits, "reducePrecision expects 'exponent_bits' to be >= 1, got {}", .{exponent_bits}); stdx.debug.assert(0 <= mantissa_bits, "reducePrecision expects 'mantissa_bits' to be positive, got {}", .{mantissa_bits}); - const op = stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.reduce_precision(mlirCtx(), self.value(), exponent_bits, mantissa_bits, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } - fn convolution(self: Tensor, other: Tensor, opts: stablehlo.ConvolutionOpts) Tensor { + fn convolution(self: Tensor, other: Tensor, opts: dialects.stablehlo.ConvolutionOpts) Tensor { stdx.debug.assert(self.rank() == other.rank(), "convolution expects tensor ranks to match, got {} and {}", .{ self.rank(), other.rank() }); const N = self.rank(); stdx.debug.guard(opts.window_strides.len == N - 2, @src()); @@ -1179,13 +821,13 @@ pub const Tensor = struct { } // inferred shape '[1, 256, 1, 12008]' is incompatible with return type of operation '[1, 256, 1, 11978]' - const op = stablehlo.convolution( + const op = dialects.stablehlo.convolution( mlirCtx(), self.value(), other.value(), - mlirx.Type.rankedTensor(mlirCtx(), new_shape), + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), used_opts, - currentLoc(), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(new_shape, op.result(0)); @@ -1280,47 +922,42 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise addition of the input Tensors. pub fn add(self: Tensor, other: Tensor) Tensor { - return binaryOp("add", stablehlo.add)(self, other); + return binaryOp("add", dialects.stablehlo.add)(self, other); } /// Returns a Tensor containing the element-wise subtraction of the input Tensors. pub fn sub(self: Tensor, other: Tensor) Tensor { - return binaryOp("subtract", stablehlo.subtract)(self, other); + return binaryOp("subtract", dialects.stablehlo.subtract)(self, other); } /// Returns a Tensor containing the element-wise multiplication of the input Tensors. pub fn mul(self: Tensor, other: Tensor) Tensor { - return binaryOp("mul", stablehlo.multiply)(self, other); + return binaryOp("mul", dialects.stablehlo.multiply)(self, other); } /// Returns a Tensor containing the element-wise division of the input Tensors. pub fn div(self: Tensor, other: Tensor) Tensor { - return binaryOp("div", stablehlo.divide)(self, other); + return binaryOp("div", dialects.stablehlo.divide)(self, other); } /// Returns a Tensor containing the element-wise exponentiation of the input Tensors. pub fn pow(self: Tensor, other: Tensor) Tensor { - return binaryOp("pow", stablehlo.power)(self, other); + return binaryOp("pow", dialects.stablehlo.power)(self, other); } /// Returns a Tensor containing the element-wise maximum operation of the input Tensors. pub fn maximum(self: Tensor, other: Tensor) Tensor { - return binaryOp("maximum", stablehlo.maximum)(self, other); + return binaryOp("maximum", dialects.stablehlo.maximum)(self, other); } /// Returns a Tensor containing the element-wise minimum operation of the input Tensors. pub fn minimum(self: Tensor, other: Tensor) Tensor { - return binaryOp("minimum", stablehlo.minimum)(self, other); + return binaryOp("minimum", dialects.stablehlo.minimum)(self, other); } /// Returns a Tensor containing the element-wise remainder of dividend 'self' and divisor 'other'. pub fn remainder(self: Tensor, other: Tensor) Tensor { - return binaryOp("remainder", stablehlo.remainder)(self, other); - } - - /// Returns a Tensor containing the element-wise substraction of the input Tensor with a constant. - pub fn remainderConst(self: Tensor, b: anytype) Tensor { - return self.remainder(.scalar(b, self.dtype())); + return binaryOp("remainder", dialects.stablehlo.remainder)(self, other); } /// Returns a Tensor containing the element-wise addition of the input Tensor with a constant. @@ -1328,11 +965,6 @@ pub const Tensor = struct { return self.add(.scalar(b, self.dtype())); } - /// Returns a Tensor containing the element-wise substraction of the input Tensor with a constant. - pub fn subConstant(self: Tensor, b: anytype) Tensor { - return self.sub(.scalar(b, self.dtype())); - } - /// Returns a Tensor containing the element-wise division of the input Tensor by a constant. pub fn divByConst(self: Tensor, b: anytype) Tensor { return self.div(.scalar(b, self.dtype())); @@ -1353,20 +985,20 @@ pub const Tensor = struct { /// Returns a Tensor containing the element-wise logical operation of the input Tensors. pub fn logical(self: Tensor, comptime logical_op: LogicalOp, other: Tensor) Tensor { return switch (logical_op) { - .OR => binaryOp("or", stablehlo.or_)(self, other), - .XOR => binaryOp("xor", stablehlo.xor)(self, other), - .AND => binaryOp("and", stablehlo.and_)(self, other), + .OR => binaryOp("or", dialects.stablehlo.or_)(self, other), + .XOR => binaryOp("xor", dialects.stablehlo.xor)(self, other), + .AND => binaryOp("and", dialects.stablehlo.and_)(self, other), }; } /// Returns a Tensor containing the element-wise floor operation of the input Tensor. pub fn floor(self: Tensor) Tensor { - return _result(self._shape, stablehlo.floor(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); + return _result(self._shape, dialects.stablehlo.floor(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise ceil operation of the input Tensor. pub fn ceil(self: Tensor) Tensor { - return _result(self._shape, stablehlo.ceil(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()).result(0)); + return _result(self._shape, dialects.stablehlo.ceil(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()).result(0)); } /// Returns a Tensor containing the element-wise conversion to another type. @@ -1375,144 +1007,68 @@ pub const Tensor = struct { return self; } - const res_type = mlirx.Type.rankedTensor(mlirCtx(), self.shape().withDtype(to)); - const op = stablehlo.convert(mlirCtx(), self.value(), res_type, currentLoc()).appendTo(currentBlock()); + const res_type = mlir.rankedTensorType(self.shape().dims(), mlirx.Type.fromDType(mlirCtx(), to)); + const op = dialects.stablehlo.convert(mlirCtx(), self.value(), res_type, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape.withDtype(to), op.result(0)); } - test "convert f32 -> f4e2m1" { + test convert { const floats = @import("floats.zig"); const zml = @import("zml.zig"); const platform = zml.testing.env(); - const x_f32 = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; - var x_f4: [x_f32.len / 2]floats.Float4E2M1.Packed = undefined; - for (&x_f4, 0..) |*f4_pack, i| { - f4_pack.* = .fromF32(x_f32[2 * i], x_f32[2 * i + 1]); + + // f4e2m1 + { + const x = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6, -0.0, -0.5, -1, -1.5, -2, -3, -4, -6 }; + var x_f4: [x.len]floats.Float4E2M1 = undefined; + for (&x_f4, &x) |*xi_f4, xi| xi_f4.* = .fromF32(xi); + + const x_d: Tensor = .init(.{x.len}, .f32); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f4e2m1 }, platform); + defer exe.deinit(); + + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); + defer x_d_buffer.deinit(); + + var x_f4_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f4_xla.free(std.testing.allocator); + + errdefer std.log.warn("convert(.f4e2m1) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f4, x_f4_xla }); + try std.testing.expectEqualDeep(&x_f4, x_f4_xla.items(floats.Float4E2M1)); } - const x_d: Tensor = .init(.{x_f32.len}, .f32); - const exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Tensor.convert, - .{ x_d, .f4e2m1 }, - platform, - .{}, - ); - defer exe.deinit(); + // f8e3m4 + { + const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; + var x_f8e3: [x.len]floats.Float8E3M4 = undefined; + for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), @ptrCast(&x_f32)); - defer x_d_buffer.deinit(); + const x_d: Tensor = .init(.{x.len}, .f32); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.convert, .{ x_d, .f8e3m4 }, platform); + defer exe.deinit(); - var x_f4_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); - defer x_f4_xla_d.deinit(); - const x_f4_xla = try x_f4_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); - defer x_f4_xla.free(std.testing.allocator); - const x_f4_items = x_f4_xla.items(floats.Float4E2M1.Packed); + var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); + defer x_d_buffer.deinit(); - errdefer std.log.warn("convert(f32 -> f4e2m1) failed !\ninput f32:\n{d}\nzml.floats computed:\n{d}\nxla computed:\n{d}", .{ stdx.fmt.slice(&x_f32), stdx.fmt.slice(&x_f4), x_f4_xla }); - try std.testing.expectEqualSlices(floats.Float4E2M1.Packed, &x_f4, x_f4_items); - } + var x_f8e3_xla_d = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.convert, .{x_d_buffer}); + const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); + defer x_f8e3_xla.free(std.testing.allocator); - test "convert f32 -> f8e3m4" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const x = [_]f32{ 1.1 / 4.0, 1.1 / 8.0, 1.1 / 16.0, 1.1 / 32.0, 1.1 / 64.0, 1.1 / 128.0 }; - var x_f8e3: [x.len]zml.floats.Float8E3M4 = undefined; - for (&x_f8e3, &x) |*xi_f8e3, xi| xi_f8e3.* = .fromF32(xi); - - const x_d: Tensor = .init(.{x.len}, .f32); - const exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Tensor.convert, - .{ x_d, .f8e3m4 }, - platform, - .{}, - ); - defer exe.deinit(); - - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_d.shape(), std.mem.sliceAsBytes(&x)); - defer x_d_buffer.deinit(); - - var x_f8e3_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); - defer x_f8e3_xla_d.deinit(); - const x_f8e3_xla = try x_f8e3_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); - defer x_f8e3_xla.free(std.testing.allocator); - - errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{d}\nzml.floats computed:\n{d}\nxla computed:\n{d}", .{ stdx.fmt.slice(&x), stdx.fmt.slice(&x_f8e3), x_f8e3_xla }); - try std.testing.expectEqualSlices(u8, @ptrCast(&x_f8e3), x_f8e3_xla.constData()); - } - - test "convert u2 -> u8" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - const x_u2: [2]stdx.Packed(u2) = .{ .pack(0, 1, 2, 3), .pack(3, 2, 1, 0) }; - comptime std.debug.assert(@sizeOf(@TypeOf(x_u2)) == 2); - const x_u8: [8]u8 = .{ 0, 1, 2, 3, 3, 2, 1, 0 }; - - const x_u2_t: Tensor = .init(.{8}, .u2); - const exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Tensor.convert, - .{ x_u2_t, .u8 }, - platform, - .{}, - ); - defer exe.deinit(); - - var x_d_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x_u2_t.shape(), @ptrCast(&x_u2)); - defer x_d_buffer.deinit(); - - var x_u8_xla_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_d_buffer}); - defer x_u8_xla_d.deinit(); - const x_u8_xla = try x_u8_xla_d.toSliceAlloc(std.testing.allocator, std.testing.io); - defer x_u8_xla.free(std.testing.allocator); - - try std.testing.expectEqualSlices(u8, &x_u8, x_u8_xla.constData()); - } - - test "convert f4e2m1 -> f32" { - const floats = @import("floats.zig"); - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - const x_f4_packed = [_]floats.Float4E2M1.Packed{ .fromF32(0, 0.5), .fromF32(1, 1.5), .fromF32(2, 3), .fromF32(4, 6) }; - - const x_t: Tensor = .init(.{x_f4_packed.len * 2}, .f4e2m1); - const exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Tensor.convert, - .{ x_t, .f32 }, - platform, - .{}, - ); - defer exe.deinit(); - - var x_f4_d: zml.Buffer = try .fromBytes(std.testing.io, platform, x_t.shape(), @ptrCast(&x_f4_packed)); - defer x_f4_d.deinit(); - - var x_f32_d = try exe.eval(std.testing.allocator, std.testing.io, .{x_f4_d}); - defer x_f32_d.deinit(); - const x_f32_h = try x_f32_d.toSliceAlloc(std.testing.allocator, std.testing.io); - defer x_f32_h.free(std.testing.allocator); - - const x_f32_expected = [_]f32{ 0.0, 0.5, 1, 1.5, 2, 3, 4, 6 }; - errdefer std.log.warn("convert(f4e2m1x2 -> f32) failed !\ninput f4e2m1x2:\n{e}\nexpected:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x_f4_packed), x_f32_expected, x_f32_h }); - try std.testing.expectEqualSlices(f32, &x_f32_expected, x_f32_h.items(f32)); + errdefer std.log.warn("convert(.f8e3m4) failed !\ninput f32:\n{e}\nzml.floats computed:\n{any}\nxla computed:\n{any}", .{ stdx.fmt.slice(&x), x_f8e3, x_f8e3_xla }); + try std.testing.expectEqualDeep(&x_f8e3, x_f8e3_xla.items(floats.Float8E3M4)); + } } /// Returns a Tensor containing the element-wise rounding operation of the input Tensor. pub fn round(self: Tensor) Tensor { - const round_op = stablehlo.round_nearest_even(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const round_op = dialects.stablehlo.round_nearest_even(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, round_op.result(0)); } /// Returns a Tensor containing the element-wise clamping operation of the input Tensor. pub fn clamp(self: Tensor, min_: Tensor, max_: Tensor) Tensor { - const op = stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.clamp(mlirCtx(), min_.value(), self.value(), max_.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -1521,52 +1077,21 @@ pub const Tensor = struct { /// Axes with the same tag on both sides, and which aren't contracting, /// are considered "batching axes". pub fn dot(lhs: Tensor, rhs: Tensor, args: anytype) Tensor { - const dot_axes = lhs.dotAxes(rhs, args); - return lhs.dotGeneral(rhs, dot_axes.contracting.constSlice(), dot_axes.batching.constSlice()); - } + stdx.debug.assert(lhs.shape().hasTag(args) != null, "Expected lhs to have {any} tag, got {f}", .{ args, lhs.shape() }); + stdx.debug.assert(rhs.shape().hasTag(args) != null, "Expected rhs to have {any} tag, got {f}", .{ args, rhs.shape() }); - pub const DotAxes = struct { - contracting: stdx.BoundedArray([2]i8, constants.MAX_RANK), - batching: stdx.BoundedArray([2]i8, constants.MAX_RANK), - }; + const lhs_contracting_dim: i8 = @intCast(lhs.shape().hasTag(args).?); + const rhs_contracting_dim: i8 = @intCast(rhs.shape().hasTag(args).?); - /// Returns contracting and batching axis pairs inferred from one or more tags. - pub fn dotAxes(lhs: Tensor, rhs: Tensor, tags: anytype) DotAxes { - const T = @TypeOf(tags); - if (comptime T == Shape.Tag or T == @EnumLiteral()) { - return lhs.dotAxes(rhs, @as([]const Shape.Tag, &.{Shape.toTag(tags)})); - } - - const lhs_contracting_axes, _ = lhs.shape().parseAxes(tags); - const rhs_contracting_axes, _ = rhs.shape().parseAxes(tags); - var contracting_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .empty; - for (lhs_contracting_axes.constSlice(), rhs_contracting_axes.constSlice()) |l, r| { - contracting_axes.appendAssumeCapacity(.{ @intCast(l), @intCast(r) }); - } - - var batching_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .empty; + var batching_axes: stdx.BoundedArray([2]i8, constants.MAX_RANK) = .{}; for (0..lhs.rank()) |lhs_tag_index| { - if (std.mem.indexOfScalar(u3, lhs_contracting_axes.constSlice(), @intCast(lhs_tag_index)) != null) { - continue; - } - const lhs_tag = lhs.shape().tag(lhs_tag_index); + if (lhs_tag == Shape.toTag(args)) continue; if (rhs.shape().hasTag(lhs_tag)) |rhs_tag_index| { batching_axes.appendAssumeCapacity(.{ @intCast(lhs_tag_index), @intCast(rhs_tag_index) }); } } - - return .{ .contracting = contracting_axes, .batching = batching_axes }; - } - - test dotAxes { - const lhs: Tensor = .init(.{ .a = 20, .b = 21, .c = 22 }, .f32); - const rhs: Tensor = .init(.{ .c = 22, .b = 21, .d = 23, .a = 20 }, .f32); - - const dot_axes = lhs.dotAxes(rhs, .{ .c, .a }); - - try std.testing.expectEqualSlices([2]i8, &.{ .{ 2, 0 }, .{ 0, 3 } }, dot_axes.contracting.constSlice()); - try std.testing.expectEqualSlices([2]i8, &.{.{ 1, 1 }}, dot_axes.batching.constSlice()); + return lhs.dotGeneral(rhs, &.{.{ lhs_contracting_dim, rhs_contracting_dim }}, batching_axes.slice()); } test dot { @@ -1605,14 +1130,7 @@ pub const Tensor = struct { } }.forward; - const exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - forward, - .{ x, y, ctr }, - platform, - .{}, - ); + const exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, y, ctr }, platform); defer exe.deinit(); try zml.testing.expectEqualShapes(Shape.init(z_shape, .f32), exe.output_shapes[0]); @@ -1633,10 +1151,10 @@ pub const Tensor = struct { const Axes = stdx.BoundedArray(i64, constants.MAX_RANK); - var res_shape: Shape = .scalar(lhs.dtype()); + var res_shape: Shape = .{ ._dtype = lhs.dtype() }; // Validate batching axes - var lhs_batching_axes: Axes = .empty; - var rhs_batching_axes: Axes = .empty; + var lhs_batching_axes: Axes = .{}; + var rhs_batching_axes: Axes = .{}; for (batching_axes) |b_axes| { const l, const r = b_axes; stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects batching dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); @@ -1648,8 +1166,8 @@ pub const Tensor = struct { } // Validate contracting axes - var lhs_contracting_axes: Axes = .empty; - var rhs_contracting_axes: Axes = .empty; + var lhs_contracting_axes: Axes = .{}; + var rhs_contracting_axes: Axes = .{}; for (contracting_axes) |c_axes| { const l, const r = c_axes; stdx.debug.assert(lhs._shape.dim(l) == rhs._shape.dim(r), "dotGeneral expects contracting dimensions to be equal, got {} and {} in {f} and {f}", .{ l, r, lhs, rhs }); @@ -1679,11 +1197,11 @@ pub const Tensor = struct { res_shape = res_shape.appendDim(rhs._shape.dim(r), rhs._shape.tag(r)); } - const op = stablehlo.dot_general( + const op = dialects.stablehlo.dot_general( mlirCtx(), lhs.value(), rhs.value(), - mlirx.Type.rankedTensor(mlirCtx(), res_shape), + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), .{ .lhs_batching_dimensions = lhs_batching_axes.constSlice(), .rhs_batching_dimensions = rhs_batching_axes.constSlice(), @@ -1691,14 +1209,14 @@ pub const Tensor = struct { .rhs_contracting_dimensions = rhs_contracting_axes.constSlice(), .dot_precision = .fast, }, - currentLoc(), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } /// Returns a Tensor containing the sigmoid function applied to each element of the input Tensor. pub fn sigmoid(self: Tensor) Tensor { - const op = stablehlo.logistic(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.logistic(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -1723,193 +1241,25 @@ pub const Tensor = struct { const platform = zml.testing.env(); const input: Tensor = .init(.{2}, .f32); - var exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Tensor.leakyReLU, - .{ input, 0.1 }, - platform, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.leakyReLU, .{ input, 0.1 }, platform); defer exe.deinit(); var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.6884, 1.6795 })); defer input_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.leakyReLU, .{input_buffer}); defer res.deinit(); const expectation: zml.Slice = .init(input.shape(), std.mem.sliceAsBytes(&[2]f32{ -0.0688, 1.6795 })); - try zml.testing.expectClose(std.testing.io, expectation, res, .{}); + try zml.testing.expectClose(std.testing.io, expectation, res, 1e-4); } /// Returns a Tensor containing the SwiGLU activation function applied to the input Tensor. - pub fn swiglu(self: Tensor, beta: f32, w: Tensor, v: Tensor, b: Tensor, c: Tensor, tag: Shape.Tag) Tensor { - var value_path = self.dot(w, tag); - value_path = value_path.add(b.broad(value_path.shape())); + pub fn swiglu(self: Tensor, beta: f32, w: Tensor, b: Tensor, tag: Shape.Tag) Tensor { + const sigmoid_tensor = self.mul(Tensor.constant(DataType.Value.init(self.dtype(), beta)).broad(.init(self._shape, self.dtype()))).sigmoid(); + const one_minus_sigmoid_tensor = Tensor.constant(.init(self.dtype(), 1)).broad(.init(self._shape, self.dtype())).sub(sigmoid_tensor); - const sigmoid_tensor = value_path.scale(beta).sigmoid(); - - var gate_path = self.dot(v, tag); - gate_path = gate_path.add(c.broad(gate_path.shape())); - - return value_path.mul(sigmoid_tensor).outer(gate_path); - } - - test swiglu { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const input: Tensor = .init(.{ .d = 4 }, .f32); - const beta = 3; - const w: Tensor = .init(.{ .c = 3, .d = 4 }, .f32); - const v: Tensor = .init(.{ .e = 3, .d = 4 }, .f32); - const b: Tensor = .init(.{ .c = 3 }, .f32); - const c: Tensor = .init(.{ .e = 3 }, .f32); - - var exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Tensor.swiglu, - .{ input, beta, w, v, b, c, Shape.toTag(.d) }, - platform, - .{}, - ); - defer exe.deinit(); - - var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&[4]f32{ 1, 0, 1, 2 })); - defer input_buffer.deinit(); - - var w_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, w.shape(), std.mem.sliceAsBytes(&[3][4]f32{ - .{ 1, 0, 2, 0 }, - .{ 2, 1, 3, 3 }, - .{ 1, 1, 0, 0 }, - })); - defer w_buffer.deinit(); - - var v_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, v.shape(), std.mem.sliceAsBytes(&[3][4]f32{ - .{ 0, 0, 2, 1 }, - .{ 1, 2, 1, 3 }, - .{ 0, 0, 2, 0 }, - })); - defer v_buffer.deinit(); - - var b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, b.shape(), std.mem.sliceAsBytes(&[3]f32{ 1, 1, 1 })); - defer b_buffer.deinit(); - - var c_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, c.shape(), std.mem.sliceAsBytes(&[3]f32{ 2, 2, 2 })); - defer c_buffer.deinit(); - - var res = try exe.eval(std.testing.allocator, std.testing.io, .{ input_buffer, w_buffer, v_buffer, b_buffer, c_buffer }); - defer res.deinit(); - - const output_shape = Shape.init(.{ .c = 3, .e = 3 }, .f32); - const expectation: zml.Slice = .init(output_shape, std.mem.sliceAsBytes(&[3][3]f32{ - .{ 23.999856, 39.99976, 15.999904 }, - .{ 72.000000, 120.000000, 48.000000 }, - .{ 11.970324, 19.950540, 7.980216 }, - })); - - try zml.testing.expectClose(std.testing.io, expectation, res, .{}); - } - - /// Returns an element-wise approximation of the error function, computed and returned in FP32. - /// Uses the Eigen rational approximation from CHLO's FP32 erf legalization: - /// https://github.com/openxla/stablehlo/blob/95ee4c70146864ed7d0a21dd135ccec37ece1b52/stablehlo/transforms/ChloLegalizeToStablehlo.cpp#L1038 - pub fn erf(input: Tensor) Tensor { - const x = input.convert(.f32).clamp(.scalar(-4, .f32), .scalar(4, .f32)); - const x2 = x.mul(x); - // Coefficients run from the highest degree to the constant term. - const alpha = [_]f32{ - -2.72614225801306e-10, - 2.77068142495902e-08, - -2.10102402082508e-06, - -5.69250639462346e-05, - -7.34990630326855e-04, - -2.95459980854025e-03, - -1.60960333262415e-02, - }; - const beta = [_]f32{ - -1.45660718464996e-05, - -2.13374055278905e-04, - -1.68282697438203e-03, - -7.37332916720468e-03, - -1.42647390514189e-02, - }; - var numerator = x2.scale(alpha[0]).addConstant(alpha[1]); - inline for (alpha[2..]) |coefficient| { - numerator = numerator.mul(x2).addConstant(coefficient); - } - var denominator = x2.scale(beta[0]).addConstant(beta[1]); - inline for (beta[2..]) |coefficient| { - denominator = denominator.mul(x2).addConstant(coefficient); - } - return x.mul(numerator).div(denominator).clamp(.scalar(-1, .f32), .scalar(1, .f32)); - } - - /// Returns an element-wise approximation of the complementary error function in FP32. - /// Uses CHLO's Cephes-based approximation to preserve the small positive tail: - /// https://github.com/openxla/stablehlo/blob/95ee4c70146864ed7d0a21dd135ccec37ece1b52/stablehlo/transforms/ChloLegalizeToStablehlo.cpp#L1018 - pub fn erfc(input: Tensor) Tensor { - const x = input.convert(.f32); - const abs_x = x.abs(); - const x2 = x.mul(x); - const one = Tensor.scalar(1, .f32); - - // For |x| < 1, erf(x) = x T(x^2). Coefficients run from highest degree to constant. - const t = [_]f32{ - 7.853861353153693e-5, - -8.010193625184903e-4, - 5.188327685732524e-3, - -2.685381193529856e-2, - 1.128358514861418e-1, - -3.761262582423300e-1, - 1.128379165726710, - }; - var small_poly = x2.scale(t[0]).addConstant(t[1]); - inline for (t[2..]) |coefficient| { - small_poly = small_poly.mul(x2).addConstant(coefficient); - } - const small = x.mul(small_poly).negate().addConstant(1); - - // For |x| >= 1, evaluate exp(-x^2) / |x| times P(1/x^2) or R(1/x^2). - const p = [_]f32{ - 2.326819970068386e-2, - -1.387039388740657e-1, - 3.687424674597105e-1, - -5.824733027278666e-1, - 6.210004621745983e-1, - -4.944515323274145e-1, - 3.404879937665872e-1, - -2.741127028184656e-1, - 5.638259427386472e-1, - }; - const r = [_]f32{ - -1.047766399936249e1, - 1.297719955372516e1, - -7.495518717768503, - 2.921019019210786, - -1.015265279202700, - 4.218463358204948e-1, - -2.820767439740514e-1, - 5.641895067754075e-1, - }; - const reciprocal_x2 = one.div(x2); - var middle_poly = reciprocal_x2.scale(p[0]).addConstant(p[1]); - inline for (p[2..]) |coefficient| { - middle_poly = middle_poly.mul(reciprocal_x2).addConstant(coefficient); - } - var tail_poly = reciprocal_x2.scale(r[0]).addConstant(r[1]); - inline for (r[2..]) |coefficient| { - tail_poly = tail_poly.mul(reciprocal_x2).addConstant(coefficient); - } - const poly = abs_x.cmp(.LT, .scalar(2, .f32)).select(middle_poly, tail_poly); - const z = x2.negate(); - const tail = z.exp().mul(one.div(abs_x)).mul(poly); - // Match CHLO's underflow cutoff, then use erfc(-x) = 2 - erfc(x). - const positive = z.cmp(.LT, .scalar(-88.72283905206835, .f32)).select(.scalar(0, .f32), tail); - const large = x.cmp(.LT, .scalar(0, .f32)).select(positive.negate().addConstant(2), positive); - return abs_x.cmp(.LT, one).select(small, large); + return self.mul(sigmoid_tensor).add(one_minus_sigmoid_tensor.mul(w.dot(self, tag).add(b))); } /// Returns a Tensor containing the Gaussian Error Linear Units (GeLU) activation function applied to each element of the input Tensor. @@ -2004,28 +1354,30 @@ pub const Tensor = struct { const rk = self.rank(); const a = self.axis(axis_); - var window_dimensions = constants.ones_i64; + const ones = [_]i64{1} ** constants.MAX_RANK; + var window_dimensions = ones; window_dimensions[a] = self.dim(a); - var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); + var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; padding[a] = .{ self.dim(a) - 1, 0 }; const result = ops.reduceWindow( - 1, .{self}, - .{.scalar(0, self.dtype())}, + .{Tensor.scalar(0, self.dtype())}, .{ - .base_dilations = constants.ones_i64[0..rk], - .window_dilations = constants.ones_i64[0..rk], - .window_strides = constants.ones_i64[0..rk], + .base_dilations = ones[0..rk], + .window_dilations = ones[0..rk], + .window_strides = ones[0..rk], .window_dimensions = window_dimensions[0..rk], .padding = padding[0..rk], }, struct { - fn add(values: ops.ReduceArgs) [1]Tensor { - return .{.add(values.left, values.right)}; + fn add(values: ops.ReduceArgs) struct { Tensor } { + return .{values.left.add(values.right)}; } }.add, + .{}, ); + return result[0]; } @@ -2045,20 +1397,13 @@ pub const Tensor = struct { const x: Tensor = .init(.{ 2, 5 }, .f32); - var exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Local._cumsum, - .{x}, - platform, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cumsum, .{x}, platform); defer exe.deinit(); - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), @ptrCast(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); + var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 1, 0, 1 }, .{ 3, 1, 0, 2, 1 } })); defer x_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cumsum, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual([2][5]f32{ .{ 0, 1, 2, 2, 3 }, .{ 3, 4, 4, 6, 7 } }, try res.getValue([2][5]f32, std.testing.io)); @@ -2090,19 +1435,19 @@ pub const Tensor = struct { return self.reshape(res_shape); } - const op = stablehlo.transpose( + const op = dialects.stablehlo.transpose( mlirCtx(), self.value(), - mlirx.Type.rankedTensor(mlirCtx(), res_shape), + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), .{ .permutation = toI64(permutation).constSlice() }, - currentLoc(), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(res_shape, op.result(0)); } pub fn swapAxes(self: Tensor, a: anytype, b: anytype) Tensor { if (self.axis(a) == self.axis(b)) return self; - var perm: Shape.AxesArray = .empty; + var perm: Shape.AxesArray = .{}; for (0..self.rank()) |i| { perm.appendAssumeCapacity(@intCast(i)); } @@ -2122,11 +1467,11 @@ pub const Tensor = struct { const new_dim = std.math.divExact(i64, self.dim(a), n) catch std.debug.panic("unflatten expects chosen dimension to be divisible by 'n' but {} is not divisible by {}", .{ self.dim(a), n }); const new_shape = self._shape.set(a, n).insert(a + 1, .{ ._ = new_dim }); - const reshaped_val = stablehlo.reshape( + const reshaped_val = dialects.stablehlo.reshape( mlirCtx(), self.value(), - mlirx.Type.rankedTensor(mlirCtx(), new_shape), - currentLoc(), + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); } @@ -2138,11 +1483,11 @@ pub const Tensor = struct { pub fn splitAxis(self: Tensor, ax: anytype, split_shape: anytype) Tensor { const new_shape = self._shape.splitAxis(ax, split_shape); - const reshaped_val = stablehlo.reshape( + const reshaped_val = dialects.stablehlo.reshape( mlirCtx(), self.value(), - mlirx.Type.rankedTensor(mlirCtx(), new_shape), - currentLoc(), + mlir.rankedTensorType(new_shape.dims(), mlirx.Type.fromDType(mlirCtx(), new_shape.dtype())), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(new_shape, reshaped_val.result(0)); } @@ -2173,143 +1518,55 @@ pub const Tensor = struct { pub const Slice = struct { start: i64 = 0, - dyn_start: ?Tensor = null, - len: i64 = to_the_end, end: i64 = to_the_end, step: i32 = 1, singleton: bool = false, - /// Select the full slice - pub const full: Slice = .{ .start = 0, .end = to_the_end, .step = 1 }; + const full = .{ .start = 0, .end = to_the_end, .step = 1 }; - /// Select a single element and drop the corresponding axis (to keep the axis use .{ .start = off, .len =1 }) pub fn single(offset: i64) Slice { return .{ .start = offset, .end = offset + 1, .singleton = true }; } - /// Select a single element and drop the corresponding axis (to keep the axis use .{ .start = off, .len =1 }) - pub fn dyn(start: Tensor, len: i64) Slice { - stdx.debug.assert(start.rank() == 0, "Slice.dyn expects scalar input, got {f}", .{start}); - return .{ .dyn_start = start, .len = len }; - } - - pub fn dynSingle(start: Tensor) Slice { - stdx.debug.assert(start.rank() == 0, "Slice.dynSingle expects scalar input, got {f}", .{start}); - return .{ .dyn_start = start, .len = 1, .singleton = true }; - } - pub fn absolute(s: Slice, d: i64) Slice { const start = if (s.start < 0) d + s.start else s.start; - const end = if (s.singleton) - start + 1 - else if (s.end == to_the_end) - d - else if (s.end < 0) - d + s.end - else if (s.len != to_the_end) - start + s.len - else - s.end; + const end = if (s.end == to_the_end) d else if (s.end < 0) d + s.end else s.end; const res: Slice = .{ .start = start, .end = end, .step = s.step, .singleton = s.singleton }; - stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f})", .{ s, d, res }); - stdx.debug.assert(end <= d, "Slice {f} is invalid for axis of dimension {d} (resolved to {f})", .{ s, d, res }); + stdx.debug.assert(start < end, "Slice {f} is invalid for axis of dimension {d} (resolved to {f}", .{ s, d, res }); return res; } const to_the_end = std.math.maxInt(i64); - pub fn format(s: Slice, writer: *std.Io.Writer) !void { - const end = if (s.len != to_the_end) s.start +| s.len else s.end; - if (s.dyn_start) |_| { - if (s.singleton) - try writer.writeAll("[x]") - else - try writer.print("[x..x+{}]", .{s.len}); - } else if (s.singleton) { - try writer.print("[{d}]", .{s.start}); - } else if (end == to_the_end and s.step == 1) { - try writer.print("[{d}..]", .{s.start}); - } else if (s.step == 1) { - try writer.print("[{d}..{d}]", .{ s.start, end }); + pub fn format(self: Slice, writer: *std.Io.Writer) !void { + if (self.singleton) { + try writer.print("[{d}]", .{self.start}); + } else if (self.end == to_the_end and self.step == 1) { + try writer.print("[{d}..]", .{self.start}); + } else if (self.step == 1) { + try writer.print("[{d}..{d}]", .{ self.start, self.end }); } else { - try writer.print("[{d}..{d}:{d}]", .{ s.start, end, s.step }); + try writer.print("[{d}..{d}:{d}]", .{ self.start, self.end, self.step }); } } }; - /// Slice a Tensor across a specific axes. - /// - /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. - /// Examples: - /// ``` - /// Tensor(.{.a=20,.b=30,.c=40 }).slice(.a, .{ .start = 5, .len = 11}); - /// Tensor(.{.a=20,.b=30,.c=40 }).slice(0, .single(5)); - /// ``` - pub fn slice(self: Tensor, axis_: anytype, slice_: Slice) Tensor { - return self.slices(@as([]const ResolvedAxis, &.{self.axis(axis_)}), &.{slice_}); + /// Slices the input Tensor over the given axis using the given parameters. + pub fn slice1d(self: Tensor, axis_: anytype, s: Slice) Tensor { + var slices = [_]Slice{.{}} ** constants.MAX_RANK; + slices[self.axis(axis_)] = s; + return self.slice(slices[0..self.rank()]); } - /// Slices a Tensor across many axes. - /// - /// ``` - /// Tensor(.{.a=20,.b=30,.c=40 }).slices(.{ .a, .b }, &.{ .start = 5, .end = 11}, .dyn(x) }); - /// Tensor(.{.a=20,.b=30,.c=40 }).slices(.{ - /// .a = .{ .start = a_off, .len = 11 }, - /// .b = .{ .start = b_off, .len = 12 }, - /// }); - /// Tensor(.{ 20,30,40}).slices(.{ 0, 2 }, .{.dyn(a_off, .len = 20), .{ .start = 8, .len = 12 }}); - /// ``` - pub fn slices(self: Tensor, axes_: anytype, slices_: []const Slice) Tensor { - // Note: with typed slices, the callsites are less akwards. - if (@TypeOf(axes_) != []const ResolvedAxis) { - const resolved_axes: []const ResolvedAxis = self._shape.parseAxes(axes_)[0].slice(); - return self.slices(resolved_axes, slices_); - } - - const rk = self.rank(); - any_dyn: { - for (slices_) |s| { - if (s.dyn_start) |_| break :any_dyn; - } - return self.staticSlices(axes_, slices_); - } - - const zero = scalar(0, .i64).value(); - var start_indices: [constants.MAX_RANK]*const mlir.Value = @splat(zero); - var new_shape: Shape = self._shape; - var to_remove: Shape.AxesArray = .empty; - - for (axes_, slices_) |a, s| { - stdx.debug.assert(s.step == 1, "Tensor.slices doesn't support mixing strided and dynamic slices, got {f}", .{stdx.fmt.slice(slices_)}); - new_shape._dims.buffer[a] = s.len; - start_indices[a] = if (s.dyn_start) |dyn| - dyn.convert(.i64).value() - else - scalar(s.start, .i64).value(); - if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); - } - - const op = stablehlo.dynamic_slice( - mlirCtx(), - self.value(), - new_shape.dims(), - start_indices[0..rk], - currentLoc(), - ).appendTo(currentBlock()); - - const res = _result(new_shape, op.result(0)); - return res.reshape(new_shape.removeMany(to_remove.constSlice())); - } - - fn staticSlices(self: Tensor, axes_: []const ResolvedAxis, slices_: []const Slice) Tensor { - var start_indices: [constants.MAX_RANK]i64 = @splat(0); - var strides: [constants.MAX_RANK]i64 = @splat(1); - var limit_indices: [constants.MAX_RANK]i64 = self._shape._dims.buffer; + /// Slices the input Tensor using the given parameters. + pub fn slice(self: Tensor, slices: []const Slice) Tensor { + var start_indices: [constants.MAX_RANK]i64 = undefined; + var strides: [constants.MAX_RANK]i64 = undefined; + var limit_indices: [constants.MAX_RANK]i64 = undefined; var res_shape: Shape = self._shape; - for (axes_, slices_) |a, s| { - stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} on axis {}", .{ s.step, a }); - stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} on axis {}", .{ s.step, a }); + for (slices, 0..) |s, a| { + stdx.debug.assert(s.step > 0, "slice expects 'step' to be positive, got {} at index {}", .{ s.step, a }); const args: Slice = s.absolute(self.dim(a)); start_indices[a] = args.start; @@ -2318,21 +1575,21 @@ pub const Tensor = struct { res_shape = res_shape.setDim(a, std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable); } - const ctx = Compiler.current(); - const slice_op = stablehlo.slice( - ctx.mlir_ctx, + const result_type = mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())); + const slice_op = dialects.stablehlo.slice( + mlirCtx(), self.value(), start_indices[0..self.rank()], limit_indices[0..self.rank()], strides[0..self.rank()], - mlirx.Type.rankedTensor(ctx.mlir_ctx, res_shape), - ctx.location, + result_type, + .unknown(mlirCtx()), ).appendTo(currentBlock()); var res = _result(res_shape, slice_op.result(0)); - var to_remove: Shape.AxesArray = .empty; - for (axes_, slices_) |a, s| { - if (s.singleton) to_remove.appendAssumeCapacity(a); + var to_remove: Shape.AxesArray = .{}; + for (slices, 0..) |s, a| { + if (s.singleton) to_remove.appendAssumeCapacity(@intCast(a)); } return res.reshape(res_shape.removeMany(to_remove.constSlice())); } @@ -2346,11 +1603,10 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); defer x_buffer.deinit(); - // Wrap slice to hide the anytype in the signature. - // TODO(wzk): stdx.meta.specifyFn (waiting 0.17 for SoA meta programming) + // Wrap slice1d to hide the anytype in the signature. const Local = struct { - fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { - return input.slice(ax, slice_); + pub fn _slice1dAxis(input: Tensor, ax: i8, slice_: Tensor.Slice) Tensor { + return input.slice1d(ax, slice_); } }; @@ -2360,50 +1616,10 @@ pub const Tensor = struct { .{ -1, Tensor.Slice{ .start = -2 }, [4]f32{ 3, 4, 8, 9 } }, }) |testcase| { const ax, const slice_, const expectation = testcase; - var exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Local._slice1dAxis, - .{ x, ax, slice_ }, - platform, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._slice1dAxis, .{ x, ax, slice_ }, platform); defer exe.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{} }); - defer res.deinit(); - - try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); - } - } - - test "dynamic slices" { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const Local = struct { - fn dynSliceResolved(x: Tensor, ax: ResolvedAxis, dyn_slice: Slice) Tensor { - return x.slice(ax, dyn_slice); - } - }; - - inline for (.{ - .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, - .{ [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }, Shape.init(.{ 2, 5 }, .f32), [4]f32{ 3, 4, 8, 9 }, 3, 1 }, - }) |testcase| { - const x_data, const x_shape, const expectation, const z_value: i32, const ax = testcase; - const x: Tensor = .fromShape(x_shape); - const z: Tensor = .init(.{}, .i32); - - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.dynSliceResolved, .{ x, ax, .dyn(z, 2) }, platform, .{}); - defer exe.deinit(); - - var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); - defer x_buffer.deinit(); - var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); - defer z_buffer.deinit(); - - var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .dyn_start = z_buffer } }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._slice1dAxis, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); @@ -2414,11 +1630,25 @@ pub const Tensor = struct { return if (idx < 0) self.dim(axis_) + idx else idx; } + pub fn choose1d(self: Tensor, axis_: anytype, i: i64) Tensor { + return self.slice1d(axis_, .single(i)); + } + + pub fn choose(self: Tensor, offsets: anytype) Tensor { + const off, const tags = Shape.parseDimensions(offsets); + var slices = [_]Slice{.{}} ** constants.MAX_RANK; + for (off.constSlice(), tags.constSlice()) |o, t| { + const ax = self.axis(t); + slices[ax] = .single(o); + } + return self.slice(slices[0..self.rank()]); + } + /// Concatenates the input Tensors along the given axis. pub fn concatenate(tensors: []const Tensor, axis_: anytype) Tensor { if (tensors.len == 1) return tensors[0]; - const ctx = Compiler.current(); - var buffer = ctx.alloc(*const mlir.Value, tensors.len); + stdx.debug.assert(tensors.len <= 32, "concatenate only supports up to 32 tensors, got {}", .{tensors.len}); + var buffer: [32]*const mlir.Value = undefined; std.debug.assert(tensors.len <= buffer.len); std.debug.assert(tensors.len > 0); const a = tensors[0].axis(axis_); @@ -2431,7 +1661,7 @@ pub const Tensor = struct { } const res_shape = tensors[0]._shape.set(a, concatenated_dim); - const op = stablehlo.concatenate(mlirCtx(), buffer, a, currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.concatenate(mlirCtx(), buffer[0..tensors.len], a, .unknown(mlirCtx())).appendTo(currentBlock()); // log.debug("concatenate({}, {}, {d}) -> {d}", .{ tensors[0], tensors[1], a, res_shape }); return _result(res_shape, op.result(0)); } @@ -2442,6 +1672,9 @@ pub const Tensor = struct { /// - Tensor.stack(&.{x, y, z}, 1, .layers) -> .{ .a, .layers, .b, .c } /// - Tensor.stack(&.{x, y, z}, .last, .layers) -> .{ .a, .b, .c, .layers } pub fn stack(tensors: []const Tensor, axis_: anytype, tag: anytype) Tensor { + // Note: we could ask the compilation context for some memory instead of stack allocating + stdx.debug.assert(tensors.len <= 32, "stack only supports up to 32 tensors, got {}", .{tensors.len}); + const shape0 = tensors[0]._shape; const res_shape = shape0.insertTag(axis_, 1, tag); @@ -2449,8 +1682,7 @@ pub const Tensor = struct { stdx.debug.assert(shape0.eqlWithTags(tensor._shape), "stack expects tensor shapes to match, got {f} and {f}", .{ shape0, tensor._shape }); } - const ctx = Compiler.current(); - var reshaped = ctx.alloc(Tensor, tensors.len); + var reshaped: [32]Tensor = undefined; for (tensors, 0..) |tensor, i| { reshaped[i] = tensor.reshape(res_shape); } @@ -2462,7 +1694,7 @@ pub const Tensor = struct { else shape0.axis(axis_); - return Tensor.concatenate(reshaped, ax); + return Tensor.concatenate(reshaped[0..tensors.len], ax); } /// Repeats a Tensor several times along the given axis. @@ -2514,21 +1746,13 @@ pub const Tensor = struct { const input_data, const shape_, const expectation, const ax, const reps = testcase; const input: Tensor = .fromShape(shape_); - var exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Local.repeat1d, - .{ input, ax, reps }, - platform, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.repeat1d, .{ input, ax, reps }, platform); defer exe.deinit(); var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); - var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); - defer output.deinit(); + const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.repeat1d, .{input_buffer}); try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); } } @@ -2537,18 +1761,17 @@ pub const Tensor = struct { /// /// * stutter1d([0, 1, 2, 3], -1, 2) = [0, 0, 1, 1, 2, 2, 3, 3] /// This is equivalent to repeat(ax+1) unless ax is the last axis. - pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u32) Tensor { - stdx.debug.assert(n_rep > 0, "stutter1d expects a positive repeat count, got {}", .{n_rep}); + pub fn stutter1d(self: Tensor, axis_: i8, n_rep: u63) Tensor { const a = self.axis(axis_); const broadshape = self._shape.insert(a + 1, .{n_rep}); - const res_shape = self._shape.setDim(a, self.dim(a) * @as(i64, n_rep)); + const res_shape = self._shape.setDim(a, self.dim(a) * n_rep); const stutter_dims = Shape.range(self.rank() + 1, self.dtype()).remove(a + 1); return self.broadcast(broadshape, stutter_dims.dims()).reshape(res_shape); } /// Repeats in line each value along the given axes. - pub fn stutter(self: Tensor, n_reps: []const u32) Tensor { + pub fn stutter(self: Tensor, n_reps: []const u63) Tensor { // TODO: this should support the tagged syntax: x.repeat(.{ .a = 3, .b = 2}); stdx.debug.assert(n_reps.len == self.rank(), "stutter expects tensor rank and 'n_reps' length to be equal, got {} and {}", .{ self.rank(), n_reps.len }); @@ -2578,76 +1801,68 @@ pub const Tensor = struct { const input_data, const shape_, const expectation, const ax, const reps = testcase; const input: Tensor = .fromShape(shape_); - var exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Local.stutter1d, - .{ input, ax, reps }, - platform, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local.stutter1d, .{ input, ax, reps }, platform); defer exe.deinit(); var input_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, input.shape(), std.mem.sliceAsBytes(&input_data)); defer input_buffer.deinit(); - var output = try exe.eval(std.testing.allocator, std.testing.io, .{input_buffer}); - defer output.deinit(); + const output = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local.stutter1d, .{input_buffer}); try std.testing.expectEqual(expectation, try output.getValue(@TypeOf(expectation), std.testing.io)); } } /// Returns a Tensor containing the element-wise negation of the input Tensor. pub fn negate(self: Tensor) Tensor { - const negate_op = stablehlo.negate(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const negate_op = dialects.stablehlo.negate(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, negate_op.result(0)); } /// Returns a Tensor containing the element-wise cosine of the input Tensor. pub fn cos(self: Tensor) Tensor { - const cosine_op = stablehlo.cosine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const cosine_op = dialects.stablehlo.cosine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, cosine_op.result(0)); } /// Returns a Tensor containing the element-wise sine of the input Tensor. pub fn sin(self: Tensor) Tensor { - const sine_op = stablehlo.sine(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const sine_op = dialects.stablehlo.sine(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, sine_op.result(0)); } /// Returns a Tensor containing the element-wise exponential operation of the input Tensor. pub fn exp(self: Tensor) Tensor { - const op = stablehlo.exponential(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.exponential(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise logarithm operation of the input Tensor. pub fn log(self: Tensor) Tensor { - const op = stablehlo.log(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.log(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } /// Returns a Tensor containing the element-wise square-root of the input Tensor. pub fn sqrt(self: Tensor) Tensor { - const sqrt_op = stablehlo.sqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const sqrt_op = dialects.stablehlo.sqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, sqrt_op.result(0)); } /// Returns a Tensor containing the element-wise reverse square-root of the input Tensor. pub fn rsqrt(self: Tensor) Tensor { - const rsqrt_op = stablehlo.rsqrt(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const rsqrt_op = dialects.stablehlo.rsqrt(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, rsqrt_op.result(0)); } /// Returns a Tensor containing the element-wise hyperbolic tangent of the input Tensor. pub fn tanh(self: Tensor) Tensor { - const tanh_op = stablehlo.tanh(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const tanh_op = dialects.stablehlo.tanh(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, tanh_op.result(0)); } /// Returns a Tensor containing the element-wise exponential minus one operation of the input Tensor. pub fn exponentialMinusOne(self: Tensor) Tensor { - const expm1_op = stablehlo.exponential_minus_one(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const expm1_op = dialects.stablehlo.exponential_minus_one(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, expm1_op.result(0)); } @@ -2656,7 +1871,7 @@ pub const Tensor = struct { end: i64, step: i64 = 1, }; - + /// /// Returns a Tensor containing evenly spaced values within a given interval. pub fn arange(args: ArangeArgs, dt: DataType) Tensor { stdx.debug.assert(args.start <= args.end, "arange expects 'args.start' to be less than 'args.end', got {} and {}", .{ args.start, args.end }); @@ -2664,11 +1879,11 @@ pub const Tensor = struct { const n_steps = std.math.divCeil(i64, args.end - args.start, args.step) catch unreachable; const sh = Shape.init(.{n_steps}, dt); - var op = stablehlo.iota( + var op = dialects.stablehlo.iota( mlirCtx(), 0, - mlirx.Type.rankedTensor(mlirCtx(), sh), - currentLoc(), + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), ).appendTo(currentBlock()); var res = _result(sh, op.result(0)); @@ -2683,31 +1898,6 @@ pub const Tensor = struct { return res; } - test arange { - const zml = @import("zml.zig"); - const platform = zml.testing.env(); - - const exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Tensor.arange, - .{ .{ .start = 2, .end = 26, .step = 4 }, .u32 }, - platform, - .{}, - ); - defer exe.deinit(); - - var output = try exe.eval(std.testing.allocator, std.testing.io, {}); - defer output.deinit(); - - try zml.testing.expectClose( - std.testing.io, - zml.Slice.initConst(.init(.{6}, .u32), @ptrCast(&[_]u32{ 2, 6, 10, 14, 18, 22 })), - output, - .exact_match, - ); - } - /// Returns a Tensor containing values in increasing order starting from 0 along the given axis. /// /// The output dtype will be `.i32`, unless the given axis has a too big dimension, in that case we use `.i64`. @@ -2718,13 +1908,13 @@ pub const Tensor = struct { const dt: DataType = if (sh.dim(a) <= std.math.maxInt(i32)) .i32 else .i64; const res_shape = sh.withDtype(dt); - var op = stablehlo.iota( + var op = dialects.stablehlo.iota( mlirCtx(), a, - mlirx.Type.rankedTensor(mlirCtx(), res_shape), - currentLoc(), + mlir.rankedTensorType(res_shape.dims(), mlirx.Type.fromDType(mlirCtx(), res_shape.dtype())), + .unknown(mlirCtx()), ).appendTo(currentBlock()); - return _resultPropagateSharding(res_shape, op.result(0)); + return _result(res_shape, op.result(0)); } pub const LinspaceArgs = struct { @@ -2740,11 +1930,11 @@ pub const Tensor = struct { stdx.debug.assert(dt.isFloat(), "linspace expects type to be a float, got {} (hint: use arange instead)", .{dt}); const sh = Shape.init(.{args.steps}, dt); - var iota_op = stablehlo.iota( + var iota_op = dialects.stablehlo.iota( mlirCtx(), 0, - mlirx.Type.rankedTensor(mlirCtx(), sh), - currentLoc(), + mlir.rankedTensorType(sh.dims(), mlirx.Type.fromDType(mlirCtx(), sh.dtype())), + .unknown(mlirCtx()), ).appendTo(currentBlock()); var res = _result(sh, iota_op.result(0)); @@ -2782,48 +1972,32 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{}, platform); defer exe.deinit(); } /// Returns a constant Tensor with the given value. pub fn constant(val: DataType.Value) Tensor { - const op = stablehlo.constant( + const op = dialects.stablehlo.constant( mlirCtx(), &.{}, mlirx.Type.fromDType(mlirCtx(), val.dtype()), val.asBytes(), - currentLoc(), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(.init(&.{}, val.dtype()), op.result(0)); } pub fn zeroes(sh: Shape) Tensor { - const result = Tensor.constant(sh.dtype().zero()).broad(sh); - return _resultPropagateSharding(sh, result.value()); - } - - pub fn uninitialized(sh: Shape) Tensor { - const ctx = Compiler.current(); - const buffer_type = mlir.Type.memRef( - mlirx.Type.fromDType(ctx.mlir_ctx, sh.dtype()), - sh.dims(), - null, - null, - ); - const buffer = stablehlo.createBuffer(ctx.mlir_ctx, buffer_type, ctx.location) - .appendTo(currentBlock()); - const tensor = stablehlo.unpin(ctx.mlir_ctx, buffer.result(0), ctx.location) - .appendTo(currentBlock()); - return _resultPropagateSharding(sh, tensor.result(0)); + return Tensor.constant(sh.dtype().zero()).broad(sh); } /// Embeds a buffer with concrete values into an Mlir program. pub fn constantTensor(sh: Shape, bytes_: []const u8) Tensor { const elem_type = mlirx.Type.fromDType(mlirCtx(), sh.dtype()); //const elem_type = mlirx.denseElementAttrType(val.dtype()) orelse std.debug.panic("constantTensor expects a dtype that can be serialized to MLIR, like f32 or i32, got {f}", .{val.shape()}); - const constant_op = stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, currentLoc()).appendTo(currentBlock()); - return _resultPropagateSharding(sh, constant_op.result(0)); + const constant_op = dialects.stablehlo.constant(mlirCtx(), sh.dims(), elem_type, bytes_, .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(sh, constant_op.result(0)); } /// Returns a Tensor containing the result of the outer product between the input Tensors. @@ -2857,8 +2031,11 @@ pub const Tensor = struct { // but we propagate output_shape tags. return _result(res_shape, self.value()); } - const result_type = mlirx.Type.rankedTensor(mlirCtx(), res_shape); - const broadcast_op = stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, currentLoc()).appendTo(currentBlock()); + const result_type = mlir.rankedTensorType( + res_shape.dims(), + mlirx.Type.fromDType(mlirCtx(), res_shape.dtype()), + ); + const broadcast_op = dialects.stablehlo.broadcast_in_dim(mlirCtx(), self.value(), axes_, result_type, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(res_shape, broadcast_op.result(0)); } @@ -2894,8 +2071,6 @@ pub const Tensor = struct { // Already the right shape if (std.mem.eql(i64, self.dims(), other.dims())) return self; - if (ops.LoweringCompatibility.preserveIntegerScalarBroadcast(self, other)) |rewritten| return rewritten; - // Non ambiguous broadcasting // TODO: broad is error prone because of this: // it will happily broadcast .{ .a = 10, .b = 1 } to .{ .b = 10, .a = 5 } @@ -2904,31 +2079,18 @@ pub const Tensor = struct { } // check that each axis of self maps to an axis of other - var axes_: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var axes_: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; for (self._shape.tags()) |t| { axes_.appendAssumeCapacity(@intCast(other.axis(t))); } return self.broadcast(other, axes_.constSlice()); } - pub fn optimizationBarrier(self: Tensor) Tensor { - const ctx = Compiler.current(); - - const op = stablehlo.optimizationBarrier( - ctx.mlir_ctx, - &.{self.value()}, - &.{self.value().type_()}, - ctx.location, - ).appendTo(ctx.currentScope().block); - - return _result(self.shape(), op.result(0)); - } - /// Reshapes the input Tensor with the given shape. pub fn reshape(self: Tensor, output_shape_: anytype) Tensor { const output_shape = self._shape.reshape(output_shape_); - const tensor_type = mlirx.Type.rankedTensor(mlirCtx(), output_shape); - const reshape_value = stablehlo.reshape(mlirCtx(), self.value(), tensor_type, currentLoc()).appendTo(currentBlock()); + const tensor_type = mlir.rankedTensorType(output_shape.dims(), mlirx.Type.fromDType(mlirCtx(), output_shape.dtype())); + const reshape_value = dialects.stablehlo.reshape(mlirCtx(), self.value(), tensor_type, .unknown(mlirCtx())).appendTo(currentBlock()); return _result(output_shape, reshape_value.result(0)); } @@ -2949,7 +2111,7 @@ pub const Tensor = struct { pub fn pad(self: Tensor, padding_value: anytype, paddings: anytype) Tensor { const _paddings = self.shape().parseAxesOptions(Pad, paddings, .{}); - const ZEROS: [constants.MAX_RANK]i64 = @splat(0); + const ZEROS = [_]i64{0} ** constants.MAX_RANK; var low = ZEROS; var high = ZEROS; var interior = ZEROS; @@ -2966,12 +2128,12 @@ pub const Tensor = struct { } const rk = self.rank(); - const pad_op = stablehlo.pad( + const pad_op = dialects.stablehlo.pad( mlirCtx(), self.value(), Tensor.scalar(padding_value, self.dtype()).value(), .{ .low = low[0..rk], .high = high[0..rk], .interior = interior[0..rk] }, - currentLoc(), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(res_shape, pad_op.result(0)); @@ -2987,9 +2149,9 @@ pub const Tensor = struct { self.axis(axis_); var res_shape = self._shape; - const ones: [constants.MAX_RANK]i64 = @splat(1); + const ones = [_]i64{1} ** constants.MAX_RANK; res_shape._dims.insertSlice(ax, ones[0..tags_.len]) catch unreachable; - res_shape._tags.insertSlice(ax, tags_) catch unreachable; + res_shape._tags.insertSlice(ax, tags_.constSlice()) catch unreachable; return self.reshape(res_shape); } @@ -3016,7 +2178,7 @@ pub const Tensor = struct { pub fn reverse(self: Tensor, axes_: anytype) Tensor { const actual_axes = self._shape.axes(axes_); - const reverse_op = stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), currentLoc()).appendTo(currentBlock()); + const reverse_op = dialects.stablehlo.reverse(mlirCtx(), self.value(), toI64(actual_axes.constSlice()).constSlice(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, reverse_op.result(0)); } @@ -3053,7 +2215,7 @@ pub const Tensor = struct { /// so that gather can pub fn gather(self: Tensor, _indices: anytype, opts: GatherOpts) Tensor { const idx_per_axis, const idx_tags = Shape.parseStruct(Tensor, _indices); - var idx_axes: Shape.AxesArray = .empty; + var idx_axes: Shape.AxesArray = .{}; for (idx_tags.slice()) |t| { idx_axes.appendAssumeCapacity(self.axis(t)); } @@ -3076,14 +2238,14 @@ pub const Tensor = struct { { // Only test shapes - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - const scope = comp.pushBlock(block); - defer scope.pop(); + comp.pushBlock(block); + defer comp.popBlock(); inline for (.{ .{ .{ .a = 10 }, .{ .a = idx(.{}) }, .{} }, @@ -3167,10 +2329,10 @@ pub const Tensor = struct { // Compute result shape var res_shape = indices._shape.remove(index_coord_axis).withDtype(self.dtype()); var slice_dims = self._shape._dims; - var self_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; - var indices_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; - var start_index_map: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; - var self_offset_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var self_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var indices_batch_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var start_index_map: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; + var self_offset_axes: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; for (self._shape.tags(), 0..self.rank()) |t, self_ax| { const maybe_slice_ax: ?u3 = if (tagged_api) slice_shape.hasTag(t) else @intCast(self_ax); @@ -3197,7 +2359,7 @@ pub const Tensor = struct { } } - const gather_op = stablehlo.gather( + const gather_op = dialects.stablehlo.gather( mlirCtx(), self.value(), indices.value(), @@ -3211,7 +2373,7 @@ pub const Tensor = struct { .index_vector_dim = index_coord_axis, .indices_are_sorted = opts.indices_are_sorted, }, - currentLoc(), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(res_shape, gather_op.result(0)); } @@ -3228,14 +2390,14 @@ pub const Tensor = struct { { // Only test shapes - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); - const scope = comp.pushBlock(block); - defer scope.pop(); + comp.pushBlock(block); + defer comp.popBlock(); inline for (.{ .{ .{ .a = 10 }, .{}, .{ ._ = 0 }, .{ .a = 10 } }, @@ -3271,14 +2433,7 @@ pub const Tensor = struct { // Test with actual values. const operand: Tensor = .init(.{ .a = 2, .b = 4, .c = 6 }, .u16); const indices: Tensor = .init(.{ .n = 2, ._ = 2 }, .i32); - var exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Local._gatherSlices, - .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, - platform, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._gatherSlices, .{ operand, Shape.init(.{ .b = 2, .c = 3 }, .u16), indices, .{} }, platform); defer exe.deinit(); var indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, indices.shape(), std.mem.sliceAsBytes(&[2][2]i32{ .{ 2, 1 }, .{ 0, 3 } })); @@ -3289,14 +2444,14 @@ pub const Tensor = struct { return Tensor.arange(.{ .end = 2 * 4 * 6 }, .u16).reshape(.{ .a = 2, .b = 4, .c = 6 }); } }.forward; - var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform, .{}); + var temp_exe = try zml.module.compile(std.testing.allocator, std.testing.io, temp_func, .{}, platform); defer temp_exe.deinit(); - const buffer = try temp_exe.eval(std.testing.allocator, std.testing.io, {}); + const buffer = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &temp_exe, temp_func, {}); break :b buffer; }; defer operand_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{ operand_buffer, indices_buffer }); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._gatherSlices, .{ operand_buffer, indices_buffer }); defer result.deinit(); const expected: zml.Slice = .init(Shape.init(.{ 2, 2, 2, 3 }, .u16), std.mem.sliceAsBytes(&[2][2][2][3]u16{ @@ -3309,7 +2464,7 @@ pub const Tensor = struct { .{ .{ 27, 28, 29 }, .{ 33, 34, 35 } }, }, })); - try zml.testing.expectClose(std.testing.io, expected, result, .{}); + try zml.testing.expectClose(std.testing.io, expected, result, 0); } pub const ScatterOpts = struct { @@ -3440,7 +2595,7 @@ pub const Tensor = struct { pub fn _scatterCB(self: Tensor, coords: Tensor, updates: Tensor) Tensor { return self.scatterSlices( - .{ .c = coords.slice(.coord, .single(0)), .b = coords.slice(.coord, .single(1)) }, + .{ .c = coords.choose1d(.coord, 0), .b = coords.choose1d(.coord, 1) }, updates, .{ .update_fn = ScatterOpts.increment }, ); @@ -3453,15 +2608,15 @@ pub const Tensor = struct { { // Only test shapes - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); defer block.deinit(); - const scope = comp.pushBlock(block); - defer scope.pop(); + comp.pushBlock(block); + defer comp.popBlock(); const idx = Local._idx; @@ -3496,16 +2651,17 @@ pub const Tensor = struct { const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); const updates = Tensor.init(.{ 2, 3 }, .i32).withTags(.{ .n, .b }); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); defer exe.deinit(); var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); defer a_buffer.deinit(); - var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), @ptrCast(&[2]i32{ 0, 2 })); + var scatter_indices_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, scatter_indices.shape(), std.mem.sliceAsBytes(&[2]i32{ 0, 2 })); defer scatter_indices_buffer.deinit(); var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2][3]i32{ .{ 10, 20, 30 }, .{ 70, 80, 90 } })); defer updates_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); const expected = [3][3]i32{ .{ 10, 21, 32 }, .{ 3, 4, 5 }, .{ 76, 87, 98 } }; @@ -3519,7 +2675,7 @@ pub const Tensor = struct { const scatter_indices = Tensor.init(.{2}, .i32).withTags(.{.n}); const updates = Tensor.init(.{2}, .i32).withTags(.{.n}); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatter, .{ a, &.{scatter_indices}, updates }, platform); defer exe.deinit(); var a_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, a.shape(), std.mem.sliceAsBytes(&[9]i32{ 0, 1, 2, 3, 4, 5, 6, 7, 8 })); @@ -3528,7 +2684,8 @@ pub const Tensor = struct { defer scatter_indices_buffer.deinit(); var updates_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, updates.shape(), std.mem.sliceAsBytes(&[2]i32{ 20, 70 })); defer updates_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); + + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatter, .{ a_buffer, &.{scatter_indices_buffer}, updates_buffer }); defer result.deinit(); const expected = [9]i32{ 0, 1, 22, 3, 4, 5, 6, 77, 8 }; @@ -3541,7 +2698,7 @@ pub const Tensor = struct { const start_indices: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .coord = 2 }, .i32); const values: Tensor = .init(.{ .n = 2, .a = 2, .m = 3, .c = 2, .d = 2 }, .u16); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._scatterCB, .{ operand, start_indices, values }, platform); defer exe.deinit(); var operand_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, operand.shape(), std.mem.sliceAsBytes(&@as([2 * 3 * 4 * 2]u16, @splat(0)))); @@ -3559,7 +2716,8 @@ pub const Tensor = struct { defer start_indices_buffer.deinit(); var values_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, values.shape(), std.mem.sliceAsBytes(&@as([2 * 2 * 3 * 2 * 2]u16, @splat(1)))); defer values_buffer.deinit(); - var result = try exe.eval(std.testing.allocator, std.testing.io, .{ operand_buffer, start_indices_buffer, values_buffer }); + + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._scatterCB, .{ operand_buffer, start_indices_buffer, values_buffer }); defer result.deinit(); const expected = [2][3][4][2]u16{ @@ -3615,18 +2773,18 @@ pub const Tensor = struct { values: Tensor, indices: Tensor, - fn cmp(values: ops.ReduceArgs, indices: ops.ReduceArgs) [2]Tensor { + fn cmp(values: ops.ReduceArgs, indices: ops.ReduceArgs) struct { Tensor, Tensor } { const left_gt_right = values.left.cmp(.GT, values.right); const is_nan = values.left.cmp(.NE, values.left); const left_gt_or_nan = left_gt_right.logical(.OR, is_nan); + // we are bubbling up Nan. + const max_val = left_gt_or_nan.select(values.left, values.right); // If values.left == values.right: keep the smallest idx. const is_same = values.left.cmp(.EQ, values.right); const is_first = indices.left.cmp(.LT, indices.right); const is_same_but_first = is_same.logical(.AND, is_first); const keep_left_idx = left_gt_or_nan.logical(.OR, is_same_but_first); - // we are bubbling up Nan. - const max_val = left_gt_or_nan.select(values.left, values.right); const max_idx = keep_left_idx.select(indices.left, indices.right); return .{ max_val, max_idx }; @@ -3658,17 +2816,14 @@ pub const Tensor = struct { }; const x: Tensor = .init(.{ 1, 5 }, .f32); - var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform, .{}); + var exe = try zml.module.compile(allocator, std.testing.io, ArgMaxTest._fwd, .{x}, platform); defer exe.deinit(); { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, 4.1, 7.9, 0, 7.9 }})); defer x_buffer.deinit(); - var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - runner.run(std.testing.io, .{x_buffer}, .{&res}, .{ .wait = true }); + var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); defer res.values.deinit(); defer res.indices.deinit(); const max_ = res.values.getValue(f32, std.testing.io); @@ -3682,10 +2837,7 @@ pub const Tensor = struct { var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[1][5]f32{.{ 5.0, std.math.nan(f32), 7.9, 0, 7.9 }})); defer x_buffer.deinit(); - var res: zml.Bufferized(Tensor.ArgMaxRes) = undefined; - var runner = try exe.runner(allocator); - defer runner.deinit(allocator); - runner.run(std.testing.io, .{x_buffer}, .{&res}, .{ .wait = true }); + var res = try zml.testing.autoCall(allocator, std.testing.io, &exe, ArgMaxTest._fwd, .{x_buffer}); defer res.values.deinit(); defer res.indices.deinit(); const max_ = try res.values.getValue(f32, std.testing.io); @@ -3703,9 +2855,9 @@ pub const Tensor = struct { pub fn sort(self: Tensor, axis_: anytype, opts: SortOpts) SortRes { const a = self.axis(axis_); const indices = Tensor.arange(.{ .end = self.dim(a) }, .i32).broadcast(self._shape, &.{a}); - const direction: stablehlo.ComparisonDirectionAttr.Direction = if (opts.descending) .GT else .LT; + const direction: dialects.stablehlo.ComparisonDirection.Direction = if (opts.descending) .GT else .LT; const res = ops.sort(.{ self, indices }, a, struct { - fn call(values: ops.SortArgs, _: ops.SortArgs, direction_: stablehlo.ComparisonDirectionAttr.Direction) Tensor { + fn call(values: ops.SortArgs, _: ops.SortArgs, direction_: dialects.stablehlo.ComparisonDirection.Direction) Tensor { return values.left.cmp(direction_, values.right); } }.call, .{direction}, true); @@ -3733,13 +2885,13 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 2, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{} }, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f32{ -0.9264, 0.7156, 1.0202, 0.3992, 1.2349, 1.0003, -0.1932, 1.3935, 0.7316, 0.0851 })); defer x_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); defer res.deinit(); const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -3752,7 +2904,7 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 1, 5, 10 }, .f16); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 1, .{ .descending = true } }, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]f16{ @@ -3764,7 +2916,7 @@ pub const Tensor = struct { })); defer x_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); defer res.deinit(); const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -3783,7 +2935,7 @@ pub const Tensor = struct { { const x: Tensor = .init(.{ 4, 2, 1, 4 }, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._argsort, .{ x, 3, .{} }, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ @@ -3798,7 +2950,7 @@ pub const Tensor = struct { })); defer x_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._argsort, .{x_buffer}); defer res.deinit(); const res_cpu = try res.toSliceAlloc(std.testing.allocator, std.testing.io); @@ -3826,58 +2978,14 @@ pub const Tensor = struct { const has_name: ?[:0]const u8, const a = switch (@typeInfo(@TypeOf(named_axis_))) { .int, .comptime_int => .{ null, self.axis(@as(i64, @intCast(named_axis_))) }, .@"struct" => |info| blk: { - stdx.debug.assertComptime(info.field_names.len == 1, err_msg, .{}); - break :blk .{ info.field_names[0], self.axis(@field(named_axis_, info.field_names[0])) }; + stdx.debug.assertComptime(info.fields.len == 1, err_msg, .{}); + break :blk .{ info.fields[0].name, self.axis(@field(named_axis_, info.fields[0].name)) }; }, else => stdx.debug.compileError(err_msg, .{}), }; - const ctx = Compiler.current(); - var result: SortRes = switch (ctx.platform.target) { - // Work around https://github.com/aws-neuron/aws-neuron-sdk/issues/1339 until Neuron's sort+slice rewrite uses the slice size as k. - .neuron => blk: { - stdx.debug.assert(self.dtype().isFloat(), "Neuron Tensor.topK only supports float tensors, got {}", .{self.dtype()}); - stdx.debug.assert(k <= 16384, "Neuron AwsNeuronTopK supports k <= 16384, got {}", .{k}); - - const last_axis: u4 = self.rank() - 1; - const topk_input = if (a == last_axis) self else self.swapAxes(a, last_axis); - const call_input = if (opts.descending) topk_input else topk_input.negate(); - const values_shape = topk_input.shape().setDim(last_axis, @intCast(k)); - const indices_shape = values_shape.withDtype(.i32); - const result_types = [_]*const mlir.Type{ - mlirx.Type.rankedTensor(ctx.mlir_ctx, values_shape), - mlirx.Type.rankedTensor(ctx.mlir_ctx, indices_shape), - }; - - var backend_config_buf: [32]u8 = undefined; - const backend_config = std.fmt.bufPrint(&backend_config_buf, "{d}", .{k}) catch unreachable; - const op = stablehlo.custom_call( - ctx.mlir_ctx, - &.{call_input.value()}, - &result_types, - .{ - .call_target_name = "AwsNeuronTopK", - .has_side_effect = false, - .backend_config = .{ .original = backend_config }, - }, - ctx.location, - ).appendTo(currentBlock()); - - var values = Tensor._result(values_shape, op.result(0)); - if (!opts.descending) values = values.negate(); - var indices = Tensor._result(indices_shape, op.result(1)); - if (a != last_axis) { - values = values.swapAxes(a, last_axis); - indices = indices.swapAxes(a, last_axis); - } - break :blk .{ .values = values, .indices = indices }; - }, - .cpu, .cuda, .rocm, .tpu, .oneapi, .metal, .furiosa => blk: { - var sorted = self.sort(a, .{ .descending = opts.descending }); - sorted.values = sorted.values.slice(a, .{ .end = k }); - sorted.indices = sorted.indices.slice(a, .{ .end = k }); - break :blk sorted; - }, - }; + var result = self.sort(a, .{ .descending = opts.descending }); + result.values = result.values.slice1d(a, .{ .end = k }); + result.indices = result.indices.slice1d(a, .{ .end = k }); if (has_name) |new_name| { result.values._shape._tags.set(a, new_name.ptr); result.indices._shape._tags.set(a, new_name.ptr); @@ -3906,7 +3014,7 @@ pub const Tensor = struct { // TODO: support maxPool on non last axis const a = self.axis(-1); - const ones: [constants.MAX_RANK]i64 = @splat(1); + const ones = [_]i64{1} ** constants.MAX_RANK; var window_dimensions = ones; window_dimensions[a] = opts.window_dimensions; var window_strides = window_dimensions; @@ -3917,13 +3025,12 @@ pub const Tensor = struct { var window_dilations = ones; window_dilations[a] = opts.window_dilations; - var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); + var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; padding[a] = opts.padding; - const values, const indices = ops.reduceWindow( - 2, + const result = ops.reduceWindow( .{ self, iota(self._shape, a) }, - .{ .constant(self.dtype().minValue()), .scalar(0, .i32) }, + .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, .{ .window_dimensions = window_dimensions[0..self.rank()], .window_strides = window_strides[0..self.rank()], @@ -3931,9 +3038,10 @@ pub const Tensor = struct { .window_dilations = window_dilations[0..self.rank()], .padding = padding[0..self.rank()], }, - &MaxPoolRes.cmp, + MaxPoolRes.cmp, + .{}, ); - return .{ .values = values, .indices = indices }; + return .{ .values = result[0], .indices = result[1] }; } /// Computes the 2d maxPool operation on the input Tensor. @@ -3956,14 +3064,13 @@ pub const Tensor = struct { const base_dilation = initPoolArg(self.rank(), &opts.base_dilations); const window_dilations = initPoolArg(self.rank(), &opts.window_dilations); - var padding: [constants.MAX_RANK][2]i64 = @splat(.{ 0, 0 }); + var padding = [_][2]i64{.{ 0, 0 }} ** constants.MAX_RANK; padding[a - 1] = opts.padding[0]; padding[a] = opts.padding[1]; - const values, const indices = ops.reduceWindow( - 2, + const result = ops.reduceWindow( .{ self, iota(self._shape, a) }, - .{ .constant(self.dtype().minValue()), .scalar(0, .i32) }, + .{ Tensor.constant(self.dtype().minValue()), Tensor.scalar(0, .i32) }, .{ .window_dimensions = window_dimensions[0..self.rank()], .window_strides = window_strides[0..self.rank()], @@ -3971,9 +3078,10 @@ pub const Tensor = struct { .window_dilations = window_dilations[0..self.rank()], .padding = padding[0..self.rank()], }, - &MaxPoolRes.cmp, + MaxPoolRes.cmp, + .{}, ); - return .{ .values = values, .indices = indices }; + return .{ .values = result[0], .indices = result[1] }; } /// Chunk a given tensor into exactly n parts of equal shape. @@ -3985,7 +3093,7 @@ pub const Tensor = struct { var chunks: [n_chunks]Tensor = undefined; for (0..n_chunks) |i| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; - chunks[i] = self.slice(a, .{ .start = start, .end = start + chunk_size }); + chunks[i] = self.slice1d(a, .{ .start = start, .end = start + chunk_size }); } return chunks; } @@ -3995,15 +3103,15 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); defer block.deinit(); - const scope = comp.pushBlock(block); - defer scope.pop(); + comp.pushBlock(block); + defer comp.popBlock(); inline for (.{ .{ .{ .a = 12 }, .a, 3, .{ .a = 4 } }, @@ -4026,21 +3134,29 @@ pub const Tensor = struct { pub fn chunkAllowTrailing( self: Tensor, axis_: i64, - n_chunks: usize, - ) []Tensor { + n_chunks: comptime_int, + ) ![]Tensor { const a = self.axis(axis_); - const d: i64 = self.dim(a); - const chunk_size: i64 = @divFloor(d, @as(i64, @intCast(n_chunks))); + const d = self.dim(a); + const chunk_size: i64 = @divFloor(d, n_chunks); const tail_chunk_size: i64 = @rem(d, chunk_size); - const len: usize = if (tail_chunk_size == 0) n_chunks else n_chunks + 1; - const chunks = Compiler.current().alloc(Tensor, len); + const allocator = CompilationContext.current().arena.allocator(); - for (0.., chunks) |i, *chunk| { + var chunks = std.ArrayList(Tensor).initCapacity(allocator, n_chunks + 1) catch @panic("OOM"); + defer chunks.deinit(allocator); + + for (0..n_chunks) |i| { const start: i64 = @as(i64, @intCast(i)) * chunk_size; - chunk.* = self.slice(a, .{ .start = start, .end = @min(start + chunk_size, d) }); + chunks.appendAssumeCapacity( + self.slice1d(a, .{ .start = start, .end = start + chunk_size }), + ); } - return chunks; + if (tail_chunk_size != 0) { + const start: i64 = n_chunks * chunk_size; + chunks.appendAssumeCapacity(self.slice1d(a, .{ .start = start })); + } + return chunks.toOwnedSlice(allocator); } test chunkAllowTrailing { @@ -4048,15 +3164,15 @@ pub const Tensor = struct { const platform = zml.testing.env(); // Only test shapes - var comp: zml.Compiler = .init(std.testing.allocator, std.testing.io, platform, .{}); + var comp = zml.module.CompilationContext.init(std.testing.allocator, platform); defer comp.deinit(); comp.activate(); defer comp.deactivate(); const block = mlir.Block.init(&.{}, &.{}); defer block.deinit(); - const scope = comp.pushBlock(block); - defer scope.pop(); + comp.pushBlock(block); + defer comp.popBlock(); inline for (.{ .{ .{ .a = 10 }, .a, 3, .{ .a = 3 }, .{ .a = 1 } }, @@ -4065,8 +3181,8 @@ pub const Tensor = struct { .{ .{ 12, 2 }, 0, 3, .{ 4, 2 }, .{} }, }) |testcase| { const x_shape, const ax, const n_chunks, const res, const trailing = testcase; - const x = Tensor.zeroes(.init(x_shape, .f16)); - const chunks = x.chunkAllowTrailing(x.axis(ax), n_chunks); + const x = Tensor.constant(.{ .f16 = 0 }).broad(Shape.init(x_shape, .f16)); + const chunks = try x.chunkAllowTrailing(x.axis(ax), n_chunks); const res_shape = Shape.init(res, .f16); for (chunks[0..n_chunks]) |chk| { @@ -4091,51 +3207,119 @@ pub const Tensor = struct { for (split_sizes) |n| split_sum += n; stdx.debug.assert(split_sum == d, "split expects sum of 'split_sizes' values and axis dimension to be equal, got {} and {}", .{ split_sum, d }); - const res = Compiler.current().alloc(Tensor, split_sizes.len); + var arena = std.heap.ArenaAllocator.init(CompilationContext.current().allocator); + defer arena.deinit(); + + const res = arena.allocator().alloc(Tensor, split_sizes.len) catch @panic("OOM"); var start: i64 = 0; for (split_sizes, 0..) |n, i| { - res[i] = self.slice(a, .{ .start = start, .end = start + n }); + res[i] = self.slice1d(a, .{ .start = start, .end = start + n }); start += n; } return res; } - test split { + /// Slices the input Tensor along a specific axis, with a start offset known at runtime. + /// Note: this doesn't support tagging, if you have tags, + /// you should use `dynamicSlice` directly. + pub fn dynamicSlice1d(self: Tensor, axis_: i8, slice_: DynSlice) Tensor { + stdx.debug.assert(slice_.start.rank() == 0, "dynamicSlice1d expects 'slice_.start' tensor rank to be a scalar, got {f}", .{slice_.start}); + + const a = self.axis(axis_); + const new_shape = self._shape.set(a, slice_.len); + + var start_indices = [_]*const mlir.Value{constant(slice_.start.dtype().zero()).value()} ** constants.MAX_RANK; + start_indices[a] = slice_.start.value(); + + const op = dialects.stablehlo.dynamic_slice( + mlirCtx(), + self.value(), + new_shape.dims(), + start_indices[0..self.rank()], + .unknown(mlirCtx()), + ).appendTo(currentBlock()); + + return _result(new_shape, op.result(0)); + } + + pub const DynSlice = struct { start: Tensor, len: i64 }; + + /// Slices a Tensor across many axes, with runtime known offsets. + /// + /// Due to the nature of stablehlo, the length of the slices need to be known when compiling the IR. + /// When using the tagged API it is allowed to not specify some axes. + /// But with the non-tagged API all slices need to be specified. + /// Examples: + /// ``` + /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ .a = .{ .start = a_off, .len = 11}); + /// Tensor(.{.a=20,.b=30,.c=40 }).dynamicSlice(.{ + /// .a = .{ .start = a_off, .len = 11 }, + /// .b = .{ .start = b_off, .len = 12 }, + /// }); + /// Tensor(.{ 20,30,40}).dynamicSlice(.{.{ .start = scalar(0, .i32), .len = 20 }, .{ .start = b_off, .len = 12 }, .{ .start = scalar(0, .i32), .len = 40 }}); + /// ``` + pub fn dynamicSlice(self: Tensor, slices_: anytype) Tensor { + // TODO: the untagged api is a bit verbose. Should I allow: `Tensor(.{ 20,30,40}).dynamicSlice(.{.{}, .{ .start = b_off, .len = 12 }, .{}});` ?? + // + const slices, const slices_tags = Shape.parseStruct(DynSlice, slices_); + + // TODO use slices and slices_tags for the format. + // Currently this prints: "dynSlice(struct{q: struct{start: tensor.Tensor, comptime len: comptime_int = 1}}{ .q = struct{start: tensor.Tensor, comptime len: comptime_int = 1}{ .start = Tensor({1,10}, dtype=.i64), .len = 1 } })" + // which is kinda ugly. + + const idx_dtype = if (slices.len > 0) slices.get(0).start.dtype() else .i32; + const zero = Tensor.scalar(0, idx_dtype).value(); + var offset_values = [_]*const mlir.Value{zero} ** constants.MAX_RANK; + var res_shape = self._shape; + for (slices.constSlice(), 0..) |slice_, i| { + const offset = slice_.start; + const len = slice_.len; + if (slices_tags.len == 0) { + stdx.debug.assert(self.rank() == slices.len, "dynamicSlice expects tensor rank and 'slices_' length to be equal, got {d} and {d}", .{ self.rank(), slices.len }); + + offset_values[i] = offset.value(); + res_shape._dims.set(i, len); + + stdx.debug.assert(len <= self.dim(i), "dynamicSlice expects slices 'len' to be less than or equal to their corresponding dimension in input tensor, got {d} and {d} for index {d}", .{ len, self.dim(i), i }); + } else { + const t = slices_tags.get(i); + const a = res_shape.hasTag(t) orelse stdx.debug.panic("dynamicSlice expects input tensor to have tags used in 'slices_' but {s} is missing (input shape is {f})", .{ t, self._shape }); + + stdx.debug.assert(len <= self.dim(a), "dynamicSlice expects slices 'len' to be less than their corresponding dimension in input tensor, got {d} and {d} for axis {s}", .{ len, self.dim(a), t }); + + offset_values[a] = offset.value(); + res_shape._dims.set(a, len); + } + } + const op = dialects.stablehlo.dynamic_slice(mlirCtx(), self.value(), res_shape.dims(), offset_values[0..self.rank()], .unknown(mlirCtx())).appendTo(currentBlock()); + return _result(res_shape, op.result(0)); + } + + test dynamicSlice { const zml = @import("zml.zig"); const platform = zml.testing.env(); - const Local = struct { - pub fn _split(x: Tensor, axis_: i8, split_sizes: []const i64) []Tensor { - return x.split(axis_, split_sizes); - } - }; - inline for (.{ - .{ [3]f32{ 0, 1, 2 }, Shape.init(.{3}, .f32), 0, [3]i64{ 1, 1, 1 }, &.{ [1]f32{0}, [1]f32{1}, [1]f32{2} } }, - .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), 0, [2]i64{ 1, 9 }, &.{ [1]f32{0}, [9]f32{ 1, 2, 3, 4, 5, 6, 7, 8, 9 } } }, - .{ [4][3]f32{ .{ 0, 1, 2 }, .{ 3, 4, 5 }, .{ 6, 7, 8 }, .{ 9, 10, 11 } }, Shape.init(.{ 4, 3 }, .f32), 0, [2]i64{ 1, 3 }, &.{ [1][3]f32{.{ 0, 1, 2 }}, [3][3]f32{ .{ 3, 4, 5 }, .{ 6, 7, 8 }, .{ 9, 10, 11 } } } }, - .{ [2][6]f32{ .{ 0, 1, 2, 3, 4, 5 }, .{ 6, 7, 8, 9, 10, 11 } }, Shape.init(.{ 2, 6 }, .f32), 1, [2]i64{ 2, 4 }, &.{ [2][2]f32{ .{ 0, 1 }, .{ 6, 7 } }, [2][4]f32{ .{ 2, 3, 4, 5 }, .{ 8, 9, 10, 11 } } } }, + .{ [10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, Shape.init(.{10}, .f32), [2]f32{ 4, 5 }, 4, 0 }, + .{ [2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } }, Shape.init(.{ 2, 5 }, .f32), [4]f32{ 3, 4, 8, 9 }, 3, 1 }, }) |testcase| { - const x_data, const x_shape, const ax, const args, const expectation = testcase; + const x_data, const x_shape, const expectation, const z_value: i32, const ax = testcase; const x: Tensor = .fromShape(x_shape); + const z: Tensor = .init(.{}, .i32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._split, .{ x, ax, &args }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Tensor.dynamicSlice1d, .{ x, ax, .{ .len = 2, .start = z } }, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&x_data)); defer x_buffer.deinit(); + var z_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, z.shape(), std.mem.asBytes(&z_value)); + defer z_buffer.deinit(); - const res = try std.testing.allocator.alloc(zml.Buffer, expectation.len); - defer std.testing.allocator.free(res); - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{x_buffer}, res, .{ .wait = true }); - defer for (res) |*buffer| buffer.deinit(); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Tensor.dynamicSlice1d, .{ x_buffer, .{ .start = z_buffer } }); + defer res.deinit(); - inline for (expectation, 0..) |expected, i| { - try std.testing.expectEqual(expected, try res[i].getValue(@TypeOf(expected), std.testing.io)); - } + try std.testing.expectEqual(expectation, try res.getValue(@TypeOf(expectation), std.testing.io)); } } @@ -4143,7 +3327,7 @@ pub const Tensor = struct { /// Note this is the untagged api, if you have tags, you should use dynamicUpdateSlice directly. pub fn dynamicUpdateSlice1d(self: Tensor, update: Tensor, axis_: i64, offset: Tensor) Tensor { const placeholder = Tensor.scalar(0, .i32); - var start_indices: [constants.MAX_RANK]Tensor = @splat(placeholder); + var start_indices = [_]Tensor{placeholder} ** constants.MAX_RANK; start_indices[self.axis(axis_)] = offset; return self.dynamicUpdateSlice(start_indices[0..self.rank()], update); } @@ -4218,19 +3402,19 @@ pub const Tensor = struct { } else { // If an axis isn't specified, update the full slice. // This is only allowed when using tagged sliced. - offset_values = @splat(zero); + offset_values = .{zero} ** constants.MAX_RANK; for (offset.constSlice(), offset_tags.constSlice()) |start, t| { const a = self._shape.hasTag(t) orelse stdx.debug.panic("dynamicUpdateSlice expects input tensor to have tags used in 'offset_' but {s} is missing (input shape is {f})", .{ t, self._shape }); offset_values[a] = start.value(); } } - const op = stablehlo.dynamic_update_slice( + const op = dialects.stablehlo.dynamic_update_slice( mlirCtx(), self.value(), update.value(), offset_values[0..self.rank()], - currentLoc(), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -4248,7 +3432,7 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = ids }, y }, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[10]f32{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 })); @@ -4258,7 +3442,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{4})); defer ids_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = ids_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqual([10]f32{ 0, 1, 2, 3, -1, -1, 6, 7, 8, 9 }, try res.getValue([10]f32, std.testing.io)); @@ -4274,7 +3458,7 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(.{ .b = idx_ }, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); @@ -4284,7 +3468,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -4303,7 +3487,7 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(.{ zml.Tensor.scalar(0, .i32), idx_ }, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, ids, y }, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); @@ -4313,7 +3497,7 @@ pub const Tensor = struct { var ids_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, ids.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer ids_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, ids_buffer, y_buffer }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, ids_buffer, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -4333,7 +3517,7 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ .a = idx_a, .b = idx_b }, y }, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); @@ -4345,7 +3529,7 @@ pub const Tensor = struct { var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ .a = idx_a_buffer, .b = idx_b_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -4365,7 +3549,7 @@ pub const Tensor = struct { return x_.dynamicUpdateSlice(&idx_, y_); } }._fwd; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, forward, .{ x, .{ idx_a, idx_b }, y }, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][5]f32{ .{ 0, 1, 2, 3, 4 }, .{ 5, 6, 7, 8, 9 } })); @@ -4377,7 +3561,7 @@ pub const Tensor = struct { var idx_b_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, idx_b.shape(), std.mem.sliceAsBytes(&[1]i32{3})); defer idx_b_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, forward, .{ x_buffer, .{ idx_a_buffer, idx_b_buffer }, y_buffer }); defer res.deinit(); try std.testing.expectEqualDeep( @@ -4388,7 +3572,7 @@ pub const Tensor = struct { } /// Returns a Tensor containing the element-wise result of the given 'cmp' comparison between the two input Tensors. - pub fn cmp(self: Tensor, direction: stablehlo.ComparisonDirectionAttr.Direction, other: Tensor) Tensor { + pub fn cmp(self: Tensor, direction: dialects.stablehlo.ComparisonDirection.Direction, other: Tensor) Tensor { stdx.debug.assert(self.dtype() == other.dtype(), "cmp expects input tensors to be of the same type, got {t} and {t}", .{ self.dtype(), other.dtype() }); if (self.rank() == 0 and other.rank() != 0) return self.broadcast(other._shape, &.{}).cmp(direction, other); @@ -4396,13 +3580,13 @@ pub const Tensor = struct { stdx.debug.assert(self._shape.eql(other._shape), "cmp expects input tensor shapes to match, got {f} and {f}", .{ self._shape, other._shape }); - const op = stablehlo.compare( + const op = dialects.stablehlo.compare( mlirCtx(), self.value(), other.value(), - stablehlo.ComparisonDirectionAttr.get(mlirCtx(), .{ .value = direction }).value(), - getComparisonType(mlirCtx(), self.dtype()).value(), - currentLoc(), + dialects.stablehlo.ComparisonDirection.init(mlirCtx(), direction).getValue(), + getComparisonType(mlirCtx(), self.dtype()).getValue(), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(self._shape.withDtype(.bool), op.result(0)); @@ -4414,12 +3598,12 @@ pub const Tensor = struct { stdx.debug.assert(self.rank() < constants.MAX_RANK - 1, "toDiagonal expects input up to {d} rank, got {f}", .{ constants.MAX_RANK - 1, self }); const a = self.axis(axis_); const d = self.dim(a); - const p = self.shape()._sharding.partition.get(a); + const p = self.shape()._partitioning.get(a); var res_shape = self._shape; res_shape._dims.replaceRange(a, 1, &.{ d, d }) catch unreachable; res_shape._tags.replaceRange(a, 1, &.{ @tagName(new_tags[0]), @tagName(new_tags[1]) }) catch unreachable; - - res_shape._sharding.partition = res_shape._sharding.partition.insert(a, p); + // TODO(Corentin): Not sure about that + res_shape._partitioning.replaceRange(a, 1, &.{ p, p }) catch unreachable; const values = self.insertAxes(a + 1, .{new_tags[1]}).broad(res_shape); const zeros = Tensor.constant(self.dtype().zero()).broad(res_shape); @@ -4445,20 +3629,23 @@ pub const Tensor = struct { }; const x: Tensor = .init(.{ 2, 2 }, .u8); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform, .{}); + + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._toDiag, .{x}, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[2][2]u8{ .{ 1, 2 }, .{ 3, 4 } })); defer x_buffer.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._toDiag, .{x_buffer}); defer res.deinit(); - try std.testing.expectEqual( - [2][2][2]u8{ - .{ .{ 1, 0 }, .{ 0, 2 } }, - .{ .{ 3, 0 }, .{ 0, 4 } }, - }, + [2][2][2]u8{ .{ + .{ 1, 0 }, + .{ 0, 2 }, + }, .{ + .{ 3, 0 }, + .{ 0, 4 }, + } }, try res.getValue([2][2][2]u8, std.testing.io), ); } @@ -4504,10 +3691,10 @@ pub const Tensor = struct { defer x_buffer.deinit(); { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 0 }, platform); defer exe.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -4520,10 +3707,10 @@ pub const Tensor = struct { ); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, 1 }, platform); defer exe.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -4536,10 +3723,10 @@ pub const Tensor = struct { ); } { - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._tri, .{ x, -1 }, platform); defer exe.deinit(); - var res = try exe.eval(std.testing.allocator, std.testing.io, .{x_buffer}); + var res = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._tri, .{x_buffer}); defer res.deinit(); try std.testing.expectEqual( @@ -4569,24 +3756,20 @@ pub const Tensor = struct { stdx.debug.assert(bool_tensor._shape.eqlDims(on_true._shape), "select expects input tensor and 'on_true' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_true._shape }); stdx.debug.assert(bool_tensor._shape.eqlDims(on_false._shape), "select expects input tensor and 'on_false' tensor dimensions to match, got {f} and {f}", .{ bool_tensor._shape, on_false._shape }); - const op = stablehlo.select( + const op = dialects.stablehlo.select( mlirCtx(), bool_tensor.value(), on_true.value(), on_false.value(), - currentLoc(), + .unknown(mlirCtx()), ).appendTo(currentBlock()); return _result(on_true._shape, op.result(0)); } - pub fn mask(x: Tensor, pred: Tensor, mask_value: anytype) Tensor { - return pred.broad(x.shape()).select(x, .scalar(mask_value, x.dtype())); - } - /// Returns a Tensor containing the element-wise not logical operation of the input Tensor. pub fn not(self: Tensor) Tensor { - const op = stablehlo.not(mlirCtx(), self.value(), currentLoc()).appendTo(currentBlock()); + const op = dialects.stablehlo.not(mlirCtx(), self.value(), .unknown(mlirCtx())).appendTo(currentBlock()); return _result(self._shape, op.result(0)); } @@ -4675,7 +3858,7 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._cartesianProduct2, .{ x, y }, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); @@ -4683,11 +3866,7 @@ pub const Tensor = struct { var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); - var xs: zml.Buffer = undefined; - var ys: zml.Buffer = undefined; - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{ x_buffer, y_buffer }, .{ &xs, &ys }, .{ .wait = true }); + var xs, var ys = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._cartesianProduct2, .{ x_buffer, y_buffer }); defer xs.deinit(); defer ys.deinit(); @@ -4717,24 +3896,13 @@ pub const Tensor = struct { ); } - pub fn rollRight1d(self: Tensor, axis_: anytype, shift: i64) Tensor { - const a = self.axis(axis_); - const n = self.dim(a); - const k = @mod(shift, n); // normalize shift to [0, n) - if (k == 0) return self; - return Tensor.concatenate(&.{ - self.slice(a, .{ .start = n - k, .end = n }), - self.slice(a, .{ .start = 0, .end = n - k }), - }, a); - } - /// Given a set of N vectors of lengths A, B, C, D, /// returns 1 tensors of rank N+1, and shape (A, B, C, D, N). /// For any coordinate (a, b, c, d), we have: /// /// - res[a, b, c, d] == (A[a], B[b], C[c], D[d]) pub fn cartesianProductStacked(vectors: []const Tensor) Tensor { - var out: stdx.BoundedArray(Tensor, constants.MAX_RANK) = .{ .buffer = undefined, .len = vectors.len }; + var out = stdx.BoundedArray(Tensor, constants.MAX_RANK).init(vectors.len) catch unreachable; _cartesianProduct(vectors, out.slice()); return Tensor.stack(out.constSlice(), .last, .coord); @@ -4752,7 +3920,7 @@ pub const Tensor = struct { } }; - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{ x, y }, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3, 4, 5 })); @@ -4760,7 +3928,7 @@ pub const Tensor = struct { var y_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, y.shape(), std.mem.sliceAsBytes(&[_]i32{ 0, 1, 2, 3 })); defer y_buffer.deinit(); - var z = try exe.eval(std.testing.allocator, std.testing.io, .{ x_buffer, y_buffer }); + var z = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, Local._fwd, .{ x_buffer, y_buffer }); defer z.deinit(); try std.testing.expectEqualDeep( @@ -4814,40 +3982,31 @@ pub const Tensor = struct { /// Only for debug purpose, it inserts device to host synchronization /// so it will slow down the program execution. pub fn print(input: Tensor, name: []const u8) void { - const ctx = Compiler.current(); - const full_name = std.fmt.allocPrint(ctx.arena.allocator(), "{s}: {f}", .{ name, input.shape() }) catch @panic("OOM"); - defer ctx.arena.allocator().free(full_name); - switch (ctx.platform.target) { - .cpu, .cuda, .rocm, .tpu, .metal => { - ops.manualComputation((struct { - input: Tensor, - name: []const u8, - - fn body(body_ctx: @This(), _: void) void { - ops.customCall("zml$print", body_ctx.input, {}, .{ .name = body_ctx.name }, .{ .has_side_effect = true }); - } - }).body, .{ .input = input, .name = full_name }, {}, ctx.platform.replicated_mesh, .{ .in_specs = .{ .input = .replicated } }); - }, - .oneapi, .neuron, .furiosa => {}, - } + _ = ops.customCall("zml$print", .{input}, .{}, .{ .name = name }, .{ .has_side_effect = true }); } fn mlirCtx() *mlir.Context { - return Compiler.current().mlir_ctx; - } - - fn currentLoc() *const mlir.Location { - return Compiler.current().location; + return CompilationContext.current().mlir_ctx; } fn currentBlock() *mlir.Block { - return Compiler.current().currentScope().block; + return CompilationContext.current().currentScope().block; + } + + /// Returns the donation data of the tensor. + pub fn donation(self: Tensor) ?usize { + return CompilationContext.current().currentScope().id_to_donation.get(self.id); + } + + /// Returns the output memory kind of the tensor. + pub fn outputMemoryKind(self: Tensor) Memory.Kind { + return CompilationContext.current().currentScope().id_to_output_memory_kind.get(self.id) orelse .device; } }; fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { // TODO use shape - var result: [constants.MAX_RANK]i64 = @splat(1); + var result = [_]i64{1} ** constants.MAX_RANK; const start = rank - data.len; @memcpy(result[start .. start + data.len], data); return result; @@ -4855,7 +4014,7 @@ fn initPoolArg(rank: usize, data: []const i64) [constants.MAX_RANK]i64 { fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(constants.MAX_RANK, i64), padding: []const i64, window_dimensions: @Vector(constants.MAX_RANK, i64), window_dilations: @Vector(constants.MAX_RANK, i64), window_strides: @Vector(constants.MAX_RANK, i64)) Shape { // TODO use shape - var input_dims: [constants.MAX_RANK]i64 = @splat(1); + var input_dims = [_]i64{1} ** constants.MAX_RANK; @memcpy(input_dims[0..in_dims.len], in_dims); const input_dims_: @Vector(constants.MAX_RANK, i64) = input_dims; @@ -4877,13 +4036,13 @@ fn getPoolResDims(dt: DataType, in_dims: []const i64, base_dilations: @Vector(co return Shape.init(dims_arr[0..in_dims.len], dt); } -fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const stablehlo.CompareTypeAttr { - return .get(ctx, .{ .value = switch (dtype.class()) { +fn getComparisonType(ctx: *mlir.Context, dtype: DataType) *const dialects.stablehlo.CompareType { + return dialects.stablehlo.CompareType.init(ctx, switch (dtype.class()) { .bool => .UNSIGNED, .integer => if (dtype.isSignedInt()) .SIGNED else .UNSIGNED, .float => .FLOAT, .complex => @panic("Can't compare complex numbers"), - } }); + }); } test "Tensor.maxPool1d" { @@ -4904,16 +4063,13 @@ test "Tensor.maxPool1d" { const x: Tensor = .init(.{ 2, 2, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); - var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{x_buffer}, .{&result}, .{ .wait = true }); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); defer result.values.deinit(); defer result.indices.deinit(); @@ -4946,16 +4102,13 @@ test "Tensor.maxPool2d" { const x: Tensor = .init(.{ 2, 2, 5, 5 }, .f32); - var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform, .{}); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, MaxPool._fwd, .{x}, platform); defer exe.deinit(); var x_buffer: zml.Buffer = try .fromBytes(std.testing.io, platform, x.shape(), std.mem.sliceAsBytes(&data)); defer x_buffer.deinit(); - var result: zml.Bufferized(Tensor.ArgMaxRes) = undefined; - var runner = try exe.runner(std.testing.allocator); - defer runner.deinit(std.testing.allocator); - runner.run(std.testing.io, .{x_buffer}, .{&result}, .{ .wait = true }); + var result = try zml.testing.autoCall(std.testing.allocator, std.testing.io, &exe, MaxPool._fwd, .{x_buffer}); defer result.values.deinit(); defer result.indices.deinit(); @@ -4991,13 +4144,13 @@ fn _parseGatherCoord(self: Tensor, axes_: anytype) struct { bool, stdx.BoundedAr } fn toI64(values: anytype) stdx.BoundedArray(i64, constants.MAX_RANK) { - var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .empty; + var res: stdx.BoundedArray(i64, constants.MAX_RANK) = .{}; for (values) |val| res.appendAssumeCapacity(@intCast(val)); return res; } fn transposeIsJustAReshape(x: Shape, permutation: []const i64) bool { - var perm: stdx.BoundedArray(struct { u8, bool }, constants.MAX_RANK) = .empty; + var perm: stdx.BoundedArray(struct { u8, bool }, constants.MAX_RANK) = .{}; // Don't rewrite on invalid inputs. if (permutation.len > x.rank()) return false; for (permutation) |ax| { @@ -5049,13 +4202,6 @@ test "unused tensor" { } }; - var exe = try zml.module.compile( - std.testing.allocator, - std.testing.io, - Local._fwd, - .{Tensor.init(.{10}, .f32)}, - platform, - .{}, - ); + var exe = try zml.module.compile(std.testing.allocator, std.testing.io, Local._fwd, .{Tensor.init(.{10}, .f32)}, platform); defer exe.deinit(); } diff --git a/zml/test_runner.zig b/zml/test_runner.zig new file mode 100644 index 00000000..121b9be4 --- /dev/null +++ b/zml/test_runner.zig @@ -0,0 +1,112 @@ +//! Test runner for unit test based on https://github.com/ziglang/zig/blob/master/lib/compiler/test_runner.zig with async +const std = @import("std"); +const testing = std.testing; +const builtin = @import("builtin"); + +// note: std_options.log_level does not respect testing.log_level +// ref: https://github.com/ziglang/zig/issues/5738 +const log_level: std.log.Level = .warn; + +pub const std_options: std.Options = .{ + .log_level = log_level, +}; + +var log_err_count: usize = 0; +var fba_buffer: [8192]u8 = undefined; +var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); + +pub fn main() !void { + testing.log_level = log_level; + + const test_fn_list: []const std.builtin.TestFn = builtin.test_functions; + var ok_count: usize = 0; + var skip_count: usize = 0; + var fail_count: usize = 0; + const root_node = std.Progress.start(std.testing.io, .{ + .root_name = "Test", + .estimated_total_items = test_fn_list.len, + }); + + const have_tty = try std.Io.File.stderr().isTty(std.testing.io); + + var args = std.process.args(); + // Skip executable path + _ = args.next().?; + + const identifier_query = if (args.next()) |arg| blk: { + std.debug.print("Only tests with identifiers that includes `{s}` will be run\n", .{arg}); + break :blk arg; + } else blk: { + break :blk ""; + }; + + var leaks: usize = 0; + + for (test_fn_list, 0..) |test_fn, i| { + if (std.mem.indexOf(u8, test_fn.name, identifier_query) == null) { + continue; + } + + testing.allocator_instance = .{}; + defer { + if (testing.allocator_instance.deinit() == .leak) { + leaks += 1; + } + } + + testing.io_instance = .init(testing.allocator_instance.allocator(), .{}); + defer testing.io_instance.deinit(); + + const test_node = root_node.start(test_fn.name, 0); + if (!have_tty) { + std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); + } + if (test_fn.func()) |_| { + ok_count += 1; + test_node.end(); + if (!have_tty) std.debug.print("OK\n", .{}); + } else |err| switch (err) { + error.SkipZigTest => { + skip_count += 1; + if (have_tty) { + std.debug.print("{d}/{d} {s}...SKIP\n", .{ i + 1, test_fn_list.len, test_fn.name }); + } else { + std.debug.print("SKIP\n", .{}); + } + test_node.end(); + }, + else => { + fail_count += 1; + if (have_tty) { + std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{ + i + 1, test_fn_list.len, test_fn.name, @errorName(err), + }); + } else { + std.debug.print("FAIL ({s})\n", .{@errorName(err)}); + } + if (@errorReturnTrace()) |trace| { + std.debug.dumpStackTrace(trace); + } + test_node.end(); + }, + } + } + root_node.end(); + if (ok_count == test_fn_list.len) { + std.debug.print("All {d} tests passed.\n", .{ok_count}); + } else { + std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count }); + } + if (log_err_count != 0) { + std.debug.print("{d} errors were logged.\n", .{log_err_count}); + } + if (leaks != 0) { + std.debug.print("{d} tests leaked memory.\n", .{leaks}); + } + if (leaks != 0 or log_err_count != 0 or fail_count != 0) { + std.process.exit(1); + } + + // Explicit exit 0 to exit from async main thread and main properly + std.process.exit(0); +} diff --git a/zml/testing.zig b/zml/testing.zig index 5e3d6333..8e33ed58 100644 --- a/zml/testing.zig +++ b/zml/testing.zig @@ -3,9 +3,8 @@ const builtin = @import("builtin"); const stdx = @import("stdx"); -const Platform = @import("platform.zig").Platform; -const Sharding = @import("Sharding.zig"); const Slice = @import("slice.zig").Slice; +const Platform = @import("platform.zig").Platform; const zml = @import("zml.zig"); const log = std.log.scoped(.@"zml/testing"); @@ -15,32 +14,14 @@ var _platform: ?*const Platform = null; pub fn env() *const Platform { if (!builtin.is_test) @compileError("Cannot use zml.testing.env outside of a test block"); if (_platform == null) { - const memory_fraction = 0.90; - const platform = Platform.auto( - std.heap.c_allocator, - std.testing.io, - .{ - .cuda = .{ .allocator = .{ .vmm = .{ .memory_fraction = memory_fraction } } }, - .rocm = .{ .allocator = .{ .vmm = .{ .memory_fraction = memory_fraction } } }, - .metal = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }, - .oneapi = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = memory_fraction } } }, - }, - ) catch @panic("Pjrt not available"); - - // Register meshes required by flash attention. - _ = platform.registerMesh("model", .mesh(.{ .model = .high_bandwidth })) catch unreachable; - _platform = platform; + _platform = Platform.auto(std.heap.c_allocator, std.testing.io, .{ + .cuda = .{ .allocator = .{ .bfc = .{ .preallocate = true, .memory_fraction = 0.85 } } }, + }) catch unreachable; } return _platform.?; } -pub fn tpMesh() *const Sharding.Mesh { - if (!builtin.is_test) @compileError("Cannot use zml.testing.tpMesh() outside of a test block"); - - return _platform.?.meshes.get("model").?; -} - /// In neural network we generally care about the relative precision, /// but on a given dimension, if the output is close to 0, then the precision /// don't matter as much. @@ -98,6 +79,7 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpt .f8e8m0, => |t| { const L = t.toZigType(); + const left_data = left.constItems(L); switch (right.dtype()) { inline .bf16, .f16, @@ -110,24 +92,22 @@ pub fn expectClose(io: std.Io, left_: anytype, right_: anytype, opts: CompareOpt .f8e5m2fnuz, => |rt| { const R = rt.toZigType(); - const report: CompareOpts.Report = try .compareSlices(true, allocator, L, R, left, right, opts); - if (!report.ok(opts)) { - try w.print("{f}\n", .{report}); + const right_data = right.constItems(R); + const compare_report = try compareSlices(allocator, L, R, left_data, right_data, opts); + const ok = !compare_report.nan_or_inf and compare_report.close_fraction >= opts.minimum_close_fraction; + if (ok) { + return; + } else { + try w.print("{f}\n", .{compare_report}); return error.TestUnexpectedResult; } - return; }, else => unreachable, } }, inline .bool, .u2, .u4, .u8, .u16, .u32, .u64, .i2, .i4, .i8, .i16, .i32, .i64 => |t| { const T = t.toZigType(); - const report: CompareOpts.Report = try .compareSlices(false, allocator, T, T, left, right, opts); - if (!report.ok(opts)) { - try w.print("{f}\n", .{report}); - return error.TestUnexpectedResult; - } - return; + return std.testing.expectEqualSlices(T, left.constItems(T), right.constItems(T)); }, .c64, .c128 => @panic("TODO: support comparison of complex"), } @@ -138,224 +118,113 @@ pub const CompareOpts = struct { relative_tolerance: f32 = 1e-2, epsilon_relative: f32 = 1e-6, minimum_close_fraction: f32 = 0.999, - - pub const exact_match: CompareOpts = .{ - .absolute_tolerance = 0, - .relative_tolerance = 0, - .epsilon_relative = 0, - .minimum_close_fraction = 1.00, - }; - - pub const Report = struct { - nan_or_inf: bool, - max_absolute_error: f32, - mean_absolute_error: f32, - rmse: f32, - - close_fraction: f32, - - p50_absolute_error: f32, - p90_absolute_error: f32, - p99_absolute_error: f32, - p999_absolute_error: f32, - - pub fn ok(report: Report, opts: CompareOpts) bool { - return !report.nan_or_inf and report.close_fraction >= opts.minimum_close_fraction; - } - - fn compareSlices(comptime float_cmp: bool, allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: Slice, right: Slice, opts: CompareOpts) !CompareOpts.Report { - var left_iter = (&left).contiguousItemsIterator(L); - var right_iter = (&right).contiguousItemsIterator(R); - var left_chunk: []const L = &.{}; - var right_chunk: []const R = &.{}; - var acc: Accumulator = if (comptime float_cmp) try .init(allocator, left.shape.count()) else .initNoQuantiles(left.shape.count()); - defer acc.deinit(allocator); - - while (true) { - if (left_chunk.len == 0) { - left_chunk = left_iter.next() orelse &.{}; - } - if (right_chunk.len == 0) { - right_chunk = right_iter.next() orelse &.{}; - } - - if (left_chunk.len == 0 or right_chunk.len == 0) { - std.debug.assert(left_chunk.len == right_chunk.len); - return acc.report(); - } - - const n = @min(left_chunk.len, right_chunk.len); - if (comptime float_cmp) { - acc.cmpFloatSlices(L, R, left_chunk[0..n], right_chunk[0..n], opts); - } else { - acc.cmpIntSlices(L, left_chunk[0..n], right_chunk[0..n]); - } - - left_chunk = left_chunk[n..]; - right_chunk = right_chunk[n..]; - } - } - - pub fn format( - self: Report, - writer: *std.Io.Writer, - ) std.Io.Writer.Error!void { - try writer.print( - \\ nan_or_inf: {} - \\ max_absolute_error: {d} - \\ mean_absolute_error: {d} - \\ rmse: {d} - \\ close_fraction: {d} - \\ p50_absolute_error: {d} - \\ p90_absolute_error: {d} - \\ p99_absolute_error: {d} - \\ p999_absolute_error: {d} - , .{ - self.nan_or_inf, - self.max_absolute_error, - self.mean_absolute_error, - self.rmse, - self.close_fraction, - self.p50_absolute_error, - self.p90_absolute_error, - self.p99_absolute_error, - self.p999_absolute_error, - }); - } - }; - - const Accumulator = struct { - absolute_errors: []f32, - relative_errors: []f32, - expected_total_count: usize, - processed_count: usize = 0, - nan_or_inf: bool = false, - max_absolute_error: f32 = 0, - sum_absolute_error: f64 = 0, - sum_squared_error: f64 = 0, - count_close: usize = 0, - - fn init(allocator: std.mem.Allocator, count: usize) !Accumulator { - const absolute_errors = try allocator.alloc(f32, count); - errdefer allocator.free(absolute_errors); - const relative_errors = try allocator.alloc(f32, count); - - @memset(absolute_errors, 0); - @memset(relative_errors, 0); - return .{ - .expected_total_count = count, - .absolute_errors = absolute_errors, - .relative_errors = relative_errors, - }; - } - - fn initNoQuantiles(count: usize) Accumulator { - return .{ - .expected_total_count = count, - .absolute_errors = &.{}, - .relative_errors = &.{}, - }; - } - - fn deinit(self: *Accumulator, allocator: std.mem.Allocator) void { - allocator.free(self.absolute_errors); - allocator.free(self.relative_errors); - } - - fn cmpFloatSlices(self: *Accumulator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) void { - std.debug.assert(left.len == right.len); - std.debug.assert(self.processed_count + left.len <= self.absolute_errors.len); - - const start = self.processed_count; - for (left, right, 0..) |l, r, i| { - const l_f32 = zml.floats.floatCast(f32, l); - const r_f32 = zml.floats.floatCast(f32, r); - if (!std.math.isFinite(l_f32) or !std.math.isFinite(r_f32)) { - self.nan_or_inf = true; - continue; - } - const absolute_error: f32 = @abs(l_f32 - r_f32); - self.max_absolute_error = @max(self.max_absolute_error, absolute_error); - self.sum_absolute_error += absolute_error; - self.sum_squared_error += absolute_error * absolute_error; - - const scale = @max(@abs(l_f32), @abs(r_f32)); - const tolerance = opts.absolute_tolerance + opts.relative_tolerance * scale; - if (absolute_error <= tolerance) { - self.count_close += 1; - } - - self.absolute_errors[start + i] = absolute_error; - const denom = @max(opts.epsilon_relative, scale); - self.relative_errors[start + i] = absolute_error / denom; - } - self.processed_count += left.len; - } - - fn cmpIntSlices(self: *Accumulator, comptime T: type, left: []const T, right: []const T) void { - std.debug.assert(left.len == right.len); - - for (left, right) |l, r| { - switch (@typeInfo(T)) { - .int, .float => self.count_close += if (l == r) 1 else 0, - .vector => |vec| self.count_close += @popCount(@as(@Int(.unsigned, vec.len), @bitCast(l == r))), - else => @panic("cmpIntSlices expect int or vector of int"), - } - } - self.processed_count += left.len; - } - - fn report(self: *Accumulator) CompareOpts.Report { - std.debug.assert(self.processed_count == self.expected_total_count); - if (self.processed_count == 0) { - return .{ - .nan_or_inf = false, - .max_absolute_error = 0, - .mean_absolute_error = std.math.nan(f32), - .rmse = std.math.nan(f32), - .close_fraction = std.math.nan(f32), - .p50_absolute_error = 0, - .p90_absolute_error = 0, - .p99_absolute_error = 0, - .p999_absolute_error = 0, - }; - } - - std.sort.heap(f32, self.absolute_errors, {}, std.sort.asc(f32)); - std.sort.heap(f32, self.relative_errors, {}, std.sort.asc(f32)); - - const q = struct { - fn q(values: []const f32, frac: f32) f32 { - if (values.len == 0) return 0; - const idx: usize = @intFromFloat(std.math.round(@as(f32, @floatFromInt(values.len - 1)) * frac)); - return values[idx]; - } - }.q; - - const mean_absolute_error = stdx.math.divFloat(f64, self.sum_absolute_error, self.processed_count); - const rmse = std.math.sqrt(stdx.math.divFloat(f64, self.sum_squared_error, self.processed_count)); - const close_fraction = stdx.math.divFloat(f64, self.count_close, self.processed_count); - - return .{ - .nan_or_inf = self.nan_or_inf, - .max_absolute_error = self.max_absolute_error, - .mean_absolute_error = @floatCast(mean_absolute_error), - .rmse = @floatCast(rmse), - .close_fraction = @floatCast(close_fraction), - .p50_absolute_error = q(self.absolute_errors, 0.5), - .p90_absolute_error = q(self.absolute_errors, 0.9), - .p99_absolute_error = q(self.absolute_errors, 0.99), - .p999_absolute_error = q(self.absolute_errors, 0.999), - }; - } - }; }; -pub fn compareFloats(allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) !CompareOpts.Report { - var acc: CompareOpts.Accumulator = try .init(allocator, left.len); - defer acc.deinit(allocator); - acc.cmpFloatSlices(true, L, R, left, right, opts); - return acc.report(); +pub const CompareReport = struct { + nan_or_inf: bool, + max_absolute_error: f32, + mean_absolute_error: f32, + rmse: f32, + + close_fraction: f32, + + p50_absolute_error: f32, + p90_absolute_error: f32, + p99_absolute_error: f32, + p999_absolute_error: f32, + + pub fn format( + self: @This(), + writer: *std.Io.Writer, + ) std.Io.Writer.Error!void { + try writer.print( + \\ nan_or_inf: {} + \\ max_absolute_error: {d} + \\ mean_absolute_error: {d} + \\ rmse: {d} + \\ close_fraction: {d} + \\ p50_absolute_error: {d} + \\ p90_absolute_error: {d} + \\ p99_absolute_error: {d} + \\ p999_absolute_error: {d} + , .{ + self.nan_or_inf, + self.max_absolute_error, + self.mean_absolute_error, + self.rmse, + self.close_fraction, + self.p50_absolute_error, + self.p90_absolute_error, + self.p99_absolute_error, + self.p999_absolute_error, + }); + } +}; + +pub fn compareSlices(allocator: std.mem.Allocator, comptime L: type, comptime R: type, left: []const L, right: []const R, opts: CompareOpts) !CompareReport { + var nan_or_inf: bool = false; + var max_absolute_error: f32 = 0; + var sum_absolute_error: f64 = 0; + var sum_squared_error: f64 = 0; + var count_close: usize = 0; + var absolute_errors = try allocator.alloc(f32, left.len); + defer allocator.free(absolute_errors); + var relative_errors = try allocator.alloc(f32, left.len); + defer allocator.free(relative_errors); + + for (left, right, 0..) |l, r, i| { + const l_f32 = zml.floats.floatCast(f32, l); + const r_f32 = zml.floats.floatCast(f32, r); + if (!std.math.isFinite(l_f32) or !std.math.isFinite(r_f32)) { + nan_or_inf = true; + continue; + } + const absolute_error = @abs(l_f32 - r_f32); + max_absolute_error = @max(max_absolute_error, absolute_error); + sum_absolute_error += @as(f64, @floatCast(absolute_error)); + sum_squared_error += @as(f64, @floatCast(absolute_error)) * @as(f64, @floatCast(absolute_error)); + + const scale = @max(@abs(l_f32), @abs(r_f32)); + const tolerance = opts.absolute_tolerance + opts.relative_tolerance * scale; + if (absolute_error <= tolerance) { + count_close += 1; + } + + absolute_errors[i] = absolute_error; + const denom = @max(opts.epsilon_relative, scale); + relative_errors[i] = absolute_error / denom; + } + + std.sort.heap(f32, absolute_errors, {}, std.sort.asc(f32)); + std.sort.heap(f32, relative_errors, {}, std.sort.asc(f32)); + + const q = struct { + pub fn q(values: []const f32, frac: f32) f32 { + if (values.len == 0) return 0; + const idx: usize = @intFromFloat(std.math.round(@as(f32, @floatFromInt(values.len - 1)) * frac)); + return values[idx]; + } + }.q; + + const mean_absolute_error: f32 = @floatCast(sum_absolute_error / @as(f64, @floatFromInt(left.len))); + const rmse: f32 = @floatCast(std.math.sqrt(sum_squared_error / @as(f64, @floatFromInt(left.len)))); + const close_fraction = @as(f32, @floatFromInt(count_close)) / @as(f32, @floatFromInt(left.len)); + + const report: CompareReport = .{ + .nan_or_inf = nan_or_inf, + .max_absolute_error = max_absolute_error, + .mean_absolute_error = mean_absolute_error, + .rmse = rmse, + .close_fraction = close_fraction, + .p50_absolute_error = q(absolute_errors, 0.5), + .p90_absolute_error = q(absolute_errors, 0.9), + .p99_absolute_error = q(absolute_errors, 0.99), + .p999_absolute_error = q(absolute_errors, 0.999), + }; + + // TODO: Move + //const ok = !report.nan_or_inf and report.close_fraction >= opts.minimum_close_fraction; + return report; } pub fn expectEqualShapes(expected: zml.Shape, actual: zml.Shape) error{TestExpectedEqual}!void { @@ -369,6 +238,25 @@ fn BufferizedWithArgs(comptime T: type) type { return zml.meta.MapType(zml.Tensor, zml.Buffer).map(T); } +/// Automatically calls the executable with the given arguments, taking care of arguments and results allocation. +/// This helper can be used in tests to make the code a bit less verbose. +/// It doesn't handle pointers in inputs/outputs structs. +pub fn autoCall(allocator: std.mem.Allocator, io: std.Io, exe: *const zml.exe.Exe, func: anytype, inputs: zml.Bufferized(stdx.meta.FnArgs(func))) !zml.Bufferized(stdx.meta.FnResult(func)) { + var args = try exe.args(allocator); + defer args.deinit(allocator); + + var results = try exe.results(allocator); + defer results.deinit(allocator); + + args.set(inputs); + exe.callOpts(io, args, &results, .{ .wait = true }); + + var output: zml.Bufferized(stdx.meta.FnResult(func)) = undefined; + results.fill(.{&output}); + + return output; +} + fn countWithPrefix(store: zml.io.TensorStore.View, prefix: []const u8) usize { var count: usize = 0; var it = store.store.registry.iterator(); @@ -401,14 +289,13 @@ pub fn testLayer( activation_store: zml.io.TensorStore.View, name: []const u8, layer_weights: zml.Bufferized(@TypeOf(layer)), - meshes: []const *const zml.Mesh, opts: CompareOpts, ) !void { var arena = std.heap.ArenaAllocator.init(allocator); defer arena.deinit(); const forward = @field(@TypeOf(layer), @tagName(func)); - const ArgsT = stdx.meta.Tail(std.meta.ArgsTuple(@TypeOf(forward))); + const ArgsT = stdx.meta.Tail(stdx.meta.FnArgs(forward)); var args: ArgsT = undefined; @@ -431,12 +318,12 @@ pub fn testLayer( var buffer: [256]u8 = undefined; const subkey = std.fmt.bufPrint(&buffer, "{d}", .{ctx_.index}) catch unreachable; - tensor.* = ctx_.activation_store.createReplicatedTensor(subkey, null); + tensor.* = ctx_.activation_store.createTensor(subkey); ctx_.index += 1; } }.cb, &ctx, &args); - const exe = try platform.compile(allocator, io, layer, func, args, .{ .meshes = if (meshes.len != 0) meshes else &.{} }); + const exe = try platform.compile(allocator, io, layer, func, args); defer exe.deinit(); const output_name = try std.fmt.allocPrint(arena.allocator(), "{s}.out", .{name}); @@ -453,7 +340,7 @@ pub fn testLayer( var exe_results = try exe.results(allocator); defer exe_results.deinit(allocator); - var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, activation_store.store, .auto); + var args_buffers = try zml.io.load(ArgsT, &args, allocator, io, platform, .{ .dma_chunks = 1, .dma_chunk_size = 4096, .store = activation_store.store, .parallelism = 1 }); defer zml.meta.visit(struct { fn cb(_: void, b: *zml.Buffer) void { b.deinit(); @@ -462,17 +349,17 @@ pub fn testLayer( exe_args.set(.{ layer_weights, args_buffers }); + exe.callOpts(io, exe_args, &exe_results, .{ .wait = true }); + var results = try allocator.alloc(zml.Buffer, output_count); defer allocator.free(results); - exe.call(io, exe_args, &exe_results, .{ .wait = true }); exe_results.fill(.{results}); - defer for (results) |*result| result.deinit(); var failed: bool = false; var reader_buffer: [4096]u8 = undefined; for (0..output_count) |i| { - const expected_slice = s: { + const expected_slice = b: { var buffer: [256]u8 = undefined; const subkey = try std.fmt.bufPrint(&buffer, "{d}", .{i}); var reader = try store_output.getReader(subkey, io, &reader_buffer); @@ -481,7 +368,7 @@ pub fn testLayer( const expected_slice: zml.Slice = try .alloc(allocator, shape); errdefer expected_slice.free(allocator); try reader.interface.readSliceAll(expected_slice.data()); - break :s expected_slice; + break :b expected_slice; }; defer expected_slice.free(allocator); diff --git a/zml/tokenizer/BUILD.bazel b/zml/tokenizer/BUILD.bazel index f2cbc132..691bd0ee 100644 --- a/zml/tokenizer/BUILD.bazel +++ b/zml/tokenizer/BUILD.bazel @@ -3,14 +3,14 @@ load("@zml//bazel:zig_srcs.bzl", "zig_srcs") zig_library( name = "tokenizer", + srcs = ["homemade.zig"], import_name = "zml/tokenizer", main = "tokenizer.zig", - tags = ["manual"], visibility = ["//visibility:public"], deps = [ "//ffi:zig", "//stdx", - "//zml/tokenizer/iree", + "//zml/tokenizer/hftokenizers", "//zml/tokenizer/sentencepiece", ], ) @@ -27,13 +27,15 @@ zig_binary( zig_test( name = "test", + main = "homemade.zig", + visibility = ["//visibility:public"], deps = [ - ":tokenizer", + "//ffi:zig", + "//stdx", ], ) -# todo: reenable after fix -# zig_srcs( -# name = "sources", -# zig_lib = ":tokenizer", -# ) +zig_srcs( + name = "sources", + zig_lib = ":tokenizer", +) diff --git a/zml/tokenizer/hftokenizers/BUILD.bazel b/zml/tokenizer/hftokenizers/BUILD.bazel new file mode 100644 index 00000000..a07d855b --- /dev/null +++ b/zml/tokenizer/hftokenizers/BUILD.bazel @@ -0,0 +1,32 @@ +load("@rules_cc//cc:cc_library.bzl", "cc_library") +load("@rules_rust//rust:defs.bzl", "rust_static_library") +load("@rules_zig//zig:defs.bzl", "zig_library") + +rust_static_library( + name = "hftokenizers_rs", + srcs = ["hftokenizers.rs"], + crate_name = "zml_tokenizer_hftokenizers", + edition = "2021", + deps = ["@crates//:tokenizers"], +) + +cc_library( + name = "hftokenizers_cc", + hdrs = ["hftokenizers.h"], + visibility = ["//visibility:public"], + deps = [ + ":hftokenizers_rs", + "//ffi:cc", + ], +) + +zig_library( + name = "hftokenizers", + main = "hftokenizers.zig", + visibility = ["//visibility:public"], + deps = [ + ":hftokenizers_cc", + "//ffi:zig", + "//stdx", + ], +) diff --git a/zml/tokenizer/hftokenizers/Cargo.lock b/zml/tokenizer/hftokenizers/Cargo.lock new file mode 100644 index 00000000..7164e77b --- /dev/null +++ b/zml/tokenizer/hftokenizers/Cargo.lock @@ -0,0 +1,713 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom", + "once_cell", + "serde", + "version_check", + "zerocopy 0.8.26", +] + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bitflags" +version = "2.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.2.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee0f8803222ba5a7e2777dd72ca451868909b1ac410621b676adf07280e9b5f" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "compact_str" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dary_heap" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" +dependencies = [ + "serde", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" +dependencies = [ + "cc", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "getrandom" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "wasi", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "macro_rules_attribute" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "monostate" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" +dependencies = [ + "monostate-impl", + "serde", +] + +[[package]] +name = "monostate-impl" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "onig" +version = "6.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy 0.7.35", +] + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rand" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +dependencies = [ + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "rustversion" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "serde" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.216" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.134" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "2.0.91" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53cbcb5a243bd33b7858b1d7f4aca2153490815872d86d955d6ea29f743c035" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokenizers" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b238e22d44a15349529690fb07bd645cf58149a1b1e44d6cb5bd1641ff1a6223" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.14.2+wasi-0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +dependencies = [ + "wit-bindgen-rt", +] + +[[package]] +name = "wit-bindgen-rt" +version = "0.39.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +dependencies = [ + "bitflags", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +dependencies = [ + "zerocopy-derive 0.8.26", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zml_tokenizer_hftokenizers" +version = "0.1.0" +dependencies = [ + "tokenizers", +] diff --git a/zml/tokenizer/hftokenizers/Cargo.toml b/zml/tokenizer/hftokenizers/Cargo.toml new file mode 100644 index 00000000..f905c004 --- /dev/null +++ b/zml/tokenizer/hftokenizers/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "zml_tokenizer_hftokenizers" +version = "0.1.0" +edition = "2021" + +[dependencies] +tokenizers = { version = "0.22.2", default-features = false, features = [ + # We don't need/want the progressbar feature + "onig", + "esaxx_fast", +] } + +[lib] +name = "zml_tokenizer_hftokenizers" +path = "hftokenizers.rs" diff --git a/zml/tokenizer/hftokenizers/hftokenizers.h b/zml/tokenizer/hftokenizers/hftokenizers.h new file mode 100644 index 00000000..f779b575 --- /dev/null +++ b/zml/tokenizer/hftokenizers/hftokenizers.h @@ -0,0 +1,17 @@ +#pragma once + +#include +#include + +#include "ffi/zig_slice.h" + +typedef struct hftokenizers hftokenizers; + +hftokenizers *hftokenizers_new(zig_slice); +hftokenizers *hftokenizers_new_from_bytes(zig_slice); +void hftokenizers_drop(hftokenizers *tokenizer); +zig_slice hftokenizers_encode(hftokenizers *tokenizer, zig_slice text); +void hftokenizers_tokens_drop(zig_slice tokens); +zig_slice hftokenizers_decode(hftokenizers *tokenizer, zig_slice tokens); +void hftokenizers_str_drop(zig_slice text); +uint32_t hftokenizers_token_to_id(hftokenizers *tokenizer, zig_slice token); diff --git a/zml/tokenizer/hftokenizers/hftokenizers.rs b/zml/tokenizer/hftokenizers/hftokenizers.rs new file mode 100644 index 00000000..048283cc --- /dev/null +++ b/zml/tokenizer/hftokenizers/hftokenizers.rs @@ -0,0 +1,109 @@ +#[repr(C)] +struct ZigSlice { + ptr: *mut T, + len: usize, +} + +impl ZigSlice { + fn as_slice(&self) -> &[T] { + unsafe { std::slice::from_raw_parts(self.ptr, self.len) } + } + + fn as_slice_mut(&self) -> &mut [T] { + unsafe { std::slice::from_raw_parts_mut(self.ptr, self.len) } + } +} + +#[no_mangle] +extern "C" fn hftokenizers_new(path: ZigSlice) -> *mut tokenizers::Tokenizer { + return Box::into_raw(Box::new( + tokenizers::Tokenizer::from_file(std::path::Path::new( + std::str::from_utf8(path.as_slice()).unwrap(), + )) + .unwrap(), + )); +} + +#[no_mangle] +extern "C" fn hftokenizers_new_from_bytes(bytes: ZigSlice) -> *mut tokenizers::Tokenizer { + return Box::into_raw(Box::new( + tokenizers::Tokenizer::from_bytes(bytes.as_slice()) + .unwrap(), + )); +} + +#[no_mangle] +extern "C" fn hftokenizers_drop(t: *mut tokenizers::Tokenizer) { + drop(unsafe { Box::from_raw(t) }); +} + +#[no_mangle] +extern "C" fn hftokenizers_encode( + t: *mut tokenizers::Tokenizer, + string: ZigSlice, +) -> ZigSlice { + let input_str = std::str::from_utf8(string.as_slice()).unwrap(); + let encoded = unsafe { t.as_ref() } + .unwrap() + .encode_fast(input_str, false) + .unwrap(); + + // Convert the result to a boxed slice + let mut ids: Box<[u32]> = encoded.get_ids().to_owned().into_boxed_slice(); + + // Retrieve the zig slice associated to the boxed slice. + let slice = ZigSlice { + ptr: ids.as_mut_ptr(), + len: ids.len(), + }; + + // Leak the box so that it's not deallocated. + Box::leak(ids); + + return slice; +} + +#[no_mangle] +extern "C" fn hftokenizers_tokens_drop(tokens: ZigSlice) { + // Reconstruct the Box from the zig slice so that it's dropped. + drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); +} + +#[no_mangle] +extern "C" fn hftokenizers_decode( + t: *mut tokenizers::Tokenizer, + ids: ZigSlice, +) -> ZigSlice { + let decoded = unsafe { t.as_ref() } + .unwrap() + .decode(ids.as_slice(), false) + .unwrap(); + + // Convert the result to a boxed slice + let mut string: Box<[u8]> = decoded.into_bytes().into_boxed_slice(); + + // Retrieve the zig slice associated to the boxed slice. + let slice = ZigSlice { + ptr: string.as_mut_ptr(), + len: string.len(), + }; + + // Leak the box so that it's not deallocated. + Box::leak(string); + + return slice; +} + +#[no_mangle] +extern "C" fn hftokenizers_str_drop(tokens: ZigSlice) { + drop(unsafe { Box::from_raw(tokens.as_slice_mut()) }); +} + +#[no_mangle] +extern "C" fn hftokenizers_token_to_id(t: *mut tokenizers::Tokenizer, token: ZigSlice) -> u32 { + let id = unsafe { t.as_ref() } + .unwrap() + .token_to_id(std::str::from_utf8(token.as_slice()).unwrap()) + .unwrap_or(u32::MAX); + return id; +} diff --git a/zml/tokenizer/hftokenizers/hftokenizers.zig b/zml/tokenizer/hftokenizers/hftokenizers.zig new file mode 100644 index 00000000..0871d6f9 --- /dev/null +++ b/zml/tokenizer/hftokenizers/hftokenizers.zig @@ -0,0 +1,120 @@ +const std = @import("std"); + +const c = @import("c"); +const ffi = @import("ffi"); +const stdx = @import("stdx"); + +pub const Encoder = struct { + inner: *HFTokenizer, + current_ids: ?[]const u32 = null, + + fn init(inner: *HFTokenizer) Encoder { + return .{ .inner = inner }; + } + + pub fn reset(self: *Encoder) void { + if (self.current_ids) |current_ids_| { + c.hftokenizers_tokens_drop(ffi.ZigSlice.from(current_ids_)); + self.current_ids = null; + } + } + + pub fn deinit(self: *Encoder) void { + self.reset(); + } + + pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { + self.reset(); + self.current_ids = ffi.ZigSlice.to(u32, c.hftokenizers_encode(@ptrCast(self.inner), ffi.ZigSlice.from(input))); + return self.ids(); + } + + pub fn ids(self: *const Encoder) []const u32 { + return self.current_ids orelse &.{}; + } +}; + +pub const Decoder = struct { + const StringBuffer = stdx.BoundedArray(u8, 128); + const TokensIdsBuffer = stdx.BoundedArray(u32, 4); + + inner: *HFTokenizer, + current_string: ?[]const u8 = null, + last_string: StringBuffer = .{ .len = 0 }, + last_token_ids: TokensIdsBuffer = .{ .len = 0 }, + + fn init(inner: *HFTokenizer) Decoder { + return .{ .inner = inner }; + } + + pub fn deinit(self: *Decoder) void { + self.reset(); + } + + pub fn reset(self: *Decoder) void { + if (self.current_string) |current_string_| { + c.hftokenizers_str_drop(ffi.ZigSlice.from(current_string_)); + self.current_string = null; + } + } + + pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { + self.reset(); + self.current_string = ffi.ZigSlice.to(u8, c.hftokenizers_decode(@ptrCast(self.inner), ffi.ZigSlice.from(ids))); + return self.string(); + } + + pub fn string(self: *const Decoder) []const u8 { + return self.current_string orelse &.{}; + } + + pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { + if (self.last_token_ids.len >= self.last_token_ids.capacity()) { + _ = self.last_token_ids.orderedRemove(0); + } + self.last_token_ids.appendAssumeCapacity(token_id); + const new_string = try self.decode(self.last_token_ids.constSlice()); + if (self.last_string.len == 0) { + self.last_string = try StringBuffer.fromSlice(new_string); + return new_string; + } + var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); + var it = view.iterator(); + while (it.nextCodepointSlice()) |cp| { + const start = it.i - cp.len; + if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { + const chunk = new_string[self.last_string.len - start ..]; + self.last_string = try StringBuffer.fromSlice(new_string); + return chunk; + } + } + return null; + } +}; + +pub const HFTokenizer = opaque { + pub fn fromFile(model: []const u8) !*HFTokenizer { + return @ptrCast(c.hftokenizers_new(ffi.ZigSlice.from(model))); + } + + pub fn fromBytes(bytes: []const u8) !*HFTokenizer { + return @ptrCast(c.hftokenizers_new_from_bytes(ffi.ZigSlice.from(bytes))); + } + + pub fn deinit(self: *HFTokenizer) void { + return c.hftokenizers_drop(@ptrCast(self)); + } + + pub fn encoder(self: *HFTokenizer) !Encoder { + return Encoder.init(self); + } + + pub fn decoder(self: *HFTokenizer) !Decoder { + return Decoder.init(self); + } + + pub fn tokenToId(self: *HFTokenizer, token: []const u8) ?u32 { + const id = c.hftokenizers_token_to_id(@ptrCast(self), ffi.ZigSlice.from(token)); + return if (id == std.math.maxInt(u32)) null else id; + } +}; diff --git a/zml/tokenizer/homemade.zig b/zml/tokenizer/homemade.zig new file mode 100644 index 00000000..bd16d183 --- /dev/null +++ b/zml/tokenizer/homemade.zig @@ -0,0 +1,1242 @@ +//! Text tokenizer implementations +//! Disclaimer this is not a very robust implementation: +//! In particular the normalization is pretty minimalist, only works with ascii, and don't do unicode normalization. +//! Mostly used for testing models that don't have an official HF/sentencepiece tokenizer. +const std = @import("std"); +const testing = std.testing; +const builtin = @import("builtin"); + +const stdx = @import("stdx"); + +const log = std.log.scoped(.@"zml/tokenizer"); + +/// Byte Pair Encoding tokenizer generally used for LLM. +pub const Tokenizer = struct { + tokens: [][]const u8, + token_lookup: std.StringHashMapUnmanaged(u32), + special_tokens: SpecialTokens, + + scores: []f32, + max_token_len: u32, + normalizer: ?Normalizer, + // Allows to split unknown unicode characters into bytes. + byte_fallback: bool = false, + + arena_state: std.heap.ArenaAllocator, + vocab_size: u32, + next_token_id: u32 = 0, + + pub const SpecialTokens = struct { + eos: u32, + bos: u32, + unk: u32, + pad: u32 = std.math.maxInt(u32), + hard_space: u32 = std.math.maxInt(u32), + }; + + pub fn init( + allocator: std.mem.Allocator, + vocab_size: u32, + max_token_len: u32, + normalizer: ?Normalizer, + special_tokens: SpecialTokens, + alloc_tokens: bool, + ) !Tokenizer { + var arena_state = std.heap.ArenaAllocator.init(allocator); + errdefer arena_state.deinit(); + const arena = arena_state.allocator(); + + var token_lookup: std.StringHashMapUnmanaged(u32) = .{}; + errdefer token_lookup.deinit(arena); + + try token_lookup.ensureTotalCapacity(arena, @intCast(vocab_size)); + + const tokens: [][]const u8 = if (alloc_tokens) try arena.alloc([]const u8, vocab_size) else &.{}; + errdefer if (alloc_tokens) arena.free(tokens); + + const scores: []f32 = if (alloc_tokens) try arena.alloc(f32, vocab_size) else &.{}; + errdefer if (alloc_tokens) arena.free(scores); + + return .{ + .tokens = tokens, + .scores = scores, + .max_token_len = max_token_len, + .token_lookup = token_lookup, + .arena_state = arena_state, + .normalizer = normalizer, + .vocab_size = vocab_size, + .special_tokens = special_tokens, + }; + } + + pub fn deinit(self: Tokenizer) void { + self.arena_state.deinit(); + } + + pub fn encoder(self: *Tokenizer) !Encoder { + return Encoder.init(self); + } + + pub fn decoder(self: *Tokenizer) !Decoder { + return Decoder.init(self); + } + + /// Reads a new word directly into the tokenizer arena. + pub fn readTokenInto(self: *Tokenizer, score: f32, len: usize, tok_reader: *std.Io.Reader) !void { + const arena = self.arena_state.allocator(); + + const token = try arena.alloc(u8, len); + try tok_reader.readSliceAll(token); + return self.addOwnedToken(score, token); + } + + /// Adds a new token (and copy it) + pub fn addToken(self: *Tokenizer, score: f32, token: []const u8) !void { + const arena = self.arena_state.allocator(); + return self.addOwnedToken(score, try arena.dupe(u8, token)); + } + + /// Adds a new token (without copying it) + pub fn addOwnedToken(self: *Tokenizer, score: f32, token: []const u8) void { + const i = self.next_token_id; + std.debug.assert(i < self.vocab_size); + self.next_token_id += 1; + + self.scores[i] = score; + self.tokens[i] = token; + const v = self.token_lookup.getOrPutAssumeCapacity(token); + if (!v.found_existing) { + v.value_ptr.* = i; + } + } + + pub fn addOwnedTokenByIndex(self: *Tokenizer, i: u32, score: f32, token: []const u8) void { + std.debug.assert(i < self.vocab_size); + self.next_token_id += 1; + self.scores[i] = score; + self.tokens[i] = token; + const v = self.token_lookup.getOrPutAssumeCapacity(token); + if (!v.found_existing) { + v.value_ptr.* = @intCast(i); + } + } + + pub fn lookup(self: *const Tokenizer, str: []const u8) ?u32 { + return self.token_lookup.get(str); + } + + pub fn tokenToId(self: *const Tokenizer, token: []const u8) ?u32 { + return self.token_lookup.get(token); + } + + pub const EncodeOptions = struct { + /// Should the beginning of sentence '' token be added. + add_bos: bool = true, + add_eos: bool = false, + pad_to: u32 = 0, + // Print tokenization intermediary steps. + debug: bool = false, + }; + + pub fn encode(self: *const Tokenizer, allocator: std.mem.Allocator, raw: []const u8, options: EncodeOptions) ![]u32 { + if (options.debug) log.debug("Tokenizer.encode('{s}')", .{raw}); + const input = if (self.normalizer) |n| try n.normalize(allocator, raw) else raw; + defer if (self.normalizer) |_| allocator.free(input); + if (options.debug) log.debug("Tokenizer.encode.normalize -> '{s}'", .{input}); + + // Allocate a buffer that can fit all indices as well as extra character if requested. + // We then slice it so that the token merging code doesn't see the bos token. + const tok_buff_alloc = try allocator.alloc(u32, @max(options.pad_to, input.len + 2)); + const tok_buff = if (options.add_bos) tok_buff_alloc[1..] else tok_buff_alloc; + + const MergeState = union(enum) { ready: u32, nope, hard_space, idk }; + const mergeable = try allocator.alloc(MergeState, tok_buff.len); + + var num_tokens: usize = 0; + var it: CharTokenIterator = .{ .input = input }; + while (try it.nextCodepointToken(self)) |token| : (num_tokens += 1) { + tok_buff[num_tokens] = token; + mergeable[num_tokens] = if (token == self.special_tokens.hard_space) + .hard_space + else + .idk; + } + + var stable_prefix: usize = 0; + var stable_off: usize = 0; + while (true) { + // This code is a bit overcomplicated cause I'm abstracting over two algorithms: + // BPE and sentencepiece unigram model. + // Normally BPE is pre-split on spaces then the regular merge algorithm is applied. + // With unigram model you work at sentence level and you handle spaces as you would any other bytes, + // hoping the final tokens mostly align with spaces. + // This seemed like a good idea, but is kinda bad because I had to add special code to speed up BPE + // by detecting when the first "word" is treated and can be safely removed from sequence. + // Also it doesn't work well with BPE vocab which have multi-space tokens (for indentation) + // and have custom splitting rules. + // This is fine for now cause we now have bindings to HF tokenizers for complexe use cases + // and are only using this for tinyllama/gguf models. + // If we come back to use this in production, the implementation would gain in speed/clarity + // by splitting in two. + // The merging token logic isn't that complicated anyway. + + // Step by step visualization of the progress. + if (options.debug) { + var _debug_buf: [256]u8 = undefined; + var debug_progress: std.Io.Writer = .fixed(&_debug_buf); + self.decodeWithOpts(tok_buff[0..num_tokens], &debug_progress, .{ .sep = "|" }) catch {}; + log.debug("tokens: {any} -> {s}", .{ tok_buff[0..num_tokens], debug_progress.buffered() }); + } + var best_score: f32 = -1e10; + var best_token: u32 = 0; + var best_idx: ?usize = null; + var input_off: usize = stable_off; + + // Find best tokens to merge in all available tokens + for (stable_prefix..num_tokens - 1) |i| { + if (tok_buff[i] == self.special_tokens.unk) { + input_off += 1; + continue; + } + const cur_tok = self.tokens[tok_buff[i]]; + defer input_off += cur_tok.len; + + // Lookup merge for current token, if not already done. + switch (mergeable[i]) { + .nope => continue, + .ready => {}, + .hard_space => { + // Since tokens are not allowed to merge through hard sep, + // we don't need to merge the sentence-wide best token. + // We can just merge the best token since beginning. + if (best_idx != null) break; + // OTOH if there was no merge possible since beginning, + // we can skip the beginning in future iterations. + stable_prefix = i + 1; + stable_off = input_off + cur_tok.len; + continue; + }, + .idk => { + + // Special tokens can't be concatenated. + if (builtin.mode == .Debug and tok_buff[i] != self.special_tokens.unk) { + // Detects memory corruption of tokens. + if (cur_tok.len == 0 or cur_tok.len > self.max_token_len) @panic("Token looks corrupted !"); + + if (!std.mem.eql(u8, cur_tok, input[input_off..][0..cur_tok.len])) { + log.err("current token '{s}' not found in input string '{s}' !", .{ cur_tok, input[input_off..] }); + @panic("invalid tokenization"); + } + } + const next_tok = self.tokens[tok_buff[i + 1]]; + // if `next_tok` is `.unk`, length is 1; otherwise, it's the length of the token. + const next_tok_len = if (tok_buff[i + 1] == self.special_tokens.unk) 1 else next_tok.len; + const concat_tokens = input[input_off..][0 .. cur_tok.len + next_tok_len]; + // Save the result + mergeable[i] = if (self.lookup(concat_tokens)) |tok| + .{ .ready = tok } + else + .nope; + }, + } + + switch (mergeable[i]) { + .idk, .hard_space => unreachable, + .nope => continue, + .ready => |tok| { + if (self.scores[tok] > best_score) { + best_score = self.scores[tok]; + best_token = tok; + best_idx = i; + } + }, + } + } + + if (best_idx) |bidx| { + // Apply the merge. + tok_buff[bidx] = best_token; + std.mem.copyForwards(u32, tok_buff[bidx + 1 ..], tok_buff[bidx + 2 .. num_tokens]); + std.mem.copyForwards(MergeState, mergeable[bidx + 1 ..], mergeable[bidx + 2 .. num_tokens]); + num_tokens -= 1; + // We got two new merge lookups to do. + mergeable[bidx] = .idk; + if (bidx > 0 and mergeable[bidx - 1] != .hard_space) mergeable[bidx - 1] = .idk; + } else { + // No merge candidate => we are done ! + break; + } + } + + if (options.add_eos) { + tok_buff[num_tokens] = self.special_tokens.eos; + num_tokens += 1; + } + if (options.add_bos) { + tok_buff_alloc[0] = self.special_tokens.bos; + num_tokens += 1; + } + if (num_tokens < options.pad_to) { + for (num_tokens..options.pad_to) |i| { + tok_buff_alloc[i] = self.special_tokens.pad; + } + num_tokens = options.pad_to; + } + + // Release extra memory we don't need anymore. + allocator.free(mergeable); + _ = allocator.resize(tok_buff_alloc, num_tokens); + return tok_buff_alloc[0..num_tokens]; + } + + /// Returns a slice corresponding to the given id. Handles unknown ids and special ids. + pub fn lookupPiece(self: *const Tokenizer, id: usize) []const u8 { + return if (id == self.special_tokens.bos or id == self.special_tokens.eos or id == self.special_tokens.pad) + "" + else if (id == self.special_tokens.unk) + "" + else if (id > self.tokens.len) + "" // this means we received an invalid id, but we didn't want to panic. + else + self.tokens[id]; + } + + /// Converts the given slice of tokens back into bytes. + /// Note that if the tokenizer allows sub-unicode bytes, it's possible + /// the output is not valid utf8. + pub fn decode(self: *const Tokenizer, input: []const u32, writer: *std.Io.Writer) std.Io.Writer.Error!void { + try self.decodeWithOpts(input, writer, .{}); + } + + pub fn decodeWithOpts( + self: *const Tokenizer, + input: []const u32, + output: *std.Io.Writer, + opts: struct { sep: []const u8 = "" }, + ) std.Io.Writer.Error!void { + const escaped = if (self.normalizer) |n| n.escapedSpace() else null; + // Flag used to indicate if the first dummy whitespace has been consumed. + var first_token: bool = true; + for (input) |id| { + // Retrieve the slice corresponding to the id. + var piece = self.lookupPiece(id); + + // Convert `▁` to a regular space. + if (escaped) |escspc| { + // we modify piece inside the loop, so we can use it in the condition + while (std.mem.startsWith(u8, piece, escspc)) { + piece = piece[escspc.len..]; + // don't output a space at beginning of text. + if (!first_token) try output.writeByte(' '); + } + } + + try output.writeAll(piece); + first_token = false; + try output.writeAll(opts.sep); + } + } + + /// Some tokenizers have bytes encoded in hex like this: "<0x40>". + /// This break the tokenization algorithm because the input text + /// will contain "@" not "<0x40>", + /// and if the input contains "<0x40>" it needs to not be treated as a single byte. + /// So we replace byte fallbacks strings, by their corresponding character. + /// This enables the normal tokenization algorithm to work. + pub fn rewriteByteFallbackTokens(tokenizer: *Tokenizer) !void { + tokenizer.byte_fallback = true; + var single_bytes = try tokenizer.arena_state.allocator().alloc(u8, 256); + var byte_fallback_buf = "<0x00>".*; + + for (0..256) |i| { + const c: u8 = @truncate(i); + single_bytes[i] = c; + + // First lookup the byte fallback entry. + // Note: we assume upper case, but we could try both upper and lower case if needed. + var writer: std.Io.Writer = .fixed(byte_fallback_buf[3..5]); + try writer.printInt(c, 16, .upper, .{ .fill = '0', .width = 2 }); + const entry = tokenizer.token_lookup.getEntry(&byte_fallback_buf) orelse { + log.err("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback token {s}", .{byte_fallback_buf}); + return error.InvalidInput; + }; + + // Check if the character is already present in the vocab. + // In that case, nothing to do, + // but note that the fallback token will be "unreachable", + // ie there is no way the tokenizer can produce it. + if (tokenizer.token_lookup.get(&.{c})) |_| continue; + + const idx: u32 = entry.value_ptr.*; + tokenizer.token_lookup.removeByPtr(entry.key_ptr); + tokenizer.addOwnedTokenByIndex(idx, tokenizer.scores[idx], single_bytes[i .. i + 1]); + } + } +}; + +test Tokenizer { + const allocator = std.testing.allocator; + const special_tokens: Tokenizer.SpecialTokens = .{ + .unk = 0, + .bos = 1, + .eos = 2, + }; + + var tokenizer = try Tokenizer.init(allocator, 10, 5, null, special_tokens, true); + defer tokenizer.deinit(); + + try tokenizer.addToken(10, "hello"); + try tokenizer.addToken(3.5, "world"); + + try testing.expect(tokenizer.lookup("hello") == 0); + try testing.expect(tokenizer.lookup("world") == 1); + + // TODO: test Tokenizer.decode, Tokenizer.encode, Tokenizer.readTokenInto +} + +pub const Encoder = struct { + inner: *Tokenizer, + arena: std.heap.ArenaAllocator, + current_ids: []const u32 = &.{}, + + fn init(inner: *Tokenizer) !Encoder { + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. + _ = try arena.allocator().alloc(u32, 4096); + std.debug.assert(arena.reset(.retain_capacity)); + return .{ .inner = inner, .arena = arena }; + } + + pub fn reset(self: *Encoder) void { + self.current_ids = &.{}; + std.debug.assert(self.arena.reset(.retain_capacity)); + } + + pub fn deinit(self: *Encoder) void { + self.arena.deinit(); + } + + pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { + self.reset(); + const res = try self.inner.encode(self.arena.allocator(), input, .{ + .add_bos = true, + .add_eos = false, + .pad_to = 0, + // Print tokenization intermediary steps. + .debug = false, + }); + self.current_ids = res; + return res; + } + + pub fn ids(self: *const Encoder) []const u32 { + return self.current_ids; + } +}; + +pub const Decoder = struct { + const StringBuffer = stdx.BoundedArray(u8, 128); + const TokensIdsBuffer = stdx.BoundedArray(u32, 4); + + inner: *Tokenizer, + arena: std.heap.ArenaAllocator, + + current_string: ?[]const u8 = null, + last_string: StringBuffer = .{ .len = 0 }, + last_token_ids: TokensIdsBuffer = .{ .len = 0 }, + + fn init(inner: *Tokenizer) !Decoder { + var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); + // Warmup the arena. Page allocator is expensive, avoid calling it for small reallocs. + _ = try arena.allocator().alloc(u32, 4096); + std.debug.assert(arena.reset(.retain_capacity)); + return .{ .inner = inner, .arena = arena }; + } + + pub fn deinit(self: *Decoder) void { + self.arena.deinit(); + } + + pub fn reset(self: *Decoder) void { + std.debug.assert(self.arena.reset(.retain_capacity)); + self.current_string = null; + } + + pub fn decode(self: *Decoder, ids: []const u32) ![]const u8 { + // TODO: revisite tokenizer api to use std.Io.Writer. + self.reset(); + // Reuse the same warmup than in init, to maximize contiguous writes. + var arena_writer = std.Io.Writer.Allocating.initCapacity(self.arena.allocator(), 4096) catch unreachable; + try self.inner.decode(ids, &arena_writer.writer); + self.current_string = arena_writer.written(); + return self.current_string.?; + } + + pub fn string(self: *const Decoder) []const u8 { + return self.current_string orelse ""; + } + + pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { + if (self.last_token_ids.len >= self.last_token_ids.capacity()) { + _ = self.last_token_ids.orderedRemove(0); + } + self.last_token_ids.appendAssumeCapacity(token_id); + const new_string = try self.decode(self.last_token_ids.constSlice()); + if (self.last_string.len == 0) { + self.last_string = try StringBuffer.fromSlice(new_string); + return new_string; + } + var view = try std.unicode.Utf8View.init(self.last_string.constSlice()); + var it = view.iterator(); + while (it.nextCodepointSlice()) |cp| { + const start = it.i - cp.len; + if (std.mem.startsWith(u8, new_string, self.last_string.constSlice()[start..])) { + const chunk = new_string[self.last_string.len - start ..]; + self.last_string = try StringBuffer.fromSlice(new_string); + return chunk; + } + } + return null; + } +}; + +/// Given a slice, split it in the most simple tokens using the given tokenizer tokens. +/// The output of this can be used to initialize the tokenization algorithm. +/// Normally we split the input text into utf8 codepoint, +/// but if we find an unknown codepoint we either split it in bytes, or use the special "unknown" token, +/// depending on the tokenizer configuration. +const CharTokenIterator = struct { + state: union(enum) { by_codepoint, by_byte: u8 } = .by_codepoint, + input: []const u8, + + fn nextCodepointToken(self: *CharTokenIterator, tokenizer: *const Tokenizer) error{ TruncatedInput, Utf8InvalidStartByte }!?u32 { + if (self.input.len == 0) return null; + return switch (self.state) { + .by_byte => |*byte_left| { + const idx = tokenizer.lookup(self.input[0..1]) orelse { + // Normally this has been caught when calling `rewriteByteFallbackTokens`. + std.debug.panic("Tokenizer has \"byte_fallback\" = true, but doesn't contains the byte fallback for token '<0x{X:02}>'", .{self.input[0]}); + }; + + self.input = self.input[1..]; + byte_left.* -|= 1; + if (byte_left.* == 0) self.state = .by_codepoint; + return idx; + }, + .by_codepoint => { + // Try to lookup valid utf8 codepoint first. + const utf8_len = try std.unicode.utf8ByteSequenceLength(self.input[0]); + if (self.input.len < utf8_len) return error.TruncatedInput; + if (tokenizer.lookup(self.input[0..utf8_len])) |idx| { + self.input = self.input[utf8_len..]; + return idx; + } + + // Otherwise split in bytes if it's allowed. + if (tokenizer.byte_fallback) { + // TODO: replace this by a continue statement next time we bump Zig. + self.state = .{ .by_byte = utf8_len }; + return self.nextCodepointToken(tokenizer); + } + + // Or mark the full utf8 codepoint as unknown. + log.debug("Token not found for char '{s}'", .{self.input[0..utf8_len]}); + self.input = self.input[utf8_len..]; + return tokenizer.special_tokens.unk; + }, + }; + } +}; + +test CharTokenIterator { + const special_tokens: Tokenizer.SpecialTokens = .{ .unk = 0, .bos = 1, .eos = 2 }; + var tokenizer = try Tokenizer.init(std.testing.allocator, 16, 4, null, special_tokens, true); + defer tokenizer.deinit(); + + tokenizer.addOwnedToken(1.0, ""); // 0 + tokenizer.addOwnedToken(1.0, ""); // 1 + tokenizer.addOwnedToken(1.0, ""); // 2 + tokenizer.addOwnedToken(1.0, "ζ"); // 3 + tokenizer.addOwnedToken(1.0, &.{0xE2}); // 4: ℳ, first byte + tokenizer.addOwnedToken(1.0, &.{0x84}); // 5: ℳ, second byte + tokenizer.addOwnedToken(1.0, &.{0xB3}); // 6: ℳ, third byte + tokenizer.addOwnedToken(1.0, "L"); // 7 + + // No byte fallback + { + tokenizer.byte_fallback = false; + var it: CharTokenIterator = .{ .input = "ζℳL" }; + var res: stdx.BoundedArray(u32, 8) = .{}; + while (try it.nextCodepointToken(&tokenizer)) |token| { + res.appendAssumeCapacity(token); + } + try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 0, 7 }, res.constSlice()); + } + + // with byte fallback + { + tokenizer.byte_fallback = true; + var it: CharTokenIterator = .{ .input = "ζℳL" }; + var res: stdx.BoundedArray(u32, 8) = .{}; + while (try it.nextCodepointToken(&tokenizer)) |token| { + res.appendAssumeCapacity(token); + } + try std.testing.expectEqualSlices(u32, &[_]u32{ 3, 4, 5, 6, 7 }, res.constSlice()); + } +} + +/// Text normalizer. +/// Most tokenizer assumes the input text have been prepocessed with on of those. +pub const Normalizer = struct { + /// Space token used by sentencepiece derived tokenizer. + pub const sentencepiece_space = "▁"; // \xe2\x96\x81 + + _whitespace: stdx.BoundedArray(u8, 8) = .{}, + + flags: packed struct { + remove_extra_whitespaces: bool, + add_dummy_prefix: bool, + add_dummy_suffix: bool, + /// Cheap lower casing. + /// TODO: try to match Python "lower" + lower_case_ascii: bool, + /// cheap ascii punct splitting. + // doing this processing ahead of time simplifies the logic + split_on_punct_ascii: bool, + }, + + pub fn init(flags: @FieldType(Normalizer, "flags"), escaped_whitespace: ?[]const u8) Normalizer { + var res: Normalizer = .{ .flags = flags }; + if (escaped_whitespace) |escaped| { + res._whitespace.appendSliceAssumeCapacity(escaped); + } + return res; + } + + pub inline fn escapedSpace(self: Normalizer) ?[]const u8 { + return if (self._whitespace.len > 1) self._whitespace.constSlice() else null; + } + + fn addSlice(allocator: std.mem.Allocator, data: []const u8, consumed: usize, normalized: *std.ArrayList(u8), normalized_to_origin: *std.ArrayList(usize)) !void { + try normalized.appendSlice(allocator, data); + for (data) |_| try normalized_to_origin.append(allocator, consumed); + } + + pub const Result = struct { + /// Normalized string + normalized: []const u8, + /// Mapping between chars in the original string and chars in the new string + normalized_to_origin: []const usize, + + pub fn deinit(self: Result, allocator: std.mem.Allocator) void { + allocator.free(self.normalized); + allocator.free(self.normalized_to_origin); + } + }; + + /// Simplifed version of Sentencepiece normalizer. + /// + /// Llama2 uses a normalizer called "identity" so this basically only handles trailing + /// whitespaces and replaces whitespace with the "▁" (U+2581) character. + pub fn normalize(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) ![]const u8 { + const res = try self.normalizeWithMapping(allocator, input); + allocator.free(res.normalized_to_origin); + return res.normalized; + } + + /// Returns both the normalized string and a mapping between the normalized string and the original. + pub fn normalizeWithMapping(self: Normalizer, allocator: std.mem.Allocator, input: []const u8) !Result { + // Number of bytes consumed from the input. + var consumed: usize = 0; + var trimmed_input = input; + + // Skip leading whitespaces. + if (self.flags.remove_extra_whitespaces) { + while (trimmed_input.len != 0) { + if (trimmed_input[0] != ' ') break; + trimmed_input = trimmed_input[1..]; + consumed += 1; + } + } + + // If the trimmed input is empty, we are done. + if (trimmed_input.len == 0) { + return .{ .normalized = &.{}, .normalized_to_origin = &.{} }; + } + + // Pre-allocate outputs + const space = self.escapedSpace() orelse " "; + const overhead = if (self.flags.split_on_punct_ascii) space.len + 1 else space.len; + var normalized: std.ArrayList(u8) = try .initCapacity(allocator, trimmed_input.len * overhead + 2 * space.len); + errdefer normalized.deinit(allocator); + var normalized_to_origin: std.ArrayList(usize) = try .initCapacity(allocator, normalized.capacity); + errdefer normalized_to_origin.deinit(allocator); + + // If the spec asks for it, add a whitespace at the beginning. + if (self.flags.add_dummy_prefix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); + + var is_prev_space: bool = true; + var is_prev_word: bool = false; + + while (trimmed_input.len != 0) { + // NOTE(Corendos): This might feel weird but normally the slice we get comes from a normalizing process and can contain multiple codepoints. + // Since we have an "identity" normalizer, each slice is actually a unicode character. + const multibyte_length = try std.unicode.utf8ByteSequenceLength(trimmed_input[0]); + var slice = trimmed_input[0..multibyte_length]; + const origin = consumed; + consumed += multibyte_length; + trimmed_input = trimmed_input[multibyte_length..]; + + if (self.flags.remove_extra_whitespaces and is_prev_space) { + while (slice.len > 0 and slice[0] == ' ') { + slice = slice[1..]; + } + if (slice.len == 0) continue; + } + is_prev_space = slice[slice.len - 1] == ' '; + + if (slice.len == 1) ascii: { + // The more advanced logic only works with ascii atm + var byte = slice[0]; + if (self.escapedSpace() != null and byte == ' ') { + // replace the space token by the special token + try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); + is_prev_word = false; + break :ascii; + } else if (self.flags.split_on_punct_ascii) { + if (is_prev_word and isPunct(slice)) { + // Insert a space, but continue handling the rest + try addSlice(allocator, space, origin, &normalized, &normalized_to_origin); + } + } + if (self.flags.lower_case_ascii) { + byte = std.ascii.toLower(byte); + } + try normalized.append(allocator, byte); + try normalized_to_origin.append(allocator, origin); + } else { + // we can safely copy to the output. + try addSlice(allocator, slice, origin, &normalized, &normalized_to_origin); + } + is_prev_word = !is_prev_space and !isPunct(slice); + } + + // Skip trailing whitespaces + if (self.flags.remove_extra_whitespaces) { + while (std.mem.endsWith(u8, normalized.items, space)) { + const length = normalized.items.len - space.len; + consumed = normalized_to_origin.items[length]; + try normalized.resize(allocator, length); + try normalized_to_origin.resize(allocator, length); + } + } + + try normalized_to_origin.append(allocator, consumed); + + std.debug.assert(normalized_to_origin.items.len == normalized.items.len + 1); + + if (self.flags.add_dummy_suffix) try addSlice(allocator, space, consumed, &normalized, &normalized_to_origin); + + return .{ + .normalized_to_origin = try normalized_to_origin.toOwnedSlice(allocator), + .normalized = try normalized.toOwnedSlice(allocator), + }; + } + + pub fn wellKnown(impl: KnownImplementation) Normalizer { + return switch (impl) { + .sentencepiece => init(.{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, sentencepiece_space), + .llama3 => init(.{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = false, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, null), + .gpt2 => init(.{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, null), + }; + } + + pub fn fromHfJson(config: std.json.ObjectMap) error{InvalidNormalizerJson}!Normalizer { + var normalizer: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = false, + .add_dummy_suffix = false, + .add_dummy_prefix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + } }; + + // Normalizer config can be a single normalizer, or a sequence of normalizers. + const maybe_steps = objectGet(config, .array, "normalizers"); + const steps = if (maybe_steps) |st| st.items else &.{std.json.Value{ .object = config }}; + + for (steps) |step_val| { + if (step_val != .object) { + return error.InvalidNormalizerJson; + } + const step = step_val.object; + + const step_type = objectGet(step, .string, "type") orelse { + return error.InvalidNormalizerJson; + }; + if (std.mem.eql(u8, "Prepend", step_type)) { + normalizer.flags.add_dummy_prefix = true; + } else if (std.mem.eql(u8, "Append", step_type)) { + normalizer.flags.add_dummy_suffix = true; + } else if (std.mem.eql(u8, "Replace", step_type)) { + const pattern = objectGet(step, .object, "pattern") orelse return error.InvalidNormalizerJson; + const str_pattern = objectGet(pattern, .string, "String") orelse return error.InvalidNormalizerJson; + + if (std.mem.eql(u8, str_pattern, " ")) { + normalizer._whitespace.appendSliceAssumeCapacity( + objectGet(step, .string, "content") orelse return error.InvalidNormalizerJson, + ); + } else { + log.warn("Normalizer Replace pattern not supported: '{s}' -> '{s}'", .{ str_pattern, objectGet(pattern, .string, "content") orelse "" }); + } + } else { + log.warn("Unknown normalizer type: {s}", .{step_type}); + } + } + + return normalizer; + } + + test "Normalizer.fromHfJson" { + const config_json = + \\{ + \\ "type": "Sequence", + \\ "normalizers": [ + \\ { + \\ "type": "Prepend", + \\ "prepend": "▁" + \\ }, + \\ { + \\ "type": "Replace", + \\ "pattern": { + \\ "String": " " + \\ }, + \\ "content": "▁" + \\ } + \\ ] + \\} + ; + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const config = try std.json.parseFromSliceLeaky(std.json.Value, arena.allocator(), config_json, .{}); + const normalizer = try Normalizer.fromHfJson(config.object); + + const expected = Normalizer{ + ._whitespace = .{ .buffer = [_]u8{ 0xe2, 0x96, 0x81 } ++ [_]u8{0} ** 5, .len = 3 }, + .flags = .{ + .remove_extra_whitespaces = false, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, + }; + try std.testing.expectEqual(expected.flags, normalizer.flags); + try std.testing.expectEqualStrings(expected.escapedSpace().?, normalizer.escapedSpace().?); + } +}; +pub const KnownImplementation = enum(u8) { + sentencepiece, + gpt2, + llama3, +}; + +fn isPunct(unicode_char: []const u8) bool { + // TODO use unicode categories + if (unicode_char.len > 1) return false; + + return switch (unicode_char[0]) { + ' ', '\t' => false, + 0...8 => true, + 10...31 => true, + '!'...'/' => true, + ':'...'@' => true, + '['...'`' => true, + '{'...'~' => true, + else => false, + }; +} + +test Normalizer { + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + } }; + const res = try n.normalizeWithMapping(testing.allocator, "Hellŏ world!"); + defer res.deinit(testing.allocator); + + try testing.expectEqualSlices(u8, " Hellŏ world!", res.normalized); + try testing.expectEqualSlices( + usize, + // H e l l ŏ ␣ w o r l d ! + &.{ 0, 0, 1, 2, 3, 4, 4, 6, 8, 9, 10, 11, 12, 13, 14 }, + res.normalized_to_origin, + ); + } + + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = true, + .add_dummy_suffix = true, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + } }; + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, " Hello world! ", res); + } + + { + const n = Normalizer.init( + .{ + .remove_extra_whitespaces = false, + .add_dummy_prefix = true, + .add_dummy_suffix = false, + .lower_case_ascii = false, + .split_on_punct_ascii = false, + }, + Normalizer.sentencepiece_space, + ); + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, "▁Hello▁▁world!", res); + } + + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = false, + .add_dummy_suffix = true, + .lower_case_ascii = true, + .split_on_punct_ascii = false, + } }; + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, "hello world! ", res); + } + + { + const n: Normalizer = .{ .flags = .{ + .remove_extra_whitespaces = true, + .add_dummy_prefix = false, + .add_dummy_suffix = true, + .lower_case_ascii = false, + .split_on_punct_ascii = true, + } }; + const res = try n.normalize(testing.allocator, "Hello world!"); + defer testing.allocator.free(res); + + try testing.expectEqualSlices(u8, "Hello world ! ", res); + } +} + +/// gpt2 had their own way of storing text. +/// Unfortunately this has contaminated other models. +/// This implementation precompupte a mapping between bytes encoded with GPT2 algorithm, +/// into utf8 bytes, and do lookups at runtime. +pub const Gpt2TextDecoder = struct { + const Code = stdx.BoundedArray(u8, 2); + + // TODO: benchmark this is more efficient than doing the conversion at runtime. + code_to_byte: std.AutoArrayHashMap(Code, u8), + + pub fn init(allocator: std.mem.Allocator) !Gpt2TextDecoder { + var self = Gpt2TextDecoder{ + .code_to_byte = std.AutoArrayHashMap(Code, u8).init(allocator), + }; + try self.code_to_byte.ensureTotalCapacity(256); + errdefer unreachable; + + var n: usize = 0; + for (0..256) |index| { + var code: Code = .{ .buffer = .{ 0, 0 }, .len = 0 }; // 0-init + const i: u8 = @intCast(index); + if (isPrintableByte(i)) { + if (std.ascii.isAscii(i)) { + code.appendAssumeCapacity(i); + } else { + const codepoint: u21 = @as(u21, @intCast(i)); + code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); + } + } else { + const codepoint: u21 = 256 + @as(u21, @intCast(n)); + code.len = @intCast(std.unicode.utf8Encode(codepoint, &code.buffer) catch unreachable); + n += 1; + } + + self.code_to_byte.putAssumeCapacityNoClobber(code, i); + } + return self; + } + + pub fn deinit(self: *Gpt2TextDecoder) void { + self.code_to_byte.deinit(); + } + + /// Transform bytes representing text under the gpt2 encoding, + /// and write to the `unicode` buffer utf-8 bytes. + pub fn decode(self: Gpt2TextDecoder, bytes: []const u8, writer: *std.Io.Writer) (error{InvalidInput} || std.Io.Writer.Error)!void { + var it = std.unicode.Utf8Iterator{ .i = 0, .bytes = bytes }; + while (it.nextCodepointSlice()) |codepoint| { + const code: Code = switch (codepoint.len) { + 1 => .{ .buffer = .{ codepoint[0], 0 }, .len = 1 }, // 0-init + 2 => .{ .buffer = .{ codepoint[0], codepoint[1] }, .len = 2 }, + else => return error.InvalidInput, + }; + const byte = self.code_to_byte.get(code) orelse return error.InvalidInput; + try writer.writeByte(byte); + } + } + + inline fn isPrintableByte(c: u8) bool { + return ('!' <= c and c <= '~') or (0xa1 <= c and c <= 0xac) or (0xae <= c and c <= 0xff); + } +}; + +test Gpt2TextDecoder { + var decoder = try Gpt2TextDecoder.init(testing.allocator); + defer decoder.deinit(); + + var buf: [128]u8 = undefined; + + // Ascii is not changed. + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("getTitle", &out); + try testing.expectEqualStrings("getTitle", out.buffered()); + } + + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("Ċ", &out); + try testing.expectEqualStrings("\n", out.buffered()); + } + + // Leading space are represented with 'Ġ' + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("ĠUINavigationController", &out); + try testing.expectEqualStrings(" UINavigationController", out.buffered()); + } + // Russian is wild + { + var out: std.Io.Writer = .fixed(&buf); + try decoder.decode("ĠÑĢабоÑĤ", &out); + try testing.expectEqualStrings(" работ", out.buffered()); + } +} + +/// Open a json file in HF format and load the vocab from it. +pub fn fromHfJson(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8) !Tokenizer { + const file = try std.fs.cwd().openFile(tokenizer_path, .{}); + defer file.close(); + + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const file_content = b: { + var reader = file.reader(io, &.{}); + var writer: std.Io.Writer.Allocating = .init(allocator); + defer writer.deinit(); + _ = try reader.interface.streamRemaining(&writer.writer); + break :b try writer.toOwnedSlice(); + }; + defer allocator.free(file_content); + + const info = try std.json.parseFromSliceLeaky(std.json.Value, arena, file_content, .{ + .duplicate_field_behavior = .use_last, + }); + const main_object = switch (info) { + .object => |obj| if (obj.get("added_tokens") == null or obj.get("model") == null) { + return error.InvalidFormat; + } else obj, + else => return error.InvalidFormat, + }; + + const model = objectGet(main_object, .object, "model") orelse return error.InvalidFormat; + const vocab = objectGet(model, .object, "vocab") orelse return error.InvalidFormat; + const added_tokens = if (objectGet(main_object, .array, "added_tokens")) |added| added.items else &.{}; + const vocab_size: u32 = @intCast(vocab.count() + added_tokens.len); + + const normalizer = if (objectGet(main_object, .object, "normalizer")) |normalizer_config| + try Normalizer.fromHfJson(normalizer_config) + else + Normalizer.wellKnown(.llama3); + + // delay init of special tokens. + var tokenizer = try Tokenizer.init(allocator, vocab_size, 256, normalizer, undefined, true); + errdefer tokenizer.deinit(); + + // Buffer containing all concatenated tokens. + // Reserve a big chunk, to avoid grow event, but release over-allocated memory. + var all_tokens: std.Io.Writer.Allocating = try .initCapacity(tokenizer.arena_state.allocator(), file_content.len); + const original_alloc = all_tokens.writer.buffer.ptr; + // A re-alloc event here means we have invalidated all slices inside the tokenizer. + // If this is too annoying we could switch to a custom type instead of slices. + defer { + std.debug.assert(all_tokens.writer.buffer.ptr == original_alloc); + } + + // gpt2 based tokenizer got a special way of encoding unicode. + // we don't know in advance if this will be used by this tokenizer or not. + // so we assume it is the case, but if we find some unicode character, + // outside of the range used by gpt2 we know it was wrong, and start over. + var is_gpt2_vocab: bool = true; + var gpt2_decoder = try Gpt2TextDecoder.init(allocator); + defer gpt2_decoder.deinit(); + var it = vocab.iterator(); + while (it.next()) |kv| { + const n = all_tokens.writer.buffered().len; + gpt2_decoder.decode(kv.key_ptr.*, &all_tokens.writer) catch |err| { + switch (err) { + error.InvalidInput => { + is_gpt2_vocab = false; + break; + }, + error.WriteFailed => return error.OutOfMemory, + } + }; + const idx: u32 = @intCast(kv.value_ptr.*.integer); + const token = all_tokens.written()[n..]; + tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); + } + + if (!is_gpt2_vocab) { + // We where wrong, this is not a gpt2 vocab, start over, + // and reset the tokenizer state. + tokenizer.next_token_id = 0; + tokenizer.token_lookup.clearRetainingCapacity(); + all_tokens.clearRetainingCapacity(); + it = vocab.iterator(); + while (it.next()) |kv| { + const idx: u32 = @intCast(kv.value_ptr.*.integer); + const token = try dup(&all_tokens, kv.key_ptr.*); + tokenizer.addOwnedTokenByIndex(idx, @floatFromInt(vocab_size - idx), token); + } + } + + // More tokens, typically added during fine tuning of the model. + for (added_tokens) |token_obj| { + if (token_obj != .object) return error.InvalidFormat; + const v = objectGet(token_obj.object, .string, "content") orelse return error.InvalidFormat; + const id: u32 = @intCast(objectGet(token_obj.object, .integer, "id") orelse return error.InvalidFormat); + const n = all_tokens.written().len; + try if (is_gpt2_vocab) + gpt2_decoder.decode(v, &all_tokens.writer) + else + all_tokens.writer.writeAll(v); + const token = all_tokens.written()[n..]; + tokenizer.addOwnedTokenByIndex(id, 0, token); + } + // We won't add more tokens here, let release. + _ = try all_tokens.toOwnedSlice(); + + var unk = tokenizer.lookup(""); + if (objectGet(model, .integer, "unk_token")) |unk_tok| { + unk = @intCast(unk_tok); + } + + tokenizer.special_tokens = .{ + // TODO allow users to specify special tokens or read them from a tokenizer_config.json file + .bos = tokenizer.lookup("") orelse tokenizer.lookup("<|begin_of_text|>") orelse @panic("bos token not found !"), + .eos = tokenizer.lookup("") orelse tokenizer.lookup("<|end_of_text|>") orelse @panic("eos token not found !"), + .unk = unk orelse std.math.maxInt(u32), + }; + + const byte_fallback = objectGet(model, .bool, "byte_fallback") orelse false; + if (!byte_fallback and unk == null) { + // GPT2 tokenizer have byte fallback already encoded in the model, + // but the json generally don't have the field set. + // We can detect it though because they don't specify an unknown token. + if (is_gpt2_vocab) { + tokenizer.byte_fallback = true; + } else { + log.warn("The given tokenizer can't handle unknown token: no unknown token was set, and byte_fallback is disabled too ! The tokenizer will panic when facing unknown tokens.", .{}); + } + } else if (byte_fallback) { + try tokenizer.rewriteByteFallbackTokens(); + } + return tokenizer; +} + +/// Returns a copy of the given string, stored inside the given ArrayList. +fn dup(allocating: *std.Io.Writer.Allocating, str: []const u8) std.Io.Writer.Error![]const u8 { + const n = allocating.written().len; + try allocating.writer.writeAll(str); + return allocating.written()[n..]; +} + +/// Returns the given entry in a json object only if it has the right type. +fn objectGet( + object: std.json.ObjectMap, + comptime kind: std.meta.FieldEnum(std.json.Value), + key: []const u8, +) ?@FieldType(std.json.Value, @tagName(kind)) { + const val = object.get(key) orelse return null; + if (val != kind) return null; + return @field(val, @tagName(kind)); +} + +pub fn fromTinyLlamaFile(allocator: std.mem.Allocator, io: std.Io, tokenizer_path: []const u8, vocab_size: u32) !Tokenizer { + const tokenizer_file = try std.fs.cwd().openFile(tokenizer_path, .{}); + defer tokenizer_file.close(); + var read_buff: [4096]u8 = undefined; + var tok_reader = tokenizer_file.reader(io, &read_buff); + const r: *std.Io.Reader = &tok_reader.interface; + + const max_token_len = try readValueLE(u32, r); + const special_tokens: Tokenizer.SpecialTokens = .{ + .unk = 0, + .bos = 1, + .eos = 2, + }; + var tokenizer = try Tokenizer.init(allocator, vocab_size, max_token_len, null, special_tokens, true); + var i: u32 = 0; + while (readToken(&tokenizer, r)) : (i += 1) { + // Pass + } else |_| { + if (i < vocab_size) { + log.info("Read {d} words out of {d}", .{ i, vocab_size }); + } + tokenizer.vocab_size = i; + } + try tokenizer.rewriteByteFallbackTokens(); + return tokenizer; +} + +fn readToken(tokenizer: *Tokenizer, tok_reader: *std.Io.Reader) !void { + const score: f32 = try readValueLE(f32, tok_reader); + const len: usize = try readValueLE(u32, tok_reader); + try tokenizer.readTokenInto(score, len, tok_reader); +} + +fn readValueLE(T: type, reader: *std.Io.Reader) !T { + var res: [1]T = undefined; + try reader.readSliceEndian(T, &res, .little); + return res[0]; +} diff --git a/zml/tokenizer/iree/BUILD.bazel b/zml/tokenizer/iree/BUILD.bazel deleted file mode 100644 index a49ccd61..00000000 --- a/zml/tokenizer/iree/BUILD.bazel +++ /dev/null @@ -1,31 +0,0 @@ -load("@rules_zig//zig:defs.bzl", "zig_library", "zig_test") - -zig_library( - name = "iree", - srcs = ["tokenizer.zig"], - main = "tokenizer.zig", - visibility = ["//visibility:public"], - deps = [ - "@iree//runtime/src/iree/tokenizer", - "@iree//runtime/src/iree/tokenizer:normalizer", - "@iree//runtime/src/iree/tokenizer/format/huggingface:normalizer_json", - "@iree//runtime/src/iree/tokenizer/format/huggingface:tokenizer_json", - "@iree//runtime/src/iree/tokenizer/normalizer:lowercase", - "@iree//runtime/src/iree/tokenizer/normalizer:nfc", - "@iree//runtime/src/iree/tokenizer/normalizer:nfd", - "@iree//runtime/src/iree/tokenizer/normalizer:nfkd", - "@iree//runtime/src/iree/tokenizer/normalizer:regex_replace", - "@iree//runtime/src/iree/tokenizer/normalizer:replace", - "@iree//runtime/src/iree/tokenizer/normalizer:sequence", - "@iree//runtime/src/iree/tokenizer/normalizer:strip", - "@iree//runtime/src/iree/tokenizer/normalizer:strip_accents", - "@iree//runtime/src/iree/tokenizer/vocab", - ], -) - -zig_test( - name = "test", - deps = [ - ":iree", - ], -) diff --git a/zml/tokenizer/iree/tokenizer.zig b/zml/tokenizer/iree/tokenizer.zig deleted file mode 100644 index b1586afb..00000000 --- a/zml/tokenizer/iree/tokenizer.zig +++ /dev/null @@ -1,1398 +0,0 @@ -const std = @import("std"); - -const c = @import("c"); - -const log = std.log.scoped(.@"zml/tokenizer"); - -pub const Error = error{ - Cancelled, - Unknown, - InvalidArgument, - DeadlineExceeded, - NotFound, - AlreadyExists, - PermissionDenied, - ResourceExhausted, - FailedPrecondition, - Aborted, - OutOfRange, - Unimplemented, - Internal, - Unavailable, - DataLoss, - Unauthenticated, -}; - -const StatusCodeMask = 0x1F; - -inline fn stringView(s: []const u8) c.iree_string_view_t { - return .{ .data = s.ptr, .size = s.len }; -} - -inline fn mutableStringView(s: []u8) c.iree_mutable_string_view_t { - return .{ .data = s.ptr, .size = s.len }; -} - -inline fn byteSpan(bytes: []u8) c.iree_byte_span_t { - return .{ .data = bytes.ptr, .data_length = bytes.len }; -} - -inline fn tokenOutput(ids: []u32) c.iree_tokenizer_token_output_t { - comptime { - if (@sizeOf(c.iree_tokenizer_token_id_t) != @sizeOf(u32) or @alignOf(c.iree_tokenizer_token_id_t) != @alignOf(u32)) { - @compileError("iree_tokenizer_token_id_t must match u32 layout"); - } - } - return .{ - .capacity = ids.len, - .token_ids = @ptrCast(ids.ptr), - .token_offsets = null, - .type_ids = null, - }; -} - -inline fn tokenIdList(ids: []const u32) c.iree_tokenizer_token_id_list_t { - comptime { - if (@sizeOf(c.iree_tokenizer_token_id_t) != @sizeOf(u32) or @alignOf(c.iree_tokenizer_token_id_t) != @alignOf(u32)) { - @compileError("iree_tokenizer_token_id_t must match u32 layout"); - } - } - return .{ .count = ids.len, .values = @ptrCast(ids.ptr) }; -} - -inline fn statusCode(status: c.iree_status_t) u32 { - return @intCast(@intFromPtr(status.?) & StatusCodeMask); -} - -fn checkOk(status: c.iree_status_t) !void { - if (status == null) return; - const code = statusCode(status); - var buffer: [1024]u8 = undefined; - var buffer_length: usize = 0; - if (c.iree_status_format(status, buffer.len, &buffer, &buffer_length)) { - log.err("IREE tokenizer error: {s}", .{buffer[0..@min(buffer_length, buffer.len - 1)]}); - } - _ = c.iree_status_ignore(status); - return switch (code) { - c.IREE_STATUS_CANCELLED => Error.Cancelled, - c.IREE_STATUS_UNKNOWN => Error.Unknown, - c.IREE_STATUS_INVALID_ARGUMENT => Error.InvalidArgument, - c.IREE_STATUS_DEADLINE_EXCEEDED => Error.DeadlineExceeded, - c.IREE_STATUS_NOT_FOUND => Error.NotFound, - c.IREE_STATUS_ALREADY_EXISTS => Error.AlreadyExists, - c.IREE_STATUS_PERMISSION_DENIED => Error.PermissionDenied, - c.IREE_STATUS_RESOURCE_EXHAUSTED => Error.ResourceExhausted, - c.IREE_STATUS_FAILED_PRECONDITION => Error.FailedPrecondition, - c.IREE_STATUS_ABORTED => Error.Aborted, - c.IREE_STATUS_OUT_OF_RANGE => Error.OutOfRange, - c.IREE_STATUS_UNIMPLEMENTED => Error.Unimplemented, - c.IREE_STATUS_INTERNAL => Error.Internal, - c.IREE_STATUS_UNAVAILABLE => Error.Unavailable, - c.IREE_STATUS_DATA_LOSS => Error.DataLoss, - c.IREE_STATUS_UNAUTHENTICATED => Error.Unauthenticated, - else => Error.Internal, - }; -} - -pub const Tokenizer = struct { - // TODO: use the ArrayList pattern. Stop copying allocator everywhere - allocator: std.mem.Allocator, - inner: *c.iree_tokenizer_t, - - pub fn fromHuggingFaceJson(allocator: std.mem.Allocator, json: []const u8) !Tokenizer { - // TODO: wrap allocator into an iree allocator - var raw: ?*c.iree_tokenizer_t = null; - try checkOk(c.iree_tokenizer_from_huggingface_json(stringView(json), c.iree_allocator_system(), &raw)); - return .{ - .allocator = allocator, - .inner = raw.?, - }; - } - - pub fn fromBytes(allocator: std.mem.Allocator, bytes: []const u8) !Tokenizer { - return fromHuggingFaceJson(allocator, bytes); - } - - pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { - const json = try std.Io.Dir.cwd().readFileAlloc(io, model, allocator, .unlimited); - defer allocator.free(json); - return try fromHuggingFaceJson(allocator, json); - } - - pub fn deinit(self: *Tokenizer) void { - c.iree_tokenizer_free(self.inner); - } - - pub fn encoder(self: *const Tokenizer) !Encoder { - return Encoder.init(self); - } - - pub fn decoder(self: *const Tokenizer) !Decoder { - return Decoder.init(self, self.allocator); - } - - pub fn normalizer(self: *const Tokenizer, sequence: []const Normalizer.Kind) !Normalizer { - return Normalizer.init(self.allocator, sequence); - } - - pub fn normalizerFromHuggingFaceJson(_: *const Tokenizer, json: []const u8) !Normalizer { - return Normalizer.fromHuggingFaceJson(json); - } - - pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { - const vocab = c.iree_tokenizer_vocab(self.inner); - if (vocab == null) return null; - - // Looks up a string in the vocabulary. - // Returns the token ID if found, or -1 if not found. - const id = c.iree_tokenizer_vocab_lookup(vocab, stringView(token)); - return if (id < 0) null else @intCast(id); - } - - pub fn decode(self: *const Tokenizer, token_id: u32) ?[]const u8 { - const vocab = c.iree_tokenizer_vocab(self.inner); - if (vocab == null) return null; - - if (token_id > std.math.maxInt(i32)) return null; - - const text = c.iree_tokenizer_vocab_token_text(vocab, @intCast(token_id)); - if (text.data == null) return null; - - return text.data[0..text.size]; - } - - pub const Encoder = struct { - tokenizer: *const Tokenizer, - state_storage: []u8, - transform_buffer: []u8, - state: *c.iree_tokenizer_encode_state_t, - - fn checkEncodeStatus(status: c.iree_status_t) !void { - // Output exhaustion is normal streaming backpressure. The caller - // commits any produced tokens and retries with more output space. - if (status != null and statusCode(status) == c.IREE_STATUS_RESOURCE_EXHAUSTED) { - _ = c.iree_status_ignore(status); - return Error.ResourceExhausted; - } - try checkOk(status); - } - - fn init(tokenizer: *const Tokenizer) !Encoder { - var state_size: usize = undefined; - try checkOk(c.iree_tokenizer_encode_state_calculate_size(tokenizer.inner, &state_size)); - const state_storage = try tokenizer.allocator.alloc(u8, state_size); - errdefer tokenizer.allocator.free(state_storage); - - const transform_size: usize = c.IREE_TOKENIZER_TRANSFORM_BUFFER_MAX_SIZE; - const transform_buffer = try tokenizer.allocator.alloc(u8, transform_size); - errdefer tokenizer.allocator.free(transform_buffer); - - var state: ?*c.iree_tokenizer_encode_state_t = null; - try checkOk(c.iree_tokenizer_encode_state_initialize( - tokenizer.inner, - byteSpan(state_storage), - byteSpan(transform_buffer), - c.iree_tokenizer_offset_run_list_empty(), - c.IREE_TOKENIZER_ENCODE_FLAG_AT_INPUT_START, - &state, - )); - - return .{ - .tokenizer = tokenizer, - .state_storage = state_storage, - .transform_buffer = transform_buffer, - .state = state.?, - }; - } - - pub fn deinit(self: *Encoder) void { - c.iree_tokenizer_encode_state_deinitialize(self.state); - self.tokenizer.allocator.free(self.state_storage); - self.tokenizer.allocator.free(self.transform_buffer); - } - - pub fn reset(self: *Encoder) void { - c.iree_tokenizer_encode_state_reset(self.state, c.IREE_TOKENIZER_ENCODE_FLAG_AT_INPUT_START); - } - - pub fn encodeAlloc(self: *Encoder, allocator: std.mem.Allocator, text: []const u8) ![]u32 { - var token_ids = std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); - errdefer token_ids.deinit(); - - try self.feed(text, &token_ids.writer); - try self.finalize(&token_ids.writer); - - const bytes = try token_ids.toOwnedSlice(); - const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..bytes.len]); - return std.mem.bytesAsSlice(u32, aligned_bytes); - } - - /// /!\ Assumes the writer provides buffers with proper alignment. - /// /!\ Remember to call the finalize() method - pub fn writer(self: *Encoder, buffer: []u8, out: *std.Io.Writer) Writer { - return Writer.init(self, buffer, out); - } - - pub const Writer = struct { - encoder: *Encoder, - out: *std.Io.Writer, - interface: std.Io.Writer, - - fn init(encoder_: *Encoder, buffer: []u8, out: *std.Io.Writer) Writer { - return .{ - .encoder = encoder_, - .out = out, - .interface = .{ - .buffer = buffer, - .vtable = &.{ - .drain = drain, - .flush = flush, - .rebase = std.Io.Writer.failingRebase, - }, - }, - }; - } - - pub fn finalize(self: *Writer) std.Io.Writer.Error!void { - try self.interface.flush(); - self.encoder.finalize(self.out) catch |e| { - log.err("Finalize failed with {s}", .{@errorName(e)}); - return error.WriteFailed; - }; - } - - fn flush(w: *std.Io.Writer) std.Io.Writer.Error!void { - if (w.end == 0) return; - const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); - try self.feed(w.buffer[0..w.end]); - w.end = 0; - } - - fn drain(w: *std.Io.Writer, data: []const []const u8, splat: usize) std.Io.Writer.Error!usize { - const self: *Writer = @alignCast(@fieldParentPtr("interface", w)); - - if (w.end != 0) { - try self.feed(w.buffer[0..w.end]); - w.end = 0; - } - - var total: usize = 0; - // Prefix chunks: written once each. - for (data[0 .. data.len - 1]) |chunk| { - try self.feed(chunk); - total += chunk.len; - } - // Last chunk is the splat pattern. - const pattern = data[data.len - 1]; - if (pattern.len == 0) return total; - for (0..splat) |_| { - try self.feed(pattern); - total += pattern.len; - } - return total; - } - - fn feed(self: *Writer, chunk: []const u8) std.Io.Writer.Error!void { - self.encoder.feed(chunk, self.out) catch |e| { - log.err("Feed failed with {s}", .{@errorName(e)}); - return error.WriteFailed; - }; - } - }; - - /// /!\ Assumes the writer provides buffers with proper alignment. - pub fn feed(self: *Encoder, text: []const u8, out: *std.Io.Writer) !void { - var min_tokens: usize = @max(text.len / 4, 32); - var remaining = text; - while (remaining.len > 0) { - var consumed: usize = 0; - var produced: usize = 0; - - checkEncodeStatus(c.iree_tokenizer_encode_state_feed( - self.state, - stringView(remaining), - tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), - &consumed, - &produced, - )) catch |e| switch (e) { - error.ResourceExhausted => { - out.advance(produced * @sizeOf(u32)); - min_tokens = @max(32, min_tokens -| produced); - remaining = remaining[consumed..]; - produced = 0; - consumed = 0; - }, - else => return e, - }; - - if (produced == 0 and consumed == 0) { - min_tokens *= 2; - std.debug.assert(min_tokens < 1 << 20); - continue; - } - - out.advance(produced * @sizeOf(u32)); - min_tokens = @max(32, min_tokens -| produced); - remaining = remaining[consumed..]; - } - } - - /// /!\ Assumes the writer provides buffers with proper alignment. - pub fn finalize(self: *Encoder, out: *std.Io.Writer) !void { - var min_tokens: usize = 4; - while (true) { - var produced: usize = 0; - - checkEncodeStatus(c.iree_tokenizer_encode_state_finalize( - self.state, - tokenOutput(try writableTokenSliceGreedy(out, min_tokens)), - &produced, - )) catch |e| switch (e) { - error.ResourceExhausted => { - out.advance(produced * @sizeOf(u32)); - min_tokens *= 2; - std.debug.assert(min_tokens < 1 << 10); - continue; - }, - else => return e, - }; - - out.advance(produced * @sizeOf(u32)); - if (!c.iree_tokenizer_encode_state_has_pending(self.state)) break; - if (produced == 0) { - min_tokens *= 2; - std.debug.assert(min_tokens < 1 << 20); - } - } - } - - fn writableTokenSliceGreedy(out: *std.Io.Writer, min_tokens: usize) std.Io.Writer.Error![]u32 { - std.debug.assert(@mod(out.end, @sizeOf(u32)) == 0); - const bytes = try out.writableSliceGreedy(min_tokens * @sizeOf(u32)); - std.debug.assert(@mod(@intFromPtr(bytes.ptr), 4) == 0); - const aligned_len = bytes.len - @mod(bytes.len, @sizeOf(u32)); - const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..aligned_len]); - return std.mem.bytesAsSlice(u32, aligned_bytes); - } - }; - - pub const Decoder = struct { - tokenizer: *const Tokenizer, - state_storage: []u8, - state: *c.iree_tokenizer_decode_state_t, - - fn init(tokenizer: *const Tokenizer, allocator: std.mem.Allocator) !Decoder { - var state_size: usize = undefined; - try checkOk(c.iree_tokenizer_decode_state_calculate_size(tokenizer.inner, &state_size)); - const state_storage = try allocator.alloc(u8, state_size); - errdefer allocator.free(state_storage); - - var state: ?*c.iree_tokenizer_decode_state_t = null; - try checkOk(c.iree_tokenizer_decode_state_initialize( - tokenizer.inner, - c.IREE_TOKENIZER_DECODE_FLAG_NONE, - byteSpan(state_storage), - &state, - )); - - return .{ - .tokenizer = tokenizer, - .state_storage = state_storage, - .state = state.?, - }; - } - - pub fn deinit(self: *Decoder) void { - c.iree_tokenizer_decode_state_deinitialize(self.state); - self.tokenizer.allocator.free(self.state_storage); - } - - pub fn reset(self: *Decoder) !void { - c.iree_tokenizer_decode_state_deinitialize(self.state); - var state: ?*c.iree_tokenizer_decode_state_t = null; - try checkOk(c.iree_tokenizer_decode_state_initialize( - self.tokenizer.inner, - c.IREE_TOKENIZER_DECODE_FLAG_NONE, - byteSpan(self.state_storage), - &state, - )); - self.state = state.?; - } - - pub fn feedOne(self: *Decoder, token_id: u32, out: []u8) ![]u8 { - std.debug.assert(out.len != 0); - var consumed: usize = 0; - var produced: usize = 0; - try checkOk(c.iree_tokenizer_decode_state_feed( - self.state, - tokenIdList(&.{token_id}), - mutableStringView(out), - &consumed, - &produced, - )); - std.debug.assert(consumed == 1); - return out[0..produced]; - } - - pub fn finalize(self: *Decoder, out: []u8) ![]u8 { - std.debug.assert(out.len != 0); - var produced: usize = 0; - try checkOk(c.iree_tokenizer_decode_state_finalize( - self.state, - mutableStringView(out), - &produced, - )); - return out[0..produced]; - } - - pub fn decode(self: *Decoder, token_ids: []const u32, out: *std.Io.Writer) !void { - var min_output: usize = @max(token_ids.len * 4, 128); - var remaining = token_ids; - while (remaining.len > 0) { - var consumed: usize = 0; - var produced: usize = 0; - - checkOk(c.iree_tokenizer_decode_state_feed( - self.state, - tokenIdList(remaining), - mutableStringView(try out.writableSliceGreedy(min_output)), - &consumed, - &produced, - )) catch |e| switch (e) { - error.ResourceExhausted => { - out.advance(produced); - min_output = @max(128, min_output -| produced); - remaining = remaining[consumed..]; - produced = 0; - consumed = 0; - }, - else => return e, - }; - - if (produced == 0 and consumed == 0) { - @branchHint(.unlikely); - min_output *= 2; - std.debug.assert(min_output < 1 << 20); - continue; - } - - out.advance(produced); - min_output = @max(128, min_output -| produced); - remaining = remaining[consumed..]; - } - - min_output = 32; - while (true) { - var produced: usize = 0; - - checkOk(c.iree_tokenizer_decode_state_finalize( - self.state, - mutableStringView(try out.writableSliceGreedy(min_output)), - &produced, - )) catch |e| switch (e) { - error.ResourceExhausted => { - out.advance(produced); - min_output *= 2; - std.debug.assert(min_output < 1 << 10); - continue; - }, - else => return e, - }; - - out.advance(produced); - break; - } - } - }; -}; - -pub const Normalizer = struct { - pub const Kind = union(enum) { - /// Unicode Normalization Form C. - NFC, - /// Unicode Normalization Form D. - NFD, - /// Unicode Normalization Form KD. - NFKD, - /// Removes Unicode combining marks. - /// filters out characters in the Unicode Mark category (Mn, Mc, Me). - StripAccents, - /// Converts text to lowercase. - Lowercase, - /// Substitutes occurrences of a literal or regular-expression pattern with the specified content. - Replace: struct { - kind: enum { - regex, - exact, - }, - pattern: []const u8, - content: []const u8, - }, - /// Removes leading and/or trailing whitespace. - Strip: enum(u8) { - left = 0b10, - right = 0b01, - both = 0b11, - }, - }; - - sequence: *c.iree_tokenizer_normalizer_t, - sequence_size: usize, - - // Supported normalizer types as of `4d4e97d00f099a21f38eeff26f82a6d9e3643a11`: - // - Sequence: Chains multiple normalizers in order - // - Lowercase: Unicode case folding - // - Strip: Leading/trailing whitespace removal - // - Prepend: Prefix string insertion - // - StripAccents: Removes combining marks (without NFD) - // - BertNormalizer: Combined BERT normalization pipeline - pub fn fromHuggingFaceJson(json: []const u8) !Normalizer { - var normalizer: ?*c.iree_tokenizer_normalizer_t = null; - errdefer if (normalizer) |ptr| c.iree_tokenizer_normalizer_free(ptr); - - try checkOk(c.iree_tokenizer_huggingface_parse_normalizer(.{ .data = json.ptr, .size = json.len }, c.iree_allocator_system(), &normalizer)); - if (normalizer == null) return error.NormalizerAllocationFailed; - - const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(normalizer.?)); - - return .{ - .sequence = normalizer.?, - .sequence_size = sequence_size, - }; - } - - fn createNormalizer(normalizer_kind: Kind) !?*c.iree_tokenizer_normalizer_t { - var norm: ?*c.iree_tokenizer_normalizer_t = null; - - const iree_allocator = c.iree_allocator_system(); - - const status = switch (normalizer_kind) { - .NFC => c.iree_tokenizer_normalizer_nfc_allocate(iree_allocator, &norm), - .NFD => c.iree_tokenizer_normalizer_nfd_allocate(iree_allocator, &norm), - .NFKD => c.iree_tokenizer_normalizer_nfkd_allocate(iree_allocator, &norm), - .StripAccents => c.iree_tokenizer_normalizer_strip_accents_allocate(iree_allocator, &norm), - .Lowercase => c.iree_tokenizer_normalizer_lowercase_allocate(iree_allocator, &norm), - .Replace => |rule| switch (rule.kind) { - .regex => c.iree_tokenizer_normalizer_regex_replace_allocate( - .{ .data = rule.pattern.ptr, .size = rule.pattern.len }, - .{ .data = rule.content.ptr, .size = rule.content.len }, - iree_allocator, - &norm, - ), - .exact => c.iree_tokenizer_normalizer_replace_allocate( - .{ .data = rule.pattern.ptr, .size = rule.pattern.len }, - .{ .data = rule.content.ptr, .size = rule.content.len }, - iree_allocator, - &norm, - ), - }, - .Strip => |direction| b: { - const strip_left = @backingInt(direction) & 0b10 != 0; - const strip_right = @backingInt(direction) & 0b01 != 0; - break :b c.iree_tokenizer_normalizer_strip_allocate(strip_left, strip_right, iree_allocator, &norm); - }, - }; - - try checkOk(status); - if (norm == null) return error.NormalizerAllocationFailed; - return norm; - } - - pub fn init(allocator: std.mem.Allocator, sequence: []const Kind) !Normalizer { - if (sequence.len == 0) return error.EmptySequence; - - if (sequence.len == 1) { - // IREE requires a sequence of at least two normalizer. - const norm = try createNormalizer(sequence[0]) orelse unreachable; - const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(norm)); - - return .{ - .sequence = norm, - .sequence_size = sequence_size, - }; - } - - var normalizers: std.ArrayList(*c.iree_tokenizer_normalizer_t) = try .initCapacity(allocator, sequence.len); - defer normalizers.deinit(allocator); - errdefer for (normalizers.items) |normalizer| c.iree_tokenizer_normalizer_free(normalizer); - - for (sequence) |normalizer| { - normalizers.appendAssumeCapacity(try createNormalizer(normalizer) orelse unreachable); - } - - var normalizer_sequence: ?*c.iree_tokenizer_normalizer_t = null; - // IREE takes ownership of all the normalizers. - try checkOk(c.iree_tokenizer_normalizer_sequence_allocate(normalizers.items.ptr, normalizers.items.len, c.iree_allocator_system(), &normalizer_sequence)); - errdefer c.iree_tokenizer_normalizer_free(normalizer_sequence); - normalizers.clearRetainingCapacity(); - - if (normalizer_sequence == null) return error.NormalizerAllocationFailed; - - const sequence_size: usize = @intCast(c.iree_tokenizer_normalizer_state_size(normalizer_sequence.?)); - if (sequence_size == 0) return error.NormalizerAllocationFailed; - - return .{ - .sequence = normalizer_sequence.?, - .sequence_size = sequence_size, - }; - } - - pub fn deinit(self: *Normalizer) void { - c.iree_tokenizer_normalizer_free(self.sequence); - } - - pub fn normalize(self: *Normalizer, allocator: std.mem.Allocator, text: []const u8) ![]const u8 { - const storage: []align(16) u8 = try allocator.alignedAlloc(u8, .@"16", self.sequence_size); - defer allocator.free(storage); - - var state: ?*c.iree_tokenizer_normalizer_state_t = null; - defer if (state) |ptr| c.iree_tokenizer_normalizer_state_deinitialize(ptr); - - try checkOk(c.iree_tokenizer_normalizer_state_initialize(self.sequence, storage.ptr, &state)); - if (state == null) return error.NormalizationFailed; - - var buffer: [2048]u8 = undefined; - var remaining = text; - - var writer: std.Io.Writer.Allocating = .init(allocator); - defer writer.deinit(); - - while (remaining.len > 0) { - var written: usize = 0; - var consumed: usize = 0; - - try checkOk(c.iree_tokenizer_normalizer_state_process( - state.?, - .{ .data = remaining.ptr, .size = remaining.len }, - .{ .data = &buffer, .size = buffer.len }, - c.IREE_TOKENIZER_NORMALIZER_FLAG_SEGMENT_END, - &consumed, - &written, - )); - - if (consumed == 0 and written == 0) return error.NormalizationFailed; - - remaining = remaining[consumed..]; - try writer.writer.writeAll(buffer[0..written]); - } - - var has_pending = true; - while (has_pending) { - var written: usize = 0; - - try checkOk(c.iree_tokenizer_normalizer_state_finalize(state.?, .{ .data = &buffer, .size = buffer.len }, &written)); - has_pending = c.iree_tokenizer_normalizer_state_has_pending(state.?); - - try writer.writer.writeAll(buffer[0..written]); - - if (written == 0 and has_pending) return error.NormalizationFailed; - } - - return writer.toOwnedSlice(); - } -}; - -test "normalizer from Hugging Face" { - const allocator = std.testing.allocator; - - const json = - \\{ - \\ "type": "Sequence", - \\ "normalizers": [ - \\ {"type":"NFKD"}, - \\ {"type":"StripAccents"}, - \\ {"type":"Lowercase"}, - \\ {"type":"Replace","pattern":{"Regex":"[ \\t\\r\\n]+"},"content":" "}, - \\ {"type":"Replace","pattern":{"Regex":"^ $"},"content":"\ue000"}, - \\ {"type":"Strip","strip_left":true,"strip_right":true}, - \\ {"type":"Replace","pattern":{"String":"\ue000"},"content":" "} - \\ ] - \\} - ; - - var normalizer = try Normalizer.fromHuggingFaceJson(json); - defer normalizer.deinit(); - - const TestCase = struct { - input: []const u8, - expected: []const u8, - }; - - for ([_]TestCase{ - .{ .input = " hello", .expected = "hello" }, - .{ .input = "hello ", .expected = "hello" }, - .{ .input = " hello ", .expected = "hello" }, - .{ .input = "\nworld\r\n", .expected = "world" }, - .{ .input = "ÉCOLE", .expected = "ecole" }, - .{ .input = " CAFÉ\tAU\nLAIT ", .expected = "cafe au lait" }, - .{ .input = "\t \r\n", .expected = " " }, - .{ .input = "", .expected = "" }, - }) |case| { - const res = try normalizer.normalize(allocator, case.input); - defer allocator.free(res); - - try std.testing.expectEqualSlices(u8, case.expected, res); - } -} - -test "normalizer sequence" { - const allocator = std.testing.allocator; - - var normalizer = try Normalizer.init(allocator, &.{ - .{ .Strip = .both }, - .Lowercase, - }); - defer normalizer.deinit(); - - const res = try normalizer.normalize(allocator, " helLO WoRLD "); - defer allocator.free(res); - - try std.testing.expectEqualSlices(u8, "hello world", res); -} - -test "normalizer preserves whitespace across output and sequence boundaries" { - const allocator = std.testing.allocator; - for ([_][]const Normalizer.Kind{ - &.{.{ .Strip = .both }}, - &.{ .Lowercase, .{ .Strip = .both } }, - &.{ .Lowercase, .{ .Strip = .both }, .NFC }, - }) |steps| { - var normalizer = try Normalizer.init(allocator, steps); - defer normalizer.deinit(); - for ([_]usize{ 63, 64, 2047, 2048, 4096 }) |prefix_len| { - const input = try allocator.alloc(u8, prefix_len + 4); - defer allocator.free(input); - @memset(input[0..prefix_len], 'a'); - @memcpy(input[prefix_len..], " b "); - const output = try normalizer.normalize(allocator, input); - defer allocator.free(output); - try std.testing.expectEqualSlices(u8, input[0 .. prefix_len + 2], output); - } - } -} - -test "normalizer sequence preserves Unicode across tiles" { - const allocator = std.testing.allocator; - var normalizer = try Normalizer.fromHuggingFaceJson( - \\{"type":"Sequence","normalizers":[{"type":"Lowercase"},{"type":"Strip","strip_left":true,"strip_right":true}]} - ); - defer normalizer.deinit(); - for ([_][]const u8{ "é", "€", "😀" }) |codepoint| { - for (61..65) |prefix_len| { - const input = try allocator.alloc(u8, prefix_len + codepoint.len + 1); - defer allocator.free(input); - @memset(input[0..prefix_len], 'a'); - @memcpy(input[prefix_len..][0..codepoint.len], codepoint); - input[input.len - 1] = 'b'; - const output = try normalizer.normalize(allocator, input); - defer allocator.free(output); - try std.testing.expectEqualSlices(u8, input, output); - } - } -} - -test "normalizer sequence preserves composition across tiles" { - const allocator = std.testing.allocator; - var normalizer = try Normalizer.init(allocator, &.{ .NFC, .Lowercase }); - defer normalizer.deinit(); - const input = @as([63]u8, @splat('a')) ++ "e\u{301}b"; - const output = try normalizer.normalize(allocator, input); - defer allocator.free(output); - try std.testing.expectEqualSlices(u8, @as([63]u8, @splat('a')) ++ "éb", output); -} - -test "normalizer sequence allocation failure retains child ownership" { - const FailAllocator = struct { - fn control(_: ?*anyopaque, _: c.iree_allocator_command_t, _: ?*const anyopaque, _: [*c]?*anyopaque) callconv(.c) c.iree_status_t { - return @ptrFromInt(c.IREE_STATUS_RESOURCE_EXHAUSTED); - } - }; - // Include a nested sequence: failure must preserve its shell and children. - var nested = try Normalizer.init(std.testing.allocator, &.{ .NFC, .Lowercase }); - defer nested.deinit(); - var strip = try Normalizer.init(std.testing.allocator, &.{.{ .Strip = .both }}); - defer strip.deinit(); - const children = [_]*c.iree_tokenizer_normalizer_t{ nested.sequence, strip.sequence }; - var sequence: ?*c.iree_tokenizer_normalizer_t = null; - const status = c.iree_tokenizer_normalizer_sequence_allocate(&children, children.len, .{ - .self = null, - .ctl = FailAllocator.control, - }, &sequence); - defer c.iree_status_free(status); - try std.testing.expect(status != null); - try std.testing.expectEqual(@as(u32, c.IREE_STATUS_RESOURCE_EXHAUSTED), statusCode(status)); - try std.testing.expect(sequence == null); - const output = try nested.normalize(std.testing.allocator, "HELLO"); - defer std.testing.allocator.free(output); - try std.testing.expectEqualStrings("hello", output); -} - -fn tokenOutputWriter(allocator: std.mem.Allocator) std.Io.Writer.Allocating { - return std.Io.Writer.Allocating.initAligned(allocator, .of(u32)); -} - -fn writtenTokenIds(out: *std.Io.Writer.Allocating) []const u32 { - const bytes = out.written(); - const aligned_bytes: []align(4) u8 = @alignCast(bytes[0..bytes.len]); - return std.mem.bytesAsSlice(u32, aligned_bytes); -} - -fn decodedTextWriter(allocator: std.mem.Allocator) std.Io.Writer.Allocating { - return std.Io.Writer.Allocating.init(allocator); -} - -test "huggingface json encode/decode" { - const allocator = std.testing.allocator; - const json = - \\{ - \\ "model": { - \\ "type": "WordPiece", - \\ "unk_token": "[UNK]", - \\ "continuing_subword_prefix": "##", - \\ "max_input_chars_per_word": 100, - \\ "vocab": { - \\ "[UNK]": 0, - \\ "hello": 1, - \\ "world": 2 - \\ } - \\ }, - \\ "pre_tokenizer": {"type": "Whitespace"}, - \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} - \\} - ; - - var tokenizer = try Tokenizer.fromHuggingFaceJson(allocator, json); - defer tokenizer.deinit(); - - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const text = "hello world"; - - const token_ids = try encoder.encodeAlloc(allocator, text); - defer allocator.free(token_ids); - try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids); - - const token_hello = tokenizer.tokenId("hello") orelse unreachable; - try std.testing.expectEqual(@as(u32, 1), token_hello); - - const unknown = tokenizer.tokenId("missing"); - try std.testing.expect(unknown == null); - - var decoder = try tokenizer.decoder(); - defer decoder.deinit(); - - var decoded = decodedTextWriter(allocator); - defer decoded.deinit(); - try decoder.decode(token_ids, &decoded.writer); - try std.testing.expectEqualStrings("hello world", decoded.written()); -} - -test "loads bpe tokenizer with newline lookahead split and added unk" { - const allocator = std.testing.allocator; - const json = - \\{ - \\ "model": { - \\ "type": "BPE", - \\ "unk_token": "[UNK]", - \\ "vocab": { - \\ "<|UNK|>": 0, - \\ "h": 1, - \\ "e": 2, - \\ "l": 3, - \\ "o": 4, - \\ "he": 5, - \\ "ll": 6, - \\ "lo": 7, - \\ "hel": 8, - \\ "hello": 9 - \\ }, - \\ "merges": ["h e", "l l", "l o", "he l", "hel lo"] - \\ }, - \\ "added_tokens": [ - \\ { - \\ "id": 0, - \\ "content": "<|UNK|>", - \\ "single_word": false, - \\ "lstrip": false, - \\ "rstrip": false, - \\ "normalized": false, - \\ "special": true - \\ } - \\ ], - \\ "pre_tokenizer": { - \\ "type": "Sequence", - \\ "pretokenizers": [ - \\ { - \\ "type": "Split", - \\ "pattern": {"Regex": "(?:\\r?\\n)+(?!\\r?\\n)"}, - \\ "behavior": "MergedWithNext", - \\ "invert": false - \\ }, - \\ {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": true} - \\ ] - \\ }, - \\ "decoder": {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true} - \\} - ; - - var tokenizer = try Tokenizer.fromBytes(allocator, json); - defer tokenizer.deinit(); - - try std.testing.expectEqual(@as(u32, 0), tokenizer.tokenId("<|UNK|>").?); - - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const ids = try encoder.encodeAlloc(allocator, "hello"); - defer allocator.free(ids); - try std.testing.expectEqualSlices(u32, &.{ 5, 6, 4 }, ids); -} - -const test_mistral_tokenizer_json = - \\{ - \\ "model": { - \\ "type": "BPE", - \\ "vocab": { - \\ "1": 0, "2": 1, "3": 2, "4": 3, "5": 4, "6": 5, "7": 6, - \\ "12": 7, "23": 8, "34": 9, "45": 10, "56": 11, "67": 12, - \\ "123": 13, "234": 14, "345": 15, "456": 16, "567": 17, - \\ "1234": 18 - \\ }, - \\ "merges": ["1 2", "2 3", "3 4", "4 5", "5 6", "6 7", - \\ "12 3", "23 4", "34 5", "45 6", "56 7", "123 4"] - \\ }, - \\ "pre_tokenizer": { - \\ "type": "Sequence", - \\ "pretokenizers": [ - \\ { - \\ "type": "Split", - \\ "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*|\\p{N}{1,3}(?=(?:\\p{N}{3})*(?!\\p{N}))|\\p{N}{1,2}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, - \\ "behavior": "Isolated", - \\ "invert": false - \\ }, - \\ {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false} - \\ ] - \\ }, - \\ "decoder": {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true} - \\} -; - -test "Mistral right-aligned number lookahead" { - const allocator = std.testing.allocator; - var tokenizer = try Tokenizer.fromBytes(allocator, test_mistral_tokenizer_json); - defer tokenizer.deinit(); - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const cases = .{ - .{ "1", &[_]u32{0} }, - .{ "12", &[_]u32{7} }, - .{ "123", &[_]u32{13} }, - .{ "1234", &[_]u32{ 0, 14 } }, - .{ "12345", &[_]u32{ 7, 15 } }, - .{ "123456", &[_]u32{ 13, 16 } }, - .{ "1234567", &[_]u32{ 0, 14, 17 } }, - }; - inline for (cases) |case| { - encoder.reset(); - const ids = try encoder.encodeAlloc(allocator, case[0]); - defer allocator.free(ids); - try std.testing.expectEqualSlices(u32, case[1], ids); - - encoder.reset(); - var streamed: std.Io.Writer.Allocating = .initAligned(allocator, .of(u32)); - defer streamed.deinit(); - for (0..case[0].len) |i| { - try encoder.feed(case[0][i .. i + 1], &streamed.writer); - } - try encoder.finalize(&streamed.writer); - try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(case[1]), streamed.written()); - } -} - -test "Mistral Unicode number grouping and output overflow" { - const allocator = std.testing.allocator; - var parsed = try std.json.parseFromSlice(std.json.Value, allocator, test_mistral_tokenizer_json, .{}); - defer parsed.deinit(); - var model = try std.json.parseFromSlice(std.json.Value, allocator, - \\{"type": "WordPiece", "unk_token": "", "continuing_subword_prefix": "", "vocab": { - \\ "": 0, "١": 1, "٢٣٤": 2, "12": 3, "345": 4, - \\ "Ⅷ": 5, "²3४": 6, "1": 7, "111": 8 - \\}} - , .{}); - defer model.deinit(); - const arena = parsed.arena.allocator(); - try parsed.value.object.put(arena, "model", model.value); - // Test the Split segmenter directly, before ByteLevel maps UTF-8 bytes. - const split = parsed.value.object.get("pre_tokenizer").?.object.get("pretokenizers").?.array.items[0]; - try parsed.value.object.put(arena, "pre_tokenizer", split); - const json = try std.json.Stringify.valueAlloc(allocator, parsed.value, .{}); - defer allocator.free(json); - - var tokenizer = try Tokenizer.fromBytes(allocator, json); - defer tokenizer.deinit(); - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const cases = .{ - .{ "١٢٣٤", &[_]u32{ 1, 2 } }, - .{ "12345", &[_]u32{ 3, 4 } }, - .{ "Ⅷ²3४", &[_]u32{ 5, 6 } }, - }; - inline for (cases) |case| { - encoder.reset(); - var streamed: std.Io.Writer.Allocating = .initAligned(allocator, .of(u32)); - defer streamed.deinit(); - for (0..case[0].len) |i| { - try encoder.feed(case[0][i .. i + 1], &streamed.writer); - } - try encoder.finalize(&streamed.writer); - try std.testing.expectEqualSlices(u8, std.mem.sliceAsBytes(case[1]), streamed.written()); - } - - encoder.reset(); - const ids = try encoder.encodeAlloc(allocator, &(@as([1000]u8, @splat('1')))); - defer allocator.free(ids); - try std.testing.expectEqual(@as(usize, 334), ids.len); - try std.testing.expectEqual(@as(u32, 7), ids[0]); - for (ids[1..]) |id| try std.testing.expectEqual(@as(u32, 8), id); -} - -test "writer" { - const allocator = std.testing.allocator; - const json = - \\{ - \\ "model": { - \\ "type": "WordPiece", - \\ "unk_token": "[UNK]", - \\ "continuing_subword_prefix": "##", - \\ "max_input_chars_per_word": 100, - \\ "vocab": { - \\ "[UNK]": 0, - \\ "hello": 1, - \\ "world": 2 - \\ } - \\ }, - \\ "pre_tokenizer": {"type": "Whitespace"}, - \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} - \\} - ; - - var tokenizer = try Tokenizer.fromBytes(allocator, json); - defer tokenizer.deinit(); - - const text = "hello world"; - - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const token_ids_alloc = try encoder.encodeAlloc(allocator, text); - defer allocator.free(token_ids_alloc); - try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids_alloc); - - encoder.reset(); - var buf: [1024]u8 = undefined; - var out = tokenOutputWriter(allocator); - defer out.deinit(); - var writer = encoder.writer(&buf, &out.writer); - try writer.interface.writeAll(text); - try writer.finalize(); - const token_ids = writtenTokenIds(&out); - - try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, token_ids); - - var decoder = try tokenizer.decoder(); - defer decoder.deinit(); - - var decoded = decodedTextWriter(allocator); - defer decoded.deinit(); - try decoder.decode(token_ids, &decoded.writer); - try std.testing.expectEqualStrings("hello world", decoded.written()); -} - -/// Shared tokenizer JSON config used by multiple tests. -const test_tokenizer_json = - \\{ - \\ "model": { - \\ "type": "WordPiece", - \\ "unk_token": "[UNK]", - \\ "continuing_subword_prefix": "##", - \\ "max_input_chars_per_word": 100, - \\ "vocab": { - \\ "[UNK]": 0, - \\ "hello": 1, - \\ "world": 2, - \\ "foo": 3, - \\ "bar": 4, - \\ "a": 5, - \\ "b": 6, - \\ "c": 7 - \\ } - \\ }, - \\ "pre_tokenizer": {"type": "Whitespace"}, - \\ "decoder": {"type": "WordPiece", "prefix": "##", "cleanup": false} - \\} -; - -test "writer with tiny buffer forces drain" { - // A buffer smaller than the input forces the drain path (not just flush). - const allocator = std.testing.allocator; - - var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); - defer tokenizer.deinit(); - - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - // Buffer of 4 bytes; "hello world" is 11 bytes, so drain must be invoked. - var buf: [4]u8 = undefined; - var out = tokenOutputWriter(allocator); - defer out.deinit(); - var writer = encoder.writer(&buf, &out.writer); - try writer.interface.writeAll("hello world"); - try writer.finalize(); - - try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); -} - -test "writer byte-by-byte" { - // Feed one byte at a time through the writer to stress buffer/drain boundaries. - const allocator = std.testing.allocator; - - var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); - defer tokenizer.deinit(); - - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - var buf: [1]u8 = undefined; - var out = tokenOutputWriter(allocator); - defer out.deinit(); - var writer = encoder.writer(&buf, &out.writer); - for ("hello world") |byte| { - try writer.interface.writeByte(byte); - } - try writer.finalize(); - - try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); -} - -test "encoder reset between multiple encodes" { - // Ensure reset properly clears state so successive encodes are independent. - const allocator = std.testing.allocator; - - var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); - defer tokenizer.deinit(); - - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - { - const ids = try encoder.encodeAlloc(allocator, "foo bar"); - defer allocator.free(ids); - try std.testing.expectEqualSlices(u32, &.{ 3, 4 }, ids); - } - - encoder.reset(); - - { - const ids = try encoder.encodeAlloc(allocator, "hello world"); - defer allocator.free(ids); - try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, ids); - } - - encoder.reset(); - - // Third encode to verify repeated resets work. - { - const ids = try encoder.encodeAlloc(allocator, "foo"); - defer allocator.free(ids); - try std.testing.expectEqualSlices(u32, &.{3}, ids); - } -} - -test "writer and encodeAlloc produce same results" { - const allocator = std.testing.allocator; - - var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); - defer tokenizer.deinit(); - - const inputs = [_][]const u8{ - "hello", - "hello world", - "foo bar", - "a b c", - "foo bar hello world a b c", - }; - - for (inputs) |text| { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const expected = try encoder.encodeAlloc(allocator, text); - defer allocator.free(expected); - - encoder.reset(); - - var buf: [8]u8 = undefined; - var out = tokenOutputWriter(allocator); - defer out.deinit(); - var writer = encoder.writer(&buf, &out.writer); - try writer.interface.writeAll(text); - try writer.finalize(); - - try std.testing.expectEqualSlices(u32, expected, writtenTokenIds(&out)); - } -} - -test "encode empty string" { - const allocator = std.testing.allocator; - - var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); - defer tokenizer.deinit(); - - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - // encodeAlloc with empty string should produce no tokens. - { - const ids = try encoder.encodeAlloc(allocator, ""); - defer allocator.free(ids); - try std.testing.expectEqual(@as(usize, 0), ids.len); - } - - encoder.reset(); - - // Writer with empty string should also produce no tokens. - { - var buf: [16]u8 = undefined; - var out = tokenOutputWriter(allocator); - defer out.deinit(); - var writer = encoder.writer(&buf, &out.writer); - try writer.interface.writeAll(""); - try writer.finalize(); - try std.testing.expectEqual(@as(usize, 0), writtenTokenIds(&out).len); - } -} - -test "unknown tokens" { - // Words not in the vocabulary should map to the [UNK] token (id 0). - const allocator = std.testing.allocator; - - var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); - defer tokenizer.deinit(); - - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const ids = try encoder.encodeAlloc(allocator, "unknown"); - defer allocator.free(ids); - try std.testing.expectEqualSlices(u32, &.{0}, ids); -} - -test "decode round-trip" { - // Encode then decode several strings and verify round-trip fidelity. - const allocator = std.testing.allocator; - - var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); - defer tokenizer.deinit(); - - const inputs = [_][]const u8{ - "hello", - "hello world", - "foo bar", - "a b c", - }; - - for (inputs) |text| { - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - const ids = try encoder.encodeAlloc(allocator, text); - defer allocator.free(ids); - - var decoder = try tokenizer.decoder(); - defer decoder.deinit(); - - var decoded = decodedTextWriter(allocator); - defer decoded.deinit(); - try decoder.decode(ids, &decoded.writer); - - try std.testing.expectEqualStrings(text, decoded.written()); - } -} - -test "decoder reset between decodes" { - const allocator = std.testing.allocator; - - var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); - defer tokenizer.deinit(); - - var decoder = try tokenizer.decoder(); - defer decoder.deinit(); - - { - var text = decodedTextWriter(allocator); - defer text.deinit(); - try decoder.decode(&.{ 1, 2 }, &text.writer); - try std.testing.expectEqualStrings("hello world", text.written()); - } - - try decoder.reset(); - - { - var text = decodedTextWriter(allocator); - defer text.deinit(); - try decoder.decode(&.{ 3, 4 }, &text.writer); - try std.testing.expectEqualStrings("foo bar", text.written()); - } -} - -test "tokenId lookup" { - const allocator = std.testing.allocator; - - var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); - defer tokenizer.deinit(); - - try std.testing.expectEqual(@as(u32, 0), tokenizer.tokenId("[UNK]").?); - try std.testing.expectEqual(@as(u32, 1), tokenizer.tokenId("hello").?); - try std.testing.expectEqual(@as(u32, 2), tokenizer.tokenId("world").?); - try std.testing.expectEqual(@as(u32, 3), tokenizer.tokenId("foo").?); - try std.testing.expectEqual(@as(u32, 4), tokenizer.tokenId("bar").?); - try std.testing.expect(tokenizer.tokenId("nonexistent") == null); - try std.testing.expect(tokenizer.tokenId("") == null); -} - -test "tokenId <-> decode" { - const allocator = std.testing.allocator; - - var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); - defer tokenizer.deinit(); - - for ([_][]const u8{ "[UNK]", "hello", "world", "foo", "bar" }) |text| { - try std.testing.expectEqualSlices(u8, text, tokenizer.decode(tokenizer.tokenId(text).?).?); - } -} - -test "writer with pre-allocated output writer" { - // Pass an already-allocated aligned output writer to the encoder stream. - const allocator = std.testing.allocator; - - var tokenizer = try Tokenizer.fromBytes(allocator, test_tokenizer_json); - defer tokenizer.deinit(); - - var encoder = try tokenizer.encoder(); - defer encoder.deinit(); - - var out = tokenOutputWriter(allocator); - defer out.deinit(); - try out.ensureUnusedCapacity(64 * @sizeOf(u32)); - - var buf: [32]u8 = undefined; - var writer = encoder.writer(&buf, &out.writer); - try writer.interface.writeAll("hello world"); - try writer.finalize(); - - try std.testing.expectEqualSlices(u32, &.{ 1, 2 }, writtenTokenIds(&out)); -} diff --git a/zml/tokenizer/main.zig b/zml/tokenizer/main.zig index 14eb1a87..a35de7d5 100644 --- a/zml/tokenizer/main.zig +++ b/zml/tokenizer/main.zig @@ -12,15 +12,16 @@ const Flags = struct { verbose: bool = false, }; -pub fn main(init: std.process.Init) !void { - const allocator = init.gpa; +pub fn main() !void { + var gpa: std.heap.GeneralPurposeAllocator(.{}) = .{}; + const allocator = gpa.allocator(); - var threaded: std.Io.Threaded = .init(allocator, .{}); + var threaded: std.Io.Threaded = .init(allocator); defer threaded.deinit(); const io = threaded.io(); - const args = stdx.flags.parseProcessArgs(init.minimal, Flags); + const args = stdx.flags.parseProcessArgs(Flags); log.info("\tLoading tokenizer from {s}", .{args.tokenizer}); var tokenizer = try zml_tokenizer.Tokenizer.fromFile(allocator, io, args.tokenizer); @@ -33,17 +34,15 @@ pub fn main(init: std.process.Init) !void { var decoder = try tokenizer.decoder(); defer decoder.deinit(); - const prompt_tok = try encoder.encodeAlloc(allocator, args.prompt); - defer allocator.free(prompt_tok); + const prompt_tok = try encoder.encode(args.prompt); log.info("Input: {s}\nOutput: {any}", .{ args.prompt, prompt_tok }); var errors: u8 = 0; { - var reconstructed = try decoder.decodeAlloc(allocator, prompt_tok); - defer reconstructed.deinit(allocator); - if (!std.mem.eql(u8, args.prompt, reconstructed.items)) { - log.err("Reconstructed string from tokens doesn't match source: {s}", .{reconstructed.items}); + const reconstructed = try decoder.decode(prompt_tok); + if (!std.mem.eql(u8, args.prompt, reconstructed)) { + log.err("Reconstructed string from tokens doesn't match source: {s}", .{reconstructed}); errors += 1; } } diff --git a/zml/tokenizer/sentencepiece/BUILD.bazel b/zml/tokenizer/sentencepiece/BUILD.bazel index dd4d2e76..0fe8e061 100644 --- a/zml/tokenizer/sentencepiece/BUILD.bazel +++ b/zml/tokenizer/sentencepiece/BUILD.bazel @@ -5,7 +5,6 @@ load("@rules_zig//zig:defs.bzl", "zig_library") swig_cc_library( name = "sentencepiece_swig", - tags = ["manual"], interface = "sentencepiece.i", module = "sentencepiece", deps = [ @@ -16,7 +15,6 @@ swig_cc_library( zig_library( name = "sentencepiece", - tags = ["manual"], import_name = "sentencepiece", main = "sentencepiece.zig", visibility = ["//visibility:public"], @@ -27,8 +25,12 @@ zig_library( ], ) -# todo: reenable after fix -# zig_srcs( -# name = "sources", -# zig_lib = ":sentencepiece", -# ) +zig_srcs( + name = "sources", + zig_lib = ":sentencepiece", +) + +cc_static_library( + name="sentencepiece_static", + deps = [":sentencepiece_swig"] +) diff --git a/zml/tokenizer/sentencepiece/sentencepiece.i b/zml/tokenizer/sentencepiece/sentencepiece.i index d89489c7..35ccc057 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.i +++ b/zml/tokenizer/sentencepiece/sentencepiece.i @@ -19,7 +19,7 @@ $1 = absl::string_view((char *)$input.ptr, $input.len); } -%typemap(out) const std::string& %{ +%typemap(out, optimal="1") const std::string& %{ $result.ptr = (void *)($1->data()); $result.len = (size_t)($1->length()); %} @@ -85,8 +85,6 @@ namespace sentencepiece { virtual sentencepiece::util::Status Encode(absl::string_view input, std::vector *ids) const; virtual sentencepiece::util::Status Decode(const std::vector &ids, std::string *detokenized) const; virtual int PieceToId(absl::string_view piece) const; - virtual const std::string &IdToPiece(int id) const; - virtual int GetPieceSize() const; virtual int unk_id() const; virtual int bos_id() const; virtual int eos_id() const; diff --git a/zml/tokenizer/sentencepiece/sentencepiece.zig b/zml/tokenizer/sentencepiece/sentencepiece.zig index a09e0c0a..c98b0740 100644 --- a/zml/tokenizer/sentencepiece/sentencepiece.zig +++ b/zml/tokenizer/sentencepiece/sentencepiece.zig @@ -89,7 +89,7 @@ pub const Decoder = struct { inner: *SentencePieceProcessor, vec: *c.std_vector_int, str: *c.std_string, - last_string: StringBuffer = .empty, + last_string: StringBuffer = .{ .len = 0 }, fn init(inner: *SentencePieceProcessor) !Decoder { const vec = try (c.std_vector_int_new() orelse std.mem.Allocator.Error.OutOfMemory); @@ -192,12 +192,7 @@ pub const SentencePieceProcessor = opaque { return try Decoder.init(self); } - pub fn tokenId(self: *SentencePieceProcessor, token: []const u8) u32 { + pub fn tokenToId(self: *SentencePieceProcessor, token: []const u8) u32 { return @intCast(c.SentencePieceProcessor_PieceToId(@ptrCast(self), ffi.ZigSlice.from(token))); } - - pub fn decode(self: *SentencePieceProcessor, token_id: u32) ?[]const u8 { - if (token_id >= @as(u32, @intCast(c.SentencePieceProcessor_GetPieceSize(@ptrCast(self))))) return null; - return ffi.ZigSlice.to(u8, c.SentencePieceProcessor_IdToPiece(@ptrCast(self), @intCast(token_id))); - } }; diff --git a/zml/tokenizer/tokenizer.zig b/zml/tokenizer/tokenizer.zig index fc304b81..c72ec803 100644 --- a/zml/tokenizer/tokenizer.zig +++ b/zml/tokenizer/tokenizer.zig @@ -1,40 +1,25 @@ const std = @import("std"); -pub const iree = @import("iree"); +pub const hftokenizers = @import("hftokenizers"); pub const sentencepiece = @import("sentencepiece"); -const log = std.log.scoped(.@"zml/tokenizer"); +pub const homemade = @import("homemade.zig"); + +test { + std.testing.refAllDecls(@This()); +} const Tokenizers = enum { - iree, + hftokenizers, sentencepiece, + homemade, }; pub const Tokenizer = union(Tokenizers) { - pub const Normalizer = union(Tokenizers) { - pub const Kind = iree.Normalizer.Kind; - - iree: iree.Normalizer, - sentencepiece: void, - - pub fn deinit(self: *Normalizer) void { - switch (self.*) { - .iree => |*n| n.deinit(), - else => {}, - } - } - - pub fn normalize(self: *Normalizer, allocator: std.mem.Allocator, text: []const u8) ![]const u8 { - return switch (self.*) { - .iree => |*n| n.normalize(allocator, text), - else => error.UnsupportedNormalizer, - }; - } - }; - pub const Encoder = union(Tokenizers) { - iree: iree.Tokenizer.Encoder, + hftokenizers: hftokenizers.Encoder, sentencepiece: sentencepiece.Encoder, + homemade: homemade.Encoder, pub fn deinit(self: *Encoder) void { switch (self.*) { @@ -48,34 +33,23 @@ pub const Tokenizer = union(Tokenizers) { } } - pub fn encodeAlloc(self: *Encoder, allocator: std.mem.Allocator, text: []const u8) ![]u32 { + pub fn encode(self: *Encoder, input: []const u8) ![]const u32 { return switch (self.*) { - .iree => |*encoder_| encoder_.encodeAlloc(allocator, text), - inline else => |*encoder_| try allocator.dupe(u32, try encoder_.encode(text)), + inline else => |*v| v.encode(input), }; } - pub fn encode(self: *Encoder, writer: *std.Io.Writer, text: []const u8) !void { - return switch (self.*) { - .iree => |*e| { - try e.feed(text, writer); - try e.finalize(writer); - }, - inline else => |*e| { - const encoded: []const u32 = try e.encode(text); - try writer.writeAll(std.mem.sliceAsBytes(encoded)); - }, + pub fn ids(self: Encoder) []const u32 { + return switch (self) { + inline else => |v| v.ids(), }; } - - pub fn appendTokens(_: *Encoder, writer: *std.Io.Writer, tokens: []const u32) !void { - try writer.writeAll(@ptrCast(tokens)); - } }; pub const Decoder = union(Tokenizers) { - iree: iree.Tokenizer.Decoder, + hftokenizers: hftokenizers.Decoder, sentencepiece: sentencepiece.Decoder, + homemade: homemade.Decoder, pub fn deinit(self: *Decoder) void { switch (self.*) { @@ -89,101 +63,81 @@ pub const Tokenizer = union(Tokenizers) { } } - pub fn decodeAlloc(self: *Decoder, allocator: std.mem.Allocator, token_ids: []const u32) !std.ArrayList(u8) { + pub fn decode(self: *Decoder, ids_: []const u32) ![]const u8 { return switch (self.*) { - .iree => |*d| { - var writer: std.Io.Writer.Allocating = .init(allocator); - errdefer writer.deinit(); - try d.decode(token_ids, &writer.writer); - return writer.toArrayList(); - }, - inline else => |*d| .fromOwnedSlice(try allocator.dupe(u8, try d.decode(token_ids))), + inline else => |*v| v.decode(ids_), }; } - // Only valid until the next decode and while the decoder AND buffer is alive. - pub fn feedOne(self: *Decoder, token_id: u32, buffer: []u8) ![]const u8 { + pub fn string(self: *Decoder) []const u8 { return switch (self.*) { - .iree => |*d| d.feedOne(token_id, buffer), - inline else => |*d| try d.next(token_id) orelse &.{}, + inline else => |v| v.string(), }; } - /// Output is only valid until the next feed/finalize call and while the decoder AND buffer is alive. - pub fn finalize(self: *Decoder, buffer: []u8) ![]const u8 { + //pub fn ids(self: *Decoder) []u32 { + // return switch (self.*) { + // inline else => |v| v.ids(), + // }; + //} + + pub fn next(self: *Decoder, token_id: u32) !?[]const u8 { return switch (self.*) { - .iree => |*v| v.finalize(buffer), - inline else => &.{}, + inline else => |*v| v.next(token_id), }; } }; - iree: iree.Tokenizer, + hftokenizers: *hftokenizers.HFTokenizer, sentencepiece: *sentencepiece.SentencePieceProcessor, + homemade: *homemade.Tokenizer, pub fn fromFile(allocator: std.mem.Allocator, io: std.Io, model: []const u8) !Tokenizer { + _ = allocator; // autofix + _ = io; // autofix if (std.mem.endsWith(u8, model, ".pb")) { return .{ .sentencepiece = try .fromFile(model) }; } if (std.mem.endsWith(u8, model, ".json")) { - return .{ .iree = try .fromFile(allocator, io, model) }; + return .{ .hftokenizers = try .fromFile(model) }; } return error.InvalidArgument; } - pub fn fromBytes(allocator: std.mem.Allocator, bytes: []const u8) !Tokenizer { + pub fn fromBytes(allocator: std.mem.Allocator, io: std.Io, bytes: []const u8) !Tokenizer { + _ = allocator; // autofix + _ = io; // autofix if (bytes[0] == '{') { - return .{ .iree = try .fromBytes(allocator, bytes) }; + return .{ .hftokenizers = try .fromBytes(bytes) }; } return .{ .sentencepiece = try .fromBytes(bytes) }; } pub fn deinit(self: *Tokenizer) void { switch (self.*) { - inline else => |*t| t.*.deinit(), + inline else => |t| t.deinit(), } } - pub fn encoder(self: *const Tokenizer) !Encoder { - return switch (self.*) { - inline else => |*v, tag| @unionInit(Encoder, @tagName(tag), try v.*.encoder()), + pub fn encoder(self: Tokenizer) !Encoder { + return switch (self) { + inline else => |v, tag| @unionInit(Encoder, @tagName(tag), try v.encoder()), }; } - pub fn decoder(self: *const Tokenizer) !Decoder { - return switch (self.*) { - inline else => |*v, tag| @unionInit(Decoder, @tagName(tag), try v.*.decoder()), + pub fn decoder(self: Tokenizer) !Decoder { + return switch (self) { + inline else => |v, tag| @unionInit(Decoder, @tagName(tag), try v.decoder()), }; } - pub fn normalizer(self: *const Tokenizer, sequence: []const Normalizer.Kind) !Normalizer { - return switch (self.*) { - .iree => |*backend| .{ .iree = try backend.normalizer(sequence) }, - else => error.UnsupportedNormalizer, - }; - } - - pub fn normalizerFromHuggingFaceJson(self: *const Tokenizer, json: []const u8) !Normalizer { - return switch (self.*) { - .iree => |*backend| .{ .iree = try backend.normalizerFromHuggingFaceJson(json) }, - else => error.UnsupportedNormalizer, - }; - } - - pub fn tokenId(self: *const Tokenizer, token: []const u8) ?u32 { - return switch (self.*) { - inline else => |v| v.tokenId(token), - }; - } - - pub fn decode(self: *const Tokenizer, token_id: u32) ?[]const u8 { - return switch (self.*) { - inline else => |v| v.decode(token_id), + pub fn tokenToId(self: Tokenizer, token: []const u8) ?u32 { + return switch (self) { + inline else => |v| v.tokenToId(token), }; } }; -test { - std.testing.refAllDecls(@This()); -} +test {} +test {} diff --git a/zml/tracer.zig b/zml/tracer.zig new file mode 100644 index 00000000..e2e173ef --- /dev/null +++ b/zml/tracer.zig @@ -0,0 +1,77 @@ +const builtin = @import("builtin"); + +const c = @import("c"); + +pub const Tracer = switch (builtin.os.tag) { + // TODO(cerisier): fix MacOsTracer + // .macos => MacOsTracer, + .linux => if (@hasDecl(c, "ZML_RUNTIME_CUDA")) CudaTracer else FakeTracer, + else => FakeTracer, +}; + +const CudaTracer = struct { + + // Those symbols are defined in cudaProfiler.h but their implementation is in libcuda.so + // They will be bound at call time after libcuda.so is loaded (as a needed dependency of libpjrt_cuda.so). + const cuProfilerStart = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStart", .linkage = .weak }) orelse unreachable; + const cuProfilerStop = @extern(*const fn () callconv(.c) c_int, .{ .name = "cuProfilerStop", .linkage = .weak }) orelse unreachable; + + // Those symbols are defined in nvToolsExt.h which we don't want to provide. + // However, we link with libnvToolsExt.so which provides them. + // They will be bound at call time after libnvToolsExt.so is loaded (manually dlopen'ed by us). + const nvtxMarkA = @extern(*const fn ([*:0]const u8) callconv(.c) void, .{ .name = "nvtxMarkA", .linkage = .weak }) orelse unreachable; + const nvtxRangeStartA = @extern(*const fn ([*:0]const u8) callconv(.c) c_int, .{ .name = "nvtxRangeStartA", .linkage = .weak }) orelse unreachable; + const nvtxRangeEnd = @extern(*const fn (c_int) callconv(.c) void, .{ .name = "nvtxRangeEnd", .linkage = .weak }) orelse unreachable; + + pub fn event(message: [:0]const u8) void { + nvtxMarkA(message.ptr); + } + + pub fn frameStart(message: [:0]const u8) u64 { + return @intCast(nvtxRangeStartA(message.ptr)); + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = message; + nvtxRangeEnd(@intCast(interval_id)); + return; + } +}; + +const MacOsTracer = struct { + logger: c.os_log_t, + + pub fn event(self: *const MacOsTracer, message: [:0]const u8) void { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_event(self.logger, interval_id, message); + } + + pub fn frameStart(self: *const MacOsTracer, message: [:0]const u8) c.os_signpost_id_t { + const interval_id = c.os_signpost_id_generate(self.logger); + c.zml_os_signpost_interval_begin(self.logger, interval_id, message); + return interval_id; + } + + pub fn frameEnd(self: *const MacOsTracer, interval_id: c.os_signpost_id_t, message: [:0]const u8) void { + c.zml_os_signpost_interval_end(self.logger, interval_id, message); + } +}; + +/// Mock tracer for OS which don't have an impl. +const FakeTracer = struct { + pub fn event(message: [:0]const u8) void { + _ = message; + return; + } + + pub fn frameStart(message: [:0]const u8) u64 { + _ = message; + return 0; + } + + pub fn frameEnd(interval_id: u64, message: [:0]const u8) void { + _ = interval_id; + _ = message; + return; + } +}; diff --git a/zml/zml.zig b/zml/zml.zig index dafc3a0f..22faac3c 100644 --- a/zml/zml.zig +++ b/zml/zml.zig @@ -13,65 +13,37 @@ const runfiles = @import("runfiles"); pub const stdx = @import("stdx"); pub const tokenizer = @import("zml/tokenizer"); -pub const attention = @import("attention/attention.zig"); -pub const paged_attention = @import("kernels/paged_attention/paged_attention.zig"); +pub const attention = @import("attention.zig"); pub const Buffer = @import("buffer.zig").Buffer; -pub const Compiler = @import("Compiler.zig"); pub const constants = @import("constants.zig"); pub const dtype = @import("dtype.zig"); pub const Data = dtype.Data; pub const DataType = dtype.DataType; pub const exe = @import("exe.zig"); pub const Exe = exe.Exe; -pub const FnExe = exe.FnExe; pub const floats = @import("floats.zig"); pub const io = @import("io.zig"); -pub const kernel = @import("kernel/kernel.zig"); -pub const logo = @import("logo.zig"); pub const mem = @import("mem.zig"); pub const Bufferized = mem.Bufferized; pub const meta = @import("meta.zig"); pub const mlir = @import("mlirx.zig"); -pub const moe = @import("kernels/moe/moe.zig"); +pub const module = @import("module.zig"); pub const nn = @import("nn.zig"); pub const ops = @import("ops.zig"); -pub const pjrtx = @import("pjrtx.zig"); pub const platform = @import("platform.zig"); pub const Memory = platform.Memory; pub const Platform = platform.Platform; pub const Target = platform.Target; -pub const tracer = @import("profiling/tracer.zig"); -pub const quantization = @import("quantization.zig"); -pub const Quantization = quantization.Quantization; +pub const CompilationOptions = platform.CompilationOptions; pub const safetensors = @import("safetensors.zig"); pub const shape = @import("shape.zig"); pub const Shape = shape.Shape; -pub const Sharding = @import("Sharding.zig"); -pub const Mesh = Sharding.Mesh; pub const slice = @import("slice.zig"); pub const Slice = slice.Slice; pub const tensor = @import("tensor.zig"); pub const Tensor = tensor.Tensor; pub const testing = @import("testing.zig"); - -/// Deprecated, use `zml.Compiler` instead -pub const module = struct { - pub const CompilationContext = Compiler; - pub const CompileError = Compiler.Error; - pub const CompilationOptions = Compiler.Options; - - /// Deprecated, use `platform.compileFn` or `zml.Compiler.compile` - pub fn compile( - allocator: std.mem.Allocator, - io_: std.Io, - comptime func: anytype, - args: std.meta.ArgsTuple(@TypeOf(func)), - platform_: *const Platform, - opts: Compiler.Options, - ) Compiler.Error!Exe { - return Compiler.compile(allocator, io_, platform_, func, args, opts); - } -}; +pub const tracer = @import("tracer.zig"); test "zml" { std.testing.refAllDecls(@This()); @@ -82,8 +54,6 @@ pub const MiB = 1024 * KiB; pub const GiB = 1024 * MiB; pub const TiB = 1024 * GiB; -comptime { - // Forces zig to compile `@zml//bazel` module even when users don't depend on it even indirectly. - // Otherwise exported functions declared in `bazel` aren't compiled. - std.testing.refAllDecls(bazel); +pub fn init(init_: std.process.Init) !void { + try bazel.initSimple(init_); }